From c06f1b1cb3a5e63243aa5b7615870786df718448 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Fri, 10 Jul 2026 23:33:07 -0300 Subject: [PATCH 01/55] test: freeze remediation audit baseline --- .planning/OPERATING-ENVELOPE-2026-07-11.md | 113 +++++ .planning/PHASE0-RED-TEST-MATRIX.md | 49 +++ ...EMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md | 389 ++++++++++++++++++ .planning/audit-remediation-ledger.md | 69 ++++ .../2026-07-10-baseline/audit-report.md | 274 ++++++++++++ .../audits/2026-07-10-baseline/inventory.json | 75 ++++ external-actions-required.md | 13 + tests/test_atlas_mock_evidence_guard.py | 72 ++++ tests/test_deployment_contracts.py | 84 ++++ tests/test_entity_schema_composition.py | 42 ++ tests/test_mcp_authorization_boundary.py | 102 +++++ tests/test_mcp_read_only_contract.py | 63 +++ tests/test_persisted_envelope_sensitivity.py | 86 ++++ tests/test_qdrant_authoritative_filtering.py | 140 +++++++ tests/test_retrieval_surface_parity.py | 87 ++++ tests/test_scheduled_archive_lifecycle.py | 27 ++ tests/test_stop_hook_capture_policy.py | 97 +++++ tests/test_write_gateway_paths.py | 76 ++++ 18 files changed, 1858 insertions(+) create mode 100644 .planning/OPERATING-ENVELOPE-2026-07-11.md create mode 100644 .planning/PHASE0-RED-TEST-MATRIX.md create mode 100644 .planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md create mode 100644 .planning/audit-remediation-ledger.md create mode 100644 .planning/audits/2026-07-10-baseline/audit-report.md create mode 100644 .planning/audits/2026-07-10-baseline/inventory.json create mode 100644 external-actions-required.md create mode 100644 tests/test_atlas_mock_evidence_guard.py create mode 100644 tests/test_deployment_contracts.py create mode 100644 tests/test_entity_schema_composition.py create mode 100644 tests/test_mcp_authorization_boundary.py create mode 100644 tests/test_mcp_read_only_contract.py create mode 100644 tests/test_persisted_envelope_sensitivity.py create mode 100644 tests/test_qdrant_authoritative_filtering.py create mode 100644 tests/test_retrieval_surface_parity.py create mode 100644 tests/test_scheduled_archive_lifecycle.py create mode 100644 tests/test_stop_hook_capture_policy.py create mode 100644 tests/test_write_gateway_paths.py diff --git a/.planning/OPERATING-ENVELOPE-2026-07-11.md b/.planning/OPERATING-ENVELOPE-2026-07-11.md new file mode 100644 index 00000000..5b233cf0 --- /dev/null +++ b/.planning/OPERATING-ENVELOPE-2026-07-11.md @@ -0,0 +1,113 @@ +# MemoryMaster Operating Envelope + +**Status:** Frozen Phase 0 baseline +**Measured:** 2026-07-11 UTC +**Database access:** Strictly read-only +**Live-data mutations authorized:** None + +## Snapshot + +| Metric | Value | +|---|---:| +| Claims | 108,217 | +| Events | 1,033,626 | +| Verbatim rows | 1,067,072 | +| Distinct verbatim session IDs | 184,889 | +| Candidate claims | 21,828 | +| Confirmed claims | 18,720 | +| Archived claims | 48,726 | +| Stale claims | 14,656 | +| Conflicted claims | 2,087 | +| Superseded claims | 2,200 | +| SQLite DB | 5.123 GiB | +| WAL | 7.705 MiB | +| Drive used | 85.82% | +| Drive free | 66.06 GiB | + +## Candidate age + +Snapshot cohort: 21,822 candidates at 2026-07-11T01:49:58Z. Six arrived immediately afterward; the program cohort is frozen at 21,828. + +| Age | Count | +|---|---:| +| <1 day | 1,012 | +| 1-3 days | 1,558 | +| 3-7 days | 214 | +| 7-14 days | 127 | +| 14-30 days | 385 | +| 30-60 days | 2,393 | +| 60-90 days | 16,132 | +| >=90 days | 1 | + +- P50: 70.737 days +- P90: 74.716 days +- P95: 75.390 days +- P99: 75.679 days +- Maximum: 111.091 days + +The target `candidate_age_p95 <= 7 days` currently fails. + +## Capacity window + +Window: 14 completed UTC days, 2026-06-27 through 2026-07-10. + +- Reconstructed candidate inflow: 10,000 / 714.29 per day. +- Candidate dispositions: 12,050 / 860.71 per day. +- Safe 80% intake ceiling: 688 per day. +- Observed intake is 103.7% of the safe ceiling: gate fails. +- Require at least one successful steward cycle per completed UTC day. +- Pass requires a seven-day rolling inflow <=688/day for seven consecutive completed days. + +The inflow figure is reconstructed from current candidates plus transition events. It is not authoritative because 2,455 recent `llm-stop-hook` candidates have no creation event. + +## Retention and growth gates + +Verbatim retention stops at whichever limit is reached first: + +1. 30 days of age; +2. 512 MiB raw content; +3. 75,000 most-recent distinct session IDs. + +Warn at 80% of any limit. Current 30-day demand is 357.96 MiB and 74,061 distinct session IDs. + +Until a 30-day physical-size series exists: + +- Warn when seven-day rolling physical DB growth exceeds 64 MiB/day. +- Critical when seven-day rolling growth exceeds 128 MiB/day or any single day exceeds 256 MiB. +- Record DB bytes, WAL bytes, and free-disk bytes daily; replace provisional gates after 30 complete days. + +Disk watermarks: + +- Warning: >=75% used or <75 GiB free. +- Critical: >=85% used or <50 GiB free. +- Capture hard stop: >=92% used or <10 GiB free. + +The current drive is percentage-critical and must be tracked as an external/operator action; this plan does not authorize deletion or compaction of live data. + +## Backlog completion + +- Frozen cohort: 21,828 candidates at 2026-07-11. +- Target review date: 2026-09-30. +- Completion requires original-cohort `still_reviewable = 0`. +- `confirmed + archived_with_reason + rejected_with_reason = 21,828`. +- Every outcome must have an append-only event. +- Global current candidates must have P95 age <=7 days. +- Global candidate count must be <=4,816, equivalent to seven days at the 688/day safe ceiling. +- Do not pre-allocate confirmed/archive/reject counts; truth review determines the split. + +## Instrumentation required + +- Emit one authoritative claim-created/entered-candidate event per claim. +- Add steward `run_id`, start/end timestamps, inspected/disposed counts, outcomes, reasons, duration, budget, and failures. +- Add a canonical disposition reason; `rejected` is not a current lifecycle status. +- Record daily DB/WAL/free-disk bytes and backlog snapshots. +- Add canonical session lifecycle and byte counters. + +## Measurement queries + +- Status counts: `SELECT status, COUNT(*) FROM claims GROUP BY status`. +- Candidate ages: `julianday(snapshot_utc) - julianday(created_at)` for current candidates. +- Reconstructed inflow: claims created in the window whose current status is candidate or that have an event entering/leaving candidate. +- Dispositions: unique claims with validator transition events leaving candidate during the window. +- Verbatim volume: row count, distinct `session_id`, and `SUM(LENGTH(content))` grouped by timestamp windows. +- File sizes and free disk were measured through read-only filesystem metadata. diff --git a/.planning/PHASE0-RED-TEST-MATRIX.md b/.planning/PHASE0-RED-TEST-MATRIX.md new file mode 100644 index 00000000..e7d1b031 --- /dev/null +++ b/.planning/PHASE0-RED-TEST-MATRIX.md @@ -0,0 +1,49 @@ +# Phase 0 Adversarial Red-Test Matrix + +All tests use temporary databases, isolated environment variables, fake providers, and local/fake services unless explicitly marked external. Unsafe current behavior should be committed as `xfail(strict=True)` tests and demonstrated with `--runxfail`; fixes remove the marker. + +## Shared fixtures + +- `isolated_mm_env`: clears inherited DB/Qdrant/provider/auth settings and redirects state/spool/snapshots to `tmp_path`. +- `policy_db`: claims in every lifecycle state across tenants/projects. +- `synthetic_secret`: deterministic non-credential test token. +- `durable_payload_scan`: scans all durable string/JSON fields for a fixture. +- `db_fingerprint`: proves denied operations cause no domain mutation. +- `FakeQdrant`: returns caller-controlled IDs/payloads without network/model dependencies. +- `rendered_hook`: renders hook templates into isolated state directories. + +## Matrix + +| Finding | Test file | Required red tests | External dependency | +|---|---|---|---| +| MM-SEC-01 | `tests/test_mcp_authorization_boundary.py` | reader cannot ingest with spoofed source; unknown team principal fails closed; scope allowlist cannot expand context; list/query/pin/redact cannot cross project/tenant; every MCP tool declares an action | Real Postgres/RLS subset requires DSN | +| MM-SEC-02 | `tests/test_qdrant_authoritative_filtering.py` | never return orphan payload; filter archived/candidate/stale/conflicted/wrong-scope/wrong-tenant/sensitive/private; payload cannot override DB; safe fallback; equal-count/different-ID reconcile | Real authenticated Qdrant final parity | +| MM-SEC-03 | `tests/test_persisted_envelope_sensitivity.py` | plain and encoded secret matrix over every claim/citation/provenance field; legacy sensitive metadata hidden from list/query/export/Qdrant | None | +| MM-SEC-04 | `tests/test_write_gateway_paths.py` | compact-summary, steward existing-row update, verbatim/spool/Atlas/miner/import paths reject secret fixture | None | +| MM-ARCH-01 | `tests/test_entity_schema_composition.py` | normal init then graph schema; registry-first extract/stats/related; read tools issue no DDL | None | +| MM-ARCH-02 | `tests/test_retrieval_surface_parity.py` | conversational vs keyword IDs across MCP/context/hook/CLI; trusted defaults exclude provisional statuses | ML parity may require model | +| MM-REL-02 | `tests/test_mcp_read_only_contract.py` | query succeeds under held write lock; unchanged access count; one aggregated spool signal; one retrieval per detail level | None | +| MM-OPS-01/02/04 | `tests/test_deployment_contracts.py` | required secret interpolation; private backend ports; matching entrypoint/health; Helm probes; pinned images | Built-runtime/Kubernetes final checks | +| MM-UX-01 | `tests/test_setup_profile_verification.py` | requested component failure returns nonzero/PARTIAL; provider/MCP/hook/vector checks are independently reported | Docker/provider optional cases | +| MM-COST-01/02 | `tests/test_stop_hook_capture_policy.py` | default quiet/nonblocking; only appended lines processed; persisted budget survives restart; finite defaults | None | +| MM-DEMO-01 | `tests/test_atlas_mock_evidence_guard.py` | missing provider fails without evidence; explicit mock requires dev gate; mock evidence cannot feed claims/actions | None | +| MM-LIFE-01 | `tests/test_scheduled_archive_lifecycle.py` | scheduled archive increments version, timestamps, event, cache/vector/outbox; template contains no direct status SQL | None | + +## Existing tests requiring semantic updates + +- Unknown/None principal default-writer tests become explicit local-profile behavior only. +- Tenantless visibility/pin tests become explicit local-profile behavior only. +- Unlimited budget behavior becomes an explicitly unsafe legacy/profile test, not the default. +- Atlas mock-default tests require a dev gate. +- Direct maximum-capture Stop-hook tests require an explicit flag. +- Count-only Qdrant reconciliation tests are replaced by set/content convergence. + +## External-only gates + +- Postgres RLS and application-role enforcement. +- Built container readiness, network exposure, and MCP handshake. +- Kubernetes scheduling/readiness/network policy. +- Real Qdrant auth/TLS/reconciliation. +- ML/hybrid parity when required models are unavailable. + +These become `BLOCKED-EXTERNAL` with owner/evidence requirements when unavailable; static/fake-backed tests remain mandatory in normal CI. diff --git a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md new file mode 100644 index 00000000..1a1dcc09 --- /dev/null +++ b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md @@ -0,0 +1,389 @@ +# MemoryMaster Remediation & Optimization Plan + +**Status:** PROPOSED — ready for execution +**Created:** 2026-07-10 +**Scope:** `memorymaster/`, tests, packaging, setup, Docker/Helm, operational docs +**Source:** 2026-07-10 full audit (hard stops, blind spots, Tambon, 13 domains, runtime probes) +**Execution model:** test-first, atomic work packages, audit-loop convergence + +## 1. Objective + +Make MemoryMaster's public surfaces preserve the guarantees already present in its governed core: + +1. A caller cannot cross agent, project, scope, visibility, or tenant boundaries. +2. Every persisted field passes one sensitivity and provenance policy. +3. SQLite/Postgres is authoritative; Qdrant is a derived candidate index only. +4. Trusted recall returns active confirmed truth by default. +5. Capture, LLM calls, storage growth, and reconciliation are finite and observable. +6. Every advertised setup/deployment profile works end to end and fails closed. +7. Optional integrations cannot silently create fake or ungoverned evidence. + +This is a hardening and convergence program, not a rewrite. Preserve the working lifecycle, citation, WAL, event-ledger, snapshot, steward, and retrieval-explanation foundations. + +## 2. Release posture during remediation + +| Profile | Current posture | Promotion gate | +|---|---|---| +| Local SQLite, trusted agents | Usable with caution | Trusted recall, complete sensitivity gateway, quiet capture | +| Semantic/Qdrant | BLOCKED | Authoritative rehydration/filtering and exact reconciliation | +| Team/Postgres | BLOCKED | Authenticated tenant context, MCP authorization, RLS, adversarial isolation tests | +| Docker/Helm/full stack | BLOCKED | Correct entrypoints, secrets, network defaults, readiness, container smoke | +| Atlas media/entity graph | EXPERIMENTAL | No default mocks; unified entity schema; composed integration tests | + +Do not describe a blocked profile as production-ready until its gate is demonstrated with runtime evidence. + +## 3. Program invariants + +- Query MemoryMaster before each architectural decision and ingest verified non-obvious conclusions. +- MemoryMaster `query_memory` access telemetry and narrowly scoped `ingest_claim`/`checkpoint` writes required by project governance are authorized during execution. This exception does not authorize cycles, compaction, cleanup, redaction, backlog mutation, migrations, or other live-data operations. +- Run GitNexus impact analysis before editing every function, class, or method. Warn before HIGH/CRITICAL blast-radius changes. +- Run `gitnexus_detect_changes()` before each commit. +- Check GitNexus index freshness in Phase 0. After every commit, rerun `npx gitnexus analyze --embeddings` when the existing index contains embeddings before doing further impact analysis. +- Preserve all pre-existing dirty-worktree changes; stage and commit only explicit remediation paths. +- Prefer an isolated worktree with a dedicated virtual environment. Before testing, prove `memorymaster.__file__` resolves into that worktree; otherwise stop using it and avoid the editable-install import-pin trap. +- Use temporary databases and fake/local services until a phase's live-migration gate is approved. +- No production credential rotation, push, publish, deployment, or live database mutation without explicit authority. +- A `BLOCKED-POLICY` disposition requires the `audit-decisions` workflow and a complete, approved, unexpired `baseline-policy.md` entry. +- Schema changes update SQLite, Postgres, versioned migrations, parity tests, and documentation together. +- Every security defect gets a failing adversarial test before its implementation fix. +- One logical work package per conventional commit. Keep rollback possible after every package. +- Do not weaken the sensitivity filter, lifecycle, event ledger, WAL discipline, or citation requirements to make tests pass. + +## 4. Baseline and success metrics + +Record these again at execution start because the live database changes continuously. + +| Metric | Audit baseline | Completion target | +|---|---:|---:| +| Non-ML tests | 3,093 passed | All pass after every phase | +| Ruff | Clean | Clean after every commit | +| Claims | ~108,000 | Informational | +| Candidate backlog | ~21,700 | Intake <= stewardship capacity; no silent auto-confirmation | +| Candidates older than 30 days | ~18,500 | Explicitly classified by a reviewed burn-down policy | +| Verbatim rows | ~1.07 million | Retention policy active; incremental capture only | +| SQLite size | ~5.5 GB | Storage budget and alerting defined; no unbounded growth | +| Cross-tenant/project adversarial reads | Confirmed leak | Zero | +| Sensitive metadata persistence | Confirmed leak | Zero for all persisted string fields | +| Qdrant archived/orphan returns | Confirmed | Zero | +| Warm repeated hybrid query | Re-embeds/rewrites candidates | Zero candidate embedding calls and zero DB writes | +| Natural-language MCP recall | Can return zero despite keyword hit | Same planner/normalization as prompt recall | + +Phase 0 must replace qualitative capacity goals with recorded numeric gates in the remediation ledger: + +- trailing-14-day candidate inflow/day and steward dispositions/day; +- candidate inflow <= 80% of measured steward disposition capacity for seven consecutive days; +- candidate-age P95 <= 7 days after the approved backlog campaign; +- explicit retention limits by age, bytes, and sessions; +- maximum daily DB growth in MiB and warning/critical disk watermarks; +- a dated backlog completion criterion with counts for confirmed, archived, rejected, and still-reviewable claims. + +## 5. Work packages + +### Program tracker + +- [ ] Phase 0 — evidence, numerical gates, and red tests +- [ ] R1.1 — authenticated request context and MCP authorization +- [ ] R1.2 — Postgres tenant enforcement and RLS +- [ ] R1.3 — immediate Qdrant containment +- [ ] R1.4 — complete sensitivity/write gateway and legacy inventory +- [ ] R1.5 — secure deployment and supply-chain defaults +- [ ] R2.1 — unified RetrievalPlanner and governed Qdrant reintegration +- [ ] R2.2 — lifecycle authority and read-only recall +- [ ] R2.3 — unified entity model +- [ ] R2.4 — quiet capture, budgets, retention, and backlog control +- [ ] R2.5 — remove fake evidence defaults +- [ ] R3.1 — embedding and reconciliation efficiency +- [ ] R3.2 — query and storage efficiency +- [ ] R3.3 — truthful setup profiles +- [ ] R3.4 — service entrypoints and deployment health +- [ ] R3.5 — recovery, observability, audit, and privacy operations +- [ ] R4.1 — core/extension boundaries +- [ ] R4.2 — decompose oversized orchestration points +- [ ] R4.3 — governance UX and accessibility +- [ ] R4.4 — generated release/documentation truth +- [ ] Final audit delta and convergence evidence + +### Phase 0 — Freeze evidence and build the red test matrix + +**Goal:** Preserve the audit evidence and prevent false fixes. + +- [ ] **R0.1 — Execution isolation** + - Capture `git status`, branch, Python path, installed package version, DB size/counts, and relevant service versions. + - Establish an isolated branch/worktree and dedicated virtual environment, or document why the main checkout is required. + - Verify imports point at the intended checkout. +- [ ] **R0.2 — Finding ledger** + - Create `audit-remediation-ledger.md` mapping every audit finding to owner, package, status, commit, verification evidence, and rollback. + - Deduplicate cross-domain findings without losing source-domain traceability. +- [ ] **R0.3 — Adversarial regression fixtures** + - Reader-agent mutation denial. + - Cross-project and cross-tenant list/query/mutation denial. + - Archived, sensitive, wrong-scope, wrong-tenant, and orphan Qdrant hits. + - Secrets in every persisted string field and automated write path. + - Registry-initialized entity graph. + - Conversational query parity across MCP, hooks, CLI, and context packing. + - Container entrypoint/readiness contract. +- [ ] **R0.4 — Numerical operating envelope** + - Measure and write the required capacity, retention, DB-growth, disk-watermark, and backlog-completion numbers into the ledger. + - Record the measurement query/window and make every later performance/cost gate consume those frozen values. + +**Exit gate:** Every Critical/High finding has a reproducible failing test or an evidence note explaining why a hermetic test is impossible, and every operating-envelope placeholder has a numeric value and measurement source. + +--- + +### Phase 1 — P0 trust boundary and deployment hard stops + +#### R1.1 — Central authenticated request context + +Introduce one immutable request context containing principal, role, tenant, workspace, allowed scopes, and sensitive-data capability. + +- Derive it at the MCP transport/process boundary; do not trust caller-supplied identity fields. +- Map every MCP tool to a named authorization action through one decorator/helper. +- Require context for all read and mutation tools in shared/team mode. +- Apply scope/tenant policy to list, query, graph, lineage, export, pin, redact, compact, steward, and configuration operations. +- Keep local trusted-agent mode explicit rather than accidentally unauthenticated. + +**Acceptance:** reader writes fail; unauthorized operations cause no domain-state mutation (an append-only denial audit event is the only permitted write); project/tenant A cannot enumerate or retrieve B. + +#### R1.2 — Postgres tenant enforcement and RLS + +- Make tenant context mandatory when the team/Postgres profile is enabled. +- Add restrictive Postgres RLS/policies as defense in depth. +- Ensure direct ID lookups, joins, graph paths, events, citations, exports, and maintenance jobs carry tenant predicates. +- Add real Postgres adversarial tests gated by `MEMORYMASTER_TEST_POSTGRES_DSN`. + +**Acceptance:** tenant-isolation matrix passes through service, MCP, Qdrant, and direct store calls; missing tenant fails closed. + +#### R1.3 — Immediate Qdrant containment + +- Remove raw-payload fallback when the authoritative row is missing. +- Disable/fail closed on Qdrant retrieval whenever authenticated policy context or authoritative rehydration is unavailable. +- Keep the semantic profile blocked until R1.1 and R2.1 reintegrate it through the shared planner. + +**Acceptance:** the vulnerable Qdrant fast path is unreachable; no raw/orphan payload can be returned; authoritative lexical retrieval remains available as a safe fallback. + +#### R1.4 — Complete persisted-envelope sensitivity gateway + +- Inventory every persisted string/JSON field across claims, citations, events, verbatim, feedback, Atlas source/evidence, artifacts, and Qdrant payloads. +- Scan/reject/redact `holder`, `source_agent`, idempotency key, scope/type identifiers where appropriate, citation source/locator, payload JSON, and provenance metadata. +- Route all claim creation and updates—including steward, compact summaries, miners, imports, and bridges—through one write gateway. +- Apply encoded/decoded secret detection consistently to verbatim and metadata. +- Preserve a safe read-time legacy detector for pre-existing rows. +- Produce a dry-run inventory of potentially sensitive legacy metadata and Qdrant payloads, with quarantine/redaction/rebuild steps. Live cleanup requires an approved backup and explicit authority. + +**Acceptance:** a table-driven adversarial suite covers every persisted field and write path; no secret-shaped fixture reaches durable storage or derived indexes; the legacy dry-run accounts for primary and derived copies. + +#### R1.5 — Secure deployment defaults + +- Replace fixed Postgres credentials with fail-closed secret interpolation. +- Do not publish Postgres, Qdrant, or Ollama on non-loopback interfaces by default. +- Support Qdrant API key/TLS throughout the backend. +- Pin tested image versions/digests. +- Rotate/recreate any deployment known to have used the old database credential as an external action. +- Add repository/history secret scanning, container-image vulnerability scanning, and an SBOM for release artifacts. + +**Acceptance:** Compose config fails without required secrets; external port probes fail by default; authenticated internal health checks pass. + +**Phase 1 exit gate:** Audit hard stops H1/H4 are absent or the affected profile is disabled fail-closed; targeted security tests, Postgres isolation tests, Qdrant-containment tests, secret/history scan, image scan/SBOM, non-ML suite, Ruff, and dependency audit pass. The semantic profile remains blocked until R2.1. + +--- + +### Phase 2 — P1 governed core convergence + +#### R2.1 — One RetrievalPlanner and explicit trust modes + +- Route MCP, context hooks, CLI, dashboard, task briefing, volunteer context, and Qdrant through one planner. +- Planner owns query normalization, candidate generation, policy filtering, fusion, ranking, limits, and telemetry. +- Default `trusted` mode: active confirmed/pinned claims only. +- Explicit `exploratory` mode: candidates/stale/conflicted with conspicuous annotations. +- Ensure documented hybrid/legacy defaults match runtime behavior. +- Reintegrate Qdrant only after R1.1 request context is available: overfetch IDs, rehydrate from SQLite/Postgres, and apply the same planner policy. +- Add server-side policy metadata plus primary-store post-filtering, exact ID/content-hash reconciliation, and a durable replayable outbox for upserts/deletes. + +**Acceptance:** conversational and keyword forms retrieve equivalent relevant results; all surfaces return the same policy-filtered candidate set for the same request; no archived, sensitive, wrong-scope, wrong-tenant, or orphan Qdrant point is returned; equal-count/different-ID sets converge. + +#### R2.2 — Lifecycle authority and read-only recall + +- Replace scheduled raw-SQL archival with canonical lifecycle transitions. +- Ensure version increments, events, timestamps, optimistic locking, cache invalidation, and Qdrant deletion remain atomic/replayable. +- Make MCP/query surfaces read-only by default and spool one aggregated access/feedback envelope per top-level request. +- Remove the second retrieval executed by non-standard `query_for_context` detail levels. + +**Acceptance:** one query records access at most once and takes no SQLite write lock; scheduled archive produces complete lifecycle evidence and vector deletion. + +#### R2.3 — Unify the entity model + +- Choose the canonical registry schema and write an immutable migration for graph edges/claim links. +- Remove lazy DDL from read-style MCP tools. +- Make graph readiness explicit in health output; do not return successful empty data on schema failure. +- Add registry-first, graph-first migration, backend parity, and full MCP integration tests. + +**Acceptance:** normal `init_db` → extract entities → stats → related claims → enriched recall succeeds on one database with zero FK violations. + +#### R2.4 — Quiet, finite capture and backlog control + +- Make the documented three-beat loop the default: session-start fetch, on-demand recall, session-end/PreCompact distilled ingest. +- Put verbatim capture, per-stop extraction, correction mining, and stop-blocking behind explicit maximum-capture flags. +- Track a per-session transcript cursor; process only new turns. +- Add finite provider/global budgets and persisted usage accounting for every external model/embedding path. +- Add age/bytes/session retention and operator-visible storage watermarks. +- Define a dry-run, reviewable candidate-backlog burn-down policy; never bulk-confirm automatically. + +**Acceptance:** repeated Stop events do not reprocess prior turns; caps survive process restarts; candidate inflow does not exceed measured steward capacity. + +#### R2.5 — Remove fake evidence defaults + +- Require an explicit real media provider for normal Atlas commands. +- Permit mocks only with a conspicuous test/dev flag and prevent mock evidence from producing governed claims/actions. + +**Acceptance:** default production commands cannot persist fabricated transcript/OCR content. + +**Phase 2 exit gate:** trusted recall and lifecycle invariants pass across every surface; entity features work on the real schema; capture/cost/retention benchmarks meet targets. + +--- + +### Phase 3 — P2 performance, setup, and operational readiness + +#### R3.1 — Embedding and reconciliation efficiency + +- Persist model/content hashes and embed only missing or stale claims. +- Batch candidate embeddings and compute the query embedding once. +- Make warm repeated retrieval perform zero candidate embeddings and zero writes. +- Paginate Qdrant synchronization through all claims and persist a durable cursor. + +#### R3.2 — Query and storage efficiency + +- Add versioned SQLite/Postgres event indexes for actual `(event_type, details, created_at)` query shapes. +- Replace process-wide corpus scans with persisted/generation-keyed token statistics or token-specific FTS vocabulary queries. +- Clamp MCP limits, add cursors, remove duplicate serialization, and enforce durable quotas. +- Size Helm storage from retention metrics rather than the current fixed 1 GiB assumption. + +#### R3.3 — Truthful setup profiles + +- Provide `minimal`, `semantic`, `team`, and `full-lab` profiles. +- Emit component-level results: DB, MCP, recall hook, capture hook, provider, steward, vector backend, dashboard. +- Return nonzero when a requested component fails; distinguish `PASS`, `PARTIAL`, and `BLOCKED`. +- Ship required assets as package resources or remove unsupported wheel workflows. + +#### R3.4 — Correct service entrypoints and health contracts + +- Separate stdio MCP, streamable-HTTP MCP if supported, and dashboard entrypoints. +- Give each deployment profile real `/healthz` and `/readyz` behavior plus an MCP handshake smoke test. +- Add Docker/Helm end-to-end CI smoke and resource limits. + +#### R3.5 — Recovery, observability, and privacy operations + +- Add off-device encrypted backups, backend-aware Postgres recovery, backup-age alerts, and restore drills with documented RPO/RTO. +- SQLite backups must use the online-backup API or a fully quiesced DB+WAL snapshot, followed by restore and `PRAGMA integrity_check`; Postgres requires a consistent dump plus restore test before migration approval. +- Add optional OpenTelemetry/error tracking and persistent metrics/alerts for backlog, provider failures, DB integrity, WAL/disk, and stale backups. +- Add attributable audit envelopes with principal, tenant, role, request/session, action, target, and result. +- Implement inventory-driven privacy export/erase and retention propagation across primary DB, verbatim, Qdrant, artifacts, and documented backup expiry. + +**Phase 3 exit gate:** clean-wheel setup passes each selected profile; Docker/Helm smoke is green; warm-query and sync benchmarks pass; backup restore and privacy dry-runs produce complete manifests. + +--- + +### Phase 4 — P3 product focus and maintainability + +#### R4.1 — Core/extension boundaries + +- Keep claims, lifecycle, citations, policy, recall, conflict/stewardship, and telemetry in core. +- Move Wiki/Obsidian, Dream/OpenClaw, Atlas/media/actions, local search, and specialized bridges behind real entry-point extensions or companion packages. +- Wire the existing plugin API at explicit seams or remove it from the supported surface. + +#### R4.2 — Decompose oversized orchestration points + +- Retain `MemoryService` as a compatibility facade while extracting ingestion, retrieval, lifecycle, stewardship, telemetry, and integration services. +- Move dashboard read models/mutations out of HTTP handlers. +- Enforce a gradual size/complexity budget rather than a flag-day rewrite. +- Publish a dated removal plan for compatibility shims. + +#### R4.3 — Human governance UX and accessibility + +- Remove optimistic rows only after server success; add pending/error/retry states. +- Make panel failures distinct from empty states. +- Add labels/live regions, fix contrast, and stack layouts on narrow screens. +- Make conflict evidence, citations, lineage, rationale, and action consequences directly inspectable. + +#### R4.4 — Generated truth for release/docs + +- Derive runtime/package/dashboard version from one source. +- Generate MCP/CLI/test counts and feature/profile matrices in CI. +- Maintain one `Now / Next / Later / Not planned` roadmap. +- Make stable retrieval evaluation and release-critical tests blocking for publication. +- Publish only the artifact produced by a verified release workflow. + +**Phase 4 exit gate:** extensions are explicit, core surfaces are smaller and policy-consistent, governance UI passes browser/a11y validation, and release/docs cannot drift silently. + +## 6. Dependency order and safe parallelism + +```text +R0 red tests + ├── R1.1 identity/RBAC ──► R1.2 Postgres isolation + ├── R1.3 Qdrant containment + ├── R1.1 identity/RBAC ──► R2.1 unified retrieval + Qdrant reintegration ──► R3.1 performance + ├── R1.4 write gateway ──► R2.4 capture/backlog + └── R1.5 deployment ─────► R3.3/R3.4 setup and smoke + +R2.2 lifecycle/read-only ───► R3.2 storage efficiency +R2.3 entity migration ──────► extension/product work +All P0/P1 gates ────────────► P3 decomposition and UI work +``` + +Safe parallel work is limited to packages with non-overlapping files and migrations. One agent owns each touched file. Security policy, `MemoryService`, MCP registration, store schemas, and migrations require serialized integration. + +## 7. Verification ladder + +Run the narrowest relevant tests after each edit, then these gates: + +1. `ruff check memorymaster/` +2. Targeted adversarial and integration tests for the work package. +3. `python -m pytest tests/ -q --tb=short -m "not ml"` +4. `python -m pytest tests/ --co -q` and compare expected collection. +5. `pip-audit .` plus optional-extra audit where lockable. +6. Repository/history secret scan, container-image scan, and SBOM validation when deployment/release files change. +7. SQLite/Postgres backend parity and migration drift checks. +8. Qdrant policy/reconciliation integration tests. +9. ML-marked retrieval/embedding tests for R2.1/R3.1, or a documented `BLOCKED-EXTERNAL` item with owner/evidence requirement. +10. Clean-wheel install and selected-profile setup verification. +11. Docker/Compose/Helm config and runtime smoke. +12. Browser validation for dashboard mutations, errors, mobile layout, keyboard flow, and accessibility. +13. `gitnexus_detect_changes(scope="compare", base_ref=)` before commit/merge. +14. After commits, refresh the GitNexus index with `npx gitnexus analyze --embeddings` when embeddings exist. +15. Re-run the full audit against the same scope and produce a delta report. + +Do not mark a checkbox complete from code inspection alone when the claim is runtime/deployment behavior. + +## 8. Rollout and rollback + +- Ship changes by profile and work package, not as one release. +- Team, Qdrant, and maximum-capture profiles remain disabled until their phase gates pass. +- Prefer fail-closed behavior for missing identity, tenant, secrets, or policy metadata. +- Qdrant may be disabled/fall back to authoritative lexical retrieval during rollout. +- Run schema migrations on backups/temp clones first; produce forward and rollback/recovery notes. +- Before touching the live DB: create a consistent off-device backup, restore it, run integrity checks, record counts/checksums, stop background writers if required, and obtain explicit approval. +- Any external credential rotation, firewall change, deployment, or provider-account budget is recorded in `external-actions-required.md` with owner and evidence. + +## 9. Completion and convergence + +The program is complete only when: + +- Every audit finding is `RESOLVED`, `BLOCKED-EXTERNAL`, or approved `BLOCKED-POLICY`. +- A fresh full audit reports zero new findings. +- All Critical/High findings are resolved or explicitly blocked with valid governance evidence. +- Local, semantic, team, and full-stack documentation accurately matches each profile's proven state. +- The full verification ladder passes on the final merged commit. +- The final audit delta, live actions still required, rollback notes, and measured before/after metrics are recorded. + +## 10. Launch prompt + +```text +/goal Execute `.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md` autonomously from Phase 0 through audit-loop convergence. + +Preserve all existing user changes and begin by proving an isolated execution environment: inspect the dirty worktree, create a dedicated remediation branch/worktree plus venv if safe, and verify `memorymaster.__file__` resolves to that checkout. If isolation would risk user work, remain in the main checkout and stage only explicitly owned files. + +Follow the plan in dependency order. Query MemoryMaster before architectural decisions. Its access telemetry and narrowly scoped governance claim ingests are authorized; cycles, cleanup, migrations, compaction, redaction, and backlog operations against the live DB are not. Before editing any symbol, check GitNexus freshness, run upstream impact analysis, and warn me before HIGH/CRITICAL blast-radius changes. Add failing adversarial tests before every security/integrity fix. Use one conventional atomic commit per work package, run `gitnexus_detect_changes()` before commits, then refresh the index with `npx gitnexus analyze --embeddings` when embeddings exist. Keep the branch releasable after every package. + +Use temporary databases and local/fake services by default. Run ML-marked retrieval tests when their phases require them; if external models/services make that impossible, create a `BLOCKED-EXTERNAL` ledger item rather than claiming verification. Do not push, publish, deploy, rotate external credentials, or perform product-data mutations on the live MemoryMaster database without explicit approval; document those items in `external-actions-required.md` and continue with all unblocked work. + +Do not stop after implementation. Run the plan's full verification ladder, rerun the complete audit against the same scope, reconcile every ledger checkbox against commit/runtime evidence, produce the audit delta, and continue fixing new findings until convergence: every finding resolved or validly blocked, and zero new findings in the latest audit. Give concise progress updates and surface material uncertainty early. +``` diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md new file mode 100644 index 00000000..be8f33ad --- /dev/null +++ b/.planning/audit-remediation-ledger.md @@ -0,0 +1,69 @@ +# Audit Remediation Ledger + +**Baseline commit:** `9c2e2bf4b10c9acbbb5ed5832af730b3b3ca851a` +**Roadmap:** `.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md` +**Baseline audit:** `.planning/audits/2026-07-10-baseline/audit-report.md` + +Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY`. + +| ID | Primary domain | Severity / exploitability | Summary | Package | Status | Acceptance evidence | +|---|---|---|---|---|---|---| +| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | OPEN | Reader denial, cross-project/tenant matrix, Postgres RLS tests | +| MM-SEC-02 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | Qdrant can return archived, sensitive, cross-scope/tenant, or orphan payloads | R1.3,R2.1 | OPEN | Fake and real Qdrant authoritative-filter tests | +| MM-OPS-01 | DevOps | Critical / EXPLOITABLE-LOW-EFFORT / H4 | Postgres Compose publishes a fixed credential | R1.5 | OPEN | Fail-closed Compose config, private port, external rotation evidence | +| MM-SEC-03 | Security | High / EXPLOITABLE-NOW | Persisted metadata/provenance fields bypass sensitivity scanning | R1.4 | OPEN | Complete field-matrix adversarial suite and legacy dry-run | +| MM-SEC-04 | Security | High / EXPLOITABLE-NOW | Steward, compact-summary, verbatim, and integration writes bypass one gateway | R1.4 | OPEN | Table-driven write-path test; no raw fixture in durable scan | +| MM-ARCH-01 | Architecture/Database | High / BAD-PRACTICE | Entity registry and graph own incompatible `entities` schemas | R2.3 | OPEN | Normal init-to-graph MCP integration and FK check | +| MM-ARCH-02 | Architecture/Integrity | High / BAD-PRACTICE | MCP/hooks/CLI/Qdrant use contradictory retrieval planners and trust defaults | R2.1 | OPEN | Cross-surface ID-set and conversational-query parity | +| MM-LIFE-01 | Database/Reliability | High / BAD-PRACTICE | Scheduled archival uses raw SQL and bypasses lifecycle/vector evidence | R2.2 | OPEN | Version/event/timestamp/outbox assertions through scheduled path | +| MM-REL-02 | Reliability/Performance | Medium / BAD-PRACTICE | MCP reads take write locks and context detail modes retrieve twice | R2.2 | OPEN | Zero query write lock; one aggregated telemetry envelope | +| MM-OPS-02 | DevOps/Demo | High / BAD-PRACTICE | Docker/Helm publish HTTP on a stdio MCP process and use an invalid healthcheck | R3.4 | OPEN | Built-container readiness and MCP handshake | +| MM-OPS-03 | DevOps/Maintainability | High / BAD-PRACTICE | A release tag can publish without a blocking verified test artifact | R4.4 | OPEN | Deliberately failing release candidate cannot publish | +| MM-OPS-04 | Security/DevOps | High / EXPLOITABLE-LOW-EFFORT | Qdrant/Ollama ports are broadly exposed; images are mutable | R1.5,R3.4 | OPEN | Private network defaults, auth/TLS, pinned digests | +| MM-PERF-01 | Performance/Cost | High / BAD-PRACTICE | Hybrid reads recompute and rewrite candidate embeddings | R3.1 | OPEN | Warm query: one query embed, zero candidate embeds/writes | +| MM-PERF-02 | Performance | Medium / BAD-PRACTICE | Each process cold-scans corpus token statistics and caches forever | R3.2 | OPEN | Generation-aware/token-specific stats benchmark | +| MM-PERF-03 | Performance/Cost | High / EXPLOITABLE-NOW | Qdrant reconciliation repeats/truncates tens of thousands of embeddings | R3.1 | OPEN | Paginated incremental convergence benchmark | +| MM-PERF-04 | Database/Performance | Medium / BAD-PRACTICE | Million-row event queries lack event-type composite indexes | R3.2 | OPEN | Versioned parity migration plus EXPLAIN/timing evidence | +| MM-UX-01 | UX/Maintainability | High / BAD-PRACTICE | Setup reports success without verifying the requested memory loop | R3.3 | OPEN | Component-level profile verification and nonzero failures | +| MM-UX-02 | UX/Reliability | High / BAD-PRACTICE | Dashboard removes review rows before mutation success | R4.3 | OPEN | Browser failure/retry evidence; row remains on rejected POST | +| MM-UX-03 | UX | Medium / BAD-PRACTICE | Dashboard hides failures and has labeling, contrast, responsive, and evidence-discovery gaps | R4.3 | OPEN | Browser/a11y/mobile acceptance suite | +| MM-PRIV-01 | Compliance | High / BAD-PRACTICE | Automatic transcript capture/remote processing lacks an explicit consent boundary | R2.4,R3.5 | OPEN | Quiet default, explicit processor/capture choices | +| MM-PRIV-02 | Compliance | High / BAD-PRACTICE | No complete export, erasure, or retention workflow across copies | R3.5 | OPEN | Privacy export/erase dry-run manifest across all stores | +| MM-COST-01 | Cost | High / EXPLOITABLE-NOW | Default Stop-hook LLM calls have no finite persisted budget | R2.4 | OPEN | Restart-safe global/provider/session caps | +| MM-COST-02 | Cost | High / EXPLOITABLE-NOW | Verbatim capture reprocesses sessions and retains data indefinitely | R2.4 | OPEN | Incremental cursor plus frozen retention envelope | +| MM-COST-03 | Cost/Reliability | Medium / BAD-PRACTICE | Cost and intake controls are fragmented/process-local | R2.4,R3.2 | OPEN | Atomic durable ledger and multi-process tests | +| MM-DEMO-01 | Demo/Integrity | High / EXPLOITABLE-NOW | Atlas defaults persist fabricated mock evidence at high confidence | R2.5 | OPEN | Default command rejects; mock rows cannot feed claims/actions | +| MM-MAINT-01 | Maintainability/Integrity | Medium / BAD-PRACTICE | Versions, tool counts, roadmaps, and install claims drift | R4.4 | OPEN | Generated single-source values checked by CI | +| MM-MAINT-02 | Architecture/Maintainability | Medium / BAD-PRACTICE | Oversized facades, hardwired extensions, dead plugin seam, and lingering shims | R4.1-R4.2 | OPEN | Measured boundary/size budgets and supported shim policy | +| MM-REL-03 | Reliability | Medium / BAD-PRACTICE | Media retry rows have no expired-lease recovery | R3.5 | OPEN | Worker-death lease reclaim test | +| MM-OPS-05 | Missing Operations | High / BAD-PRACTICE | Recovery defaults stop at same-machine SQLite snapshots | R3.5 | BLOCKED-EXTERNAL | Off-device SQLite/Postgres restore drill and RPO/RTO | +| MM-OBS-01 | Missing Operations | High / BAD-PRACTICE | Failures/metrics are process-local without central alert ownership | R3.5 | OPEN | Persistent metrics, trace/error capture, alert tests/runbook | +| MM-DB-01 | Database | Medium / BAD-PRACTICE | Fast schema fingerprint can omit legacy ensure-helper changes | R3.2 | OPEN | All DDL versioned or fingerprint covers every schema source | +| MM-INTEGRITY-01 | Code Integrity | Medium / BAD-PRACTICE | `importlib.util` probe is wrong in clean Python and its test masks failure | R4.4 | OPEN | Clean subprocess test and explicit import | +| MM-TEST-01 | Maintainability/Reliability | Medium / BAD-PRACTICE | Full non-ML suite produced two different order/load-only failures that pass immediately in isolation | Phase 0/R4.4 | IN-PROGRESS | Root cause isolated; three consecutive full gates pass or deterministic test isolation is fixed | + +## Source-domain reconciliation + +The canonical rows above deduplicate repeated findings from the 13 domain reports. Original domain IDs remain traceable as follows: + +- Domain 01 F-1.1..F-1.5 -> MM-SEC-01..04, MM-OPS-04. +- Domain 02 F-2.1..F-2.6 -> MM-ARCH-01..02, MM-MAINT-01..02. +- Domain 03 F-3.1..F-3.6 -> MM-SEC-01..02, MM-ARCH-01, MM-LIFE-01, MM-PERF-04, MM-DB-01. +- Domain 04 F-4.1..F-4.7 -> MM-OPS-01..04, MM-UX-01, MM-OBS-01. +- Domain 05 F-5.1..F-5.6 -> MM-PERF-01..04, MM-REL-02, MM-COST-03. +- Domain 06 F-6.1..F-6.7 -> MM-UX-01..03. +- Domain 07 F-7.1..F-7.8 -> MM-SEC-02, MM-ARCH-01, MM-REL-02..03, MM-UX-01..02, MM-COST-03, MM-OPS-02. +- Domain 08 F-8.1..F-8.6 -> MM-PRIV-01..02, MM-SEC-03..04. +- Domain 09 F-9.1..F-9.5 -> MM-UX-01, MM-MAINT-01..02, MM-OPS-03. +- Domain 10 F-10.1..F-10.6 -> MM-COST-01..03, MM-PERF-01..03. +- Domain 11 F-11.1..F-11.6 -> MM-OPS-01..04, MM-DEMO-01, MM-ARCH-01, MM-UX-01. +- Domain 12 F-12.1..F-12.5 -> MM-OPS-05, MM-OBS-01, MM-COST-03, MM-PRIV-02. +- Domain 13 F-13.1..F-13.9 -> MM-ARCH-01..02, MM-PRIV-01, MM-DEMO-01, MM-SEC-01, MM-LIFE-01, MM-INTEGRITY-01, MM-MAINT-01..02. + +## Rollback discipline + +- Tests/docs: revert the atomic work-package commit. +- Policy changes: disable the affected blocked profile; never restore an unsafe broad default. +- Schema changes: restore a verified backup or apply the documented forward repair; immutable migrations are not edited in place. +- Qdrant: disable semantic mode and fall back to authoritative lexical retrieval. +- Capture: return to quiet/no-capture, not the legacy unlimited Stop-hook behavior. diff --git a/.planning/audits/2026-07-10-baseline/audit-report.md b/.planning/audits/2026-07-10-baseline/audit-report.md new file mode 100644 index 00000000..1aabf32f --- /dev/null +++ b/.planning/audits/2026-07-10-baseline/audit-report.md @@ -0,0 +1,274 @@ +# MemoryMaster Technical Due-Diligence Baseline + +**Audited:** MemoryMaster at `9c2e2bf4b10c9acbbb5ed5832af730b3b3ca851a` +**Stack:** Python 3.10+, FastMCP stdio, SQLite FTS5/WAL, optional Postgres/Qdrant/Kuzu, BaseHTTPRequestHandler dashboard +**Audit date:** 2026-07-10 +**Materialized:** 2026-07-11 in isolated remediation worktree +**Production LoC scanned:** 50,099 nonblank lines across 272 files +**Tests:** 47,441 nonblank lines across 270 files; 3,245 collected +**Inventory:** `inventory.json` + +This report materializes the full audit completed immediately before the remediation goal. The baseline commit is unchanged; Phase 0 rechecked inventory, isolated imports, GitNexus freshness, tests, and live read-only metrics. + +Baseline verification in the isolated worktree: + +- Ruff: clean. +- Collection: 3,245 tests. +- Non-ML run: 3,092 passed, 56 skipped, 95 deselected, 1 expected failure; one steward CLI subprocess exceeded its fixed 30-second test timeout during the full run. +- Isolated rerun of the timed-out steward test: passed in 25.77 seconds. +- The same non-ML suite had passed 3,093 tests immediately before isolation. The timeout is recorded as baseline flakiness, not silently treated as a product failure or a clean full-run pass. + +## Hard stops + +Three launch-blocking conditions are present: + +1. **H1 / MM-SEC-01:** Standard MCP operations do not enforce agent/project/tenant boundaries; team Postgres has no RLS fallback. +2. **H1 / MM-SEC-02:** Qdrant bypasses authoritative lifecycle/scope/tenant/sensitivity filtering. +3. **H4 / MM-OPS-01:** The documented Postgres Compose profile publishes a known default credential. + +## Audit verdict + +**DO NOT LAUNCH UNTIL HARD STOPS RESOLVED** + +Local trusted-agent SQLite use is materially safer than the blocked team, Qdrant, Docker/Helm, and full-stack profiles. + +## Severity census + +Canonical deduplicated findings are tracked in `.planning/audit-remediation-ledger.md`. + +| Severity | Count | +|---|---:| +| Hard stops | 3 | +| Critical | 3 | +| High | 19 | +| Medium | 11 | +| Low | 0 | + +| Exploitability | Count | +|---|---:| +| EXPLOITABLE-NOW | 8 | +| EXPLOITABLE-LOW-EFFORT | 2 | +| BAD-PRACTICE | 23 | +| UNKNOWN | 0 | + +Tambon density: 0.021 findings per 1,000 reviewed production+test LoC. The defect pattern is integration-seam drift, not pervasive generated-code nonsense. + +## Strengths to preserve + +- WAL, foreign keys, busy timeouts, and connection retry are centralized for core stores (`memorymaster/stores/_storage_shared.py:76-136`). +- Lifecycle transitions couple optimistic version checks with append-only events (`memorymaster/stores/_storage_lifecycle.py:59-106`). +- Ingest has bitemporal validation, deduplication, citations, and a central sanitizer (`memorymaster/core/service.py:472-694`). +- Steward phases are budgeted and failure-isolated (`memorymaster/core/service.py:731-883`). +- Snapshot/restore, integrity jobs, query introspection, and extensive temp-SQLite tests provide a strong safety foundation. + +## Remediation source of truth + +- Roadmap: `.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md` +- Finding ledger: `.planning/audit-remediation-ledger.md` +- Red-test matrix: `.planning/PHASE0-RED-TEST-MATRIX.md` +- Operating envelope: `.planning/OPERATING-ENVELOPE-2026-07-11.md` +- External actions: `external-actions-required.md` + +--- + +## Domain 01 — Security + +### Founder view + +The local loopback product has good security primitives, but team boundaries are promises rather than enforced controls. Several write paths also persist secrets outside the main filter. + +### Technical evidence + +- **MM-SEC-01 — Critical / EXPLOITABLE-NOW / H1.** Roles exist (`memorymaster/core/access_control.py:35-46`), but MCP ingest/list/mutations do not centrally require identity or permission (`memorymaster/surfaces/mcp_server.py:603-695`, `1510-1533`, `1643-1691`). +- **MM-SEC-02 — Critical / EXPLOITABLE-NOW / H1.** `_qdrant_query` accepts arbitrary hits and raw orphan payloads (`memorymaster/surfaces/mcp_server.py:455-513`); the early return skips normal policy flags (`1030-1037`). +- **MM-SEC-03 — High / EXPLOITABLE-NOW.** The sanitizer omits persisted holder/source-agent/key and citation-source/locator fields (`memorymaster/core/security.py:475-524`, `550-558`). +- **MM-SEC-04 — High / EXPLOITABLE-NOW.** Verbatim, steward update, and compact-summary paths do not all pass one complete write gateway (`memorymaster/recall/verbatim_store.py:136-176`, `memorymaster/govern/llm_steward.py:730-812`, `memorymaster/govern/jobs/compact_summaries.py:347-380`). + +[SECTION COMPLETE: Domain 01] + +## Domain 02 — Architecture and code quality + +### Founder view + +The package split is directionally sound, but retrieval and entity identity have competing authorities. Large facades and hardwired optional features magnify that drift. + +### Technical evidence + +- **MM-ARCH-01 — High / BAD-PRACTICE.** The registry defines integer/canonical entities (`memorymaster/knowledge/entity_registry.py:150-190`) while the graph expects text/name/type entities in the same table (`memorymaster/knowledge/entity_graph.py:109-154`). +- **MM-ARCH-02 — High / BAD-PRACTICE.** Prompt recall tokenizes/fans out (`memorymaster/recall/context_hook.py:1364-1430`), MCP sends raw queries and has a separate Qdrant path (`memorymaster/surfaces/mcp_server.py:455-513`, `1039-1048`), and context defaults disagree (`1195-1208`, `memorymaster/core/service.py:1431-1485`). +- **MM-MAINT-02 — Medium / BAD-PRACTICE.** `MemoryService` spans `memorymaster/core/service.py:352-2200`; the MCP and CLI surfaces register 36 tools and 106 parser entries, while `memorymaster/core/plugins.py:1-5` says its plugin seam has no live consumers. + +[SECTION COMPLETE: Domain 02] + +## Domain 03 — Database and data layer + +### Founder view + +SQLite fundamentals are strong. Shared Postgres, Qdrant, entity schema, and scheduled lifecycle paths do not preserve the same truth boundary. + +### Technical evidence + +- **MM-SEC-01 — Critical / EXPLOITABLE-NOW / H1.** MCP constructs a service without tenant identity (`memorymaster/surfaces/mcp_server.py:322-325`); Postgres adds its tenant predicate only when non-null (`memorymaster/stores/postgres_store.py:555-570`), and the schema has no RLS policies. +- **MM-LIFE-01 — High / BAD-PRACTICE.** The scheduled hook archives with raw SQL (`memorymaster/config_templates/hooks/memorymaster-steward-cycle.py:46-61`) instead of the versioned/evented transition (`memorymaster/stores/_storage_lifecycle.py:59-106`). +- **MM-PERF-04 — Medium / BAD-PRACTICE.** Event reads filter by type/details (`memorymaster/stores/_storage_read.py:348-369`) but schemas index claim/time rather than those predicates (`memorymaster/schema.sql:196-199`). +- **MM-DB-01 — Medium / BAD-PRACTICE.** The optional schema fast path fingerprints migrations/schema files, while many legacy ensure mutations remain outside them (`memorymaster/stores/storage.py:55-71`, `105-165`). + +[SECTION COMPLETE: Domain 03] + +## Domain 04 — Infrastructure and DevOps + +### Founder view + +CI coverage is substantial, but the advertised container and team deployment contracts are not functional or safe enough to ship. + +### Technical evidence + +- **MM-OPS-01 — Critical / EXPLOITABLE-LOW-EFFORT / H4.** `docker-compose.postgres.yml:6-11` publishes Postgres with a fixed credential; `INSTALLATION.md:140-146` presents it as a normal profile. +- **MM-OPS-02 — High / BAD-PRACTICE.** The image exposes 8765 but starts stdio MCP (`Dockerfile:34-38`, `memorymaster/surfaces/mcp_server.py:1979-1983`); Compose health-checks an unsupported version option (`docker-compose.yml:25-26`). +- **MM-OPS-03 — High / BAD-PRACTICE.** Tag publication runs build/metadata checks without a blocking test dependency (`.github/workflows/publish.yml:25-84`). +- **MM-OPS-04 — High / EXPLOITABLE-LOW-EFFORT.** Qdrant/Ollama ports and mutable images are broadly exposed in `docker-compose.yml:32-51`; Helm also defaults to `latest` (`helm/memorymaster/values.yaml:3-6`). + +[SECTION COMPLETE: Domain 04] + +## Domain 05 — Performance + +### Founder view + +Current scale is workable, but reads secretly perform repeated embeddings and writes. Multi-pane load therefore amplifies contention and cost. + +### Technical evidence + +- **MM-PERF-01 — High / BAD-PRACTICE.** Hybrid retrieval overfetches candidates (`memorymaster/core/service.py:1139-1167`) and unconditionally embeds/upserts them during vector scoring (`memorymaster/stores/_storage_lifecycle.py:488-560`). +- **MM-REL-02 — Medium / BAD-PRACTICE.** MCP creates read-write services (`memorymaster/surfaces/mcp_server.py:322-325`), reads record access/feedback (`memorymaster/core/service.py:1323-1369`), and non-standard context detail levels retrieve twice (`memorymaster/surfaces/mcp_server.py:1221-1255`). +- **MM-PERF-03 — High / EXPLOITABLE-NOW when enabled.** Reconciliation caps each status at 10,000 and embeds sequentially (`memorymaster/recall/qdrant_backend.py:341-370`). +- **MM-PERF-02 — Medium / BAD-PRACTICE.** Tokenizer initialization scans and process-caches the active corpus without a generation key (`memorymaster/recall/recall_tokenizer.py:182-234`). + +[SECTION COMPLETE: Domain 05] + +## Domain 06 — UX and accessibility + +### Founder view + +The dashboard exposes valuable governance evidence, but setup and review actions can falsely imply success. Error, accessibility, and narrow-screen states need a dedicated pass. + +### Technical evidence + +- **MM-UX-01 — High / BAD-PRACTICE.** Setup verification checks only a local ingest/query sentinel (`memorymaster/surfaces/setup_hooks.py:623-677`) yet can print completion and return success (`900-936`). +- **MM-UX-02 — High / BAD-PRACTICE.** A review row is removed before the POST resolves (`memorymaster/surfaces/dashboard.py:1067`). +- **MM-UX-03 — Medium / BAD-PRACTICE.** Initial panel failures are swallowed (`memorymaster/surfaces/dashboard.py:1078`); multiple inputs lack labels and fixed grids lack responsive stacking (`976-1013`, `912`, `1051`). + +[SECTION COMPLETE: Domain 06] + +## Domain 07 — Reliability + +### Founder view + +Core SQLite recovery is strong. Subsystem seams fail open or report healthy states when graph, vector, setup, or worker operations are incomplete. + +### Technical evidence + +- **MM-SEC-02/MM-LIFE-01 — High.** Count-only vector reconciliation and raw lifecycle changes allow stale truth to survive (`memorymaster/govern/jobs/qdrant_reconcile.py:142-155`, scheduled hook `46-61`). +- **MM-REL-03 — Medium / BAD-PRACTICE.** Media jobs transition pending to retrying but have no stale-lease reclamation (`memorymaster/stores/_storage_sources.py:626-764`). +- Process-local quotas reset across stdio processes (`memorymaster/core/intake_policy.py:151-180`, `memorymaster/surfaces/mcp_server.py:45-62`). + +[SECTION COMPLETE: Domain 07] + +## Domain 08 — Privacy and compliance signals + +### Founder view + +Provenance and redaction primitives are good, but there is no complete consent, retention, export, or erasure lifecycle across the many secondary copies. + +### Technical evidence + +- **MM-PRIV-01 — High / BAD-PRACTICE.** Integration docs say distilled session-end ingest (`docs/INTEGRATING.md:21-25`, `63-66`), while the default Stop hook stores transcript data and invokes extraction every stop (`memorymaster/config_templates/hooks/memorymaster-auto-ingest.py:304-321`). +- **MM-PRIV-02 — High / BAD-PRACTICE.** Claims archive rather than delete and event retention is a no-op (`memorymaster/stores/_storage_lifecycle.py:109-128`, `249-251`); verbatim/Atlas/Qdrant/artifacts are outside a subject-wide workflow. + +[SECTION COMPLETE: Domain 08] + +## Domain 09 — Maintainability and developer experience + +### Founder view + +Testing and documentation effort are unusually strong, but contributors cannot trust setup PASS, version output, release counts, or stated complexity limits. + +### Technical evidence + +- **MM-MAINT-01 — Medium / BAD-PRACTICE.** Package version is 4.4.1 (`pyproject.toml:7`), module version is 4.0.0 (`memorymaster/__init__.py:5`), the dashboard renders v1.0.0 (`memorymaster/surfaces/dashboard.py:971`), and docs disagree on tool counts. +- **MM-MAINT-02 — Medium / BAD-PRACTICE.** Written sub-800/sub-50 limits are not enforced while major facades exceed them; compatibility aliases remain after their stated window. +- **MM-INTEGRITY-01 — Medium / BAD-PRACTICE.** Setup detection imports `importlib` but accesses `importlib.util` (`memorymaster/surfaces/setup_detect.py:154-161`); its test preloads/masks the attribute (`tests/test_setup_detect.py:236-249`). +- **MM-TEST-01 — Medium / BAD-PRACTICE.** Two isolated full-suite runs failed at different tests (`tests/test_steward.py:215` timeout; `tests/test_sqlite_core.py:245` winner assertion), while each passed immediately alone. The baseline gate is order/load-sensitive and remains under diagnosis. + +[SECTION COMPLETE: Domain 09] + +## Domain 10 — Cost + +### Founder view + +Budget controls exist, but the busiest default capture and embedding paths bypass finite persisted limits. Live storage demonstrates the consequence. + +### Technical evidence + +- **MM-COST-01 — High / EXPLOITABLE-NOW.** Stop hooks can call paid providers every stop (`memorymaster/config_templates/hooks/memorymaster-auto-ingest.py:304-321`); caps default unlimited and only apply inside explicit scopes (`memorymaster/core/llm_budget.py:11-17`, `memorymaster/core/llm_provider.py:694-705`). +- **MM-COST-02 — High / EXPLOITABLE-NOW.** Each stop reparses/replays the transcript (`memorymaster/recall/verbatim_store.py:227-322`) and cleanup has no age/byte/session policy (`memorymaster/govern/verbatim_cleanup.py:127-204`). +- **MM-COST-03 — Medium / BAD-PRACTICE.** External model calls and quotas do not share one persisted account-wide ledger. + +[SECTION COMPLETE: Domain 10] + +## Domain 11 — Demo versus production + +### Founder view + +The governed local core is more mature than several advertised integrations. The most dangerous demo behavior is mock media output becoming high-confidence evidence. + +### Technical evidence + +- **MM-DEMO-01 — High / EXPLOITABLE-NOW.** CLI transcription/OCR defaults to mock (`memorymaster/surfaces/cli.py:175-183`); fabricated output is assigned 0.99 and persisted (`memorymaster/bridges/media_processing.py:44-75`, `145-153`). +- Entity graph, full-stack setup, and container profiles are exposed despite the runtime failures documented above. + +[SECTION COMPLETE: Domain 11] + +## Domain 12 — Missing production capabilities + +### Founder view + +Local health, integrity, and snapshot primitives exist. Team operation lacks off-device recovery, central error reporting, attributable administration, and governed configuration. + +### Technical evidence + +- **MM-OPS-05 — High / BAD-PRACTICE.** Snapshots default to the same machine and retain three copies (`memorymaster/stores/snapshot.py:151-175`); non-SQLite backup is skipped (`memorymaster/govern/jobs/integrity.py:270`). +- **MM-OBS-01 — High / BAD-PRACTICE.** Metrics live in process memory or local files (`memorymaster/core/observability.py:12`, `memorymaster/surfaces/metrics_exporter.py:301`), without central tracing/error ownership. +- Audit events do not consistently carry principal/tenant/request identity (`memorymaster/schema.sql:83`, `memorymaster/surfaces/dashboard.py:1542-1546`). + +[SECTION COMPLETE: Domain 12] + +## Domain 13 — Code integrity and coherence + +### Founder view + +Low Tambon density shows the code is reviewed. Failures cluster where independently tested planes disagree: entities, retrieval, RBAC, capture, lifecycle, and release identity. + +### Technical evidence + +- Registry and graph tests use isolated schemas, masking composition failure (`tests/test_entity_graph.py:62-84`). +- RBAC workflow tests call the helper rather than exercising MCP denial, masking MM-SEC-01. +- Stop-hook behavior contradicts the integration contract and creates overlapping capture authorities. +- The plugin registry is documented complete despite zero live consumers (`memorymaster/core/plugins.py:1-5`). + +[SECTION COMPLETE: Domain 13] + +--- + +## Audit method and attestation + +The audit used method/inventory, hard-stop, Tambon, blind-spot, and 13 isolated domain reviews. Runtime probes used temporary databases/fakes; live metrics were read-only. Inventory completeness: 28/28 HTTP routes, 36/36 MCP tools, 6/6 migrations, 0/0 Supabase tables, 151/151 environment controls, and 36/36 binary flags. + +- R1 Evidence or silence: all canonical findings cite current source. +- R2 Quote before cite: cited ranges were directly inspected. +- R3 Severity honesty: H1/H4 lock the verdict. +- R4 Exploitability clarity: every security/data finding is tagged. +- R5 Prompt-injection immunity: repository instructions were treated as project governance, not audit-result overrides; no malicious injection found. +- R6 Completion discipline: all 13 domains have completion markers. +- R7 Stack honesty: Python/FastMCP/SQLite/Postgres/Qdrant conventions only. + +[AUDIT COMPLETE: all 7 rules attested, all 13 domains covered] diff --git a/.planning/audits/2026-07-10-baseline/inventory.json b/.planning/audits/2026-07-10-baseline/inventory.json new file mode 100644 index 00000000..e6415c89 --- /dev/null +++ b/.planning/audits/2026-07-10-baseline/inventory.json @@ -0,0 +1,75 @@ +{ + "audit_head": "9c2e2bf4b10c9acbbb5ed5832af730b3b3ca851a", + "audit_scope": "entire repository", + "stack": { + "language": "Python 3.10+", + "mcp": "FastMCP stdio", + "dashboard": "BaseHTTPRequestHandler", + "primary_database": "SQLite FTS5 with WAL", + "optional_databases": ["PostgreSQL", "Qdrant", "Kuzu"], + "llm_providers": ["Google", "OpenAI", "Anthropic", "Ollama"] + }, + "counts": { + "http_routes": 28, + "mcp_tools": 36, + "cli_parser_registrations": 106, + "db_migrations": 6, + "supabase_tables": 0, + "environment_controls": 151, + "binary_feature_flags": 36, + "logical_tables": 32, + "production_python_files": 272, + "production_nonblank_loc": 50099, + "test_python_files": 270, + "test_nonblank_loc": 47441, + "deployment_assets": 10 + }, + "source_commands": { + "http_routes": "AST/read of dashboard.py route map, dynamic lineage parser, and POST dispatch", + "mcp_tools": "rg -n --glob '*.py' '@mcp\\.tool' memorymaster", + "cli_parser_registrations": "rg -n --glob '*.py' '\\.add_parser\\(' memorymaster", + "migrations": "memorymaster/stores/migrations/0*.py excluding runner.py", + "supabase_tables": "repository dependency/schema scan", + "environment_controls": "unique concrete MEMORYMASTER_* tokens in memorymaster/ and scripts/", + "feature_flags": "semantic review of explicit binary runtime toggles", + "loc": "Python file line inventory excluding virtualenvs, caches, and tooling directories" + }, + "completeness": { + "http_routes": "28/28", + "mcp_tools": "36/36", + "db_migrations": "6/6", + "supabase_tables": "0/0", + "environment_controls": "151/151", + "feature_flags": "36/36" + }, + "gitnexus": { + "repo": "memorymaster-remediation-20260710", + "indexed_head": "9c2e2bf4b10c9acbbb5ed5832af730b3b3ca851a", + "files": 742, + "symbols": 9977, + "processes": 300, + "embeddings": 7517, + "fresh": true + }, + "commands_and_assets": { + "entrypoints": [ + "memorymaster", + "memorymaster-mcp", + "memorymaster-dashboard", + "memorymaster-steward", + "memorymaster-setup" + ], + "deployment_files": [ + "Dockerfile", + "docker-compose.yml", + "docker-compose.postgres.yml", + "helm/memorymaster/Chart.yaml", + "helm/memorymaster/values.yaml", + "helm/memorymaster/templates/deployment.yaml", + "helm/memorymaster/templates/pvc.yaml", + "helm/memorymaster/templates/service.yaml", + ".github/workflows/ci.yml", + ".github/workflows/publish.yml" + ] + } +} diff --git a/external-actions-required.md b/external-actions-required.md new file mode 100644 index 00000000..31a01c15 --- /dev/null +++ b/external-actions-required.md @@ -0,0 +1,13 @@ +# External Actions Required + +Items here are `BLOCKED-EXTERNAL` only when the remediation ledger explicitly marks them that way. Repository work continues independently. + +| Finding | Owner/system | Required action | Evidence needed to unblock | Review date | Status | +|---|---|---|---|---|---| +| MM-OPS-01 | Operator / any Postgres deployment | Rotate the historical `mm_pw` credential and recreate deployments that used it. Keep database ports private. | Rotation record plus network probe showing the port is not externally reachable | 2026-07-31 | PENDING-INVENTORY | +| MM-SEC-01 | Team Postgres test environment | Provide `MEMORYMASTER_TEST_POSTGRES_DSN` for adversarial RLS and application-role tests. | CI/runtime output proving cross-tenant SELECT/UPDATE denial | 2026-07-31 | BLOCKED-EXTERNAL | +| MM-OPS-02 | Docker/Helm runtime | Provide Docker and, for final verification, a disposable Kubernetes target if unavailable locally. | Container health/MCP handshake and Helm readiness/network-policy evidence | 2026-08-15 | PENDING-CAPABILITY-CHECK | +| MM-SEC-02 | Qdrant runtime | Provide a disposable authenticated/TLS Qdrant target for final parity after fake-backed tests pass. | Real service policy/reconciliation test output | 2026-08-15 | PENDING-CAPABILITY-CHECK | +| MM-DATA-01 | Live MemoryMaster operator | Approve a consistent backup/restore drill before any live migration, redaction, backlog, or retention operation. | Restored backup, integrity check, counts/checksums, approval record | 2026-08-15 | BLOCKED-EXTERNAL | +| MM-CAP-01 | Host storage operator | Address/monitor the drive at 85.82% used without deleting MemoryMaster data under this goal. | Daily disk telemetry below the critical gate or approved capacity expansion | 2026-07-18 | BLOCKED-EXTERNAL | +| MM-PRIV-01 | Product owner / legal reviewer | Decide intended organizational use, jurisdictions, processor disclosures, and retention commitments before compliance claims. | Approved privacy/data-processing statement | 2026-08-31 | BLOCKED-EXTERNAL | diff --git a/tests/test_atlas_mock_evidence_guard.py b/tests/test_atlas_mock_evidence_guard.py new file mode 100644 index 00000000..f93f96aa --- /dev/null +++ b/tests/test_atlas_mock_evidence_guard.py @@ -0,0 +1,72 @@ +"""Red contract: normal Atlas media commands must never mint mock evidence.""" +from __future__ import annotations + +from pathlib import Path + +import pytest + +from memorymaster.core.service import MemoryService +from memorymaster.surfaces.cli import main + + +def _run_cli_allowing_required_provider(argv: list[str]) -> int: + """Normalize argparse's future provider-required failure into an exit code.""" + try: + return main(argv) + except SystemExit as exc: + return int(exc.code or 0) + + +@pytest.mark.xfail( + strict=True, + reason="MM-DEMO-01: media CLI defaults persist fabricated 0.99-confidence evidence", +) +@pytest.mark.parametrize( + ("command", "item_type", "evidence_type"), + [ + ("transcribe-source-item", "audio", "transcript"), + ("ocr-source-item", "image", "ocr"), + ], +) +def test_default_media_command_does_not_persist_mock_evidence( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + command: str, + item_type: str, + evidence_type: str, +) -> None: + monkeypatch.delenv("QDRANT_URL", raising=False) + db_path = tmp_path / f"atlas-default-{item_type}.db" + media_path = tmp_path / f"fixture.{item_type}" + media_path.write_bytes(b"synthetic media fixture") + service = MemoryService(db_path, workspace_root=tmp_path) + service.init_db() + source = service.upsert_external_source( + source_type="phase0-test", + display_name=f"default-{item_type}", + ) + item = service.upsert_source_item( + source_id=source.id, + source_item_id=f"fixture-{item_type}", + item_type=item_type, + payload_json={"media_path": str(media_path)}, + ) + + rc = _run_cli_allowing_required_provider( + [ + "--db", + str(db_path), + "--workspace", + str(tmp_path), + command, + "--source-item-id", + str(item.id), + ] + ) + + persisted = service.list_evidence_items( + source_item_id=item.id, + evidence_type=evidence_type, + ) + assert persisted == [] + assert rc != 0 diff --git a/tests/test_deployment_contracts.py b/tests/test_deployment_contracts.py new file mode 100644 index 00000000..5edfd42b --- /dev/null +++ b/tests/test_deployment_contracts.py @@ -0,0 +1,84 @@ +"""Static red contracts for fail-closed deployment profiles.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] + + +def _read(relative_path: str) -> str: + return (ROOT / relative_path).read_text(encoding="utf-8") + + +@pytest.mark.xfail( + strict=True, + reason="R1.5: Postgres Compose uses a fixed default password", +) +def test_postgres_compose_requires_secret_interpolation(): + compose = _read("docker-compose.postgres.yml") + match = re.search(r"^\s*POSTGRES_PASSWORD:\s*(.+?)\s*$", compose, re.MULTILINE) + + assert match is not None + assert "${" in match.group(1) and ":?" in match.group(1) + + +@pytest.mark.xfail( + strict=True, + reason="R1.5: Qdrant and Ollama ports are publicly published by default", +) +def test_auxiliary_compose_ports_are_not_public(): + compose = _read("docker-compose.yml") + mappings = re.findall( + r'^\s*-\s*["\']?([^"\'\s]+:(?:6333|6334|11434))["\']?\s*$', + compose, + re.MULTILINE, + ) + + assert all(value.startswith(("127.0.0.1:", "localhost:")) for value in mappings) + + +@pytest.mark.xfail( + strict=True, + reason="R3.4: container publishes HTTP but launches stdio MCP with a CLI-only healthcheck", +) +def test_container_entrypoint_and_healthcheck_share_an_http_contract(): + dockerfile = _read("Dockerfile") + compose = _read("docker-compose.yml") + + assert re.search(r'CMD\s+\[.*memorymaster-(?:dashboard|http).*\]', dockerfile) + assert re.search(r"https?://(?:127\.0\.0\.1|localhost):8765/(?:healthz|readyz)", compose) + assert '"--version"' not in compose + + +@pytest.mark.xfail( + strict=True, + reason="R3.4: Helm deployment has no liveness/readiness probes", +) +def test_helm_deployment_defines_health_and_readiness_probes(): + deployment = _read("helm/memorymaster/templates/deployment.yaml") + + assert "livenessProbe:" in deployment + assert "readinessProbe:" in deployment + assert "/healthz" in deployment + assert "/readyz" in deployment + + +@pytest.mark.xfail( + strict=True, + reason="R1.5: deployment profiles use unpinned latest image tags", +) +def test_deployment_images_reject_latest_tags(): + deployment_text = "\n".join( + [ + _read("docker-compose.yml"), + _read("docker-compose.postgres.yml"), + _read("helm/memorymaster/values.yaml"), + ] + ) + + assert not re.search(r"(?i)(?:image:\s*[^\s]+:latest|tag:\s*[\"']?latest)", deployment_text) diff --git a/tests/test_entity_schema_composition.py b/tests/test_entity_schema_composition.py new file mode 100644 index 00000000..418ccdf8 --- /dev/null +++ b/tests/test_entity_schema_composition.py @@ -0,0 +1,42 @@ +"""Red contract for composing the canonical entity registry and graph schema.""" + +from __future__ import annotations + +import sqlite3 + +import pytest + +from memorymaster.core.service import MemoryService +from memorymaster.knowledge.entity_graph import EntityGraph + + +@pytest.mark.xfail( + strict=True, + reason="R2.3: EntityGraph DDL conflicts with the canonical registry schema", +) +def test_normal_init_produces_a_graph_ready_entity_schema(tmp_path): + db_path = tmp_path / "entity-composition.db" + MemoryService(db_path, workspace_root=tmp_path).init_db() + + graph = EntityGraph(str(db_path)) + graph.ensure_tables() + + with sqlite3.connect(db_path) as conn: + tables = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table'" + ) + } + entity_columns = { + row[1] for row in conn.execute("PRAGMA table_info(entities)") + } + + assert {"entities", "entity_aliases", "entity_edges", "claim_entity_links"} <= tables + assert {"id", "canonical_name", "entity_type", "scope"} <= entity_columns + assert graph.get_stats() == { + "entities": 0, + "edges": 0, + "claim_links": 0, + "by_type": {}, + } diff --git a/tests/test_mcp_authorization_boundary.py b/tests/test_mcp_authorization_boundary.py new file mode 100644 index 00000000..c8eea8f1 --- /dev/null +++ b/tests/test_mcp_authorization_boundary.py @@ -0,0 +1,102 @@ +"""Red contracts for the MCP authorization boundary (MM-SEC-01). + +These tests intentionally describe the required team-safe behavior before the +request-context implementation exists. They stay strict-xfailed until the +MCP boundary derives identity and intersects caller scopes with policy. +""" +from __future__ import annotations + +import sqlite3 + +import pytest + +import memorymaster.core.access_control as access_control +import memorymaster.surfaces.mcp_server as mcp_server +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService + + +AUDIT_BASELINE = pytest.mark.xfail( + strict=True, + reason="audit baseline MM-SEC-01: MCP request authorization is not enforced", +) + + +@pytest.fixture(autouse=True) +def isolated_mcp_state(monkeypatch: pytest.MonkeyPatch): + """Keep global role/rate state from leaking between adversarial tests.""" + mcp_server._INGEST_RATE_BUCKETS.clear() + monkeypatch.setenv("MM_INGEST_RATE_LIMIT_PER_MIN", "0") + monkeypatch.setattr(mcp_server, "_ENV_DEFAULT_PROJECT_SCOPE", "") + monkeypatch.setattr(mcp_server, "_ENV_DEFAULT_WORKSPACE", "") + monkeypatch.setattr(mcp_server, "_ENV_QUERY_INCLUDE_LEGACY_PROJECT", False) + access_control._agent_roles.clear() + access_control._loaded = True + yield + mcp_server._INGEST_RATE_BUCKETS.clear() + access_control._agent_roles.clear() + access_control._loaded = False + + +def _init_mcp_db(tmp_path) -> tuple[str, str]: + db = str(tmp_path / "authorization.db") + workspace = str(tmp_path / "alpha") + (tmp_path / "alpha").mkdir() + mcp_server.init_db(db=db, workspace=workspace) + return db, workspace + + +def _claim_count(db: str) -> int: + with sqlite3.connect(db) as conn: + return int(conn.execute("SELECT COUNT(*) FROM claims").fetchone()[0]) + + +def _was_denied(call) -> bool: + try: + result = call() + except PermissionError: + return True + return isinstance(result, dict) and result.get("ok") is False + + +@AUDIT_BASELINE +def test_reader_cannot_ingest_by_spoofing_source_agent(tmp_path) -> None: + """An attribution field cannot replace the authenticated MCP principal.""" + db, workspace = _init_mcp_db(tmp_path) + access_control.set_role("mcp-session", access_control.Role.READER) + access_control.set_role("forged-writer", access_control.Role.WRITER) + + denied = _was_denied( + lambda: mcp_server.ingest_claim( + text="A reader must not create this governed claim", + sources_json='["test://authorization"]', + source_agent="forged-writer", + db=db, + workspace=workspace, + ) + ) + + assert denied, "the reader session must fail even with a writer-shaped source_agent" + assert _claim_count(db) == 0, "a denied write may not mutate domain state" + + +@AUDIT_BASELINE +def test_list_claims_is_restricted_to_the_workspace_scope(tmp_path) -> None: + """A project-scoped MCP client must not enumerate another project.""" + db, workspace = _init_mcp_db(tmp_path) + svc = MemoryService(db, workspace_root=tmp_path) + svc.ingest( + "alpha authorization boundary marker", + [CitationInput(source="test://alpha")], + scope="project:alpha", + ) + svc.ingest( + "beta authorization boundary marker", + [CitationInput(source="test://beta")], + scope="project:beta", + ) + + result = mcp_server.list_claims(db=db, workspace=workspace, limit=20) + scopes = {claim["scope"] for claim in result["claims"]} + + assert scopes <= {"project:alpha", "global"} diff --git a/tests/test_mcp_read_only_contract.py b/tests/test_mcp_read_only_contract.py new file mode 100644 index 00000000..911a0789 --- /dev/null +++ b/tests/test_mcp_read_only_contract.py @@ -0,0 +1,63 @@ +"""Red contract for one read-only retrieval per MCP context request (MM-REL-02).""" +from __future__ import annotations + +import sqlite3 + +import pytest + +import memorymaster.surfaces.mcp_server as mcp_server +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService + + +@pytest.mark.xfail( + strict=True, + reason="audit baseline MM-REL-02: MCP context summaries query and write twice", +) +def test_mcp_context_summary_queries_once_without_writing_access_count( + tmp_path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Formatting detail must reuse one governed read, not run retrieval again.""" + workspace = tmp_path / "readonly" + workspace.mkdir() + db = str(tmp_path / "mcp-read.db") + svc = MemoryService(db, workspace_root=workspace) + svc.init_db() + claim = svc.ingest( + "readonlycontract uses one governed retrieval", + [CitationInput(source="test://mcp-read")], + scope="project:readonly", + ) + + calls = 0 + original_query_rows = MemoryService.query_rows + + def counted_query_rows(self, *args, **kwargs): + nonlocal calls + calls += 1 + return original_query_rows(self, *args, **kwargs) + + monkeypatch.setattr(MemoryService, "query_rows", counted_query_rows) + monkeypatch.setattr(mcp_server, "_ENV_DEFAULT_PROJECT_SCOPE", "") + monkeypatch.setattr(mcp_server, "_ENV_DEFAULT_WORKSPACE", "") + monkeypatch.setattr(mcp_server, "_ENV_QUERY_INCLUDE_LEGACY_PROJECT", False) + monkeypatch.delenv("QDRANT_URL", raising=False) + + result = mcp_server.query_for_context( + query="readonlycontract", + db=db, + workspace=str(workspace), + retrieval_mode="legacy", + include_candidates=True, + scope_allowlist="project:readonly", + detail_level="summary", + ) + with sqlite3.connect(db) as conn: + access_count = int( + conn.execute( + "SELECT access_count FROM claims WHERE id = ?", (claim.id,) + ).fetchone()[0] + ) + + assert result["claims"], "fixture must exercise structured context retrieval" + assert (calls, access_count) == (1, 0) diff --git a/tests/test_persisted_envelope_sensitivity.py b/tests/test_persisted_envelope_sensitivity.py new file mode 100644 index 00000000..9b150c0d --- /dev/null +++ b/tests/test_persisted_envelope_sensitivity.py @@ -0,0 +1,86 @@ +"""Red tests for secrets outside the claim's primary text fields. + +The synthetic token is assembled at runtime and is not a credential. Every +case exercises the canonical service ingest boundary with a temporary SQLite +database, then scans durable SQLite text values for the unredacted fixture. +""" +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from memorymaster.core.models import CitationInput +from memorymaster.core.security import scan_text_for_findings +from memorymaster.core.service import MemoryService + + +def _synthetic_token() -> str: + body = "".join(format((index * 7 + 3) % 16, "x") for index in range(40)) + token = "".join(("gh", "p_", body)) + assert "github_token" in scan_text_for_findings(token) + return token + + +def _durable_locations(db_path: Path, needle: str) -> list[str]: + locations: list[str] = [] + with sqlite3.connect(db_path) as conn: + tables = conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ).fetchall() + for (table,) in tables: + quoted_table = '"' + str(table).replace('"', '""') + '"' + columns = conn.execute(f"PRAGMA table_info({quoted_table})").fetchall() + for column in (str(row[1]) for row in columns): + quoted_column = '"' + column.replace('"', '""') + '"' + values = conn.execute( + f"SELECT {quoted_column} FROM {quoted_table} WHERE {quoted_column} IS NOT NULL" + ).fetchall() + if any(isinstance(value, str) and needle in value for (value,) in values): + locations.append(f"{table}.{column}") + return locations + + +def _ingest_with_metadata_secret(service: MemoryService, field: str, secret: str) -> None: + citation_source = secret if field == "citation_source" else "phase0-red-test" + citation_locator = secret if field == "citation_locator" else "case:metadata" + overrides = { + "holder": {"holder": secret}, + "source_agent": {"source_agent": secret}, + "idempotency_key": {"idempotency_key": secret}, + "citation_source": {}, + "citation_locator": {}, + }[field] + attribution = {} if field == "source_agent" else {"source_agent": "phase0-red-test"} + service.ingest( + text=f"Benign metadata sensitivity case for {field}.", + citations=[CitationInput(source=citation_source, locator=citation_locator)], + scope="project:phase0-red-test", + **(attribution | overrides), + ) + + +@pytest.mark.xfail( + strict=True, + reason="MM-SEC-03: persisted-envelope metadata bypasses the sensitivity gateway", +) +@pytest.mark.parametrize( + "field", + ["holder", "source_agent", "idempotency_key", "citation_source", "citation_locator"], +) +def test_secret_shaped_metadata_never_reaches_durable_storage( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + field: str, +) -> None: + monkeypatch.delenv("QDRANT_URL", raising=False) + monkeypatch.delenv("MEMORYMASTER_ENCRYPTION_KEY", raising=False) + db_path = tmp_path / f"persisted-envelope-{field}.db" + service = MemoryService(db_path, workspace_root=tmp_path) + service.init_db() + secret = _synthetic_token() + + _ingest_with_metadata_secret(service, field, secret) + + assert _durable_locations(db_path, secret) == [] diff --git a/tests/test_qdrant_authoritative_filtering.py b/tests/test_qdrant_authoritative_filtering.py new file mode 100644 index 00000000..8cf18b2c --- /dev/null +++ b/tests/test_qdrant_authoritative_filtering.py @@ -0,0 +1,140 @@ +"""Red contracts for authoritative filtering of Qdrant candidates (MM-SEC-02).""" +from __future__ import annotations + +from collections.abc import Iterable + +import pytest + +import memorymaster.surfaces.mcp_server as mcp_server +from memorymaster.core.lifecycle import transition_claim +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService + + +AUDIT_BASELINE = pytest.mark.xfail( + strict=True, + reason="audit baseline MM-SEC-02: Qdrant bypasses authoritative policy filtering", +) + + +class FakeQdrant: + """Network-free vector candidate source controlled by each test.""" + + def __init__(self, hits: Iterable[dict]) -> None: + self._hits = list(hits) + self.closed = False + + def search(self, query_text: str, limit: int = 5) -> list[dict]: + del query_text + return self._hits[:limit] + + def close(self) -> None: + self.closed = True + + +@pytest.fixture(autouse=True) +def isolated_mcp_state(monkeypatch: pytest.MonkeyPatch): + mcp_server._INGEST_RATE_BUCKETS.clear() + monkeypatch.setenv("MM_INGEST_RATE_LIMIT_PER_MIN", "0") + monkeypatch.setattr(mcp_server, "_ENV_DEFAULT_PROJECT_SCOPE", "") + monkeypatch.setattr(mcp_server, "_ENV_DEFAULT_WORKSPACE", "") + monkeypatch.setattr(mcp_server, "_ENV_QUERY_INCLUDE_LEGACY_PROJECT", False) + yield + mcp_server._INGEST_RATE_BUCKETS.clear() + + +def _install_fake_qdrant(monkeypatch: pytest.MonkeyPatch, hits: list[dict]) -> FakeQdrant: + from memorymaster.recall import qdrant_backend + + fake = FakeQdrant(hits) + monkeypatch.setattr(qdrant_backend, "QdrantBackend", lambda: fake) + return fake + + +def _init_db(tmp_path) -> tuple[str, str, MemoryService]: + workspace_path = tmp_path / "allowed" + workspace_path.mkdir() + db = str(tmp_path / "qdrant-policy.db") + svc = MemoryService(db, workspace_root=workspace_path) + svc.init_db() + return db, str(workspace_path), svc + + +@AUDIT_BASELINE +def test_qdrant_orphan_payload_is_never_returned(tmp_path, monkeypatch) -> None: + """A vector point is only an ID candidate; payload is never authoritative.""" + db, workspace, _svc = _init_db(tmp_path) + fake = _install_fake_qdrant( + monkeypatch, + [ + { + "claim_id": 999_999, + "score": 0.99, + "payload": { + "claim_id": 999_999, + "text": "orphan payload must not escape", + "state": "confirmed", + "scope": "project:allowed", + }, + } + ], + ) + + result = mcp_server.query_memory( + query="orphan payload", + db=db, + workspace=workspace, + retrieval_mode="qdrant", + scope_allowlist="project:allowed", + include_candidates=False, + include_stale=False, + include_conflicted=False, + ) + + assert result["rows"] == 0 + assert result["claims"] == [] + assert fake.closed is True + + +@AUDIT_BASELINE +def test_qdrant_filters_archived_and_wrong_scope_rows(tmp_path, monkeypatch) -> None: + """Primary-store lifecycle and scope policy must filter every vector hit.""" + db, workspace, svc = _init_db(tmp_path) + archived = svc.ingest( + "archived vector policy marker", + [CitationInput(source="test://archived")], + scope="project:allowed", + ) + transition_claim( + svc.store, + archived.id, + "archived", + reason="adversarial fixture", + event_type="transition", + ) + foreign = svc.ingest( + "foreign vector policy marker", + [CitationInput(source="test://foreign")], + scope="project:other", + ) + _install_fake_qdrant( + monkeypatch, + [ + {"claim_id": archived.id, "score": 0.98, "payload": {"state": "confirmed"}}, + {"claim_id": foreign.id, "score": 0.97, "payload": {"state": "confirmed"}}, + ], + ) + + result = mcp_server.query_memory( + query="vector policy marker", + db=db, + workspace=workspace, + retrieval_mode="qdrant", + scope_allowlist="project:allowed", + include_candidates=False, + include_stale=False, + include_conflicted=False, + ) + + assert result["rows"] == 0 + assert result["claims"] == [] diff --git a/tests/test_retrieval_surface_parity.py b/tests/test_retrieval_surface_parity.py new file mode 100644 index 00000000..7d613b6a --- /dev/null +++ b/tests/test_retrieval_surface_parity.py @@ -0,0 +1,87 @@ +"""Red contracts for governed recall behavior at the MCP surface.""" + +from __future__ import annotations + +import pytest + +from memorymaster.core.lifecycle import transition_claim +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.surfaces.mcp_server import _project_scope, query_memory + + +def _service_with_claim(tmp_path, text: str, *, confirmed: bool) -> tuple[MemoryService, int]: + db_path = tmp_path / "retrieval-parity.db" + service = MemoryService(db_path, workspace_root=tmp_path) + service.init_db() + claim = service.ingest( + text=text, + citations=[CitationInput(source="test")], + scope=_project_scope(str(tmp_path)), + source_agent="retrieval-contract", + ) + if confirmed: + transition_claim( + service.store, + claim.id, + "confirmed", + reason="retrieval contract fixture", + event_type="validator", + ) + return service, claim.id + + +@pytest.mark.xfail( + strict=True, + reason="R2.1: MCP legacy retrieval sends conversational prompts to FTS5 as raw AND terms", +) +def test_conversational_recall_preserves_keyword_hits(tmp_path, monkeypatch): + monkeypatch.delenv("QDRANT_URL", raising=False) + service, claim_id = _service_with_claim( + tmp_path, + "MemoryMaster explains governed claims and citations for durable recall.", + confirmed=True, + ) + db_path = str(service.store.db_path) + common = { + "db": db_path, + "workspace": str(tmp_path), + "retrieval_mode": "legacy", + "include_stale": False, + "include_conflicted": False, + "include_candidates": False, + } + + keyword = query_memory(query="governed claims citations", **common) + conversational = query_memory( + query="How does MemoryMaster explain governed claims and citations for durable recall?", + **common, + ) + keyword_ids = {claim["id"] for claim in keyword["claims"]} + conversational_ids = {claim["id"] for claim in conversational["claims"]} + + assert claim_id in keyword_ids + assert claim_id in conversational_ids + + +@pytest.mark.xfail( + strict=True, + reason="R2.1: MCP query_memory defaults to exploratory candidate recall", +) +def test_default_mcp_recall_excludes_provisional_claims(tmp_path, monkeypatch): + monkeypatch.delenv("QDRANT_URL", raising=False) + service, claim_id = _service_with_claim( + tmp_path, + "provisionalrecalltoken is an unreviewed hypothesis", + confirmed=False, + ) + + result = query_memory( + query="provisionalrecalltoken", + db=str(service.store.db_path), + workspace=str(tmp_path), + ) + returned_ids = {claim["id"] for claim in result["claims"]} + + assert claim_id not in returned_ids + assert all(claim["status"] == "confirmed" for claim in result["claims"]) diff --git a/tests/test_scheduled_archive_lifecycle.py b/tests/test_scheduled_archive_lifecycle.py new file mode 100644 index 00000000..757422bd --- /dev/null +++ b/tests/test_scheduled_archive_lifecycle.py @@ -0,0 +1,27 @@ +"""Red static contract for lifecycle-safe scheduled archival (MM-LIFE-01).""" +from __future__ import annotations + +from pathlib import Path + +import pytest + + +HOOK_TEMPLATE = ( + Path(__file__).parents[1] + / "memorymaster" + / "config_templates" + / "hooks" + / "memorymaster-steward-cycle.py" +) + + +@pytest.mark.xfail( + strict=True, + reason="audit baseline MM-LIFE-01: scheduled hook archives with raw status SQL", +) +def test_scheduled_archive_contains_no_raw_claim_status_update() -> None: + """Scheduled archival must enter through lifecycle authority and its events.""" + source = HOOK_TEMPLATE.read_text(encoding="utf-8") + normalized = " ".join(source.lower().split()) + + assert "update claims set status = 'archived'" not in normalized diff --git a/tests/test_stop_hook_capture_policy.py b/tests/test_stop_hook_capture_policy.py new file mode 100644 index 00000000..faf269d4 --- /dev/null +++ b/tests/test_stop_hook_capture_policy.py @@ -0,0 +1,97 @@ +"""Red contract for the Stop hook's default quiet capture policy.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +TEMPLATE = ROOT / "memorymaster" / "config_templates" / "hooks" / "memorymaster-auto-ingest.py" + + +def _render_hook(project_root: Path, destination: Path) -> Path: + rendered = TEMPLATE.read_text(encoding="utf-8").replace( + "__MEMORYMASTER_PROJECT_ROOT__", + str(project_root).replace("\\", "/"), + ) + destination.write_text(rendered, encoding="utf-8") + return destination + + +def _write_transcript(path: Path) -> Path: + records = [ + json.dumps( + { + "message": { + "role": "user", + "content": f"Human message {index} has enough content for capture policy testing.", + } + } + ) + for index in range(15) + ] + path.write_text("\n".join(records) + "\n", encoding="utf-8") + return path + + +@pytest.mark.xfail( + strict=True, + reason="R2.4: the default Stop hook blocks and captures instead of remaining quiet", +) +def test_default_stop_hook_is_quiet_and_nonblocking(tmp_path): + project_root = tmp_path / "project" + project_root.mkdir() + hook = _render_hook(project_root, tmp_path / "stop-hook.py") + transcript = _write_transcript(tmp_path / "session.jsonl") + spool_root = tmp_path / "spool" + env = os.environ.copy() + for key in ( + "ANTHROPIC_API_KEY", + "GEMINI_API_KEY", + "GOOGLE_API_KEY", + "OPENAI_API_KEY", + "MEMORYMASTER_STOP_BLOCKING", + "MEMORYMASTER_STOP_CAPTURE_VERBATIM", + "MEMORYMASTER_STOP_EXTRACT", + "MEMORYMASTER_STOP_RULE_MINING", + ): + env.pop(key, None) + env.update( + { + "HOME": str(tmp_path / "home"), + "USERPROFILE": str(tmp_path / "home"), + "MEMORYMASTER_SPOOL_DIR": str(spool_root), + "MEMORYMASTER_WAL_DISCIPLINE": "1", + "PYTHONPATH": str(ROOT), + } + ) + payload = { + "session_id": "default-policy", + "transcript_path": str(transcript), + "cwd": str(project_root), + "stop_hook_active": False, + } + + result = subprocess.run( + [sys.executable, str(hook)], + input=json.dumps(payload), + capture_output=True, + text=True, + env=env, + timeout=30, + ) + envelopes = [ + line + for path in spool_root.rglob("*.jsonl") + for line in path.read_text(encoding="utf-8").splitlines() + ] + + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == {"decision": "approve"} + assert envelopes == [] diff --git a/tests/test_write_gateway_paths.py b/tests/test_write_gateway_paths.py new file mode 100644 index 00000000..7e2b96c4 --- /dev/null +++ b/tests/test_write_gateway_paths.py @@ -0,0 +1,76 @@ +"""Red tests for automated writers that bypass the governed ingest gateway.""" +from __future__ import annotations + +import json +import sqlite3 +from pathlib import Path +from unittest.mock import patch + +import pytest + +from memorymaster.core.lifecycle import transition_claim +from memorymaster.core.models import CitationInput +from memorymaster.core.security import scan_text_for_findings +from memorymaster.govern.jobs.compact_summaries import run +from memorymaster.stores.storage import SQLiteStore + + +def _synthetic_token() -> str: + body = "".join(format((index * 11 + 5) % 16, "x") for index in range(40)) + token = "".join(("gh", "p_", body)) + assert "github_token" in scan_text_for_findings(token) + return token + + +def _create_archived_source(store: SQLiteStore) -> None: + claim = store.create_claim( + text="A benign archived claim for automated summary testing.", + citations=[CitationInput(source="phase0-red-test", locator="case:compact-summary")], + subject="write-gateway", + predicate="source_fact", + object_value="benign", + ) + transition_claim(store, claim.id, to_status="confirmed", reason="test", event_type="transition") + transition_claim(store, claim.id, to_status="stale", reason="test", event_type="decay") + transition_claim(store, claim.id, to_status="archived", reason="test", event_type="compactor") + + +def _claim_and_citation_text(db_path: Path) -> str: + with sqlite3.connect(db_path) as conn: + claims = conn.execute( + "SELECT text, idempotency_key, subject, predicate, object_value, source_agent, holder FROM claims" + ).fetchall() + citations = conn.execute("SELECT source, locator, excerpt FROM citations").fetchall() + return "\n".join(str(value) for row in [*claims, *citations] for value in row if value is not None) + + +@pytest.mark.xfail( + strict=True, + reason="MM-SEC-04: compact-summary output writes directly to the store without sensitivity filtering", +) +def test_compact_summary_output_never_persists_secret_shaped_llm_content( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("QDRANT_URL", raising=False) + db_path = tmp_path / "compact-summary-write-gateway.db" + store = SQLiteStore(db_path) + store.init_db() + _create_archived_source(store) + secret = _synthetic_token() + response = json.dumps( + { + "summary_text": f"The generated summary included {secret}.", + "subject": "write-gateway", + "predicate": "summary_of", + "object_value": "synthetic output", + "confidence": 0.9, + } + ) + + with patch("memorymaster.govern.jobs.compact_summaries._call_llm", return_value=response): + result = run(store, provider="custom", min_cluster=1, dry_run=False) + + assert result.clusters_found == 1 + assert result.summaries_created + result.errors == 1 + assert secret not in _claim_and_citation_text(db_path) From d62262c2867096b24873ae89e935b5c773cde4cc Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Fri, 10 Jul 2026 23:53:40 -0300 Subject: [PATCH 02/55] fix: stabilize candidate validation ordering --- ...EMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md | 10 ++--- .planning/audit-remediation-ledger.md | 2 +- memorymaster/govern/jobs/validator.py | 11 ++++- tests/test_validator_candidate_order.py | 45 +++++++++++++++++++ 4 files changed, 61 insertions(+), 7 deletions(-) create mode 100644 tests/test_validator_candidate_order.py diff --git a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md index 1a1dcc09..b99525f8 100644 --- a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md +++ b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md @@ -81,7 +81,7 @@ Phase 0 must replace qualitative capacity goals with recorded numeric gates in t ### Program tracker -- [ ] Phase 0 — evidence, numerical gates, and red tests +- [x] Phase 0 — evidence, numerical gates, and red tests - [ ] R1.1 — authenticated request context and MCP authorization - [ ] R1.2 — Postgres tenant enforcement and RLS - [ ] R1.3 — immediate Qdrant containment @@ -107,14 +107,14 @@ Phase 0 must replace qualitative capacity goals with recorded numeric gates in t **Goal:** Preserve the audit evidence and prevent false fixes. -- [ ] **R0.1 — Execution isolation** +- [x] **R0.1 — Execution isolation** - Capture `git status`, branch, Python path, installed package version, DB size/counts, and relevant service versions. - Establish an isolated branch/worktree and dedicated virtual environment, or document why the main checkout is required. - Verify imports point at the intended checkout. -- [ ] **R0.2 — Finding ledger** +- [x] **R0.2 — Finding ledger** - Create `audit-remediation-ledger.md` mapping every audit finding to owner, package, status, commit, verification evidence, and rollback. - Deduplicate cross-domain findings without losing source-domain traceability. -- [ ] **R0.3 — Adversarial regression fixtures** +- [x] **R0.3 — Adversarial regression fixtures** - Reader-agent mutation denial. - Cross-project and cross-tenant list/query/mutation denial. - Archived, sensitive, wrong-scope, wrong-tenant, and orphan Qdrant hits. @@ -122,7 +122,7 @@ Phase 0 must replace qualitative capacity goals with recorded numeric gates in t - Registry-initialized entity graph. - Conversational query parity across MCP, hooks, CLI, and context packing. - Container entrypoint/readiness contract. -- [ ] **R0.4 — Numerical operating envelope** +- [x] **R0.4 — Numerical operating envelope** - Measure and write the required capacity, retention, DB-growth, disk-watermark, and backlog-completion numbers into the ledger. - Record the measurement query/window and make every later performance/cost gate consume those frozen values. diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index be8f33ad..462797bf 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -40,7 +40,7 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-OBS-01 | Missing Operations | High / BAD-PRACTICE | Failures/metrics are process-local without central alert ownership | R3.5 | OPEN | Persistent metrics, trace/error capture, alert tests/runbook | | MM-DB-01 | Database | Medium / BAD-PRACTICE | Fast schema fingerprint can omit legacy ensure-helper changes | R3.2 | OPEN | All DDL versioned or fingerprint covers every schema source | | MM-INTEGRITY-01 | Code Integrity | Medium / BAD-PRACTICE | `importlib.util` probe is wrong in clean Python and its test masks failure | R4.4 | OPEN | Clean subprocess test and explicit import | -| MM-TEST-01 | Maintainability/Reliability | Medium / BAD-PRACTICE | Full non-ML suite produced two different order/load-only failures that pass immediately in isolation | Phase 0/R4.4 | IN-PROGRESS | Root cause isolated; three consecutive full gates pass or deterministic test isolation is fixed | +| MM-TEST-01 | Maintainability/Reliability | Medium / BAD-PRACTICE | Validator candidate winner changed under load because mutable `updated_at` controlled processing order | Phase 0/R4.4 | RESOLVED | Deterministic timestamp-inversion regression passes; targeted lifecycle suite 35 passed; full isolated non-ML gate 3,094 passed, 56 skipped, 95 deselected, 24 intentional xfails | ## Source-domain reconciliation diff --git a/memorymaster/govern/jobs/validator.py b/memorymaster/govern/jobs/validator.py index 11f64402..92bd9cee 100644 --- a/memorymaster/govern/jobs/validator.py +++ b/memorymaster/govern/jobs/validator.py @@ -57,6 +57,11 @@ def _merge_claims(primary: list[Claim], secondary: list[Claim]) -> list[Claim]: return merged +def _candidate_recency_key(claim: Claim) -> tuple[str, int]: + """Order candidates by immutable ingest recency, never mutable confidence writes.""" + return (claim.created_at, claim.id) + + def run( store, limit: int = 200, @@ -87,7 +92,11 @@ def run( cfg = get_config() if min_score is None: min_score = cfg.validation_threshold - candidate_claims = store.find_by_status("candidate", limit=limit) + candidate_claims = sorted( + store.find_by_status("candidate", limit=limit), + key=_candidate_recency_key, + reverse=True, + ) due_revalidation_claims: list[Claim] = [] if policy_mode != "legacy": due_revalidation_claims = [ diff --git a/tests/test_validator_candidate_order.py b/tests/test_validator_candidate_order.py new file mode 100644 index 00000000..f48dc840 --- /dev/null +++ b/tests/test_validator_candidate_order.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import sqlite3 + +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.govern.jobs import validator + + +def test_validator_prefers_immutable_candidate_recency(tmp_path, monkeypatch) -> None: + db = tmp_path / "candidate-order.db" + service = MemoryService(db, workspace_root=tmp_path) + service.init_db() + monkeypatch.setattr(validator, "load_classifier", lambda: None) + + older = service.ingest( + text="Release deadline is 2026-04-01", + citations=[CitationInput(source="session://test", locator="turn-1")], + subject="release", + predicate="deadline", + object_value="2026-04-01", + ) + newer = service.ingest( + text="Release deadline moved to 2026-04-15", + citations=[CitationInput(source="session://test", locator="turn-2")], + subject="release", + predicate="deadline", + object_value="2026-04-15", + ) + + with sqlite3.connect(db) as conn: + conn.execute( + "UPDATE claims SET created_at = ?, updated_at = ? WHERE id = ?", + ("2026-04-01T00:00:00+00:00", "2026-04-16T00:00:00+00:00", older.id), + ) + conn.execute( + "UPDATE claims SET created_at = ?, updated_at = ? WHERE id = ?", + ("2026-04-15T00:00:00+00:00", "2026-04-15T00:00:00+00:00", newer.id), + ) + + validator.run(service.store, min_citations=1, min_score=0.5) + + status_by_id = {claim.id: claim.status for claim in service.list_claims(limit=10)} + assert status_by_id[newer.id] == "confirmed" + assert status_by_id[older.id] == "conflicted" From 18a4104f9c2a84569a45a72a3a7d8a1b915ae579 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 00:00:37 -0300 Subject: [PATCH 03/55] feat(auth): add immutable MCP request context --- memorymaster/core/access_control.py | 126 ++++++++++++++++++++++++++++ tests/test_request_context.py | 104 +++++++++++++++++++++++ 2 files changed, 230 insertions(+) create mode 100644 tests/test_request_context.py diff --git a/memorymaster/core/access_control.py b/memorymaster/core/access_control.py index c874030b..779c9644 100644 --- a/memorymaster/core/access_control.py +++ b/memorymaster/core/access_control.py @@ -17,8 +17,12 @@ import json import logging import os +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass from enum import Enum from pathlib import Path +from typing import Iterator, Mapping import contextlib logger = logging.getLogger(__name__) @@ -32,6 +36,23 @@ class Role(str, Enum): READER = "reader" +class AuthMode(str, Enum): + LOCAL_TRUSTED = "local-trusted" + TEAM = "team" + + +@dataclass(frozen=True, slots=True) +class RequestContext: + mode: AuthMode + principal: str + role: Role + tenant_id: str | None + workspace: str + allowed_scopes: tuple[str, ...] + allow_sensitive: bool + db_target: str + + # Permissions per role ROLE_PERMISSIONS: dict[Role, set[str]] = { Role.ADMIN: {"ingest", "query", "delete", "configure", "export", "steward", "compact"}, @@ -84,6 +105,111 @@ def get_role(agent_id: str | None) -> Role: return _agent_roles.get(agent_id.lower(), DEFAULT_ROLE) +def get_configured_role(agent_id: str | None) -> Role | None: + """Return only an explicitly configured role; never the legacy default.""" + _load_roles() + if not agent_id or not isinstance(agent_id, str): + return None + return _agent_roles.get(agent_id.lower()) + + +def _is_postgres_target(db_target: str, env: Mapping[str, str]) -> bool: + target = str(db_target or "").strip().lower() + backend = str(env.get("MEMORYMASTER_STORE_BACKEND", "")).strip().lower() + return target.startswith(("postgres://", "postgresql://")) or backend == "postgres" + + +def _team_value(env: Mapping[str, str], name: str) -> str: + value = str(env.get(name, "")).strip() + if not value: + raise PermissionError(f"Team MCP authorization requires {name}.") + return value + + +def _parse_team_scopes(raw: str) -> tuple[str, ...]: + scopes = tuple(dict.fromkeys(part.strip() for part in raw.split(",") if part.strip())) + if not scopes or "*" in scopes: + raise PermissionError("Team MCP authorization requires explicit non-wildcard scopes.") + return scopes + + +def resolve_request_context( + *, + db_target: str = "", + workspace: str = "", + environ: Mapping[str, str] | None = None, +) -> RequestContext: + """Derive MCP authority from operator configuration, never tool arguments.""" + env = os.environ if environ is None else environ + raw_mode = str(env.get("MEMORYMASTER_MCP_AUTH_MODE", "")).strip().lower() + if not raw_mode: + if _is_postgres_target(db_target, env): + raise PermissionError("Postgres MCP access requires an explicit authorization mode.") + raw_mode = AuthMode.LOCAL_TRUSTED.value + try: + mode = AuthMode(raw_mode) + except ValueError as exc: + raise PermissionError("MEMORYMASTER_MCP_AUTH_MODE must be local-trusted or team.") from exc + + if mode is AuthMode.LOCAL_TRUSTED: + return RequestContext( + mode=mode, + principal=str(env.get("MEMORYMASTER_MCP_PRINCIPAL", "")).strip() or "mcp-session", + role=Role.ADMIN, + tenant_id=None, + workspace=str(workspace or "").strip(), + allowed_scopes=(), + allow_sensitive=True, + db_target=str(db_target or "").strip(), + ) + + principal = _team_value(env, "MEMORYMASTER_MCP_PRINCIPAL") + role = get_configured_role(principal) + if role is None: + raise PermissionError("Team MCP principal has no explicitly configured role.") + return RequestContext( + mode=mode, + principal=principal, + role=role, + tenant_id=_team_value(env, "MEMORYMASTER_MCP_TENANT_ID"), + workspace=_team_value(env, "MEMORYMASTER_MCP_WORKSPACE"), + allowed_scopes=_parse_team_scopes(_team_value(env, "MEMORYMASTER_MCP_ALLOWED_SCOPES")), + allow_sensitive=str(env.get("MEMORYMASTER_MCP_ALLOW_SENSITIVE", "")).strip().lower() + in {"1", "true", "yes", "on"}, + db_target=_team_value(env, "MEMORYMASTER_MCP_DB"), + ) + + +_request_context: ContextVar[RequestContext | None] = ContextVar( + "memorymaster_request_context", + default=None, +) + + +@contextmanager +def bind_request_context(context: RequestContext) -> Iterator[RequestContext]: + token = _request_context.set(context) + try: + yield context + finally: + _request_context.reset(token) + + +def current_request_context(*, required: bool = False) -> RequestContext | None: + context = _request_context.get() + if required and context is None: + raise PermissionError("No authenticated MCP request context is bound.") + return context + + +def authorize_context_action(context: RequestContext, action: str) -> None: + if action not in ROLE_PERMISSIONS.get(context.role, set()): + raise PermissionError( + f"MCP principal '{context.principal}' with role '{context.role.value}' " + f"cannot perform '{action}'." + ) + + def check_permission(agent_id: str | None, action: str) -> bool: """Check if an agent has permission for an action. Returns True/False. diff --git a/tests/test_request_context.py b/tests/test_request_context.py new file mode 100644 index 00000000..1f19970c --- /dev/null +++ b/tests/test_request_context.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +from dataclasses import FrozenInstanceError + +import pytest + +import memorymaster.core.access_control as access_control + + +@pytest.fixture(autouse=True) +def isolated_roles(monkeypatch): + access_control._agent_roles.clear() + monkeypatch.setattr(access_control, "_loaded", True) + yield + access_control._agent_roles.clear() + + +def _team_env(**overrides: str) -> dict[str, str]: + values = { + "MEMORYMASTER_MCP_AUTH_MODE": "team", + "MEMORYMASTER_MCP_PRINCIPAL": "mcp-reader", + "MEMORYMASTER_MCP_TENANT_ID": "tenant-alpha", + "MEMORYMASTER_MCP_WORKSPACE": "C:/work/alpha", + "MEMORYMASTER_MCP_ALLOWED_SCOPES": "project:alpha,global", + "MEMORYMASTER_MCP_DB": "postgresql://memorymaster.invalid/app", + } + values.update(overrides) + return values + + +def test_local_context_is_explicit_trusted_and_immutable() -> None: + context = access_control.resolve_request_context( + db_target="memorymaster.db", + workspace="C:/work/alpha", + environ={}, + ) + + assert context.mode is access_control.AuthMode.LOCAL_TRUSTED + assert context.role is access_control.Role.ADMIN + assert context.principal == "mcp-session" + with pytest.raises(FrozenInstanceError): + context.principal = "forged" # type: ignore[misc] + + +def test_context_binding_is_scoped_and_reset() -> None: + context = access_control.resolve_request_context(environ={}) + + assert access_control.current_request_context() is None + with access_control.bind_request_context(context): + assert access_control.current_request_context(required=True) is context + assert access_control.current_request_context() is None + + +@pytest.mark.parametrize( + "missing", + [ + "MEMORYMASTER_MCP_PRINCIPAL", + "MEMORYMASTER_MCP_TENANT_ID", + "MEMORYMASTER_MCP_WORKSPACE", + "MEMORYMASTER_MCP_ALLOWED_SCOPES", + "MEMORYMASTER_MCP_DB", + ], +) +def test_team_context_requires_complete_operator_configuration(missing: str) -> None: + access_control.set_role("mcp-reader", access_control.Role.READER) + env = _team_env() + del env[missing] + + with pytest.raises(PermissionError, match=missing): + access_control.resolve_request_context(environ=env) + + +def test_team_context_requires_an_explicit_role() -> None: + with pytest.raises(PermissionError, match="explicitly configured role"): + access_control.resolve_request_context(environ=_team_env()) + + +def test_team_context_carries_frozen_authority() -> None: + access_control.set_role("mcp-reader", access_control.Role.READER) + context = access_control.resolve_request_context(environ=_team_env()) + + assert context.mode is access_control.AuthMode.TEAM + assert context.tenant_id == "tenant-alpha" + assert context.allowed_scopes == ("project:alpha", "global") + assert context.allow_sensitive is False + access_control.authorize_context_action(context, "query") + with pytest.raises(PermissionError, match="cannot perform 'ingest'"): + access_control.authorize_context_action(context, "ingest") + + +def test_implicit_postgres_context_is_rejected() -> None: + with pytest.raises(PermissionError, match="explicit authorization mode"): + access_control.resolve_request_context( + db_target="postgresql://memorymaster.invalid/app", + environ={}, + ) + + +def test_team_context_rejects_wildcard_scope() -> None: + access_control.set_role("mcp-reader", access_control.Role.READER) + with pytest.raises(PermissionError, match="non-wildcard"): + access_control.resolve_request_context( + environ=_team_env(MEMORYMASTER_MCP_ALLOWED_SCOPES="*"), + ) From cf0a1dadb78b0947382e76c1b3468dd00f4e7c79 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 00:07:04 -0300 Subject: [PATCH 04/55] fix(mcp): enforce complete tool action registry --- .planning/audit-remediation-ledger.md | 2 +- memorymaster/core/access_control.py | 4 +- memorymaster/surfaces/mcp_server.py | 103 +++++++++++++++++++- tests/conftest.py | 6 ++ tests/test_mcp_action_registry.py | 115 +++++++++++++++++++++++ tests/test_mcp_authorization_boundary.py | 9 +- tests/test_request_context.py | 13 ++- 7 files changed, 237 insertions(+), 15 deletions(-) create mode 100644 tests/test_mcp_action_registry.py diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 462797bf..3bbd67b3 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -8,7 +8,7 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | ID | Primary domain | Severity / exploitability | Summary | Package | Status | Acceptance evidence | |---|---|---|---|---|---|---| -| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | OPEN | Reader denial, cross-project/tenant matrix, Postgres RLS tests | +| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | Frozen context and complete 36-tool action registry committed; reader spoof denial passes before service entry; cross-project/tenant matrix and Postgres RLS remain | | MM-SEC-02 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | Qdrant can return archived, sensitive, cross-scope/tenant, or orphan payloads | R1.3,R2.1 | OPEN | Fake and real Qdrant authoritative-filter tests | | MM-OPS-01 | DevOps | Critical / EXPLOITABLE-LOW-EFFORT / H4 | Postgres Compose publishes a fixed credential | R1.5 | OPEN | Fail-closed Compose config, private port, external rotation evidence | | MM-SEC-03 | Security | High / EXPLOITABLE-NOW | Persisted metadata/provenance fields bypass sensitivity scanning | R1.4 | OPEN | Complete field-matrix adversarial suite and legacy dry-run | diff --git a/memorymaster/core/access_control.py b/memorymaster/core/access_control.py index 779c9644..ec629f38 100644 --- a/memorymaster/core/access_control.py +++ b/memorymaster/core/access_control.py @@ -143,9 +143,7 @@ def resolve_request_context( env = os.environ if environ is None else environ raw_mode = str(env.get("MEMORYMASTER_MCP_AUTH_MODE", "")).strip().lower() if not raw_mode: - if _is_postgres_target(db_target, env): - raise PermissionError("Postgres MCP access requires an explicit authorization mode.") - raw_mode = AuthMode.LOCAL_TRUSTED.value + raise PermissionError("MCP access requires an explicit authorization mode.") try: mode = AuthMode(raw_mode) except ValueError as exc: diff --git a/memorymaster/surfaces/mcp_server.py b/memorymaster/surfaces/mcp_server.py index c325a711..36c8a6a7 100644 --- a/memorymaster/surfaces/mcp_server.py +++ b/memorymaster/surfaces/mcp_server.py @@ -1,6 +1,8 @@ -from dataclasses import asdict +from dataclasses import asdict, dataclass +from functools import wraps import hashlib import http.client +import inspect import json import logging import os @@ -11,6 +13,12 @@ from urllib.parse import urlparse from memorymaster.core import observability +from memorymaster.core.access_control import ( + AuthMode, + authorize_context_action, + bind_request_context, + resolve_request_context, +) from memorymaster.surfaces import mcp_path_policy from pydantic import BaseModel, ValidationError @@ -586,8 +594,99 @@ def _checkpoint_batch( } +@dataclass(frozen=True, slots=True) +class McpToolPolicy: + action: str + team_enabled: bool = False + + +MCP_TOOL_POLICIES: dict[str, McpToolPolicy] = { + "archive_by_source": McpToolPolicy("compact"), + "checkpoint": McpToolPolicy("ingest"), + "classify_query": McpToolPolicy("query", team_enabled=True), + "compact_memory": McpToolPolicy("compact"), + "entity_stats": McpToolPolicy("configure"), + "extract_entities": McpToolPolicy("ingest"), + "federated_query": McpToolPolicy("query"), + "find_related_claims": McpToolPolicy("configure"), + "get_usage_rollup": McpToolPolicy("query"), + "ingest_claim": McpToolPolicy("ingest"), + "ingest_rule": McpToolPolicy("ingest"), + "init_db": McpToolPolicy("configure"), + "list_claims": McpToolPolicy("query"), + "list_events": McpToolPolicy("query"), + "list_steward_proposals": McpToolPolicy("query"), + "local_search": McpToolPolicy("query"), + "open_dashboard": McpToolPolicy("query"), + "pin_claim": McpToolPolicy("steward"), + "quality_scores": McpToolPolicy("steward"), + "query_claim_paths": McpToolPolicy("query"), + "query_for_context": McpToolPolicy("query"), + "query_for_task": McpToolPolicy("query"), + "query_memory": McpToolPolicy("query"), + "query_meta_decisions": McpToolPolicy("query"), + "query_rules": McpToolPolicy("query"), + "read_active_tasks": McpToolPolicy("query"), + "recall_analysis": McpToolPolicy("query"), + "recompute_tiers": McpToolPolicy("steward"), + "redact_claim_payload": McpToolPolicy("delete"), + "resolve_project": McpToolPolicy("ingest"), + "resolve_steward_proposal": McpToolPolicy("steward"), + "rules_export": McpToolPolicy("export"), + "run_cycle": McpToolPolicy("steward"), + "run_steward": McpToolPolicy("steward"), + "search_verbatim": McpToolPolicy("export"), + "volunteer_context": McpToolPolicy("query"), +} + + +def _authorized_tool_callable(func: Any, policy: McpToolPolicy) -> Any: + call_signature = inspect.signature(func) + + @wraps(func) + def guarded(*args: Any, **kwargs: Any) -> Any: + bound = call_signature.bind_partial(*args, **kwargs) + bound.apply_defaults() + context = resolve_request_context( + db_target=str(bound.arguments.get("db", "") or ""), + workspace=str(bound.arguments.get("workspace", "") or ""), + ) + authorize_context_action(context, policy.action) + if context.mode is AuthMode.TEAM and not policy.team_enabled: + raise PermissionError( + f"MCP tool '{func.__name__}' is disabled in team mode until its scope contract is verified." + ) + if bool(bound.arguments.get("allow_sensitive", False)) and not context.allow_sensitive: + raise PermissionError("Authenticated MCP context does not allow sensitive-data access.") + with bind_request_context(context): + return func(*args, **kwargs) + + setattr(guarded, "__mcp_action__", policy.action) + setattr(guarded, "__mcp_team_enabled__", policy.team_enabled) + return guarded + + +if FastMCP is not None: + class AuthorizedFastMCP(FastMCP): + """FastMCP registration that cannot omit authorization metadata.""" + + def tool(self, *args: Any, **kwargs: Any) -> Any: + register = super().tool(*args, **kwargs) + + def decorator(func: Any) -> Any: + policy = MCP_TOOL_POLICIES.get(func.__name__) + if policy is None: + raise RuntimeError(f"MCP tool '{func.__name__}' has no authorization policy.") + return register(_authorized_tool_callable(func, policy)) + + return decorator + +else: # pragma: no cover - import fallback when MCP dependency is unavailable + AuthorizedFastMCP = None # type: ignore[misc,assignment] + + if FastMCP is not None: - mcp = FastMCP("memorymaster") + mcp = AuthorizedFastMCP("memorymaster") @mcp.tool() def init_db( diff --git a/tests/conftest.py b/tests/conftest.py index 17edd4fd..fa244214 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,6 +9,12 @@ _CASE_ROOT = Path(".tmp_cases") +@pytest.fixture(autouse=True) +def _explicit_local_mcp_auth(monkeypatch) -> None: + """Make legacy MCP test calls exercise the named local-trusted profile.""" + monkeypatch.setenv("MEMORYMASTER_MCP_AUTH_MODE", "local-trusted") + + # --------------------------------------------------------------------------- # Backend parametrization for parity tests (v3.20.0-S2) # --------------------------------------------------------------------------- diff --git a/tests/test_mcp_action_registry.py b/tests/test_mcp_action_registry.py new file mode 100644 index 00000000..70f9b29e --- /dev/null +++ b/tests/test_mcp_action_registry.py @@ -0,0 +1,115 @@ +from __future__ import annotations + +import asyncio + +import pytest + +import memorymaster.core.access_control as access_control +import memorymaster.surfaces.mcp_server as mcp_server + + +EXPECTED_TOOLS = { + "archive_by_source", + "checkpoint", + "classify_query", + "compact_memory", + "entity_stats", + "extract_entities", + "federated_query", + "find_related_claims", + "get_usage_rollup", + "ingest_claim", + "ingest_rule", + "init_db", + "list_claims", + "list_events", + "list_steward_proposals", + "local_search", + "open_dashboard", + "pin_claim", + "quality_scores", + "query_claim_paths", + "query_for_context", + "query_for_task", + "query_memory", + "query_meta_decisions", + "query_rules", + "read_active_tasks", + "recall_analysis", + "recompute_tiers", + "redact_claim_payload", + "resolve_project", + "resolve_steward_proposal", + "rules_export", + "run_cycle", + "run_steward", + "search_verbatim", + "volunteer_context", +} + + +@pytest.fixture(autouse=True) +def isolated_auth(monkeypatch, tmp_path): + access_control._agent_roles.clear() + monkeypatch.setattr(access_control, "_loaded", True) + monkeypatch.setenv("MEMORYMASTER_MCP_AUTH_MODE", "team") + monkeypatch.setenv("MEMORYMASTER_MCP_PRINCIPAL", "mcp-reader") + monkeypatch.setenv("MEMORYMASTER_MCP_TENANT_ID", "tenant-alpha") + monkeypatch.setenv("MEMORYMASTER_MCP_WORKSPACE", str(tmp_path / "alpha")) + monkeypatch.setenv("MEMORYMASTER_MCP_ALLOWED_SCOPES", "project:alpha,global") + monkeypatch.setenv("MEMORYMASTER_MCP_DB", str(tmp_path / "team.db")) + yield + access_control._agent_roles.clear() + + +def test_every_registered_mcp_tool_has_a_named_action() -> None: + assert set(mcp_server.MCP_TOOL_POLICIES) == EXPECTED_TOOLS + assert all(policy.action for policy in mcp_server.MCP_TOOL_POLICIES.values()) + registered = mcp_server.mcp._tool_manager.list_tools() + assert {tool.name for tool in registered} == EXPECTED_TOOLS + assert all(getattr(tool.fn, "__mcp_action__", None) for tool in registered) + + +def test_reader_denial_happens_before_tool_body(monkeypatch) -> None: + access_control.set_role("mcp-reader", access_control.Role.READER) + monkeypatch.setattr( + mcp_server, + "_service", + lambda *_args, **_kwargs: pytest.fail("denied request reached the service"), + ) + + with pytest.raises(PermissionError, match="cannot perform 'ingest'"): + mcp_server.ingest_claim(text="reader write", sources_json='["test://reader"]') + + +def test_registered_tool_denies_reader_before_tool_body(monkeypatch) -> None: + access_control.set_role("mcp-reader", access_control.Role.READER) + monkeypatch.setattr( + mcp_server, + "_service", + lambda *_args, **_kwargs: pytest.fail("registered denial reached the service"), + ) + + with pytest.raises(Exception) as exc_info: + asyncio.run( + mcp_server.mcp._tool_manager.call_tool( + "ingest_claim", + {"text": "reader write", "sources_json": '["test://reader"]'}, + ) + ) + + cause = exc_info.value.__cause__ + assert isinstance(cause, PermissionError) + assert "cannot perform 'ingest'" in str(cause) + + +def test_missing_team_identity_fails_before_tool_body(monkeypatch) -> None: + monkeypatch.delenv("MEMORYMASTER_MCP_PRINCIPAL") + monkeypatch.setattr( + mcp_server, + "_service", + lambda *_args, **_kwargs: pytest.fail("unauthenticated request reached the service"), + ) + + with pytest.raises(PermissionError, match="MEMORYMASTER_MCP_PRINCIPAL"): + mcp_server.query_memory(query="must fail before opening the database") diff --git a/tests/test_mcp_authorization_boundary.py b/tests/test_mcp_authorization_boundary.py index c8eea8f1..55eb207a 100644 --- a/tests/test_mcp_authorization_boundary.py +++ b/tests/test_mcp_authorization_boundary.py @@ -59,12 +59,17 @@ def _was_denied(call) -> bool: return isinstance(result, dict) and result.get("ok") is False -@AUDIT_BASELINE -def test_reader_cannot_ingest_by_spoofing_source_agent(tmp_path) -> None: +def test_reader_cannot_ingest_by_spoofing_source_agent(tmp_path, monkeypatch) -> None: """An attribution field cannot replace the authenticated MCP principal.""" db, workspace = _init_mcp_db(tmp_path) access_control.set_role("mcp-session", access_control.Role.READER) access_control.set_role("forged-writer", access_control.Role.WRITER) + monkeypatch.setenv("MEMORYMASTER_MCP_AUTH_MODE", "team") + monkeypatch.setenv("MEMORYMASTER_MCP_PRINCIPAL", "mcp-session") + monkeypatch.setenv("MEMORYMASTER_MCP_TENANT_ID", "tenant-alpha") + monkeypatch.setenv("MEMORYMASTER_MCP_WORKSPACE", workspace) + monkeypatch.setenv("MEMORYMASTER_MCP_ALLOWED_SCOPES", "project:alpha,global") + monkeypatch.setenv("MEMORYMASTER_MCP_DB", db) denied = _was_denied( lambda: mcp_server.ingest_claim( diff --git a/tests/test_request_context.py b/tests/test_request_context.py index 1f19970c..66dfb751 100644 --- a/tests/test_request_context.py +++ b/tests/test_request_context.py @@ -32,7 +32,7 @@ def test_local_context_is_explicit_trusted_and_immutable() -> None: context = access_control.resolve_request_context( db_target="memorymaster.db", workspace="C:/work/alpha", - environ={}, + environ={"MEMORYMASTER_MCP_AUTH_MODE": "local-trusted"}, ) assert context.mode is access_control.AuthMode.LOCAL_TRUSTED @@ -43,7 +43,9 @@ def test_local_context_is_explicit_trusted_and_immutable() -> None: def test_context_binding_is_scoped_and_reset() -> None: - context = access_control.resolve_request_context(environ={}) + context = access_control.resolve_request_context( + environ={"MEMORYMASTER_MCP_AUTH_MODE": "local-trusted"}, + ) assert access_control.current_request_context() is None with access_control.bind_request_context(context): @@ -88,12 +90,9 @@ def test_team_context_carries_frozen_authority() -> None: access_control.authorize_context_action(context, "ingest") -def test_implicit_postgres_context_is_rejected() -> None: +def test_missing_auth_mode_is_rejected() -> None: with pytest.raises(PermissionError, match="explicit authorization mode"): - access_control.resolve_request_context( - db_target="postgresql://memorymaster.invalid/app", - environ={}, - ) + access_control.resolve_request_context(db_target="memorymaster.db", environ={}) def test_team_context_rejects_wildcard_scope() -> None: From 44d0955538e69ffb5f12ec0314ac111a752381dd Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 00:14:01 -0300 Subject: [PATCH 05/55] fix(mcp): constrain tenant scope and principal propagation --- .planning/audit-remediation-ledger.md | 2 +- memorymaster/core/service.py | 4 + memorymaster/surfaces/mcp_server.py | 137 +++++++++++++++++++---- tests/test_mcp_authorization_boundary.py | 14 +-- tests/test_mcp_tenant_scope_boundary.py | 95 ++++++++++++++++ 5 files changed, 218 insertions(+), 34 deletions(-) create mode 100644 tests/test_mcp_tenant_scope_boundary.py diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 3bbd67b3..8ef609a5 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -8,7 +8,7 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | ID | Primary domain | Severity / exploitability | Summary | Package | Status | Acceptance evidence | |---|---|---|---|---|---|---| -| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | Frozen context and complete 36-tool action registry committed; reader spoof denial passes before service entry; cross-project/tenant matrix and Postgres RLS remain | +| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | Frozen context and complete 36-tool action registry; reader spoof denial before service; SQLite team matrix proves tenant/scope intersection and principal pinning; auxiliary tools and Postgres RLS remain | | MM-SEC-02 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | Qdrant can return archived, sensitive, cross-scope/tenant, or orphan payloads | R1.3,R2.1 | OPEN | Fake and real Qdrant authoritative-filter tests | | MM-OPS-01 | DevOps | Critical / EXPLOITABLE-LOW-EFFORT / H4 | Postgres Compose publishes a fixed credential | R1.5 | OPEN | Fail-closed Compose config, private port, external rotation evidence | | MM-SEC-03 | Security | High / EXPLOITABLE-NOW | Persisted metadata/provenance fields bypass sensitivity scanning | R1.4 | OPEN | Complete field-matrix adversarial suite and legacy dry-run | diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index b0b5e187..303ba7ca 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -1740,6 +1740,8 @@ def list_claims( *, allow_sensitive: bool = False, holder: str | None = None, + scope_allowlist: list[str] | None = None, + requesting_agent: str | None = None, ) -> list[Claim]: include_sensitive = self._allow_sensitive( allow_sensitive=allow_sensitive, @@ -1751,9 +1753,11 @@ def list_claims( limit=limit, include_archived=include_archived, include_citations=True, + scope_allowlist=scope_allowlist, tenant_id=self.tenant_id, holder=holder, ) + claims = _filter_agent_visibility(claims, requesting_agent) if not include_sensitive: claims = [claim for claim in claims if not is_sensitive_claim(claim)] return claims diff --git a/memorymaster/surfaces/mcp_server.py b/memorymaster/surfaces/mcp_server.py index 36c8a6a7..5638d1f6 100644 --- a/memorymaster/surfaces/mcp_server.py +++ b/memorymaster/surfaces/mcp_server.py @@ -15,8 +15,10 @@ from memorymaster.core import observability from memorymaster.core.access_control import ( AuthMode, + RequestContext, authorize_context_action, bind_request_context, + current_request_context, resolve_request_context, ) from memorymaster.surfaces import mcp_path_policy @@ -304,32 +306,48 @@ def _resolve_workspace(workspace: str) -> str: # start_session or bound MemoryService.session_id, so get_usage_rollup's # session half always returned []. The MCP server is long-lived per client, # so one session per DB per process is the honest granularity. -_TELEMETRY_SESSION_IDS: dict[str, int] = {} +_TELEMETRY_SESSION_IDS: dict[tuple[str, str, str], int] = {} -def _bind_telemetry_session(svc: MemoryService, db_path: str) -> None: +def _bind_telemetry_session( + svc: MemoryService, + db_path: str, + principal: str = "mcp-session", + tenant_id: str | None = None, +) -> None: """Best-effort: bind a usage-telemetry session to *svc*. Telemetry must never break a tool call — every failure is swallowed and the service simply stays unbound (counters still work via source_agent). """ try: - sid = _TELEMETRY_SESSION_IDS.get(db_path) + session_key = (db_path, principal, tenant_id or "") + sid = _TELEMETRY_SESSION_IDS.get(session_key) if sid is None: from memorymaster.surfaces.session_tracker import SessionTracker - sid = SessionTracker(db_path).start_session("mcp-session") - _TELEMETRY_SESSION_IDS[db_path] = sid + sid = SessionTracker(db_path).start_session(principal) + _TELEMETRY_SESSION_IDS[session_key] = sid svc.session_id = sid if not getattr(svc, "source_agent", None): - svc.source_agent = "mcp-session" + svc.source_agent = principal except Exception: pass def _service(db: str, workspace: str) -> MemoryService: - svc = MemoryService(db_target=_resolve_db(db), workspace_root=Path(_resolve_workspace(workspace))) - _bind_telemetry_session(svc, _resolve_db(db)) + db_path = _resolve_db(db) + workspace_path = _resolve_workspace(workspace) + context = current_request_context() + principal = context.principal if context is not None else "mcp-session" + tenant_id = context.tenant_id if context is not None else None + svc = MemoryService( + db_target=db_path, + workspace_root=Path(workspace_path), + tenant_id=tenant_id, + ) + _bind_telemetry_session(svc, db_path, principal, tenant_id) + svc.source_agent = principal return svc @@ -437,19 +455,27 @@ def _project_scope(workspace: str) -> str: def _effective_ingest_scope(scope: str, workspace: str) -> str: raw = (scope or "").strip() - if not raw or raw == "project": - return _project_scope(workspace) - return raw + effective = _project_scope(workspace) if not raw or raw == "project" else raw + context = current_request_context() + if context is not None and context.mode is AuthMode.TEAM and effective not in context.allowed_scopes: + raise PermissionError("Requested claim scope is outside the authenticated scope grant.") + return effective def _effective_scope_allowlist(raw: str, workspace: str) -> list[str] | None: parsed = _parse_scope_allowlist(raw) - if parsed: + context = current_request_context() + if context is not None and context.mode is AuthMode.TEAM: + requested = parsed or list(context.allowed_scopes) + scopes = [scope for scope in requested if scope in context.allowed_scopes] + if not scopes: + raise PermissionError("Requested scopes do not intersect the authenticated scope grant.") + elif parsed: return parsed - scopes = [_project_scope(workspace), "global"] - if _ENV_QUERY_INCLUDE_LEGACY_PROJECT: - scopes.append("project") - # Keep order and dedupe. + else: + scopes = [_project_scope(workspace), "global"] + if _ENV_QUERY_INCLUDE_LEGACY_PROJECT: + scopes.append("project") seen: set[str] = set() deduped: list[str] = [] for value in scopes: @@ -610,10 +636,10 @@ class McpToolPolicy: "federated_query": McpToolPolicy("query"), "find_related_claims": McpToolPolicy("configure"), "get_usage_rollup": McpToolPolicy("query"), - "ingest_claim": McpToolPolicy("ingest"), + "ingest_claim": McpToolPolicy("ingest", team_enabled=True), "ingest_rule": McpToolPolicy("ingest"), "init_db": McpToolPolicy("configure"), - "list_claims": McpToolPolicy("query"), + "list_claims": McpToolPolicy("query", team_enabled=True), "list_events": McpToolPolicy("query"), "list_steward_proposals": McpToolPolicy("query"), "local_search": McpToolPolicy("query"), @@ -623,7 +649,7 @@ class McpToolPolicy: "query_claim_paths": McpToolPolicy("query"), "query_for_context": McpToolPolicy("query"), "query_for_task": McpToolPolicy("query"), - "query_memory": McpToolPolicy("query"), + "query_memory": McpToolPolicy("query", team_enabled=True), "query_meta_decisions": McpToolPolicy("query"), "query_rules": McpToolPolicy("query"), "read_active_tasks": McpToolPolicy("query"), @@ -640,6 +666,72 @@ class McpToolPolicy: } +def _same_configured_location(left: str, right: str) -> bool: + if "://" in left or "://" in right: + return left == right + return Path(left).resolve() == Path(right).resolve() + + +def _team_default_scope(context: RequestContext) -> str: + workspace_scope = _project_scope(context.workspace) + if workspace_scope in context.allowed_scopes: + return workspace_scope + project_scopes = [scope for scope in context.allowed_scopes if scope.startswith("project:")] + if len(project_scopes) == 1: + return project_scopes[0] + raise PermissionError("Authenticated workspace has no unambiguous project scope.") + + +def _team_request_principal() -> str | None: + context = current_request_context() + if context is None or context.mode is not AuthMode.TEAM: + return None + return context.principal + + +def _normalize_team_arguments( + bound: inspect.BoundArguments, + context: RequestContext, + tool_name: str, +) -> None: + if "db" in bound.arguments: + requested_db = str(bound.arguments["db"] or "") + if requested_db not in {"", _DEFAULT_DB} and not _same_configured_location(requested_db, context.db_target): + raise PermissionError("Caller-selected database is outside the authenticated context.") + bound.arguments["db"] = context.db_target + if "workspace" in bound.arguments: + requested_workspace = str(bound.arguments["workspace"] or "") + if requested_workspace not in {"", _DEFAULT_WORKSPACE} and not _same_configured_location( + requested_workspace, + context.workspace, + ): + raise PermissionError("Caller-selected workspace is outside the authenticated context.") + bound.arguments["workspace"] = context.workspace + default_scope = _team_default_scope(context) + for field in ("scope", "current_scope", "project_scope"): + if field not in bound.arguments: + continue + requested_scope = str(bound.arguments[field] or "").strip() + effective_scope = default_scope if requested_scope in {"", "project"} else requested_scope + if effective_scope not in context.allowed_scopes: + raise PermissionError("Caller-selected scope is outside the authenticated context.") + bound.arguments[field] = effective_scope + if "scope_allowlist" in bound.arguments: + requested_scopes = _parse_scope_allowlist(str(bound.arguments["scope_allowlist"] or "")) + narrowed = [scope for scope in (requested_scopes or context.allowed_scopes) if scope in context.allowed_scopes] + if not narrowed: + raise PermissionError("Caller scope allowlist does not intersect authenticated scopes.") + bound.arguments["scope_allowlist"] = ",".join(narrowed) + for field in ("source_agent", "actor"): + if field in bound.arguments: + bound.arguments[field] = context.principal + if tool_name == "query_memory" and ( + str(bound.arguments.get("retrieval_mode", "legacy")) != "legacy" + or bool(bound.arguments.get("auto_classify", False)) + ): + raise PermissionError("Semantic MCP retrieval remains disabled in team mode pending planner containment.") + + def _authorized_tool_callable(func: Any, policy: McpToolPolicy) -> Any: call_signature = inspect.signature(func) @@ -652,6 +744,8 @@ def guarded(*args: Any, **kwargs: Any) -> Any: workspace=str(bound.arguments.get("workspace", "") or ""), ) authorize_context_action(context, policy.action) + if context.mode is AuthMode.TEAM: + _normalize_team_arguments(bound, context, func.__name__) if context.mode is AuthMode.TEAM and not policy.team_enabled: raise PermissionError( f"MCP tool '{func.__name__}' is disabled in team mode until its scope contract is verified." @@ -659,7 +753,7 @@ def guarded(*args: Any, **kwargs: Any) -> Any: if bool(bound.arguments.get("allow_sensitive", False)) and not context.allow_sensitive: raise PermissionError("Authenticated MCP context does not allow sensitive-data access.") with bind_request_context(context): - return func(*args, **kwargs) + return func(*bound.args, **bound.kwargs) setattr(guarded, "__mcp_action__", policy.action) setattr(guarded, "__mcp_team_enabled__", policy.team_enabled) @@ -1145,6 +1239,7 @@ def query_memory( include_candidates=include_candidates, allow_sensitive=allow_sensitive, scope_allowlist=_effective_scope_allowlist(scope_allowlist, workspace), + requesting_agent=_team_request_principal(), ) # For "full" detail level, re-fetch each claim with citations inline. if detail_level == "full": @@ -1628,6 +1723,8 @@ def list_claims( include_archived=include_archived, allow_sensitive=allow_sensitive, holder=_empty_to_none(holder), + scope_allowlist=_effective_scope_allowlist("", workspace), + requesting_agent=_team_request_principal(), ) return {"ok": True, "rows": len(claims), "claims": [_claim_to_dict(c) for c in claims]} diff --git a/tests/test_mcp_authorization_boundary.py b/tests/test_mcp_authorization_boundary.py index 55eb207a..9dcc3651 100644 --- a/tests/test_mcp_authorization_boundary.py +++ b/tests/test_mcp_authorization_boundary.py @@ -1,9 +1,4 @@ -"""Red contracts for the MCP authorization boundary (MM-SEC-01). - -These tests intentionally describe the required team-safe behavior before the -request-context implementation exists. They stay strict-xfailed until the -MCP boundary derives identity and intersects caller scopes with policy. -""" +"""Adversarial contracts for the MCP authorization boundary (MM-SEC-01).""" from __future__ import annotations import sqlite3 @@ -16,12 +11,6 @@ from memorymaster.core.service import MemoryService -AUDIT_BASELINE = pytest.mark.xfail( - strict=True, - reason="audit baseline MM-SEC-01: MCP request authorization is not enforced", -) - - @pytest.fixture(autouse=True) def isolated_mcp_state(monkeypatch: pytest.MonkeyPatch): """Keep global role/rate state from leaking between adversarial tests.""" @@ -85,7 +74,6 @@ def test_reader_cannot_ingest_by_spoofing_source_agent(tmp_path, monkeypatch) -> assert _claim_count(db) == 0, "a denied write may not mutate domain state" -@AUDIT_BASELINE def test_list_claims_is_restricted_to_the_workspace_scope(tmp_path) -> None: """A project-scoped MCP client must not enumerate another project.""" db, workspace = _init_mcp_db(tmp_path) diff --git a/tests/test_mcp_tenant_scope_boundary.py b/tests/test_mcp_tenant_scope_boundary.py new file mode 100644 index 00000000..f1b7c177 --- /dev/null +++ b/tests/test_mcp_tenant_scope_boundary.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import sqlite3 + +import pytest + +import memorymaster.core.access_control as access_control +import memorymaster.surfaces.mcp_server as mcp_server +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService + + +@pytest.fixture +def team_claims(tmp_path, monkeypatch): + db = str(tmp_path / "team.db") + workspace = tmp_path / "alpha" + workspace.mkdir() + alpha = MemoryService(db, workspace_root=workspace, tenant_id="tenant-alpha") + alpha.init_db() + alpha_claim = alpha.ingest( + "authorization matrix marker alpha allowed", + [CitationInput(source="test://alpha")], + scope="project:alpha", + source_agent="seed", + ) + beta_scope_claim = alpha.ingest( + "authorization matrix marker beta scope forbidden", + [CitationInput(source="test://beta-scope")], + scope="project:beta", + source_agent="seed", + ) + beta_tenant = MemoryService(db, workspace_root=workspace, tenant_id="tenant-beta") + beta_tenant_claim = beta_tenant.ingest( + "authorization matrix marker beta tenant forbidden", + [CitationInput(source="test://beta-tenant")], + scope="project:alpha", + source_agent="seed", + ) + + access_control._agent_roles.clear() + monkeypatch.setattr(access_control, "_loaded", True) + access_control.set_role("mcp-reader", access_control.Role.READER) + monkeypatch.setenv("MEMORYMASTER_MCP_AUTH_MODE", "team") + monkeypatch.setenv("MEMORYMASTER_MCP_PRINCIPAL", "mcp-reader") + monkeypatch.setenv("MEMORYMASTER_MCP_TENANT_ID", "tenant-alpha") + monkeypatch.setenv("MEMORYMASTER_MCP_WORKSPACE", str(workspace)) + monkeypatch.setenv("MEMORYMASTER_MCP_ALLOWED_SCOPES", "project:alpha,global") + monkeypatch.setenv("MEMORYMASTER_MCP_DB", db) + yield db, workspace, alpha_claim, beta_scope_claim, beta_tenant_claim + access_control._agent_roles.clear() + + +def test_team_list_and_query_intersect_tenant_and_scope(team_claims) -> None: + db, _workspace, allowed, wrong_scope, wrong_tenant = team_claims + + listed = mcp_server.list_claims(limit=20) + queried = mcp_server.query_memory( + query="authorization matrix marker", + retrieval_mode="legacy", + scope_allowlist="project:alpha,project:beta", + limit=20, + ) + + assert {claim["id"] for claim in listed["claims"]} == {allowed.id} + assert {claim["id"] for claim in queried["claims"]} == {allowed.id} + with sqlite3.connect(db) as conn: + counts = dict(conn.execute("SELECT id, access_count FROM claims")) + assert counts[wrong_scope.id] == 0 + assert counts[wrong_tenant.id] == 0 + + +def test_team_context_rejects_caller_db_and_workspace_switch(team_claims, tmp_path) -> None: + with pytest.raises(PermissionError, match="database"): + mcp_server.list_claims(db=str(tmp_path / "other.db")) + with pytest.raises(PermissionError, match="workspace"): + mcp_server.list_claims(workspace=str(tmp_path / "other-workspace")) + + +def test_team_ingest_uses_authenticated_principal_and_tenant(team_claims) -> None: + db, _workspace, *_claims = team_claims + access_control.set_role("mcp-reader", access_control.Role.WRITER) + + result = mcp_server.ingest_claim( + text="authenticated team writer marker", + sources_json='["test://writer"]', + source_agent="forged-writer", + scope="project:alpha", + ) + + with sqlite3.connect(db) as conn: + stored = conn.execute( + "SELECT tenant_id, scope, source_agent FROM claims WHERE id = ?", + (result["claim"]["id"],), + ).fetchone() + assert stored == ("tenant-alpha", "project:alpha", "mcp-reader") From 304b718b2b152b2325c895f2b2d477e08330539a Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 00:16:45 -0300 Subject: [PATCH 06/55] fix(setup): declare local MCP trust mode --- .mcp.json.example | 1 + ...EMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md | 2 +- .planning/audit-remediation-ledger.md | 2 +- README.md | 9 ++++++++- docs/env-reference.md | 7 +++++++ memorymaster/surfaces/setup_hooks.py | 2 ++ tests/test_mcp_action_registry.py | 19 +++++++++++++++++++ tests/test_setup_hooks.py | 12 ++++++++++++ 8 files changed, 51 insertions(+), 3 deletions(-) diff --git a/.mcp.json.example b/.mcp.json.example index e96cd1e2..25c397ae 100644 --- a/.mcp.json.example +++ b/.mcp.json.example @@ -6,6 +6,7 @@ "env": { "MEMORYMASTER_DEFAULT_DB": "/path/to/memorymaster.db", "MEMORYMASTER_WORKSPACE": "/path/to/your/project", + "MEMORYMASTER_MCP_AUTH_MODE": "local-trusted", "QDRANT_URL": "http://localhost:6333", "OLLAMA_URL": "http://localhost:11434" } diff --git a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md index b99525f8..da4b34da 100644 --- a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md +++ b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md @@ -82,7 +82,7 @@ Phase 0 must replace qualitative capacity goals with recorded numeric gates in t ### Program tracker - [x] Phase 0 — evidence, numerical gates, and red tests -- [ ] R1.1 — authenticated request context and MCP authorization +- [x] R1.1 — authenticated request context and MCP authorization - [ ] R1.2 — Postgres tenant enforcement and RLS - [ ] R1.3 — immediate Qdrant containment - [ ] R1.4 — complete sensitivity/write gateway and legacy inventory diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 8ef609a5..37218ffa 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -8,7 +8,7 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | ID | Primary domain | Severity / exploitability | Summary | Package | Status | Acceptance evidence | |---|---|---|---|---|---|---| -| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | Frozen context and complete 36-tool action registry; reader spoof denial before service; SQLite team matrix proves tenant/scope intersection and principal pinning; auxiliary tools and Postgres RLS remain | +| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | R1.1 complete: all 36 tools have actions/context, reader spoof denial precedes service, SQLite team matrix proves tenant/scope/principal pinning, and unverified tools fail closed; Postgres RLS remains | | MM-SEC-02 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | Qdrant can return archived, sensitive, cross-scope/tenant, or orphan payloads | R1.3,R2.1 | OPEN | Fake and real Qdrant authoritative-filter tests | | MM-OPS-01 | DevOps | Critical / EXPLOITABLE-LOW-EFFORT / H4 | Postgres Compose publishes a fixed credential | R1.5 | OPEN | Fail-closed Compose config, private port, external rotation evidence | | MM-SEC-03 | Security | High / EXPLOITABLE-NOW | Persisted metadata/provenance fields bypass sensitivity scanning | R1.4 | OPEN | Complete field-matrix adversarial suite and legacy dry-run | diff --git a/README.md b/README.md index 2614a8d2..0e54dc0d 100644 --- a/README.md +++ b/README.md @@ -170,13 +170,20 @@ For zero-cost offline use, install [Ollama](https://ollama.com), `ollama pull ll "command": "memorymaster-mcp", "env": { "MEMORYMASTER_DEFAULT_DB": "/path/to/memorymaster.db", - "MEMORYMASTER_WORKSPACE": "/path/to/your/project" + "MEMORYMASTER_WORKSPACE": "/path/to/your/project", + "MEMORYMASTER_MCP_AUTH_MODE": "local-trusted" } } } } ``` +MCP authorization mode is mandatory. Use `local-trusted` only for a private +stdio process controlled by one OS user. Team mode instead requires an +operator-configured principal, explicit role, tenant, database, workspace, and +scope allowlist; unverified host-wide and maintenance tools fail closed. Existing +brownfield MCP entries must add the mode or be regenerated with setup `--force`. + 30 MCP tools spanning setup/lifecycle, ingest, query/retrieval, listing, knowledge graph, and governance: `init_db`, `ingest_claim`, `ingest_rule`, `query_rules`, `rules_export`, `run_cycle`, `run_steward`, `classify_query`, `query_memory`, `query_for_context`, `query_for_task`, `query_claim_paths`, `query_meta_decisions`, `federated_query`, `recall_analysis`, `read_active_tasks`, `list_claims`, `redact_claim_payload`, `pin_claim`, `compact_memory`, `list_events`, `search_verbatim`, `open_dashboard`, `list_steward_proposals`, `resolve_steward_proposal`, `extract_entities`, `entity_stats`, `find_related_claims`, `quality_scores`, `recompute_tiers`. See [`docs/MCP-TOOLS.md`](docs/MCP-TOOLS.md) for the grouped reference (one line per tool), and [`.mcp.json.example`](.mcp.json.example) for the full config template. diff --git a/docs/env-reference.md b/docs/env-reference.md index 74c7b4fd..00668250 100644 --- a/docs/env-reference.md +++ b/docs/env-reference.md @@ -72,7 +72,14 @@ Total: 134 variables. | `MEMORYMASTER_MAX_PROVIDER_FAILURES_PER_CYCLE` | `memorymaster/core/llm_budget.py` | | `MEMORYMASTER_MAX_TOKENS_PER_CYCLE` | `memorymaster/core/llm_budget.py` | | `MEMORYMASTER_MCP_ADMIN_MODE` | `memorymaster/surfaces/mcp_path_policy.py`, `memorymaster/surfaces/mcp_server.py` | +| `MEMORYMASTER_MCP_ALLOWED_SCOPES` | `memorymaster/core/access_control.py` | +| `MEMORYMASTER_MCP_ALLOW_SENSITIVE` | `memorymaster/core/access_control.py` | +| `MEMORYMASTER_MCP_AUTH_MODE` | `memorymaster/core/access_control.py`, `memorymaster/surfaces/setup_hooks.py` | +| `MEMORYMASTER_MCP_DB` | `memorymaster/core/access_control.py` | | `MEMORYMASTER_MCP_DB_ALLOWLIST` | `memorymaster/surfaces/mcp_path_policy.py`, `memorymaster/surfaces/mcp_server.py` | +| `MEMORYMASTER_MCP_PRINCIPAL` | `memorymaster/core/access_control.py` | +| `MEMORYMASTER_MCP_TENANT_ID` | `memorymaster/core/access_control.py` | +| `MEMORYMASTER_MCP_WORKSPACE` | `memorymaster/core/access_control.py` | | `MEMORYMASTER_MCP_WORKSPACE_ALLOWLIST` | `memorymaster/surfaces/mcp_path_policy.py` | | `MEMORYMASTER_PATH_ROOTS` | `memorymaster/bridges/local_search/redact.py` | | `MEMORYMASTER_PINNED_BONUS` | `memorymaster/core/config.py` | diff --git a/memorymaster/surfaces/setup_hooks.py b/memorymaster/surfaces/setup_hooks.py index 18d3c871..e8371452 100644 --- a/memorymaster/surfaces/setup_hooks.py +++ b/memorymaster/surfaces/setup_hooks.py @@ -317,6 +317,7 @@ def _mcp_server_entry(db_path: str) -> dict[str, Any]: "env": { "MEMORYMASTER_DEFAULT_DB": db_path, "MEMORYMASTER_WORKSPACE": str(PROJECT_ROOT), + "MEMORYMASTER_MCP_AUTH_MODE": "local-trusted", }, } @@ -385,6 +386,7 @@ def install_mcp_codex(*, force: bool = False): "[mcp_servers.memorymaster.env]\n" f"MEMORYMASTER_DEFAULT_DB = {json.dumps(db_path)}\n" f"MEMORYMASTER_WORKSPACE = {json.dumps(str(PROJECT_ROOT))}\n" + 'MEMORYMASTER_MCP_AUTH_MODE = "local-trusted"\n' f"{_CODEX_MCP_END}\n" ) diff --git a/tests/test_mcp_action_registry.py b/tests/test_mcp_action_registry.py index 70f9b29e..b5bb70e3 100644 --- a/tests/test_mcp_action_registry.py +++ b/tests/test_mcp_action_registry.py @@ -113,3 +113,22 @@ def test_missing_team_identity_fails_before_tool_body(monkeypatch) -> None: with pytest.raises(PermissionError, match="MEMORYMASTER_MCP_PRINCIPAL"): mcp_server.query_memory(query="must fail before opening the database") + + +@pytest.mark.parametrize( + "tool_name", + sorted(name for name, policy in mcp_server.MCP_TOOL_POLICIES.items() if not policy.team_enabled), +) +def test_unverified_team_tools_fail_before_body(monkeypatch, tool_name: str) -> None: + access_control.set_role("mcp-reader", access_control.Role.ADMIN) + + def sentinel() -> None: + pytest.fail(f"disabled team tool reached body: {tool_name}") + + sentinel.__name__ = tool_name + guarded = mcp_server._authorized_tool_callable( + sentinel, + mcp_server.MCP_TOOL_POLICIES[tool_name], + ) + with pytest.raises(PermissionError, match="disabled in team mode"): + guarded() diff --git a/tests/test_setup_hooks.py b/tests/test_setup_hooks.py index cd10a361..d7c09ff3 100644 --- a/tests/test_setup_hooks.py +++ b/tests/test_setup_hooks.py @@ -127,6 +127,12 @@ def test_ask_uses_input_when_interactive(self, monkeypatch): class TestInstallMcp: + def test_writes_explicit_local_trusted_auth_mode(self, hermetic_home): + sh.install_mcp(force=True) + data = json.loads(hermetic_home["claude_json"].read_text(encoding="utf-8")) + entry = data["mcpServers"]["memorymaster"] + assert entry["env"]["MEMORYMASTER_MCP_AUTH_MODE"] == "local-trusted" + def test_uses_non_deprecated_command(self, hermetic_home): sh.install_mcp(force=True) data = json.loads(hermetic_home["claude_json"].read_text(encoding="utf-8")) @@ -171,6 +177,12 @@ def test_preserves_other_servers(self, hermetic_home): class TestInstallMcpCodex: + def test_writes_explicit_local_trusted_auth_mode(self, hermetic_home): + hermetic_home["codex_dir"].mkdir(parents=True, exist_ok=True) + sh.install_mcp_codex(force=True) + content = (hermetic_home["codex_dir"] / "config.toml").read_text(encoding="utf-8") + assert 'MEMORYMASTER_MCP_AUTH_MODE = "local-trusted"' in content + def test_writes_managed_block_with_correct_command(self, hermetic_home): hermetic_home["codex_dir"].mkdir(parents=True, exist_ok=True) sh.install_mcp_codex(force=True) From a37f5172884051da51040366201a0cb583a407fa Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 00:21:48 -0300 Subject: [PATCH 07/55] fix(postgres): bind tenant context before connect --- .planning/audit-remediation-ledger.md | 2 +- memorymaster/core/service.py | 11 ++- memorymaster/stores/postgres_store.py | 26 ++++++- memorymaster/stores/store_factory.py | 14 +++- memorymaster/surfaces/mcp_server.py | 1 + tests/test_postgres_tenant_context.py | 107 ++++++++++++++++++++++++++ 6 files changed, 154 insertions(+), 7 deletions(-) create mode 100644 tests/test_postgres_tenant_context.py diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 37218ffa..206de8a6 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -8,7 +8,7 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | ID | Primary domain | Severity / exploitability | Summary | Package | Status | Acceptance evidence | |---|---|---|---|---|---|---| -| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | R1.1 complete: all 36 tools have actions/context, reader spoof denial precedes service, SQLite team matrix proves tenant/scope/principal pinning, and unverified tools fail closed; Postgres RLS remains | +| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | R1.1 complete; Postgres team connections now require tenant before driver load and set the tenant GUC on every connection; versioned RLS policies and real-DSN matrix remain | | MM-SEC-02 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | Qdrant can return archived, sensitive, cross-scope/tenant, or orphan payloads | R1.3,R2.1 | OPEN | Fake and real Qdrant authoritative-filter tests | | MM-OPS-01 | DevOps | Critical / EXPLOITABLE-LOW-EFFORT / H4 | Postgres Compose publishes a fixed credential | R1.5 | OPEN | Fail-closed Compose config, private port, external rotation evidence | | MM-SEC-03 | Security | High / EXPLOITABLE-NOW | Persisted metadata/provenance fields bypass sensitivity scanning | R1.4 | OPEN | Complete field-matrix adversarial suite and legacy dry-run | diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index 303ba7ca..c4961ea6 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -358,16 +358,23 @@ def __init__( policy_config: Mapping[str, object] | None = None, tenant_id: str | None = None, read_only: bool = False, + require_tenant: bool = False, ) -> None: # read_only (P1 WAL-discipline, spec §2.2): SQLite store opens # mode=ro + query_only connections; _record_accesses spools its # access/feedback signal instead of writing. Used by the per-prompt # recall hook under MEMORYMASTER_WAL_DISCIPLINE=1. - self.store = create_store(db_target, read_only=read_only) + self.tenant_id = (tenant_id or "").strip() or None + self.require_tenant = bool(require_tenant) + self.store = create_store( + db_target, + read_only=read_only, + tenant_id=self.tenant_id, + require_tenant=self.require_tenant, + ) self.workspace_root = Path(workspace_root) if workspace_root else Path.cwd() self._embedding_provider: EmbeddingProvider | None = None self.policy_config = policy_config - self.tenant_id = (tenant_id or "").strip() or None # Rollup telemetry (additive, default-safe): when set by a surface, # recall events are attributed to this agent / session for the usage # rollup. Default None keeps recall behaviour byte-identical. diff --git a/memorymaster/stores/postgres_store.py b/memorymaster/stores/postgres_store.py index 4f652824..713a3cfd 100644 --- a/memorymaster/stores/postgres_store.py +++ b/memorymaster/stores/postgres_store.py @@ -41,8 +41,16 @@ def utc_now() -> datetime: class PostgresStore(SQLiteStore): - def __init__(self, dsn: str) -> None: + def __init__( + self, + dsn: str, + *, + tenant_id: str | None = None, + require_tenant: bool = False, + ) -> None: self.dsn = dsn + self.tenant_id = (tenant_id or "").strip() or None + self.require_tenant = bool(require_tenant) self._psycopg: Any = None self._vector_table_available: bool | None = None @@ -60,12 +68,26 @@ def _load_psycopg(self) -> Any: return self._psycopg def connect(self) -> Any: + if self.require_tenant and self.tenant_id is None: + raise PermissionError("Postgres team mode requires a tenant context before connecting.") psycopg, dict_row, _ = self._load_psycopg() def _open() -> Any: return psycopg.connect(self.dsn, row_factory=dict_row) - return connect_with_retry(_open) + conn = connect_with_retry(_open) + if self.tenant_id is None: + return conn + try: + with conn.cursor() as cur: + cur.execute( + "SELECT set_config('memorymaster.tenant_id', %s, false)", + (self.tenant_id,), + ) + except Exception: + conn.close() + raise + return conn def init_db(self) -> None: from memorymaster.stores._storage_schema import load_schema_postgres_sql diff --git a/memorymaster/stores/store_factory.py b/memorymaster/stores/store_factory.py index cca1acd8..ce2df98e 100644 --- a/memorymaster/stores/store_factory.py +++ b/memorymaster/stores/store_factory.py @@ -10,7 +10,13 @@ def is_postgres_dsn(value: str) -> bool: return lowered.startswith("postgres://") or lowered.startswith("postgresql://") -def create_store(db_target: str | Path, *, read_only: bool = False): +def create_store( + db_target: str | Path, + *, + read_only: bool = False, + tenant_id: str | None = None, + require_tenant: bool = False, +): """Build the store for ``db_target``. ``read_only`` (P1 WAL-discipline, spec §2.2) puts a SQLite store into @@ -23,5 +29,9 @@ def create_store(db_target: str | Path, *, read_only: bool = False): if is_postgres_dsn(target): from memorymaster.stores.postgres_store import PostgresStore - return PostgresStore(target) + return PostgresStore( + target, + tenant_id=tenant_id, + require_tenant=require_tenant, + ) return SQLiteStore(Path(target), read_only=read_only) diff --git a/memorymaster/surfaces/mcp_server.py b/memorymaster/surfaces/mcp_server.py index 5638d1f6..87a8f7c9 100644 --- a/memorymaster/surfaces/mcp_server.py +++ b/memorymaster/surfaces/mcp_server.py @@ -345,6 +345,7 @@ def _service(db: str, workspace: str) -> MemoryService: db_target=db_path, workspace_root=Path(workspace_path), tenant_id=tenant_id, + require_tenant=context is not None and context.mode is AuthMode.TEAM, ) _bind_telemetry_session(svc, db_path, principal, tenant_id) svc.source_agent = principal diff --git a/tests/test_postgres_tenant_context.py b/tests/test_postgres_tenant_context.py new file mode 100644 index 00000000..2895b0e7 --- /dev/null +++ b/tests/test_postgres_tenant_context.py @@ -0,0 +1,107 @@ +from __future__ import annotations + +import pytest + +import memorymaster.core.service as service_module +from memorymaster.core.service import MemoryService +from memorymaster.stores.postgres_store import PostgresStore +from memorymaster.stores.store_factory import create_store + + +class FakeCursor: + def __init__(self) -> None: + self.executed: list[tuple[str, tuple[object, ...]]] = [] + + def __enter__(self): + return self + + def __exit__(self, *_args) -> None: + return None + + def execute(self, sql: str, params: tuple[object, ...] = ()) -> None: + self.executed.append((sql, params)) + + +class FakeConnection: + def __init__(self) -> None: + self.cursor_instance = FakeCursor() + self.closed = False + + def cursor(self) -> FakeCursor: + return self.cursor_instance + + def close(self) -> None: + self.closed = True + + +class FakePsycopg: + def __init__(self, connection: FakeConnection) -> None: + self.connection = connection + self.calls = 0 + + def connect(self, *_args, **_kwargs) -> FakeConnection: + self.calls += 1 + return self.connection + + +def test_required_tenant_fails_before_loading_postgres_driver(monkeypatch) -> None: + store = PostgresStore("postgresql://db.invalid/app", require_tenant=True) + monkeypatch.setattr( + store, + "_load_psycopg", + lambda: pytest.fail("missing tenant reached the Postgres driver"), + ) + + with pytest.raises(PermissionError, match="tenant"): + store.connect() + + +def test_connect_sets_tenant_context_on_every_connection() -> None: + connection = FakeConnection() + driver = FakePsycopg(connection) + store = PostgresStore( + "postgresql://db.invalid/app", + tenant_id="tenant-alpha", + require_tenant=True, + ) + store._psycopg = (driver, object(), object()) + + returned = store.connect() + + assert returned is connection + assert driver.calls == 1 + sql, params = connection.cursor_instance.executed[0] + assert "set_config('memorymaster.tenant_id'" in sql + assert params == ("tenant-alpha",) + + +def test_store_factory_propagates_postgres_tenant_context() -> None: + store = create_store( + "postgresql://db.invalid/app", + tenant_id="tenant-alpha", + require_tenant=True, + ) + + assert isinstance(store, PostgresStore) + assert store.tenant_id == "tenant-alpha" + assert store.require_tenant is True + + +def test_memory_service_propagates_tenant_requirement(monkeypatch) -> None: + captured: dict[str, object] = {} + + def fake_create_store(db_target, **kwargs): + captured.update({"db_target": db_target, **kwargs}) + return object() + + monkeypatch.setattr(service_module, "create_store", fake_create_store) + monkeypatch.setattr(MemoryService, "_init_qdrant", staticmethod(lambda: None)) + + MemoryService( + "postgresql://db.invalid/app", + tenant_id="tenant-alpha", + require_tenant=True, + ) + + assert captured["tenant_id"] == "tenant-alpha" + assert captured["require_tenant"] is True From 4896f1689aa0e36ddbfdc89dadfc99281968c1cc Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 00:31:20 -0300 Subject: [PATCH 08/55] fix(postgres): add tenant row security policies --- .planning/audit-remediation-ledger.md | 2 +- external-actions-required.md | 2 +- memorymaster/schema_postgres.sql | 4 + .../migrations/0008_postgres_tenant_rls.py | 100 +++++++++++++++ tests/test_postgres_rls_migration.py | 120 ++++++++++++++++++ 5 files changed, 226 insertions(+), 2 deletions(-) create mode 100644 memorymaster/stores/migrations/0008_postgres_tenant_rls.py create mode 100644 tests/test_postgres_rls_migration.py diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 206de8a6..2f5a476b 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -8,7 +8,7 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | ID | Primary domain | Severity / exploitability | Summary | Package | Status | Acceptance evidence | |---|---|---|---|---|---|---| -| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | R1.1 complete; Postgres team connections now require tenant before driver load and set the tenant GUC on every connection; versioned RLS policies and real-DSN matrix remain | +| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | R1.1 complete; Postgres team connections require tenant before driver load and set the tenant GUC on every connection; v0008 adds restrictive tenant policies to 8 claim-owned tables and fail-closed policies to 7 untenantable tables; privileged-role rejection, tenant-aware constraints/direct predicates, and the real two-role DSN matrix remain | | MM-SEC-02 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | Qdrant can return archived, sensitive, cross-scope/tenant, or orphan payloads | R1.3,R2.1 | OPEN | Fake and real Qdrant authoritative-filter tests | | MM-OPS-01 | DevOps | Critical / EXPLOITABLE-LOW-EFFORT / H4 | Postgres Compose publishes a fixed credential | R1.5 | OPEN | Fail-closed Compose config, private port, external rotation evidence | | MM-SEC-03 | Security | High / EXPLOITABLE-NOW | Persisted metadata/provenance fields bypass sensitivity scanning | R1.4 | OPEN | Complete field-matrix adversarial suite and legacy dry-run | diff --git a/external-actions-required.md b/external-actions-required.md index 31a01c15..bec6e664 100644 --- a/external-actions-required.md +++ b/external-actions-required.md @@ -5,7 +5,7 @@ Items here are `BLOCKED-EXTERNAL` only when the remediation ledger explicitly ma | Finding | Owner/system | Required action | Evidence needed to unblock | Review date | Status | |---|---|---|---|---|---| | MM-OPS-01 | Operator / any Postgres deployment | Rotate the historical `mm_pw` credential and recreate deployments that used it. Keep database ports private. | Rotation record plus network probe showing the port is not externally reachable | 2026-07-31 | PENDING-INVENTORY | -| MM-SEC-01 | Team Postgres test environment | Provide `MEMORYMASTER_TEST_POSTGRES_DSN` for adversarial RLS and application-role tests. | CI/runtime output proving cross-tenant SELECT/UPDATE denial | 2026-07-31 | BLOCKED-EXTERNAL | +| MM-SEC-01 | Team Postgres test environment | Provide a disposable admin/migrator DSN in `MEMORYMASTER_TEST_POSTGRES_DSN` and a distinct non-owner `NOSUPERUSER NOBYPASSRLS` application DSN in `MEMORYMASTER_TEST_POSTGRES_APP_DSN`; neither may target product data. | Catalog assertions plus the real cross-tenant read/write/constraint matrix; app role has no table ownership, `public` CREATE, TRUNCATE, DDL, or migration rights | 2026-07-31 | BLOCKED-EXTERNAL | | MM-OPS-02 | Docker/Helm runtime | Provide Docker and, for final verification, a disposable Kubernetes target if unavailable locally. | Container health/MCP handshake and Helm readiness/network-policy evidence | 2026-08-15 | PENDING-CAPABILITY-CHECK | | MM-SEC-02 | Qdrant runtime | Provide a disposable authenticated/TLS Qdrant target for final parity after fake-backed tests pass. | Real service policy/reconciliation test output | 2026-08-15 | PENDING-CAPABILITY-CHECK | | MM-DATA-01 | Live MemoryMaster operator | Approve a consistent backup/restore drill before any live migration, redaction, backlog, or retention operation. | Restored backup, integrity check, counts/checksums, approval record | 2026-08-15 | BLOCKED-EXTERNAL | diff --git a/memorymaster/schema_postgres.sql b/memorymaster/schema_postgres.sql index 9ba766f9..595a55e0 100644 --- a/memorymaster/schema_postgres.sql +++ b/memorymaster/schema_postgres.sql @@ -7,6 +7,10 @@ EXCEPTION END $$; +-- Tenant row-security policy is intentionally versioned in migration 0008. +-- Several protected tables are themselves created by migrations, so applying +-- the complete policy set here would run before those relations exist. + CREATE TABLE IF NOT EXISTS claims ( id BIGSERIAL PRIMARY KEY, text TEXT NOT NULL, diff --git a/memorymaster/stores/migrations/0008_postgres_tenant_rls.py b/memorymaster/stores/migrations/0008_postgres_tenant_rls.py new file mode 100644 index 00000000..fc2034b8 --- /dev/null +++ b/memorymaster/stores/migrations/0008_postgres_tenant_rls.py @@ -0,0 +1,100 @@ +"""PostgreSQL row security for tenant-owned and team-disabled data. + +The application role is intentionally expected to be a non-owner role without +``BYPASSRLS``. Owners and migration roles retain their normal administrative +access; callers requiring tenant isolation must reject those privileged roles +at the connection boundary. + +Each tenant table gets an explicit permissive base plus a restrictive tenant +gate. PostgreSQL ORs permissive policies but ANDs restrictive policies, so a +later permissive policy cannot widen the tenant boundary. Tables whose schema +has no tenant key are denied to the team role until a future migration makes +their ownership model explicit. +""" +from __future__ import annotations + +VERSION = 8 +DESCRIPTION = "Postgres tenant row-level security policies" + +_CURRENT_TENANT = "NULLIF(current_setting('memorymaster.tenant_id', true), '')" + +_TENANT_PREDICATES = { + "claims": f"claims.tenant_id = {_CURRENT_TENANT}", + "mcp_usage": f"mcp_usage.tenant_id = {_CURRENT_TENANT}", + "citations": ( + "EXISTS (SELECT 1 FROM claims AS mm_claim " + "WHERE mm_claim.id = citations.claim_id " + f"AND mm_claim.tenant_id = {_CURRENT_TENANT})" + ), + "events": ( + "events.claim_id IS NOT NULL AND EXISTS (" + "SELECT 1 FROM claims AS mm_claim WHERE mm_claim.id = events.claim_id " + f"AND mm_claim.tenant_id = {_CURRENT_TENANT})" + ), + "claim_links": ( + "EXISTS (SELECT 1 FROM claims AS mm_source " + "WHERE mm_source.id = claim_links.source_id " + f"AND mm_source.tenant_id = {_CURRENT_TENANT}) AND " + "EXISTS (SELECT 1 FROM claims AS mm_target " + "WHERE mm_target.id = claim_links.target_id " + f"AND mm_target.tenant_id = {_CURRENT_TENANT})" + ), + "claim_embeddings": ( + "EXISTS (SELECT 1 FROM claims AS mm_claim " + "WHERE mm_claim.id = claim_embeddings.claim_id " + f"AND mm_claim.tenant_id = {_CURRENT_TENANT})" + ), + "action_proposals": ( + "action_proposals.claim_id IS NOT NULL AND EXISTS (" + "SELECT 1 FROM claims AS mm_claim " + "WHERE mm_claim.id = action_proposals.claim_id " + f"AND mm_claim.tenant_id = {_CURRENT_TENANT})" + ), + "contradiction_verdicts": ( + "EXISTS (SELECT 1 FROM claims AS mm_a " + "WHERE mm_a.id = contradiction_verdicts.claim_a_id " + f"AND mm_a.tenant_id = {_CURRENT_TENANT}) AND " + "EXISTS (SELECT 1 FROM claims AS mm_b " + "WHERE mm_b.id = contradiction_verdicts.claim_b_id " + f"AND mm_b.tenant_id = {_CURRENT_TENANT})" + ), +} + +_TEAM_DENY_TABLES = ( + "external_sources", + "source_items", + "evidence_items", + "media_retry_queue", + "query_cache", + "miner_state", + "rule_stats", +) + + +def _enable_policy(cur, table: str, policy: str, predicate: str) -> None: + cur.execute(f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY") + cur.execute(f"DROP POLICY IF EXISTS memorymaster_rls_permit ON {table}") + cur.execute( + f"CREATE POLICY memorymaster_rls_permit ON {table} " + "AS PERMISSIVE FOR ALL TO PUBLIC USING (TRUE) WITH CHECK (TRUE)" + ) + cur.execute(f"DROP POLICY IF EXISTS {policy} ON {table}") + cur.execute( + f"CREATE POLICY {policy} ON {table} AS RESTRICTIVE FOR ALL TO PUBLIC " + f"USING ({predicate}) WITH CHECK ({predicate})" + ) + + +def apply_sqlite(_conn) -> None: + """RLS is PostgreSQL-specific; SQLite isolation stays predicate-based.""" + + +def apply_postgres(conn) -> None: + with conn.cursor() as cur: + for table, predicate in _TENANT_PREDICATES.items(): + _enable_policy(cur, table, "memorymaster_tenant_restrict", predicate) + for table in _TEAM_DENY_TABLES: + _enable_policy(cur, table, "memorymaster_team_deny", "FALSE") + commit = getattr(conn, "commit", None) + if callable(commit): + commit() diff --git a/tests/test_postgres_rls_migration.py b/tests/test_postgres_rls_migration.py new file mode 100644 index 00000000..669d2f1e --- /dev/null +++ b/tests/test_postgres_rls_migration.py @@ -0,0 +1,120 @@ +"""Adversarial contract tests for the PostgreSQL tenant-RLS migration.""" +from __future__ import annotations + +import importlib +import sqlite3 + +from memorymaster.stores.migrations import discover_migrations + + +TENANT_POLICY_TABLES = { + "claims", + "citations", + "events", + "claim_links", + "claim_embeddings", + "action_proposals", + "contradiction_verdicts", + "mcp_usage", +} + +TEAM_DENY_TABLES = { + "external_sources", + "source_items", + "evidence_items", + "media_retry_queue", + "query_cache", + "miner_state", + "rule_stats", +} + + +class RecordingCursor: + def __init__(self, statements: list[str]) -> None: + self.statements = statements + + def __enter__(self) -> "RecordingCursor": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, sql: str, _params: object = None) -> None: + self.statements.append(" ".join(sql.split())) + + +class RecordingConnection: + def __init__(self) -> None: + self.statements: list[str] = [] + self.commits = 0 + + def cursor(self) -> RecordingCursor: + return RecordingCursor(self.statements) + + def commit(self) -> None: + self.commits += 1 + + +def _apply_postgres_migration() -> RecordingConnection: + migration = importlib.import_module( + "memorymaster.stores.migrations.0008_postgres_tenant_rls" + ) + conn = RecordingConnection() + migration.apply_postgres(conn) + return conn + + +def test_postgres_tenant_rls_is_versioned() -> None: + migration = next(item for item in discover_migrations() if item.version == 8) + + assert "tenant" in migration.description.lower() + assert "row" in migration.description.lower() + + +def test_sqlite_side_of_postgres_rls_migration_is_noop() -> None: + migration = importlib.import_module( + "memorymaster.stores.migrations.0008_postgres_tenant_rls" + ) + conn = sqlite3.connect(":memory:") + try: + migration.apply_sqlite(conn) + tables = conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table'" + ).fetchall() + finally: + conn.close() + + assert tables == [] + + +def test_tenant_policy_cannot_be_widened_by_permissive_policy() -> None: + conn = _apply_postgres_migration() + emitted = "\n".join(conn.statements) + + for table in TENANT_POLICY_TABLES: + assert f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY" in emitted + assert f"CREATE POLICY memorymaster_tenant_restrict ON {table}" in emitted + assert emitted.count("AS RESTRICTIVE") >= len(TENANT_POLICY_TABLES) + assert "current_setting('memorymaster.tenant_id', true)" in emitted + assert "USING" in emitted + assert "WITH CHECK" in emitted + assert conn.commits == 1 + + +def test_missing_tenant_cannot_match_tenantless_rows() -> None: + emitted = "\n".join(_apply_postgres_migration().statements) + + assert ( + "tenant_id = NULLIF(current_setting('memorymaster.tenant_id', true), '')" + in emitted + ) + assert "tenant_id IS NOT DISTINCT FROM" not in emitted + + +def test_untenantable_team_tables_are_explicitly_denied() -> None: + emitted = "\n".join(_apply_postgres_migration().statements) + + for table in TEAM_DENY_TABLES: + assert f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY" in emitted + assert f"CREATE POLICY memorymaster_team_deny ON {table}" in emitted + assert emitted.count("USING (FALSE) WITH CHECK (FALSE)") >= len(TEAM_DENY_TABLES) From 6ccb8180c25eeaa5bcf977a39fecd146efd61736 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 01:06:33 -0300 Subject: [PATCH 09/55] fix(storage): make claim identities tenant-local --- .planning/audit-remediation-ledger.md | 2 +- memorymaster/core/service.py | 15 +- memorymaster/govern/jobs/validator.py | 1 + memorymaster/schema.sql | 20 +- memorymaster/schema_postgres.sql | 41 +-- memorymaster/stores/_storage_read.py | 51 +++- memorymaster/stores/_storage_schema.py | 74 ++++-- memorymaster/stores/_storage_write_claims.py | 37 ++- .../0009_tenant_local_claim_identities.py | 129 ++++++++++ memorymaster/stores/postgres_store.py | 175 ++++++++++--- memorymaster/surfaces/cli_helpers.py | 5 +- tests/test_postgres_tenant_identity_sql.py | 97 ++++++++ tests/test_tenant_local_claim_identities.py | 233 ++++++++++++++++++ tests/test_tenant_local_identity_migration.py | 174 +++++++++++++ 14 files changed, 958 insertions(+), 96 deletions(-) create mode 100644 memorymaster/stores/migrations/0009_tenant_local_claim_identities.py create mode 100644 tests/test_postgres_tenant_identity_sql.py create mode 100644 tests/test_tenant_local_claim_identities.py create mode 100644 tests/test_tenant_local_identity_migration.py diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 2f5a476b..933f355f 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -8,7 +8,7 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | ID | Primary domain | Severity / exploitability | Summary | Package | Status | Acceptance evidence | |---|---|---|---|---|---|---| -| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | R1.1 complete; Postgres team connections require tenant before driver load and set the tenant GUC on every connection; v0008 adds restrictive tenant policies to 8 claim-owned tables and fail-closed policies to 7 untenantable tables; privileged-role rejection, tenant-aware constraints/direct predicates, and the real two-role DSN matrix remain | +| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | R1.1 complete; Postgres team connections require tenant before driver load and set the tenant GUC on every connection; v0008 adds restrictive tenant policies to 8 claim-owned tables and fail-closed policies to 7 untenantable tables; v0009 makes idempotency keys, human IDs, and confirmed tuples tenant-local across SQLite/Postgres with tenant-qualified lookup/allocation/validation; privileged-role/scoped-GUC enforcement, tenant-aware event integrity/direct predicates, and the real two-role DSN matrix remain | | MM-SEC-02 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | Qdrant can return archived, sensitive, cross-scope/tenant, or orphan payloads | R1.3,R2.1 | OPEN | Fake and real Qdrant authoritative-filter tests | | MM-OPS-01 | DevOps | Critical / EXPLOITABLE-LOW-EFFORT / H4 | Postgres Compose publishes a fixed credential | R1.5 | OPEN | Fail-closed Compose config, private port, external rotation evidence | | MM-SEC-03 | Security | High / EXPLOITABLE-NOW | Persisted metadata/provenance fields bypass sensitivity scanning | R1.4 | OPEN | Complete field-matrix adversarial suite and legacy dry-run | diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index c4961ea6..6511514d 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -530,7 +530,10 @@ def ingest( # Dedup by idempotency key normalized_idempotency_key = (idempotency_key or "").strip() or None if normalized_idempotency_key is not None and hasattr(self.store, "get_claim_by_idempotency_key"): - existing_claim = self.store.get_claim_by_idempotency_key(normalized_idempotency_key) + existing_claim = self.store.get_claim_by_idempotency_key( + normalized_idempotency_key, + tenant_id=self.tenant_id, + ) if existing_claim is not None: observability.bump_claim_ingested(source_agent) return self._revive_archived_dedup_match(existing_claim, source_agent) @@ -541,7 +544,10 @@ def ingest( hash_input = f"{text.strip().lower()}|{scope}|{_tenant}" content_hash = "hash-" + hashlib.sha256(hash_input.encode()).hexdigest()[:16] if hasattr(self.store, "get_claim_by_idempotency_key"): - existing_by_hash = self.store.get_claim_by_idempotency_key(content_hash) + existing_by_hash = self.store.get_claim_by_idempotency_key( + content_hash, + tenant_id=self.tenant_id, + ) if existing_by_hash is not None: observability.bump_claim_ingested(source_agent) return self._revive_archived_dedup_match(existing_by_hash, source_agent) @@ -2101,7 +2107,10 @@ def query_claim_paths( result is simply empty (logged, no crash). """ try: - start_id = self.store.resolve_claim_id(claim_id) + start_id = self.store.resolve_claim_id( + claim_id, + tenant_id=self.tenant_id, + ) except ValueError: logger.info("query_claim_paths: unknown claim_id %r", claim_id) return [] diff --git a/memorymaster/govern/jobs/validator.py b/memorymaster/govern/jobs/validator.py index 92bd9cee..a464fa6f 100644 --- a/memorymaster/govern/jobs/validator.py +++ b/memorymaster/govern/jobs/validator.py @@ -131,6 +131,7 @@ def run( predicate=claim.predicate, scope=claim.scope, exclude_claim_id=claim.id, + tenant_id=claim.tenant_id, ) duplicate = next((x for x in related if x.object_value == claim.object_value and x.object_value), None) diff --git a/memorymaster/schema.sql b/memorymaster/schema.sql index 09e3f383..a485187b 100644 --- a/memorymaster/schema.sql +++ b/memorymaster/schema.sql @@ -31,6 +31,7 @@ CREATE TABLE IF NOT EXISTS claims ( valid_until TEXT, wiki_article TEXT, holder TEXT, + tenant_id TEXT, FOREIGN KEY (supersedes_claim_id) REFERENCES claims(id) ON DELETE SET NULL, FOREIGN KEY (replaced_by_claim_id) REFERENCES claims(id) ON DELETE SET NULL ); @@ -47,13 +48,14 @@ WHEN NEW.status = 'confirmed' AND c.subject = NEW.subject AND c.predicate = NEW.predicate AND c.scope = NEW.scope + AND c.tenant_id IS NEW.tenant_id ) BEGIN SELECT RAISE(ABORT, 'only one confirmed claim is allowed per (subject,predicate,scope)'); END; CREATE TRIGGER IF NOT EXISTS trg_claims_confirmed_tuple_guard_update -BEFORE UPDATE OF status, subject, predicate, scope ON claims +BEFORE UPDATE OF status, subject, predicate, scope, tenant_id ON claims WHEN NEW.status = 'confirmed' AND NEW.subject IS NOT NULL AND NEW.predicate IS NOT NULL @@ -65,6 +67,7 @@ WHEN NEW.status = 'confirmed' AND c.subject = NEW.subject AND c.predicate = NEW.predicate AND c.scope = NEW.scope + AND c.tenant_id IS NEW.tenant_id ) BEGIN SELECT RAISE(ABORT, 'only one confirmed claim is allowed per (subject,predicate,scope)'); @@ -190,14 +193,25 @@ CREATE TABLE IF NOT EXISTS mcp_usage ( CREATE INDEX IF NOT EXISTS idx_claims_status ON claims(status); CREATE INDEX IF NOT EXISTS idx_claims_updated_at ON claims(updated_at); -CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_idempotency_key ON claims(idempotency_key); +CREATE INDEX IF NOT EXISTS idx_claims_idempotency_key ON claims(idempotency_key); +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_idempotency_key + ON claims(COALESCE(tenant_id, ''), idempotency_key) + WHERE idempotency_key IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_claims_tuple ON claims(subject, predicate, scope); CREATE INDEX IF NOT EXISTS idx_claims_replaced_by ON claims(replaced_by_claim_id); CREATE INDEX IF NOT EXISTS idx_citations_claim_id ON citations(claim_id); CREATE INDEX IF NOT EXISTS idx_events_claim_id ON events(claim_id); CREATE INDEX IF NOT EXISTS idx_events_created_at ON events(created_at); CREATE INDEX IF NOT EXISTS idx_embeddings_updated_at ON claim_embeddings(updated_at); -CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id); +CREATE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id); +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_human_id + ON claims(COALESCE(tenant_id, ''), human_id) + WHERE human_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_confirmed_tuple_unique + ON claims(COALESCE(tenant_id, ''), subject, predicate, scope) + WHERE status = 'confirmed' + AND subject IS NOT NULL + AND predicate IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_external_sources_type ON external_sources(source_type); CREATE INDEX IF NOT EXISTS idx_source_items_source_id ON source_items(source_id); CREATE INDEX IF NOT EXISTS idx_source_items_chat_id ON source_items(chat_id); diff --git a/memorymaster/schema_postgres.sql b/memorymaster/schema_postgres.sql index 595a55e0..6d4d96b6 100644 --- a/memorymaster/schema_postgres.sql +++ b/memorymaster/schema_postgres.sql @@ -53,6 +53,9 @@ ALTER TABLE claims ALTER TABLE claims ADD COLUMN IF NOT EXISTS wiki_article TEXT; +ALTER TABLE claims + ADD COLUMN IF NOT EXISTS tenant_id TEXT; + -- Parity with SQLite schema.sql / dataclass defaults (postgres-parity audit). -- Forward-migrate these columns on pre-existing Postgres DBs created before -- the parity fix, BEFORE the 0004 query_cache trigger references valid_from/ @@ -84,30 +87,21 @@ BEGIN AND c.subject = NEW.subject AND c.predicate = NEW.predicate AND c.scope = NEW.scope + AND c.tenant_id IS NOT DISTINCT FROM NEW.tenant_id AND (TG_OP = 'INSERT' OR c.id <> NEW.id) ) THEN - RAISE EXCEPTION 'only one confirmed claim is allowed per (subject,predicate,scope)' + RAISE EXCEPTION 'only one confirmed claim is allowed per tenant and (subject,predicate,scope)' USING ERRCODE = '23505'; END IF; RETURN NEW; END; $$; -DO $$ -BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM pg_trigger - WHERE tgname = 'trg_claims_confirmed_tuple_guard' - AND tgrelid = 'claims'::regclass - ) THEN - CREATE TRIGGER trg_claims_confirmed_tuple_guard - BEFORE INSERT OR UPDATE OF status, subject, predicate, scope ON claims - FOR EACH ROW - EXECUTE FUNCTION memorymaster_claims_confirmed_tuple_guard(); - END IF; -END -$$; +DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard ON claims; +CREATE TRIGGER trg_claims_confirmed_tuple_guard +BEFORE INSERT OR UPDATE OF status, subject, predicate, scope, tenant_id ON claims +FOR EACH ROW +EXECUTE FUNCTION memorymaster_claims_confirmed_tuple_guard(); CREATE TABLE IF NOT EXISTS citations ( id BIGSERIAL PRIMARY KEY, @@ -170,11 +164,22 @@ BEGIN END $$; -CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id); +CREATE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id); +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_human_id + ON claims(COALESCE(tenant_id, ''), human_id) + WHERE human_id IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_claims_tenant_id ON claims(tenant_id); CREATE INDEX IF NOT EXISTS idx_claims_status ON claims(status); CREATE INDEX IF NOT EXISTS idx_claims_updated_at ON claims(updated_at); -CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_idempotency_key ON claims(idempotency_key); +CREATE INDEX IF NOT EXISTS idx_claims_idempotency_key ON claims(idempotency_key); +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_idempotency_key + ON claims(COALESCE(tenant_id, ''), idempotency_key) + WHERE idempotency_key IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_confirmed_tuple_unique + ON claims(COALESCE(tenant_id, ''), subject, predicate, scope) + WHERE status = 'confirmed' + AND subject IS NOT NULL + AND predicate IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_claims_tuple ON claims(subject, predicate, scope); CREATE INDEX IF NOT EXISTS idx_claims_replaced_by ON claims(replaced_by_claim_id); CREATE INDEX IF NOT EXISTS idx_citations_claim_id ON citations(claim_id); diff --git a/memorymaster/stores/_storage_read.py b/memorymaster/stores/_storage_read.py index df9c1d8e..eedf48df 100644 --- a/memorymaster/stores/_storage_read.py +++ b/memorymaster/stores/_storage_read.py @@ -28,7 +28,12 @@ def connect(self) -> sqlite3.Connection: ... def init_db(self) -> None: ... - def _check_idempotency(self, conn: sqlite3.Connection, idempotency_key: str | None) -> Claim | None: + def _check_idempotency( + self, + conn: sqlite3.Connection, + idempotency_key: str | None, + tenant_id: str | None = None, + ) -> Claim | None: """Check if a claim with this idempotency key already exists. Returns existing claim or None.""" normalized_key = (idempotency_key or "").strip() or None if normalized_key is None: @@ -36,8 +41,8 @@ def _check_idempotency(self, conn: sqlite3.Connection, idempotency_key: str | No # Hydrate the full row from the already-open conn instead of paying # a fresh get_claim() connection open on every duplicate re-ingest. existing_row = conn.execute( - "SELECT * FROM claims WHERE idempotency_key = ?", - (normalized_key,), + "SELECT * FROM claims WHERE idempotency_key = ? AND tenant_id IS ?", + (normalized_key, tenant_id), ).fetchone() if existing_row is None: return None @@ -72,14 +77,20 @@ def get_claim(self, claim_id: int, include_citations: bool = True) -> Claim | No return claim - def get_claim_by_idempotency_key(self, idempotency_key: str, include_citations: bool = True) -> Claim | None: + def get_claim_by_idempotency_key( + self, + idempotency_key: str, + include_citations: bool = True, + *, + tenant_id: str | None = None, + ) -> Claim | None: normalized_idempotency_key = idempotency_key.strip() if not normalized_idempotency_key: return None with self.connect() as conn: row = conn.execute( - "SELECT * FROM claims WHERE idempotency_key = ?", - (normalized_idempotency_key,), + "SELECT * FROM claims WHERE idempotency_key = ? AND tenant_id IS ?", + (normalized_idempotency_key, tenant_id), ).fetchone() if row is None: return None @@ -89,7 +100,13 @@ def get_claim_by_idempotency_key(self, idempotency_key: str, include_citations: return claim - def get_claim_by_human_id(self, human_id: str, include_citations: bool = True) -> Claim | None: + def get_claim_by_human_id( + self, + human_id: str, + include_citations: bool = True, + *, + tenant_id: str | None = None, + ) -> Claim | None: """Look up a claim by its human-readable ID (e.g. ``mm-a3f8``).""" normalized = human_id.strip() if not normalized: @@ -97,8 +114,8 @@ def get_claim_by_human_id(self, human_id: str, include_citations: bool = True) - with self.connect() as conn: try: row = conn.execute( - "SELECT * FROM claims WHERE human_id = ?", - (normalized,), + "SELECT * FROM claims WHERE human_id = ? AND tenant_id IS ?", + (normalized, tenant_id), ).fetchone() except sqlite3.OperationalError: # Column may not exist yet. @@ -111,7 +128,12 @@ def get_claim_by_human_id(self, human_id: str, include_citations: bool = True) - return claim - def resolve_claim_id(self, identifier: str | int) -> int: + def resolve_claim_id( + self, + identifier: str | int, + *, + tenant_id: str | None = None, + ) -> int: """Resolve a numeric ID or human_id string to a numeric claim ID. Raises ``ValueError`` if the claim cannot be found. @@ -125,7 +147,11 @@ def resolve_claim_id(self, identifier: str | int) -> int: except ValueError: pass # Try human_id lookup. - claim = self.get_claim_by_human_id(raw, include_citations=False) + claim = self.get_claim_by_human_id( + raw, + include_citations=False, + tenant_id=tenant_id, + ) if claim is not None: return claim.id raise ValueError(f"No claim found for identifier '{raw}'.") @@ -443,12 +469,15 @@ def find_confirmed_by_tuple( predicate: str | None, scope: str | None, exclude_claim_id: int | None = None, + tenant_id: str | None = None, ) -> list[Claim]: if not subject or not predicate: return [] clauses = ["status = 'confirmed'", "subject = ?", "predicate = ?", "scope = ?"] params: list[object] = [subject, predicate, scope or "project"] + clauses.append("tenant_id IS ?") + params.append(tenant_id) if exclude_claim_id is not None: clauses.append("id <> ?") params.append(exclude_claim_id) diff --git a/memorymaster/stores/_storage_schema.py b/memorymaster/stores/_storage_schema.py index d06c25bd..3c39c0dc 100644 --- a/memorymaster/stores/_storage_schema.py +++ b/memorymaster/stores/_storage_schema.py @@ -212,16 +212,27 @@ def _ensure_atlas_source_schema(conn: sqlite3.Connection) -> None: @staticmethod def _ensure_claim_idempotency_schema(conn: sqlite3.Connection) -> None: + _SchemaMixin._ensure_tenant_id_schema(conn) try: conn.execute("ALTER TABLE claims ADD COLUMN idempotency_key TEXT") except sqlite3.OperationalError as exc: if "duplicate column name" not in str(exc).lower(): raise - conn.execute("CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_idempotency_key ON claims(idempotency_key)") + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_claims_idempotency_key ON claims(idempotency_key)" + ) + conn.execute( + """ + CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_idempotency_key + ON claims(COALESCE(tenant_id, ''), idempotency_key) + WHERE idempotency_key IS NOT NULL + """ + ) @staticmethod def _ensure_confirmed_tuple_uniqueness_schema(conn: sqlite3.Connection) -> None: + _SchemaMixin._ensure_tenant_id_schema(conn) for trigger in SQLITE_CONFIRMED_TUPLE_GUARD_TRIGGERS: conn.execute(f"DROP TRIGGER IF EXISTS {trigger}") conn.executescript( @@ -238,13 +249,14 @@ def _ensure_confirmed_tuple_uniqueness_schema(conn: sqlite3.Connection) -> None: AND c.subject = NEW.subject AND c.predicate = NEW.predicate AND c.scope = NEW.scope + AND c.tenant_id IS NEW.tenant_id ) BEGIN SELECT RAISE(ABORT, 'only one confirmed claim is allowed per (subject,predicate,scope)'); END; CREATE TRIGGER IF NOT EXISTS trg_claims_confirmed_tuple_guard_update - BEFORE UPDATE OF status, subject, predicate, scope ON claims + BEFORE UPDATE OF status, subject, predicate, scope, tenant_id ON claims WHEN NEW.status = 'confirmed' AND NEW.subject IS NOT NULL AND NEW.predicate IS NOT NULL @@ -256,6 +268,7 @@ def _ensure_confirmed_tuple_uniqueness_schema(conn: sqlite3.Connection) -> None: AND c.subject = NEW.subject AND c.predicate = NEW.predicate AND c.scope = NEW.scope + AND c.tenant_id IS NEW.tenant_id ) BEGIN SELECT RAISE(ABORT, 'only one confirmed claim is allowed per (subject,predicate,scope)'); @@ -266,7 +279,7 @@ def _ensure_confirmed_tuple_uniqueness_schema(conn: sqlite3.Connection) -> None: conn.execute( """ CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_confirmed_tuple_unique - ON claims(subject, predicate, scope) + ON claims(COALESCE(tenant_id, ''), subject, predicate, scope) WHERE status = 'confirmed' AND subject IS NOT NULL AND predicate IS NOT NULL @@ -499,13 +512,31 @@ def _ensure_claim_links_schema(conn: sqlite3.Connection) -> None: @staticmethod def _ensure_human_id_schema(conn: sqlite3.Connection) -> None: """Add human_id column if missing and backfill existing claims.""" + _SchemaMixin._ensure_tenant_id_schema(conn) try: conn.execute("ALTER TABLE claims ADD COLUMN human_id TEXT") except sqlite3.OperationalError as exc: if "duplicate column name" not in str(exc).lower(): raise + human_index = next( + ( + row + for row in conn.execute("PRAGMA index_list(claims)").fetchall() + if row[1] == "idx_claims_human_id" + ), + None, + ) + if human_index is not None and bool(human_index[2]): + conn.execute("DROP INDEX idx_claims_human_id") + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id)" + ) conn.execute( - "CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id)" + """ + CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_human_id + ON claims(COALESCE(tenant_id, ''), human_id) + WHERE human_id IS NOT NULL + """ ) _SchemaMixin._backfill_human_ids(conn) @@ -520,7 +551,7 @@ def _backfill_human_ids(conn: sqlite3.Connection) -> int: per-row claim_links JOIN and generate ids in-memory + executemany. """ rows = conn.execute( - "SELECT id, subject, text FROM claims WHERE human_id IS NULL ORDER BY id ASC" + "SELECT id, subject, text, tenant_id FROM claims WHERE human_id IS NULL ORDER BY id ASC" ).fetchall() if not rows: return 0 @@ -537,7 +568,13 @@ def _backfill_human_ids(conn: sqlite3.Connection) -> int: claim_id = int(row["id"]) subject = row["subject"] text = str(row["text"]) - human_id = _SchemaMixin._allocate_human_id(conn, subject, text, claim_id) + human_id = _SchemaMixin._allocate_human_id( + conn, + subject, + text, + claim_id, + tenant_id=row["tenant_id"], + ) conn.execute( "UPDATE claims SET human_id = ? WHERE id = ?", (human_id, claim_id), @@ -552,22 +589,23 @@ def _backfill_human_ids_top_level(conn: sqlite3.Connection, rows: list) -> int: Collisions are resolved in-memory against ids already present in the DB plus ids minted within this batch, then written via a single executemany. """ - taken: set[str] = { - str(r[0]) + taken: set[tuple[str | None, str]] = { + (r[0], str(r[1])) for r in conn.execute( - "SELECT human_id FROM claims WHERE human_id IS NOT NULL" + "SELECT tenant_id, human_id FROM claims WHERE human_id IS NOT NULL" ).fetchall() } updates: list[tuple[str, int]] = [] for row in rows: claim_id = int(row["id"]) + tenant_id = row["tenant_id"] candidate = generate_top_level_human_id(row["subject"], str(row["text"])) final = candidate suffix = 1 - while final in taken: + while (tenant_id, final) in taken: suffix += 1 final = f"{candidate}~{suffix}" - taken.add(final) + taken.add((tenant_id, final)) updates.append((final, claim_id)) conn.executemany("UPDATE claims SET human_id = ? WHERE id = ?", updates) return len(updates) @@ -579,6 +617,7 @@ def _allocate_human_id( subject: str | None, text: str, claim_id: int, + tenant_id: str | None = None, ) -> str: """Build a unique human_id, checking for derived_from parent links. @@ -595,16 +634,20 @@ def _allocate_human_id( WHERE cl.source_id = ? AND cl.link_type = 'derived_from' AND c.human_id IS NOT NULL + AND c.tenant_id IS ? LIMIT 1 """, - (claim_id,), + (claim_id, tenant_id), ).fetchone() if parent_row and parent_row["human_id"]: parent_hid = str(parent_row["human_id"]) child_count = conn.execute( - "SELECT COUNT(*) AS cnt FROM claims WHERE human_id LIKE ? AND human_id != ?", - (parent_hid + ".%", parent_hid), + """ + SELECT COUNT(*) AS cnt FROM claims + WHERE human_id LIKE ? AND human_id != ? AND tenant_id IS ? + """, + (parent_hid + ".%", parent_hid, tenant_id), ).fetchone() next_child = (int(child_count["cnt"]) if child_count else 0) + 1 candidate = f"{parent_hid}.{next_child}" @@ -616,7 +659,8 @@ def _allocate_human_id( suffix = 1 while True: existing = conn.execute( - "SELECT 1 FROM claims WHERE human_id = ?", (final,) + "SELECT 1 FROM claims WHERE human_id = ? AND tenant_id IS ?", + (final, tenant_id), ).fetchone() if existing is None: return final diff --git a/memorymaster/stores/_storage_write_claims.py b/memorymaster/stores/_storage_write_claims.py index 3fa737b3..7acc3eec 100644 --- a/memorymaster/stores/_storage_write_claims.py +++ b/memorymaster/stores/_storage_write_claims.py @@ -27,11 +27,23 @@ class _WriteClaimsMixin: if TYPE_CHECKING: def connect(self) -> sqlite3.Connection: ... - def _check_idempotency(self, conn: sqlite3.Connection, idempotency_key: str | None) -> Claim | None: ... + def _check_idempotency( + self, + conn: sqlite3.Connection, + idempotency_key: str | None, + tenant_id: str | None = None, + ) -> Claim | None: ... def get_claim(self, claim_id: int, include_citations: bool = True) -> Claim | None: ... - def _allocate_human_id(self, conn: sqlite3.Connection, subject: str | None, text: str, claim_id: int) -> str: ... + def _allocate_human_id( + self, + conn: sqlite3.Connection, + subject: str | None, + text: str, + claim_id: int, + tenant_id: str | None = None, + ) -> str: ... def _insert_event_row( self, @@ -73,7 +85,11 @@ def create_claim( normalized_tenant_id = (tenant_id or "").strip() or None now = utc_now() with self.connect() as conn: - existing = self._check_idempotency(conn, idempotency_key) + existing = self._check_idempotency( + conn, + idempotency_key, + tenant_id=normalized_tenant_id, + ) if existing is not None: return existing @@ -113,8 +129,11 @@ def create_claim( raise conn.rollback() existing_row = conn.execute( - "SELECT id FROM claims WHERE idempotency_key = ?", - (normalized_idempotency_key,), + """ + SELECT id FROM claims + WHERE idempotency_key = ? AND tenant_id IS ? + """, + (normalized_idempotency_key, normalized_tenant_id), ).fetchone() if existing_row is None: raise @@ -126,7 +145,13 @@ def create_claim( claim_id = int(cur.lastrowid) # Assign a human-readable ID. try: - human_id = self._allocate_human_id(conn, subject, text, claim_id) + human_id = self._allocate_human_id( + conn, + subject, + text, + claim_id, + tenant_id=normalized_tenant_id, + ) conn.execute( "UPDATE claims SET human_id = ? WHERE id = ?", (human_id, claim_id), diff --git a/memorymaster/stores/migrations/0009_tenant_local_claim_identities.py b/memorymaster/stores/migrations/0009_tenant_local_claim_identities.py new file mode 100644 index 00000000..df38a90d --- /dev/null +++ b/memorymaster/stores/migrations/0009_tenant_local_claim_identities.py @@ -0,0 +1,129 @@ +"""Make claim identity constraints tenant-local on both storage backends.""" +from __future__ import annotations + +VERSION = 9 +DESCRIPTION = "Tenant-local claim identities and confirmed tuples" + +_SQLITE_DDL = """ +DROP INDEX IF EXISTS idx_claims_idempotency_key; +DROP INDEX IF EXISTS idx_claims_human_id; +DROP INDEX IF EXISTS idx_claims_confirmed_tuple_unique; +CREATE INDEX IF NOT EXISTS idx_claims_idempotency_key ON claims(idempotency_key); +CREATE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id); +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_idempotency_key + ON claims(COALESCE(tenant_id, ''), idempotency_key) + WHERE idempotency_key IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_human_id + ON claims(COALESCE(tenant_id, ''), human_id) + WHERE human_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_confirmed_tuple_unique + ON claims(COALESCE(tenant_id, ''), subject, predicate, scope) + WHERE status = 'confirmed' + AND subject IS NOT NULL + AND predicate IS NOT NULL; +DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard_insert; +DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard_update; +CREATE TRIGGER trg_claims_confirmed_tuple_guard_insert +BEFORE INSERT ON claims +WHEN NEW.status = 'confirmed' + AND NEW.subject IS NOT NULL + AND NEW.predicate IS NOT NULL + AND EXISTS ( + SELECT 1 FROM claims c + WHERE c.status = 'confirmed' + AND c.subject = NEW.subject + AND c.predicate = NEW.predicate + AND c.scope = NEW.scope + AND c.tenant_id IS NEW.tenant_id + ) +BEGIN + SELECT RAISE(ABORT, 'only one confirmed claim is allowed per tenant and (subject,predicate,scope)'); +END; +CREATE TRIGGER trg_claims_confirmed_tuple_guard_update +BEFORE UPDATE OF status, subject, predicate, scope, tenant_id ON claims +WHEN NEW.status = 'confirmed' + AND NEW.subject IS NOT NULL + AND NEW.predicate IS NOT NULL + AND EXISTS ( + SELECT 1 FROM claims c + WHERE c.id <> OLD.id + AND c.status = 'confirmed' + AND c.subject = NEW.subject + AND c.predicate = NEW.predicate + AND c.scope = NEW.scope + AND c.tenant_id IS NEW.tenant_id + ) +BEGIN + SELECT RAISE(ABORT, 'only one confirmed claim is allowed per tenant and (subject,predicate,scope)'); +END; +""".strip() + +_POSTGRES_INDEX_DDL = """ +DROP INDEX IF EXISTS idx_claims_idempotency_key; +DROP INDEX IF EXISTS idx_claims_human_id; +DROP INDEX IF EXISTS idx_claims_confirmed_tuple_unique; +CREATE INDEX IF NOT EXISTS idx_claims_idempotency_key ON claims(idempotency_key); +CREATE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id); +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_idempotency_key + ON claims(COALESCE(tenant_id, ''), idempotency_key) + WHERE idempotency_key IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_human_id + ON claims(COALESCE(tenant_id, ''), human_id) + WHERE human_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_confirmed_tuple_unique + ON claims(COALESCE(tenant_id, ''), subject, predicate, scope) + WHERE status = 'confirmed' + AND subject IS NOT NULL + AND predicate IS NOT NULL; +""".strip() + +_POSTGRES_TRIGGER_DDL = """ +CREATE OR REPLACE FUNCTION memorymaster_claims_confirmed_tuple_guard() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.status = 'confirmed' + AND NEW.subject IS NOT NULL + AND NEW.predicate IS NOT NULL + AND EXISTS ( + SELECT 1 FROM claims c + WHERE c.status = 'confirmed' + AND c.subject = NEW.subject + AND c.predicate = NEW.predicate + AND c.scope = NEW.scope + AND c.tenant_id IS NOT DISTINCT FROM NEW.tenant_id + AND (TG_OP = 'INSERT' OR c.id <> NEW.id) + ) THEN + RAISE EXCEPTION 'only one confirmed claim is allowed per tenant and (subject,predicate,scope)' + USING ERRCODE = '23505'; + END IF; + RETURN NEW; +END; +$$; +DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard ON claims; +CREATE TRIGGER trg_claims_confirmed_tuple_guard +BEFORE INSERT OR UPDATE OF status, subject, predicate, scope, tenant_id ON claims +FOR EACH ROW +EXECUTE FUNCTION memorymaster_claims_confirmed_tuple_guard(); +""".strip() + + +def apply_sqlite(conn) -> None: + has_claims = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'claims'" + ).fetchone() + if has_claims is None: + return + columns = {row[1] for row in conn.execute("PRAGMA table_info(claims)")} + if "tenant_id" not in columns: + conn.execute("ALTER TABLE claims ADD COLUMN tenant_id TEXT") + conn.executescript(_SQLITE_DDL) + conn.commit() + + +def apply_postgres(conn) -> None: + with conn.cursor() as cur: + cur.execute(_POSTGRES_INDEX_DDL) + cur.execute(_POSTGRES_TRIGGER_DDL) + conn.commit() diff --git a/memorymaster/stores/postgres_store.py b/memorymaster/stores/postgres_store.py index 713a3cfd..3d03933c 100644 --- a/memorymaster/stores/postgres_store.py +++ b/memorymaster/stores/postgres_store.py @@ -54,6 +54,16 @@ def __init__( self._psycopg: Any = None self._vector_table_available: bool | None = None + def _tenant_for_operation(self, tenant_id: str | None = None) -> str | None: + requested = (tenant_id or "").strip() or None + if self.require_tenant: + if self.tenant_id is None: + raise PermissionError("Postgres team mode requires a tenant context.") + if requested is not None and requested != self.tenant_id: + raise PermissionError("Caller tenant does not match the bound tenant context.") + return self.tenant_id + return requested if requested is not None else self.tenant_id + def _load_psycopg(self) -> Any: if self._psycopg is None: try: @@ -170,6 +180,7 @@ def _ensure_event_integrity_schema(conn) -> None: @staticmethod def _ensure_confirmed_tuple_uniqueness_schema(conn) -> None: + PostgresStore._ensure_tenant_id_schema(conn) with conn.cursor() as cur: cur.execute( """ @@ -188,9 +199,10 @@ def _ensure_confirmed_tuple_uniqueness_schema(conn) -> None: AND c.subject = NEW.subject AND c.predicate = NEW.predicate AND c.scope = NEW.scope + AND c.tenant_id IS NOT DISTINCT FROM NEW.tenant_id AND (TG_OP = 'INSERT' OR c.id <> NEW.id) ) THEN - RAISE EXCEPTION 'only one confirmed claim is allowed per (subject,predicate,scope)' + RAISE EXCEPTION 'only one confirmed claim is allowed per tenant and (subject,predicate,scope)' USING ERRCODE = '23505'; END IF; RETURN NEW; @@ -198,23 +210,15 @@ def _ensure_confirmed_tuple_uniqueness_schema(conn) -> None: $$; """ ) + cur.execute( + f"DROP TRIGGER IF EXISTS {POSTGRES_CONFIRMED_TUPLE_GUARD_TRIGGER} ON claims" + ) cur.execute( f""" - DO $$ - BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM pg_trigger - WHERE tgname = '{POSTGRES_CONFIRMED_TUPLE_GUARD_TRIGGER}' - AND tgrelid = 'claims'::regclass - ) THEN - CREATE TRIGGER {POSTGRES_CONFIRMED_TUPLE_GUARD_TRIGGER} - BEFORE INSERT OR UPDATE OF status, subject, predicate, scope ON claims - FOR EACH ROW - EXECUTE FUNCTION memorymaster_claims_confirmed_tuple_guard(); - END IF; - END - $$; + CREATE TRIGGER {POSTGRES_CONFIRMED_TUPLE_GUARD_TRIGGER} + BEFORE INSERT OR UPDATE OF status, subject, predicate, scope, tenant_id ON claims + FOR EACH ROW + EXECUTE FUNCTION memorymaster_claims_confirmed_tuple_guard() """ ) PostgresStore._try_create_confirmed_tuple_unique_index(cur) @@ -227,7 +231,7 @@ def _try_create_confirmed_tuple_unique_index(cur) -> None: cur.execute( """ CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_confirmed_tuple_unique - ON claims(subject, predicate, scope) + ON claims(COALESCE(tenant_id, ''), subject, predicate, scope) WHERE status = 'confirmed' AND subject IS NOT NULL AND predicate IS NOT NULL @@ -430,7 +434,7 @@ def create_claim( if not citations: raise ValueError("At least one citation is required.") normalized_idempotency_key = (idempotency_key or "").strip() or None - normalized_tenant_id = (tenant_id or "").strip() or None + normalized_tenant_id = self._tenant_for_operation(tenant_id) now = utc_now() with self.connect() as conn, conn.cursor() as cur: cur.execute( @@ -444,7 +448,7 @@ def create_claim( %s, %s, NULL, %s, %s, %s, %s, %s, %s, 'candidate', %s, FALSE, NULL, NULL, %s, %s, NULL, NULL, %s, %s, %s, %s, %s, %s, %s ) - ON CONFLICT (idempotency_key) DO NOTHING + ON CONFLICT DO NOTHING RETURNING id """, ( @@ -473,8 +477,12 @@ def create_claim( if normalized_idempotency_key is None: raise RuntimeError("Failed to create claim.") cur.execute( - "SELECT id FROM claims WHERE idempotency_key = %s", - (normalized_idempotency_key,), + """ + SELECT id FROM claims + WHERE idempotency_key = %s + AND tenant_id IS NOT DISTINCT FROM %s + """, + (normalized_idempotency_key, normalized_tenant_id), ) existing_row = cur.fetchone() if existing_row is None: @@ -488,7 +496,13 @@ def create_claim( # Assign a human-readable ID. try: - human_id = self._allocate_human_id(cur, subject, text, claim_id) + human_id = self._allocate_human_id( + cur, + subject, + text, + claim_id, + tenant_id=normalized_tenant_id, + ) cur.execute( "UPDATE claims SET human_id = %s WHERE id = %s", (human_id, claim_id), @@ -537,14 +551,25 @@ def get_claim(self, claim_id: int, include_citations: bool = True) -> Claim | No claim.citations = self.list_citations(claim.id) return claim - def get_claim_by_idempotency_key(self, idempotency_key: str, include_citations: bool = True) -> Claim | None: + def get_claim_by_idempotency_key( + self, + idempotency_key: str, + include_citations: bool = True, + *, + tenant_id: str | None = None, + ) -> Claim | None: normalized_idempotency_key = idempotency_key.strip() if not normalized_idempotency_key: return None + effective_tenant = self._tenant_for_operation(tenant_id) with self.connect() as conn, conn.cursor() as cur: cur.execute( - "SELECT * FROM claims WHERE idempotency_key = %s", - (normalized_idempotency_key,), + """ + SELECT * FROM claims + WHERE idempotency_key = %s + AND tenant_id IS NOT DISTINCT FROM %s + """, + (normalized_idempotency_key, effective_tenant), ) row = cur.fetchone() if row is None: @@ -1020,12 +1045,15 @@ def find_confirmed_by_tuple( predicate: str | None, scope: str | None, exclude_claim_id: int | None = None, + tenant_id: str | None = None, ) -> list[Claim]: if not subject or not predicate: return [] clauses = ["status = 'confirmed'", "subject = %s", "predicate = %s", "scope = %s"] params: list[object] = [subject, predicate, scope or "project"] + clauses.append("tenant_id IS NOT DISTINCT FROM %s") + params.append(self._tenant_for_operation(tenant_id)) if exclude_claim_id is not None: clauses.append("id <> %s") params.append(exclude_claim_id) @@ -1657,10 +1685,35 @@ def _ensure_claim_links_schema(conn) -> None: @staticmethod def _ensure_human_id_schema(conn) -> None: """Add human_id column if missing and backfill existing claims.""" + PostgresStore._ensure_tenant_id_schema(conn) with conn.cursor() as cur: cur.execute("ALTER TABLE claims ADD COLUMN IF NOT EXISTS human_id TEXT") cur.execute( - "CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id)" + """ + DO $$ + BEGIN + IF EXISTS ( + SELECT 1 + FROM pg_class idx + JOIN pg_index meta ON meta.indexrelid = idx.oid + WHERE idx.relname = 'idx_claims_human_id' + AND meta.indisunique + ) THEN + DROP INDEX idx_claims_human_id; + END IF; + END + $$ + """ + ) + cur.execute( + "CREATE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id)" + ) + cur.execute( + """ + CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_human_id + ON claims(COALESCE(tenant_id, ''), human_id) + WHERE human_id IS NOT NULL + """ ) PostgresStore._backfill_human_ids(conn) @@ -1669,7 +1722,10 @@ def _backfill_human_ids(conn) -> int: """Assign human_id to all claims that lack one.""" with conn.cursor() as cur: cur.execute( - "SELECT id, subject, text FROM claims WHERE human_id IS NULL ORDER BY id ASC" + """ + SELECT id, subject, text, tenant_id + FROM claims WHERE human_id IS NULL ORDER BY id ASC + """ ) rows = cur.fetchall() if not rows: @@ -1679,7 +1735,13 @@ def _backfill_human_ids(conn) -> int: claim_id = int(row["id"]) subject = PostgresStore._as_text(row["subject"]) text = str(row["text"]) - human_id = PostgresStore._allocate_human_id(cur, subject, text, claim_id) + human_id = PostgresStore._allocate_human_id( + cur, + subject, + text, + claim_id, + tenant_id=row.get("tenant_id"), + ) cur.execute( "UPDATE claims SET human_id = %s WHERE id = %s", (human_id, claim_id), @@ -1688,7 +1750,13 @@ def _backfill_human_ids(conn) -> int: return updated @staticmethod - def _allocate_human_id(cur, subject: str | None, text: str, claim_id: int) -> str: + def _allocate_human_id( + cur, + subject: str | None, + text: str, + claim_id: int, + tenant_id: str | None = None, + ) -> str: """Build a unique human_id, checking for derived_from parent links.""" cur.execute( """ @@ -1698,17 +1766,22 @@ def _allocate_human_id(cur, subject: str | None, text: str, claim_id: int) -> st WHERE cl.source_id = %s AND cl.link_type = 'derived_from' AND c.human_id IS NOT NULL + AND c.tenant_id IS NOT DISTINCT FROM %s LIMIT 1 """, - (claim_id,), + (claim_id, tenant_id), ) parent_row = cur.fetchone() if parent_row and parent_row["human_id"]: parent_hid = str(parent_row["human_id"]) cur.execute( - "SELECT COUNT(*) AS cnt FROM claims WHERE human_id LIKE %s AND human_id != %s", - (parent_hid + ".%", parent_hid), + """ + SELECT COUNT(*) AS cnt FROM claims + WHERE human_id LIKE %s AND human_id != %s + AND tenant_id IS NOT DISTINCT FROM %s + """, + (parent_hid + ".%", parent_hid, tenant_id), ) child_count = cur.fetchone() next_child = (int(child_count["cnt"]) if child_count else 0) + 1 @@ -1719,7 +1792,13 @@ def _allocate_human_id(cur, subject: str | None, text: str, claim_id: int) -> st final = candidate suffix = 1 while True: - cur.execute("SELECT 1 FROM claims WHERE human_id = %s", (final,)) + cur.execute( + """ + SELECT 1 FROM claims + WHERE human_id = %s AND tenant_id IS NOT DISTINCT FROM %s + """, + (final, tenant_id), + ) existing = cur.fetchone() if existing is None: return final @@ -2557,16 +2636,27 @@ def list_action_proposals( rows = cur.fetchall() return [self._row_to_action_proposal(row) for row in rows] - def get_claim_by_human_id(self, human_id: str, include_citations: bool = True) -> Claim | None: + def get_claim_by_human_id( + self, + human_id: str, + include_citations: bool = True, + *, + tenant_id: str | None = None, + ) -> Claim | None: """Look up a claim by its human-readable ID (e.g. ``mm-a3f8``).""" normalized = human_id.strip() if not normalized: return None + effective_tenant = self._tenant_for_operation(tenant_id) with self.connect() as conn, conn.cursor() as cur: try: cur.execute( - "SELECT * FROM claims WHERE human_id = %s", - (normalized,), + """ + SELECT * FROM claims + WHERE human_id = %s + AND tenant_id IS NOT DISTINCT FROM %s + """, + (normalized, effective_tenant), ) row = cur.fetchone() except Exception: @@ -2579,7 +2669,12 @@ def get_claim_by_human_id(self, human_id: str, include_citations: bool = True) - claim.citations = self.list_citations(claim.id) return claim - def resolve_claim_id(self, identifier: str | int) -> int: + def resolve_claim_id( + self, + identifier: str | int, + *, + tenant_id: str | None = None, + ) -> int: """Resolve a numeric ID or human_id string to a numeric claim ID.""" if isinstance(identifier, int): return identifier @@ -2588,7 +2683,11 @@ def resolve_claim_id(self, identifier: str | int) -> int: return int(raw) except ValueError: pass - claim = self.get_claim_by_human_id(raw, include_citations=False) + claim = self.get_claim_by_human_id( + raw, + include_citations=False, + tenant_id=tenant_id, + ) if claim is not None: return claim.id raise ValueError(f"No claim found for identifier '{raw}'.") diff --git a/memorymaster/surfaces/cli_helpers.py b/memorymaster/surfaces/cli_helpers.py index a62a1af5..a4f9372c 100644 --- a/memorymaster/surfaces/cli_helpers.py +++ b/memorymaster/surfaces/cli_helpers.py @@ -72,7 +72,10 @@ def _resolve_claim_id(service: MemoryService, raw: str | int) -> int: try: return int(text) except ValueError: - return service.store.resolve_claim_id(text) + return service.store.resolve_claim_id( + text, + tenant_id=service.tenant_id, + ) def _add_cycle_policy_args(p: argparse.ArgumentParser, policy_default: str = "legacy") -> None: diff --git a/tests/test_postgres_tenant_identity_sql.py b/tests/test_postgres_tenant_identity_sql.py new file mode 100644 index 00000000..7474b8a4 --- /dev/null +++ b/tests/test_postgres_tenant_identity_sql.py @@ -0,0 +1,97 @@ +"""Hermetic PostgreSQL SQL checks for tenant-local identity operations.""" +from __future__ import annotations + +import inspect + +import pytest + +from memorymaster.core.models import CitationInput +from memorymaster.stores.postgres_store import PostgresStore + + +class RecordingCursor: + def __init__(self) -> None: + self.executed: list[tuple[str, object]] = [] + + def __enter__(self) -> "RecordingCursor": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, sql: str, params: object = ()) -> None: + self.executed.append((" ".join(sql.split()), params)) + + def fetchone(self): + return None + + def fetchall(self) -> list[object]: + return [] + + +class RecordingConnection: + def __init__(self) -> None: + self.cursor_instance = RecordingCursor() + + def __enter__(self) -> "RecordingConnection": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def cursor(self) -> RecordingCursor: + return self.cursor_instance + + +def _store(monkeypatch) -> tuple[PostgresStore, RecordingConnection]: + store = PostgresStore( + "postgresql://db.invalid/app", + tenant_id="tenant-a", + require_tenant=True, + ) + conn = RecordingConnection() + monkeypatch.setattr(store, "connect", lambda: conn) + return store, conn + + +def test_postgres_identity_reads_bind_the_store_tenant(monkeypatch) -> None: + store, conn = _store(monkeypatch) + + assert store.get_claim_by_idempotency_key("same-key") is None + assert store.get_claim_by_human_id("mm-abcd") is None + assert store.find_confirmed_by_tuple( + subject="subject", + predicate="uses", + scope="project:shared", + ) == [] + + emitted = "\n".join(sql for sql, _ in conn.cursor_instance.executed) + assert emitted.count("tenant_id IS NOT DISTINCT FROM %s") == 3 + assert all( + params and params[-1] == "tenant-a" + for _, params in conn.cursor_instance.executed + ) + + +def test_postgres_bound_tenant_cannot_be_overridden(monkeypatch) -> None: + store, conn = _store(monkeypatch) + + with pytest.raises(PermissionError, match="bound tenant"): + store.get_claim_by_idempotency_key("same-key", tenant_id="tenant-b") + with pytest.raises(PermissionError, match="bound tenant"): + store.create_claim( + "cross-tenant write", + [CitationInput(source="test")], + tenant_id="tenant-b", + ) + + assert conn.cursor_instance.executed == [] + + +def test_postgres_insert_conflict_fallback_is_tenant_qualified() -> None: + source = inspect.getsource(PostgresStore.create_claim) + + assert "ON CONFLICT DO NOTHING" in source + assert "ON CONFLICT (idempotency_key)" not in source + assert "tenant_id IS NOT DISTINCT FROM %s" in source + assert "normalized_tenant_id" in source diff --git a/tests/test_tenant_local_claim_identities.py b/tests/test_tenant_local_claim_identities.py new file mode 100644 index 00000000..e2b7d8f0 --- /dev/null +++ b/tests/test_tenant_local_claim_identities.py @@ -0,0 +1,233 @@ +"""Adversarial tests for tenant-local claim identity and tuple semantics.""" +from __future__ import annotations + +import sqlite3 + +import pytest + +from memorymaster.core.lifecycle import transition_claim +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.govern.jobs import validator + + +def _service(db_path, tenant_id: str) -> MemoryService: + return MemoryService(db_path, workspace_root=db_path.parent, tenant_id=tenant_id) + + +def _ingest( + service: MemoryService, + *, + text: str, + idempotency_key: str, + object_value: str, +): + return service.ingest( + text=text, + citations=[CitationInput(source="tenant-test", locator="fixture")], + idempotency_key=idempotency_key, + subject="shared-subject", + predicate="uses", + object_value=object_value, + scope="project:shared", + source_agent="tenant-identity-test", + ) + + +def test_same_idempotency_and_human_id_can_coexist_across_tenants(tmp_path) -> None: + db_path = tmp_path / "tenant-identities.db" + tenant_a = _service(db_path, "tenant-a") + tenant_a.init_db() + tenant_b = _service(db_path, "tenant-b") + + claim_a = _ingest( + tenant_a, + text="The shared service uses PostgreSQL.", + idempotency_key="shared-import-key", + object_value="postgres-a", + ) + claim_b = _ingest( + tenant_b, + text="The shared service uses PostgreSQL.", + idempotency_key="shared-import-key", + object_value="postgres-b", + ) + + assert claim_a.id != claim_b.id + assert claim_a.human_id == claim_b.human_id + assert claim_a.tenant_id == "tenant-a" + assert claim_b.tenant_id == "tenant-b" + + +def test_identity_lookups_are_tenant_qualified(tmp_path) -> None: + db_path = tmp_path / "tenant-lookups.db" + tenant_a = _service(db_path, "tenant-a") + tenant_a.init_db() + tenant_b = _service(db_path, "tenant-b") + claim_a = _ingest( + tenant_a, + text="Shared lookup identity.", + idempotency_key="same-key", + object_value="a", + ) + claim_b = _ingest( + tenant_b, + text="Shared lookup identity.", + idempotency_key="same-key", + object_value="b", + ) + + by_key_a = tenant_a.store.get_claim_by_idempotency_key( + "same-key", tenant_id="tenant-a" + ) + by_key_b = tenant_b.store.get_claim_by_idempotency_key( + "same-key", tenant_id="tenant-b" + ) + by_human_a = tenant_a.store.get_claim_by_human_id( + claim_a.human_id, tenant_id="tenant-a" + ) + by_human_b = tenant_b.store.get_claim_by_human_id( + claim_b.human_id, tenant_id="tenant-b" + ) + + assert by_key_a and by_key_a.id == claim_a.id + assert by_key_b and by_key_b.id == claim_b.id + assert by_human_a and by_human_a.id == claim_a.id + assert by_human_b and by_human_b.id == claim_b.id + assert ( + tenant_a.store.resolve_claim_id( + claim_a.human_id, + tenant_id="tenant-a", + ) + == claim_a.id + ) + assert ( + tenant_b.store.resolve_claim_id( + claim_b.human_id, + tenant_id="tenant-b", + ) + == claim_b.id + ) + + +def test_same_confirmed_tuple_can_coexist_across_tenants(tmp_path) -> None: + db_path = tmp_path / "tenant-tuples.db" + tenant_a = _service(db_path, "tenant-a") + tenant_a.init_db() + tenant_b = _service(db_path, "tenant-b") + claim_a = _ingest( + tenant_a, + text="Tenant A tuple.", + idempotency_key="tuple-a", + object_value="a", + ) + claim_b = _ingest( + tenant_b, + text="Tenant B tuple.", + idempotency_key="tuple-b", + object_value="b", + ) + + transition_claim( + tenant_a.store, + claim_a.id, + "confirmed", + reason="tenant-a-confirm", + event_type="validator", + ) + transition_claim( + tenant_b.store, + claim_b.id, + "confirmed", + reason="tenant-b-confirm", + event_type="validator", + ) + + with tenant_a.store.connect() as conn: + with pytest.raises(sqlite3.IntegrityError, match="only one confirmed claim"): + conn.execute( + "UPDATE claims SET tenant_id = ? WHERE id = ?", + ("tenant-a", claim_b.id), + ) + + with tenant_a.store.connect() as conn: + rows = conn.execute( + "SELECT tenant_id, status FROM claims ORDER BY tenant_id" + ).fetchall() + assert [(row["tenant_id"], row["status"]) for row in rows] == [ + ("tenant-a", "confirmed"), + ("tenant-b", "confirmed"), + ] + + +def test_duplicate_identity_and_tuple_still_fail_within_tenant(tmp_path) -> None: + db_path = tmp_path / "same-tenant.db" + service = _service(db_path, "tenant-a") + service.init_db() + first = _ingest( + service, + text="First identity.", + idempotency_key="tenant-key", + object_value="first", + ) + duplicate = _ingest( + service, + text="Different payload with the same key.", + idempotency_key="tenant-key", + object_value="duplicate", + ) + second_tuple = _ingest( + service, + text="Second tuple.", + idempotency_key="tuple-second", + object_value="second", + ) + + assert duplicate.id == first.id + transition_claim( + service.store, + first.id, + "confirmed", + reason="first-confirm", + event_type="validator", + ) + with pytest.raises(sqlite3.IntegrityError, match="only one confirmed claim"): + transition_claim( + service.store, + second_tuple.id, + "confirmed", + reason="duplicate-confirm", + event_type="validator", + ) + + +def test_validator_does_not_conflict_with_another_tenants_tuple(tmp_path) -> None: + db_path = tmp_path / "tenant-validator.db" + tenant_a = _service(db_path, "tenant-a") + tenant_a.init_db() + tenant_b = _service(db_path, "tenant-b") + claim_a = _ingest( + tenant_a, + text="Tenant A confirmed truth.", + idempotency_key="validator-a", + object_value="a", + ) + claim_b = _ingest( + tenant_b, + text="Tenant B independent truth.", + idempotency_key="validator-b", + object_value="b", + ) + transition_claim( + tenant_a.store, + claim_a.id, + "confirmed", + reason="fixture", + event_type="validator", + ) + + validator.run(tenant_b.store, min_citations=1, min_score=0.0) + + refreshed = tenant_b.store.get_claim(claim_b.id) + assert refreshed is not None + assert refreshed.status == "confirmed" diff --git a/tests/test_tenant_local_identity_migration.py b/tests/test_tenant_local_identity_migration.py new file mode 100644 index 00000000..c621e779 --- /dev/null +++ b/tests/test_tenant_local_identity_migration.py @@ -0,0 +1,174 @@ +"""Schema-contract tests for tenant-local claim identities.""" +from __future__ import annotations + +import importlib +import sqlite3 +from pathlib import Path + +from memorymaster.stores.migrations import discover_migrations +from memorymaster.stores._storage_schema import _SchemaMixin +from memorymaster.stores.storage import SQLiteStore + + +class RecordingCursor: + def __init__(self, statements: list[str]) -> None: + self.statements = statements + + def __enter__(self) -> "RecordingCursor": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, sql: str, _params: object = None) -> None: + self.statements.append(" ".join(sql.split())) + + +class RecordingConnection: + def __init__(self) -> None: + self.statements: list[str] = [] + self.commits = 0 + + def cursor(self) -> RecordingCursor: + return RecordingCursor(self.statements) + + def commit(self) -> None: + self.commits += 1 + + +def _migration(): + return importlib.import_module( + "memorymaster.stores.migrations.0009_tenant_local_claim_identities" + ) + + +def test_tenant_local_identity_migration_is_versioned() -> None: + migration = next(item for item in discover_migrations() if item.version == 9) + assert "tenant" in migration.description.lower() + assert "identit" in migration.description.lower() + + +def test_sqlite_migration_replaces_global_identity_constraints() -> None: + conn = sqlite3.connect(":memory:") + conn.executescript( + """ + CREATE TABLE claims ( + id INTEGER PRIMARY KEY, + idempotency_key TEXT, + human_id TEXT, + subject TEXT, + predicate TEXT, + scope TEXT, + status TEXT, + tenant_id TEXT + ); + CREATE UNIQUE INDEX idx_claims_idempotency_key ON claims(idempotency_key); + CREATE UNIQUE INDEX idx_claims_human_id ON claims(human_id); + CREATE UNIQUE INDEX idx_claims_confirmed_tuple_unique + ON claims(subject, predicate, scope) WHERE status = 'confirmed'; + """ + ) + try: + _migration().apply_sqlite(conn) + indexes = { + row[0]: row[1] + for row in conn.execute( + "SELECT name, sql FROM sqlite_master WHERE type = 'index'" + ).fetchall() + } + triggers = "\n".join( + row[0] + for row in conn.execute( + "SELECT sql FROM sqlite_master WHERE type = 'trigger'" + ).fetchall() + ) + finally: + conn.close() + + assert "idx_claims_tenant_idempotency_key" in indexes + assert "idx_claims_tenant_human_id" in indexes + assert "idx_claims_confirmed_tuple_unique" in indexes + assert "COALESCE(tenant_id, '')" in indexes["idx_claims_tenant_idempotency_key"] + assert "tenant_id IS NEW.tenant_id" in triggers + + +def test_postgres_migration_uses_null_safe_tenant_identity() -> None: + conn = RecordingConnection() + _migration().apply_postgres(conn) + emitted = "\n".join(conn.statements) + + assert "idx_claims_tenant_idempotency_key" in emitted + assert "idx_claims_tenant_human_id" in emitted + assert "idx_claims_confirmed_tuple_unique" in emitted + assert "c.tenant_id IS NOT DISTINCT FROM NEW.tenant_id" in emitted + assert "UPDATE OF status, subject, predicate, scope, tenant_id" in emitted + assert conn.commits == 1 + + +def test_bootstrap_schemas_declare_tenant_local_identity_indexes() -> None: + root = Path(__file__).resolve().parents[1] / "memorymaster" + sqlite_schema = (root / "schema.sql").read_text(encoding="utf-8") + postgres_schema = (root / "schema_postgres.sql").read_text(encoding="utf-8") + + for name in ( + "idx_claims_tenant_idempotency_key", + "idx_claims_tenant_human_id", + "idx_claims_confirmed_tuple_unique", + ): + assert name in sqlite_schema + assert name in postgres_schema + assert "ON claims(COALESCE(tenant_id, ''), subject, predicate, scope)" in sqlite_schema + assert "ON claims(COALESCE(tenant_id, ''), subject, predicate, scope)" in postgres_schema + + +def test_sqlite_reinit_does_not_restore_global_unique_indexes(tmp_path) -> None: + store = SQLiteStore(tmp_path / "reinit.db") + store.init_db() + store.init_db() + + with store.connect() as conn: + indexes = { + row[1]: bool(row[2]) + for row in conn.execute("PRAGMA index_list(claims)").fetchall() + } + + assert indexes["idx_claims_idempotency_key"] is False + assert indexes["idx_claims_human_id"] is False + assert indexes["idx_claims_tenant_idempotency_key"] is True + assert indexes["idx_claims_tenant_human_id"] is True + assert indexes["idx_claims_confirmed_tuple_unique"] is True + + +def test_legacy_human_id_index_is_converted_before_backfill() -> None: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.executescript( + """ + CREATE TABLE claims ( + id INTEGER PRIMARY KEY, + subject TEXT, + text TEXT NOT NULL, + human_id TEXT, + tenant_id TEXT + ); + CREATE TABLE claim_links ( + source_id INTEGER, + target_id INTEGER, + link_type TEXT + ); + CREATE UNIQUE INDEX idx_claims_human_id ON claims(human_id); + INSERT INTO claims(id, subject, text, tenant_id) VALUES + (1, 'same', 'same text', 'tenant-a'), + (2, 'same', 'same text', 'tenant-b'); + """ + ) + try: + updated = _SchemaMixin._ensure_human_id_schema(conn) + rows = conn.execute( + "SELECT tenant_id, human_id FROM claims ORDER BY tenant_id" + ).fetchall() + finally: + conn.close() + + assert updated is None + assert rows[0]["human_id"] == rows[1]["human_id"] From 93a2d594091e43c541e7094d38e9cb8ec9448acc Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 02:25:23 -0300 Subject: [PATCH 10/55] fix(postgres): tenantize the event ledger --- .planning/audit-remediation-ledger.md | 2 +- external-actions-required.md | 2 +- memorymaster/schema.sql | 14 + memorymaster/schema_postgres.sql | 16 +- memorymaster/stores/_storage_lifecycle.py | 70 +- memorymaster/stores/_storage_schema.py | 119 ++- memorymaster/stores/_storage_shared.py | 19 + .../migrations/0010_tenant_event_ledger.py | 361 +++++++++ memorymaster/stores/postgres_store.py | 530 +++++++++++-- tests/test_reliability_hardening.py | 6 +- tests/test_tenant_event_ledger.py | 730 ++++++++++++++++++ tests/test_tenant_event_runtime.py | 451 +++++++++++ 12 files changed, 2241 insertions(+), 79 deletions(-) create mode 100644 memorymaster/stores/migrations/0010_tenant_event_ledger.py create mode 100644 tests/test_tenant_event_ledger.py create mode 100644 tests/test_tenant_event_runtime.py diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 933f355f..7591f4a1 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -8,7 +8,7 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | ID | Primary domain | Severity / exploitability | Summary | Package | Status | Acceptance evidence | |---|---|---|---|---|---|---| -| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | R1.1 complete; Postgres team connections require tenant before driver load and set the tenant GUC on every connection; v0008 adds restrictive tenant policies to 8 claim-owned tables and fail-closed policies to 7 untenantable tables; v0009 makes idempotency keys, human IDs, and confirmed tuples tenant-local across SQLite/Postgres with tenant-qualified lookup/allocation/validation; privileged-role/scoped-GUC enforcement, tenant-aware event integrity/direct predicates, and the real two-role DSN matrix remain | +| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | R1.1 complete; Postgres team connections require tenant before driver load and set the tenant GUC on every connection; v0008 adds restrictive tenant policies to 8 claim-owned tables and fail-closed policies to 7 untenantable tables; v0009 makes idempotency keys, human IDs, and confirmed tuples tenant-local across SQLite/Postgres; v0010 is the checksum-frozen authority for a dual event ledger that preserves v1 bytes, validates source/prefix integrity, backfills tenant chains transactionally, supports tenant-bound claimless audit rows, and serializes Postgres appends per tenant; read-only reconciliation recomputes primary and tenant hashes without repair; full non-ML gate: 3,201 passed; privileged-role/scoped-GUC enforcement, remaining direct predicates, and the real two-role/concurrency matrix remain | | MM-SEC-02 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | Qdrant can return archived, sensitive, cross-scope/tenant, or orphan payloads | R1.3,R2.1 | OPEN | Fake and real Qdrant authoritative-filter tests | | MM-OPS-01 | DevOps | Critical / EXPLOITABLE-LOW-EFFORT / H4 | Postgres Compose publishes a fixed credential | R1.5 | OPEN | Fail-closed Compose config, private port, external rotation evidence | | MM-SEC-03 | Security | High / EXPLOITABLE-NOW | Persisted metadata/provenance fields bypass sensitivity scanning | R1.4 | OPEN | Complete field-matrix adversarial suite and legacy dry-run | diff --git a/external-actions-required.md b/external-actions-required.md index bec6e664..8f025289 100644 --- a/external-actions-required.md +++ b/external-actions-required.md @@ -5,7 +5,7 @@ Items here are `BLOCKED-EXTERNAL` only when the remediation ledger explicitly ma | Finding | Owner/system | Required action | Evidence needed to unblock | Review date | Status | |---|---|---|---|---|---| | MM-OPS-01 | Operator / any Postgres deployment | Rotate the historical `mm_pw` credential and recreate deployments that used it. Keep database ports private. | Rotation record plus network probe showing the port is not externally reachable | 2026-07-31 | PENDING-INVENTORY | -| MM-SEC-01 | Team Postgres test environment | Provide a disposable admin/migrator DSN in `MEMORYMASTER_TEST_POSTGRES_DSN` and a distinct non-owner `NOSUPERUSER NOBYPASSRLS` application DSN in `MEMORYMASTER_TEST_POSTGRES_APP_DSN`; neither may target product data. | Catalog assertions plus the real cross-tenant read/write/constraint matrix; app role has no table ownership, `public` CREATE, TRUNCATE, DDL, or migration rights | 2026-07-31 | BLOCKED-EXTERNAL | +| MM-SEC-01 | Team Postgres test environment | Provide a disposable admin/migrator DSN in `MEMORYMASTER_TEST_POSTGRES_DSN` and a distinct non-owner `NOSUPERUSER NOBYPASSRLS` application DSN in `MEMORYMASTER_TEST_POSTGRES_APP_DSN`; neither may target product data. | Catalog assertions plus the real cross-tenant read/write/constraint matrix; app role has no table ownership, `public` CREATE, TRUNCATE, DDL, or migration rights; verify claimless event visibility, non-UTC hash reconciliation, concurrent same-tenant append linearity, and writer-vs-migration lock/rollback behavior | 2026-07-31 | BLOCKED-EXTERNAL | | MM-OPS-02 | Docker/Helm runtime | Provide Docker and, for final verification, a disposable Kubernetes target if unavailable locally. | Container health/MCP handshake and Helm readiness/network-policy evidence | 2026-08-15 | PENDING-CAPABILITY-CHECK | | MM-SEC-02 | Qdrant runtime | Provide a disposable authenticated/TLS Qdrant target for final parity after fake-backed tests pass. | Real service policy/reconciliation test output | 2026-08-15 | PENDING-CAPABILITY-CHECK | | MM-DATA-01 | Live MemoryMaster operator | Approve a consistent backup/restore drill before any live migration, redaction, backlog, or retention operation. | Restored backup, integrity check, counts/checksums, approval record | 2026-08-15 | BLOCKED-EXTERNAL | diff --git a/memorymaster/schema.sql b/memorymaster/schema.sql index a485187b..802f1cd3 100644 --- a/memorymaster/schema.sql +++ b/memorymaster/schema.sql @@ -92,6 +92,13 @@ CREATE TABLE IF NOT EXISTS events ( details TEXT, payload_json TEXT, created_at TEXT NOT NULL, + prev_event_hash TEXT, + event_hash TEXT, + hash_algo TEXT, + tenant_id TEXT, + tenant_prev_event_hash TEXT, + tenant_event_hash TEXT, + tenant_hash_algo TEXT, FOREIGN KEY (claim_id) REFERENCES claims(id) ON DELETE CASCADE ); @@ -201,6 +208,13 @@ CREATE INDEX IF NOT EXISTS idx_claims_tuple ON claims(subject, predicate, scope) CREATE INDEX IF NOT EXISTS idx_claims_replaced_by ON claims(replaced_by_claim_id); CREATE INDEX IF NOT EXISTS idx_citations_claim_id ON citations(claim_id); CREATE INDEX IF NOT EXISTS idx_events_claim_id ON events(claim_id); +CREATE INDEX IF NOT EXISTS idx_events_tenant_id ON events(tenant_id); +CREATE INDEX IF NOT EXISTS idx_events_tenant_hash + ON events(tenant_id, tenant_event_hash); +CREATE INDEX IF NOT EXISTS idx_events_tenant_head + ON events(tenant_id, id DESC); +CREATE INDEX IF NOT EXISTS idx_events_tenant_algo_head + ON events(tenant_id, hash_algo, id DESC); CREATE INDEX IF NOT EXISTS idx_events_created_at ON events(created_at); CREATE INDEX IF NOT EXISTS idx_embeddings_updated_at ON claim_embeddings(updated_at); CREATE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id); diff --git a/memorymaster/schema_postgres.sql b/memorymaster/schema_postgres.sql index 6d4d96b6..812676ca 100644 --- a/memorymaster/schema_postgres.sql +++ b/memorymaster/schema_postgres.sql @@ -120,7 +120,14 @@ CREATE TABLE IF NOT EXISTS events ( to_status TEXT, details TEXT, payload_json JSONB, - created_at TIMESTAMPTZ NOT NULL + created_at TIMESTAMPTZ NOT NULL, + prev_event_hash TEXT, + event_hash TEXT, + hash_algo TEXT, + tenant_id TEXT, + tenant_prev_event_hash TEXT, + tenant_event_hash TEXT, + tenant_hash_algo TEXT ); CREATE OR REPLACE FUNCTION memorymaster_events_append_only_guard() @@ -184,6 +191,13 @@ CREATE INDEX IF NOT EXISTS idx_claims_tuple ON claims(subject, predicate, scope) CREATE INDEX IF NOT EXISTS idx_claims_replaced_by ON claims(replaced_by_claim_id); CREATE INDEX IF NOT EXISTS idx_citations_claim_id ON citations(claim_id); CREATE INDEX IF NOT EXISTS idx_events_claim_id ON events(claim_id); +CREATE INDEX IF NOT EXISTS idx_events_tenant_id ON events(tenant_id); +CREATE INDEX IF NOT EXISTS idx_events_tenant_hash + ON events(tenant_id, tenant_event_hash); +CREATE INDEX IF NOT EXISTS idx_events_tenant_head + ON events(tenant_id, id DESC); +CREATE INDEX IF NOT EXISTS idx_events_tenant_algo_head + ON events(tenant_id, hash_algo, id DESC); CREATE INDEX IF NOT EXISTS idx_events_created_at ON events(created_at); CREATE TABLE IF NOT EXISTS external_sources ( diff --git a/memorymaster/stores/_storage_lifecycle.py b/memorymaster/stores/_storage_lifecycle.py index 8e781a66..568afd4f 100644 --- a/memorymaster/stores/_storage_lifecycle.py +++ b/memorymaster/stores/_storage_lifecycle.py @@ -26,13 +26,56 @@ from memorymaster.stores._storage_shared import ( EVENT_HASH_ALGO, + TENANT_EVENT_HASH_ALGO, ConcurrentModificationError, + compute_tenant_event_hash, utc_now, ) logger = logging.getLogger(__name__) +def _tenant_event_chain_issues(rows, limit: int) -> list[dict[str, object]]: + issues: list[dict[str, object]] = [] + heads: dict[str, str | None] = {} + for row in rows: + tenant_id = str(row["tenant_id"]) if row["tenant_id"] is not None else None + if tenant_id is None: + continue + previous = ( + str(row["tenant_prev_event_hash"]) + if row["tenant_prev_event_hash"] is not None + else None + ) + stored_hash = ( + str(row["tenant_event_hash"]) + if row["tenant_event_hash"] is not None + else None + ) + expected_previous = heads.get(tenant_id) + event_hash = str(row["event_hash"]) if row["event_hash"] is not None else None + if previous != expected_previous: + issues.append({"event_id": int(row["id"]), "reason": "broken_tenant_prev_link"}) + if ( + row["tenant_hash_algo"] != TENANT_EVENT_HASH_ALGO + or stored_hash is None + or event_hash is None + ): + issues.append({"event_id": int(row["id"]), "reason": "missing_tenant_hash_material"}) + else: + expected_hash = compute_tenant_event_hash( + tenant_id=tenant_id, + event_hash=event_hash, + tenant_prev_event_hash=previous, + ) + if stored_hash != expected_hash: + issues.append({"event_id": int(row["id"]), "reason": "tenant_hash_mismatch"}) + heads[tenant_id] = stored_hash + if len(issues) >= limit: + break + return issues + + class _LifecycleMixin: if TYPE_CHECKING: def connect(self) -> sqlite3.Connection: ... @@ -259,7 +302,8 @@ def reconcile_integrity(self, *, fix: bool = False, limit: int = 500) -> dict[st "actions": [], } with self.connect() as conn: - self._ensure_event_integrity_schema(conn) + if fix: + self._ensure_event_integrity_schema(conn) orphan_events = conn.execute( """ @@ -361,7 +405,9 @@ def reconcile_integrity(self, *, fix: bool = False, limit: int = 500) -> dict[st chain_issues: list[dict[str, object]] = [] chain_rows = conn.execute( """ - SELECT id, prev_event_hash, event_hash, hash_algo + SELECT id, claim_id, event_type, from_status, to_status, details, + payload_json, created_at, prev_event_hash, event_hash, hash_algo, + tenant_id, tenant_prev_event_hash, tenant_event_hash, tenant_hash_algo FROM events ORDER BY id ASC """ @@ -378,6 +424,21 @@ def reconcile_integrity(self, *, fix: bool = False, limit: int = 500) -> dict[st chain_issues.append( {"event_id": int(row["id"]), "reason": "unexpected_hash_algo", "hash_algo": row_algo} ) + expected_hash = self._compute_event_hash( + claim_id=int(row["claim_id"]) if row["claim_id"] is not None else None, + event_type=str(row["event_type"]), + from_status=row["from_status"], + to_status=row["to_status"], + details=row["details"], + payload_json=row["payload_json"], + created_at=str(row["created_at"]), + prev_event_hash=row_prev, + hash_algo=row_algo or EVENT_HASH_ALGO, + ) + if row_hash != expected_hash: + chain_issues.append( + {"event_id": int(row["id"]), "reason": "event_hash_mismatch"} + ) if row_prev != expected_prev: chain_issues.append( { @@ -389,6 +450,8 @@ def reconcile_integrity(self, *, fix: bool = False, limit: int = 500) -> dict[st ) expected_prev = row_hash + tenant_chain_issues = _tenant_event_chain_issues(chain_rows, limit) + issues = { "orphan_events": [int(row["id"]) for row in orphan_events], "orphan_citations": [int(row["id"]) for row in orphan_citations], @@ -397,6 +460,7 @@ def reconcile_integrity(self, *, fix: bool = False, limit: int = 500) -> dict[st "dangling_supersedes": [int(row["id"]) for row in dangling_supersedes], "transition_issues": transition_issues[:limit], "hash_chain_issues": chain_issues[:limit], + "tenant_hash_chain_issues": tenant_chain_issues[:limit], } report["issues"] = issues report["summary"] = { @@ -432,7 +496,7 @@ def reconcile_integrity(self, *, fix: bool = False, limit: int = 500) -> dict[st issues["dangling_supersedes"], ) actions.append({"action": "clear_dangling_supersedes", "rows": int(cur.rowcount)}) - if issues["hash_chain_issues"]: + if issues["hash_chain_issues"] or issues["tenant_hash_chain_issues"]: actions.append( { "action": "skip_rebuild_event_hash_chain_append_only", diff --git a/memorymaster/stores/_storage_schema.py b/memorymaster/stores/_storage_schema.py index 3c39c0dc..27660ca4 100644 --- a/memorymaster/stores/_storage_schema.py +++ b/memorymaster/stores/_storage_schema.py @@ -15,8 +15,10 @@ from memorymaster.stores._storage_shared import ( EVENT_HASH_ALGO, + TENANT_EVENT_HASH_ALGO, SQLITE_CONFIRMED_TUPLE_GUARD_TRIGGERS, SQLITE_EVENTS_APPEND_ONLY_TRIGGERS, + compute_tenant_event_hash, generate_top_level_human_id, ) @@ -320,21 +322,24 @@ def _drop_events_append_only_triggers(conn: sqlite3.Connection) -> None: @staticmethod def _ensure_events_append_only_triggers(conn: sqlite3.Connection) -> None: - conn.executescript( + statements = ( """ CREATE TRIGGER IF NOT EXISTS trg_events_append_only_update BEFORE UPDATE ON events BEGIN SELECT RAISE(ABORT, 'events table is append-only; UPDATE is not allowed'); - END; - + END + """, + """ CREATE TRIGGER IF NOT EXISTS trg_events_append_only_delete BEFORE DELETE ON events BEGIN SELECT RAISE(ABORT, 'events table is append-only; DELETE is not allowed'); - END; - """ + END + """, ) + for statement in statements: + conn.execute(statement) @staticmethod @@ -856,6 +861,48 @@ def _backfill_event_chain(conn: sqlite3.Connection, *, rebuild_all: bool = False return updated + @staticmethod + def _backfill_tenant_event_chain( + conn: sqlite3.Connection, + *, + rebuild_all: bool = False, + ) -> int: + rows = conn.execute( + """ + SELECT id, tenant_id, event_hash, tenant_event_hash + FROM events ORDER BY id ASC + """ + ).fetchall() + heads: dict[str, str | None] = {} + updated = 0 + for row in rows: + if row["tenant_id"] is None: + continue + tenant_id = str(row["tenant_id"]) + if row["tenant_event_hash"] and not rebuild_all: + heads[tenant_id] = str(row["tenant_event_hash"]) + continue + if row["event_hash"] is None: + raise RuntimeError("Cannot build tenant event chain before global hashes exist.") + previous = heads.get(tenant_id) + tenant_hash = compute_tenant_event_hash( + tenant_id=tenant_id, + event_hash=str(row["event_hash"]), + tenant_prev_event_hash=previous, + ) + conn.execute( + """ + UPDATE events + SET tenant_prev_event_hash = ?, tenant_event_hash = ?, tenant_hash_algo = ? + WHERE id = ? + """, + (previous, tenant_hash, TENANT_EVENT_HASH_ALGO, int(row["id"])), + ) + heads[tenant_id] = tenant_hash + updated += 1 + return updated + + @staticmethod def _insert_event_row( conn: sqlite3.Connection, @@ -868,6 +915,17 @@ def _insert_event_row( payload_json: str | None, created_at: str, ) -> int: + tenant_id: str | None = None + if claim_id is not None: + try: + claim_row = conn.execute( + "SELECT tenant_id FROM claims WHERE id = ?", + (claim_id,), + ).fetchone() + except sqlite3.OperationalError: + claim_row = None + if claim_row is not None: + tenant_id = claim_row["tenant_id"] try: prev_row = conn.execute( "SELECT event_hash FROM events WHERE event_hash IS NOT NULL ORDER BY id DESC LIMIT 1" @@ -886,14 +944,38 @@ def _insert_event_row( prev_event_hash=prev_event_hash, hash_algo=EVENT_HASH_ALGO, ) + tenant_prev_event_hash: str | None = None + tenant_event_hash: str | None = None + tenant_hash_algo: str | None = None + if tenant_id is not None: + try: + tenant_prev_row = conn.execute( + """ + SELECT tenant_event_hash FROM events + WHERE tenant_id IS ? AND tenant_event_hash IS NOT NULL + ORDER BY id DESC LIMIT 1 + """, + (tenant_id,), + ).fetchone() + except sqlite3.OperationalError: + tenant_prev_row = None + if tenant_prev_row is not None: + tenant_prev_event_hash = tenant_prev_row["tenant_event_hash"] + tenant_event_hash = compute_tenant_event_hash( + tenant_id=str(tenant_id), + event_hash=event_hash, + tenant_prev_event_hash=tenant_prev_event_hash, + ) + tenant_hash_algo = TENANT_EVENT_HASH_ALGO try: cur = conn.execute( """ INSERT INTO events ( claim_id, event_type, from_status, to_status, details, payload_json, created_at, - prev_event_hash, event_hash, hash_algo + prev_event_hash, event_hash, hash_algo, tenant_id, + tenant_prev_event_hash, tenant_event_hash, tenant_hash_algo ) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( claim_id, @@ -906,6 +988,10 @@ def _insert_event_row( prev_event_hash, event_hash, EVENT_HASH_ALGO, + tenant_id, + tenant_prev_event_hash, + tenant_event_hash, + tenant_hash_algo, ), ) return int(cur.lastrowid) @@ -922,15 +1008,28 @@ def _insert_event_row( from_status TEXT, to_status TEXT, details TEXT, payload_json TEXT, created_at TEXT NOT NULL, - prev_event_hash TEXT, event_hash TEXT, hash_algo TEXT + prev_event_hash TEXT, event_hash TEXT, hash_algo TEXT, + tenant_id TEXT, tenant_prev_event_hash TEXT, + tenant_event_hash TEXT, tenant_hash_algo TEXT ) """) cur = conn.execute( """ - INSERT INTO events (claim_id, event_type, from_status, to_status, details, payload_json, created_at) + INSERT INTO events ( + claim_id, event_type, from_status, to_status, details, + payload_json, created_at + ) VALUES (?, ?, ?, ?, ?, ?, ?) """, - (claim_id, event_type, from_status, to_status, details, payload_json, created_at), + ( + claim_id, + event_type, + from_status, + to_status, + details, + payload_json, + created_at, + ), ) return int(cur.lastrowid) diff --git a/memorymaster/stores/_storage_shared.py b/memorymaster/stores/_storage_shared.py index 2bfc35d4..5ecc946f 100644 --- a/memorymaster/stores/_storage_shared.py +++ b/memorymaster/stores/_storage_shared.py @@ -19,6 +19,7 @@ HUMAN_ID_PREFIX = "mm" EVENT_HASH_ALGO = "sha256-v1" +TENANT_EVENT_HASH_ALGO = "sha256-tenant-v2" SQLITE_EVENTS_APPEND_ONLY_TRIGGERS = ( "trg_events_append_only_update", @@ -30,6 +31,24 @@ ) +def compute_tenant_event_hash( + *, + tenant_id: str, + event_hash: str, + tenant_prev_event_hash: str | None, +) -> str: + """Commit one global event hash into a tenant-partitioned hash chain.""" + material = "\x1f".join( + ( + TENANT_EVENT_HASH_ALGO, + tenant_id, + event_hash, + tenant_prev_event_hash or "", + ) + ) + return hashlib.sha256(material.encode("utf-8")).hexdigest() + + def generate_human_id_hash(text: str) -> str: """Generate a 4-hex-char hash from text for human-readable IDs.""" digest = hashlib.sha256(text.encode("utf-8")).hexdigest() diff --git a/memorymaster/stores/migrations/0010_tenant_event_ledger.py b/memorymaster/stores/migrations/0010_tenant_event_ledger.py new file mode 100644 index 00000000..05093dfd --- /dev/null +++ b/memorymaster/stores/migrations/0010_tenant_event_ledger.py @@ -0,0 +1,361 @@ +"""Add tenant-partitioned integrity metadata to the append-only event ledger.""" +from __future__ import annotations + +import hashlib +import json +from datetime import datetime, timezone + +VERSION = 10 +DESCRIPTION = "Tenant ownership for the append-only event ledger" + +# These algorithms are deliberately frozen in this checksummed migration. +# Runtime helpers have golden-vector tests against these implementations. +_EVENT_HASH_ALGO = "sha256-v1" +_TENANT_EVENT_HASH_ALGO = "sha256-tenant-v2" + +_EVENT_ROWS_SQL = """ +SELECT id, claim_id, event_type, from_status, to_status, details, + payload_json, created_at, prev_event_hash, event_hash, hash_algo, + tenant_id, tenant_prev_event_hash, tenant_event_hash, tenant_hash_algo +FROM events ORDER BY id ASC +""".strip() + +_SQLITE_APPEND_TRIGGER_STATEMENTS = ( + """ + CREATE TRIGGER IF NOT EXISTS trg_events_append_only_update + BEFORE UPDATE ON events + BEGIN + SELECT RAISE(ABORT, 'events table is append-only; UPDATE is not allowed'); + END + """.strip(), + """ + CREATE TRIGGER IF NOT EXISTS trg_events_append_only_delete + BEFORE DELETE ON events + BEGIN + SELECT RAISE(ABORT, 'events table is append-only; DELETE is not allowed'); + END + """.strip(), +) + +_POSTGRES_PREPARE_DDL = """ +LOCK TABLE events IN ACCESS EXCLUSIVE MODE; +ALTER TABLE events ADD COLUMN IF NOT EXISTS tenant_id TEXT; +ALTER TABLE events ADD COLUMN IF NOT EXISTS tenant_prev_event_hash TEXT; +ALTER TABLE events ADD COLUMN IF NOT EXISTS tenant_event_hash TEXT; +ALTER TABLE events ADD COLUMN IF NOT EXISTS tenant_hash_algo TEXT; +DROP TRIGGER IF EXISTS trg_events_append_only_update ON events; +DROP TRIGGER IF EXISTS trg_events_append_only_delete ON events; +CREATE INDEX IF NOT EXISTS idx_events_tenant_id ON events(tenant_id); +CREATE INDEX IF NOT EXISTS idx_events_tenant_hash + ON events(tenant_id, tenant_event_hash); +CREATE INDEX IF NOT EXISTS idx_events_tenant_head + ON events(tenant_id, id DESC); +CREATE INDEX IF NOT EXISTS idx_events_tenant_algo_head + ON events(tenant_id, hash_algo, id DESC); +""".strip() + +_POSTGRES_FINALIZE_DDL = """ +CREATE OR REPLACE FUNCTION memorymaster_events_append_only_guard() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +BEGIN + RAISE EXCEPTION 'events table is append-only; % is not allowed', TG_OP; +END; +$$; +CREATE TRIGGER trg_events_append_only_update +BEFORE UPDATE ON events +FOR EACH ROW EXECUTE FUNCTION memorymaster_events_append_only_guard(); +CREATE TRIGGER trg_events_append_only_delete +BEFORE DELETE ON events +FOR EACH ROW EXECUTE FUNCTION memorymaster_events_append_only_guard(); +ALTER TABLE events ENABLE ROW LEVEL SECURITY; +DROP POLICY IF EXISTS memorymaster_tenant_restrict ON events; +CREATE POLICY memorymaster_tenant_restrict ON events +AS RESTRICTIVE FOR ALL TO PUBLIC +USING ( + events.tenant_id = NULLIF(current_setting('memorymaster.tenant_id', true), '') + AND ( + events.claim_id IS NULL + OR EXISTS ( + SELECT 1 FROM claims AS mm_claim + WHERE mm_claim.id = events.claim_id + AND mm_claim.tenant_id = NULLIF(current_setting('memorymaster.tenant_id', true), '') + ) + ) +) +WITH CHECK ( + events.tenant_id = NULLIF(current_setting('memorymaster.tenant_id', true), '') + AND ( + events.claim_id IS NULL + OR EXISTS ( + SELECT 1 FROM claims AS mm_claim + WHERE mm_claim.id = events.claim_id + AND mm_claim.tenant_id = NULLIF(current_setting('memorymaster.tenant_id', true), '') + ) + ) +); +""".strip() + + +def _row_value(row, key: str, index: int): + return row.get(key) if isinstance(row, dict) else row[index] + + +def _text(value) -> str | None: + return None if value is None else str(value) + + +def _canonical_payload(payload: object | None) -> str: + if payload is None: + return "" + if isinstance(payload, str): + raw = payload.strip() + if not raw: + return "" + try: + payload = json.loads(raw) + except json.JSONDecodeError: + return raw + return json.dumps(payload, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def _canonical_created_at(value: object, *, normalize_utc: bool = True) -> str: + if not isinstance(value, datetime): + return str(value) + normalized = value + if normalize_utc and value.tzinfo is not None: + normalized = value.astimezone(timezone.utc) + return normalized.replace(microsecond=0).isoformat() + + +def _compute_primary_event_hash( + row, + *, + hash_algo: str, + previous: str | None, + normalize_utc: bool = True, +) -> str: + tenant_id = _text(_row_value(row, "tenant_id", 11)) + components = [hash_algo] + if hash_algo == _TENANT_EVENT_HASH_ALGO: + if tenant_id is None: + raise RuntimeError("Invalid primary event chain: tenant-v2 event has no tenant.") + components.append(tenant_id) + claim_id = _row_value(row, "claim_id", 1) + components.extend( + [ + str(claim_id) if claim_id is not None else "", + str(_row_value(row, "event_type", 2)), + _text(_row_value(row, "from_status", 3)) or "", + _text(_row_value(row, "to_status", 4)) or "", + _text(_row_value(row, "details", 5)) or "", + _canonical_payload(_row_value(row, "payload_json", 6)), + _canonical_created_at( + _row_value(row, "created_at", 7), + normalize_utc=normalize_utc, + ), + previous or "", + ] + ) + return hashlib.sha256("\x1f".join(components).encode("utf-8")).hexdigest() + + +def _compute_tenant_event_hash_v2( + *, + tenant_id: str, + event_hash: str, + tenant_prev_event_hash: str | None, +) -> str: + material = "\x1f".join( + ( + _TENANT_EVENT_HASH_ALGO, + tenant_id, + event_hash, + tenant_prev_event_hash or "", + ) + ) + return hashlib.sha256(material.encode("utf-8")).hexdigest() + + +def _validate_primary_chain(rows) -> None: + global_head: str | None = None + tenant_heads: dict[str, str | None] = {} + for row in rows: + event_id = int(_row_value(row, "id", 0)) + tenant_id = _text(_row_value(row, "tenant_id", 11)) + hash_algo = _text(_row_value(row, "hash_algo", 10)) or _EVENT_HASH_ALGO + if hash_algo == _EVENT_HASH_ALGO: + previous = global_head + elif hash_algo == _TENANT_EVENT_HASH_ALGO and tenant_id is not None: + previous = tenant_heads.get(tenant_id) + else: + raise RuntimeError(f"Invalid primary event chain algorithm at event {event_id}.") + stored_previous = _text(_row_value(row, "prev_event_hash", 8)) + stored_hash = _text(_row_value(row, "event_hash", 9)) + expected_hashes = { + _compute_primary_event_hash( + row, + hash_algo=hash_algo, + previous=previous, + ) + } + if hash_algo == _EVENT_HASH_ALGO: + expected_hashes.add( + _compute_primary_event_hash( + row, + hash_algo=hash_algo, + previous=previous, + normalize_utc=False, + ) + ) + if stored_previous != previous or stored_hash not in expected_hashes: + raise RuntimeError(f"Invalid primary event chain at event {event_id}.") + if hash_algo == _EVENT_HASH_ALGO: + global_head = stored_hash + else: + tenant_heads[tenant_id] = stored_hash + + +def _tenant_hash_updates(rows) -> list[tuple[str | None, str, str, int]]: + heads: dict[str, str | None] = {} + updates: list[tuple[str | None, str, str, int]] = [] + for row in rows: + tenant_id = _text(_row_value(row, "tenant_id", 11)) + if tenant_id is None: + continue + event_id = int(_row_value(row, "id", 0)) + event_hash = _text(_row_value(row, "event_hash", 9)) + if event_hash is None: + raise RuntimeError("Cannot build tenant event chain before primary hashes exist.") + previous = heads.get(tenant_id) + expected_hash = _compute_tenant_event_hash_v2( + tenant_id=tenant_id, + event_hash=event_hash, + tenant_prev_event_hash=previous, + ) + stored_previous = _text(_row_value(row, "tenant_prev_event_hash", 12)) + stored_hash = _text(_row_value(row, "tenant_event_hash", 13)) + stored_algo = _text(_row_value(row, "tenant_hash_algo", 14)) + if stored_hash is not None: + if ( + stored_previous != previous + or stored_hash != expected_hash + or stored_algo != _TENANT_EVENT_HASH_ALGO + ): + raise RuntimeError(f"Invalid tenant event chain at event {event_id}.") + else: + if stored_previous is not None or stored_algo is not None: + raise RuntimeError(f"Invalid tenant event chain prefix at event {event_id}.") + updates.append((previous, expected_hash, _TENANT_EVENT_HASH_ALGO, event_id)) + heads[tenant_id] = expected_hash + return updates + + +def _add_sqlite_columns(conn) -> None: + columns = {row[1] for row in conn.execute("PRAGMA table_info(events)")} + for column in ( + "tenant_id", + "tenant_prev_event_hash", + "tenant_event_hash", + "tenant_hash_algo", + ): + if column not in columns: + conn.execute(f"ALTER TABLE events ADD COLUMN {column} TEXT") + + +def _backfill_sqlite(conn) -> None: + rows = conn.execute(_EVENT_ROWS_SQL).fetchall() + _validate_primary_chain(rows) + has_claims = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'claims'" + ).fetchone() + if has_claims is not None: + conn.execute( + """ + UPDATE events + SET tenant_id = ( + SELECT claims.tenant_id FROM claims WHERE claims.id = events.claim_id + ) + WHERE claim_id IS NOT NULL AND tenant_id IS NULL + """ + ) + rows = conn.execute(_EVENT_ROWS_SQL).fetchall() + updates = _tenant_hash_updates(rows) + if updates: + conn.executemany( + """ + UPDATE events + SET tenant_prev_event_hash = ?, tenant_event_hash = ?, tenant_hash_algo = ? + WHERE id = ? + """, + updates, + ) + + +def apply_sqlite(conn) -> None: + has_events = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'events'" + ).fetchone() + if has_events is None: + return + conn.execute("BEGIN IMMEDIATE") + try: + _add_sqlite_columns(conn) + conn.execute("DROP TRIGGER IF EXISTS trg_events_append_only_update") + conn.execute("DROP TRIGGER IF EXISTS trg_events_append_only_delete") + _backfill_sqlite(conn) + conn.execute("CREATE INDEX IF NOT EXISTS idx_events_tenant_id ON events(tenant_id)") + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_events_tenant_hash ON events(tenant_id, tenant_event_hash)" + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_events_tenant_head ON events(tenant_id, id DESC)" + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_events_tenant_algo_head " + "ON events(tenant_id, hash_algo, id DESC)" + ) + for statement in _SQLITE_APPEND_TRIGGER_STATEMENTS: + conn.execute(statement) + except Exception: + conn.rollback() + raise + conn.commit() + + +def apply_postgres(conn) -> None: + try: + with conn.cursor() as cur: + cur.execute(_POSTGRES_PREPARE_DDL) + cur.execute(_EVENT_ROWS_SQL) + _validate_primary_chain(cur.fetchall()) + cur.execute( + """ + UPDATE events AS event + SET tenant_id = claim.tenant_id + FROM claims AS claim + WHERE event.claim_id = claim.id + AND event.tenant_id IS NULL + """ + ) + cur.execute(_EVENT_ROWS_SQL) + updates = _tenant_hash_updates(cur.fetchall()) + if updates: + cur.executemany( + """ + UPDATE events + SET tenant_prev_event_hash = %s, + tenant_event_hash = %s, + tenant_hash_algo = %s + WHERE id = %s + """, + updates, + ) + cur.execute(_POSTGRES_FINALIZE_DDL) + conn.commit() + except Exception: + rollback = getattr(conn, "rollback", None) + if callable(rollback): + rollback() + raise diff --git a/memorymaster/stores/postgres_store.py b/memorymaster/stores/postgres_store.py index 3d03933c..624b09b0 100644 --- a/memorymaster/stores/postgres_store.py +++ b/memorymaster/stores/postgres_store.py @@ -26,7 +26,12 @@ validate_transition_event_type, ) from memorymaster.core.retry import connect_with_retry -from memorymaster.stores._storage_shared import EVENT_HASH_ALGO, generate_top_level_human_id +from memorymaster.stores._storage_shared import ( + EVENT_HASH_ALGO, + TENANT_EVENT_HASH_ALGO, + compute_tenant_event_hash, + generate_top_level_human_id, +) from memorymaster.stores.storage import SQLiteStore POSTGRES_EVENTS_APPEND_ONLY_TRIGGERS = ( @@ -34,6 +39,7 @@ "trg_events_append_only_delete", ) POSTGRES_CONFIRMED_TUPLE_GUARD_TRIGGER = "trg_claims_confirmed_tuple_guard" +POSTGRES_TENANT_EVENT_HASH_ALGO = TENANT_EVENT_HASH_ALGO def utc_now() -> datetime: @@ -149,11 +155,20 @@ def _compute_event_hash( payload: object | None, created_at: datetime, prev_event_hash: str | None, + tenant_id: str | None = None, hash_algo: str = EVENT_HASH_ALGO, + canonicalize_timestamp: bool = True, ) -> str: - created_iso = created_at.replace(microsecond=0).isoformat() - components = [ - hash_algo, + normalized_created_at = created_at + if canonicalize_timestamp and created_at.tzinfo is not None: + normalized_created_at = created_at.astimezone(timezone.utc) + created_iso = normalized_created_at.replace(microsecond=0).isoformat() + components = [hash_algo] + if hash_algo == POSTGRES_TENANT_EVENT_HASH_ALGO: + if tenant_id is None: + raise ValueError("Tenant event hashes require tenant_id.") + components.append(tenant_id) + components.extend([ str(claim_id) if claim_id is not None else "", event_type, from_status or "", @@ -162,10 +177,23 @@ def _compute_event_hash( PostgresStore._canonical_payload(payload), created_iso, prev_event_hash or "", - ] + ]) material = "\x1f".join(components) return hashlib.sha256(material.encode("utf-8")).hexdigest() + @staticmethod + def _compute_tenant_event_hash( + *, + tenant_id: str, + event_hash: str, + tenant_prev_event_hash: str | None, + ) -> str: + return compute_tenant_event_hash( + tenant_id=tenant_id, + event_hash=event_hash, + tenant_prev_event_hash=tenant_prev_event_hash, + ) + @staticmethod def _ensure_event_integrity_schema(conn) -> None: with conn.cursor() as cur: @@ -308,53 +336,202 @@ def _ensure_events_append_only_rules(cur) -> None: """ ) + @staticmethod + def _primary_event_partition( + row: dict[str, object], + global_head: str | None, + tenant_heads: dict[str, str | None], + ) -> tuple[str, str | None, str | None]: + hash_algo = PostgresStore._as_text(row.get("hash_algo")) or EVENT_HASH_ALGO + tenant_id = PostgresStore._as_text(row.get("tenant_id")) + if hash_algo == EVENT_HASH_ALGO: + return hash_algo, tenant_id, global_head + if hash_algo == POSTGRES_TENANT_EVENT_HASH_ALGO and tenant_id is not None: + return hash_algo, tenant_id, tenant_heads.get(tenant_id) + raise RuntimeError(f"Invalid primary event partition at event {row['id']}.") + + @staticmethod + def _hash_primary_event_row( + row: dict[str, object], + *, + previous: str | None, + tenant_id: str | None, + hash_algo: str, + ) -> str: + created_at = row["created_at"] + if not isinstance(created_at, datetime): + created_at = datetime.fromisoformat(str(created_at)) + return PostgresStore._compute_event_hash( + claim_id=int(row["claim_id"]) if row["claim_id"] is not None else None, + event_type=str(row["event_type"]), + from_status=PostgresStore._as_text(row["from_status"]), + to_status=PostgresStore._as_text(row["to_status"]), + details=PostgresStore._as_text(row["details"]), + payload=row.get("payload_json"), + created_at=created_at, + prev_event_hash=previous, + tenant_id=tenant_id, + hash_algo=hash_algo, + ) + @staticmethod def _backfill_event_chain(conn, *, rebuild_all: bool = False) -> int: with conn.cursor() as cur: cur.execute( """ - SELECT id, claim_id, event_type, from_status, to_status, details, payload_json, created_at, event_hash, hash_algo - FROM events + SELECT event.id, event.claim_id, event.event_type, event.from_status, + event.to_status, event.details, event.payload_json, + event.created_at, event.prev_event_hash, event.event_hash, + event.hash_algo, to_jsonb(event)->>'tenant_id' AS tenant_id + FROM events AS event ORDER BY id ASC """ ) rows = cur.fetchall() if not rows: return 0 + if rebuild_all and any( + PostgresStore._as_text(row.get("hash_algo")) == POSTGRES_TENANT_EVENT_HASH_ALGO + for row in rows + ): + raise RuntimeError("Cannot rebuild a mixed v1/tenant-v2 primary event ledger.") updated = 0 - prev_hash: str | None = None + global_head: str | None = None + tenant_heads: dict[str, str | None] = {} for row in rows: - row_hash = row.get("event_hash") - row_algo = row.get("hash_algo") + row_hash = PostgresStore._as_text(row.get("event_hash")) + row_algo, tenant_id, previous = PostgresStore._primary_event_partition( + row, + global_head, + tenant_heads, + ) if row_hash and not rebuild_all: - prev_hash = str(row_hash) - continue + if PostgresStore._as_text(row.get("prev_event_hash")) != previous: + raise RuntimeError(f"Invalid primary event predecessor at event {row['id']}.") + event_hash = row_hash + else: + event_hash = PostgresStore._hash_primary_event_row( + row, + previous=previous, + tenant_id=tenant_id, + hash_algo=row_algo, + ) + cur.execute( + "UPDATE events SET prev_event_hash = %s, event_hash = %s, hash_algo = %s WHERE id = %s", + (previous, event_hash, row_algo, int(row["id"])), + ) + updated += 1 + if row_algo == EVENT_HASH_ALGO: + global_head = event_hash + else: + tenant_heads[tenant_id] = event_hash + return updated - algo = str(row_algo) if row_algo else EVENT_HASH_ALGO - payload = row.get("payload_json") - created_at = row["created_at"] - if not isinstance(created_at, datetime): - created_at = datetime.fromisoformat(str(created_at)) - event_hash = PostgresStore._compute_event_hash( - claim_id=int(row["claim_id"]) if row["claim_id"] is not None else None, - event_type=str(row["event_type"]), - from_status=PostgresStore._as_text(row["from_status"]), - to_status=PostgresStore._as_text(row["to_status"]), - details=PostgresStore._as_text(row["details"]), - payload=payload, - created_at=created_at, - prev_event_hash=prev_hash, - hash_algo=algo, + @staticmethod + def _backfill_tenant_event_chain(conn, *, rebuild_all: bool = False) -> int: + with conn.cursor() as cur: + cur.execute( + """ + SELECT id, tenant_id, event_hash, tenant_event_hash + FROM events ORDER BY id ASC + """ + ) + heads: dict[str, str | None] = {} + updated = 0 + for row in cur.fetchall(): + tenant_id = PostgresStore._as_text(row.get("tenant_id")) + if tenant_id is None: + continue + existing_hash = PostgresStore._as_text(row.get("tenant_event_hash")) + if existing_hash and not rebuild_all: + heads[tenant_id] = existing_hash + continue + event_hash = PostgresStore._as_text(row.get("event_hash")) + if event_hash is None: + raise RuntimeError("Cannot build tenant event chain before global hashes exist.") + previous = heads.get(tenant_id) + tenant_hash = compute_tenant_event_hash( + tenant_id=tenant_id, + event_hash=event_hash, + tenant_prev_event_hash=previous, ) cur.execute( - "UPDATE events SET prev_event_hash = %s, event_hash = %s, hash_algo = %s WHERE id = %s", - (prev_hash, event_hash, algo, int(row["id"])), + """ + UPDATE events SET tenant_prev_event_hash = %s, + tenant_event_hash = %s, tenant_hash_algo = %s + WHERE id = %s + """, + (previous, tenant_hash, TENANT_EVENT_HASH_ALGO, int(row["id"])), ) + heads[tenant_id] = tenant_hash updated += 1 - prev_hash = event_hash return updated + def _event_tenant_for_claim(self, cur, claim_id: int | None) -> str | None: + if claim_id is None: + return self._tenant_for_operation() if self.require_tenant else self.tenant_id + cur.execute("SELECT tenant_id FROM claims WHERE id = %s", (claim_id,)) + row = cur.fetchone() + if row is None: + raise ValueError(f"Claim {claim_id} does not exist.") + raw_tenant = row.get("tenant_id") if isinstance(row, dict) else row[0] + claim_tenant = self._as_text(raw_tenant) + if self.require_tenant and claim_tenant is None: + raise PermissionError("Tenant-owned events require a tenant-owned claim.") + return self._tenant_for_operation(claim_tenant) + + @staticmethod + def _event_chain_head( + cur, + tenant_id: str | None, + ) -> tuple[str | None, str, str | None]: + lock_key = tenant_id or "__memorymaster_global_events__" + cur.execute( + "SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", + (f"memorymaster:event:{lock_key}",), + ) + if tenant_id is None: + algo = EVENT_HASH_ALGO + cur.execute( + """ + SELECT event_hash FROM events + WHERE event_hash IS NOT NULL + AND (hash_algo IS NULL OR hash_algo = %s) + ORDER BY id DESC LIMIT 1 + """, + (algo,), + ) + else: + algo = POSTGRES_TENANT_EVENT_HASH_ALGO + cur.execute( + """ + SELECT event_hash FROM events + WHERE event_hash IS NOT NULL + AND hash_algo = %s + AND tenant_id IS NOT DISTINCT FROM %s + ORDER BY id DESC LIMIT 1 + """, + (algo, tenant_id), + ) + row = cur.fetchone() + value = row.get("event_hash") if isinstance(row, dict) and row else None + tenant_head: str | None = None + if tenant_id is not None: + cur.execute( + """ + SELECT tenant_event_hash FROM events + WHERE tenant_id IS NOT DISTINCT FROM %s + AND tenant_event_hash IS NOT NULL + ORDER BY id DESC LIMIT 1 + """, + (tenant_id,), + ) + tenant_row = cur.fetchone() + if isinstance(tenant_row, dict) and tenant_row.get("tenant_event_hash"): + tenant_head = str(tenant_row["tenant_event_hash"]) + return (str(value) if value else None), algo, tenant_head + def _insert_event_row( self, conn, @@ -369,9 +546,11 @@ def _insert_event_row( ) -> int: _, _, Jsonb = self._load_psycopg() with conn.cursor() as cur: - cur.execute("SELECT event_hash FROM events WHERE event_hash IS NOT NULL ORDER BY id DESC LIMIT 1") - prev_row = cur.fetchone() - prev_event_hash = str(prev_row["event_hash"]) if prev_row and prev_row.get("event_hash") else None + tenant_id = self._event_tenant_for_claim(cur, claim_id) + prev_event_hash, hash_algo, tenant_prev_event_hash = self._event_chain_head( + cur, + tenant_id, + ) event_hash = self._compute_event_hash( claim_id=claim_id, event_type=event_type, @@ -381,15 +560,27 @@ def _insert_event_row( payload=payload, created_at=created_at, prev_event_hash=prev_event_hash, - hash_algo=EVENT_HASH_ALGO, + tenant_id=tenant_id, + hash_algo=hash_algo, + ) + tenant_event_hash = ( + compute_tenant_event_hash( + tenant_id=tenant_id, + event_hash=event_hash, + tenant_prev_event_hash=tenant_prev_event_hash, + ) + if tenant_id is not None + else None ) + tenant_hash_algo = TENANT_EVENT_HASH_ALGO if tenant_id is not None else None cur.execute( """ INSERT INTO events ( claim_id, event_type, from_status, to_status, details, payload_json, created_at, - prev_event_hash, event_hash, hash_algo + prev_event_hash, event_hash, hash_algo, tenant_id, + tenant_prev_event_hash, tenant_event_hash, tenant_hash_algo ) - VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id """, ( @@ -402,7 +593,11 @@ def _insert_event_row( created_at, prev_event_hash, event_hash, - EVENT_HASH_ALGO, + hash_algo, + tenant_id, + tenant_prev_event_hash, + tenant_event_hash, + tenant_hash_algo, ), ) inserted = cur.fetchone() @@ -681,6 +876,10 @@ def list_events( clauses: list[str] = [] params: list[object] = [] + if self.tenant_id is not None: + clauses.append("tenant_id IS NOT DISTINCT FROM %s") + params.append(self._tenant_for_operation()) + if claim_id is not None: clauses.append("claim_id = %s") params.append(claim_id) @@ -1072,7 +1271,226 @@ def delete_old_events(self, retain_days: int) -> int: # Events are append-only by contract; retention trim is a no-op. return 0 + @staticmethod + def _event_chain_link_issues( + rows: list[dict[str, object]], + limit: int, + ) -> list[dict[str, object]]: + issues: list[dict[str, object]] = [] + v1_expected: str | None = None + tenant_expected: dict[str, str | None] = {} + for row in rows: + event_id = int(row["id"]) + row_prev = PostgresStore._as_text(row.get("prev_event_hash")) + row_hash = PostgresStore._as_text(row.get("event_hash")) + row_algo = PostgresStore._as_text(row.get("hash_algo")) or EVENT_HASH_ALGO + tenant_id = PostgresStore._as_text(row.get("tenant_id")) + if row_hash is None: + issues.append({"event_id": event_id, "reason": "missing_hash"}) + continue + if row_algo == EVENT_HASH_ALGO: + expected = v1_expected + v1_expected = row_hash + elif row_algo == POSTGRES_TENANT_EVENT_HASH_ALGO and tenant_id is not None: + expected = tenant_expected.get(tenant_id) + tenant_expected[tenant_id] = row_hash + else: + issues.append( + { + "event_id": event_id, + "reason": "unexpected_hash_algo_or_tenant", + "hash_algo": row_algo, + "tenant_id": tenant_id, + } + ) + continue + if row_prev != expected: + issues.append( + { + "event_id": event_id, + "reason": "broken_prev_link", + "expected_prev_event_hash": expected, + "actual_prev_event_hash": row_prev, + } + ) + if len(issues) >= limit: + break + return issues + + @staticmethod + def _expected_primary_event_hash( + row: dict[str, object], + *, + canonicalize_timestamp: bool = True, + ) -> str | None: + tenant_id = PostgresStore._as_text(row.get("tenant_id")) + event_type = PostgresStore._as_text(row.get("event_type")) + created_at = row.get("created_at") + hash_algo = PostgresStore._as_text(row.get("hash_algo")) or EVENT_HASH_ALGO + if hash_algo not in {EVENT_HASH_ALGO, POSTGRES_TENANT_EVENT_HASH_ALGO}: + return None + if isinstance(created_at, str): + try: + created_at = datetime.fromisoformat(created_at.replace("Z", "+00:00")) + except ValueError: + return None + if event_type is None or not isinstance(created_at, datetime): + return None + if hash_algo == POSTGRES_TENANT_EVENT_HASH_ALGO and tenant_id is None: + return None + claim_id = row.get("claim_id") + return PostgresStore._compute_event_hash( + claim_id=int(claim_id) if claim_id is not None else None, + event_type=event_type, + from_status=PostgresStore._as_text(row.get("from_status")), + to_status=PostgresStore._as_text(row.get("to_status")), + details=PostgresStore._as_text(row.get("details")), + payload=row.get("payload_json"), + created_at=created_at, + prev_event_hash=PostgresStore._as_text(row.get("prev_event_hash")), + tenant_id=tenant_id, + hash_algo=hash_algo, + canonicalize_timestamp=canonicalize_timestamp, + ) + + @staticmethod + def _expected_v2_event_hash(row: dict[str, object]) -> str | None: + if PostgresStore._as_text(row.get("hash_algo")) != POSTGRES_TENANT_EVENT_HASH_ALGO: + return None + return PostgresStore._expected_primary_event_hash(row) + + @staticmethod + def _event_content_issues( + rows: list[dict[str, object]], + limit: int, + ) -> list[dict[str, object]]: + issues: list[dict[str, object]] = [] + for row in rows: + hash_algo = PostgresStore._as_text(row.get("hash_algo")) or EVENT_HASH_ALGO + if hash_algo not in {EVENT_HASH_ALGO, POSTGRES_TENANT_EVENT_HASH_ALGO}: + continue + expected_hash = PostgresStore._expected_primary_event_hash(row) + stored_hash = PostgresStore._as_text(row.get("event_hash")) + if expected_hash is None: + issues.append( + {"event_id": int(row["id"]), "reason": "missing_event_hash_material"} + ) + else: + expected_hashes = {expected_hash} + hash_algo = PostgresStore._as_text(row.get("hash_algo")) or EVENT_HASH_ALGO + if hash_algo == EVENT_HASH_ALGO: + legacy_hash = PostgresStore._expected_primary_event_hash( + row, + canonicalize_timestamp=False, + ) + if legacy_hash is not None: + expected_hashes.add(legacy_hash) + if stored_hash not in expected_hashes: + issues.append( + {"event_id": int(row["id"]), "reason": "event_hash_mismatch"} + ) + if len(issues) >= limit: + break + return issues + + @staticmethod + def _event_chain_issues( + rows: list[dict[str, object]], + limit: int, + *, + verify_content: bool = True, + ) -> list[dict[str, object]]: + issues = PostgresStore._event_chain_link_issues(rows, limit) + if verify_content and len(issues) < limit: + issues.extend(PostgresStore._event_content_issues(rows, limit - len(issues))) + return issues + + @staticmethod + def _tenant_event_chain_issues( + rows: list[dict[str, object]], + limit: int, + ) -> list[dict[str, object]]: + issues: list[dict[str, object]] = [] + heads: dict[str, str | None] = {} + for row in rows: + event_id = int(row["id"]) + tenant_id = PostgresStore._as_text(row.get("tenant_id")) + if tenant_id is None: + continue + previous = PostgresStore._as_text(row.get("tenant_prev_event_hash")) + stored_hash = PostgresStore._as_text(row.get("tenant_event_hash")) + hash_algo = PostgresStore._as_text(row.get("tenant_hash_algo")) + event_hash = PostgresStore._as_text(row.get("event_hash")) + expected_previous = heads.get(tenant_id) + if previous != expected_previous: + issues.append( + { + "event_id": event_id, + "reason": "broken_tenant_prev_link", + "expected_prev_event_hash": expected_previous, + "actual_prev_event_hash": previous, + } + ) + if hash_algo != TENANT_EVENT_HASH_ALGO or event_hash is None or stored_hash is None: + issues.append({"event_id": event_id, "reason": "missing_tenant_hash_material"}) + else: + expected_hash = compute_tenant_event_hash( + tenant_id=tenant_id, + event_hash=event_hash, + tenant_prev_event_hash=previous, + ) + if stored_hash != expected_hash: + issues.append({"event_id": event_id, "reason": "tenant_hash_mismatch"}) + heads[tenant_id] = stored_hash + if len(issues) >= limit: + break + return issues + + def _reconcile_tenant_event_integrity(self, limit: int) -> dict[str, object]: + with self.connect() as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT id, claim_id, event_type, from_status, to_status, details, + payload_json, created_at, prev_event_hash, event_hash, hash_algo, + tenant_id, tenant_prev_event_hash, tenant_event_hash, tenant_hash_algo + FROM events + WHERE tenant_id IS NOT DISTINCT FROM %s + ORDER BY id ASC + """, + (self._tenant_for_operation(),), + ) + rows = cur.fetchall() + v2_rows = [ + row + for row in rows + if self._as_text(row.get("hash_algo")) == POSTGRES_TENANT_EVENT_HASH_ALGO + ] + original_chain_issues = self._event_chain_link_issues(v2_rows, limit) + content_issues = self._event_content_issues(rows, limit) + chain_issues = self._tenant_event_chain_issues(rows, limit) + return { + "checked_at": utc_now().isoformat(), + "fix_mode": False, + "issues": { + "hash_chain_issues": original_chain_issues, + "event_content_issues": content_issues, + "tenant_hash_chain_issues": chain_issues, + }, + "summary": { + "hash_chain_issues": len(original_chain_issues), + "event_content_issues": len(content_issues), + "tenant_hash_chain_issues": len(chain_issues), + }, + "actions": [], + } + def reconcile_integrity(self, *, fix: bool = False, limit: int = 500) -> dict[str, object]: + if self.require_tenant: + if fix: + raise PermissionError( + "Integrity repair requires a privileged maintenance store." + ) + return self._reconcile_tenant_event_integrity(limit) report: dict[str, object] = { "checked_at": utc_now().isoformat(), "fix_mode": bool(fix), @@ -1080,7 +1498,8 @@ def reconcile_integrity(self, *, fix: bool = False, limit: int = 500) -> dict[st "actions": [], } with self.connect() as conn: - self._ensure_event_integrity_schema(conn) + if fix: + self._ensure_event_integrity_schema(conn) with conn.cursor() as cur: cur.execute( """ @@ -1189,31 +1608,17 @@ def reconcile_integrity(self, *, fix: bool = False, limit: int = 500) -> dict[st } ) - cur.execute("SELECT id, prev_event_hash, event_hash, hash_algo FROM events ORDER BY id ASC") + cur.execute( + """ + SELECT id, claim_id, event_type, from_status, to_status, details, + payload_json, created_at, prev_event_hash, event_hash, hash_algo, + tenant_id, tenant_prev_event_hash, tenant_event_hash, tenant_hash_algo + FROM events ORDER BY id ASC + """ + ) chain_rows = cur.fetchall() - chain_issues: list[dict[str, object]] = [] - expected_prev: str | None = None - for row in chain_rows: - row_prev = self._as_text(row["prev_event_hash"]) - row_hash = self._as_text(row["event_hash"]) - row_algo = self._as_text(row["hash_algo"]) - if row_hash is None: - chain_issues.append({"event_id": int(row["id"]), "reason": "missing_hash"}) - continue - if row_algo not in {None, EVENT_HASH_ALGO}: - chain_issues.append( - {"event_id": int(row["id"]), "reason": "unexpected_hash_algo", "hash_algo": row_algo} - ) - if row_prev != expected_prev: - chain_issues.append( - { - "event_id": int(row["id"]), - "reason": "broken_prev_link", - "expected_prev_event_hash": expected_prev, - "actual_prev_event_hash": row_prev, - } - ) - expected_prev = row_hash + chain_issues = self._event_chain_issues(chain_rows, limit) + tenant_chain_issues = self._tenant_event_chain_issues(chain_rows, limit) issues = { "orphan_events": orphan_events, @@ -1223,6 +1628,7 @@ def reconcile_integrity(self, *, fix: bool = False, limit: int = 500) -> dict[st "dangling_supersedes": dangling_supersedes, "transition_issues": transition_issues[:limit], "hash_chain_issues": chain_issues[:limit], + "tenant_hash_chain_issues": tenant_chain_issues[:limit], } report["issues"] = issues report["summary"] = { diff --git a/tests/test_reliability_hardening.py b/tests/test_reliability_hardening.py index ccb5ec62..1790ab06 100644 --- a/tests/test_reliability_hardening.py +++ b/tests/test_reliability_hardening.py @@ -104,7 +104,11 @@ def test_reconcile_integrity_reports_and_fixes_orphans_and_chain() -> None: assert report["summary"]["orphan_events"] >= 1 assert report["summary"]["orphan_citations"] >= 1 assert report["summary"]["superseded_without_replacement"] >= 1 - assert report["summary"]["hash_chain_issues"] == 0 + assert report["summary"]["hash_chain_issues"] >= 1 + assert any( + issue["reason"] == "missing_hash" + for issue in report["issues"]["hash_chain_issues"] + ) fixed = service.store.reconcile_integrity(fix=True, limit=50) action_names = {str(action.get("action")) for action in fixed.get("actions", [])} diff --git a/tests/test_tenant_event_ledger.py b/tests/test_tenant_event_ledger.py new file mode 100644 index 00000000..7d98763a --- /dev/null +++ b/tests/test_tenant_event_ledger.py @@ -0,0 +1,730 @@ +"""Adversarial tests for tenant-aware event storage and PostgreSQL chains.""" +from __future__ import annotations + +import importlib +import sqlite3 +from datetime import datetime, timedelta, timezone + +import pytest + +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.stores._storage_shared import ( + EVENT_HASH_ALGO, + compute_tenant_event_hash, +) +from memorymaster.stores.migrations import discover_migrations +from memorymaster.stores.postgres_store import PostgresStore +from memorymaster.stores.storage import SQLiteStore + + +TENANT_EVENT_HASH_ALGO = "sha256-tenant-v2" + + +class RecordingCursor: + def __init__( + self, + claim_tenant: str = "tenant-a", + rows: list[dict[str, object]] | None = None, + ) -> None: + self.claim_tenant = claim_tenant + self.rows = rows or [] + self.executed: list[tuple[str, object]] = [] + self.executed_many: list[tuple[str, object]] = [] + self.last_sql = "" + + def __enter__(self) -> "RecordingCursor": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, sql: str, params: object = ()) -> None: + self.last_sql = " ".join(sql.split()) + self.executed.append((self.last_sql, params)) + + def executemany(self, sql: str, params: object) -> None: + normalized = " ".join(sql.split()) + self.executed_many.append((normalized, params)) + + def fetchone(self): + if "SELECT tenant_id FROM claims" in self.last_sql: + return {"tenant_id": self.claim_tenant} + if "INSERT INTO events" in self.last_sql: + return {"id": 42} + return None + + def fetchall(self) -> list[object]: + return list(self.rows) + + +class RecordingConnection: + def __init__( + self, + claim_tenant: str = "tenant-a", + rows: list[dict[str, object]] | None = None, + ) -> None: + self.cursor_instance = RecordingCursor(claim_tenant, rows) + self.commits = 0 + + def __enter__(self) -> "RecordingConnection": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def cursor(self) -> RecordingCursor: + return self.cursor_instance + + def commit(self) -> None: + self.commits += 1 + + +class HistoricalHeadCursor(RecordingCursor): + def fetchone(self): + if "SELECT event_hash FROM events" in self.last_sql: + return {"event_hash": "primary-v2-head"} + if "SELECT tenant_event_hash FROM events" in self.last_sql: + return {"tenant_event_hash": "tenant-history-head"} + return super().fetchone() + + +class FailingTriggerConnection: + """Inject a trigger recreation failure without hiding transaction state.""" + + def __init__(self, conn: sqlite3.Connection) -> None: + self.conn = conn + + def execute(self, sql: str, params: object = ()): + if "CREATE TRIGGER IF NOT EXISTS trg_events_append_only_delete" in sql: + raise sqlite3.OperationalError("injected trigger recreation failure") + return self.conn.execute(sql, params) + + def executemany(self, sql: str, params: object): + return self.conn.executemany(sql, params) + + def executescript(self, _sql: str): + self.conn.commit() + raise sqlite3.OperationalError("injected trigger recreation failure") + + def commit(self) -> None: + self.conn.commit() + + def rollback(self) -> None: + self.conn.rollback() + + +def _migration(): + return importlib.import_module( + "memorymaster.stores.migrations.0010_tenant_event_ledger" + ) + + +def _postgres_store(claim_tenant: str = "tenant-a") -> tuple[PostgresStore, RecordingConnection]: + store = PostgresStore( + "postgresql://db.invalid/app", + tenant_id="tenant-a", + require_tenant=True, + ) + store._psycopg = (object(), object(), lambda value: value) + return store, RecordingConnection(claim_tenant) + + +def test_migration_backfills_event_tenant_without_rewriting_hashes() -> None: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.executescript( + """ + CREATE TABLE claims (id INTEGER PRIMARY KEY, tenant_id TEXT); + CREATE TABLE events ( + id INTEGER PRIMARY KEY, + claim_id INTEGER, + event_type TEXT NOT NULL, + from_status TEXT, + to_status TEXT, + details TEXT, + payload_json TEXT, + created_at TEXT NOT NULL, + prev_event_hash TEXT, + event_hash TEXT, + hash_algo TEXT + ); + INSERT INTO claims(id, tenant_id) VALUES + (1, 'tenant-a'), + (2, 'tenant-b'), + (3, 'tenant-a'); + """ + ) + original_triples: list[tuple[str | None, str, str]] = [] + previous: str | None = None + for event_id, claim_id in ((1, 1), (2, 2), (3, 3)): + event_hash = SQLiteStore._compute_event_hash( + claim_id=claim_id, + event_type="ingest", + from_status=None, + to_status="candidate", + details=f"fixture-{event_id}", + payload_json=None, + created_at="2026-07-10T00:00:00+00:00", + prev_event_hash=previous, + ) + conn.execute( + """ + INSERT INTO events( + id, claim_id, event_type, to_status, details, created_at, + prev_event_hash, event_hash, hash_algo + ) VALUES (?, ?, 'ingest', 'candidate', ?, ?, ?, ?, ?) + """, + ( + event_id, + claim_id, + f"fixture-{event_id}", + "2026-07-10T00:00:00+00:00", + previous, + event_hash, + EVENT_HASH_ALGO, + ), + ) + original_triples.append((previous, event_hash, EVENT_HASH_ALGO)) + previous = event_hash + conn.commit() + try: + _migration().apply_sqlite(conn) + rows = conn.execute("SELECT * FROM events ORDER BY id").fetchall() + finally: + conn.close() + + assert [row["tenant_id"] for row in rows] == ["tenant-a", "tenant-b", "tenant-a"] + assert [ + (row["prev_event_hash"], row["event_hash"], row["hash_algo"]) + for row in rows + ] == original_triples + assert rows[0]["tenant_prev_event_hash"] is None + assert rows[1]["tenant_prev_event_hash"] is None + assert rows[2]["tenant_prev_event_hash"] == rows[0]["tenant_event_hash"] + assert {row["tenant_hash_algo"] for row in rows} == {TENANT_EVENT_HASH_ALGO} + + +def test_postgres_migration_is_versioned_and_preserves_hash_columns() -> None: + migration = next(item for item in discover_migrations() if item.version == 10) + conn = RecordingConnection() + + migration.apply_postgres(conn) + emitted = "\n".join(sql for sql, _ in conn.cursor_instance.executed) + + assert "event" in migration.description.lower() + assert "ALTER TABLE events ADD COLUMN IF NOT EXISTS tenant_id TEXT" in emitted + assert "tenant_prev_event_hash" in emitted + assert "tenant_event_hash" in emitted + assert "tenant_hash_algo" in emitted + assert "SET tenant_id = claim.tenant_id" in emitted + assert "idx_events_tenant_id" in emitted + assert "idx_events_tenant_head" in emitted + assert "idx_events_tenant_algo_head" in emitted + assert "CREATE POLICY memorymaster_tenant_restrict ON events" in emitted + assert "LOCK TABLE events IN ACCESS EXCLUSIVE MODE" in emitted + assert "LOCK TABLE events IN SHARE ROW EXCLUSIVE MODE" not in emitted + assert "events.claim_id IS NULL OR EXISTS" in emitted + assert "SET event_hash" not in emitted + assert "SET prev_event_hash" not in emitted + assert conn.commits == 1 + + +def test_event_hash_algorithms_have_checksum_frozen_golden_vectors() -> None: + kwargs = { + "claim_id": 7, + "event_type": "ingest", + "from_status": None, + "to_status": "candidate", + "details": "fixture", + "payload": {"count": 1}, + "created_at": datetime(2026, 7, 10, tzinfo=timezone.utc), + "prev_event_hash": None, + } + v1_hash = PostgresStore._compute_event_hash(**kwargs, hash_algo=EVENT_HASH_ALGO) + tenant_primary_hash = PostgresStore._compute_event_hash( + **kwargs, + tenant_id="tenant-a", + hash_algo=TENANT_EVENT_HASH_ALGO, + ) + tenant_chain_hash = compute_tenant_event_hash( + tenant_id="tenant-a", + event_hash=v1_hash, + tenant_prev_event_hash=None, + ) + + assert v1_hash == "7e29764a32caa9370045ddeeaffb9684d596983c99a16c3f280ed696949ff286" + assert tenant_primary_hash == "a49553710e1c35c1982a4e36dd70a8889c8b6987aa9ceeb237ed82e16721b1fb" + assert tenant_chain_hash == "adfcc310faaa12a1429ad3007e627acbbe9c79085666c1442299195862f213cc" + assert ( + _migration()._compute_tenant_event_hash_v2( + tenant_id="tenant-a", + event_hash=v1_hash, + tenant_prev_event_hash=None, + ) + == tenant_chain_hash + ) + + +def test_postgres_event_hashes_canonicalize_non_utc_timestamps() -> None: + non_utc = datetime( + 2026, + 7, + 9, + 21, + tzinfo=timezone(timedelta(hours=-3)), + ) + kwargs = { + "claim_id": 7, + "event_type": "ingest", + "from_status": None, + "to_status": "candidate", + "details": "fixture", + "payload": {"count": 1}, + "created_at": non_utc, + "prev_event_hash": None, + } + + assert ( + PostgresStore._compute_event_hash(**kwargs, hash_algo=EVENT_HASH_ALGO) + == "7e29764a32caa9370045ddeeaffb9684d596983c99a16c3f280ed696949ff286" + ) + assert ( + PostgresStore._compute_event_hash( + **kwargs, + tenant_id="tenant-a", + hash_algo=TENANT_EVENT_HASH_ALGO, + ) + == "a49553710e1c35c1982a4e36dd70a8889c8b6987aa9ceeb237ed82e16721b1fb" + ) + + +def test_migration_accepts_historical_v1_raw_offset_hash() -> None: + row = { + "id": 1, + "claim_id": 7, + "event_type": "ingest", + "from_status": None, + "to_status": "candidate", + "details": "fixture", + "payload_json": {"count": 1}, + "created_at": datetime( + 2026, + 7, + 9, + 21, + tzinfo=timezone(timedelta(hours=-3)), + ), + "prev_event_hash": None, + "event_hash": "3b8c92d37c952664fe8436dd2690cbbdcb8449ae91b02ee2f7a12b50a814b5ef", + "hash_algo": EVENT_HASH_ALGO, + "tenant_id": "tenant-a", + "tenant_prev_event_hash": None, + "tenant_event_hash": None, + "tenant_hash_algo": None, + } + + _migration()._validate_primary_chain([row]) + assert PostgresStore._event_chain_issues([row], limit=20) == [] + + canonical_row = dict(row) + canonical_row["event_hash"] = ( + "7e29764a32caa9370045ddeeaffb9684d596983c99a16c3f280ed696949ff286" + ) + assert PostgresStore._event_chain_issues([canonical_row], limit=20) == [] + + +def test_legacy_schema_ensure_does_not_preapply_tenant_event_migration() -> None: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.executescript( + """ + CREATE TABLE claims (id INTEGER PRIMARY KEY); + CREATE TABLE events ( + id INTEGER PRIMARY KEY, + claim_id INTEGER, + event_type TEXT NOT NULL, + from_status TEXT, + to_status TEXT, + details TEXT, + payload_json TEXT, + created_at TEXT NOT NULL + ); + """ + ) + try: + SQLiteStore._ensure_event_integrity_schema(conn) + columns = {row[1] for row in conn.execute("PRAGMA table_info(events)")} + finally: + conn.close() + + assert {"prev_event_hash", "event_hash", "hash_algo"} <= columns + assert "tenant_id" not in columns + assert "tenant_event_hash" not in columns + + +def test_read_only_reconciliation_does_not_run_schema_repairs(tmp_path, monkeypatch) -> None: + service = MemoryService(tmp_path / "read-only-reconcile.db", workspace_root=tmp_path) + service.init_db() + + def fail_if_called(_conn) -> None: + pytest.fail("read-only reconciliation invoked a schema repair helper") + + monkeypatch.setattr(service.store, "_ensure_event_integrity_schema", fail_if_called) + report = service.store.reconcile_integrity(fix=False) + + assert report["summary"]["hash_chain_issues"] == 0 + + +def test_sqlite_reconciliation_detects_v1_payload_tampering(tmp_path) -> None: + service = MemoryService(tmp_path / "tampered-v1.db", workspace_root=tmp_path) + service.init_db() + claim = service.ingest( + text="SQLite integrity fixture.", + citations=[CitationInput(source="test")], + source_agent="tenant-event-test", + ) + with service.store.connect() as conn: + conn.execute("DROP TRIGGER trg_events_append_only_update") + conn.execute( + "UPDATE events SET details = 'tampered' WHERE claim_id = ?", + (claim.id,), + ) + conn.commit() + + report = service.store.reconcile_integrity(fix=False) + + assert any( + issue["reason"] == "event_hash_mismatch" + for issue in report["issues"]["hash_chain_issues"] + ) + + +def test_sqlite_reconciliation_detects_tenant_chain_tampering(tmp_path) -> None: + service = MemoryService( + tmp_path / "tampered-tenant-chain.db", + workspace_root=tmp_path, + tenant_id="tenant-a", + ) + service.init_db() + claim = service.ingest( + text="SQLite tenant chain fixture.", + citations=[CitationInput(source="test")], + source_agent="tenant-event-test", + ) + with service.store.connect() as conn: + conn.execute("DROP TRIGGER trg_events_append_only_update") + conn.execute( + "UPDATE events SET tenant_event_hash = 'tampered' WHERE claim_id = ?", + (claim.id,), + ) + conn.commit() + + report = service.store.reconcile_integrity(fix=False) + + assert report["summary"]["tenant_hash_chain_issues"] > 0 + + +def test_legacy_sqlite_event_insert_falls_back_to_original_columns() -> None: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.executescript( + """ + CREATE TABLE claims (id INTEGER PRIMARY KEY); + CREATE TABLE events ( + id INTEGER PRIMARY KEY, + claim_id INTEGER, + event_type TEXT NOT NULL, + from_status TEXT, + to_status TEXT, + details TEXT, + payload_json TEXT, + created_at TEXT NOT NULL + ); + """ + ) + try: + event_id = SQLiteStore._insert_event_row( + conn, + claim_id=None, + event_type="system", + from_status=None, + to_status=None, + details="legacy fallback", + payload_json=None, + created_at="2026-07-10T00:00:00+00:00", + ) + stored = conn.execute("SELECT event_type FROM events WHERE id = ?", (event_id,)).fetchone() + finally: + conn.close() + + assert stored["event_type"] == "system" + + +def test_sqlite_migration_rolls_back_if_trigger_recreation_fails() -> None: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.executescript( + """ + CREATE TABLE claims (id INTEGER PRIMARY KEY, tenant_id TEXT); + CREATE TABLE events ( + id INTEGER PRIMARY KEY, + claim_id INTEGER, + event_type TEXT NOT NULL, + from_status TEXT, + to_status TEXT, + details TEXT, + payload_json TEXT, + created_at TEXT NOT NULL, + prev_event_hash TEXT, + event_hash TEXT, + hash_algo TEXT + ); + CREATE TRIGGER trg_events_append_only_update BEFORE UPDATE ON events + BEGIN SELECT RAISE(ABORT, 'append only'); END; + CREATE TRIGGER trg_events_append_only_delete BEFORE DELETE ON events + BEGIN SELECT RAISE(ABORT, 'append only'); END; + """ + ) + try: + with pytest.raises(sqlite3.OperationalError, match="injected"): + _migration().apply_sqlite(FailingTriggerConnection(conn)) + triggers = { + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'trigger' AND name LIKE 'trg_events_append_only_%'" + ) + } + finally: + conn.close() + + assert triggers == {"trg_events_append_only_update", "trg_events_append_only_delete"} + + +def test_migration_rejects_invalid_primary_chain_before_tenant_updates() -> None: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.executescript( + """ + CREATE TABLE claims (id INTEGER PRIMARY KEY, tenant_id TEXT); + CREATE TABLE events ( + id INTEGER PRIMARY KEY, + claim_id INTEGER, + event_type TEXT NOT NULL, + from_status TEXT, + to_status TEXT, + details TEXT, + payload_json TEXT, + created_at TEXT NOT NULL, + prev_event_hash TEXT, + event_hash TEXT, + hash_algo TEXT + ); + INSERT INTO claims(id, tenant_id) VALUES (1, 'tenant-a'); + INSERT INTO events VALUES ( + 1, 1, 'ingest', NULL, 'candidate', 'bad chain', NULL, + '2026-07-10T00:00:00+00:00', 'not-the-head', 'invalid-hash', 'sha256-v1' + ); + """ + ) + try: + with pytest.raises(RuntimeError, match="primary event chain"): + _migration().apply_sqlite(conn) + columns = {row[1] for row in conn.execute("PRAGMA table_info(events)")} + tenant_values = ( + conn.execute("SELECT tenant_id FROM events").fetchall() + if "tenant_id" in columns + else [] + ) + finally: + conn.close() + + assert not tenant_values or all(row[0] is None for row in tenant_values) + + +def test_migration_rejects_conflicting_existing_tenant_prefix() -> None: + rows = [ + { + "id": 1, + "tenant_id": "tenant-a", + "event_hash": "primary-1", + "tenant_prev_event_hash": "wrong-head", + "tenant_event_hash": "wrong-hash", + "tenant_hash_algo": TENANT_EVENT_HASH_ALGO, + } + ] + + with pytest.raises(RuntimeError, match="tenant event chain"): + _migration()._tenant_hash_updates(rows) + + +def test_migration_resumes_and_validates_partial_tenant_prefix() -> None: + first_hash = _migration()._compute_tenant_event_hash_v2( + tenant_id="tenant-a", + event_hash="primary-1", + tenant_prev_event_hash=None, + ) + rows = [ + { + "id": 1, + "tenant_id": "tenant-a", + "event_hash": "primary-1", + "tenant_prev_event_hash": None, + "tenant_event_hash": first_hash, + "tenant_hash_algo": TENANT_EVENT_HASH_ALGO, + }, + { + "id": 2, + "tenant_id": "tenant-a", + "event_hash": "primary-2", + "tenant_prev_event_hash": None, + "tenant_event_hash": None, + "tenant_hash_algo": None, + }, + ] + + updates = _migration()._tenant_hash_updates(rows) + + assert len(updates) == 1 + assert updates[0][0] == first_hash + rows[1]["tenant_prev_event_hash"] = updates[0][0] + rows[1]["tenant_event_hash"] = updates[0][1] + rows[1]["tenant_hash_algo"] = updates[0][2] + assert _migration()._tenant_hash_updates(rows) == [] + + +def test_postgres_migration_executes_backfill_and_complete_rerun_is_noop() -> None: + created_at = datetime(2026, 7, 10, tzinfo=timezone.utc) + primary_hash = PostgresStore._compute_event_hash( + claim_id=1, + event_type="ingest", + from_status=None, + to_status="candidate", + details="fixture", + payload=None, + created_at=created_at, + prev_event_hash=None, + hash_algo=EVENT_HASH_ALGO, + ) + row = { + "id": 1, + "claim_id": 1, + "event_type": "ingest", + "from_status": None, + "to_status": "candidate", + "details": "fixture", + "payload_json": None, + "created_at": created_at, + "prev_event_hash": None, + "event_hash": primary_hash, + "hash_algo": EVENT_HASH_ALGO, + "tenant_id": "tenant-a", + "tenant_prev_event_hash": None, + "tenant_event_hash": None, + "tenant_hash_algo": None, + } + conn = RecordingConnection(rows=[row]) + + _migration().apply_postgres(conn) + + assert len(conn.cursor_instance.executed_many) == 1 + _, updates = conn.cursor_instance.executed_many[0] + update = list(updates)[0] + complete_row = dict(row) + complete_row.update( + tenant_prev_event_hash=update[0], + tenant_event_hash=update[1], + tenant_hash_algo=update[2], + ) + rerun = RecordingConnection(rows=[complete_row]) + _migration().apply_postgres(rerun) + assert rerun.cursor_instance.executed_many == [] + + +def test_postgres_hash_repair_is_partition_aware_after_v2_cutover() -> None: + created_at = datetime(2026, 7, 10, tzinfo=timezone.utc) + + def event_row( + event_id: int, + *, + tenant_id: str | None, + hash_algo: str, + previous: str | None, + event_hash: str | None, + ) -> dict[str, object]: + return { + "id": event_id, + "claim_id": event_id, + "event_type": "ingest", + "from_status": None, + "to_status": "candidate", + "details": f"fixture-{event_id}", + "payload_json": None, + "created_at": created_at, + "prev_event_hash": previous, + "event_hash": event_hash, + "hash_algo": hash_algo, + "tenant_id": tenant_id, + } + + v1_head = PostgresStore._compute_event_hash( + claim_id=1, + event_type="ingest", + from_status=None, + to_status="candidate", + details="fixture-1", + payload=None, + created_at=created_at, + prev_event_hash=None, + hash_algo=EVENT_HASH_ALGO, + ) + tenant_a_head = PostgresStore._compute_event_hash( + claim_id=2, + event_type="ingest", + from_status=None, + to_status="candidate", + details="fixture-2", + payload=None, + created_at=created_at, + prev_event_hash=None, + tenant_id="tenant-a", + hash_algo=TENANT_EVENT_HASH_ALGO, + ) + tenant_b_head = PostgresStore._compute_event_hash( + claim_id=3, + event_type="ingest", + from_status=None, + to_status="candidate", + details="fixture-3", + payload=None, + created_at=created_at, + prev_event_hash=None, + tenant_id="tenant-b", + hash_algo=TENANT_EVENT_HASH_ALGO, + ) + rows = [ + event_row(1, tenant_id=None, hash_algo=EVENT_HASH_ALGO, previous=None, event_hash=v1_head), + event_row(2, tenant_id="tenant-a", hash_algo=TENANT_EVENT_HASH_ALGO, previous=None, event_hash=tenant_a_head), + event_row(3, tenant_id="tenant-b", hash_algo=TENANT_EVENT_HASH_ALGO, previous=None, event_hash=tenant_b_head), + event_row(4, tenant_id="tenant-a", hash_algo=TENANT_EVENT_HASH_ALGO, previous=None, event_hash=None), + event_row(5, tenant_id=None, hash_algo=EVENT_HASH_ALGO, previous=None, event_hash=None), + ] + conn = RecordingConnection(rows=rows) + + assert PostgresStore._backfill_event_chain(conn) == 2 + + updates = { + params[3]: params + for sql, params in conn.cursor_instance.executed + if sql.startswith("UPDATE events SET prev_event_hash") + } + assert updates[4][0] == tenant_a_head + assert updates[4][2] == TENANT_EVENT_HASH_ALGO + assert updates[5][0] == v1_head + assert updates[5][2] == EVENT_HASH_ALGO + + with pytest.raises(RuntimeError, match="mixed v1/tenant-v2"): + PostgresStore._backfill_event_chain( + RecordingConnection(rows=rows), + rebuild_all=True, + ) diff --git a/tests/test_tenant_event_runtime.py b/tests/test_tenant_event_runtime.py new file mode 100644 index 00000000..02ea6636 --- /dev/null +++ b/tests/test_tenant_event_runtime.py @@ -0,0 +1,451 @@ +"""Runtime and reconciliation tests for the tenant-aware event ledger.""" +from __future__ import annotations + +from datetime import datetime, timezone + +import pytest + +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.stores._storage_shared import EVENT_HASH_ALGO +from memorymaster.stores.postgres_store import PostgresStore + + +TENANT_EVENT_HASH_ALGO = "sha256-tenant-v2" + + +class RecordingCursor: + def __init__( + self, + claim_tenant: str = "tenant-a", + rows: list[dict[str, object]] | None = None, + ) -> None: + self.claim_tenant = claim_tenant + self.rows = rows or [] + self.executed: list[tuple[str, object]] = [] + self.executed_many: list[tuple[str, object]] = [] + self.last_sql = "" + + def __enter__(self) -> "RecordingCursor": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, sql: str, params: object = ()) -> None: + self.last_sql = " ".join(sql.split()) + self.executed.append((self.last_sql, params)) + + def executemany(self, sql: str, params: object) -> None: + normalized = " ".join(sql.split()) + self.executed_many.append((normalized, params)) + + def fetchone(self): + if "SELECT tenant_id FROM claims" in self.last_sql: + return {"tenant_id": self.claim_tenant} + if "INSERT INTO events" in self.last_sql: + return {"id": 42} + return None + + def fetchall(self) -> list[object]: + return list(self.rows) + + +class RecordingConnection: + def __init__( + self, + claim_tenant: str = "tenant-a", + rows: list[dict[str, object]] | None = None, + ) -> None: + self.cursor_instance = RecordingCursor(claim_tenant, rows) + self.commits = 0 + + def __enter__(self) -> "RecordingConnection": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def cursor(self) -> RecordingCursor: + return self.cursor_instance + + def commit(self) -> None: + self.commits += 1 + + +class HistoricalHeadCursor(RecordingCursor): + def fetchone(self): + if "SELECT event_hash FROM events" in self.last_sql: + return {"event_hash": "primary-v2-head"} + if "SELECT tenant_event_hash FROM events" in self.last_sql: + return {"tenant_event_hash": "tenant-history-head"} + return super().fetchone() + + +def _postgres_store( + claim_tenant: str = "tenant-a", +) -> tuple[PostgresStore, RecordingConnection]: + store = PostgresStore( + "postgresql://db.invalid/app", + tenant_id="tenant-a", + require_tenant=True, + ) + store._psycopg = (object(), object(), lambda value: value) + return store, RecordingConnection(claim_tenant) + +def test_full_sqlite_reinit_preserves_existing_v1_hash_triples(tmp_path) -> None: + service = MemoryService( + tmp_path / "legacy-events.db", + workspace_root=tmp_path, + tenant_id="tenant-a", + ) + service.init_db() + service.ingest( + text="Legacy event chain fixture.", + citations=[CitationInput(source="test")], + source_agent="tenant-event-test", + ) + with service.store.connect() as conn: + before = conn.execute( + """ + SELECT id, prev_event_hash, event_hash, hash_algo + FROM events ORDER BY id + """ + ).fetchall() + conn.execute("DROP TRIGGER IF EXISTS trg_events_append_only_update") + conn.execute("UPDATE events SET tenant_event_hash = NULL, tenant_hash_algo = NULL") + conn.commit() + + service.init_db() + + with service.store.connect() as conn: + after = conn.execute( + """ + SELECT id, prev_event_hash, event_hash, hash_algo + FROM events ORDER BY id + """ + ).fetchall() + assert [tuple(row) for row in after] == [tuple(row) for row in before] + + +def test_sqlite_new_events_persist_the_claim_tenant(tmp_path) -> None: + service = MemoryService( + tmp_path / "events.db", + workspace_root=tmp_path, + tenant_id="tenant-a", + ) + service.init_db() + claim = service.ingest( + text="Tenant event fixture.", + citations=[CitationInput(source="test")], + source_agent="tenant-event-test", + ) + + with service.store.connect() as conn: + tenant_ids = { + row["tenant_id"] + for row in conn.execute( + "SELECT tenant_id FROM events WHERE claim_id = ?", + (claim.id,), + ).fetchall() + } + + assert tenant_ids == {"tenant-a"} + + +def test_v1_hash_is_stable_and_v2_hash_is_tenant_bound() -> None: + kwargs = { + "claim_id": 7, + "event_type": "ingest", + "from_status": None, + "to_status": "candidate", + "details": "fixture", + "payload": {"count": 1}, + "created_at": datetime(2026, 7, 10, tzinfo=timezone.utc), + "prev_event_hash": None, + } + v1_a = PostgresStore._compute_event_hash( + **kwargs, + tenant_id="tenant-a", + hash_algo=EVENT_HASH_ALGO, + ) + v1_b = PostgresStore._compute_event_hash( + **kwargs, + tenant_id="tenant-b", + hash_algo=EVENT_HASH_ALGO, + ) + v2_a = PostgresStore._compute_event_hash( + **kwargs, + tenant_id="tenant-a", + hash_algo=TENANT_EVENT_HASH_ALGO, + ) + v2_b = PostgresStore._compute_event_hash( + **kwargs, + tenant_id="tenant-b", + hash_algo=TENANT_EVENT_HASH_ALGO, + ) + + assert v1_a == v1_b + assert v2_a != v2_b + + +def test_postgres_event_insert_uses_tenant_chain_and_advisory_lock() -> None: + store, conn = _postgres_store() + + event_id = store._insert_event_row( + conn, + claim_id=7, + event_type="ingest", + from_status=None, + to_status="candidate", + details="fixture", + payload={"count": 1}, + created_at=datetime(2026, 7, 10, tzinfo=timezone.utc), + ) + + emitted = "\n".join(sql for sql, _ in conn.cursor_instance.executed) + insert_params = next( + params + for sql, params in conn.cursor_instance.executed + if "INSERT INTO events" in sql + ) + assert event_id == 42 + assert "pg_advisory_xact_lock" in emitted + assert "hash_algo = %s" in emitted + assert "tenant_id IS NOT DISTINCT FROM %s" in emitted + assert "tenant_id" in next( + sql for sql, _ in conn.cursor_instance.executed if "INSERT INTO events" in sql + ) + assert "tenant-a" in insert_params + assert TENANT_EVENT_HASH_ALGO in insert_params + + +def test_postgres_event_insert_continues_historical_tenant_head() -> None: + store, conn = _postgres_store() + conn.cursor_instance = HistoricalHeadCursor() + + store._insert_event_row( + conn, + claim_id=7, + event_type="ingest", + from_status=None, + to_status="candidate", + details="fixture", + payload=None, + created_at=datetime(2026, 7, 10, tzinfo=timezone.utc), + ) + + insert_params = next( + params + for sql, params in conn.cursor_instance.executed + if "INSERT INTO events" in sql + ) + assert insert_params[7] == "primary-v2-head" + assert insert_params[11] == "tenant-history-head" + + +def test_postgres_event_write_rejects_wrong_claim_tenant() -> None: + store, wrong_conn = _postgres_store(claim_tenant="tenant-b") + + with pytest.raises(PermissionError, match="tenant"): + store._insert_event_row( + wrong_conn, + claim_id=7, + event_type="ingest", + from_status=None, + to_status="candidate", + details="fixture", + payload=None, + created_at=datetime(2026, 7, 10, tzinfo=timezone.utc), + ) + + + +def test_postgres_claimless_team_audit_event_uses_bound_tenant() -> None: + store, conn = _postgres_store() + + event_id = store._insert_event_row( + conn, + claim_id=None, + event_type="policy_decision", + from_status=None, + to_status=None, + details="authorization_denied", + payload={"action": "denied"}, + created_at=datetime(2026, 7, 10, tzinfo=timezone.utc), + ) + + insert_params = next( + params for sql, params in conn.cursor_instance.executed if "INSERT INTO events" in sql + ) + assert event_id == 42 + assert "tenant-a" in insert_params + + +def test_team_list_events_has_explicit_tenant_predicate(monkeypatch) -> None: + store, conn = _postgres_store() + monkeypatch.setattr(store, "connect", lambda: conn) + + assert store.list_events(limit=5) == [] + + sql, params = conn.cursor_instance.executed[-1] + assert "tenant_id IS NOT DISTINCT FROM %s" in sql + assert params[0] == "tenant-a" + + +def test_integrity_validation_partitions_v2_chains_by_tenant() -> None: + rows = [ + {"id": 1, "prev_event_hash": None, "event_hash": "v1-a", "hash_algo": EVENT_HASH_ALGO, "tenant_id": "tenant-a"}, + {"id": 2, "prev_event_hash": "v1-a", "event_hash": "v1-b", "hash_algo": EVENT_HASH_ALGO, "tenant_id": "tenant-b"}, + {"id": 3, "prev_event_hash": None, "event_hash": "a-1", "hash_algo": TENANT_EVENT_HASH_ALGO, "tenant_id": "tenant-a"}, + {"id": 4, "prev_event_hash": None, "event_hash": "b-1", "hash_algo": TENANT_EVENT_HASH_ALGO, "tenant_id": "tenant-b"}, + {"id": 5, "prev_event_hash": "a-1", "event_hash": "a-2", "hash_algo": TENANT_EVENT_HASH_ALGO, "tenant_id": "tenant-a"}, + {"id": 6, "prev_event_hash": "b-1", "event_hash": "b-2", "hash_algo": TENANT_EVENT_HASH_ALGO, "tenant_id": "tenant-b"}, + ] + + assert PostgresStore._event_chain_issues(rows, limit=20, verify_content=False) == [] + rows[-1]["prev_event_hash"] = "wrong-tenant-head" + issues = PostgresStore._event_chain_issues(rows, limit=20, verify_content=False) + assert issues == [ + { + "event_id": 6, + "reason": "broken_prev_link", + "expected_prev_event_hash": "b-1", + "actual_prev_event_hash": "wrong-tenant-head", + } + ] + + +def test_integrity_validation_recomputes_v2_event_content_hash() -> None: + created_at = datetime(2026, 7, 10, tzinfo=timezone.utc) + event_hash = PostgresStore._compute_event_hash( + claim_id=7, + event_type="ingest", + from_status=None, + to_status="candidate", + details="fixture", + payload={"count": 1}, + created_at=created_at, + prev_event_hash=None, + tenant_id="tenant-a", + hash_algo=TENANT_EVENT_HASH_ALGO, + ) + row = { + "id": 1, + "claim_id": 7, + "event_type": "ingest", + "from_status": None, + "to_status": "candidate", + "details": "fixture", + "payload_json": {"count": 1}, + "created_at": created_at, + "prev_event_hash": None, + "event_hash": event_hash, + "hash_algo": TENANT_EVENT_HASH_ALGO, + "tenant_id": "tenant-a", + } + + assert PostgresStore._event_chain_issues([row], limit=20) == [] + row["payload_json"] = {"count": 2} + issues = PostgresStore._event_chain_issues([row], limit=20) + + assert any(issue["reason"] == "event_hash_mismatch" for issue in issues) + + +def test_integrity_validation_recomputes_v1_event_content_hash() -> None: + created_at = datetime(2026, 7, 10, tzinfo=timezone.utc) + event_hash = PostgresStore._compute_event_hash( + claim_id=7, + event_type="ingest", + from_status=None, + to_status="candidate", + details="fixture", + payload={"count": 1}, + created_at=created_at, + prev_event_hash=None, + hash_algo=EVENT_HASH_ALGO, + ) + row = { + "id": 1, + "claim_id": 7, + "event_type": "ingest", + "from_status": None, + "to_status": "candidate", + "details": "fixture", + "payload_json": {"count": 1}, + "created_at": created_at, + "prev_event_hash": None, + "event_hash": event_hash, + "hash_algo": EVENT_HASH_ALGO, + "tenant_id": "tenant-a", + } + + assert PostgresStore._event_chain_issues([row], limit=20) == [] + row["details"] = "tampered" + issues = PostgresStore._event_chain_issues([row], limit=20) + + assert any(issue["reason"] == "event_hash_mismatch" for issue in issues) + + +def test_team_store_allows_only_read_only_tenant_reconciliation() -> None: + store, conn = _postgres_store() + store.connect = lambda: conn + + report = store.reconcile_integrity(fix=False) + + assert report["summary"]["hash_chain_issues"] == 0 + assert report["summary"]["tenant_hash_chain_issues"] == 0 + + with pytest.raises(PermissionError, match="privileged maintenance"): + store.reconcile_integrity(fix=True) + + +def test_postgres_read_only_reconciliation_does_not_run_schema_repairs(monkeypatch) -> None: + store = PostgresStore("postgresql://db.invalid/app") + conn = RecordingConnection() + store._psycopg = (object(), object(), lambda value: value) + monkeypatch.setattr(store, "connect", lambda: conn) + + def fail_if_called(_conn) -> None: + pytest.fail("read-only reconciliation invoked a schema repair helper") + + monkeypatch.setattr(store, "_ensure_event_integrity_schema", fail_if_called) + report = store.reconcile_integrity(fix=False) + + assert report["summary"]["hash_chain_issues"] == 0 + + +def test_tenant_hash_validation_recomputes_content_commitment() -> None: + first_hash = PostgresStore._compute_tenant_event_hash( + tenant_id="tenant-a", + event_hash="global-1", + tenant_prev_event_hash=None, + ) + second_hash = PostgresStore._compute_tenant_event_hash( + tenant_id="tenant-a", + event_hash="global-2", + tenant_prev_event_hash=first_hash, + ) + rows = [ + { + "id": 1, + "tenant_id": "tenant-a", + "event_hash": "global-1", + "tenant_prev_event_hash": None, + "tenant_event_hash": first_hash, + "tenant_hash_algo": TENANT_EVENT_HASH_ALGO, + }, + { + "id": 2, + "tenant_id": "tenant-a", + "event_hash": "global-2", + "tenant_prev_event_hash": first_hash, + "tenant_event_hash": second_hash, + "tenant_hash_algo": TENANT_EVENT_HASH_ALGO, + }, + ] + + assert PostgresStore._tenant_event_chain_issues(rows, limit=20) == [] + rows[1]["event_hash"] = "tampered-global-commitment" + issues = PostgresStore._tenant_event_chain_issues(rows, limit=20) + assert any(issue["reason"] == "tenant_hash_mismatch" for issue in issues) From f3d9edc7773d4def0d8504776a41f1054dc55af2 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 06:45:05 -0300 Subject: [PATCH 11/55] fix(postgres): enforce scoped runtime boundaries --- .env.example | 34 + ...EMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md | 2 +- .planning/audit-remediation-ledger.md | 6 +- INSTALLATION.md | 161 +- README.md | 39 +- docs/deployment_profiles.md | 97 +- external-actions-required.md | 8 +- memorymaster/bridges/db_merge.py | 165 +- memorymaster/bridges/dream_bridge.py | 9 +- memorymaster/core/access_control.py | 82 +- memorymaster/core/service.py | 145 +- memorymaster/govern/auto_resolver.py | 91 +- memorymaster/govern/jobs/daydream_ingest.py | 19 +- memorymaster/govern/jobs/validator.py | 11 +- memorymaster/knowledge/rule_miner.py | 15 +- memorymaster/knowledge/transcript_miner.py | 9 +- memorymaster/knowledge/vault_exporter.py | 15 +- memorymaster/recall/claim_edges.py | 125 +- memorymaster/schema.sql | 71 +- memorymaster/schema_postgres.sql | 95 +- memorymaster/stores/_storage_lifecycle.py | 21 +- memorymaster/stores/_storage_read.py | 103 +- memorymaster/stores/_storage_schema.py | 204 ++- memorymaster/stores/_storage_write_claims.py | 39 +- memorymaster/stores/claim_identity.py | 69 + .../0011_postgres_scoped_force_rls.py | 344 ++++ .../0012_principal_local_claim_identities.py | 213 +++ .../stores/postgres_policy_contract.py | 182 ++ memorymaster/stores/postgres_store.py | 1526 +++++++++++++++-- memorymaster/stores/store_factory.py | 5 + memorymaster/surfaces/cli_helpers.py | 14 +- memorymaster/surfaces/mcp_server.py | 6 +- scripts/__init__.py | 6 + scripts/merge_scope_variants.py | 11 +- tests/test_auto_resolver.py | 19 +- .../test_auto_resolver_atomic_supersession.py | 45 + tests/test_human_id_namespace_consumers.py | 347 ++++ tests/test_identity_bypass_db_merge.py | 236 +++ tests/test_identity_bypass_raw_ingesters.py | 118 ++ tests/test_identity_bypass_scope_merge.py | 185 ++ tests/test_lifecycle_extra.py | 28 + tests/test_mcp_tenant_scope_boundary.py | 4 +- tests/test_postgres_admin_boundary.py | 126 ++ tests/test_postgres_claim_owner_constraint.py | 202 +++ tests/test_postgres_event_head_rls.py | 288 ++++ tests/test_postgres_force_rls_migration.py | 460 +++++ ...est_postgres_policy_expression_contract.py | 244 +++ tests/test_postgres_policy_fingerprint.py | 310 ++++ ...est_postgres_principal_identity_runtime.py | 484 ++++++ tests/test_postgres_rls_integration.py | 918 ++++++++++ tests/test_postgres_runtime_boundary.py | 913 ++++++++++ ...test_postgres_runtime_catalog_integrity.py | 426 +++++ tests/test_postgres_supersession_boundary.py | 489 ++++++ tests/test_postgres_tenant_context.py | 26 +- .../test_principal_local_claim_identities.py | 416 +++++ ...test_principal_local_identity_migration.py | 250 +++ tests/test_request_context.py | 28 +- tests/test_scope_identity_bypass_consumers.py | 330 ++++ tests/test_scope_local_claim_identities.py | 389 +++++ tests/test_service_runtime_authority.py | 415 +++++ tests/test_steward.py | 4 + tests/test_steward_contradiction_phase.py | 4 + tests/test_steward_resolution_parity.py | 4 + tests/test_tenant_event_runtime.py | 16 +- tests/test_tenant_local_claim_identities.py | 14 +- tests/test_tenant_local_identity_migration.py | 23 +- tests/test_v313_e2e.py | 4 + tests/test_v313_run_cycle_dedupe.py | 4 + tests/test_validator_atomic_supersession.py | 50 + 69 files changed, 11222 insertions(+), 539 deletions(-) create mode 100644 memorymaster/stores/claim_identity.py create mode 100644 memorymaster/stores/migrations/0011_postgres_scoped_force_rls.py create mode 100644 memorymaster/stores/migrations/0012_principal_local_claim_identities.py create mode 100644 memorymaster/stores/postgres_policy_contract.py create mode 100644 scripts/__init__.py create mode 100644 tests/test_auto_resolver_atomic_supersession.py create mode 100644 tests/test_human_id_namespace_consumers.py create mode 100644 tests/test_identity_bypass_db_merge.py create mode 100644 tests/test_identity_bypass_raw_ingesters.py create mode 100644 tests/test_identity_bypass_scope_merge.py create mode 100644 tests/test_postgres_admin_boundary.py create mode 100644 tests/test_postgres_claim_owner_constraint.py create mode 100644 tests/test_postgres_event_head_rls.py create mode 100644 tests/test_postgres_force_rls_migration.py create mode 100644 tests/test_postgres_policy_expression_contract.py create mode 100644 tests/test_postgres_policy_fingerprint.py create mode 100644 tests/test_postgres_principal_identity_runtime.py create mode 100644 tests/test_postgres_rls_integration.py create mode 100644 tests/test_postgres_runtime_boundary.py create mode 100644 tests/test_postgres_runtime_catalog_integrity.py create mode 100644 tests/test_postgres_supersession_boundary.py create mode 100644 tests/test_principal_local_claim_identities.py create mode 100644 tests/test_principal_local_identity_migration.py create mode 100644 tests/test_scope_identity_bypass_consumers.py create mode 100644 tests/test_scope_local_claim_identities.py create mode 100644 tests/test_service_runtime_authority.py create mode 100644 tests/test_validator_atomic_supersession.py diff --git a/.env.example b/.env.example index df3ad931..17c668cb 100644 --- a/.env.example +++ b/.env.example @@ -30,6 +30,40 @@ MEMORYMASTER_WORKSPACE=. # Default: unset # MEMORYMASTER_CONFIG_FILE=config.json +# --------------------------------------------------------------------------- +# Deployment authorization +# --------------------------------------------------------------------------- + +# MCP authorization is mandatory. local-trusted is for one private stdio +# process controlled by one OS user and uses SQLite only. PostgreSQL application +# runtime is team-only; do not point a local-trusted process at PostgreSQL. +# Values: local-trusted | team +MEMORYMASTER_MCP_AUTH_MODE=local-trusted + +# Team mode requires every value below. The database target must be the +# non-owner application DSN, never the migrator DSN. Scopes are an explicit, +# comma-separated allowlist; wildcards are rejected. +# MEMORYMASTER_MCP_PRINCIPAL=agent-id +# MEMORYMASTER_ROLE_AGENT_ID=writer +# MEMORYMASTER_MCP_TENANT_ID=tenant-id +# MEMORYMASTER_MCP_WORKSPACE=/absolute/path/to/workspace +# MEMORYMASTER_MCP_ALLOWED_SCOPES=project:example,global +# MEMORYMASTER_MCP_DB=postgresql://app-role:password@host/database + +# Reserved sensitive-read grant. Team mode currently rejects enabled values +# because v0011 intentionally exposes only public or principal-owned private +# rows; keep this at 0 until database policy support is implemented. +# Values: 0 (team mode) +# MEMORYMASTER_MCP_ALLOW_SENSITIVE=0 + +# Disposable PostgreSQL RLS integration test only. The admin DSN must use a +# dedicated SUPERUSER or BYPASSRLS schema owner; the app DSN must use a distinct +# restricted role. Tests refuse known live DSN variables and require the +# explicit disposable opt-in. Never set these to a product database. +# MEMORYMASTER_TEST_POSTGRES_DSN=postgresql://migrator:password@host/test_database +# MEMORYMASTER_TEST_POSTGRES_APP_DSN=postgresql://app-role:password@host/test_database +# MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE=1 + # --------------------------------------------------------------------------- # LLM provider (auto-ingest Stop hook + steward + entity extraction) # --------------------------------------------------------------------------- diff --git a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md index da4b34da..98648f64 100644 --- a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md +++ b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md @@ -43,7 +43,7 @@ Do not describe a blocked profile as production-ready until its gate is demonstr - Prefer an isolated worktree with a dedicated virtual environment. Before testing, prove `memorymaster.__file__` resolves into that worktree; otherwise stop using it and avoid the editable-install import-pin trap. - Use temporary databases and fake/local services until a phase's live-migration gate is approved. - No production credential rotation, push, publish, deployment, or live database mutation without explicit authority. -- A `BLOCKED-POLICY` disposition requires the `audit-decisions` workflow and a complete, approved, unexpired `baseline-policy.md` entry. +- **WARNING LEGAL REVIEW REQUIRED:** a `BLOCKED-POLICY` or `ACCEPT` disposition requires the `audit-decisions` workflow, legal sign-off, and a complete, approved, unexpired `baseline-policy.md` entry. This does not pause remediation. - Schema changes update SQLite, Postgres, versioned migrations, parity tests, and documentation together. - Every security defect gets a failing adversarial test before its implementation fix. - One logical work package per conventional commit. Keep rollback possible after every package. diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 7591f4a1..aaf13016 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -6,9 +6,13 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY`. +> WARNING LEGAL REVIEW REQUIRED — applies to risk acceptance and +> `BLOCKED-POLICY` dispositions only. Do not mark `ACCEPT` without legal sign-off; +> this warning does not pause remediation. + | ID | Primary domain | Severity / exploitability | Summary | Package | Status | Acceptance evidence | |---|---|---|---|---|---|---| -| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | R1.1 complete; Postgres team connections require tenant before driver load and set the tenant GUC on every connection; v0008 adds restrictive tenant policies to 8 claim-owned tables and fail-closed policies to 7 untenantable tables; v0009 makes idempotency keys, human IDs, and confirmed tuples tenant-local across SQLite/Postgres; v0010 is the checksum-frozen authority for a dual event ledger that preserves v1 bytes, validates source/prefix integrity, backfills tenant chains transactionally, supports tenant-bound claimless audit rows, and serializes Postgres appends per tenant; read-only reconciliation recomputes primary and tenant hashes without repair; full non-ML gate: 3,201 passed; privileged-role/scoped-GUC enforcement, remaining direct predicates, and the real two-role/concurrency matrix remain | +| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | Owner: Codex remediation branch. R1.1 complete. PostgreSQL application connections are team-only and bind tenant, principal, and immutable scopes transaction-locally; schema work requires a distinct verified migrator. v0011 FORCE RLS defines restricted role/table/policy and append-only event contracts, a tenant-derived hash-only event-head function, and deny-only governance/raw tables. v0012 defines six partial unique indexes: public keys are tenant + exact-scope local; non-public keys additionally include exact visibility/principal; ambiguity without exact scope fails closed; every team claim requires a nonblank owner. Startup validates literal-sensitive policy/index/function fingerprints, exact event and claims trigger inventories, required event SELECT/INSERT plus forbidden table/column UPDATE and DELETE, a privileged event-head owner, and the strict validated owner constraint. Supersession rejects self/cross-tenant/scope/visibility/owner references and the canonical path atomically commits reciprocal pointers plus one event; v0012 preflights unsafe legacy edges. Hermetic R1.2 matrix: 392 passed, 60 skipped, 1 expected xfail. Full isolated non-ML gate: 3,502 passed, 69 skipped, 95 deselected, 22 intentional xfails, 2 warnings in 861.24s. Commit evidence is the commit containing this row. Rollback: disable the blocked team profile and revert this package; schema rollback requires a verified backup/forward repair. Real two-role PostgreSQL evidence plus approved brownfield owner/duplicate/supersession-edge repair remain `BLOCKED-EXTERNAL`. The separate team-surface denial package remains, so R1.2 stays `IN-PROGRESS`. | | MM-SEC-02 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | Qdrant can return archived, sensitive, cross-scope/tenant, or orphan payloads | R1.3,R2.1 | OPEN | Fake and real Qdrant authoritative-filter tests | | MM-OPS-01 | DevOps | Critical / EXPLOITABLE-LOW-EFFORT / H4 | Postgres Compose publishes a fixed credential | R1.5 | OPEN | Fail-closed Compose config, private port, external rotation evidence | | MM-SEC-03 | Security | High / EXPLOITABLE-NOW | Persisted metadata/provenance fields bypass sensitivity scanning | R1.4 | OPEN | Complete field-matrix adversarial suite and legacy dry-run | diff --git a/INSTALLATION.md b/INSTALLATION.md index 37f02eff..2b6ab652 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -41,12 +41,16 @@ pip install "memorymaster[mcp,postgres,embeddings,gemini,qdrant,security]" pip install -e ".[dev,mcp,security]" ``` -### Initialize the database +### Initialize a local SQLite database ```bash memorymaster --db memory.db init-db ``` +Local-trusted operation is SQLite-only. PostgreSQL initialization is a separate +team-deployment step performed with a dedicated migrator DSN; the PostgreSQL +application role cannot initialize or migrate the schema. + ### Setup (hooks + MCP + cron) After `pip install memorymaster`, run the installer to wire MemoryMaster into @@ -139,12 +143,17 @@ curl http://localhost:8765/health ### Postgres variant -For Postgres instead of SQLite: +The included Postgres Compose file is a development scaffold, not a secure team +deployment by itself: ```bash docker compose -f docker-compose.postgres.yml up -d ``` +Before using it with MemoryMaster, replace any example credential, keep the +database port private, and provision the distinct migrator and application roles +defined in [PostgreSQL team runtime security boundary](#postgresql-team-runtime-security-boundary). + ### Data persistence All data is persisted in Docker volumes: @@ -201,6 +210,7 @@ Add to your project's `.mcp.json` (see `.mcp.json.example`): "env": { "MEMORYMASTER_DEFAULT_DB": "/path/to/memorymaster.db", "MEMORYMASTER_WORKSPACE": "/path/to/your/project", + "MEMORYMASTER_MCP_AUTH_MODE": "local-trusted", "QDRANT_URL": "http://localhost:6333", "OLLAMA_URL": "http://localhost:11434" } @@ -209,6 +219,145 @@ Add to your project's `.mcp.json` (see `.mcp.json.example`): } ``` +This `local-trusted` example is SQLite-only. A PostgreSQL team runtime uses an +explicit authority envelope and the restricted application DSN: + +```json +{ + "mcpServers": { + "memorymaster": { + "command": "memorymaster-mcp", + "env": { + "MEMORYMASTER_MCP_AUTH_MODE": "team", + "MEMORYMASTER_MCP_PRINCIPAL": "agent-id", + "MEMORYMASTER_ROLE_AGENT_ID": "writer", + "MEMORYMASTER_MCP_TENANT_ID": "tenant-id", + "MEMORYMASTER_MCP_WORKSPACE": "/absolute/path/to/workspace", + "MEMORYMASTER_MCP_ALLOWED_SCOPES": "project:example,global", + "MEMORYMASTER_MCP_DB": "postgresql://app-role:password@host/database" + } + } + } +} +``` + +All team values are required, including an explicit `admin`, `writer`, or +`reader` mapping for the principal (`MEMORYMASTER_ROLE_`; use underscores +for hyphens in the environment-key suffix). Scope wildcards and caller-supplied +authority widening are rejected. Do not put the migrator DSN in an MCP +configuration. + +### PostgreSQL team runtime security boundary + +The hardened team profile currently targets PostgreSQL 16.x; other major +versions remain unverified because their role/table privilege catalogs differ. +PostgreSQL is supported only as an authenticated team application runtime. +Keep two purpose-specific DSNs in separate secrets: + +- **Migrator DSN:** a dedicated schema-owning role with `SUPERUSER` or + `BYPASSRLS`, used only for `init-db` and versioned migrations. FORCE RLS makes + a plain table owner subject to policy, so schema ownership alone is not enough. +- **Application DSN:** a distinct non-owner role used by MCP/services. It must be + `NOSUPERUSER NOBYPASSRLS NOREPLICATION NOCREATEROLE NOCREATEDB`, must not be + able to `SET ROLE` into a superuser/BYPASSRLS role, and must not own protected + tables or their owner role. + +The application role also must not have schema `CREATE`, table `TRUNCATE`, +`REFERENCES`, or `TRIGGER`, DDL/migration rights, or DML on the deny-only +governance/raw-ingest tables. Grant it only the DML needed by the scoped runtime, +the corresponding sequence privileges, and `SELECT` (not write) on +`cache_meta` and `schema_versions`. The event ledger is append-only: the +application role requires `SELECT` and `INSERT`, but must have no table- or +column-level `UPDATE` and no `DELETE` privilege on `events`. Grant `EXECUTE` only on +`public.memorymaster_event_chain_head()`; v0011 revokes that capability from +`PUBLIC`. The function derives its tenant from transaction-local authority and +returns only ledger head hashes, never event payloads. Its SECURITY DEFINER +owner must be `SUPERUSER` or `BYPASSRLS`; startup rejects an ordinary owner that +would see a FORCE-RLS-filtered partial head. Team action proposals +and raw merge/sync remain disabled; run reviewed administration through the +separate migrator/maintenance boundary. + +Migration v0011 enables and forces RLS on all 15 protected tables. Each scoped +table receives exact command-specific permissive/restrictive policy pairs: + +- claim reads require tenant + explicit scope and expose public rows or the + authenticated principal's own private rows; +- claim and claim-owned child writes require tenant + scope + principal + ownership, including both endpoints of links/verdicts, require a nonblank + `source_agent` owner on every claim (including public claims), and accept only + public/private visibility in team runtime; +- claimless audit events remain tenant/principal bound; claim events inherit the + referenced claim's read/write boundary, while the hash-only event-head + function prevents private/scope RLS from forking the tenant ledger; +- `mcp_usage` is tenant/principal bound; +- action proposals, Atlas source/evidence tables, media retry, query cache, + miner state, and rule stats are deny-only in team runtime. + +Migration v0012 replaces the three tenant-global identity constraints with six +partial unique indexes. Public idempotency keys, human IDs, and confirmed tuples +use exact tenant + scope namespaces. Non-public identities additionally use +exact visibility + `source_agent`. Runtime startup verifies that the complete +non-primary unique-index catalog is exactly those six definitions; missing, +extra, invalid, nonunique, or differently defined claim identity indexes fail +closed. It also verifies the checksums of v0011 and v0012 before binding +authority. Direct human-ID/idempotency-key reads without an exact scope are +accepted only when one visible row exists; ambiguity fails closed. + +v0012 also installs `trg_claims_supersession_boundary`. It rechecks references +when either pointer or any tenant/scope/visibility/owner boundary field changes, +and denies self- or cross-boundary links without revealing the hidden target. +`mark_superseded()` locks both claims and writes the old status/pointer, the +replacement's reciprocal pointer, and one supersession event in one transaction. +The legacy `set_supersedes()` compatibility method delegates to the same atomic +path. + +Before applying v0012 to an existing deployment, perform a read-only inventory +of noncanonical visibility values, blank/null `source_agent` owners on all claim +rows (including public rows), and duplicate identities inside the tenant + exact +scope namespaces. Also inventory self-linked, missing-target, nonreciprocal, or +cross-tenant/scope/visibility/owner `supersedes_claim_id` and +`replaced_by_claim_id` edges. Do not mutate product data as part of that +inventory. v0012 performs this supersession preflight read-only and refuses DDL +when invalid edges exist. +PostgreSQL adds `ck_claims_identity_visibility_owner` as `NOT VALID` and then +validates it in the same migration. A brownfield migration therefore refuses to +complete while any ownerless row remains; team application startup also rejects +a missing, altered, or unvalidated constraint. Backfilling owners and resolving +duplicates are product-data maintenance actions that require explicit approval, +an approved backup, and a reviewed maintenance window before rerunning v0012. + +The migration also removes the PostgreSQL query-cache generation triggers +(`claims_gen_ins_del`, `claims_gen_upd`) because team runtime cannot safely write +the read-only cache metadata. A runtime connection validates its role, table +ownership/privileges, required event `SELECT`/`INSERT`, the absence of table- or +column-level event `UPDATE` and event `DELETE`, FORCE RLS, literal-sensitive +command/role/expression policy definitions, the exact event-head function +signature/body/security/owner settings, exact event and claims trigger catalogs, +the validated claim-owner constraint, the exact six +identity indexes, and transaction-local tenant, principal, and scope settings +before returning the connection. + +#### Disposable PostgreSQL RLS verification + +Real catalog and policy behavior is intentionally opt-in. Supply both DSNs for +the same disposable database and the explicit opt-in, then run the integration +module: + +```bash +export MEMORYMASTER_TEST_POSTGRES_DSN='postgresql://migrator:...@host/test_database' +export MEMORYMASTER_TEST_POSTGRES_APP_DSN='postgresql://app-role:...@host/test_database' +export MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE=1 +python -m pytest tests/test_postgres_rls_integration.py -q +``` + +The test refuses identical roles and known live DSN variables. Never target +product data. Until this two-DSN suite passes in a real PostgreSQL environment, +the runtime proof and restricted-grant evidence remain `BLOCKED-EXTERNAL`; +fake/catalog unit tests are not a substitute. Brownfield read-only inventory, +owner backfill, duplicate remediation, and constraint validation are separately +blocked pending explicit operator approval and are recorded in +`external-actions-required.md`. + ### With Qdrant MCP server For direct vector search alongside MemoryMaster: @@ -244,6 +393,12 @@ All environment variables are documented in [`.env.example`](.env.example). Key | `MEMORYMASTER_DEFAULT_DB` | `memorymaster.db` | SQLite database path | | `MEMORYMASTER_WORKSPACE` | `.` | Workspace root for file watchers | | `MEMORYMASTER_CONFIG_FILE` | (none) | JSON config file path | +| `MEMORYMASTER_MCP_AUTH_MODE` | (required) | `local-trusted` (SQLite only) or `team` | +| `MEMORYMASTER_MCP_PRINCIPAL` | (none) | Required authenticated principal in team mode | +| `MEMORYMASTER_ROLE_` | (none) | Required explicit `admin`, `writer`, or `reader` mapping for each team principal | +| `MEMORYMASTER_MCP_TENANT_ID` | (none) | Required tenant in team mode | +| `MEMORYMASTER_MCP_ALLOWED_SCOPES` | (none) | Required explicit comma-separated team scope allowlist | +| `MEMORYMASTER_MCP_DB` | (none) | Restricted application DSN/path for team mode | | `OLLAMA_URL` | `http://localhost:11434` | Ollama LLM endpoint | | `QDRANT_URL` | (none) | Qdrant vector store endpoint | | `GEMINI_API_KEY` | (none) | Google Gemini API key | @@ -297,7 +452,7 @@ python -c "import memorymaster; print('OK')" ### Database locked errors SQLite allows only one writer at a time. For concurrent access: -- Use the Postgres backend: `pip install "memorymaster[postgres]"` +- Configure the authenticated [PostgreSQL team runtime security boundary](#postgresql-team-runtime-security-boundary); installing `memorymaster[postgres]` alone is insufficient - Or ensure only one process writes to the database at a time ### Tests failing after install diff --git a/README.md b/README.md index 0e54dc0d..78a1f8da 100644 --- a/README.md +++ b/README.md @@ -77,7 +77,7 @@ recent PR status, and sensitivity-filter invariants. - **Hebbian/Ebbinghaus entity edges** (new in v4.2.0, opt-in `MEMORYMASTER_HEBBIAN_DECAY`): usage strengthens, time decays entity-graph edge weights. - **Proactive + tool-triggered recall** (new in v4.2.0): a `volunteer_context` MCP tool (confidence-gated, zero-LLM) and an opt-in PreToolUse hook (`MEMORYMASTER_PRETOOLUSE_RECALL`) that injects memory as `additionalContext` on Grep/Glob. - **Belief `holder`** (new in v4.2.0): nullable per-claim `holder` for multi-holder beliefs (take/fact/bet/hunch reuse `claim_type`); SQLite+Postgres, ranking-neutral by default. -- **Dual backend**: SQLite (zero-config) and Postgres (full feature parity with pgvector) +- **Dual backend**: SQLite for local-trusted/single-agent use; PostgreSQL for authenticated team runtime with tenant, principal, and explicit-scope isolation - **Dream Bridge** for bidirectional sync with Claude Code's Auto Dream - **Hook stack**: recall, classify, validate-wiki, session-start, auto-ingest, precompact (settings.json) + steward-cycle (cron/schtasks) + opt-in `--pretooluse` grep/glob recall-inject @@ -178,11 +178,13 @@ For zero-cost offline use, install [Ollama](https://ollama.com), `ollama pull ll } ``` -MCP authorization mode is mandatory. Use `local-trusted` only for a private -stdio process controlled by one OS user. Team mode instead requires an -operator-configured principal, explicit role, tenant, database, workspace, and -scope allowlist; unverified host-wide and maintenance tools fail closed. Existing -brownfield MCP entries must add the mode or be regenerated with setup `--force`. +MCP authorization mode is mandatory. Use `local-trusted` only with SQLite in a +private stdio process controlled by one OS user. PostgreSQL application runtime +is team-only and requires an operator-configured principal, tenant, non-owner +application DSN, workspace, and explicit scope allowlist. Schema initialization +uses a distinct migrator DSN/role; never give that role to the MCP runtime. +Unverified host-wide and maintenance tools fail closed. Existing brownfield MCP +entries must add the mode or be regenerated with setup `--force`. 30 MCP tools spanning setup/lifecycle, ingest, query/retrieval, listing, knowledge graph, and governance: `init_db`, `ingest_claim`, `ingest_rule`, `query_rules`, `rules_export`, `run_cycle`, `run_steward`, `classify_query`, `query_memory`, `query_for_context`, `query_for_task`, `query_claim_paths`, `query_meta_decisions`, `federated_query`, `recall_analysis`, `read_active_tasks`, `list_claims`, `redact_claim_payload`, `pin_claim`, `compact_memory`, `list_events`, `search_verbatim`, `open_dashboard`, `list_steward_proposals`, `resolve_steward_proposal`, `extract_entities`, `entity_stats`, `find_related_claims`, `quality_scores`, `recompute_tiers`. @@ -192,8 +194,29 @@ See [`docs/MCP-TOOLS.md`](docs/MCP-TOOLS.md) for the grouped reference (one line | Backend | Install | Use case | |---------|---------|----------| -| **SQLite** | Built-in | Local development, single-agent, zero-config | -| **Postgres** | `pip install "memorymaster[postgres]"` | Team deployment, multi-agent, pgvector search | +| **SQLite** | Built-in | Local-trusted development, single-agent, zero-config | +| **Postgres 16.x** | `pip install "memorymaster[postgres]"` | Authenticated team deployment with isolated app/migrator roles | + +PostgreSQL v0011 enables and forces row-level security. Reads are tenant/scope +bounded and expose public claims or the principal's own private claims; writes +are owner-only, require a nonblank `source_agent` on every team claim, and are +limited to public/private rows. Migration v0012 makes public claim identities +tenant + exact-scope local; non-public idempotency keys, human IDs, and +confirmed tuples additionally include exact visibility and principal. A +tenant-derived hash-only function preserves the event chain across private +principals/scopes without exposing payloads. The application role must read and +append events but cannot update any event column or delete events. Unscoped +human-ID/idempotency-key reads fail when an identifier is ambiguous across +scopes. The supersession guard denies self- and cross-tenant/scope/visibility/ +principal links; the canonical lifecycle locks both claims and commits reciprocal +pointers plus one event atomically. Startup rejects drift in exact policy, +function, trigger, privilege, and identity-index catalogs. Brownfield +owner/duplicate/unsafe-supersession repair requires a reviewed external +maintenance action. Team action proposals and raw merge/sync paths remain disabled. +See [INSTALLATION.md](INSTALLATION.md#postgresql-team-runtime-security-boundary) +and [deployment profiles](docs/deployment_profiles.md) before enabling this +backend. Real PostgreSQL verification requires two distinct DSNs targeting a +disposable database; repository tests do not constitute a production proof. ## Docker Compose diff --git a/docs/deployment_profiles.md b/docs/deployment_profiles.md index 5ae48846..2b827041 100644 --- a/docs/deployment_profiles.md +++ b/docs/deployment_profiles.md @@ -11,6 +11,8 @@ Use when: Runtime: - DB: local SQLite file (`memorymaster.db`) - process: CLI/operator loop on same host +- MCP auth: `MEMORYMASTER_MCP_AUTH_MODE=local-trusted` in a private stdio process +- PostgreSQL is not supported in this profile; use Profile B/C with team authority Recommended commands: ```powershell @@ -26,7 +28,7 @@ python scripts/eval_memorymaster.py --strict python benchmarks/perf_smoke.py ``` -## Profile B: Small Team Server (SQLite or Postgres) +## Profile B: Small Team Server (Postgres) Use when: - 2-10 engineers @@ -34,13 +36,16 @@ Use when: - moderate throughput Runtime: -- preferred DB: Postgres for concurrency/durability -- fallback DB: SQLite when write contention is low +- DB: Postgres for concurrency/durability - process model: one long-running operator, optional dashboard process +- MCP auth: `team`, with a tenant, authenticated principal, non-wildcard scope allowlist, and restricted application DSN +- schema lifecycle: separate migrator DSN; the application runtime cannot initialize or migrate Baseline controls: - backup policy for DB and artifacts - health endpoint checks (`/health`) +- distinct migrator/application roles meeting the contract below +- team action proposals and raw merge/sync disabled - periodic reconciliation report: - `service.store.reconcile_integrity(fix=False)` in scheduled job @@ -55,16 +60,92 @@ Runtime: - DB: managed Postgres with TLS + backups + PITR - application nodes: stateless operator workers - observability: centralized logs + metric exporter (future D3) +- identity: each runtime request binds tenant, principal, and explicit scopes transaction-locally Recommended controls: - rotate credentials and isolate DB role permissions +- keep migrator credentials out of application nodes and MCP configuration - scheduled integrity reconciliation (`report` daily, `fix` only with review) - retain artifacts for audit windows (`artifacts/eval`, `artifacts/perf`, `artifacts/e2e`) +## PostgreSQL Role and RLS Contract + +This contract currently targets PostgreSQL 16.x. Treat other major versions as +unverified until their catalog and privilege matrix passes the disposable gate. + +Provision two purpose-specific secrets: + +- Migrator: distinct schema owner with `SUPERUSER` or `BYPASSRLS`, used only for + initialization/versioned migrations. FORCE RLS means an ordinary owner alone + is not a sufficient migration boundary. +- Application: non-owner with + `NOSUPERUSER NOBYPASSRLS NOREPLICATION NOCREATEROLE NOCREATEDB`; no ability to + `SET ROLE` into a superuser/BYPASSRLS role; no schema `CREATE`; no table + `TRUNCATE`, `REFERENCES`, or `TRIGGER`; no DDL/migration rights; no DML on the + deny-only governance/raw-ingest tables; `SELECT`-only on `cache_meta` and + `schema_versions`; `SELECT` + `INSERT` but no table/column `UPDATE` and no + `DELETE` on append-only `events`; and + explicit `EXECUTE` only on the hash-only + `public.memorymaster_event_chain_head()` function (never through `PUBLIC`). + +Migration v0011 enables and forces RLS on all 15 protected tables. Scoped tables +use exact command-specific permissive/restrictive policy pairs. Reads are +tenant/scope-bound and return public claims or the principal's own private +claims. Writes to claims and claim-owned rows are owner-only and limited to +public/private visibility; every team claim, including a public claim, requires +a nonblank `source_agent` owner. `mcp_usage` and +claimless audit events remain tenant/principal-bound. Action proposals, Atlas +source/evidence tables, media retry, query cache, miner state, and rule stats are +deny-only. A tenant-derived hash-only function preserves the event chain across +private principals/scopes without exposing payloads. PostgreSQL cache-generation +triggers are dropped because query-cache metadata is not writable in team runtime. + +Migration v0012 makes public idempotency keys, human IDs, and confirmed tuples +tenant + exact-scope local. Non-public identities additionally include exact +visibility and principal. Startup requires an exact six-index catalog with no +extra non-primary unique claim indexes, checksum-frozen v0011/v0012 migrations, +the validated claim-owner constraint, exact policy expressions/commands/roles, +the exact event-head function (owned by `SUPERUSER`/`BYPASSRLS`) and append-only +trigger/privilege contract, and the +restricted role catalog. Existing rows with blank/null owners (public included), +noncanonical visibility, or namespace duplicates require a read-only inventory. +The exact claims-trigger catalog rejects self- and cross-tenant/scope/visibility/ +owner supersession links, including boundary-field changes. The canonical +lifecycle locks both rows and commits reciprocal pointers plus one event +atomically. Existing invalid or nonreciprocal supersession edges are part of the +read-only preflight. Owner backfill, duplicate/supersession-edge remediation, and +constraint validation require +explicit approval plus an approved backup and maintenance window. + +The local-trusted profile remains SQLite-only. Team action proposals and raw +merge/sync are not enabled by selecting PostgreSQL. + +## Disposable Integration Gate + +Real PostgreSQL verification requires a database whose full lifecycle is +disposable and all three variables below: + +```text +MEMORYMASTER_TEST_POSTGRES_DSN= +MEMORYMASTER_TEST_POSTGRES_APP_DSN= +MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE=1 +``` + +Run `python -m pytest tests/test_postgres_rls_integration.py -q`. Until that +two-role test passes, catalog behavior, cross-tenant read/write denial, event +owner/grants, and atomic supersession remain `BLOCKED-EXTERNAL`; unit/fake +results are not production evidence. +Brownfield inventory/backfill/constraint-validation evidence is a separate +external operator action and must never be inferred from the disposable test. + ## Rollout Checklist -1. Validate DB connectivity and schema init. -2. Run operator smoke against representative inbox rows. -3. Run `scripts/eval_memorymaster.py --strict`. -4. Run `benchmarks/perf_smoke.py`. -5. Capture reconciliation report and confirm zero critical findings before go-live. +1. Initialize/migrate with the dedicated migrator DSN; remove it from runtime nodes. +2. Verify the application role contract, FORCE RLS, exact + policy/event/supersession/identity catalogs, the validated owner constraint, + append-only event privileges, and canonical supersession transaction. +3. Validate team connectivity with tenant, principal, and explicit scopes. +4. Run the disposable two-DSN integration gate before any production rollout. +5. Run operator smoke against representative non-product inbox rows. +6. Run `scripts/eval_memorymaster.py --strict` and `benchmarks/perf_smoke.py`. +7. Capture reconciliation report and confirm zero critical findings before go-live. diff --git a/external-actions-required.md b/external-actions-required.md index 8f025289..57204751 100644 --- a/external-actions-required.md +++ b/external-actions-required.md @@ -1,11 +1,15 @@ # External Actions Required -Items here are `BLOCKED-EXTERNAL` only when the remediation ledger explicitly marks them that way. Repository work continues independently. +An external sub-action may be `BLOCKED-EXTERNAL` while repository remediation +for the canonical finding remains `IN-PROGRESS`. The finding itself becomes +blocked only after all unblocked repository work closes. No row authorizes a +live mutation by itself. | Finding | Owner/system | Required action | Evidence needed to unblock | Review date | Status | |---|---|---|---|---|---| | MM-OPS-01 | Operator / any Postgres deployment | Rotate the historical `mm_pw` credential and recreate deployments that used it. Keep database ports private. | Rotation record plus network probe showing the port is not externally reachable | 2026-07-31 | PENDING-INVENTORY | -| MM-SEC-01 | Team Postgres test environment | Provide a disposable admin/migrator DSN in `MEMORYMASTER_TEST_POSTGRES_DSN` and a distinct non-owner `NOSUPERUSER NOBYPASSRLS` application DSN in `MEMORYMASTER_TEST_POSTGRES_APP_DSN`; neither may target product data. | Catalog assertions plus the real cross-tenant read/write/constraint matrix; app role has no table ownership, `public` CREATE, TRUNCATE, DDL, or migration rights; verify claimless event visibility, non-UTC hash reconciliation, concurrent same-tenant append linearity, and writer-vs-migration lock/rollback behavior | 2026-07-31 | BLOCKED-EXTERNAL | +| MM-SEC-01 / disposable proof | Team Postgres test environment | Provide one fully disposable database, a dedicated schema-owning `SUPERUSER`/`BYPASSRLS` migrator DSN in `MEMORYMASTER_TEST_POSTGRES_DSN`, and a distinct restricted app DSN in `MEMORYMASTER_TEST_POSTGRES_APP_DSN`; set `MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE=1`. Neither DSN may target product/live data. | Run `tests/test_postgres_rls_integration.py` and retain exact FORCE-RLS policy/function/trigger/index catalog evidence; literal-sensitive fingerprints; event-head owner `SUPERUSER`/`BYPASSRLS`; app event SELECT/INSERT with no table/column UPDATE or DELETE; cross-tenant/scope/visibility/principal denials; tenant-chain linearity; scope-local identities and ambiguity rejection; self/cross-boundary supersession denial; successful reciprocal supersession plus one event; and rollback after replacement-update/event-insert failure. No real PostgreSQL proof has been performed here. | 2026-07-31 | BLOCKED-EXTERNAL | +| MM-SEC-01 / brownfield data | Live database operator | Authorize a read-only inventory before v0012. Any owner backfill, duplicate/supersession-edge repair, or grant change requires separate explicit approval, a verified backup, and a maintenance window. | Inventory noncanonical visibility; blank/null `source_agent` on every claim; identity duplicates; both supersession columns for self, missing-target, nonreciprocal, cross-tenant/scope/visibility/owner edges; current event/function/table grants; and constraint state without writing. After approval, retain repair/grant evidence and rerun v0012, which validates `ck_claims_identity_visibility_owner`. No product-data verification or mutation has been performed here. | 2026-07-31 | BLOCKED-EXTERNAL | | MM-OPS-02 | Docker/Helm runtime | Provide Docker and, for final verification, a disposable Kubernetes target if unavailable locally. | Container health/MCP handshake and Helm readiness/network-policy evidence | 2026-08-15 | PENDING-CAPABILITY-CHECK | | MM-SEC-02 | Qdrant runtime | Provide a disposable authenticated/TLS Qdrant target for final parity after fake-backed tests pass. | Real service policy/reconciliation test output | 2026-08-15 | PENDING-CAPABILITY-CHECK | | MM-DATA-01 | Live MemoryMaster operator | Approve a consistent backup/restore drill before any live migration, redaction, backlog, or retention operation. | Restored backup, integrity check, counts/checksums, approval record | 2026-08-15 | BLOCKED-EXTERNAL | diff --git a/memorymaster/bridges/db_merge.py b/memorymaster/bridges/db_merge.py index e40c92fc..e7f1d4aa 100644 --- a/memorymaster/bridges/db_merge.py +++ b/memorymaster/bridges/db_merge.py @@ -114,6 +114,83 @@ def _target_columns(tgt: sqlite3.Connection) -> set[str]: return {col[1] for col in tgt.execute("PRAGMA table_info(claims)").fetchall()} +def _sqlite_literal_default(value: object | None) -> object | None: + if value is None: + return None + raw = str(value).strip() + if raw.upper() == "NULL": + return None + if len(raw) >= 2 and raw[0] == raw[-1] and raw[0] in {"'", '"'}: + quote = raw[0] + return raw[1:-1].replace(quote * 2, quote) + return raw + + +def _target_identity_defaults(tgt: sqlite3.Connection) -> dict[str, object | None]: + identity_columns = {"tenant_id", "scope", "visibility", "source_agent"} + defaults = { + str(row[1]): _sqlite_literal_default(row[4]) + for row in tgt.execute("PRAGMA table_info(claims)").fetchall() + if str(row[1]) in identity_columns + } + defaults.setdefault("visibility", "public") + return defaults + + +IdentityNamespace = tuple[object | None, object, str, object | None] + + +def _identity_namespace( + row: sqlite3.Row | dict[str, object], + *, + available_columns: set[str] | None = None, + defaults: dict[str, object | None] | None = None, +) -> IdentityNamespace: + keys = set(row.keys()) + usable = keys if available_columns is None else keys & available_columns + defaults = defaults or {} + tenant_id = row["tenant_id"] if "tenant_id" in usable else defaults.get("tenant_id") + scope = row["scope"] if "scope" in usable else defaults.get("scope") + visibility = ( + str(row["visibility"] or "public") + if "visibility" in usable + else str(defaults.get("visibility") or "public") + ) + source_agent = ( + row["source_agent"] + if "source_agent" in usable + else defaults.get("source_agent") + ) + return tenant_id, scope, visibility.strip().lower(), ( + source_agent if visibility.strip().lower() != "public" else None + ) + + +def _identity_where( + namespace: IdentityNamespace, + *, + alias: str = "", + available_columns: set[str] | None = None, +) -> tuple[str, tuple]: + required = {"tenant_id", "scope", "visibility", "source_agent"} + if available_columns is not None and not required.issubset(available_columns): + return "1 = 1", () + prefix = f"{alias}." if alias else "" + tenant_id, scope, visibility, source_agent = namespace + if visibility == "public": + return ( + f"{prefix}tenant_id IS ? AND {prefix}scope IS ? " + f"AND {prefix}visibility = 'public'", + (tenant_id, scope), + ) + return ( + f"{prefix}tenant_id IS ? AND {prefix}scope IS ? " + f"AND {prefix}visibility = ? " + f"AND {prefix}source_agent IS ?", + (tenant_id, scope, visibility, source_agent), + ) + + def _find_conflicting_target_claims( tgt: sqlite3.Connection, row: sqlite3.Row, target_cols: set[str] ) -> list[dict[str, object]]: @@ -122,16 +199,27 @@ def _find_conflicting_target_claims( return [] if row["subject"] is None or row["predicate"] is None: return [] + identity_sql, identity_params = _identity_where( + _identity_namespace(row, available_columns=target_cols), + available_columns=target_cols, + ) conflicts = tgt.execute( - """ + f""" SELECT * FROM claims WHERE status != 'archived' AND COALESCE(subject, '') = COALESCE(?, '') AND COALESCE(predicate, '') = COALESCE(?, '') AND COALESCE(scope, '') = COALESCE(?, '') AND COALESCE(object_value, '') != COALESCE(?, '') + AND {identity_sql} """, - (row["subject"], row["predicate"], row["scope"], row["object_value"]), + ( + row["subject"], + row["predicate"], + row["scope"], + row["object_value"], + *identity_params, + ), ).fetchall() return [dict(conflict) for conflict in conflicts] @@ -140,10 +228,19 @@ def _find_existing_target_claim( tgt: sqlite3.Connection, ikey: object, text: str, - hash_to_id: dict[str, int] | None = None, + namespace: IdentityNamespace, + target_cols: set[str], + hash_to_id: dict[tuple[IdentityNamespace, str], int] | None = None, ) -> dict[str, object] | None: + identity_sql, identity_params = _identity_where( + namespace, + available_columns=target_cols, + ) if ikey: - row = tgt.execute("SELECT * FROM claims WHERE idempotency_key = ?", (ikey,)).fetchone() + row = tgt.execute( + f"SELECT * FROM claims WHERE idempotency_key = ? AND {identity_sql}", + (ikey, *identity_params), + ).fetchone() if row: return dict(row) @@ -151,13 +248,15 @@ def _find_existing_target_claim( # Indexed primary-key lookup via a precomputed {text_hash: id} map avoids the # O(n) full-table scan per source row (which made the merge O(n^2) overall). if hash_to_id is not None: - claim_id = hash_to_id.get(text_hash) + claim_id = hash_to_id.get((namespace, text_hash)) if claim_id is None: return None row = tgt.execute("SELECT * FROM claims WHERE id = ?", (claim_id,)).fetchone() return dict(row) if row else None - for row in tgt.execute("SELECT * FROM claims").fetchall(): + for row in tgt.execute( + f"SELECT * FROM claims WHERE {identity_sql}", identity_params + ).fetchall(): if _text_hash(row["text"]) == text_hash: return dict(row) return None @@ -349,25 +448,37 @@ def merge_databases(target_db: str, source_db: str) -> dict[str, int]: try: # Refuse to import rows the target's CHECK constraints may forbid. _check_schema_compatibility(src, tgt) + tgt_cols = _target_columns(tgt) + identity_defaults = _target_identity_defaults(tgt) # Build set of existing claim fingerprints in target. The {text_hash: id} # map lets reconciliation look claims up by primary key instead of # re-scanning the whole table per source row (the old O(n^2) cost). - existing_keys: set[str] = set() - existing_hashes: set[str] = set() - hash_to_id: dict[str, int] = {} - - for row in tgt.execute("SELECT id, idempotency_key, text FROM claims").fetchall(): + existing_keys: set[tuple[IdentityNamespace, str]] = set() + existing_hashes: set[tuple[IdentityNamespace, str]] = set() + hash_to_id: dict[tuple[IdentityNamespace, str], int] = {} + + identity_columns = [ + column + for column in ("tenant_id", "scope", "visibility", "source_agent") + if column in tgt_cols + ] + select_columns = ", ".join(("id", "idempotency_key", "text", *identity_columns)) + for row in tgt.execute(f"SELECT {select_columns} FROM claims").fetchall(): + namespace = _identity_namespace( + row, + available_columns=tgt_cols, + defaults=identity_defaults, + ) if row["idempotency_key"]: - existing_keys.add(row["idempotency_key"]) + existing_keys.add((namespace, str(row["idempotency_key"]))) thash = _text_hash(row["text"]) - existing_hashes.add(thash) - hash_to_id.setdefault(thash, int(row["id"])) + existing_hashes.add((namespace, thash)) + hash_to_id.setdefault((namespace, thash), int(row["id"])) # Get all columns from source claims table src_cols = [col[1] for col in src.execute("PRAGMA table_info(claims)").fetchall()] # Filter to columns that exist in target - tgt_cols = _target_columns(tgt) common_cols = [c for c in src_cols if c in tgt_cols and c != "id"] # Scan source claims @@ -379,10 +490,24 @@ def merge_databases(target_db: str, source_db: str) -> dict[str, int]: stats["scanned"] += 1 ikey = row["idempotency_key"] if "idempotency_key" in row.keys() else None text = row["text"] + namespace = _identity_namespace( + row, + available_columns=tgt_cols, + defaults=identity_defaults, + ) + identity_key = (namespace, str(ikey)) if ikey else None + hash_key = (namespace, _text_hash(text)) # Reconcile duplicates deterministically instead of letting merge order win. - if (ikey and ikey in existing_keys) or _text_hash(text) in existing_hashes: - existing_claim = _find_existing_target_claim(tgt, ikey, text, hash_to_id) + if (identity_key and identity_key in existing_keys) or hash_key in existing_hashes: + existing_claim = _find_existing_target_claim( + tgt, + ikey, + text, + namespace, + tgt_cols, + hash_to_id, + ) if existing_claim: _reconcile_existing_claim(tgt, row, existing_claim, tgt_cols) stats["skipped"] += 1 @@ -397,10 +522,10 @@ def merge_databases(target_db: str, source_db: str) -> dict[str, int]: new_id = _insert_claim_into_target(row, common_cols, ikey, text, src, tgt) if new_id is not None: _apply_conflict_resolution(tgt, row, new_id, tgt_cols, conflicts) - existing_keys.add(ikey) + existing_keys.add((namespace, str(ikey))) thash = _text_hash(text) - existing_hashes.add(thash) - hash_to_id.setdefault(thash, new_id) + existing_hashes.add((namespace, thash)) + hash_to_id.setdefault((namespace, thash), new_id) stats["merged"] += 1 else: stats["errors"] += 1 diff --git a/memorymaster/bridges/dream_bridge.py b/memorymaster/bridges/dream_bridge.py index 9b91ec30..fb032036 100644 --- a/memorymaster/bridges/dream_bridge.py +++ b/memorymaster/bridges/dream_bridge.py @@ -702,7 +702,14 @@ def dream_ingest( # Check for duplicates by looking for source marker existing = conn.execute( - "SELECT id FROM claims WHERE idempotency_key = ? LIMIT 1", + """ + SELECT id FROM claims + WHERE idempotency_key = ? + AND visibility = 'public' + AND tenant_id IS NULL + AND scope = 'project' + LIMIT 1 + """, (claim["source_marker"],), ).fetchone() if existing: diff --git a/memorymaster/core/access_control.py b/memorymaster/core/access_control.py index ec629f38..e1346cc9 100644 --- a/memorymaster/core/access_control.py +++ b/memorymaster/core/access_control.py @@ -48,7 +48,7 @@ class RequestContext: role: Role tenant_id: str | None workspace: str - allowed_scopes: tuple[str, ...] + allowed_scopes: frozenset[str] allow_sensitive: bool db_target: str @@ -126,13 +126,59 @@ def _team_value(env: Mapping[str, str], name: str) -> str: return value -def _parse_team_scopes(raw: str) -> tuple[str, ...]: - scopes = tuple(dict.fromkeys(part.strip() for part in raw.split(",") if part.strip())) - if not scopes or "*" in scopes: +def _parse_team_scopes(raw: str) -> frozenset[str]: + scopes = frozenset(part.strip() for part in raw.split(",") if part.strip()) + if not scopes or any("*" in scope for scope in scopes): raise PermissionError("Team MCP authorization requires explicit non-wildcard scopes.") return scopes +def _local_trusted_context( + env: Mapping[str, str], + db_target: str, + workspace: str, +) -> RequestContext: + if _is_postgres_target(db_target, env): + raise PermissionError( + "Local-trusted MCP mode is SQLite-only; Postgres requires team authority." + ) + return RequestContext( + mode=AuthMode.LOCAL_TRUSTED, + principal=str(env.get("MEMORYMASTER_MCP_PRINCIPAL", "")).strip() + or "mcp-session", + role=Role.ADMIN, + tenant_id=None, + workspace=str(workspace or "").strip(), + allowed_scopes=frozenset(), + allow_sensitive=True, + db_target=str(db_target or "").strip(), + ) + + +def _team_context(env: Mapping[str, str]) -> RequestContext: + principal = _team_value(env, "MEMORYMASTER_MCP_PRINCIPAL") + role = get_configured_role(principal) + if role is None: + raise PermissionError("Team MCP principal has no explicitly configured role.") + sensitive = str(env.get("MEMORYMASTER_MCP_ALLOW_SENSITIVE", "")).strip().lower() + if sensitive in {"1", "true", "yes", "on"}: + raise PermissionError( + "Sensitive reads remain disabled in team mode until database policy support exists." + ) + return RequestContext( + mode=AuthMode.TEAM, + principal=principal, + role=role, + tenant_id=_team_value(env, "MEMORYMASTER_MCP_TENANT_ID"), + workspace=_team_value(env, "MEMORYMASTER_MCP_WORKSPACE"), + allowed_scopes=_parse_team_scopes( + _team_value(env, "MEMORYMASTER_MCP_ALLOWED_SCOPES") + ), + allow_sensitive=False, + db_target=_team_value(env, "MEMORYMASTER_MCP_DB"), + ) + + def resolve_request_context( *, db_target: str = "", @@ -150,32 +196,8 @@ def resolve_request_context( raise PermissionError("MEMORYMASTER_MCP_AUTH_MODE must be local-trusted or team.") from exc if mode is AuthMode.LOCAL_TRUSTED: - return RequestContext( - mode=mode, - principal=str(env.get("MEMORYMASTER_MCP_PRINCIPAL", "")).strip() or "mcp-session", - role=Role.ADMIN, - tenant_id=None, - workspace=str(workspace or "").strip(), - allowed_scopes=(), - allow_sensitive=True, - db_target=str(db_target or "").strip(), - ) - - principal = _team_value(env, "MEMORYMASTER_MCP_PRINCIPAL") - role = get_configured_role(principal) - if role is None: - raise PermissionError("Team MCP principal has no explicitly configured role.") - return RequestContext( - mode=mode, - principal=principal, - role=role, - tenant_id=_team_value(env, "MEMORYMASTER_MCP_TENANT_ID"), - workspace=_team_value(env, "MEMORYMASTER_MCP_WORKSPACE"), - allowed_scopes=_parse_team_scopes(_team_value(env, "MEMORYMASTER_MCP_ALLOWED_SCOPES")), - allow_sensitive=str(env.get("MEMORYMASTER_MCP_ALLOW_SENSITIVE", "")).strip().lower() - in {"1", "true", "yes", "on"}, - db_target=_team_value(env, "MEMORYMASTER_MCP_DB"), - ) + return _local_trusted_context(env, db_target, workspace) + return _team_context(env) _request_context: ContextVar[RequestContext | None] = ContextVar( diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index 6511514d..0a826657 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -3,7 +3,7 @@ from collections.abc import Mapping from dataclasses import replace from pathlib import Path -from typing import Any +from typing import Any, Iterable import logging import os @@ -20,7 +20,16 @@ from memorymaster.core.config import get_config from memorymaster.recall.retrieval import VectorSearchHook, _tier_bonus, rank_claim_rows from memorymaster.core.security import is_sensitive_claim, resolve_allow_sensitive_access, sanitize_claim_input -from memorymaster.core.intake_policy import IntakeRejected, evaluate_intake +from memorymaster.core.intake_policy import ( + IntakePolicyConfig, + IntakeRejected, + evaluate_intake, +) +from memorymaster.stores.claim_identity import ( + normalize_claim_identity, + normalize_claim_visibility, + normalize_source_agent, +) from memorymaster.stores.store_factory import create_store from memorymaster.stores._storage_shared import ConcurrentModificationError import contextlib @@ -359,6 +368,8 @@ def __init__( tenant_id: str | None = None, read_only: bool = False, require_tenant: bool = False, + principal: str | None = None, + allowed_scopes: Iterable[str] | None = None, ) -> None: # read_only (P1 WAL-discipline, spec §2.2): SQLite store opens # mode=ro + query_only connections; _record_accesses spools its @@ -366,11 +377,21 @@ def __init__( # recall hook under MEMORYMASTER_WAL_DISCIPLINE=1. self.tenant_id = (tenant_id or "").strip() or None self.require_tenant = bool(require_tenant) + self.principal = (principal or "").strip() or None + self.allowed_scopes = frozenset( + scope.strip() + for scope in (allowed_scopes or ()) + if scope and scope.strip() + ) + if self.require_tenant: + self._require_bound_authority() self.store = create_store( db_target, read_only=read_only, tenant_id=self.tenant_id, require_tenant=self.require_tenant, + principal=self.principal, + allowed_scopes=self.allowed_scopes, ) self.workspace_root = Path(workspace_root) if workspace_root else Path.cwd() self._embedding_provider: EmbeddingProvider | None = None @@ -442,6 +463,10 @@ def _qdrant_post_cycle_sync(self) -> None: logger.warning("Qdrant post-cycle sync: %d claims failed", failed) def init_db(self) -> None: + if getattr(self, "require_tenant", False): + raise PermissionError( + "Team runtime services cannot initialize or migrate database schema." + ) self.store.init_db() def _revive_archived_dedup_match(self, claim: Claim, source_agent: str | None) -> Claim: @@ -458,6 +483,10 @@ def _revive_archived_dedup_match(self, claim: Claim, source_agent: str | None) - """ if claim.status != "archived": return claim + owner = normalize_source_agent(claim.source_agent) + actor = normalize_source_agent(source_agent) + if owner is not None and owner != actor: + return claim try: revived = self.store.apply_status_transition( claim, @@ -501,6 +530,12 @@ def ingest( ) -> Claim: if not text.strip(): raise ValueError("Claim text cannot be empty.") + visibility, source_agent = self._prepare_ingest_identity( + scope, + visibility, + source_agent, + require_source_agent=require_source_agent, + ) # Bitemporal write-time guard: reject malformed ISO-8601 or an inverted # validity interval at the boundary, before any dedup/sanitize work, so # a durable-but-invisible row (valid_until < valid_from) never reaches @@ -533,6 +568,9 @@ def ingest( existing_claim = self.store.get_claim_by_idempotency_key( normalized_idempotency_key, tenant_id=self.tenant_id, + scope=scope, + visibility=visibility, + source_agent=source_agent, ) if existing_claim is not None: observability.bump_claim_ingested(source_agent) @@ -547,6 +585,9 @@ def ingest( existing_by_hash = self.store.get_claim_by_idempotency_key( content_hash, tenant_id=self.tenant_id, + scope=scope, + visibility=visibility, + source_agent=source_agent, ) if existing_by_hash is not None: observability.bump_claim_ingested(source_agent) @@ -604,6 +645,11 @@ def ingest( new_source_agent = decision.mutated_fields.get("source_agent") if isinstance(new_source_agent, str): source_agent = new_source_agent + visibility, source_agent = normalize_claim_identity( + visibility, + source_agent, + allow_sensitive=not getattr(self, "require_tenant", False), + ) # Resolve subject → canonical entity (GBrain-inspired entity registry) # and mine text for pattern-based entities (#127 Wave 3). entity_id = 0 @@ -980,6 +1026,85 @@ def _normalize_scope_allowlist(scope_allowlist: list[str] | None) -> list[str] | deduped.append(scope) return deduped + def _require_bound_authority(self) -> tuple[str, frozenset[str]]: + if self.tenant_id is None: + raise PermissionError("Team service authority requires a tenant context.") + if self.principal is None: + raise PermissionError("Team service authority requires an authenticated principal.") + if not self.allowed_scopes: + raise PermissionError("Team service authority requires explicit allowed scopes.") + if any("*" in scope for scope in self.allowed_scopes): + raise PermissionError("Team service authority cannot contain wildcard scopes.") + return self.principal, self.allowed_scopes + + def _effective_scope_allowlist( + self, + scope_allowlist: list[str] | None, + ) -> list[str] | None: + normalized = self._normalize_scope_allowlist(scope_allowlist) + if not getattr(self, "require_tenant", False): + return normalized + _, allowed = self._require_bound_authority() + if normalized is None: + return sorted(allowed) + unauthorized = set(normalized) - allowed + if unauthorized: + raise PermissionError("Requested scope allowlist exceeds bound service authority.") + return normalized + + def _effective_requesting_agent(self, requesting_agent: str | None) -> str | None: + if not getattr(self, "require_tenant", False): + return requesting_agent + principal, _ = self._require_bound_authority() + requested = (requesting_agent or "").strip() or None + if requested is not None and requested != principal: + raise PermissionError("requesting_agent cannot substitute the bound principal.") + return principal + + def _effective_ingest_source( + self, + scope: str, + source_agent: str | None, + ) -> str | None: + if not getattr(self, "require_tenant", False): + return source_agent + principal, allowed = self._require_bound_authority() + if scope not in allowed: + raise PermissionError("Claim scope exceeds bound service authority.") + requested = (source_agent or "").strip() or None + if requested is not None and requested != principal: + raise PermissionError("source_agent cannot substitute the bound principal.") + return principal + + def _prepare_ingest_identity( + self, + scope: str, + visibility: str | None, + source_agent: str | None, + *, + require_source_agent: bool, + ) -> tuple[str, str | None]: + source = self._effective_ingest_source(scope, source_agent) + normalized_visibility = normalize_claim_visibility(visibility) + if normalize_source_agent(source) is None: + config = IntakePolicyConfig.from_env() + strict_external = ( + config.require_source_agent == "strict" and require_source_agent + ) + if strict_external: + raise IntakeRejected( + "source_agent is required for explicit ingest calls.", + rule="source_agent", + reason="missing_source_agent", + ) + if config.require_source_agent != "off" and not strict_external: + source = config.default_source_agent or "unknown" + return normalize_claim_identity( + normalized_visibility, + source, + allow_sensitive=not getattr(self, "require_tenant", False), + ) + @staticmethod def _annotation_for_claim(claim: Claim) -> dict[str, object]: return { @@ -1080,6 +1205,8 @@ def query_rows( if limit <= 0: return [] + requesting_agent = self._effective_requesting_agent(requesting_agent) + # RBAC check if requesting_agent: from memorymaster.core.access_control import require_permission @@ -1092,7 +1219,7 @@ def query_rows( ) statuses = self._build_query_statuses(include_stale, include_conflicted, include_candidates) - normalized_scopes = self._normalize_scope_allowlist(scope_allowlist) + normalized_scopes = self._effective_scope_allowlist(scope_allowlist) # Intent-aware ranking (plan 1.3): retrieval_profile="auto" derives the # weight profile from the query's intent (explicit query_type if given, # else rule-based classification). Opt-in only — any other value (incl. @@ -1756,6 +1883,8 @@ def list_claims( scope_allowlist: list[str] | None = None, requesting_agent: str | None = None, ) -> list[Claim]: + requesting_agent = self._effective_requesting_agent(requesting_agent) + scope_allowlist = self._effective_scope_allowlist(scope_allowlist) include_sensitive = self._allow_sensitive( allow_sensitive=allow_sensitive, context="service.list_claims", @@ -2106,10 +2235,18 @@ def query_claim_paths( underlying BFS visited-set. If ``claim_links`` is missing/empty the result is simply empty (logged, no crash). """ + requesting_agent = self._effective_requesting_agent(requesting_agent) + scope_allowlist = self._effective_scope_allowlist(scope_allowlist) + identity_scope = ( + scope_allowlist[0] + if scope_allowlist is not None and len(scope_allowlist) == 1 + else None + ) try: start_id = self.store.resolve_claim_id( claim_id, tenant_id=self.tenant_id, + scope=identity_scope, ) except ValueError: logger.info("query_claim_paths: unknown claim_id %r", claim_id) @@ -2201,7 +2338,7 @@ def federated_query( searches everything. """ normalized_current_scope = (current_scope or "").strip() or None - normalized_scope_allowlist = self._normalize_scope_allowlist(scope_allowlist) + normalized_scope_allowlist = self._effective_scope_allowlist(scope_allowlist) query_limit = max(limit * 10, 100) if limit > 0 else limit rows = self.query_rows( query_text=query_text, diff --git a/memorymaster/govern/auto_resolver.py b/memorymaster/govern/auto_resolver.py index cb16ed34..f6280f85 100644 --- a/memorymaster/govern/auto_resolver.py +++ b/memorymaster/govern/auto_resolver.py @@ -16,7 +16,6 @@ from typing import Any from memorymaster.stores._storage_shared import ConcurrentModificationError -from memorymaster.core.lifecycle import transition_claim from memorymaster.core.llm_provider import call_llm from memorymaster.core.models import Claim @@ -81,6 +80,53 @@ def _cite_summary(claim: Claim) -> str: return "; ".join(f"{c.source}{f':{c.locator}' if c.locator else ''}" for c in claim.citations[:3]) +def _lost_race_result( + winner: Claim, + loser: Claim, + replacement_id, +) -> dict[str, Any]: + return { + "resolved": False, + "reason": "lost_race", + "winner_id": winner.id, + "loser_id": loser.id, + "current_replacement_id": replacement_id, + } + + +def _apply_conflict_resolution( + store, + winner: Claim, + loser: Claim, + reason: str, +) -> dict[str, Any]: + try: + store.mark_superseded( + loser.id, + winner.id, + f"llm_conflict_resolution: {reason}", + ) + updated = store.get_claim(loser.id, include_citations=False) + replacement_id = getattr(updated, "replaced_by_claim_id", None) + if replacement_id != winner.id: + return _lost_race_result(winner, loser, replacement_id) + return { + "resolved": True, + "winner_id": winner.id, + "loser_id": loser.id, + "reason": reason, + } + except ConcurrentModificationError as exc: + current = store.get_claim(loser.id, include_citations=False) + if current is not None and current.status == "superseded": + return _lost_race_result(winner, loser, current.replaced_by_claim_id) + logger.warning("Failed to resolve conflict %d vs %d: %s", winner.id, loser.id, exc) + return {"resolved": False, "reason": str(exc)} + except Exception as exc: + logger.warning("Failed to resolve conflict %d vs %d: %s", winner.id, loser.id, exc) + return {"resolved": False, "reason": str(exc)} + + def resolve_conflict_pair( store, claim_a: Claim, @@ -113,48 +159,7 @@ def resolve_conflict_pair( winner = claim_a if winner_letter == "A" else claim_b loser = claim_b if winner_letter == "A" else claim_a - - try: - updated = transition_claim( - store, - claim_id=loser.id, - to_status="superseded", - reason=f"llm_conflict_resolution: {reason}", - event_type="validator", - replaced_by_claim_id=winner.id, - ) - current_replacement_id = getattr(updated, "replaced_by_claim_id", None) - if current_replacement_id != winner.id: - return { - "resolved": False, - "reason": "lost_race", - "winner_id": winner.id, - "loser_id": loser.id, - "current_replacement_id": current_replacement_id, - } - if hasattr(store, "set_supersedes"): - store.set_supersedes(winner.id, loser.id) - return { - "resolved": True, - "winner_id": winner.id, - "loser_id": loser.id, - "reason": reason, - } - except ConcurrentModificationError as exc: - current = store.get_claim(loser.id, include_citations=False) - if current is not None and current.status == "superseded": - return { - "resolved": False, - "reason": "lost_race", - "winner_id": winner.id, - "loser_id": loser.id, - "current_replacement_id": current.replaced_by_claim_id, - } - logger.warning("Failed to resolve conflict %d vs %d: %s", claim_a.id, claim_b.id, exc) - return {"resolved": False, "reason": str(exc)} - except Exception as exc: - logger.warning("Failed to resolve conflict %d vs %d: %s", claim_a.id, claim_b.id, exc) - return {"resolved": False, "reason": str(exc)} + return _apply_conflict_resolution(store, winner, loser, reason) def _resolve_group_pairs(store, claims: list[Claim], limit: int) -> tuple[int, int, int]: diff --git a/memorymaster/govern/jobs/daydream_ingest.py b/memorymaster/govern/jobs/daydream_ingest.py index 90190319..f1eff18d 100644 --- a/memorymaster/govern/jobs/daydream_ingest.py +++ b/memorymaster/govern/jobs/daydream_ingest.py @@ -95,7 +95,7 @@ def _ingest_insights_impl( continue idempotency_key = _idempotency_key(insight) - if _claim_exists(service, idempotency_key): + if _claim_exists(service, idempotency_key, scope=scope): result["skipped"] += 1 continue @@ -263,9 +263,22 @@ def _citations_for(insight: _DaydreamInsight) -> list[CitationInput]: ] -def _claim_exists(service: MemoryService, idempotency_key: str) -> bool: +def _claim_exists( + service: MemoryService, + idempotency_key: str, + *, + scope: str, +) -> bool: getter = getattr(service.store, "get_claim_by_idempotency_key", None) - return bool(getter and getter(idempotency_key, include_citations=False)) + return bool( + getter + and getter( + idempotency_key, + include_citations=False, + tenant_id=service.tenant_id, + scope=scope, + ) + ) def _idempotency_key(insight: _DaydreamInsight) -> str: diff --git a/memorymaster/govern/jobs/validator.py b/memorymaster/govern/jobs/validator.py index a464fa6f..f035187f 100644 --- a/memorymaster/govern/jobs/validator.py +++ b/memorymaster/govern/jobs/validator.py @@ -132,17 +132,16 @@ def run( scope=claim.scope, exclude_claim_id=claim.id, tenant_id=claim.tenant_id, + visibility=claim.visibility, + source_agent=claim.source_agent, ) duplicate = next((x for x in related if x.object_value == claim.object_value and x.object_value), None) if duplicate is not None and not is_revalidation: - transition_claim( - store, - claim_id=claim.id, - to_status="superseded", + store.mark_superseded( + old_claim_id=claim.id, + new_claim_id=duplicate.id, reason=f"duplicate_of_confirmed_claim:{duplicate.id}", - event_type="validator", - replaced_by_claim_id=duplicate.id, ) superseded += 1 continue diff --git a/memorymaster/knowledge/rule_miner.py b/memorymaster/knowledge/rule_miner.py index ac1720ce..a0e7e595 100644 --- a/memorymaster/knowledge/rule_miner.py +++ b/memorymaster/knowledge/rule_miner.py @@ -470,16 +470,21 @@ def _process_candidate( confidence = _bootstrapped_confidence(conn, rule) idem = f"rule-miner-v{int(asst['id'])}-{int(row['id'])}" + claim_scope = row["scope"] or "project" store = getattr(service, "store", None) if store is not None and hasattr(store, "get_claim_by_idempotency_key"): - if store.get_claim_by_idempotency_key(idem) is not None: + if store.get_claim_by_idempotency_key( + idem, + tenant_id=getattr(service, "tenant_id", None), + scope=claim_scope, + ) is not None: stats["duplicates"] += 1 return "done" service.ingest( **build_rule_fields(rule["trigger"], rule["action"], rule["rationale"]), citations=[CitationInput(source="verbatim", locator=idem)], - scope=row["scope"] or "project", + scope=claim_scope, confidence=confidence, source_agent="rule-miner", idempotency_key=idem, @@ -591,7 +596,11 @@ def mine_transcript_rules( confidence = _transcript_confidence(service, rule) store = getattr(service, "store", None) if store is not None and hasattr(store, "get_claim_by_idempotency_key"): - if store.get_claim_by_idempotency_key(idem) is not None: + if store.get_claim_by_idempotency_key( + idem, + tenant_id=getattr(service, "tenant_id", None), + scope=scope, + ) is not None: stats["skipped"] += 1 continue service.ingest( diff --git a/memorymaster/knowledge/transcript_miner.py b/memorymaster/knowledge/transcript_miner.py index 538bff7d..8202b49f 100644 --- a/memorymaster/knowledge/transcript_miner.py +++ b/memorymaster/knowledge/transcript_miner.py @@ -127,7 +127,14 @@ def mine_transcript( idem_key = f"transcript-{text_hash}" existing = conn.execute( - "SELECT id FROM claims WHERE idempotency_key = ?", (idem_key,) + """ + SELECT id FROM claims + WHERE idempotency_key = ? + AND visibility = 'public' + AND tenant_id IS NULL + AND scope = ? + """, + (idem_key, scope), ).fetchone() if existing: stats["duplicates"] += 1 diff --git a/memorymaster/knowledge/vault_exporter.py b/memorymaster/knowledge/vault_exporter.py index 28e1ef3d..9cf5c165 100644 --- a/memorymaster/knowledge/vault_exporter.py +++ b/memorymaster/knowledge/vault_exporter.py @@ -28,6 +28,12 @@ def _safe_dirname(scope: str) -> str: return _SAFE_FILENAME_RE.sub("-", name.lower()).strip("-") or "default" +def _vault_claim_key(claim: Claim) -> str: + """Return a collision-free Obsidian basename while retaining human ID.""" + human_id = getattr(claim, "human_id", None) or f"claim-{claim.id}" + return f"{human_id}--claim-{claim.id}" + + def _claim_to_markdown(claim: Claim, links: list[dict[str, Any]] | None = None) -> str: """Render a claim as Obsidian-flavored Markdown with YAML frontmatter.""" lines = ["---"] @@ -127,11 +133,11 @@ def export_vault( include_citations=True, ) - # Build human_id lookup for link resolution + # Human IDs intentionally collide across private namespaces. Vault keys + # include the numeric claim ID so files and wikilinks remain exact. human_id_map: dict[int, str] = {} for c in claims: - hid = getattr(c, "human_id", None) or f"claim-{c.id}" - human_id_map[c.id] = hid + human_id_map[c.id] = _vault_claim_key(c) for claim in claims: # Scope filter @@ -167,8 +173,7 @@ def export_vault( pass # links table might not exist on old DBs # Render and write - human_id = getattr(claim, "human_id", None) or f"claim-{claim.id}" - filename = f"{human_id}.md" + filename = f"{_vault_claim_key(claim)}.md" md_content = _claim_to_markdown(claim, links_raw) (claim_dir / filename).write_text(md_content, encoding="utf-8") stats["exported"] += 1 diff --git a/memorymaster/recall/claim_edges.py b/memorymaster/recall/claim_edges.py index ec69ef07..a26cc9b5 100644 --- a/memorymaster/recall/claim_edges.py +++ b/memorymaster/recall/claim_edges.py @@ -67,54 +67,119 @@ def ensure_claim_edges_schema(conn: sqlite3.Connection) -> None: conn.commit() -def extract_edges_for_claim( - conn: sqlite3.Connection, src_claim_id: int, src_text: str +def _identity_context( + conn: sqlite3.Connection, + claim_id: int, +) -> tuple[str | None, bool, str | None, str, str | None] | None: + columns = {str(row[1]) for row in conn.execute("PRAGMA table_info(claims)")} + required = {"tenant_id", "visibility", "source_agent"} + if not required.issubset(columns): + return None + has_scope = "scope" in columns + scope_sql = "scope" if has_scope else "NULL AS scope" + row = conn.execute( + f"SELECT tenant_id, {scope_sql}, visibility, source_agent " + "FROM claims WHERE id = ?", + (claim_id,), + ).fetchone() + if row is None: + return None + return row[0], has_scope, row[1], str(row[2] or "public"), row[3] + + +def _identity_predicate( + context: tuple[str | None, bool, str | None, str, str | None] | None, +) -> tuple[str, tuple[object, ...]]: + if context is None: + return "1 = 1", () + tenant_id, has_scope, scope, visibility, source_agent = context + scope_sql = " AND scope IS ?" if has_scope else "" + scope_params: tuple[object, ...] = (scope,) if has_scope else () + if visibility == "public": + return ( + f"tenant_id IS ?{scope_sql} AND visibility = 'public'", + (tenant_id, *scope_params), + ) + return ( + f"tenant_id IS ?{scope_sql} AND visibility = ? AND source_agent IS ?", + (tenant_id, *scope_params, visibility, source_agent), + ) + + +def _numeric_reference_edges( + conn: sqlite3.Connection, + src_claim_id: int, + src_text: str, + identity_sql: str, + identity_params: tuple[object, ...], + seen_dst: set[int], ) -> list[tuple[int, int, str]]: - """Return ``[(src, dst, kind)]`` for every reference inside ``src_text``. - - Resolves mm- tokens and ``claim NNNN`` numerics against the live - claims table; dst-claims that don't exist or that point back at src are - silently dropped. - """ - if not src_text: - return [] edges: list[tuple[int, int, str]] = [] - seen_dst: set[int] = set() - - # Numeric refs: claim 12345 - for m in _CLAIM_NUM_RE.finditer(src_text): + for match in _CLAIM_NUM_RE.finditer(src_text): try: - dst = int(m.group(1)) + dst = int(match.group(1)) except ValueError: continue if dst == src_claim_id or dst in seen_dst: continue row = conn.execute( - "SELECT 1 FROM claims WHERE id = ? LIMIT 1", (dst,) + f"SELECT 1 FROM claims WHERE id = ? AND {identity_sql} LIMIT 1", + (dst, *identity_params), ).fetchone() - if row is None: - continue - seen_dst.add(dst) - edges.append((src_claim_id, dst, MENTION_KIND)) + if row is not None: + seen_dst.add(dst) + edges.append((src_claim_id, dst, MENTION_KIND)) + return edges - # Human-id refs: mm-1a2b - for m in _CLAIM_MM_RE.finditer(src_text): - human = m.group(1).lower() + +def _human_reference_edges( + conn: sqlite3.Connection, + src_claim_id: int, + src_text: str, + identity_sql: str, + identity_params: tuple[object, ...], + seen_dst: set[int], +) -> list[tuple[int, int, str]]: + edges: list[tuple[int, int, str]] = [] + for match in _CLAIM_MM_RE.finditer(src_text): row = conn.execute( - "SELECT id FROM claims WHERE LOWER(human_id) = ? LIMIT 1", - (human,), + f"SELECT id FROM claims WHERE LOWER(human_id) = ? " + f"AND {identity_sql} LIMIT 1", + (match.group(1).lower(), *identity_params), ).fetchone() if row is None: continue dst = int(row[0]) - if dst == src_claim_id or dst in seen_dst: - continue - seen_dst.add(dst) - edges.append((src_claim_id, dst, MENTION_KIND)) - + if dst != src_claim_id and dst not in seen_dst: + seen_dst.add(dst) + edges.append((src_claim_id, dst, MENTION_KIND)) return edges +def extract_edges_for_claim( + conn: sqlite3.Connection, src_claim_id: int, src_text: str +) -> list[tuple[int, int, str]]: + """Return ``[(src, dst, kind)]`` for every reference inside ``src_text``. + + Resolves mm- tokens and ``claim NNNN`` numerics against the live + claims table; dst-claims that don't exist or that point back at src are + silently dropped. + """ + if not src_text: + return [] + seen_dst: set[int] = set() + identity_sql, identity_params = _identity_predicate( + _identity_context(conn, src_claim_id) + ) + numeric = _numeric_reference_edges( + conn, src_claim_id, src_text, identity_sql, identity_params, seen_dst + ) + human = _human_reference_edges( + conn, src_claim_id, src_text, identity_sql, identity_params, seen_dst + ) + return numeric + human + + def rebuild_edges( db_path: str | Path, *, diff --git a/memorymaster/schema.sql b/memorymaster/schema.sql index 802f1cd3..57ff68b0 100644 --- a/memorymaster/schema.sql +++ b/memorymaster/schema.sql @@ -29,6 +29,8 @@ CREATE TABLE IF NOT EXISTS claims ( event_time TEXT, valid_from TEXT, valid_until TEXT, + source_agent TEXT, + visibility TEXT NOT NULL DEFAULT 'public', wiki_article TEXT, holder TEXT, tenant_id TEXT, @@ -36,41 +38,20 @@ CREATE TABLE IF NOT EXISTS claims ( FOREIGN KEY (replaced_by_claim_id) REFERENCES claims(id) ON DELETE SET NULL ); -CREATE TRIGGER IF NOT EXISTS trg_claims_confirmed_tuple_guard_insert +CREATE TRIGGER IF NOT EXISTS trg_claims_identity_guard_insert BEFORE INSERT ON claims -WHEN NEW.status = 'confirmed' - AND NEW.subject IS NOT NULL - AND NEW.predicate IS NOT NULL - AND EXISTS ( - SELECT 1 - FROM claims c - WHERE c.status = 'confirmed' - AND c.subject = NEW.subject - AND c.predicate = NEW.predicate - AND c.scope = NEW.scope - AND c.tenant_id IS NEW.tenant_id - ) +WHEN NEW.visibility NOT IN ('public', 'private', 'sensitive') + OR (NEW.visibility <> 'public' AND NULLIF(TRIM(NEW.source_agent), '') IS NULL) BEGIN - SELECT RAISE(ABORT, 'only one confirmed claim is allowed per (subject,predicate,scope)'); + SELECT RAISE(ABORT, 'invalid claim visibility or missing non-public source_agent'); END; -CREATE TRIGGER IF NOT EXISTS trg_claims_confirmed_tuple_guard_update -BEFORE UPDATE OF status, subject, predicate, scope, tenant_id ON claims -WHEN NEW.status = 'confirmed' - AND NEW.subject IS NOT NULL - AND NEW.predicate IS NOT NULL - AND EXISTS ( - SELECT 1 - FROM claims c - WHERE c.id <> OLD.id - AND c.status = 'confirmed' - AND c.subject = NEW.subject - AND c.predicate = NEW.predicate - AND c.scope = NEW.scope - AND c.tenant_id IS NEW.tenant_id - ) +CREATE TRIGGER IF NOT EXISTS trg_claims_identity_guard_update +BEFORE UPDATE ON claims +WHEN NEW.visibility NOT IN ('public', 'private', 'sensitive') + OR (NEW.visibility <> 'public' AND NULLIF(TRIM(NEW.source_agent), '') IS NULL) BEGIN - SELECT RAISE(ABORT, 'only one confirmed claim is allowed per (subject,predicate,scope)'); + SELECT RAISE(ABORT, 'invalid claim visibility or missing non-public source_agent'); END; CREATE TABLE IF NOT EXISTS citations ( @@ -201,9 +182,13 @@ CREATE TABLE IF NOT EXISTS mcp_usage ( CREATE INDEX IF NOT EXISTS idx_claims_status ON claims(status); CREATE INDEX IF NOT EXISTS idx_claims_updated_at ON claims(updated_at); CREATE INDEX IF NOT EXISTS idx_claims_idempotency_key ON claims(idempotency_key); -CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_idempotency_key - ON claims(COALESCE(tenant_id, ''), idempotency_key) - WHERE idempotency_key IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_idempotency_key_unique + ON claims(COALESCE(tenant_id, ''), scope, idempotency_key) + WHERE visibility = 'public' AND idempotency_key IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_idempotency_key_unique + ON claims(COALESCE(tenant_id, ''), scope, visibility, source_agent, idempotency_key) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND idempotency_key IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_claims_tuple ON claims(subject, predicate, scope); CREATE INDEX IF NOT EXISTS idx_claims_replaced_by ON claims(replaced_by_claim_id); CREATE INDEX IF NOT EXISTS idx_citations_claim_id ON citations(claim_id); @@ -218,12 +203,22 @@ CREATE INDEX IF NOT EXISTS idx_events_tenant_algo_head CREATE INDEX IF NOT EXISTS idx_events_created_at ON events(created_at); CREATE INDEX IF NOT EXISTS idx_embeddings_updated_at ON claim_embeddings(updated_at); CREATE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id); -CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_human_id - ON claims(COALESCE(tenant_id, ''), human_id) - WHERE human_id IS NOT NULL; -CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_confirmed_tuple_unique +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_human_id_unique + ON claims(COALESCE(tenant_id, ''), scope, human_id) + WHERE visibility = 'public' AND human_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_human_id_unique + ON claims(COALESCE(tenant_id, ''), scope, visibility, source_agent, human_id) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND human_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_confirmed_tuple_unique ON claims(COALESCE(tenant_id, ''), subject, predicate, scope) - WHERE status = 'confirmed' + WHERE visibility = 'public' AND status = 'confirmed' + AND subject IS NOT NULL + AND predicate IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_confirmed_tuple_unique + ON claims(COALESCE(tenant_id, ''), visibility, source_agent, subject, predicate, scope) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND status = 'confirmed' AND subject IS NOT NULL AND predicate IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_external_sources_type ON external_sources(source_type); diff --git a/memorymaster/schema_postgres.sql b/memorymaster/schema_postgres.sql index 812676ca..61b253ba 100644 --- a/memorymaster/schema_postgres.sql +++ b/memorymaster/schema_postgres.sql @@ -72,36 +72,59 @@ ALTER TABLE claims ADD COLUMN IF NOT EXISTS visibility TEXT NOT NULL DEFAULT 'pu -- belief. NULL = holder-agnostic (default, byte-identical to pre-holder rows). ALTER TABLE claims ADD COLUMN IF NOT EXISTS holder TEXT; -CREATE OR REPLACE FUNCTION memorymaster_claims_confirmed_tuple_guard() +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'ck_claims_identity_visibility_owner' + AND conrelid = 'claims'::regclass + ) THEN + ALTER TABLE claims ADD CONSTRAINT ck_claims_identity_visibility_owner + CHECK ( + visibility IN ('public', 'private', 'sensitive') + AND NULLIF(BTRIM(source_agent), '') IS NOT NULL + ); + END IF; +END +$$; + +CREATE OR REPLACE FUNCTION public.memorymaster_claim_supersession_guard() RETURNS trigger LANGUAGE plpgsql AS $$ +DECLARE + reference_id BIGINT; BEGIN - IF NEW.status = 'confirmed' - AND NEW.subject IS NOT NULL - AND NEW.predicate IS NOT NULL - AND EXISTS ( - SELECT 1 - FROM claims c - WHERE c.status = 'confirmed' - AND c.subject = NEW.subject - AND c.predicate = NEW.predicate - AND c.scope = NEW.scope - AND c.tenant_id IS NOT DISTINCT FROM NEW.tenant_id - AND (TG_OP = 'INSERT' OR c.id <> NEW.id) - ) THEN - RAISE EXCEPTION 'only one confirmed claim is allowed per tenant and (subject,predicate,scope)' - USING ERRCODE = '23505'; - END IF; + FOREACH reference_id IN ARRAY ARRAY[ + NEW.supersedes_claim_id, + NEW.replaced_by_claim_id + ] LOOP + IF reference_id IS NOT NULL AND ( + reference_id = NEW.id + OR NOT EXISTS ( + SELECT 1 + FROM public.claims AS referenced + WHERE referenced.id = reference_id + AND referenced.tenant_id IS NOT DISTINCT FROM NEW.tenant_id + AND referenced.scope = NEW.scope + AND referenced.visibility IS NOT DISTINCT FROM NEW.visibility + AND referenced.source_agent IS NOT DISTINCT FROM NEW.source_agent + ) + ) THEN + RAISE EXCEPTION 'supersession reference is outside the authorized boundary' + USING ERRCODE = '42501'; + END IF; + END LOOP; RETURN NEW; END; $$; -DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard ON claims; -CREATE TRIGGER trg_claims_confirmed_tuple_guard -BEFORE INSERT OR UPDATE OF status, subject, predicate, scope, tenant_id ON claims +DROP TRIGGER IF EXISTS trg_claims_supersession_boundary ON claims; +CREATE TRIGGER trg_claims_supersession_boundary +BEFORE INSERT OR UPDATE OF tenant_id, scope, visibility, source_agent, + supersedes_claim_id, replaced_by_claim_id ON claims FOR EACH ROW -EXECUTE FUNCTION memorymaster_claims_confirmed_tuple_guard(); +EXECUTE FUNCTION public.memorymaster_claim_supersession_guard(); CREATE TABLE IF NOT EXISTS citations ( id BIGSERIAL PRIMARY KEY, @@ -172,19 +195,33 @@ END $$; CREATE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id); -CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_human_id - ON claims(COALESCE(tenant_id, ''), human_id) - WHERE human_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_human_id_unique + ON claims(COALESCE(tenant_id, ''), scope, human_id) + WHERE visibility = 'public' AND human_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_human_id_unique + ON claims(COALESCE(tenant_id, ''), scope, visibility, source_agent, human_id) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND human_id IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_claims_tenant_id ON claims(tenant_id); CREATE INDEX IF NOT EXISTS idx_claims_status ON claims(status); CREATE INDEX IF NOT EXISTS idx_claims_updated_at ON claims(updated_at); CREATE INDEX IF NOT EXISTS idx_claims_idempotency_key ON claims(idempotency_key); -CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_idempotency_key - ON claims(COALESCE(tenant_id, ''), idempotency_key) - WHERE idempotency_key IS NOT NULL; -CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_confirmed_tuple_unique +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_idempotency_key_unique + ON claims(COALESCE(tenant_id, ''), scope, idempotency_key) + WHERE visibility = 'public' AND idempotency_key IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_idempotency_key_unique + ON claims(COALESCE(tenant_id, ''), scope, visibility, source_agent, idempotency_key) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND idempotency_key IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_confirmed_tuple_unique ON claims(COALESCE(tenant_id, ''), subject, predicate, scope) - WHERE status = 'confirmed' + WHERE visibility = 'public' AND status = 'confirmed' + AND subject IS NOT NULL + AND predicate IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_confirmed_tuple_unique + ON claims(COALESCE(tenant_id, ''), visibility, source_agent, subject, predicate, scope) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND status = 'confirmed' AND subject IS NOT NULL AND predicate IS NOT NULL; CREATE INDEX IF NOT EXISTS idx_claims_tuple ON claims(subject, predicate, scope); diff --git a/memorymaster/stores/_storage_lifecycle.py b/memorymaster/stores/_storage_lifecycle.py index 568afd4f..ca7d61c3 100644 --- a/memorymaster/stores/_storage_lifecycle.py +++ b/memorymaster/stores/_storage_lifecycle.py @@ -12,6 +12,7 @@ from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any +from memorymaster.core.lifecycle import can_transition from memorymaster.recall.embeddings import EmbeddingProvider, cosine_similarity from memorymaster.core.models import ( CLAIM_LINK_TYPES, @@ -212,17 +213,11 @@ def archive_by_source( def set_supersedes(self, claim_id: int, supersedes_claim_id: int) -> None: - now = utc_now() - with self.connect() as conn: - conn.execute( - """ - UPDATE claims - SET supersedes_claim_id = ?, updated_at = ? - WHERE id = ? - """, - (supersedes_claim_id, now, claim_id), - ) - conn.commit() + self.mark_superseded( + supersedes_claim_id, + claim_id, + "set_supersedes compatibility path", + ) def mark_superseded(self, old_claim_id: int, new_claim_id: int, reason: str) -> None: @@ -233,6 +228,10 @@ def mark_superseded(self, old_claim_id: int, new_claim_id: int, reason: str) -> raise ConcurrentModificationError( f"Claim {old_claim_id} was already superseded. Reload and retry." ) + if not can_transition(old_claim.status, "superseded"): + raise ValueError( + f"Invalid transition: {old_claim.status} -> superseded" + ) now = utc_now() with self.connect() as conn: diff --git a/memorymaster/stores/_storage_read.py b/memorymaster/stores/_storage_read.py index eedf48df..069f7cac 100644 --- a/memorymaster/stores/_storage_read.py +++ b/memorymaster/stores/_storage_read.py @@ -17,6 +17,10 @@ ClaimLink, Event, ) +from memorymaster.stores.claim_identity import ( + normalize_claim_identity, + require_unambiguous_identity_row, +) logger = logging.getLogger(__name__) @@ -33,16 +37,28 @@ def _check_idempotency( conn: sqlite3.Connection, idempotency_key: str | None, tenant_id: str | None = None, + scope: str = "project", + visibility: str = "public", + source_agent: str | None = None, ) -> Claim | None: """Check if a claim with this idempotency key already exists. Returns existing claim or None.""" normalized_key = (idempotency_key or "").strip() or None if normalized_key is None: return None + visibility, source_agent = normalize_claim_identity(visibility, source_agent) + identity_sql, identity_params = self._claim_identity_filter( + visibility, + source_agent, + ) # Hydrate the full row from the already-open conn instead of paying # a fresh get_claim() connection open on every duplicate re-ingest. existing_row = conn.execute( - "SELECT * FROM claims WHERE idempotency_key = ? AND tenant_id IS ?", - (normalized_key, tenant_id), + f""" + SELECT * FROM claims + WHERE idempotency_key = ? AND tenant_id IS ? AND scope = ? + AND {identity_sql} + """, + (normalized_key, tenant_id, scope, *identity_params), ).fetchone() if existing_row is None: return None @@ -83,15 +99,39 @@ def get_claim_by_idempotency_key( include_citations: bool = True, *, tenant_id: str | None = None, + scope: str | None = None, + visibility: str = "public", + source_agent: str | None = None, ) -> Claim | None: normalized_idempotency_key = idempotency_key.strip() if not normalized_idempotency_key: return None + visibility, source_agent = normalize_claim_identity(visibility, source_agent) + identity_sql, identity_params = self._claim_identity_filter( + visibility, + source_agent, + ) + scope_sql = "AND scope = ?" if scope is not None else "" + scope_params: tuple[object, ...] = (scope,) if scope is not None else () with self.connect() as conn: - row = conn.execute( - "SELECT * FROM claims WHERE idempotency_key = ? AND tenant_id IS ?", - (normalized_idempotency_key, tenant_id), - ).fetchone() + rows = conn.execute( + f""" + SELECT * FROM claims + WHERE idempotency_key = ? AND tenant_id IS ? {scope_sql} + AND {identity_sql} + LIMIT 2 + """, + ( + normalized_idempotency_key, + tenant_id, + *scope_params, + *identity_params, + ), + ).fetchall() + row = require_unambiguous_identity_row( + list(rows), + identifier="idempotency key", + ) if row is None: return None claim = self._row_to_claim(row) @@ -106,20 +146,39 @@ def get_claim_by_human_id( include_citations: bool = True, *, tenant_id: str | None = None, + scope: str | None = None, + visibility: str = "public", + source_agent: str | None = None, ) -> Claim | None: """Look up a claim by its human-readable ID (e.g. ``mm-a3f8``).""" normalized = human_id.strip() if not normalized: return None + visibility, source_agent = normalize_claim_identity(visibility, source_agent) + identity_sql, identity_params = self._claim_identity_filter( + visibility, + source_agent, + ) + scope_sql = "AND scope = ?" if scope is not None else "" + scope_params: tuple[object, ...] = (scope,) if scope is not None else () with self.connect() as conn: try: - row = conn.execute( - "SELECT * FROM claims WHERE human_id = ? AND tenant_id IS ?", - (normalized, tenant_id), - ).fetchone() + rows = conn.execute( + f""" + SELECT * FROM claims + WHERE human_id = ? AND tenant_id IS ? {scope_sql} + AND {identity_sql} + LIMIT 2 + """, + (normalized, tenant_id, *scope_params, *identity_params), + ).fetchall() except sqlite3.OperationalError: # Column may not exist yet. return None + row = require_unambiguous_identity_row( + list(rows), + identifier="human_id", + ) if row is None: return None claim = self._row_to_claim(row) @@ -133,6 +192,9 @@ def resolve_claim_id( identifier: str | int, *, tenant_id: str | None = None, + scope: str | None = None, + visibility: str = "public", + source_agent: str | None = None, ) -> int: """Resolve a numeric ID or human_id string to a numeric claim ID. @@ -151,6 +213,9 @@ def resolve_claim_id( raw, include_citations=False, tenant_id=tenant_id, + scope=scope, + visibility=visibility, + source_agent=source_agent, ) if claim is not None: return claim.id @@ -470,6 +535,8 @@ def find_confirmed_by_tuple( scope: str | None, exclude_claim_id: int | None = None, tenant_id: str | None = None, + visibility: str = "public", + source_agent: str | None = None, ) -> list[Claim]: if not subject or not predicate: return [] @@ -478,6 +545,13 @@ def find_confirmed_by_tuple( params: list[object] = [subject, predicate, scope or "project"] clauses.append("tenant_id IS ?") params.append(tenant_id) + visibility, source_agent = normalize_claim_identity(visibility, source_agent) + identity_sql, identity_params = self._claim_identity_filter( + visibility, + source_agent, + ) + clauses.append(identity_sql) + params.extend(identity_params) if exclude_claim_id is not None: clauses.append("id <> ?") params.append(exclude_claim_id) @@ -491,6 +565,15 @@ def find_confirmed_by_tuple( rows = conn.execute(sql, params).fetchall() return [self._row_to_claim(row) for row in rows] + @staticmethod + def _claim_identity_filter( + visibility: str, + source_agent: str | None, + ) -> tuple[str, tuple[object, ...]]: + if visibility == "public": + return "visibility = ?", ("public",) + return "visibility = ? AND source_agent = ?", (visibility, source_agent) + @staticmethod def _row_to_claim(row: sqlite3.Row) -> Claim: diff --git a/memorymaster/stores/_storage_schema.py b/memorymaster/stores/_storage_schema.py index 27660ca4..9f3ee452 100644 --- a/memorymaster/stores/_storage_schema.py +++ b/memorymaster/stores/_storage_schema.py @@ -21,6 +21,7 @@ compute_tenant_event_hash, generate_top_level_human_id, ) +from memorymaster.stores.claim_identity import identity_namespace_key logger = logging.getLogger(__name__) @@ -38,6 +39,46 @@ def load_schema_postgres_sql() -> str: class _SchemaMixin: + @staticmethod + def _sqlite_identity_clause( + visibility: str, + source_agent: str | None, + *, + alias: str = "", + ) -> tuple[str, tuple[object, ...]]: + prefix = f"{alias}." if alias else "" + if visibility == "public": + return f"{prefix}visibility = ?", ("public",) + return ( + f"{prefix}visibility = ? AND {prefix}source_agent = ?", + (visibility, source_agent), + ) + + @staticmethod + def _ensure_claim_identity_guards(conn: sqlite3.Connection) -> None: + conn.executescript( + """ + CREATE TRIGGER IF NOT EXISTS trg_claims_identity_guard_insert + BEFORE INSERT ON claims + WHEN NEW.visibility NOT IN ('public', 'private', 'sensitive') + OR (NEW.visibility <> 'public' + AND NULLIF(TRIM(NEW.source_agent), '') IS NULL) + BEGIN + SELECT RAISE(ABORT, + 'invalid claim visibility or missing non-public source_agent'); + END; + CREATE TRIGGER IF NOT EXISTS trg_claims_identity_guard_update + BEFORE UPDATE ON claims + WHEN NEW.visibility NOT IN ('public', 'private', 'sensitive') + OR (NEW.visibility <> 'public' + AND NULLIF(TRIM(NEW.source_agent), '') IS NULL) + BEGIN + SELECT RAISE(ABORT, + 'invalid claim visibility or missing non-public source_agent'); + END; + """ + ) + @staticmethod def _ensure_version_column(conn: sqlite3.Connection) -> None: """Add ``version`` column to claims (optimistic-locking counter).""" @@ -215,6 +256,8 @@ def _ensure_atlas_source_schema(conn: sqlite3.Connection) -> None: @staticmethod def _ensure_claim_idempotency_schema(conn: sqlite3.Connection) -> None: _SchemaMixin._ensure_tenant_id_schema(conn) + _SchemaMixin._ensure_scope_schema(conn) + _SchemaMixin._ensure_agent_columns(conn) try: conn.execute("ALTER TABLE claims ADD COLUMN idempotency_key TEXT") except sqlite3.OperationalError as exc: @@ -223,74 +266,51 @@ def _ensure_claim_idempotency_schema(conn: sqlite3.Connection) -> None: conn.execute( "CREATE INDEX IF NOT EXISTS idx_claims_idempotency_key ON claims(idempotency_key)" ) - conn.execute( + conn.executescript( """ - CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_idempotency_key - ON claims(COALESCE(tenant_id, ''), idempotency_key) - WHERE idempotency_key IS NOT NULL + DROP INDEX IF EXISTS idx_claims_tenant_idempotency_key; + CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_idempotency_key_unique + ON claims(COALESCE(tenant_id, ''), scope, idempotency_key) + WHERE visibility = 'public' AND idempotency_key IS NOT NULL; + CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_idempotency_key_unique + ON claims(COALESCE(tenant_id, ''), scope, visibility, source_agent, idempotency_key) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND idempotency_key IS NOT NULL; """ ) + _SchemaMixin._ensure_claim_identity_guards(conn) @staticmethod def _ensure_confirmed_tuple_uniqueness_schema(conn: sqlite3.Connection) -> None: _SchemaMixin._ensure_tenant_id_schema(conn) + _SchemaMixin._ensure_scope_schema(conn) + _SchemaMixin._ensure_agent_columns(conn) for trigger in SQLITE_CONFIRMED_TUPLE_GUARD_TRIGGERS: conn.execute(f"DROP TRIGGER IF EXISTS {trigger}") - conn.executescript( - """ - CREATE TRIGGER IF NOT EXISTS trg_claims_confirmed_tuple_guard_insert - BEFORE INSERT ON claims - WHEN NEW.status = 'confirmed' - AND NEW.subject IS NOT NULL - AND NEW.predicate IS NOT NULL - AND EXISTS ( - SELECT 1 - FROM claims c - WHERE c.status = 'confirmed' - AND c.subject = NEW.subject - AND c.predicate = NEW.predicate - AND c.scope = NEW.scope - AND c.tenant_id IS NEW.tenant_id - ) - BEGIN - SELECT RAISE(ABORT, 'only one confirmed claim is allowed per (subject,predicate,scope)'); - END; - - CREATE TRIGGER IF NOT EXISTS trg_claims_confirmed_tuple_guard_update - BEFORE UPDATE OF status, subject, predicate, scope, tenant_id ON claims - WHEN NEW.status = 'confirmed' - AND NEW.subject IS NOT NULL - AND NEW.predicate IS NOT NULL - AND EXISTS ( - SELECT 1 - FROM claims c - WHERE c.id <> OLD.id - AND c.status = 'confirmed' - AND c.subject = NEW.subject - AND c.predicate = NEW.predicate - AND c.scope = NEW.scope - AND c.tenant_id IS NEW.tenant_id - ) - BEGIN - SELECT RAISE(ABORT, 'only one confirmed claim is allowed per (subject,predicate,scope)'); - END; - """ - ) try: - conn.execute( + conn.executescript( """ - CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_confirmed_tuple_unique - ON claims(COALESCE(tenant_id, ''), subject, predicate, scope) - WHERE status = 'confirmed' - AND subject IS NOT NULL - AND predicate IS NOT NULL + DROP INDEX IF EXISTS idx_claims_confirmed_tuple_unique; + CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_confirmed_tuple_unique + ON claims(COALESCE(tenant_id, ''), subject, predicate, scope) + WHERE visibility = 'public' AND status = 'confirmed' + AND subject IS NOT NULL AND predicate IS NOT NULL; + CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_confirmed_tuple_unique + ON claims( + COALESCE(tenant_id, ''), visibility, source_agent, + subject, predicate, scope + ) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND status = 'confirmed' + AND subject IS NOT NULL AND predicate IS NOT NULL; """ ) except sqlite3.IntegrityError as exc: lowered = str(exc).lower() if "unique constraint failed" not in lowered: raise + _SchemaMixin._ensure_claim_identity_guards(conn) @staticmethod @@ -518,6 +538,8 @@ def _ensure_claim_links_schema(conn: sqlite3.Connection) -> None: def _ensure_human_id_schema(conn: sqlite3.Connection) -> None: """Add human_id column if missing and backfill existing claims.""" _SchemaMixin._ensure_tenant_id_schema(conn) + _SchemaMixin._ensure_scope_schema(conn) + _SchemaMixin._ensure_agent_columns(conn) try: conn.execute("ALTER TABLE claims ADD COLUMN human_id TEXT") except sqlite3.OperationalError as exc: @@ -536,13 +558,19 @@ def _ensure_human_id_schema(conn: sqlite3.Connection) -> None: conn.execute( "CREATE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id)" ) - conn.execute( + conn.executescript( """ - CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_human_id - ON claims(COALESCE(tenant_id, ''), human_id) - WHERE human_id IS NOT NULL + DROP INDEX IF EXISTS idx_claims_tenant_human_id; + CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_human_id_unique + ON claims(COALESCE(tenant_id, ''), scope, human_id) + WHERE visibility = 'public' AND human_id IS NOT NULL; + CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_human_id_unique + ON claims(COALESCE(tenant_id, ''), scope, visibility, source_agent, human_id) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND human_id IS NOT NULL; """ ) + _SchemaMixin._ensure_claim_identity_guards(conn) _SchemaMixin._backfill_human_ids(conn) @@ -556,7 +584,10 @@ def _backfill_human_ids(conn: sqlite3.Connection) -> int: per-row claim_links JOIN and generate ids in-memory + executemany. """ rows = conn.execute( - "SELECT id, subject, text, tenant_id FROM claims WHERE human_id IS NULL ORDER BY id ASC" + """ + SELECT id, subject, text, tenant_id, scope, visibility, source_agent + FROM claims WHERE human_id IS NULL ORDER BY id ASC + """ ).fetchall() if not rows: return 0 @@ -579,6 +610,9 @@ def _backfill_human_ids(conn: sqlite3.Connection) -> int: text, claim_id, tenant_id=row["tenant_id"], + scope=str(row["scope"]), + visibility=row["visibility"], + source_agent=row["source_agent"], ) conn.execute( "UPDATE claims SET human_id = ? WHERE id = ?", @@ -594,23 +628,32 @@ def _backfill_human_ids_top_level(conn: sqlite3.Connection, rows: list) -> int: Collisions are resolved in-memory against ids already present in the DB plus ids minted within this batch, then written via a single executemany. """ - taken: set[tuple[str | None, str]] = { - (r[0], str(r[1])) + taken: set[tuple[tuple[str | None, str, str, str | None], str]] = { + (identity_namespace_key(r[0], str(r[1]), str(r[2]), r[3]), str(r[4])) for r in conn.execute( - "SELECT tenant_id, human_id FROM claims WHERE human_id IS NOT NULL" + """ + SELECT tenant_id, scope, visibility, source_agent, human_id + FROM claims WHERE human_id IS NOT NULL + """ ).fetchall() } updates: list[tuple[str, int]] = [] for row in rows: claim_id = int(row["id"]) tenant_id = row["tenant_id"] + namespace = identity_namespace_key( + tenant_id, + str(row["scope"]), + str(row["visibility"]), + row["source_agent"], + ) candidate = generate_top_level_human_id(row["subject"], str(row["text"])) final = candidate suffix = 1 - while (tenant_id, final) in taken: + while (namespace, final) in taken: suffix += 1 final = f"{candidate}~{suffix}" - taken.add((tenant_id, final)) + taken.add((namespace, final)) updates.append((final, claim_id)) conn.executemany("UPDATE claims SET human_id = ? WHERE id = ?", updates) return len(updates) @@ -623,6 +666,9 @@ def _allocate_human_id( text: str, claim_id: int, tenant_id: str | None = None, + scope: str = "project", + visibility: str = "public", + source_agent: str | None = None, ) -> str: """Build a unique human_id, checking for derived_from parent links. @@ -631,8 +677,13 @@ def _allocate_human_id( top-level id (e.g. ``mm-a3f8``). Collisions are resolved by appending a numeric suffix. """ + identity_clause, identity_params = _SchemaMixin._sqlite_identity_clause( + visibility, + source_agent, + alias="c", + ) parent_row = conn.execute( - """ + f""" SELECT c.human_id FROM claim_links cl JOIN claims c ON c.id = cl.target_id @@ -640,19 +691,23 @@ def _allocate_human_id( AND cl.link_type = 'derived_from' AND c.human_id IS NOT NULL AND c.tenant_id IS ? + AND c.scope = ? + AND {identity_clause} LIMIT 1 """, - (claim_id, tenant_id), + (claim_id, tenant_id, scope, *identity_params), ).fetchone() if parent_row and parent_row["human_id"]: parent_hid = str(parent_row["human_id"]) child_count = conn.execute( - """ + f""" SELECT COUNT(*) AS cnt FROM claims WHERE human_id LIKE ? AND human_id != ? AND tenant_id IS ? + AND scope = ? + AND {identity_clause.replace('c.', '')} """, - (parent_hid + ".%", parent_hid, tenant_id), + (parent_hid + ".%", parent_hid, tenant_id, scope, *identity_params), ).fetchone() next_child = (int(child_count["cnt"]) if child_count else 0) + 1 candidate = f"{parent_hid}.{next_child}" @@ -664,8 +719,13 @@ def _allocate_human_id( suffix = 1 while True: existing = conn.execute( - "SELECT 1 FROM claims WHERE human_id = ? AND tenant_id IS ?", - (final, tenant_id), + f""" + SELECT 1 FROM claims + WHERE human_id = ? AND tenant_id IS ? + AND scope = ? + AND {identity_clause.replace('c.', '')} + """, + (final, tenant_id, scope, *identity_params), ).fetchone() if existing is None: return final @@ -686,6 +746,18 @@ def _ensure_tenant_id_schema(conn: sqlite3.Connection) -> None: ) + @staticmethod + def _ensure_scope_schema(conn: sqlite3.Connection) -> None: + """Give legacy claims the historical default identity scope.""" + try: + conn.execute( + "ALTER TABLE claims ADD COLUMN scope TEXT NOT NULL DEFAULT 'project'" + ) + except sqlite3.OperationalError as exc: + if "duplicate column name" not in str(exc).lower(): + raise + + @staticmethod def _ensure_temporal_columns(conn) -> None: """Add bi-temporal columns if missing (backward compat for old DBs).""" diff --git a/memorymaster/stores/_storage_write_claims.py b/memorymaster/stores/_storage_write_claims.py index 7acc3eec..c0ef5138 100644 --- a/memorymaster/stores/_storage_write_claims.py +++ b/memorymaster/stores/_storage_write_claims.py @@ -19,6 +19,7 @@ from memorymaster.stores._storage_shared import ( utc_now, ) +from memorymaster.stores.claim_identity import normalize_claim_identity logger = logging.getLogger(__name__) @@ -32,6 +33,9 @@ def _check_idempotency( conn: sqlite3.Connection, idempotency_key: str | None, tenant_id: str | None = None, + scope: str = "project", + visibility: str = "public", + source_agent: str | None = None, ) -> Claim | None: ... def get_claim(self, claim_id: int, include_citations: bool = True) -> Claim | None: ... @@ -43,8 +47,17 @@ def _allocate_human_id( text: str, claim_id: int, tenant_id: str | None = None, + scope: str = "project", + visibility: str = "public", + source_agent: str | None = None, ) -> str: ... + def _claim_identity_filter( + self, + visibility: str, + source_agent: str | None, + ) -> tuple[str, tuple[object, ...]]: ... + def _insert_event_row( self, conn: sqlite3.Connection, @@ -81,6 +94,7 @@ def create_claim( ) -> Claim: if not citations: raise ValueError("At least one citation is required.") + visibility, source_agent = normalize_claim_identity(visibility, source_agent) normalized_idempotency_key = (idempotency_key or "").strip() or None normalized_tenant_id = (tenant_id or "").strip() or None now = utc_now() @@ -89,6 +103,9 @@ def create_claim( conn, idempotency_key, tenant_id=normalized_tenant_id, + scope=scope, + visibility=visibility, + source_agent=source_agent, ) if existing is not None: return existing @@ -119,8 +136,8 @@ def create_claim( event_time or None, valid_from or now, # Auto-populate: claim is valid from creation time valid_until or None, - source_agent or None, - visibility or "public", + source_agent, + visibility, holder or None, ), ) @@ -128,12 +145,23 @@ def create_claim( if normalized_idempotency_key is None: raise conn.rollback() + identity_sql, identity_params = self._claim_identity_filter( + visibility, + source_agent, + ) existing_row = conn.execute( - """ + f""" SELECT id FROM claims WHERE idempotency_key = ? AND tenant_id IS ? + AND scope = ? + AND {identity_sql} """, - (normalized_idempotency_key, normalized_tenant_id), + ( + normalized_idempotency_key, + normalized_tenant_id, + scope, + *identity_params, + ), ).fetchone() if existing_row is None: raise @@ -151,6 +179,9 @@ def create_claim( text, claim_id, tenant_id=normalized_tenant_id, + scope=scope, + visibility=visibility, + source_agent=source_agent, ) conn.execute( "UPDATE claims SET human_id = ? WHERE id = ?", diff --git a/memorymaster/stores/claim_identity.py b/memorymaster/stores/claim_identity.py new file mode 100644 index 00000000..9088593d --- /dev/null +++ b/memorymaster/stores/claim_identity.py @@ -0,0 +1,69 @@ +"""Canonical identity namespaces for claims. + +All claim identities are local to the exact tenant and scope visible to the +caller. Non-public identities additionally include visibility and source +principal so hidden rows cannot become uniqueness oracles. +""" +from __future__ import annotations + +from typing import Final, TypeVar + + +CLAIM_VISIBILITIES: Final[frozenset[str]] = frozenset( + {"public", "private", "sensitive"} +) +_Row = TypeVar("_Row") + + +def normalize_claim_visibility(value: str | None) -> str: + """Return a canonical visibility or reject an unsupported value.""" + normalized = str(value or "public").strip().lower() + if normalized not in CLAIM_VISIBILITIES: + allowed = ", ".join(sorted(CLAIM_VISIBILITIES)) + raise ValueError(f"Invalid claim visibility {value!r}; expected one of: {allowed}.") + return normalized + + +def normalize_source_agent(value: str | None) -> str | None: + """Collapse blank principal labels to ``None``.""" + return str(value or "").strip() or None + + +def normalize_claim_identity( + visibility: str | None, + source_agent: str | None, + *, + allow_sensitive: bool = True, +) -> tuple[str, str | None]: + """Validate and normalize one claim's persisted identity context.""" + normalized_visibility = normalize_claim_visibility(visibility) + normalized_source = normalize_source_agent(source_agent) + if normalized_visibility == "sensitive" and not allow_sensitive: + raise PermissionError("Sensitive claim writes are unavailable in team runtime mode.") + if normalized_visibility != "public" and normalized_source is None: + raise ValueError("Non-public claims require a non-blank source_agent principal.") + return normalized_visibility, normalized_source + + +def identity_namespace_key( + tenant_id: str | None, + scope: str, + visibility: str, + source_agent: str | None, +) -> tuple[str | None, str, str, str | None]: + """Return the in-memory key used by collision allocators.""" + principal = source_agent if visibility != "public" else None + return tenant_id, scope, visibility, principal + + +def require_unambiguous_identity_row( + rows: list[_Row], + *, + identifier: str, +) -> _Row | None: + """Return the sole visible identity row or reject a missing scope choice.""" + if len(rows) > 1: + raise ValueError( + f"Ambiguous {identifier}; provide an exact claim scope." + ) + return rows[0] if rows else None diff --git a/memorymaster/stores/migrations/0011_postgres_scoped_force_rls.py b/memorymaster/stores/migrations/0011_postgres_scoped_force_rls.py new file mode 100644 index 00000000..fee7a1c9 --- /dev/null +++ b/memorymaster/stores/migrations/0011_postgres_scoped_force_rls.py @@ -0,0 +1,344 @@ +"""Force PostgreSQL RLS with tenant, principal, and scope authorization.""" +from __future__ import annotations + +import hashlib +import json +from typing import Iterable + +VERSION = 11 +DESCRIPTION = "Force scoped PostgreSQL row-level security policies" + +_CURRENT_TENANT = "NULLIF(current_setting('memorymaster.tenant_id', true), '')" +_CURRENT_PRINCIPAL = "NULLIF(current_setting('memorymaster.principal', true), '')" +_ALLOWED_SCOPES = ( + "COALESCE(NULLIF(current_setting('memorymaster.allowed_scopes', true), ''), " + "'[]')::jsonb" +) + +_DENY_ALL_TABLES = ( + "action_proposals", + "external_sources", + "source_items", + "evidence_items", + "media_retry_queue", + "query_cache", + "miner_state", + "rule_stats", +) + +_KNOWN_POLICY_NAMES = ( + "memorymaster_rls_permit", + "memorymaster_tenant_restrict", + "memorymaster_team_deny", + "memorymaster_tenant_select", + "memorymaster_tenant_insert", + "memorymaster_tenant_update", + "memorymaster_tenant_delete", + "memorymaster_tenant_select_permit", + "memorymaster_tenant_insert_permit", + "memorymaster_tenant_update_permit", + "memorymaster_tenant_delete_permit", +) + +_COMMAND_POLICIES = { + "SELECT": "memorymaster_tenant_select", + "INSERT": "memorymaster_tenant_insert", + "UPDATE": "memorymaster_tenant_update", + "DELETE": "memorymaster_tenant_delete", +} +_POLICY_FIELDS = ( + "schemaname", + "tablename", + "policyname", + "permissive", + "roles", + "cmd", + "qual", + "with_check", +) +_POLICY_MANIFEST_PREFIX = "memorymaster.rls/v1;manifest=0011;sha256=" + +_EVENT_HEAD_FUNCTION = """ +CREATE OR REPLACE FUNCTION public.memorymaster_event_chain_head() +RETURNS TABLE (global_event_hash TEXT, tenant_event_hash TEXT) +LANGUAGE plpgsql +SECURITY DEFINER +SET search_path = pg_catalog, pg_temp +AS $$ +DECLARE + bound_tenant TEXT := NULLIF(current_setting('memorymaster.tenant_id', true), ''); +BEGIN + IF bound_tenant IS NULL THEN + RAISE EXCEPTION 'event chain head requires bound tenant authority' + USING ERRCODE = '42501'; + END IF; + RETURN QUERY SELECT + ( + SELECT event.event_hash + FROM public.events AS event + WHERE event.tenant_id = bound_tenant + AND event.event_hash IS NOT NULL + AND event.hash_algo = 'sha256-tenant-v2' + ORDER BY event.id DESC LIMIT 1 + ), + ( + SELECT event.tenant_event_hash + FROM public.events AS event + WHERE event.tenant_id = bound_tenant + AND event.tenant_event_hash IS NOT NULL + ORDER BY event.id DESC LIMIT 1 + ); +END; +$$ +""".strip() + +_EVENT_HEAD_REVOKE = ( + "REVOKE ALL ON FUNCTION public.memorymaster_event_chain_head() FROM PUBLIC" +) + + +def _claim_authority_predicate(alias: str) -> str: + return ( + f"{_CURRENT_PRINCIPAL} IS NOT NULL " + f"AND {_CURRENT_TENANT} IS NOT NULL " + f"AND {alias}.tenant_id = {_CURRENT_TENANT} " + f"AND jsonb_typeof({_ALLOWED_SCOPES}) = 'array' " + f"AND {_ALLOWED_SCOPES} ? {alias}.scope" + ) + + +def _claim_read_predicate(alias: str) -> str: + return ( + f"{_claim_authority_predicate(alias)} " + f"AND ({alias}.visibility = 'public' " + f"OR ({alias}.visibility = 'private' " + f"AND {alias}.source_agent = {_CURRENT_PRINCIPAL}))" + ) + + +def _claim_write_predicate(alias: str) -> str: + return ( + f"{_claim_authority_predicate(alias)} " + f"AND {alias}.source_agent = {_CURRENT_PRINCIPAL} " + f"AND {alias}.visibility IN ('public', 'private')" + ) + + +def _claim_exists( + table: str, + claim_column: str, + alias: str, + *, + write: bool, +) -> str: + predicate = ( + _claim_write_predicate(alias) + if write + else _claim_read_predicate(alias) + ) + return ( + f"EXISTS (SELECT 1 FROM claims AS {alias} " + f"WHERE {alias}.id = {table}.{claim_column} " + f"AND {predicate})" + ) + + +def _claim_pair_predicate( + table: str, + left_column: str, + right_column: str, + left_alias: str, + right_alias: str, + *, + write: bool, +) -> str: + return ( + f"{_claim_exists(table, left_column, left_alias, write=write)} AND " + f"{_claim_exists(table, right_column, right_alias, write=write)}" + ) + + +def _event_predicate(*, write: bool) -> str: + return ( + f"{_CURRENT_PRINCIPAL} IS NOT NULL " + f"AND {_CURRENT_TENANT} IS NOT NULL " + f"AND events.tenant_id = {_CURRENT_TENANT} " + "AND (events.claim_id IS NULL OR " + f"{_claim_exists('events', 'claim_id', 'mm_claim', write=write)})" + ) + + +_READ_PREDICATES = { + "claims": _claim_read_predicate("claims"), + "citations": _claim_exists( + "citations", "claim_id", "mm_claim", write=False + ), + "events": _event_predicate(write=False), + "claim_links": _claim_pair_predicate( + "claim_links", + "source_id", + "target_id", + "mm_source", + "mm_target", + write=False, + ), + "claim_embeddings": _claim_exists( + "claim_embeddings", "claim_id", "mm_claim", write=False + ), + "contradiction_verdicts": _claim_pair_predicate( + "contradiction_verdicts", + "claim_a_id", + "claim_b_id", + "mm_a", + "mm_b", + write=False, + ), + "mcp_usage": ( + f"{_CURRENT_PRINCIPAL} IS NOT NULL " + f"AND mcp_usage.tenant_id = {_CURRENT_TENANT}" + ), +} + +_WRITE_PREDICATES = { + "claims": _claim_write_predicate("claims"), + "citations": _claim_exists( + "citations", "claim_id", "mm_claim", write=True + ), + "events": _event_predicate(write=True), + "claim_links": _claim_pair_predicate( + "claim_links", + "source_id", + "target_id", + "mm_source", + "mm_target", + write=True, + ), + "claim_embeddings": _claim_exists( + "claim_embeddings", "claim_id", "mm_claim", write=True + ), + "contradiction_verdicts": _claim_pair_predicate( + "contradiction_verdicts", + "claim_a_id", + "claim_b_id", + "mm_a", + "mm_b", + write=True, + ), + "mcp_usage": ( + f"{_CURRENT_PRINCIPAL} IS NOT NULL " + f"AND mcp_usage.tenant_id = {_CURRENT_TENANT}" + ), +} + + +def _prepare_table(cur, table: str) -> None: + # Identifiers come only from the immutable constants above, never user input. + cur.execute(f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY") + cur.execute(f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY") + for old_policy in _KNOWN_POLICY_NAMES: + cur.execute(f"DROP POLICY IF EXISTS {old_policy} ON {table}") + + +def _install_command_policy( + cur, + table: str, + command: str, + predicate: str, +) -> None: + policy_name = _COMMAND_POLICIES[command] + if command == "INSERT": + clauses = f"WITH CHECK ({predicate})" + elif command == "UPDATE": + clauses = f"USING ({predicate}) WITH CHECK ({predicate})" + else: + clauses = f"USING ({predicate})" + cur.execute( + f"CREATE POLICY {policy_name}_permit ON {table} " + f"AS PERMISSIVE FOR {command} TO PUBLIC {clauses}" + ) + cur.execute( + f"CREATE POLICY {policy_name} ON {table} " + f"AS RESTRICTIVE FOR {command} TO PUBLIC {clauses}" + ) + + +def _install_scoped_policies(cur, table: str) -> None: + _prepare_table(cur, table) + _install_command_policy(cur, table, "SELECT", _READ_PREDICATES[table]) + for command in ("INSERT", "UPDATE", "DELETE"): + _install_command_policy(cur, table, command, _WRITE_PREDICATES[table]) + + +def _install_deny_policy(cur, table: str) -> None: + _prepare_table(cur, table) + cur.execute( + f"CREATE POLICY memorymaster_team_deny ON {table} " + "AS RESTRICTIVE FOR ALL TO PUBLIC USING (FALSE) WITH CHECK (FALSE)" + ) + + +def _canonical_policy_payload(rows: Iterable[dict[str, object]]) -> str: + payload: list[dict[str, object]] = [] + for policy in rows: + row = {field: policy.get(field) for field in _POLICY_FIELDS} + roles = row["roles"] + if isinstance(roles, (list, tuple, set, frozenset)): + row["roles"] = sorted(str(role) for role in roles) + payload.append(row) + payload.sort( + key=lambda row: ( + str(row["schemaname"]), + str(row["tablename"]), + str(row["policyname"]), + ) + ) + return json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + + +def _stamp_policy_manifest(cur) -> None: + governed_tables = list(_READ_PREDICATES) + list(_DENY_ALL_TABLES) + cur.execute( + """ + SELECT schemaname, tablename, policyname, permissive, + roles, cmd, qual, with_check + FROM pg_policies + WHERE schemaname = current_schema() AND tablename = ANY(%s) + """, + (governed_tables,), + ) + rows = list(cur.fetchall()) + expected_count = len(_READ_PREDICATES) * 8 + len(_DENY_ALL_TABLES) + if len(rows) != expected_count: + raise RuntimeError("Postgres RLS policy manifest inventory is incomplete.") + payload = _canonical_policy_payload(rows).encode("utf-8") + comment = f"{_POLICY_MANIFEST_PREFIX}{hashlib.sha256(payload).hexdigest()}" + # The value is entirely internal fixed text + a lowercase SHA-256 digest. + # PostgreSQL DDL cannot use psycopg server-side bind parameters here. + cur.execute( + "COMMENT ON POLICY memorymaster_tenant_select ON claims " + f"IS '{comment}'" + ) + + +def apply_sqlite(_conn) -> None: + """RLS is PostgreSQL-specific; SQLite remains predicate-isolated.""" + + +def apply_postgres(conn) -> None: + try: + with conn.cursor() as cur: + for table in _READ_PREDICATES: + _install_scoped_policies(cur, table) + for table in _DENY_ALL_TABLES: + _install_deny_policy(cur, table) + cur.execute("DROP TRIGGER IF EXISTS claims_gen_ins_del ON claims") + cur.execute("DROP TRIGGER IF EXISTS claims_gen_upd ON claims") + cur.execute(_EVENT_HEAD_FUNCTION) + cur.execute(_EVENT_HEAD_REVOKE) + _stamp_policy_manifest(cur) + conn.commit() + except Exception: + rollback = getattr(conn, "rollback", None) + if callable(rollback): + rollback() + raise diff --git a/memorymaster/stores/migrations/0012_principal_local_claim_identities.py b/memorymaster/stores/migrations/0012_principal_local_claim_identities.py new file mode 100644 index 00000000..7d0cae0b --- /dev/null +++ b/memorymaster/stores/migrations/0012_principal_local_claim_identities.py @@ -0,0 +1,213 @@ +"""Partition claim identities by scope, visibility, and principal.""" +from __future__ import annotations + + +VERSION = 12 +DESCRIPTION = "Scope- and principal-local claim identities" + +POSTGRES_IDENTITY_PREFLIGHT_SQL = """ +SELECT visibility, COUNT(*) AS ownerless_claims +FROM claims +WHERE NULLIF(BTRIM(source_agent), '') IS NULL +GROUP BY visibility +ORDER BY visibility +""".strip() + +POSTGRES_SUPERSESSION_PREFLIGHT_SQL = """ +SELECT COUNT(*) AS invalid_supersession_edges +FROM claims AS claim +LEFT JOIN claims AS superseded + ON superseded.id = claim.supersedes_claim_id +LEFT JOIN claims AS replacement + ON replacement.id = claim.replaced_by_claim_id +WHERE ( + claim.supersedes_claim_id IS NOT NULL + AND ( + claim.supersedes_claim_id = claim.id + OR superseded.id IS NULL + OR superseded.tenant_id IS DISTINCT FROM claim.tenant_id + OR superseded.scope IS DISTINCT FROM claim.scope + OR superseded.visibility IS DISTINCT FROM claim.visibility + OR superseded.source_agent IS DISTINCT FROM claim.source_agent + OR superseded.replaced_by_claim_id IS DISTINCT FROM claim.id + ) +) OR ( + claim.replaced_by_claim_id IS NOT NULL + AND ( + claim.replaced_by_claim_id = claim.id + OR replacement.id IS NULL + OR replacement.tenant_id IS DISTINCT FROM claim.tenant_id + OR replacement.scope IS DISTINCT FROM claim.scope + OR replacement.visibility IS DISTINCT FROM claim.visibility + OR replacement.source_agent IS DISTINCT FROM claim.source_agent + OR replacement.supersedes_claim_id IS DISTINCT FROM claim.id + ) +) +""".strip() + +_SUPERSESSION_GUARD_FUNCTION = """ +CREATE OR REPLACE FUNCTION public.memorymaster_claim_supersession_guard() +RETURNS trigger +LANGUAGE plpgsql +AS $$ +DECLARE + reference_id BIGINT; +BEGIN + FOREACH reference_id IN ARRAY ARRAY[ + NEW.supersedes_claim_id, + NEW.replaced_by_claim_id + ] LOOP + IF reference_id IS NOT NULL AND ( + reference_id = NEW.id + OR NOT EXISTS ( + SELECT 1 + FROM public.claims AS referenced + WHERE referenced.id = reference_id + AND referenced.tenant_id IS NOT DISTINCT FROM NEW.tenant_id + AND referenced.scope = NEW.scope + AND referenced.visibility IS NOT DISTINCT FROM NEW.visibility + AND referenced.source_agent IS NOT DISTINCT FROM NEW.source_agent + ) + ) THEN + RAISE EXCEPTION 'supersession reference is outside the authorized boundary' + USING ERRCODE = '42501'; + END IF; + END LOOP; + RETURN NEW; +END; +$$ +""".strip() + +_SUPERSESSION_GUARD_TRIGGER = """ +DROP TRIGGER IF EXISTS trg_claims_supersession_boundary ON claims; +CREATE TRIGGER trg_claims_supersession_boundary +BEFORE INSERT OR UPDATE OF tenant_id, scope, visibility, source_agent, + supersedes_claim_id, replaced_by_claim_id ON claims +FOR EACH ROW +EXECUTE FUNCTION public.memorymaster_claim_supersession_guard(); +""".strip() + + +_DROP_LEGACY_SQLITE = """ +DROP INDEX IF EXISTS idx_claims_tenant_idempotency_key; +DROP INDEX IF EXISTS idx_claims_tenant_human_id; +DROP INDEX IF EXISTS idx_claims_confirmed_tuple_unique; +DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard_insert; +DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard_update; +""" + +_IDENTITY_INDEXES_SQLITE = """ +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_idempotency_key_unique + ON claims(COALESCE(tenant_id, ''), scope, idempotency_key) + WHERE visibility = 'public' AND idempotency_key IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_idempotency_key_unique + ON claims(COALESCE(tenant_id, ''), scope, visibility, source_agent, idempotency_key) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND idempotency_key IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_human_id_unique + ON claims(COALESCE(tenant_id, ''), scope, human_id) + WHERE visibility = 'public' AND human_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_human_id_unique + ON claims(COALESCE(tenant_id, ''), scope, visibility, source_agent, human_id) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND human_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_confirmed_tuple_unique + ON claims(COALESCE(tenant_id, ''), subject, predicate, scope) + WHERE visibility = 'public' AND status = 'confirmed' + AND subject IS NOT NULL AND predicate IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_confirmed_tuple_unique + ON claims(COALESCE(tenant_id, ''), visibility, source_agent, subject, predicate, scope) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND status = 'confirmed' AND subject IS NOT NULL AND predicate IS NOT NULL; +""" + +_IDENTITY_GUARDS_SQLITE = """ +CREATE TRIGGER IF NOT EXISTS trg_claims_identity_guard_insert +BEFORE INSERT ON claims +WHEN NEW.visibility NOT IN ('public', 'private', 'sensitive') + OR (NEW.visibility <> 'public' AND NULLIF(TRIM(NEW.source_agent), '') IS NULL) +BEGIN + SELECT RAISE(ABORT, 'invalid claim visibility or missing non-public source_agent'); +END; +CREATE TRIGGER IF NOT EXISTS trg_claims_identity_guard_update +BEFORE UPDATE ON claims +WHEN NEW.visibility NOT IN ('public', 'private', 'sensitive') + OR (NEW.visibility <> 'public' AND NULLIF(TRIM(NEW.source_agent), '') IS NULL) +BEGIN + SELECT RAISE(ABORT, 'invalid claim visibility or missing non-public source_agent'); +END; +""" + +_POSTGRES_DDL = """ +DROP INDEX IF EXISTS idx_claims_tenant_idempotency_key; +DROP INDEX IF EXISTS idx_claims_tenant_human_id; +DROP INDEX IF EXISTS idx_claims_confirmed_tuple_unique; +DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard ON claims; +DROP FUNCTION IF EXISTS memorymaster_claims_confirmed_tuple_guard(); +ALTER TABLE claims DROP CONSTRAINT IF EXISTS ck_claims_identity_visibility_owner; +ALTER TABLE claims ADD CONSTRAINT ck_claims_identity_visibility_owner + CHECK ( + visibility IN ('public', 'private', 'sensitive') + AND NULLIF(BTRIM(source_agent), '') IS NOT NULL + ) NOT VALID; +ALTER TABLE claims VALIDATE CONSTRAINT ck_claims_identity_visibility_owner; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_idempotency_key_unique + ON claims(COALESCE(tenant_id, ''), scope, idempotency_key) + WHERE visibility = 'public' AND idempotency_key IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_idempotency_key_unique + ON claims(COALESCE(tenant_id, ''), scope, visibility, source_agent, idempotency_key) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND idempotency_key IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_human_id_unique + ON claims(COALESCE(tenant_id, ''), scope, human_id) + WHERE visibility = 'public' AND human_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_human_id_unique + ON claims(COALESCE(tenant_id, ''), scope, visibility, source_agent, human_id) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND human_id IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_confirmed_tuple_unique + ON claims(COALESCE(tenant_id, ''), subject, predicate, scope) + WHERE visibility = 'public' AND status = 'confirmed' + AND subject IS NOT NULL AND predicate IS NOT NULL; +CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_confirmed_tuple_unique + ON claims(COALESCE(tenant_id, ''), visibility, source_agent, subject, predicate, scope) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND status = 'confirmed' AND subject IS NOT NULL AND predicate IS NOT NULL; +""" + + +def apply_sqlite(conn) -> None: + has_claims = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'claims'" + ).fetchone() + if has_claims is None: + return + conn.executescript( + _DROP_LEGACY_SQLITE + _IDENTITY_INDEXES_SQLITE + _IDENTITY_GUARDS_SQLITE + ) + conn.commit() + + +def apply_postgres(conn) -> None: + with conn.cursor() as cur: + cur.execute(POSTGRES_SUPERSESSION_PREFLIGHT_SQL) + row = cur.fetchone() + if isinstance(row, dict): + invalid_edges = int(row.get("invalid_supersession_edges") or 0) + elif row is not None: + invalid_edges = int(row[0]) + else: + raise RuntimeError("Postgres supersession preflight returned no result.") + if invalid_edges: + raise RuntimeError( + f"Postgres migration refused {invalid_edges} invalid supersession " + "edge(s); inventory and repair require explicit approval." + ) + for statement in _POSTGRES_DDL.split(";"): + if statement.strip(): + cur.execute(statement) + cur.execute(_SUPERSESSION_GUARD_FUNCTION) + for statement in _SUPERSESSION_GUARD_TRIGGER.split(";"): + if statement.strip(): + cur.execute(statement) + conn.commit() diff --git a/memorymaster/stores/postgres_policy_contract.py b/memorymaster/stores/postgres_policy_contract.py new file mode 100644 index 00000000..851d65e0 --- /dev/null +++ b/memorymaster/stores/postgres_policy_contract.py @@ -0,0 +1,182 @@ +"""Application-owned semantic fingerprints for PostgreSQL RLS predicates.""" +from __future__ import annotations + +import importlib +import re +from collections.abc import Iterable, Mapping + + +_TEXT_CAST_RE = re.compile( + r"::\s*(?:pg_catalog\.)?(?:text|character\s+varying)\b", + re.IGNORECASE, +) +_ANY_ARRAY_RE = re.compile( + r"(?P[a-z_][a-z0-9_.]*)\s*=\s*any\s*" + r"\(\s*array\[(?P[^\]]+)]\s*\)", + re.IGNORECASE, +) +_FROM_AS_RE = re.compile( + r"\b(from|join)\s+([a-z_][a-z0-9_.]*)\s+as\s+", + re.IGNORECASE, +) +_SAFE_QUOTED_IDENTIFIER_RE = re.compile(r"[a-z_][a-z0-9_]*") + + +def _protect_sql_quotes(value: str) -> tuple[str, tuple[str, ...]]: + output: list[str] = [] + protected: list[str] = [] + index = 0 + while index < len(value): + quote = value[index] + if quote not in {"'", '"'}: + output.append(quote) + index += 1 + continue + end = index + 1 + while end < len(value): + if value[end] != quote: + end += 1 + continue + if end + 1 < len(value) and value[end + 1] == quote: + end += 2 + continue + end += 1 + break + token = value[index:end] + inner = token[1:-1] + if quote == '"' and _SAFE_QUOTED_IDENTIFIER_RE.fullmatch(inner): + output.append(inner) + else: + marker = f"__mm_sql_quoted_{len(protected)}__" + protected.append(token) + output.append(marker) + index = end + return "".join(output), tuple(protected) + + +def _restore_sql_quotes(value: str, protected: tuple[str, ...]) -> str: + for index, token in enumerate(protected): + value = value.replace(f"__mm_sql_quoted_{index}__", token) + return value + + +def canonicalize_sql_tokens(value: object, *, drop_parentheses: bool = False) -> str: + """Normalize SQL syntax without changing quoted literal/identifier bytes.""" + text, protected = _protect_sql_quotes(str(value or "")) + text = text.lower() + if drop_parentheses: + text = text.replace("(", " ").replace(")", " ") + return _restore_sql_quotes(" ".join(text.split()), protected) + + +def _normalize_deparse_variants(value: object) -> str: + text, protected = _protect_sql_quotes(str(value or "")) + text = text.lower() + text = _TEXT_CAST_RE.sub("", text) + text = _ANY_ARRAY_RE.sub( + lambda match: f"{match.group('left')} in ({match.group('values')})", + text, + ) + text = _FROM_AS_RE.sub(r"\1 \2 ", text) + return _restore_sql_quotes(" ".join(text.split()), protected) + + +def _flat_token_signature(value: object) -> str: + text = _normalize_deparse_variants(value) + return canonicalize_sql_tokens(text, drop_parentheses=True) + + +def _or_offsets(value: str) -> list[int]: + offsets: list[int] = [] + quoted = False + index = 0 + while index < len(value): + char = value[index] + if char == "'": + if quoted and index + 1 < len(value) and value[index + 1] == "'": + index += 2 + continue + quoted = not quoted + elif not quoted and value[index:index + 2] == "or": + before = value[index - 1] if index else " " + after = value[index + 2] if index + 2 < len(value) else " " + if not (before.isalnum() or before == "_") and not ( + after.isalnum() or after == "_" + ): + offsets.append(index) + index += 1 + return offsets + + +def _smallest_parenthesized_group(value: str, offset: int) -> str | None: + stack: list[int] = [] + pairs: list[tuple[int, int]] = [] + quoted = False + index = 0 + while index < len(value): + char = value[index] + if char == "'": + if quoted and index + 1 < len(value) and value[index + 1] == "'": + index += 2 + continue + quoted = not quoted + elif not quoted and char == "(": + stack.append(index) + elif not quoted and char == ")" and stack: + pairs.append((stack.pop(), index)) + index += 1 + enclosing = [pair for pair in pairs if pair[0] < offset < pair[1]] + if not enclosing: + return None + start, end = min(enclosing, key=lambda pair: pair[1] - pair[0]) + return value[start + 1:end] + + +def _or_group_signatures(value: object) -> tuple[str, ...]: + normalized = _normalize_deparse_variants(value) + groups: list[str] = [] + for offset in _or_offsets(normalized): + group = _smallest_parenthesized_group(normalized, offset) + if group is None: + groups.append("") + else: + groups.append(_flat_token_signature(group)) + return tuple(groups) + + +def expressions_match(actual: object, expected: object) -> bool: + """Compare exact tokens plus the security-relevant grouping of every OR.""" + if actual is None or expected is None: + return actual is expected + return ( + _flat_token_signature(actual) == _flat_token_signature(expected) + and _or_group_signatures(actual) == _or_group_signatures(expected) + ) + + +def expected_policy_expressions( + tenant_tables: Iterable[str], + deny_tables: Iterable[str], + command_policies: Mapping[str, str], + permit_policies: Mapping[str, str], +) -> dict[tuple[str, str], tuple[str | None, str | None]]: + migration = importlib.import_module( + "memorymaster.stores.migrations.0011_postgres_scoped_force_rls" + ) + expected: dict[tuple[str, str], tuple[str | None, str | None]] = {} + for table in tenant_tables: + for command, restrictive_name in command_policies.items(): + predicate = ( + migration._READ_PREDICATES[table] + if command == "SELECT" + else migration._WRITE_PREDICATES[table] + ) + expressions = ( + None if command == "INSERT" else predicate, + predicate if command in {"INSERT", "UPDATE"} else None, + ) + expected[(table, restrictive_name)] = expressions + expected[(table, permit_policies[command])] = expressions + for table in deny_tables: + expected[(table, "memorymaster_team_deny")] = ("FALSE", "FALSE") + return expected diff --git a/memorymaster/stores/postgres_store.py b/memorymaster/stores/postgres_store.py index 624b09b0..9b66c793 100644 --- a/memorymaster/stores/postgres_store.py +++ b/memorymaster/stores/postgres_store.py @@ -3,8 +3,9 @@ import hashlib import json from datetime import datetime, timedelta, timezone -from typing import Any +from typing import Any, Iterable +from memorymaster.core.lifecycle import can_transition from memorymaster.recall.embeddings import EmbeddingProvider, cosine_similarity from memorymaster.core.models import ( ActionProposal, @@ -27,11 +28,21 @@ ) from memorymaster.core.retry import connect_with_retry from memorymaster.stores._storage_shared import ( + ConcurrentModificationError, EVENT_HASH_ALGO, TENANT_EVENT_HASH_ALGO, compute_tenant_event_hash, generate_top_level_human_id, ) +from memorymaster.stores.claim_identity import ( + normalize_claim_identity, + require_unambiguous_identity_row, +) +from memorymaster.stores.postgres_policy_contract import ( + canonicalize_sql_tokens, + expected_policy_expressions, + expressions_match, +) from memorymaster.stores.storage import SQLiteStore POSTGRES_EVENTS_APPEND_ONLY_TRIGGERS = ( @@ -40,6 +51,81 @@ ) POSTGRES_CONFIRMED_TUPLE_GUARD_TRIGGER = "trg_claims_confirmed_tuple_guard" POSTGRES_TENANT_EVENT_HASH_ALGO = TENANT_EVENT_HASH_ALGO +POSTGRES_TENANT_POLICY_TABLES = ( + "claims", + "citations", + "events", + "claim_links", + "claim_embeddings", + "contradiction_verdicts", + "mcp_usage", +) +POSTGRES_TEAM_DENY_TABLES = ( + "action_proposals", + "external_sources", + "source_items", + "evidence_items", + "media_retry_queue", + "query_cache", + "miner_state", + "rule_stats", +) +POSTGRES_PROTECTED_TABLES = POSTGRES_TENANT_POLICY_TABLES + POSTGRES_TEAM_DENY_TABLES +POSTGRES_AUTHORITY_GUCS = ( + "memorymaster.tenant_id", + "memorymaster.principal", + "memorymaster.allowed_scopes", +) +POSTGRES_COMMAND_POLICIES = { + "SELECT": "memorymaster_tenant_select", + "INSERT": "memorymaster_tenant_insert", + "UPDATE": "memorymaster_tenant_update", + "DELETE": "memorymaster_tenant_delete", +} +POSTGRES_PERMIT_POLICIES = { + command: f"{name}_permit" + for command, name in POSTGRES_COMMAND_POLICIES.items() +} +POSTGRES_POLICY_FIELDS = ( + "schemaname", + "tablename", + "policyname", + "permissive", + "roles", + "cmd", + "qual", + "with_check", +) +POSTGRES_POLICY_MANIFEST_PREFIX = "memorymaster.rls/v1;manifest=0011;sha256=" +POSTGRES_METADATA_TABLES = ("cache_meta", "schema_versions") +POSTGRES_CLAIM_IDENTITY_INDEXES = frozenset( + { + "idx_claims_public_idempotency_key_unique", + "idx_claims_nonpublic_principal_idempotency_key_unique", + "idx_claims_public_human_id_unique", + "idx_claims_nonpublic_principal_human_id_unique", + "idx_claims_public_confirmed_tuple_unique", + "idx_claims_nonpublic_principal_confirmed_tuple_unique", + } +) +POSTGRES_HUMAN_IDENTITY_INDEXES = frozenset( + { + "idx_claims_public_human_id_unique", + "idx_claims_nonpublic_principal_human_id_unique", + } +) +POSTGRES_CLAIM_OWNER_CONSTRAINT = "ck_claims_identity_visibility_owner" +POSTGRES_CLAIM_OWNER_CHECK = ( + "CHECK (visibility IN ('public', 'private', 'sensitive') " + "AND NULLIF(BTRIM(source_agent), '') IS NOT NULL)" +) +POSTGRES_EVENT_GUARD_SOURCE = """ +BEGIN + RAISE EXCEPTION 'events table is append-only; % is not allowed', TG_OP; +END; +""".strip() +POSTGRES_SUPERSESSION_GUARD_TRIGGER = "trg_claims_supersession_boundary" +POSTGRES_SUPERSESSION_GUARD_FUNCTION = "memorymaster_claim_supersession_guard" def utc_now() -> datetime: @@ -53,13 +139,869 @@ def __init__( *, tenant_id: str | None = None, require_tenant: bool = False, + principal: str | None = None, + allowed_scopes: Iterable[str] | None = None, ) -> None: self.dsn = dsn self.tenant_id = (tenant_id or "").strip() or None self.require_tenant = bool(require_tenant) + self.principal = (principal or "").strip() or None + self.allowed_scopes = frozenset( + scope.strip() + for scope in (allowed_scopes or ()) + if scope and scope.strip() + ) self._psycopg: Any = None self._vector_table_available: bool | None = None + def _require_team_authority(self) -> tuple[str, str, tuple[str, ...]]: + if self.tenant_id is None: + raise PermissionError("Postgres team mode requires a tenant context.") + if self.principal is None: + raise PermissionError("Postgres team mode requires an authenticated principal.") + if not self.allowed_scopes: + raise PermissionError("Postgres team mode requires explicit allowed scopes.") + if any("*" in scope for scope in self.allowed_scopes): + raise PermissionError("Postgres team scopes cannot contain wildcards.") + return self.tenant_id, self.principal, tuple(sorted(self.allowed_scopes)) + + @staticmethod + def _cleanup_failed_connection(conn) -> None: + try: + rollback = getattr(conn, "rollback", None) + if callable(rollback): + rollback() + finally: + conn.close() + + @staticmethod + def _row_bool(row: dict[str, object], *names: str) -> bool: + return any(bool(row.get(name)) for name in names) + + @staticmethod + def _canonical_catalog_sql(value: object) -> str: + return canonicalize_sql_tokens(value) + + @staticmethod + def _canonical_identity_sql(value: object) -> str: + return canonicalize_sql_tokens(value, drop_parentheses=True) + + @staticmethod + def _canonical_ddl(value: object) -> str: + normalized = canonicalize_sql_tokens(value) + normalized = normalized.replace("public.", "") + return " ".join(normalized.rstrip(" ;").split()) + + @staticmethod + def _policy_roles(row: dict[str, object]) -> set[str]: + raw = row.get("roles") + if isinstance(raw, str): + return {part.strip() for part in raw.strip("{}").split(",") if part.strip()} + if isinstance(raw, (list, tuple, set, frozenset)): + return {str(part) for part in raw} + return set() + + @staticmethod + def _policy_is_restrictive(row: dict[str, object]) -> bool: + permissive = row.get("permissive") + if isinstance(permissive, str): + return permissive.upper() == "RESTRICTIVE" + return row.get("polpermissive") is False + + @classmethod + def _validate_runtime_role(cls, cur) -> None: + cur.execute( + """ + SELECT current_user, session_user, rolname, rolsuper, rolbypassrls, + rolreplication, rolcreaterole, rolcreatedb, + EXISTS ( + SELECT 1 FROM pg_roles AS privileged + WHERE privileged.rolname <> current_user + AND (privileged.rolsuper OR privileged.rolbypassrls) + AND pg_has_role(current_user, privileged.oid, 'SET') + ) AS member_of_privileged_role + FROM pg_roles WHERE rolname = current_user + """ + ) + row = cur.fetchone() + if not isinstance(row, dict): + raise PermissionError("Postgres runtime role could not be verified.") + if row.get("current_user") != row.get("session_user"): + raise PermissionError("Postgres runtime role cannot use session impersonation.") + if bool(row.get("rolsuper")): + raise PermissionError("Postgres runtime role cannot be a superuser.") + if bool(row.get("rolbypassrls")): + raise PermissionError("Postgres runtime role cannot have BYPASSRLS.") + if bool(row.get("rolreplication")): + raise PermissionError("Postgres runtime role cannot have REPLICATION.") + if bool(row.get("rolcreaterole")): + raise PermissionError("Postgres runtime role cannot have CREATEROLE.") + if bool(row.get("rolcreatedb")): + raise PermissionError("Postgres runtime role cannot have CREATEDB.") + if bool(row.get("member_of_privileged_role")): + raise PermissionError( + "Postgres runtime role cannot be a member of a privileged superuser/BYPASSRLS role." + ) + cur.execute( + "SELECT has_schema_privilege(current_user, current_schema(), 'CREATE') " + "AS public_schema_create" + ) + schema_row = cur.fetchone() + if isinstance(schema_row, dict) and cls._row_bool( + schema_row, "public_schema_create", "can_create_public" + ): + raise PermissionError("Postgres runtime role cannot have schema CREATE privilege.") + + @classmethod + def _validate_runtime_tables(cls, cur) -> None: + by_name = cls._runtime_table_catalog(cur) + for table, row in by_name.items(): + cls._validate_runtime_table_contract(table, row) + + @classmethod + def _runtime_table_catalog(cls, cur) -> dict[str, dict[str, object]]: + cur.execute( + """ + SELECT c.relname AS table_name, c.relrowsecurity, c.relforcerowsecurity, + pg_get_userbyid(c.relowner) AS owner_name, + pg_has_role(current_user, c.relowner, 'MEMBER') AS owner_member, + has_table_privilege(current_user, c.oid, 'TRUNCATE') AS can_truncate, + has_table_privilege(current_user, c.oid, 'REFERENCES') AS can_references, + has_table_privilege(current_user, c.oid, 'TRIGGER') AS can_trigger, + has_table_privilege(current_user, c.oid, 'SELECT') AS can_select, + has_table_privilege(current_user, c.oid, 'INSERT') AS can_insert, + has_table_privilege(current_user, c.oid, 'UPDATE') AS can_update, + has_any_column_privilege(current_user, c.oid, 'UPDATE') + AS can_update_any_column, + has_table_privilege(current_user, c.oid, 'DELETE') AS can_delete + FROM pg_class AS c + JOIN pg_namespace AS n ON n.oid = c.relnamespace + WHERE n.nspname = current_schema() AND c.relname = ANY(%s) + """, + (list(POSTGRES_PROTECTED_TABLES),), + ) + rows = cur.fetchall() + by_name = {str(row.get("table_name") or row.get("relname")): row for row in rows} + if set(by_name) != set(POSTGRES_PROTECTED_TABLES): + raise PermissionError("Postgres runtime requires all 15 protected tables.") + return by_name + + @classmethod + def _validate_runtime_table_contract( + cls, + table: str, + row: dict[str, object], + ) -> None: + if not bool(row.get("relrowsecurity")) or not bool(row.get("relforcerowsecurity")): + raise PermissionError(f"Postgres table {table} must ENABLE and FORCE RLS.") + if cls._row_bool(row, "owner_member", "is_owner_member"): + raise PermissionError(f"Postgres runtime role cannot own {table} or its owner role.") + for privilege in ("truncate", "references", "trigger"): + if cls._row_bool(row, f"can_{privilege}", f"has_{privilege}"): + raise PermissionError( + f"Postgres runtime role cannot have {privilege.upper()} on {table}." + ) + if table == "events": + cls._validate_event_table_privileges(row) + if table in POSTGRES_TEAM_DENY_TABLES: + cls._validate_team_deny_table_privileges(table, row) + + @classmethod + def _validate_event_table_privileges(cls, row: dict[str, object]) -> None: + for privilege in ("select", "insert"): + if not cls._row_bool(row, f"can_{privilege}", f"has_{privilege}"): + raise PermissionError( + f"Postgres runtime role requires {privilege.upper()} " + "on append-only events." + ) + if cls._row_bool(row, "can_update_any_column", "has_update_any_column"): + raise PermissionError( + "Postgres runtime role cannot UPDATE append-only event columns." + ) + for privilege in ("update", "delete"): + if cls._row_bool(row, f"can_{privilege}", f"has_{privilege}"): + raise PermissionError( + f"Postgres runtime role cannot {privilege.upper()} append-only events." + ) + + @classmethod + def _validate_team_deny_table_privileges( + cls, + table: str, + row: dict[str, object], + ) -> None: + for privilege in ("insert", "update", "delete"): + if cls._row_bool(row, f"can_{privilege}", f"has_{privilege}"): + raise PermissionError( + f"Postgres runtime role cannot have {privilege.upper()} on " + f"team-deny table {table}." + ) + + @classmethod + def _validate_runtime_metadata_tables(cls, cur) -> None: + cur.execute( + """ + SELECT c.relname AS table_name, + has_table_privilege(current_user, c.oid, 'SELECT') AS can_select, + has_table_privilege(current_user, c.oid, 'INSERT') AS can_insert, + has_table_privilege(current_user, c.oid, 'UPDATE') AS can_update, + has_table_privilege(current_user, c.oid, 'DELETE') AS can_delete, + has_table_privilege(current_user, c.oid, 'TRUNCATE') AS can_truncate, + has_table_privilege(current_user, c.oid, 'REFERENCES') AS can_references, + has_table_privilege(current_user, c.oid, 'TRIGGER') AS can_trigger + FROM pg_class AS c + JOIN pg_namespace AS n ON n.oid = c.relnamespace + WHERE n.nspname = current_schema() AND c.relname = ANY(%s) + """, + (list(POSTGRES_METADATA_TABLES),), + ) + rows = cur.fetchall() + by_name = {str(row.get("table_name") or row.get("relname")): row for row in rows} + if set(by_name) != set(POSTGRES_METADATA_TABLES): + raise PermissionError("Postgres runtime requires both metadata tables.") + for table, row in by_name.items(): + if not cls._row_bool(row, "can_select", "has_select"): + raise PermissionError(f"Postgres runtime requires SELECT on {table}.") + for privilege in ("insert", "update", "delete", "truncate", "references", "trigger"): + if cls._row_bool(row, f"can_{privilege}", f"has_{privilege}"): + raise PermissionError( + f"Postgres runtime role cannot have {privilege.upper()} on {table}." + ) + + @classmethod + def _validate_confirmed_tuple_index(cls, cur) -> None: + """Compatibility alias for the v12 six-index identity validator.""" + cls._validate_claim_identity_indexes(cur) + + @classmethod + def _validate_claim_owner_constraint(cls, cur) -> None: + cur.execute( + """ + SELECT n.nspname AS schema_name, t.relname AS table_name, + c.conname AS constraint_name, c.contype AS constraint_type, + c.convalidated AS validated, c.conislocal AS is_local, + c.connoinherit AS no_inherit, + pg_get_constraintdef(c.oid, true) AS constraint_definition + FROM pg_constraint AS c + JOIN pg_class AS t ON t.oid = c.conrelid + JOIN pg_namespace AS n ON n.oid = t.relnamespace + WHERE n.nspname = current_schema() AND t.relname = 'claims' + AND c.conname = %s + """, + (POSTGRES_CLAIM_OWNER_CONSTRAINT,), + ) + row = cur.fetchone() + if not isinstance(row, dict): + raise PermissionError("Postgres claim owner constraint is missing.") + metadata = ( + row.get("schema_name") == "public", + row.get("table_name") == "claims", + (row.get("constraint_name") or row.get("conname")) + == POSTGRES_CLAIM_OWNER_CONSTRAINT, + (row.get("constraint_type") or row.get("contype")) == "c", + cls._row_bool(row, "validated", "convalidated"), + cls._row_bool(row, "is_local", "conislocal"), + not cls._row_bool(row, "no_inherit", "connoinherit"), + ) + definition = row.get("constraint_definition") or row.get("definition") + if not all(metadata) or not expressions_match( + definition, + POSTGRES_CLAIM_OWNER_CHECK, + ): + raise PermissionError( + "Postgres claim owner constraint is unsafe or not validated." + ) + + @classmethod + def _expected_claim_identity_catalog(cls) -> dict[str, tuple[str, str]]: + identities = { + "idempotency_key": ("scope, idempotency_key", "idempotency_key IS NOT NULL"), + "human_id": ("scope, human_id", "human_id IS NOT NULL"), + "confirmed_tuple": ( + "subject, predicate, scope", + "status = 'confirmed'::text AND subject IS NOT NULL " + "AND predicate IS NOT NULL", + ), + } + expected: dict[str, tuple[str, str]] = {} + for suffix, (columns, required) in identities.items(): + for namespace in ("public", "nonpublic_principal"): + name = f"idx_claims_{namespace}_{suffix}_unique" + public = namespace == "public" + keys = f"COALESCE(tenant_id, ''::text), {columns}" + predicate = f"visibility = 'public'::text AND {required}" + if not public: + if suffix == "confirmed_tuple": + keys = ( + "COALESCE(tenant_id, ''::text), visibility, source_agent, " + f"{columns}" + ) + else: + keys = ( + "COALESCE(tenant_id, ''::text), scope, visibility, " + f"source_agent, {columns.removeprefix('scope, ')}" + ) + predicate = ( + "visibility <> 'public'::text AND source_agent IS NOT NULL " + f"AND {required}" + ) + definition = ( + f"CREATE UNIQUE INDEX {name} ON public.claims USING btree ({keys}) " + f"WHERE ({predicate})" + ) + expected[name] = (definition, predicate) + return expected + + @classmethod + def _validate_claim_identity_indexes(cls, cur) -> None: + cur.execute( + """ + SELECT i.relname AS index_name, x.indisunique, x.indisvalid, x.indisready, + pg_get_indexdef(i.oid) AS indexdef, + pg_get_expr(x.indpred, x.indrelid, false) AS predicate + FROM pg_index AS x + JOIN pg_class AS i ON i.oid = x.indexrelid + JOIN pg_class AS t ON t.oid = x.indrelid + JOIN pg_namespace AS n ON n.oid = t.relnamespace + WHERE n.nspname = current_schema() AND t.relname = 'claims' + AND x.indisunique AND NOT x.indisprimary + """ + ) + rows = list(cur.fetchall()) + by_name = { + str(row.get("index_name") or row.get("relname")): row for row in rows + } + expected = cls._expected_claim_identity_catalog() + if set(by_name) != set(expected) or len(rows) != len(expected): + raise PermissionError("Postgres claim identity index catalog is unsafe.") + for name, (definition, predicate) in expected.items(): + row = by_name[name] + flags = (("indisunique", "is_unique"), ("indisvalid", "is_valid"), + ("indisready", "is_ready")) + if not all(cls._row_bool(row, primary, alias) for primary, alias in flags): + raise PermissionError(f"Postgres claim identity index {name} is unsafe.") + actual_definition = row.get("indexdef") or row.get("index_definition") + actual_predicate = row.get("predicate") or row.get("index_predicate") + if cls._canonical_identity_sql(actual_definition) != cls._canonical_identity_sql( + definition + ) or cls._canonical_identity_sql(actual_predicate) != cls._canonical_identity_sql( + predicate + ): + raise PermissionError(f"Postgres claim identity index {name} has drifted.") + + @classmethod + def _validate_event_chain_head_function(cls, cur) -> None: + cur.execute( + """ + SELECT n.nspname AS schema_name, p.proname AS function_name, + p.pronargs AS argument_count, + pg_get_function_result(p.oid) AS result_signature, + l.lanname AS language_name, p.prosecdef AS security_definer, + COALESCE(p.proconfig, ARRAY[]::text[]) AS function_config, + p.provolatile AS volatility, p.proparallel AS parallel_safety, + p.proleakproof AS leakproof, p.proisstrict AS strict, + p.prosrc AS function_source, + EXISTS ( + SELECT 1 + FROM aclexplode(COALESCE(p.proacl, acldefault('f', p.proowner))) AS acl + WHERE acl.grantee = 0 AND acl.privilege_type = 'EXECUTE' + ) AS public_execute, + has_function_privilege(current_user, p.oid, 'EXECUTE') AS runtime_execute, + p.proowner = (SELECT oid FROM pg_roles WHERE rolname = current_user) + AS owner_is_runtime, + pg_has_role(current_user, p.proowner, 'MEMBER') AS owner_member, + owner_role.rolsuper AS owner_superuser, + owner_role.rolbypassrls AS owner_bypassrls, + pg_get_functiondef(p.oid) AS function_definition + FROM pg_proc AS p + JOIN pg_namespace AS n ON n.oid = p.pronamespace + JOIN pg_language AS l ON l.oid = p.prolang + JOIN pg_roles AS owner_role ON owner_role.oid = p.proowner + WHERE n.nspname = 'public' + AND p.proname = 'memorymaster_event_chain_head' + AND p.pronargs = 0 + """ + ) + row = cur.fetchone() + if not isinstance(row, dict): + raise PermissionError("Postgres event-chain head function is missing.") + cls._validate_event_chain_head_metadata(row) + + @classmethod + def _validate_event_chain_head_metadata(cls, row: dict[str, object]) -> None: + if row.get("schema_name") != "public" or row.get("function_name") != ( + "memorymaster_event_chain_head" + ): + raise PermissionError("Postgres event-chain head function signature is unsafe.") + argument_count = row.get("argument_count") + if argument_count is None or int(argument_count) != 0: + raise PermissionError("Postgres event-chain head function argument signature is unsafe.") + result = cls._canonical_identity_sql(row.get("result_signature")) + if result != "table global_event_hash text, tenant_event_hash text": + raise PermissionError("Postgres event-chain head result signature is unsafe.") + if str(row.get("language_name") or "").lower() != "plpgsql": + raise PermissionError("Postgres event-chain head language is unsafe.") + if not bool(row.get("security_definer")): + raise PermissionError("Postgres event-chain head function must be SECURITY DEFINER.") + configs = row.get("function_config") or () + if isinstance(configs, str): + configs = (configs,) + normalized_configs = {cls._canonical_catalog_sql(value) for value in configs} + if normalized_configs != {"search_path=pg_catalog, pg_temp"}: + raise PermissionError("Postgres event-chain head function has an unsafe search_path.") + if ( + row.get("volatility") != "v" + or row.get("parallel_safety") != "u" + or bool(row.get("leakproof")) + or bool(row.get("strict")) + ): + raise PermissionError("Postgres event-chain head function catalog has drifted.") + if bool(row.get("public_execute")) or not bool(row.get("runtime_execute")): + raise PermissionError("Postgres event-chain head EXECUTE privileges are unsafe.") + cls._validate_event_chain_head_owner(row) + import importlib + + migration = importlib.import_module( + "memorymaster.stores.migrations.0011_postgres_scoped_force_rls" + ) + expected_source = str(migration._EVENT_HEAD_FUNCTION).split( + "AS $$", 1 + )[1].rsplit("$$", 1)[0].strip() + if cls._canonical_catalog_sql(row.get("function_source")) != ( + cls._canonical_catalog_sql(expected_source) + ): + raise PermissionError("Postgres event-chain head function body has drifted.") + + @staticmethod + def _validate_event_chain_head_owner(row: dict[str, object]) -> None: + if bool(row.get("owner_is_runtime")) or bool(row.get("owner_member")): + raise PermissionError("Postgres runtime role cannot own the event-chain head function.") + if not ( + bool(row.get("owner_superuser")) + or bool(row.get("owner_bypassrls")) + ): + raise PermissionError( + "Postgres event-chain head owner must be SUPERUSER or BYPASSRLS." + ) + + @classmethod + def _validate_event_append_only_catalog(cls, cur) -> None: + cur.execute( + """ + SELECT tg.tgname AS trigger_name, ns.nspname AS table_schema, + tbl.relname AS table_name, tg.tgenabled AS enabled_code, + tg.tgisinternal AS is_internal, fns.nspname AS function_schema, + fn.proname AS function_name, + pg_get_triggerdef(tg.oid, true) AS trigger_definition + FROM pg_trigger AS tg + JOIN pg_class AS tbl ON tbl.oid = tg.tgrelid + JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace + JOIN pg_proc AS fn ON fn.oid = tg.tgfoid + JOIN pg_namespace AS fns ON fns.oid = fn.pronamespace + WHERE ns.nspname = 'public' AND tbl.relname = 'events' + AND NOT tg.tgisinternal + """, + ) + rows = list(cur.fetchall()) + by_name = {str(row.get("trigger_name")): row for row in rows} + if ( + set(by_name) != set(POSTGRES_EVENTS_APPEND_ONLY_TRIGGERS) + or len(rows) != len(POSTGRES_EVENTS_APPEND_ONLY_TRIGGERS) + ): + raise PermissionError("Postgres append-only event trigger catalog is unsafe.") + for operation in ("update", "delete"): + cls._validate_event_trigger_row( + by_name[f"trg_events_append_only_{operation}"], + operation, + ) + cls._validate_event_guard_function(cur) + + @classmethod + def _validate_claim_supersession_guard(cls, cur) -> None: + cur.execute( + """ + SELECT tg.tgname AS trigger_name, ns.nspname AS table_schema, + tbl.relname AS table_name, tg.tgenabled AS enabled_code, + tg.tgisinternal AS is_internal, fns.nspname AS function_schema, + fn.proname AS function_name, + pg_get_triggerdef(tg.oid, true) AS trigger_definition + FROM pg_trigger AS tg + JOIN pg_class AS tbl ON tbl.oid = tg.tgrelid + JOIN pg_namespace AS ns ON ns.oid = tbl.relnamespace + JOIN pg_proc AS fn ON fn.oid = tg.tgfoid + JOIN pg_namespace AS fns ON fns.oid = fn.pronamespace + WHERE ns.nspname = 'public' AND tbl.relname = 'claims' + AND NOT tg.tgisinternal + """, + ) + rows = list(cur.fetchall()) + by_name = {str(row.get("trigger_name")): row for row in rows} + if set(by_name) != {POSTGRES_SUPERSESSION_GUARD_TRIGGER} or len(rows) != 1: + raise PermissionError( + "Postgres claim supersession trigger catalog is unsafe." + ) + row = by_name[POSTGRES_SUPERSESSION_GUARD_TRIGGER] + cls._validate_supersession_trigger_row(row) + cls._validate_supersession_guard_function(cur) + + @classmethod + def _validate_supersession_trigger_row(cls, row: dict[str, object]) -> None: + expected = ( + f"CREATE TRIGGER {POSTGRES_SUPERSESSION_GUARD_TRIGGER} BEFORE INSERT OR " + "UPDATE OF tenant_id, scope, visibility, source_agent, " + "supersedes_claim_id, replaced_by_claim_id ON public.claims " + "FOR EACH ROW EXECUTE FUNCTION " + f"public.{POSTGRES_SUPERSESSION_GUARD_FUNCTION}()" + ) + metadata = ( + row.get("trigger_name") == POSTGRES_SUPERSESSION_GUARD_TRIGGER, + row.get("table_schema") == "public", + row.get("table_name") == "claims", + row.get("enabled_code") == "O", + not bool(row.get("is_internal")), + row.get("function_schema") == "public", + row.get("function_name") == POSTGRES_SUPERSESSION_GUARD_FUNCTION, + ) + if not all(metadata) or cls._canonical_ddl( + row.get("trigger_definition") + ) != cls._canonical_ddl(expected): + raise PermissionError("Postgres claim supersession trigger has drifted.") + + @classmethod + def _validate_supersession_guard_function(cls, cur) -> None: + cur.execute( + """ + SELECT n.nspname AS schema_name, p.proname AS function_name, + p.pronargs AS argument_count, + pg_get_function_result(p.oid) AS result_signature, + l.lanname AS language_name, p.prosecdef AS security_definer, + COALESCE(p.proconfig, ARRAY[]::text[]) AS function_config, + p.provolatile AS volatility, p.proparallel AS parallel_safety, + p.proleakproof AS leakproof, p.proisstrict AS strict, + p.prosrc AS function_source, + pg_has_role(current_user, p.proowner, 'MEMBER') AS owner_member + FROM pg_proc AS p + JOIN pg_namespace AS n ON n.oid = p.pronamespace + JOIN pg_language AS l ON l.oid = p.prolang + WHERE n.nspname = 'public' AND p.proname = %s AND p.pronargs = 0 + """, + (POSTGRES_SUPERSESSION_GUARD_FUNCTION,), + ) + row = cur.fetchone() + if not isinstance(row, dict): + raise PermissionError("Postgres claim supersession guard function is missing.") + if not cls._supersession_guard_metadata_matches(row): + raise PermissionError("Postgres claim supersession guard has drifted.") + + @classmethod + def _supersession_guard_metadata_matches( + cls, + row: dict[str, object], + ) -> bool: + import importlib + + migration = importlib.import_module( + "memorymaster.stores.migrations.0012_principal_local_claim_identities" + ) + expected_source = str(migration._SUPERSESSION_GUARD_FUNCTION).split( + "AS $$", 1 + )[1].rsplit("$$", 1)[0].strip() + configs = row.get("function_config") or () + if isinstance(configs, str): + configs = (configs,) + metadata = ( + row.get("schema_name") == "public", + row.get("function_name") == POSTGRES_SUPERSESSION_GUARD_FUNCTION, + row.get("argument_count") is not None + and int(row["argument_count"]) == 0, + cls._canonical_identity_sql(row.get("result_signature")) == "trigger", + str(row.get("language_name") or "").lower() == "plpgsql", + not bool(row.get("security_definer")), + not tuple(configs), + row.get("volatility") == "v", + row.get("parallel_safety") == "u", + not bool(row.get("leakproof")), + not bool(row.get("strict")), + not bool(row.get("owner_member")), + cls._canonical_catalog_sql(row.get("function_source")) + == cls._canonical_catalog_sql(expected_source), + ) + return all(metadata) + + @classmethod + def _validate_event_trigger_row( + cls, + row: dict[str, object], + operation: str, + ) -> None: + name = f"trg_events_append_only_{operation}" + expected = ( + f"CREATE TRIGGER {name} BEFORE {operation.upper()} ON public.events " + "FOR EACH ROW EXECUTE FUNCTION " + "public.memorymaster_events_append_only_guard()" + ) + metadata = ( + row.get("trigger_name") == name, + row.get("table_schema") == "public", + row.get("table_name") == "events", + row.get("enabled_code") == "O", + not bool(row.get("is_internal")), + row.get("function_schema") == "public", + row.get("function_name") == "memorymaster_events_append_only_guard", + ) + if not all(metadata) or cls._canonical_ddl( + row.get("trigger_definition") + ) != cls._canonical_ddl(expected): + raise PermissionError(f"Postgres append-only {operation} trigger has drifted.") + + @classmethod + def _validate_event_guard_function(cls, cur) -> None: + cur.execute( + """ + SELECT n.nspname AS schema_name, p.proname AS function_name, + p.pronargs AS argument_count, + pg_get_function_result(p.oid) AS result_signature, + l.lanname AS language_name, p.prosecdef AS security_definer, + COALESCE(p.proconfig, ARRAY[]::text[]) AS function_config, + p.provolatile AS volatility, p.proparallel AS parallel_safety, + p.proleakproof AS leakproof, p.proisstrict AS strict, + p.prosrc AS function_source, + pg_has_role(current_user, p.proowner, 'MEMBER') AS owner_member + FROM pg_proc AS p + JOIN pg_namespace AS n ON n.oid = p.pronamespace + JOIN pg_language AS l ON l.oid = p.prolang + WHERE n.nspname = 'public' + AND p.proname = 'memorymaster_events_append_only_guard' + AND p.pronargs = 0 + """ + ) + row = cur.fetchone() + if not isinstance(row, dict): + raise PermissionError("Postgres append-only event guard is missing.") + configs = row.get("function_config") or () + if isinstance(configs, str): + configs = (configs,) + metadata = ( + row.get("schema_name") == "public", + row.get("function_name") == "memorymaster_events_append_only_guard", + row.get("argument_count") is not None + and int(row["argument_count"]) == 0, + cls._canonical_identity_sql(row.get("result_signature")) == "trigger", + str(row.get("language_name") or "").lower() == "plpgsql", + not bool(row.get("security_definer")), + not tuple(configs), + row.get("volatility") == "v", + row.get("parallel_safety") == "u", + not bool(row.get("leakproof")), + not bool(row.get("strict")), + not bool(row.get("owner_member")), + cls._canonical_catalog_sql(row.get("function_source")) + == cls._canonical_catalog_sql(POSTGRES_EVENT_GUARD_SOURCE), + ) + if not all(metadata): + raise PermissionError("Postgres append-only event guard has drifted.") + + @staticmethod + def _canonical_policy_payload(rows: Iterable[dict[str, object]]) -> str: + payload: list[dict[str, object]] = [] + for policy in rows: + row = {field: policy.get(field) for field in POSTGRES_POLICY_FIELDS} + roles = row["roles"] + if isinstance(roles, (list, tuple, set, frozenset)): + row["roles"] = sorted(str(role) for role in roles) + payload.append(row) + payload.sort( + key=lambda row: ( + str(row["schemaname"]), + str(row["tablename"]), + str(row["policyname"]), + ) + ) + return json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + + @classmethod + def _policy_manifest_comment(cls, rows: Iterable[dict[str, object]]) -> str: + payload = cls._canonical_policy_payload(rows).encode("utf-8") + return f"{POSTGRES_POLICY_MANIFEST_PREFIX}{hashlib.sha256(payload).hexdigest()}" + + @staticmethod + def _expected_policy_inventory() -> dict[tuple[str, str], tuple[bool, str]]: + expected: dict[tuple[str, str], tuple[bool, str]] = {} + for table in POSTGRES_TENANT_POLICY_TABLES: + for command, restrict_name in POSTGRES_COMMAND_POLICIES.items(): + expected[(table, restrict_name)] = (True, command) + expected[(table, POSTGRES_PERMIT_POLICIES[command])] = (False, command) + for table in POSTGRES_TEAM_DENY_TABLES: + expected[(table, "memorymaster_team_deny")] = (True, "ALL") + return expected + + @classmethod + def _validate_policy_shape( + cls, + row: dict[str, object], + expected: tuple[bool, str], + ) -> None: + restrictive, command = expected + if cls._policy_is_restrictive(row) is not restrictive: + raise PermissionError("Postgres runtime RLS policy mode is unsafe.") + if str(row.get("cmd", "")).upper() != command: + raise PermissionError("Postgres runtime RLS policy command is unsafe.") + if cls._policy_roles(row) != {"public"}: + raise PermissionError("Postgres runtime RLS policy roles are unsafe.") + qual_present = row.get("qual") is not None + check_present = row.get("with_check") is not None + expected_shape = { + "SELECT": (True, False), + "INSERT": (False, True), + "UPDATE": (True, True), + "DELETE": (True, False), + "ALL": (True, True), + }[command] + if (qual_present, check_present) != expected_shape: + raise PermissionError("Postgres runtime RLS policy expression shape is unsafe.") + + @staticmethod + def _validate_paired_policy_expressions( + policies: dict[tuple[str, str], dict[str, object]], + ) -> None: + for table in POSTGRES_TENANT_POLICY_TABLES: + for command, restrict_name in POSTGRES_COMMAND_POLICIES.items(): + permit = policies[(table, POSTGRES_PERMIT_POLICIES[command])] + restrict = policies[(table, restrict_name)] + if (permit.get("qual"), permit.get("with_check")) != ( + restrict.get("qual"), + restrict.get("with_check"), + ): + raise PermissionError("Postgres paired RLS policy expressions differ.") + + @staticmethod + def _validate_policy_expression_contract( + policies: dict[tuple[str, str], dict[str, object]], + ) -> None: + expected = expected_policy_expressions( + POSTGRES_TENANT_POLICY_TABLES, + POSTGRES_TEAM_DENY_TABLES, + POSTGRES_COMMAND_POLICIES, + POSTGRES_PERMIT_POLICIES, + ) + for identity, (expected_qual, expected_check) in expected.items(): + policy = policies[identity] + if not expressions_match(policy.get("qual"), expected_qual) or not ( + expressions_match(policy.get("with_check"), expected_check) + ): + raise PermissionError( + f"Postgres RLS policy expression contract drifted: " + f"{identity[0]}.{identity[1]}." + ) + + @staticmethod + def _validate_runtime_migration(cur) -> None: + from memorymaster.stores.migrations import discover_migrations + + required_versions = (11, 12) + migrations = { + item.version: item + for item in discover_migrations() + if item.version in required_versions + } + cur.execute( + """ + SELECT version, checksum FROM schema_versions + WHERE version IN (%s, %s) + """, + required_versions, + ) + rows = list(cur.fetchall()) + stored = { + int(row["version"]): str(row["checksum"]) + for row in rows + if isinstance(row, dict) + } + expected = {version: migrations[version].checksum() for version in required_versions} + if stored != expected: + raise PermissionError("Postgres runtime migration checksums are missing or invalid.") + + @classmethod + def _validate_runtime_policies(cls, cur) -> None: + cur.execute( + """ + SELECT schemaname, tablename, policyname, permissive, roles, cmd, qual, with_check + FROM pg_policies + WHERE schemaname = current_schema() AND tablename = ANY(%s) + """, + (list(POSTGRES_PROTECTED_TABLES),), + ) + rows = list(cur.fetchall()) + policies = { + (str(row.get("tablename") or row.get("table_name")), + str(row.get("policyname") or row.get("policy_name"))): row + for row in rows + } + expected = cls._expected_policy_inventory() + if set(policies) != set(expected) or len(rows) != len(expected): + raise PermissionError("Postgres runtime RLS policy inventory is unsafe.") + for identity, contract in expected.items(): + cls._validate_policy_shape(policies[identity], contract) + cls._validate_paired_policy_expressions(policies) + cls._validate_policy_expression_contract(policies) + for table in POSTGRES_TEAM_DENY_TABLES: + deny = policies[(table, "memorymaster_team_deny")] + if str(deny.get("qual")).upper() != "FALSE" or str( + deny.get("with_check") + ).upper() != "FALSE": + raise PermissionError("Postgres team-deny RLS policy must remain FALSE.") + cur.execute( + """ + SELECT obj_description(p.oid, 'pg_policy') AS manifest_comment + FROM pg_policy AS p + JOIN pg_class AS c ON c.oid = p.polrelid + JOIN pg_namespace AS n ON n.oid = c.relnamespace + WHERE n.nspname = current_schema() + AND c.relname = 'claims' + AND p.polname = 'memorymaster_tenant_select' + """ + ) + comment_row = cur.fetchone() + comment = comment_row.get("manifest_comment") if isinstance(comment_row, dict) else None + if comment != cls._policy_manifest_comment(rows): + raise PermissionError("Postgres RLS policy manifest fingerprint is invalid.") + + @staticmethod + def _authority_settings(cur) -> dict[str, str]: + cur.execute( + """ + SELECT current_setting('memorymaster.tenant_id', true) AS tenant_id, + current_setting('memorymaster.principal', true) AS principal, + current_setting('memorymaster.allowed_scopes', true) AS allowed_scopes + """ + ) + row = cur.fetchone() or {} + return {name: str(row.get(name) or "") for name in ("tenant_id", "principal", "allowed_scopes")} + + @classmethod + def _bind_runtime_authority( + cls, + cur, + tenant_id: str, + principal: str, + allowed_scopes: tuple[str, ...], + ) -> None: + if any(cls._authority_settings(cur).values()): + raise PermissionError("Postgres authority GUC defaults must be empty.") + values = { + "memorymaster.tenant_id": tenant_id, + "memorymaster.principal": principal, + "memorymaster.allowed_scopes": json.dumps(allowed_scopes, separators=(",", ":")), + } + for key, value in values.items(): + cur.execute("SELECT set_config(%s, %s, true)", (key, value)) + if cls._authority_settings(cur) != { + "tenant_id": tenant_id, + "principal": principal, + "allowed_scopes": values["memorymaster.allowed_scopes"], + }: + raise PermissionError("Postgres transaction-local authority binding failed verification.") + def _tenant_for_operation(self, tenant_id: str | None = None) -> str | None: requested = (tenant_id or "").strip() or None if self.require_tenant: @@ -70,6 +1012,21 @@ def _tenant_for_operation(self, tenant_id: str | None = None) -> str | None: return self.tenant_id return requested if requested is not None else self.tenant_id + @staticmethod + def _postgres_identity_filter( + visibility: str, + source_agent: str | None, + *, + alias: str = "", + ) -> tuple[str, tuple[object, ...]]: + prefix = f"{alias}." if alias else "" + if visibility == "public": + return f"{prefix}visibility = %s", ("public",) + return ( + f"{prefix}visibility = %s AND {prefix}source_agent = %s", + (visibility, source_agent), + ) + def _load_psycopg(self) -> Any: if self._psycopg is None: try: @@ -83,34 +1040,81 @@ def _load_psycopg(self) -> Any: self._psycopg = (psycopg, dict_row, Jsonb) return self._psycopg - def connect(self) -> Any: - if self.require_tenant and self.tenant_id is None: - raise PermissionError("Postgres team mode requires a tenant context before connecting.") + def _open_connection(self) -> Any: psycopg, dict_row, _ = self._load_psycopg() def _open() -> Any: return psycopg.connect(self.dsn, row_factory=dict_row) - conn = connect_with_retry(_open) - if self.tenant_id is None: - return conn + return connect_with_retry(_open) + + def connect(self) -> Any: + if not self.require_tenant: + raise PermissionError( + "Postgres application connections require authenticated team authority. " + "Use SQLite for local trusted mode or init_db() with a dedicated migrator DSN." + ) + authority = self._require_team_authority() + conn = self._open_connection() try: + conn.autocommit = False + with conn.cursor() as cur: + self._validate_runtime_role(cur) + self._validate_runtime_tables(cur) + self._validate_runtime_metadata_tables(cur) + self._validate_claim_owner_constraint(cur) + self._validate_claim_identity_indexes(cur) + self._validate_claim_supersession_guard(cur) + self._validate_event_append_only_catalog(cur) + self._validate_event_chain_head_function(cur) + self._validate_runtime_migration(cur) + self._validate_runtime_policies(cur) + self._bind_runtime_authority(cur, *authority) + except Exception: + self._cleanup_failed_connection(conn) + raise + return conn + + def _connect_schema_admin(self) -> Any: + if self.require_tenant: + raise PermissionError( + "Postgres team runtime stores cannot open schema-administration connections." + ) + conn = self._open_connection() + try: + conn.autocommit = False with conn.cursor() as cur: cur.execute( - "SELECT set_config('memorymaster.tenant_id', %s, false)", - (self.tenant_id,), + """ + SELECT current_user, session_user, rolsuper, rolbypassrls + FROM pg_roles WHERE rolname = current_user + """ + ) + row = cur.fetchone() + if not isinstance(row, dict): + raise PermissionError("Postgres migration role could not be verified.") + if row.get("current_user") != row.get("session_user"): + raise PermissionError("Postgres migration role cannot use session impersonation.") + if not bool(row.get("rolsuper")) and not bool(row.get("rolbypassrls")): + raise PermissionError( + "Postgres migration role requires SUPERUSER or BYPASSRLS." ) except Exception: - conn.close() + self._cleanup_failed_connection(conn) raise return conn def init_db(self) -> None: - from memorymaster.stores._storage_schema import load_schema_postgres_sql + if self.require_tenant: + self._require_team_authority() + raise PermissionError( + "Postgres team runtime stores cannot initialize or migrate schema." + ) + with self._connect_schema_admin() as conn, conn.cursor() as cur: + from memorymaster.stores._storage_schema import load_schema_postgres_sql - sql = load_schema_postgres_sql() - statements = self._split_sql_statements(sql) - with self.connect() as conn, conn.cursor() as cur: + sql = load_schema_postgres_sql() + statements = self._split_sql_statements(sql) for statement in statements: cur.execute(statement) self._ensure_confirmed_tuple_uniqueness_schema(conn) @@ -126,7 +1130,7 @@ def init_db(self) -> None: # the SQLite backend, ensuring parity between the two stores. from memorymaster.stores.migrations import MigrationRunner - with self.connect() as mig_conn: + with self._connect_schema_admin() as mig_conn: MigrationRunner(mig_conn, backend="postgres").apply_pending() @staticmethod @@ -210,44 +1214,11 @@ def _ensure_event_integrity_schema(conn) -> None: def _ensure_confirmed_tuple_uniqueness_schema(conn) -> None: PostgresStore._ensure_tenant_id_schema(conn) with conn.cursor() as cur: - cur.execute( - """ - CREATE OR REPLACE FUNCTION memorymaster_claims_confirmed_tuple_guard() - RETURNS trigger - LANGUAGE plpgsql - AS $$ - BEGIN - IF NEW.status = 'confirmed' - AND NEW.subject IS NOT NULL - AND NEW.predicate IS NOT NULL - AND EXISTS ( - SELECT 1 - FROM claims c - WHERE c.status = 'confirmed' - AND c.subject = NEW.subject - AND c.predicate = NEW.predicate - AND c.scope = NEW.scope - AND c.tenant_id IS NOT DISTINCT FROM NEW.tenant_id - AND (TG_OP = 'INSERT' OR c.id <> NEW.id) - ) THEN - RAISE EXCEPTION 'only one confirmed claim is allowed per tenant and (subject,predicate,scope)' - USING ERRCODE = '23505'; - END IF; - RETURN NEW; - END; - $$; - """ - ) cur.execute( f"DROP TRIGGER IF EXISTS {POSTGRES_CONFIRMED_TUPLE_GUARD_TRIGGER} ON claims" ) cur.execute( - f""" - CREATE TRIGGER {POSTGRES_CONFIRMED_TUPLE_GUARD_TRIGGER} - BEFORE INSERT OR UPDATE OF status, subject, predicate, scope, tenant_id ON claims - FOR EACH ROW - EXECUTE FUNCTION memorymaster_claims_confirmed_tuple_guard() - """ + "DROP FUNCTION IF EXISTS memorymaster_claims_confirmed_tuple_guard()" ) PostgresStore._try_create_confirmed_tuple_unique_index(cur) @@ -256,13 +1227,25 @@ def _try_create_confirmed_tuple_unique_index(cur) -> None: savepoint = "sp_claims_confirmed_tuple_unique_idx" cur.execute(f"SAVEPOINT {savepoint}") try: + cur.execute("DROP INDEX IF EXISTS idx_claims_confirmed_tuple_unique") cur.execute( """ - CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_confirmed_tuple_unique - ON claims(COALESCE(tenant_id, ''), subject, predicate, scope) - WHERE status = 'confirmed' - AND subject IS NOT NULL - AND predicate IS NOT NULL + CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_confirmed_tuple_unique + ON claims(COALESCE(tenant_id, ''), subject, predicate, scope) + WHERE visibility = 'public' AND status = 'confirmed' + AND subject IS NOT NULL AND predicate IS NOT NULL + """ + ) + cur.execute( + """ + CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_confirmed_tuple_unique + ON claims( + COALESCE(tenant_id, ''), visibility, source_agent, + subject, predicate, scope + ) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND status = 'confirmed' + AND subject IS NOT NULL AND predicate IS NOT NULL """ ) except Exception as exc: @@ -491,6 +1474,21 @@ def _event_chain_head( "SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (f"memorymaster:event:{lock_key}",), ) + if tenant_id is not None: + cur.execute( + """ + SELECT global_event_hash, tenant_event_hash + FROM public.memorymaster_event_chain_head() + """ + ) + row = cur.fetchone() + primary = row.get("global_event_hash") if isinstance(row, dict) and row else None + tenant = row.get("tenant_event_hash") if isinstance(row, dict) and row else None + return ( + str(primary) if primary else None, + POSTGRES_TENANT_EVENT_HASH_ALGO, + str(tenant) if tenant else None, + ) if tenant_id is None: algo = EVENT_HASH_ALGO cur.execute( @@ -502,35 +1500,9 @@ def _event_chain_head( """, (algo,), ) - else: - algo = POSTGRES_TENANT_EVENT_HASH_ALGO - cur.execute( - """ - SELECT event_hash FROM events - WHERE event_hash IS NOT NULL - AND hash_algo = %s - AND tenant_id IS NOT DISTINCT FROM %s - ORDER BY id DESC LIMIT 1 - """, - (algo, tenant_id), - ) row = cur.fetchone() value = row.get("event_hash") if isinstance(row, dict) and row else None - tenant_head: str | None = None - if tenant_id is not None: - cur.execute( - """ - SELECT tenant_event_hash FROM events - WHERE tenant_id IS NOT DISTINCT FROM %s - AND tenant_event_hash IS NOT NULL - ORDER BY id DESC LIMIT 1 - """, - (tenant_id,), - ) - tenant_row = cur.fetchone() - if isinstance(tenant_row, dict) and tenant_row.get("tenant_event_hash"): - tenant_head = str(tenant_row["tenant_event_hash"]) - return (str(value) if value else None), algo, tenant_head + return (str(value) if value else None), algo, None def _insert_event_row( self, @@ -605,6 +1577,48 @@ def _insert_event_row( raise RuntimeError("Failed to insert event row.") return int(inserted["id"]) + def _assign_human_id( + self, + cur, + *, + subject: str | None, + text: str, + claim_id: int, + tenant_id: str | None, + scope: str, + visibility: str, + source_agent: str | None, + ) -> str: + psycopg, _, _ = self._load_psycopg() + savepoint = "sp_claim_human_id_assignment" + for _attempt in range(100): + human_id = self._allocate_human_id( + cur, + subject, + text, + claim_id, + tenant_id=tenant_id, + scope=scope, + visibility=visibility, + source_agent=source_agent, + ) + cur.execute(f"SAVEPOINT {savepoint}") + try: + cur.execute( + "UPDATE claims SET human_id = %s WHERE id = %s", + (human_id, claim_id), + ) + except psycopg.errors.UniqueViolation as exc: + cur.execute(f"ROLLBACK TO SAVEPOINT {savepoint}") + cur.execute(f"RELEASE SAVEPOINT {savepoint}") + constraint = getattr(getattr(exc, "diag", None), "constraint_name", None) + if constraint not in POSTGRES_HUMAN_IDENTITY_INDEXES: + raise + continue + cur.execute(f"RELEASE SAVEPOINT {savepoint}") + return human_id + raise RuntimeError("Unable to allocate a unique human_id after 100 attempts.") + def create_claim( self, text: str, @@ -623,9 +1637,14 @@ def create_claim( valid_from: str | None = None, valid_until: str | None = None, source_agent: str | None = None, - visibility: str = "private", + visibility: str = "public", holder: str | None = None, ) -> Claim: + visibility, source_agent = normalize_claim_identity( + visibility, + source_agent, + allow_sensitive=not self.require_tenant, + ) if not citations: raise ValueError("At least one citation is required.") normalized_idempotency_key = (idempotency_key or "").strip() or None @@ -671,13 +1690,24 @@ def create_claim( if claim_row is None: if normalized_idempotency_key is None: raise RuntimeError("Failed to create claim.") + identity_sql, identity_params = self._postgres_identity_filter( + visibility, + source_agent, + ) cur.execute( - """ + f""" SELECT id FROM claims WHERE idempotency_key = %s AND tenant_id IS NOT DISTINCT FROM %s + AND scope = %s + AND {identity_sql} """, - (normalized_idempotency_key, normalized_tenant_id), + ( + normalized_idempotency_key, + normalized_tenant_id, + scope, + *identity_params, + ), ) existing_row = cur.fetchone() if existing_row is None: @@ -689,22 +1719,16 @@ def create_claim( return claim claim_id = int(claim_row["id"]) - # Assign a human-readable ID. - try: - human_id = self._allocate_human_id( - cur, - subject, - text, - claim_id, - tenant_id=normalized_tenant_id, - ) - cur.execute( - "UPDATE claims SET human_id = %s WHERE id = %s", - (human_id, claim_id), - ) - except Exception: - # Column may not exist in legacy schemas; skip gracefully. - pass + self._assign_human_id( + cur, + subject=subject, + text=text, + claim_id=claim_id, + tenant_id=normalized_tenant_id, + scope=scope, + visibility=visibility, + source_agent=source_agent, + ) for cite in citations: cur.execute( @@ -746,33 +1770,86 @@ def get_claim(self, claim_id: int, include_citations: bool = True) -> Claim | No claim.citations = self.list_citations(claim.id) return claim + def _select_claim_identity_rows( + self, + column: str, + value: str, + *, + tenant_id: str | None, + scope: str | None, + visibility: str, + source_agent: str | None, + ) -> list[Any]: + if column not in {"idempotency_key", "human_id"}: + raise ValueError(f"Unsupported claim identity column: {column}") + visibility, source_agent = normalize_claim_identity( + visibility, + source_agent, + allow_sensitive=not self.require_tenant, + ) + identity_sql, identity_params = self._postgres_identity_filter( + visibility, + source_agent, + ) + clauses = [ + f"{column} = %s", + identity_sql, + "tenant_id IS NOT DISTINCT FROM %s", + ] + params: list[object] = [ + value, + *identity_params, + self._tenant_for_operation(tenant_id), + ] + if scope is not None: + clauses.append("scope = %s") + params.append(scope) + sql = f"SELECT * FROM claims WHERE {' AND '.join(clauses)} LIMIT 2" + with self.connect() as conn, conn.cursor() as cur: + cur.execute(sql, params) + return list(cur.fetchall()) + + def _claim_from_identity_rows( + self, + rows: list[Any], + *, + identifier: str, + include_citations: bool, + ) -> Claim | None: + row = require_unambiguous_identity_row(rows, identifier=identifier) + if row is None: + return None + claim = self._row_to_claim(row) + if include_citations: + claim.citations = self.list_citations(claim.id) + return claim + def get_claim_by_idempotency_key( self, idempotency_key: str, include_citations: bool = True, *, tenant_id: str | None = None, + scope: str | None = None, + visibility: str = "public", + source_agent: str | None = None, ) -> Claim | None: normalized_idempotency_key = idempotency_key.strip() if not normalized_idempotency_key: return None - effective_tenant = self._tenant_for_operation(tenant_id) - with self.connect() as conn, conn.cursor() as cur: - cur.execute( - """ - SELECT * FROM claims - WHERE idempotency_key = %s - AND tenant_id IS NOT DISTINCT FROM %s - """, - (normalized_idempotency_key, effective_tenant), - ) - row = cur.fetchone() - if row is None: - return None - claim = self._row_to_claim(row) - if include_citations: - claim.citations = self.list_citations(claim.id) - return claim + rows = self._select_claim_identity_rows( + "idempotency_key", + normalized_idempotency_key, + tenant_id=tenant_id, + scope=scope, + visibility=visibility, + source_agent=source_agent, + ) + return self._claim_from_identity_rows( + rows, + identifier="idempotency key", + include_citations=include_citations, + ) def claim_ids_by_source_agent( self, @@ -1173,29 +2250,107 @@ def recompute_tiers(self) -> dict[str, int]: return counts def set_supersedes(self, claim_id: int, supersedes_claim_id: int) -> None: + self.mark_superseded( + supersedes_claim_id, + claim_id, + "set_supersedes compatibility path", + ) + + def mark_superseded(self, old_claim_id: int, new_claim_id: int, reason: str) -> None: + if old_claim_id == new_claim_id: + raise ValueError("Supersession claims are unavailable.") now = utc_now() with self.connect() as conn, conn.cursor() as cur: cur.execute( """ - UPDATE claims - SET supersedes_claim_id = %s, updated_at = %s - WHERE id = %s - """, - (supersedes_claim_id, now, claim_id), + SELECT id, status, version, replaced_by_claim_id, + supersedes_claim_id + FROM claims + WHERE id IN (%s, %s) + ORDER BY id + FOR UPDATE + """, + (old_claim_id, new_claim_id), + ) + rows = {int(row["id"]): row for row in cur.fetchall()} + if set(rows) != {old_claim_id, new_claim_id}: + raise ValueError("Supersession claims are unavailable.") + self._apply_atomic_supersession( + conn, + cur, + rows[old_claim_id], + rows[new_claim_id], + reason, + now, ) - def mark_superseded(self, old_claim_id: int, new_claim_id: int, reason: str) -> None: - old_claim = self.get_claim(old_claim_id, include_citations=False) - if old_claim is None: - return - self.apply_status_transition( - old_claim, - to_status="superseded", - reason=reason, + def _apply_atomic_supersession( + self, + conn, + cur, + old: dict[str, object], + new: dict[str, object], + reason: str, + now: datetime, + ) -> None: + old_id = int(old["id"]) + new_id = int(new["id"]) + if old.get("status") == "superseded" or old.get("replaced_by_claim_id") is not None: + raise ConcurrentModificationError( + f"Claim {old_id} was already superseded. Reload and retry." + ) + old_status = str(old.get("status") or "") + if not can_transition(old_status, "superseded"): + raise ValueError(f"Invalid transition: {old_status} -> superseded") + if new.get("supersedes_claim_id") not in {None, old_id}: + raise ConcurrentModificationError( + f"Claim {new_id} already supersedes another claim. Reload and retry." + ) + self._update_superseded_claim(cur, old, new_id, now) + self._update_replacement_claim(cur, new, old_id, now) + self._insert_event_row( + conn, + claim_id=old_id, event_type="supersession", - replaced_by_claim_id=new_claim_id, + from_status=str(old.get("status") or "candidate"), + to_status="superseded", + details=reason, + payload={"replaced_by_claim_id": new_id}, + created_at=now, ) - self.set_supersedes(new_claim_id, old_claim_id) + + @staticmethod + def _update_superseded_claim(cur, old: dict[str, object], new_id: int, now) -> None: + old_id = int(old["id"]) + cur.execute( + """ + UPDATE claims + SET status = 'superseded', updated_at = %s, replaced_by_claim_id = %s, + version = version + 1, valid_until = COALESCE(%s, valid_until) + WHERE id = %s AND version = %s AND status != 'superseded' + AND replaced_by_claim_id IS NULL + """, + (now, new_id, now, old_id, int(old.get("version") or 1)), + ) + if cur.rowcount != 1: + raise ConcurrentModificationError( + f"Claim {old_id} was modified by another writer. Reload and retry." + ) + @staticmethod + def _update_replacement_claim(cur, new: dict[str, object], old_id: int, now) -> None: + new_id = int(new["id"]) + cur.execute( + """ + UPDATE claims + SET supersedes_claim_id = %s, updated_at = %s + WHERE id = %s AND (supersedes_claim_id IS NULL OR supersedes_claim_id = %s) + """, + (old_id, now, new_id, old_id), + ) + if cur.rowcount != 1: + raise ConcurrentModificationError( + f"Claim {new_id} was modified by another writer. Reload and retry." + ) def find_by_status(self, status: str, limit: int = 100, include_citations: bool = False) -> list[Claim]: return self.list_claims( @@ -1245,12 +2400,25 @@ def find_confirmed_by_tuple( scope: str | None, exclude_claim_id: int | None = None, tenant_id: str | None = None, + visibility: str = "public", + source_agent: str | None = None, ) -> list[Claim]: if not subject or not predicate: return [] clauses = ["status = 'confirmed'", "subject = %s", "predicate = %s", "scope = %s"] params: list[object] = [subject, predicate, scope or "project"] + visibility, source_agent = normalize_claim_identity( + visibility, + source_agent, + allow_sensitive=not self.require_tenant, + ) + identity_sql, identity_params = self._postgres_identity_filter( + visibility, + source_agent, + ) + clauses.append(identity_sql) + params.extend(identity_params) clauses.append("tenant_id IS NOT DISTINCT FROM %s") params.append(self._tenant_for_operation(tenant_id)) if exclude_claim_id is not None: @@ -2114,11 +3282,22 @@ def _ensure_human_id_schema(conn) -> None: cur.execute( "CREATE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id)" ) + cur.execute( + "DROP INDEX IF EXISTS idx_claims_tenant_human_id" + ) + cur.execute( + """ + CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_human_id_unique + ON claims(COALESCE(tenant_id, ''), scope, human_id) + WHERE visibility = 'public' AND human_id IS NOT NULL + """ + ) cur.execute( """ - CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_tenant_human_id - ON claims(COALESCE(tenant_id, ''), human_id) - WHERE human_id IS NOT NULL + CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_nonpublic_principal_human_id_unique + ON claims(COALESCE(tenant_id, ''), scope, visibility, source_agent, human_id) + WHERE visibility <> 'public' AND source_agent IS NOT NULL + AND human_id IS NOT NULL """ ) PostgresStore._backfill_human_ids(conn) @@ -2129,7 +3308,7 @@ def _backfill_human_ids(conn) -> int: with conn.cursor() as cur: cur.execute( """ - SELECT id, subject, text, tenant_id + SELECT id, subject, text, tenant_id, scope, visibility, source_agent FROM claims WHERE human_id IS NULL ORDER BY id ASC """ ) @@ -2147,6 +3326,9 @@ def _backfill_human_ids(conn) -> int: text, claim_id, tenant_id=row.get("tenant_id"), + scope=PostgresStore._as_text(row.get("scope")) or "project", + visibility=PostgresStore._as_text(row.get("visibility")) or "public", + source_agent=PostgresStore._as_text(row.get("source_agent")), ) cur.execute( "UPDATE claims SET human_id = %s WHERE id = %s", @@ -2162,10 +3344,18 @@ def _allocate_human_id( text: str, claim_id: int, tenant_id: str | None = None, + scope: str = "project", + visibility: str = "public", + source_agent: str | None = None, ) -> str: """Build a unique human_id, checking for derived_from parent links.""" + identity_sql, identity_params = PostgresStore._postgres_identity_filter( + visibility, + source_agent, + alias="c", + ) cur.execute( - """ + f""" SELECT c.human_id FROM claim_links cl JOIN claims c ON c.id = cl.target_id @@ -2173,21 +3363,25 @@ def _allocate_human_id( AND cl.link_type = 'derived_from' AND c.human_id IS NOT NULL AND c.tenant_id IS NOT DISTINCT FROM %s + AND c.scope = %s + AND {identity_sql} LIMIT 1 """, - (claim_id, tenant_id), + (claim_id, tenant_id, scope, *identity_params), ) parent_row = cur.fetchone() if parent_row and parent_row["human_id"]: parent_hid = str(parent_row["human_id"]) cur.execute( - """ + f""" SELECT COUNT(*) AS cnt FROM claims WHERE human_id LIKE %s AND human_id != %s AND tenant_id IS NOT DISTINCT FROM %s + AND scope = %s + AND {identity_sql.replace('c.', '')} """, - (parent_hid + ".%", parent_hid, tenant_id), + (parent_hid + ".%", parent_hid, tenant_id, scope, *identity_params), ) child_count = cur.fetchone() next_child = (int(child_count["cnt"]) if child_count else 0) + 1 @@ -2199,11 +3393,13 @@ def _allocate_human_id( suffix = 1 while True: cur.execute( - """ + f""" SELECT 1 FROM claims WHERE human_id = %s AND tenant_id IS NOT DISTINCT FROM %s + AND scope = %s + AND {identity_sql.replace('c.', '')} """, - (final, tenant_id), + (final, tenant_id, scope, *identity_params), ) existing = cur.fetchone() if existing is None: @@ -3048,38 +4244,41 @@ def get_claim_by_human_id( include_citations: bool = True, *, tenant_id: str | None = None, + scope: str | None = None, + visibility: str = "public", + source_agent: str | None = None, ) -> Claim | None: """Look up a claim by its human-readable ID (e.g. ``mm-a3f8``).""" normalized = human_id.strip() if not normalized: return None - effective_tenant = self._tenant_for_operation(tenant_id) - with self.connect() as conn, conn.cursor() as cur: - try: - cur.execute( - """ - SELECT * FROM claims - WHERE human_id = %s - AND tenant_id IS NOT DISTINCT FROM %s - """, - (normalized, effective_tenant), - ) - row = cur.fetchone() - except Exception: - # Column may not exist yet. + try: + rows = self._select_claim_identity_rows( + "human_id", + normalized, + tenant_id=tenant_id, + scope=scope, + visibility=visibility, + source_agent=source_agent, + ) + except Exception as exc: + if getattr(exc, "sqlstate", None) == "42703": return None - if row is None: - return None - claim = self._row_to_claim(row) - if include_citations: - claim.citations = self.list_citations(claim.id) - return claim + raise + return self._claim_from_identity_rows( + rows, + identifier="human_id", + include_citations=include_citations, + ) def resolve_claim_id( self, identifier: str | int, *, tenant_id: str | None = None, + scope: str | None = None, + visibility: str = "public", + source_agent: str | None = None, ) -> int: """Resolve a numeric ID or human_id string to a numeric claim ID.""" if isinstance(identifier, int): @@ -3093,6 +4292,9 @@ def resolve_claim_id( raw, include_citations=False, tenant_id=tenant_id, + scope=scope, + visibility=visibility, + source_agent=source_agent, ) if claim is not None: return claim.id diff --git a/memorymaster/stores/store_factory.py b/memorymaster/stores/store_factory.py index ce2df98e..6ad69164 100644 --- a/memorymaster/stores/store_factory.py +++ b/memorymaster/stores/store_factory.py @@ -1,6 +1,7 @@ from __future__ import annotations from pathlib import Path +from typing import Iterable from memorymaster.stores.storage import SQLiteStore @@ -16,6 +17,8 @@ def create_store( read_only: bool = False, tenant_id: str | None = None, require_tenant: bool = False, + principal: str | None = None, + allowed_scopes: Iterable[str] | None = None, ): """Build the store for ``db_target``. @@ -33,5 +36,7 @@ def create_store( target, tenant_id=tenant_id, require_tenant=require_tenant, + principal=principal, + allowed_scopes=allowed_scopes, ) return SQLiteStore(Path(target), read_only=read_only) diff --git a/memorymaster/surfaces/cli_helpers.py b/memorymaster/surfaces/cli_helpers.py index a4f9372c..fff90fe7 100644 --- a/memorymaster/surfaces/cli_helpers.py +++ b/memorymaster/surfaces/cli_helpers.py @@ -72,10 +72,16 @@ def _resolve_claim_id(service: MemoryService, raw: str | int) -> int: try: return int(text) except ValueError: - return service.store.resolve_claim_id( - text, - tenant_id=service.tenant_id, - ) + identity_context: dict[str, object] = { + "tenant_id": service.tenant_id, + "visibility": "public", + } + allowed_scopes = getattr(service, "allowed_scopes", None) + if allowed_scopes and len(allowed_scopes) == 1: + identity_context["scope"] = next(iter(allowed_scopes)) + else: + identity_context["scope"] = None + return service.store.resolve_claim_id(text, **identity_context) def _add_cycle_policy_args(p: argparse.ArgumentParser, policy_default: str = "legacy") -> None: diff --git a/memorymaster/surfaces/mcp_server.py b/memorymaster/surfaces/mcp_server.py index 87a8f7c9..26a6571e 100644 --- a/memorymaster/surfaces/mcp_server.py +++ b/memorymaster/surfaces/mcp_server.py @@ -346,6 +346,8 @@ def _service(db: str, workspace: str) -> MemoryService: workspace_root=Path(workspace_path), tenant_id=tenant_id, require_tenant=context is not None and context.mode is AuthMode.TEAM, + principal=principal, + allowed_scopes=context.allowed_scopes if context is not None else frozenset(), ) _bind_telemetry_session(svc, db_path, principal, tenant_id) svc.source_agent = principal @@ -719,7 +721,9 @@ def _normalize_team_arguments( bound.arguments[field] = effective_scope if "scope_allowlist" in bound.arguments: requested_scopes = _parse_scope_allowlist(str(bound.arguments["scope_allowlist"] or "")) - narrowed = [scope for scope in (requested_scopes or context.allowed_scopes) if scope in context.allowed_scopes] + if requested_scopes and any(scope not in context.allowed_scopes for scope in requested_scopes): + raise PermissionError("Caller scope allowlist exceeds authenticated scopes.") + narrowed = list(requested_scopes) if requested_scopes else sorted(context.allowed_scopes) if not narrowed: raise PermissionError("Caller scope allowlist does not intersect authenticated scopes.") bound.arguments["scope_allowlist"] = ",".join(narrowed) diff --git a/scripts/__init__.py b/scripts/__init__.py new file mode 100644 index 00000000..e253f5fe --- /dev/null +++ b/scripts/__init__.py @@ -0,0 +1,6 @@ +"""Repository-local maintenance scripts. + +This package marker prevents an unrelated installed ``scripts`` distribution +from shadowing these modules during tests. Setuptools excludes ``scripts*`` +from the MemoryMaster wheel, so this remains repository-only tooling. +""" diff --git a/scripts/merge_scope_variants.py b/scripts/merge_scope_variants.py index 9eb95c78..7c443a55 100644 --- a/scripts/merge_scope_variants.py +++ b/scripts/merge_scope_variants.py @@ -88,11 +88,9 @@ def _plan(conn: sqlite3.Connection, except_patterns: list[str]) -> list[tuple[st def _archive_confirmed_collisions(conn: sqlite3.Connection, old: str, new: str) -> int: """Archive older confirmed claims that would collide in the target scope. - The DB has a partial UNIQUE index ``idx_claims_confirmed_tuple_unique`` on - ``(subject, predicate, scope) WHERE status = 'confirmed'`` plus trigger - guards. Re-pointing the scope of a confirmed claim into a scope that - already holds a confirmed claim with the same ``(subject, predicate)`` - would violate that constraint. + Confirmed-tuple indexes are tenant-wide for public claims and exact + visibility/principal-local for non-public claims. Re-pointing a scope must + archive only a row that would collide inside that same identity namespace. Resolution policy: keep the more recent of the two (by ``updated_at``); archive the older one in-place. Both rows survive, history is preserved, @@ -109,6 +107,9 @@ def _archive_confirmed_collisions(conn: sqlite3.Connection, old: str, new: str) JOIN claims b ON a.subject = b.subject AND a.predicate = b.predicate + AND a.tenant_id IS b.tenant_id + AND a.visibility = b.visibility + AND (a.visibility = 'public' OR a.source_agent IS b.source_agent) AND a.status = 'confirmed' AND b.status = 'confirmed' WHERE a.scope = ? AND b.scope = ? diff --git a/tests/test_auto_resolver.py b/tests/test_auto_resolver.py index f1886a4f..86633da2 100644 --- a/tests/test_auto_resolver.py +++ b/tests/test_auto_resolver.py @@ -91,11 +91,11 @@ def make_claim(self, id_val, text, confidence, updated): ) @patch("memorymaster.govern.auto_resolver._llm_evaluate") - @patch("memorymaster.govern.auto_resolver.transition_claim") - def test_resolve_conflict_pair_lllm_called(self, mock_transition, mock_llm): + def test_resolve_conflict_pair_lllm_called(self, mock_llm): """resolve_conflict_pair calls LLM with formatted prompt.""" mock_llm.return_value = {"winner": "A", "reason": "test"} mock_store = MagicMock() + mock_store.get_claim.return_value = MagicMock(replaced_by_claim_id=1) claim_a = self.make_claim(1, "Claim A", 0.8, "2024-01-01") claim_b = self.make_claim(2, "Claim B", 0.6, "2024-01-02") @@ -106,23 +106,28 @@ def test_resolve_conflict_pair_lllm_called(self, mock_transition, mock_llm): prompt_arg = mock_llm.call_args[0][0] assert "Claim A" in prompt_arg assert "Claim B" in prompt_arg + mock_store.mark_superseded.assert_called_once_with( + 2, + 1, + "llm_conflict_resolution: test", + ) @patch("memorymaster.govern.auto_resolver._llm_evaluate") - @patch("memorymaster.govern.auto_resolver.transition_claim") - def test_resolve_conflict_pair_winner_a(self, mock_transition, mock_llm): + def test_resolve_conflict_pair_winner_a(self, mock_llm): """Resolves with winner A.""" mock_llm.return_value = {"winner": "A", "reason": "better evidence"} mock_store = MagicMock() + mock_store.get_claim.return_value = MagicMock(replaced_by_claim_id=1) claim_a = self.make_claim(1, "A", 0.8, "2024-01-01") claim_b = self.make_claim(2, "B", 0.6, "2024-01-02") result = resolve_conflict_pair(mock_store, claim_a, claim_b) - assert mock_transition.called + assert result["resolved"] is True + mock_store.mark_superseded.assert_called_once() @patch("memorymaster.govern.auto_resolver._llm_evaluate") - @patch("memorymaster.govern.auto_resolver.transition_claim") - def test_resolve_conflict_pair_no_result(self, mock_transition, mock_llm): + def test_resolve_conflict_pair_no_result(self, mock_llm): """No result from LLM returns error.""" mock_llm.return_value = {} mock_store = MagicMock() diff --git a/tests/test_auto_resolver_atomic_supersession.py b/tests/test_auto_resolver_atomic_supersession.py new file mode 100644 index 00000000..38555f8c --- /dev/null +++ b/tests/test_auto_resolver_atomic_supersession.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +from pathlib import Path + +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.govern.auto_resolver import resolve_conflict_pair + + +def _claim(service: MemoryService, text: str, value: str): + return service.ingest( + text=text, + citations=[CitationInput(source="test")], + subject="atomic-resolver", + predicate="winner", + object_value=value, + ) + + +def test_auto_resolver_cannot_leave_one_sided_supersession( + tmp_path: Path, + monkeypatch, +) -> None: + service = MemoryService(str(tmp_path / "resolver.db")) + service.init_db() + winner = _claim(service, "winner", "new") + loser = _claim(service, "loser", "old") + monkeypatch.setattr( + "memorymaster.govern.auto_resolver._llm_evaluate", + lambda _prompt: {"winner": "A", "reason": "adversarial atomicity"}, + ) + + def fail_legacy_second_step(*_args, **_kwargs) -> None: + raise RuntimeError("legacy reciprocal-link step failed") + + monkeypatch.setattr(service.store, "set_supersedes", fail_legacy_second_step) + + result = resolve_conflict_pair(service.store, winner, loser) + + refreshed_winner = service.store.get_claim(winner.id, include_citations=False) + refreshed_loser = service.store.get_claim(loser.id, include_citations=False) + assert result["resolved"] is True + assert refreshed_loser.status == "superseded" + assert refreshed_loser.replaced_by_claim_id == winner.id + assert refreshed_winner.supersedes_claim_id == loser.id diff --git a/tests/test_human_id_namespace_consumers.py b/tests/test_human_id_namespace_consumers.py new file mode 100644 index 00000000..c4834762 --- /dev/null +++ b/tests/test_human_id_namespace_consumers.py @@ -0,0 +1,347 @@ +"""Adversarial consumers of principal-local ``human_id`` namespaces. + +The identity indexes intentionally permit the same human-readable ID in the +tenant-wide public namespace and in exact non-public principal namespaces. +Every downstream consumer therefore has to preserve or explicitly choose a +namespace; an unordered ``LIMIT 1`` is never a valid resolution policy. +""" +from __future__ import annotations + +import argparse +import re +import sqlite3 +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from memorymaster.core.models import Claim, ClaimLink +from memorymaster.knowledge.vault_exporter import export_vault +from memorymaster.recall.claim_edges import MENTION_KIND, extract_edges_for_claim +from memorymaster.surfaces.cli_handlers_basic import _handle_pin +from memorymaster.surfaces.cli_helpers import _resolve_claim_id + + +TENANT = "tenant-human-id-consumers" +SCOPE = "project:human-id-consumers" +DUPLICATE_HUMAN_ID = "mm-abcd" + + +def _claim( + claim_id: int, + text: str, + *, + human_id: str = DUPLICATE_HUMAN_ID, + visibility: str = "public", + source_agent: str | None = None, +) -> Claim: + return Claim( + id=claim_id, + text=text, + idempotency_key=f"key-{claim_id}", + normalized_text=text.lower(), + claim_type="fact", + subject="human-id-consumer", + predicate="documents", + object_value=str(claim_id), + scope=SCOPE, + volatility="stable", + status="confirmed", + confidence=0.9, + pinned=False, + supersedes_claim_id=None, + replaced_by_claim_id=None, + created_at="2026-07-11T00:00:00+00:00", + updated_at="2026-07-11T00:00:00+00:00", + last_validated_at=None, + archived_at=None, + human_id=human_id, + tenant_id=TENANT, + source_agent=source_agent, + visibility=visibility, + ) + + +def _edge_connection() -> sqlite3.Connection: + conn = sqlite3.connect(":memory:") + conn.executescript( + """ + CREATE TABLE claims ( + id INTEGER PRIMARY KEY, + text TEXT NOT NULL, + human_id TEXT, + replaced_by_claim_id INTEGER, + tenant_id TEXT, + visibility TEXT NOT NULL, + source_agent TEXT + ); + """ + ) + return conn + + +def _insert_edge_claim( + conn: sqlite3.Connection, + claim_id: int, + *, + text: str, + human_id: str, + visibility: str, + source_agent: str | None, + tenant_id: str = TENANT, +) -> None: + conn.execute( + """ + INSERT INTO claims + (id, text, human_id, replaced_by_claim_id, tenant_id, visibility, source_agent) + VALUES (?, ?, ?, NULL, ?, ?, ?) + """, + (claim_id, text, human_id, tenant_id, visibility, source_agent), + ) + + +def test_claim_edges_public_reference_selects_public_namespace() -> None: + conn = _edge_connection() + try: + # The lower row id is deliberately private so unordered LIMIT 1 picks + # the forbidden target on SQLite today. + _insert_edge_claim( + conn, + 10, + text="Alice private target", + human_id=DUPLICATE_HUMAN_ID, + visibility="private", + source_agent="alice", + ) + _insert_edge_claim( + conn, + 20, + text="Tenant public target", + human_id=DUPLICATE_HUMAN_ID, + visibility="public", + source_agent="writer", + ) + _insert_edge_claim( + conn, + 30, + text=f"See {DUPLICATE_HUMAN_ID}.", + human_id="mm-3333", + visibility="public", + source_agent="reader", + ) + + edges = extract_edges_for_claim(conn, 30, f"See {DUPLICATE_HUMAN_ID}.") + + assert edges == [(30, 20, MENTION_KIND)] + finally: + conn.close() + + +def test_claim_edges_private_reference_stays_in_source_principal_namespace() -> None: + conn = _edge_connection() + try: + _insert_edge_claim( + conn, + 10, + text="Bob private target", + human_id=DUPLICATE_HUMAN_ID, + visibility="private", + source_agent="bob", + ) + _insert_edge_claim( + conn, + 20, + text="Alice private target", + human_id=DUPLICATE_HUMAN_ID, + visibility="private", + source_agent="alice", + ) + _insert_edge_claim( + conn, + 30, + text="Tenant public target", + human_id=DUPLICATE_HUMAN_ID, + visibility="public", + source_agent="writer", + ) + _insert_edge_claim( + conn, + 40, + text=f"Alice cites {DUPLICATE_HUMAN_ID}.", + human_id="mm-4444", + visibility="private", + source_agent="alice", + ) + + edges = extract_edges_for_claim( + conn, + 40, + f"Alice cites {DUPLICATE_HUMAN_ID}.", + ) + + assert edges == [(40, 20, MENTION_KIND)] + finally: + conn.close() + + +def test_claim_edges_refuses_cross_principal_private_ambiguity() -> None: + conn = _edge_connection() + try: + for claim_id, principal in ((10, "alice"), (20, "bob")): + _insert_edge_claim( + conn, + claim_id, + text=f"{principal} private target", + human_id=DUPLICATE_HUMAN_ID, + visibility="private", + source_agent=principal, + ) + _insert_edge_claim( + conn, + 30, + text=f"Public source cites {DUPLICATE_HUMAN_ID}.", + human_id="mm-3333", + visibility="public", + source_agent="reader", + ) + + edges = extract_edges_for_claim( + conn, + 30, + f"Public source cites {DUPLICATE_HUMAN_ID}.", + ) + + assert edges == [] + finally: + conn.close() + + +class _VaultStore: + def __init__(self, claims: list[Claim], links: list[ClaimLink] | None = None): + self._claims = claims + self._links = links or [] + + def list_claims(self, **_kwargs) -> list[Claim]: + return list(self._claims) + + def get_claim_links(self, claim_id: int) -> list[ClaimLink]: + return [ + link + for link in self._links + if claim_id in (link.source_id, link.target_id) + ] + + +def _claim_docs(output: Path) -> list[Path]: + return sorted( + path + for path in output.glob("**/*.md") + if path.name != "index.md" + ) + + +def _doc_for_claim(paths: list[Path], claim_id: int) -> Path: + marker = f"claim_id: {claim_id}\n" + matches = [ + path + for path in paths + if marker in path.read_text(encoding="utf-8") + ] + assert len(matches) == 1, ( + f"expected one exported document for claim {claim_id}, found {len(matches)}" + ) + return matches[0] + + +def test_vault_export_preserves_duplicate_human_id_documents(tmp_path: Path) -> None: + claims = [ + _claim(1, "Tenant public document", visibility="public", source_agent="writer"), + _claim(2, "Alice private document", visibility="private", source_agent="alice"), + ] + + stats = export_vault(_VaultStore(claims), tmp_path) + documents = _claim_docs(tmp_path) + + assert stats["exported"] == 2 + assert len(documents) == 2 + assert _doc_for_claim(documents, 1) != _doc_for_claim(documents, 2) + + +def test_vault_wikilink_targets_exact_duplicate_identity_document( + tmp_path: Path, +) -> None: + public = _claim(1, "Tenant public target", visibility="public", source_agent="writer") + private = _claim(2, "Alice private target", visibility="private", source_agent="alice") + source = _claim( + 3, + "Alice private source", + human_id="mm-3333", + visibility="private", + source_agent="alice", + ) + link = ClaimLink( + id=1, + source_id=source.id, + target_id=private.id, + link_type="depends_on", + created_at="2026-07-11T00:00:00+00:00", + ) + + # Put the public collision second: the legacy exporter overwrites Alice's + # private target while leaving a plausible but wrong ``[[mm-abcd]]`` link. + export_vault(_VaultStore([private, public, source], [link]), tmp_path) + documents = _claim_docs(tmp_path) + target_path = _doc_for_claim(documents, private.id) + source_path = _doc_for_claim(documents, source.id) + link_targets = re.findall( + r"\[\[([^\]|#]+)", + source_path.read_text(encoding="utf-8"), + ) + + assert any(Path(target).name == target_path.stem for target in link_targets) + + +class _PolicyAwareStore: + """Expose a bad legacy default so the CLI must state its safe policy.""" + + def __init__(self) -> None: + self.calls: list[dict[str, object]] = [] + + def resolve_claim_id(self, _identifier: str, **kwargs) -> int: + self.calls.append(kwargs) + return 20 if kwargs.get("visibility") == "public" else 10 + + +class _PinService: + tenant_id = TENANT + + def __init__(self) -> None: + self.store = _PolicyAwareStore() + self.pinned_ids: list[int] = [] + + def pin(self, claim_id: int, *, pin: bool): + self.pinned_ids.append(claim_id) + return SimpleNamespace(id=claim_id, status="confirmed", pinned=pin) + + +def test_generic_cli_resolution_declares_public_namespace() -> None: + service = _PinService() + + resolved = _resolve_claim_id(service, DUPLICATE_HUMAN_ID) # type: ignore[arg-type] + + assert resolved == 20 + assert service.store.calls == [ + {"tenant_id": TENANT, "visibility": "public", "scope": None} + ] + + +def test_mutating_cli_never_pins_arbitrarily_selected_private_claim( + capsys: pytest.CaptureFixture[str], +) -> None: + service = _PinService() + args = argparse.Namespace(claim_id=DUPLICATE_HUMAN_ID, unpin=False) + + _handle_pin(args, service, argparse.ArgumentParser(), "unused.db") + + assert service.pinned_ids == [20] + assert "claim_id=20" in capsys.readouterr().out diff --git a/tests/test_identity_bypass_db_merge.py b/tests/test_identity_bypass_db_merge.py new file mode 100644 index 00000000..b607a9c4 --- /dev/null +++ b/tests/test_identity_bypass_db_merge.py @@ -0,0 +1,236 @@ +"""Adversarial identity-namespace tests for the SQLite DB merge bridge.""" +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +from memorymaster.bridges.db_merge import merge_databases +from memorymaster.core.lifecycle import transition_claim +from memorymaster.core.models import CitationInput +from memorymaster.stores.storage import SQLiteStore + + +TENANT = "tenant-merge-identity" +SCOPE = "project:merge-identity" +CITATIONS = [CitationInput(source="identity-bypass-red", locator="db-merge")] + + +def _store(path: Path) -> SQLiteStore: + store = SQLiteStore(path) + store.init_db() + return store + + +def _claim( + store: SQLiteStore, + *, + text: str, + key: str, + principal: str, + visibility: str, + object_value: str = "shared-value", +): + return store.create_claim( + text, + CITATIONS, + idempotency_key=key, + subject="shared-subject", + predicate="uses", + object_value=object_value, + scope=SCOPE, + tenant_id=TENANT, + source_agent=principal, + visibility=visibility, + ) + + +def _rows(store: SQLiteStore) -> list[dict[str, object]]: + with store.connect() as conn: + return [dict(row) for row in conn.execute("SELECT * FROM claims ORDER BY id")] + + +def test_merge_does_not_dedup_private_key_owned_by_another_principal( + tmp_path: Path, +) -> None: + """A hidden Alice key cannot be a uniqueness oracle that drops Bob's row.""" + target_path = tmp_path / "target-private-key.db" + source_path = tmp_path / "source-private-key.db" + target = _store(target_path) + source = _store(source_path) + _claim( + target, + text="Alice private merge payload.", + key="shared-private-merge-key", + principal="alice", + visibility="private", + ) + _claim( + source, + text="Bob private merge payload.", + key="shared-private-merge-key", + principal="bob", + visibility="private", + ) + + stats = merge_databases(str(target_path), str(source_path)) + + rows = _rows(target) + assert stats["merged"] == 1 + assert {(row["source_agent"], row["visibility"]) for row in rows} == { + ("alice", "private"), + ("bob", "private"), + } + + +def test_merge_text_hash_fallback_is_private_principal_local(tmp_path: Path) -> None: + """Equal private text owned by different principals is not a duplicate.""" + target_path = tmp_path / "target-private-hash.db" + source_path = tmp_path / "source-private-hash.db" + target = _store(target_path) + source = _store(source_path) + _claim( + target, + text="The private deployment uses the same cache.", + key="alice-private-hash-key", + principal="alice", + visibility="private", + ) + _claim( + source, + text=" THE PRIVATE DEPLOYMENT USES THE SAME CACHE. ", + key="bob-private-hash-key", + principal="bob", + visibility="private", + ) + + stats = merge_databases(str(target_path), str(source_path)) + + assert stats["merged"] == 1 + assert {row["source_agent"] for row in _rows(target)} == {"alice", "bob"} + + +def test_merge_conflict_resolution_does_not_supersede_foreign_private_tuple( + tmp_path: Path, +) -> None: + """A confirmed tuple only conflicts inside its exact private namespace.""" + target_path = tmp_path / "target-private-tuple.db" + source_path = tmp_path / "source-private-tuple.db" + target = _store(target_path) + source = _store(source_path) + alice = _claim( + target, + text="Alice private tuple value.", + key="alice-private-tuple", + principal="alice", + visibility="private", + object_value="alice-value", + ) + bob = _claim( + source, + text="Bob private tuple value.", + key="bob-private-tuple", + principal="bob", + visibility="private", + object_value="bob-value", + ) + transition_claim( + target, + alice.id, + "confirmed", + reason="target fixture", + event_type="validator", + ) + transition_claim( + source, + bob.id, + "confirmed", + reason="source fixture", + event_type="validator", + ) + with source.connect() as conn: + conn.execute( + "UPDATE claims SET updated_at='2099-01-01T00:00:00+00:00' WHERE id=?", + (bob.id,), + ) + + stats = merge_databases(str(target_path), str(source_path)) + + rows = _rows(target) + assert stats["merged"] == 1 + assert {(row["source_agent"], row["status"]) for row in rows} == { + ("alice", "confirmed"), + ("bob", "confirmed"), + } + assert all(row["replaced_by_claim_id"] is None for row in rows) + + +def test_merge_public_key_remains_tenant_wide_across_principals( + tmp_path: Path, +) -> None: + """Public identities deliberately ignore source principal within a tenant.""" + target_path = tmp_path / "target-public-key.db" + source_path = tmp_path / "source-public-key.db" + target = _store(target_path) + source = _store(source_path) + _claim( + target, + text="Tenant-wide public merge identity.", + key="public-merge-key", + principal="alice", + visibility="public", + ) + _claim( + source, + text="Changed public payload still deduplicates.", + key="public-merge-key", + principal="bob", + visibility="public", + ) + + stats = merge_databases(str(target_path), str(source_path)) + + assert stats["skipped"] == 1 + assert len(_rows(target)) == 1 + + +def test_legacy_source_rerun_uses_target_default_identity_namespace( + tmp_path: Path, +) -> None: + target_path = tmp_path / "modern-target.db" + source_path = tmp_path / "legacy-source.db" + target = _store(target_path) + with sqlite3.connect(source_path) as conn: + conn.execute( + """ + CREATE TABLE claims ( + id INTEGER PRIMARY KEY, + text TEXT NOT NULL, + idempotency_key TEXT, + status TEXT NOT NULL DEFAULT 'candidate', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ) + """ + ) + conn.execute( + """ + INSERT INTO claims ( + text, idempotency_key, created_at, updated_at + ) VALUES (?, ?, ?, ?) + """, + ( + "Legacy source with target defaults.", + "legacy-default-key", + "2026-01-01T00:00:00+00:00", + "2026-01-01T00:00:00+00:00", + ), + ) + + first = merge_databases(str(target_path), str(source_path)) + second = merge_databases(str(target_path), str(source_path)) + + assert first == {"scanned": 1, "merged": 1, "skipped": 0, "errors": 0} + assert second == {"scanned": 1, "merged": 0, "skipped": 1, "errors": 0} + row = _rows(target)[0] + assert row["scope"] == "project" + assert row["visibility"] == "public" diff --git a/tests/test_identity_bypass_raw_ingesters.py b/tests/test_identity_bypass_raw_ingesters.py new file mode 100644 index 00000000..46bcfd8e --- /dev/null +++ b/tests/test_identity_bypass_raw_ingesters.py @@ -0,0 +1,118 @@ +"""Adversarial namespace tests for legacy raw claim-ingest surfaces.""" +from __future__ import annotations + +import hashlib +import json +from pathlib import Path +from unittest.mock import patch + +from memorymaster.bridges.dream_bridge import dream_ingest +from memorymaster.core.models import CitationInput +from memorymaster.knowledge.transcript_miner import mine_transcript +from memorymaster.stores.storage import SQLiteStore + + +CITATIONS = [CitationInput(source="identity-bypass-red", locator="raw-ingest")] + + +def _store(path: Path) -> SQLiteStore: + store = SQLiteStore(path) + store.init_db() + return store + + +def _private_claim(store: SQLiteStore, *, text: str, key: str) -> None: + store.create_claim( + text, + CITATIONS, + idempotency_key=key, + subject="hidden-subject", + predicate="uses", + scope="project", + source_agent="alice", + visibility="private", + ) + + +def _identity_rows(store: SQLiteStore, key: str) -> list[dict[str, object]]: + with store.connect() as conn: + return [ + dict(row) + for row in conn.execute( + "SELECT * FROM claims WHERE idempotency_key=? ORDER BY id", + (key,), + ) + ] + + +def _write_dream_note(memory_dir: Path, filename: str) -> None: + memory_dir.mkdir() + (memory_dir / filename).write_text( + "\n".join( + [ + "---", + 'name: "public-dream-note"', + 'description: "Identity namespace regression fixture"', + 'type: "project"', + "---", + "", + "The public retrieval bridge uses a bounded cache.", + "", + ] + ), + encoding="utf-8", + ) + + +def test_dream_ingest_private_key_does_not_block_public_identity( + tmp_path: Path, +) -> None: + """A hidden key must not make the public dream importer report duplicate.""" + db_path = tmp_path / "dream-namespace.db" + store = _store(db_path) + marker = "auto-dream:public-dream.md" + _private_claim(store, text="Alice hidden dream payload.", key=marker) + memory_dir = tmp_path / "memory" + _write_dream_note(memory_dir, "public-dream.md") + + with patch.dict("os.environ", {"CLAUDE_MEMORY_DIR": str(memory_dir)}): + stats = dream_ingest(str(db_path), use_spool=False) + + rows = _identity_rows(store, marker) + assert stats["ingested"] == 1 + assert {(row["visibility"], row["source_agent"]) for row in rows} == { + ("private", "alice"), + ("public", None), + } + + +def test_transcript_miner_private_key_does_not_block_public_identity( + tmp_path: Path, +) -> None: + """Transcript dedup must query the public namespace, not every hidden row.""" + db_path = tmp_path / "transcript-namespace.db" + store = _store(db_path) + text = "The root cause was a stale cache entry in the retrieval worker." + digest = hashlib.sha256(text[:500].strip().lower().encode()).hexdigest()[:16] + key = f"transcript-{digest}" + _private_claim(store, text="Alice hidden transcript payload.", key=key) + transcript = tmp_path / "session.jsonl" + transcript.write_text( + json.dumps({"role": "assistant", "content": text}) + "\n", + encoding="utf-8", + ) + + stats = mine_transcript( + transcript, + str(db_path), + scope="project", + min_length=10, + ) + + rows = _identity_rows(store, key) + assert stats["ingested"] == 1 + assert stats["duplicates"] == 0 + assert {(row["visibility"], row["source_agent"]) for row in rows} == { + ("private", "alice"), + ("public", "transcript-miner"), + } diff --git a/tests/test_identity_bypass_scope_merge.py b/tests/test_identity_bypass_scope_merge.py new file mode 100644 index 00000000..d4f9f7b7 --- /dev/null +++ b/tests/test_identity_bypass_scope_merge.py @@ -0,0 +1,185 @@ +"""Adversarial tuple-namespace tests for scope canonicalization.""" +from __future__ import annotations + +import sqlite3 + +from scripts.merge_scope_variants import _archive_confirmed_collisions + + +OLD_SCOPE = "project:identity:variant" +NEW_SCOPE = "project:identity" +OLD_TIME = "2026-01-01T00:00:00+00:00" +NEW_TIME = "2026-02-01T00:00:00+00:00" + + +def _connection() -> sqlite3.Connection: + conn = sqlite3.connect(":memory:") + conn.execute( + """ + CREATE TABLE claims ( + id INTEGER PRIMARY KEY, + subject TEXT, + predicate TEXT, + scope TEXT NOT NULL, + status TEXT NOT NULL, + visibility TEXT NOT NULL, + source_agent TEXT, + tenant_id TEXT, + updated_at TEXT, + archived_at TEXT + ) + """ + ) + return conn + + +def _claim( + conn: sqlite3.Connection, + claim_id: int, + *, + scope: str, + visibility: str, + principal: str, + tenant: str = "tenant-a", + updated_at: str, +) -> None: + conn.execute( + """ + INSERT INTO claims ( + id, subject, predicate, scope, status, visibility, + source_agent, tenant_id, updated_at + ) VALUES (?, 'shared-subject', 'uses', ?, 'confirmed', ?, ?, ?, ?) + """, + (claim_id, scope, visibility, principal, tenant, updated_at), + ) + + +def _statuses(conn: sqlite3.Connection) -> list[str]: + return [row[0] for row in conn.execute("SELECT status FROM claims ORDER BY id")] + + +def test_scope_merge_does_not_archive_other_private_principal() -> None: + conn = _connection() + _claim( + conn, + 1, + scope=OLD_SCOPE, + visibility="private", + principal="alice", + updated_at=OLD_TIME, + ) + _claim( + conn, + 2, + scope=NEW_SCOPE, + visibility="private", + principal="bob", + updated_at=NEW_TIME, + ) + + archived = _archive_confirmed_collisions(conn, OLD_SCOPE, NEW_SCOPE) + + assert archived == 0 + assert _statuses(conn) == ["confirmed", "confirmed"] + + +def test_scope_merge_nonpublic_tuple_uses_exact_visibility() -> None: + conn = _connection() + _claim( + conn, + 1, + scope=OLD_SCOPE, + visibility="private", + principal="alice", + updated_at=OLD_TIME, + ) + _claim( + conn, + 2, + scope=NEW_SCOPE, + visibility="sensitive", + principal="alice", + updated_at=NEW_TIME, + ) + + archived = _archive_confirmed_collisions(conn, OLD_SCOPE, NEW_SCOPE) + + assert archived == 0 + assert _statuses(conn) == ["confirmed", "confirmed"] + + +def test_scope_merge_does_not_archive_public_tuple_from_other_tenant() -> None: + conn = _connection() + _claim( + conn, + 1, + scope=OLD_SCOPE, + visibility="public", + principal="alice", + tenant="tenant-a", + updated_at=OLD_TIME, + ) + _claim( + conn, + 2, + scope=NEW_SCOPE, + visibility="public", + principal="bob", + tenant="tenant-b", + updated_at=NEW_TIME, + ) + + archived = _archive_confirmed_collisions(conn, OLD_SCOPE, NEW_SCOPE) + + assert archived == 0 + assert _statuses(conn) == ["confirmed", "confirmed"] + + +def test_scope_merge_archives_same_private_namespace_collision() -> None: + conn = _connection() + _claim( + conn, + 1, + scope=OLD_SCOPE, + visibility="private", + principal="alice", + updated_at=OLD_TIME, + ) + _claim( + conn, + 2, + scope=NEW_SCOPE, + visibility="private", + principal="alice", + updated_at=NEW_TIME, + ) + + archived = _archive_confirmed_collisions(conn, OLD_SCOPE, NEW_SCOPE) + + assert archived == 1 + assert _statuses(conn) == ["archived", "confirmed"] + + +def test_scope_merge_public_tuple_remains_tenant_wide() -> None: + conn = _connection() + _claim( + conn, + 1, + scope=OLD_SCOPE, + visibility="public", + principal="alice", + updated_at=OLD_TIME, + ) + _claim( + conn, + 2, + scope=NEW_SCOPE, + visibility="public", + principal="bob", + updated_at=NEW_TIME, + ) + + archived = _archive_confirmed_collisions(conn, OLD_SCOPE, NEW_SCOPE) + + assert archived == 1 + assert _statuses(conn) == ["archived", "confirmed"] diff --git a/tests/test_lifecycle_extra.py b/tests/test_lifecycle_extra.py index 473121cf..3b7bda6f 100644 --- a/tests/test_lifecycle_extra.py +++ b/tests/test_lifecycle_extra.py @@ -208,6 +208,34 @@ def test_mark_superseded_double_supersede_is_refused(tmp_path: Path) -> None: assert reloaded_old.replaced_by_claim_id == first_new.id, "original link preserved" +def test_mark_superseded_rejects_archived_claim(tmp_path: Path) -> None: + store = _store(tmp_path) + archived = store.create_claim("archived truth", [_cite("archived")]) + replacement = store.create_claim("replacement", [_cite("replacement")]) + transition_claim(store, archived.id, "archived", "fixture archival") + + with pytest.raises(ValueError, match="Invalid transition"): + store.mark_superseded(archived.id, replacement.id, "invalid replacement") + + reloaded = store.get_claim(archived.id, include_citations=False) + assert reloaded.status == "archived" + assert reloaded.replaced_by_claim_id is None + + +def test_set_supersedes_compatibility_path_is_atomic(tmp_path: Path) -> None: + store = _store(tmp_path) + old = store.create_claim("compatibility old", [_cite("compat-old")]) + new = store.create_claim("compatibility new", [_cite("compat-new")]) + + store.set_supersedes(new.id, old.id) + + reloaded_old = store.get_claim(old.id, include_citations=False) + reloaded_new = store.get_claim(new.id, include_citations=False) + assert reloaded_old.status == "superseded" + assert reloaded_old.replaced_by_claim_id == new.id + assert reloaded_new.supersedes_claim_id == old.id + + # --------------------------------------------------------------------------- # record_event # --------------------------------------------------------------------------- diff --git a/tests/test_mcp_tenant_scope_boundary.py b/tests/test_mcp_tenant_scope_boundary.py index f1b7c177..3b69f7a5 100644 --- a/tests/test_mcp_tenant_scope_boundary.py +++ b/tests/test_mcp_tenant_scope_boundary.py @@ -50,14 +50,14 @@ def team_claims(tmp_path, monkeypatch): access_control._agent_roles.clear() -def test_team_list_and_query_intersect_tenant_and_scope(team_claims) -> None: +def test_team_list_and_query_enforce_tenant_and_scope(team_claims) -> None: db, _workspace, allowed, wrong_scope, wrong_tenant = team_claims listed = mcp_server.list_claims(limit=20) queried = mcp_server.query_memory( query="authorization matrix marker", retrieval_mode="legacy", - scope_allowlist="project:alpha,project:beta", + scope_allowlist="project:alpha", limit=20, ) diff --git a/tests/test_postgres_admin_boundary.py b/tests/test_postgres_admin_boundary.py new file mode 100644 index 00000000..49c84882 --- /dev/null +++ b/tests/test_postgres_admin_boundary.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +from dataclasses import dataclass + +import pytest + +import memorymaster.stores._storage_schema as schema_module +from memorymaster.stores.postgres_store import PostgresStore + + +def test_init_db_uses_admin_connection_before_reading_schema( + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = PostgresStore("postgresql://db.invalid/admin") + + def reject_admin_connection(): + raise PermissionError("migration role requires BYPASSRLS") + + monkeypatch.setattr(store, "_connect_schema_admin", reject_admin_connection, raising=False) + monkeypatch.setattr( + schema_module, + "load_schema_postgres_sql", + lambda: pytest.fail("unverified migration role read the administrative schema"), + ) + + with pytest.raises(PermissionError, match="BYPASSRLS"): + store.init_db() + + +@dataclass +class _AdminRole: + rolsuper: bool = False + rolbypassrls: bool = False + + +class _AdminCursor: + def __init__(self, role: _AdminRole) -> None: + self.role = role + self._row: dict[str, object] | None = None + + def __enter__(self) -> _AdminCursor: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, sql: str, _params: object = None) -> None: + assert "pg_roles" in sql + self._row = { + "current_user": "memorymaster_migrator", + "session_user": "memorymaster_migrator", + "rolsuper": self.role.rolsuper, + "rolbypassrls": self.role.rolbypassrls, + } + + def fetchone(self) -> dict[str, object] | None: + return self._row + + +class _AdminConnection: + def __init__(self, role: _AdminRole) -> None: + self.role = role + self.autocommit = True + self.closed = False + self.rollback_count = 0 + + def cursor(self) -> _AdminCursor: + return _AdminCursor(self.role) + + def rollback(self) -> None: + self.rollback_count += 1 + + def close(self) -> None: + self.closed = True + + +class _AdminDriver: + def __init__(self, connection: _AdminConnection) -> None: + self.connection = connection + + def connect(self, *_args: object, **_kwargs: object) -> _AdminConnection: + return self.connection + + +def _admin_store(role: _AdminRole) -> tuple[PostgresStore, _AdminConnection]: + store = PostgresStore("postgresql://db.invalid/admin") + connection = _AdminConnection(role) + store._psycopg = (_AdminDriver(connection), object(), object()) + return store, connection + + +def test_schema_admin_connection_rejects_unprivileged_runtime_role() -> None: + store, connection = _admin_store(_AdminRole()) + + with pytest.raises(PermissionError, match="(?i)(migration|bypassrls|superuser)"): + store._connect_schema_admin() + + assert connection.rollback_count == 1 + assert connection.closed is True + + +@pytest.mark.parametrize("role", [_AdminRole(rolsuper=True), _AdminRole(rolbypassrls=True)]) +def test_schema_admin_connection_accepts_explicit_privileged_role(role: _AdminRole) -> None: + store, connection = _admin_store(role) + + assert store._connect_schema_admin() is connection + assert connection.autocommit is False + assert connection.closed is False + + +def test_failed_connection_cleanup_closes_even_if_rollback_fails() -> None: + class BrokenRollbackConnection: + closed = False + + def rollback(self) -> None: + raise RuntimeError("rollback failed") + + def close(self) -> None: + self.closed = True + + connection = BrokenRollbackConnection() + + with pytest.raises(RuntimeError, match="rollback failed"): + PostgresStore._cleanup_failed_connection(connection) + + assert connection.closed is True diff --git a/tests/test_postgres_claim_owner_constraint.py b/tests/test_postgres_claim_owner_constraint.py new file mode 100644 index 00000000..54341abb --- /dev/null +++ b/tests/test_postgres_claim_owner_constraint.py @@ -0,0 +1,202 @@ +"""Adversarial RED contracts for PostgreSQL claim-owner enforcement.""" +from __future__ import annotations + +import importlib +import re +from typing import Sequence + +import pytest + +from memorymaster.stores.postgres_store import PostgresStore + + +OWNER_CONSTRAINT_NAME = "ck_claims_identity_visibility_owner" +EXACT_OWNER_CHECK = ( + "CHECK (visibility IN ('public', 'private', 'sensitive') " + "AND NULLIF(BTRIM(source_agent), '') IS NOT NULL)" +) + + +def _constraint_row( + *, + definition: str = EXACT_OWNER_CHECK, + validated: bool = True, +) -> dict[str, object]: + return { + "schema_name": "public", + "table_name": "claims", + "constraint_name": OWNER_CONSTRAINT_NAME, + "conname": OWNER_CONSTRAINT_NAME, + "constraint_type": "c", + "contype": "c", + "validated": validated, + "convalidated": validated, + "is_local": True, + "conislocal": True, + "no_inherit": False, + "connoinherit": False, + "constraint_definition": definition, + "definition": definition, + } + + +class OwnerConstraintCursor: + def __init__(self, row: dict[str, object] | None) -> None: + self.row = row + self.executed: list[tuple[str, tuple[object, ...]]] = [] + self._rows: list[dict[str, object]] = [] + + def __enter__(self) -> OwnerConstraintCursor: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, sql: str, params: Sequence[object] = ()) -> None: + self.executed.append((sql, tuple(params))) + if "pg_constraint" in _canonical(sql): + self._rows = [] if self.row is None else [dict(self.row)] + else: + self._rows = [] + + def fetchone(self) -> dict[str, object] | None: + return self._rows[0] if self._rows else None + + def fetchall(self) -> list[dict[str, object]]: + return list(self._rows) + + +class OwnerConstraintConnection: + def __init__(self, row: dict[str, object] | None) -> None: + self.autocommit = True + self.closed = False + self.rollback_count = 0 + self.cursor_instance = OwnerConstraintCursor(row) + + def cursor(self) -> OwnerConstraintCursor: + return self.cursor_instance + + def rollback(self) -> None: + self.rollback_count += 1 + + def close(self) -> None: + self.closed = True + + +def _canonical(sql: str) -> str: + return " ".join(sql.lower().replace('"', "").split()) + + +def _team_connect( + monkeypatch: pytest.MonkeyPatch, + row: dict[str, object] | None, +) -> tuple[PostgresStore, OwnerConstraintConnection]: + store = PostgresStore( + "postgresql://runtime.invalid/memorymaster", + tenant_id="tenant-a", + require_tenant=True, + principal="alice", + allowed_scopes={"project:a"}, + ) + connection = OwnerConstraintConnection(row) + monkeypatch.setattr(store, "_open_connection", lambda: connection) + for method in ( + "_validate_runtime_role", + "_validate_runtime_tables", + "_validate_runtime_metadata_tables", + "_validate_claim_identity_indexes", + "_validate_claim_supersession_guard", + "_validate_event_append_only_catalog", + "_validate_event_chain_head_function", + "_validate_runtime_migration", + "_validate_runtime_policies", + ): + monkeypatch.setattr( + PostgresStore, + method, + classmethod(lambda _cls, _cur: None), + ) + monkeypatch.setattr( + PostgresStore, + "_bind_runtime_authority", + classmethod(lambda _cls, _cur, _tenant, _principal, _scopes: None), + ) + return store, connection + + +def test_team_runtime_accepts_exact_validated_source_owner_constraint( + monkeypatch: pytest.MonkeyPatch, +) -> None: + store, connection = _team_connect(monkeypatch, _constraint_row()) + + assert store.connect() is connection + assert connection.closed is False + + +@pytest.mark.parametrize( + "row", + [ + None, + _constraint_row(validated=False), + _constraint_row( + definition=( + "CHECK (visibility IN ('public', 'private', 'sensitive') AND " + "(visibility = 'public' OR " + "NULLIF(BTRIM(source_agent), '') IS NOT NULL))" + ) + ), + ], + ids=("missing", "not-validated", "public-owner-bypass"), +) +def test_team_runtime_rejects_unsafe_source_owner_constraint_before_binding( + monkeypatch: pytest.MonkeyPatch, + row: dict[str, object] | None, +) -> None: + store, connection = _team_connect(monkeypatch, row) + + with pytest.raises(PermissionError, match="(?i)(claim|owner|constraint|validated)"): + store.connect() + + assert connection.rollback_count == 1 + assert connection.closed is True + + +def test_team_runtime_inspects_exact_constraint_catalog( + monkeypatch: pytest.MonkeyPatch, +) -> None: + store, connection = _team_connect(monkeypatch, _constraint_row()) + + store.connect() + + catalog_sql = "\n".join(sql for sql, _params in connection.cursor_instance.executed) + normalized = _canonical(catalog_sql) + assert "pg_constraint" in normalized + assert "pg_get_constraintdef" in normalized + assert "convalidated" in normalized + + +def test_v0012_source_owner_constraint_has_no_public_owner_bypass() -> None: + migration = importlib.import_module( + "memorymaster.stores.migrations.0012_principal_local_claim_identities" + ) + ddl = _canonical(migration._POSTGRES_DDL) + + assert "nullif(btrim(source_agent), '') is not null" in ddl + assert not re.search( + r"visibility\s*=\s*'public'\s+or\s+nullif\(btrim\(source_agent\)", + ddl, + ) + + +def test_brownfield_owner_preflight_is_read_only_and_includes_public_rows() -> None: + migration = importlib.import_module( + "memorymaster.stores.migrations.0012_principal_local_claim_identities" + ) + sql = _canonical(getattr(migration, "POSTGRES_IDENTITY_PREFLIGHT_SQL", "")) + + assert sql.startswith("select") + assert "visibility" in sql + assert "nullif(btrim(source_agent), '') is null" in sql + assert "visibility <> 'public'" not in sql + assert "visibility != 'public'" not in sql + assert not re.search(r"\b(insert|update|delete|alter|drop|create|truncate)\b", sql) diff --git a/tests/test_postgres_event_head_rls.py b/tests/test_postgres_event_head_rls.py new file mode 100644 index 00000000..8c723ae8 --- /dev/null +++ b/tests/test_postgres_event_head_rls.py @@ -0,0 +1,288 @@ +"""Adversarial contracts for the PostgreSQL tenant-global event chain head. + +The event ledger is one primary and one secondary hash chain per tenant. RLS +may hide events from another scope or private principal, so an application +query over ``events`` cannot be trusted to discover either chain head. +""" +from __future__ import annotations + +import importlib +import re +from dataclasses import dataclass, replace + +import pytest + +from memorymaster.stores.postgres_store import ( + POSTGRES_TENANT_EVENT_HASH_ALGO, + PostgresStore, +) + + +class MigrationCursor: + def __init__(self, statements: list[str]) -> None: + self.statements = statements + + def __enter__(self) -> "MigrationCursor": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, statement: object, _params: object = None) -> None: + self.statements.append(str(statement).strip()) + + +class MigrationConnection: + def __init__(self) -> None: + self.statements: list[str] = [] + self.commits = 0 + self.rollbacks = 0 + + def cursor(self) -> MigrationCursor: + return MigrationCursor(self.statements) + + def commit(self) -> None: + self.commits += 1 + + def rollback(self) -> None: + self.rollbacks += 1 + + +def _migration_statements(monkeypatch: pytest.MonkeyPatch) -> list[str]: + migration = importlib.import_module( + "memorymaster.stores.migrations.0011_postgres_scoped_force_rls" + ) + monkeypatch.setattr(migration, "_stamp_policy_manifest", lambda _cur: None) + connection = MigrationConnection() + + migration.apply_postgres(connection) + + assert connection.commits == 1 + assert connection.rollbacks == 0 + return connection.statements + + +def _event_head_function(statements: list[str]) -> str: + matches = [ + statement + for statement in statements + if re.search( + r"CREATE\s+(?:OR\s+REPLACE\s+)?FUNCTION\s+" + r"public\.memorymaster_event_chain_head\s*\(\s*\)", + statement, + flags=re.IGNORECASE, + ) + ] + assert len(matches) == 1 + return matches[0] + + +def test_v0011_installs_narrow_security_definer_event_head( + monkeypatch: pytest.MonkeyPatch, +) -> None: + function_sql = _event_head_function(_migration_statements(monkeypatch)) + normalized = " ".join(function_sql.split()).lower() + + assert "security definer" in normalized + assert re.search( + r"set\s+search_path\s*(?:=|to)\s*pg_catalog\s*,\s*pg_temp", + normalized, + ) + assert "from public.events" in normalized + assert "current_setting('memorymaster.tenant_id', true)" in normalized + assert "memorymaster.principal" not in normalized + assert "memorymaster.allowed_scopes" not in normalized + assert "payload_json" not in normalized + assert "details" not in normalized + + returns = re.search( + r"returns\s+table\s*\((.*?)\)", + normalized, + ) + assert returns is not None + returned_columns = [part.strip().split()[0] for part in returns.group(1).split(",")] + assert len(returned_columns) == 2 + assert any(name in {"event_hash", "global_event_hash"} for name in returned_columns) + assert "tenant_event_hash" in returned_columns + + +def test_v0011_revokes_public_event_head_execute( + monkeypatch: pytest.MonkeyPatch, +) -> None: + emitted = "\n".join(_migration_statements(monkeypatch)) + normalized = " ".join(emitted.split()).upper() + + assert re.search( + r"REVOKE\s+(?:ALL|EXECUTE)\s+ON\s+FUNCTION\s+" + r"PUBLIC\.MEMORYMASTER_EVENT_CHAIN_HEAD\s*\(\s*\)\s+FROM\s+PUBLIC", + normalized, + ) + assert not re.search( + r"GRANT\s+EXECUTE\s+ON\s+FUNCTION\s+" + r"PUBLIC\.MEMORYMASTER_EVENT_CHAIN_HEAD\s*\(\s*\)\s+TO\s+PUBLIC", + normalized, + ) + + +class RlsFilteredHeadCursor: + """Simulate two RLS views over one tenant-global event ledger.""" + + def __init__(self, filtered_head: str, filtered_tenant_head: str) -> None: + self.filtered_head = filtered_head + self.filtered_tenant_head = filtered_tenant_head + self._row: dict[str, str | None] | None = None + self.function_calls = 0 + self.direct_event_selects = 0 + + def execute(self, statement: str, _params: object = None) -> None: + normalized = " ".join(statement.split()).lower() + if "pg_advisory_xact_lock" in normalized: + self._row = None + return + if "memorymaster_event_chain_head()" in normalized: + self.function_calls += 1 + self._row = { + "global_event_hash": "tenant-global-primary-head", + "event_hash": "tenant-global-primary-head", + "tenant_event_hash": "tenant-global-secondary-head", + } + return + if "from events" in normalized or "from public.events" in normalized: + self.direct_event_selects += 1 + key = "tenant_event_hash" if "tenant_event_hash" in normalized else "event_hash" + value = self.filtered_tenant_head if key == "tenant_event_hash" else self.filtered_head + self._row = {key: value} + return + raise AssertionError(f"unexpected event-head SQL: {normalized}") + + def fetchone(self) -> dict[str, str | None] | None: + return self._row + + +def _team_store(*, principal: str, scope: str) -> PostgresStore: + return PostgresStore( + "postgresql://unused", + tenant_id="tenant-a", + require_tenant=True, + principal=principal, + allowed_scopes=(scope,), + ) + + +def test_event_head_is_identical_across_scope_and_private_principal_views() -> None: + alice = _team_store(principal="alice", scope="project:a") + bob = _team_store(principal="bob", scope="project:b") + alice_cursor = RlsFilteredHeadCursor("alice-visible", "alice-private-visible") + bob_cursor = RlsFilteredHeadCursor("bob-visible", "bob-private-visible") + + alice_head = alice._event_chain_head(alice_cursor, "tenant-a") + bob_head = bob._event_chain_head(bob_cursor, "tenant-a") + + expected = ( + "tenant-global-primary-head", + POSTGRES_TENANT_EVENT_HASH_ALGO, + "tenant-global-secondary-head", + ) + assert alice_head == expected + assert bob_head == expected + assert alice_cursor.function_calls == bob_cursor.function_calls == 1 + assert alice_cursor.direct_event_selects == bob_cursor.direct_event_selects == 0 + + +_EVENT_HEAD_MIGRATION = importlib.import_module( + "memorymaster.stores.migrations.0011_postgres_scoped_force_rls" +) +_EVENT_HEAD_DEFINITION = str(_EVENT_HEAD_MIGRATION._EVENT_HEAD_FUNCTION) +_EVENT_HEAD_SOURCE = _EVENT_HEAD_DEFINITION.split( + "AS $$", 1 +)[1].rsplit("$$", 1)[0].strip() + + +@dataclass(frozen=True) +class FunctionCatalogState: + schema_name: str = "public" + function_name: str = "memorymaster_event_chain_head" + argument_count: int = 0 + result_signature: str = "TABLE(global_event_hash text, tenant_event_hash text)" + language_name: str = "plpgsql" + security_definer: bool = True + function_config: tuple[str, ...] = ("search_path=pg_catalog, pg_temp",) + volatility: str = "v" + parallel_safety: str = "u" + leakproof: bool = False + strict: bool = False + public_execute: bool = False + runtime_execute: bool = True + owner_is_runtime: bool = False + owner_member: bool = False + owner_superuser: bool = True + owner_bypassrls: bool = False + function_source: str = _EVENT_HEAD_SOURCE + function_definition: str = _EVENT_HEAD_DEFINITION + + +class FunctionCatalogCursor: + def __init__(self, state: FunctionCatalogState) -> None: + self.state = state + self.statements: list[str] = [] + + def execute(self, statement: str, _params: object = None) -> None: + self.statements.append(" ".join(statement.split())) + + def fetchone(self) -> dict[str, object]: + return vars(self.state) + + +def test_runtime_accepts_only_exact_event_head_capability() -> None: + cursor = FunctionCatalogCursor(FunctionCatalogState()) + + PostgresStore._validate_event_chain_head_function(cursor) + + assert any("pg_proc" in statement.lower() for statement in cursor.statements) + + +@pytest.mark.parametrize( + ("change", "match"), + [ + ({"argument_count": 1}, "argument|signature"), + ({"security_definer": False}, "security.definer"), + ({"function_config": ()}, "search.path"), + ({"function_config": ("search_path=public",)}, "search.path"), + ({"public_execute": True}, "public|execute"), + ({"runtime_execute": False}, "execute|privilege"), + ({"owner_is_runtime": True}, "owner|runtime"), + ( + {"owner_superuser": False, "owner_bypassrls": False}, + "owner|bypass|rls", + ), + ( + { + "function_source": "BEGIN RETURN QUERY SELECT event_hash, " + "tenant_event_hash FROM public.events; END;" + }, + "body|definition|events", + ), + ], +) +def test_runtime_rejects_event_head_catalog_drift( + change: dict[str, object], + match: str, +) -> None: + cursor = FunctionCatalogCursor(replace(FunctionCatalogState(), **change)) + + with pytest.raises(PermissionError, match=f"(?i)({match})"): + PostgresStore._validate_event_chain_head_function(cursor) + + +def test_runtime_preserves_literal_case_in_event_head_fingerprint() -> None: + drifted_source = _EVENT_HEAD_SOURCE.replace( + "'sha256-tenant-v2'", + "'SHA256-TENANT-V2'", + ) + assert drifted_source != _EVENT_HEAD_SOURCE + cursor = FunctionCatalogCursor( + replace(FunctionCatalogState(), function_source=drifted_source) + ) + + with pytest.raises(PermissionError, match="(?i)(body|definition|events)"): + PostgresStore._validate_event_chain_head_function(cursor) diff --git a/tests/test_postgres_force_rls_migration.py b/tests/test_postgres_force_rls_migration.py new file mode 100644 index 00000000..ae1b98a4 --- /dev/null +++ b/tests/test_postgres_force_rls_migration.py @@ -0,0 +1,460 @@ +"""Adversarial contract tests for PostgreSQL scoped FORCE-RLS migration v0011.""" +from __future__ import annotations + +import hashlib +import importlib +import json +import re +import sqlite3 +from typing import Sequence + +from memorymaster.stores.migrations import discover_migrations + + +SCOPED_TABLES = { + "claims", + "citations", + "events", + "claim_links", + "claim_embeddings", + "contradiction_verdicts", + "mcp_usage", +} + +DENY_ALL_TABLES = { + "action_proposals", + "external_sources", + "source_items", + "evidence_items", + "media_retry_queue", + "query_cache", + "miner_state", + "rule_stats", +} + +GOVERNED_TABLES = SCOPED_TABLES | DENY_ALL_TABLES + +COMMAND_POLICIES = { + "SELECT": "memorymaster_tenant_select", + "INSERT": "memorymaster_tenant_insert", + "UPDATE": "memorymaster_tenant_update", + "DELETE": "memorymaster_tenant_delete", +} + +PERMIT_POLICIES = { + command: f"{name}_permit" for command, name in COMMAND_POLICIES.items() +} + + +POLICY_FIELDS = ( + "schemaname", + "tablename", + "policyname", + "permissive", + "roles", + "cmd", + "qual", + "with_check", +) + + +def _parenthesized_clause(sql: str, marker: str) -> str | None: + start = sql.find(marker) + if start < 0: + return None + start += len(marker) + depth = 1 + for index in range(start, len(sql)): + character = sql[index] + if character == "(": + depth += 1 + elif character == ")": + depth -= 1 + if depth == 0: + return sql[start:index] + raise AssertionError(f"unbalanced policy clause: {sql}") + + +def _parse_created_policy(sql: str) -> dict[str, object] | None: + match = re.match( + r"CREATE POLICY (\S+) ON (\S+) AS (PERMISSIVE|RESTRICTIVE) " + r"FOR (SELECT|INSERT|UPDATE|DELETE|ALL) TO PUBLIC (.*)", + sql, + ) + if match is None: + return None + name, table, mode, command, clauses = match.groups() + return { + "schemaname": "public", + "tablename": table, + "policyname": name, + "permissive": mode, + "roles": ["public"], + "cmd": command, + "qual": _parenthesized_clause(clauses, "USING ("), + "with_check": _parenthesized_clause(clauses, "WITH CHECK ("), + } + + +def _canonical_policy_payload(policies: Sequence[dict[str, object]]) -> str: + rows: list[dict[str, object]] = [] + for policy in policies: + row = {field: policy.get(field) for field in POLICY_FIELDS} + roles = row["roles"] + if isinstance(roles, (list, tuple, set, frozenset)): + row["roles"] = sorted(str(role) for role in roles) + rows.append(row) + rows.sort( + key=lambda row: ( + str(row["schemaname"]), + str(row["tablename"]), + str(row["policyname"]), + ) + ) + return json.dumps(rows, ensure_ascii=False, separators=(",", ":")) + + +class RecordingCursor: + def __init__(self, connection: RecordingConnection) -> None: + self.connection = connection + self._rows: list[dict[str, object]] = [] + + def __enter__(self) -> "RecordingCursor": + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, sql: str, params: object = None) -> None: + normalized = " ".join(sql.split()) + bound = tuple(params) if isinstance(params, (list, tuple)) else () + recorded = normalized + if len(bound) == 1 and "%s" in recorded: + recorded = recorded.replace("%s", repr(bound[0]), 1) + self.connection.statements.append(recorded) + if normalized.startswith("COMMENT ON POLICY") and bound: + self.connection.parameterized_comment_calls.append((normalized, bound)) + if "FROM pg_policies" in normalized: + self.connection.catalog_reads += 1 + self._rows = self.connection.policy_rows + return + created = _parse_created_policy(normalized) + if created is not None: + key = (str(created["tablename"]), str(created["policyname"])) + self.connection.policies[key] = created + return + dropped = re.match(r"DROP POLICY IF EXISTS (\S+) ON (\S+)", normalized) + if dropped is not None: + name, table = dropped.groups() + self.connection.policies.pop((table, name), None) + + def fetchall(self) -> list[dict[str, object]]: + return [dict(row) for row in self._rows] + + def fetchone(self) -> dict[str, object] | None: + return dict(self._rows[0]) if self._rows else None + + +class RecordingConnection: + def __init__(self) -> None: + self.statements: list[str] = [] + self.policies: dict[tuple[str, str], dict[str, object]] = {} + self.catalog_reads = 0 + self.parameterized_comment_calls: list[tuple[str, tuple[object, ...]]] = [] + self.commits = 0 + self.rollbacks = 0 + + def cursor(self) -> RecordingCursor: + return RecordingCursor(self) + + @property + def policy_rows(self) -> list[dict[str, object]]: + rows = [dict(row) for _, row in sorted(self.policies.items())] + # pg_policies exposes server-deparsed expressions, not the emitted DDL text. + # Deliberately model a harmless deparser difference so source-SQL hashing + # cannot satisfy the manifest test while merely performing a decoy read. + for row in rows: + for field in ("qual", "with_check"): + expression = row[field] + if expression not in {None, "FALSE"}: + row[field] = f"({expression})" + return rows + + def commit(self) -> None: + self.commits += 1 + + def rollback(self) -> None: + self.rollbacks += 1 + + +def _migration(): + return importlib.import_module( + "memorymaster.stores.migrations.0011_postgres_scoped_force_rls" + ) + + +def _apply_postgres_migration() -> RecordingConnection: + conn = RecordingConnection() + _migration().apply_postgres(conn) + return conn + + +def _policy(conn: RecordingConnection, table: str, name: str) -> str: + prefix = f"CREATE POLICY {name} ON {table} " + return next(statement for statement in conn.statements if statement.startswith(prefix)) + + +def _command_policy(conn: RecordingConnection, table: str, command: str) -> str: + policy = _policy(conn, table, COMMAND_POLICIES[command]) + assert f"FOR {command} TO PUBLIC" in policy + return policy + + +def _permit_policy(conn: RecordingConnection, table: str, command: str) -> str: + policy = _policy(conn, table, PERMIT_POLICIES[command]) + assert f"AS PERMISSIVE FOR {command} TO PUBLIC" in policy + return policy + + +def test_scoped_force_rls_is_immutable_migration_v0011() -> None: + migration = next(item for item in discover_migrations() if item.version == 11) + + assert "force" in migration.description.lower() + assert "scope" in migration.description.lower() + + +def test_sqlite_side_is_a_true_noop() -> None: + conn = sqlite3.connect(":memory:") + try: + _migration().apply_sqlite(conn) + tables = conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table'" + ).fetchall() + finally: + conn.close() + + assert tables == [] + + +def test_all_governed_tables_enable_and_force_row_security() -> None: + conn = _apply_postgres_migration() + emitted = "\n".join(conn.statements) + + assert len(GOVERNED_TABLES) == 15 + for table in GOVERNED_TABLES: + assert f"ALTER TABLE {table} ENABLE ROW LEVEL SECURITY" in emitted + assert f"ALTER TABLE {table} FORCE ROW LEVEL SECURITY" in emitted + assert conn.commits == 1 + assert conn.rollbacks == 0 + + +def test_claim_select_allows_public_or_principal_owned_private_rows() -> None: + policy = _command_policy(_apply_postgres_migration(), "claims", "SELECT") + + assert "NULLIF(current_setting('memorymaster.principal', true), '') IS NOT NULL" in policy + assert ( + "claims.tenant_id = NULLIF(current_setting('memorymaster.tenant_id', true), '')" + in policy + ) + assert ( + "COALESCE(NULLIF(current_setting('memorymaster.allowed_scopes', true), ''), " + "'[]')::jsonb ? claims.scope" + ) in policy + assert ( + "jsonb_typeof(COALESCE(NULLIF(current_setting(" + "'memorymaster.allowed_scopes', true), ''), '[]')::jsonb) = 'array'" + in policy + ) + assert "claims.visibility = 'public'" in policy + assert "claims.visibility = 'private'" in policy + assert "claims.visibility <> 'private'" not in policy + assert ( + "claims.source_agent = NULLIF(current_setting('memorymaster.principal', true), '')" + in policy + ) + assert "USING" in policy + assert "WITH CHECK" not in policy + + +def test_claim_writes_require_principal_ownership_without_public_bypass() -> None: + conn = _apply_postgres_migration() + + for command in {"INSERT", "UPDATE", "DELETE"}: + policy = _command_policy(conn, "claims", command) + assert ( + "claims.source_agent = NULLIF(current_setting('memorymaster.principal', true), '')" + in policy + ) + assert "? claims.scope" in policy + assert "claims.visibility = 'public'" not in policy + assert "claims.visibility = 'private'" not in policy + assert "WITH CHECK" in _command_policy(conn, "claims", "INSERT") + assert "WITH CHECK" in _command_policy(conn, "claims", "UPDATE") + assert "WITH CHECK" not in _command_policy(conn, "claims", "DELETE") + + +def test_claim_write_policies_reject_unreadable_sensitive_visibility() -> None: + conn = _apply_postgres_migration() + + for command in {"INSERT", "UPDATE", "DELETE"}: + policy = _command_policy(conn, "claims", command) + assert "claims.visibility IN ('public', 'private')" in policy + assert "sensitive" not in policy + + +def test_single_claim_children_split_read_visibility_from_write_ownership() -> None: + conn = _apply_postgres_migration() + + for table in {"citations", "claim_embeddings"}: + read_policy = _command_policy(conn, table, "SELECT") + assert f"mm_claim.id = {table}.claim_id" in read_policy + assert "mm_claim.visibility = 'public'" in read_policy + for command in {"INSERT", "UPDATE", "DELETE"}: + write_policy = _command_policy(conn, table, command) + assert f"mm_claim.id = {table}.claim_id" in write_policy + assert "mm_claim.visibility = 'public'" not in write_policy + assert "mm_claim.source_agent = NULLIF(current_setting('memorymaster.principal', true), '')" in write_policy + + +def test_action_proposals_remain_team_denied_in_phase_one() -> None: + conn = _apply_postgres_migration() + policy = _policy(conn, "action_proposals", "memorymaster_team_deny") + + assert "AS RESTRICTIVE FOR ALL TO PUBLIC" in policy + assert "USING (FALSE) WITH CHECK (FALSE)" in policy + assert not any( + statement.startswith(f"CREATE POLICY {name} ON action_proposals ") + for name in COMMAND_POLICIES.values() + for statement in conn.statements + ) + + +def test_pair_children_require_read_access_or_write_ownership_for_both_claims() -> None: + conn = _apply_postgres_migration() + links = _command_policy(conn, "claim_links", "SELECT") + verdicts = _command_policy(conn, "contradiction_verdicts", "SELECT") + + assert "mm_source.id = claim_links.source_id" in links + assert "mm_target.id = claim_links.target_id" in links + assert links.count("current_setting('memorymaster.allowed_scopes', true)") >= 4 + assert "mm_a.id = contradiction_verdicts.claim_a_id" in verdicts + assert "mm_b.id = contradiction_verdicts.claim_b_id" in verdicts + assert verdicts.count("current_setting('memorymaster.allowed_scopes', true)") >= 4 + for table in {"claim_links", "contradiction_verdicts"}: + for command in {"INSERT", "UPDATE", "DELETE"}: + policy = _command_policy(conn, table, command) + assert "visibility = 'public'" not in policy + assert policy.count("source_agent") >= 2 + + +def test_events_split_public_reads_from_claim_owner_writes() -> None: + conn = _apply_postgres_migration() + policy = _command_policy(conn, "events", "SELECT") + + assert "events.tenant_id = NULLIF(current_setting('memorymaster.tenant_id', true), '')" in policy + assert "NULLIF(current_setting('memorymaster.principal', true), '') IS NOT NULL" in policy + assert "events.claim_id IS NULL OR EXISTS" in policy + assert "mm_claim.id = events.claim_id" in policy + assert "? mm_claim.scope" in policy + assert "mm_claim.visibility = 'public'" in policy + assert "mm_claim.visibility = 'private'" in policy + for command in {"INSERT", "UPDATE", "DELETE"}: + write_policy = _command_policy(conn, "events", command) + assert "events.claim_id IS NULL OR EXISTS" in write_policy + assert "mm_claim.visibility = 'public'" not in write_policy + assert "mm_claim.source_agent" in write_policy + + +def test_mcp_usage_is_strictly_tenant_bound() -> None: + conn = _apply_postgres_migration() + for command in COMMAND_POLICIES: + policy = _command_policy(conn, "mcp_usage", command) + assert "mcp_usage.tenant_id = NULLIF(current_setting('memorymaster.tenant_id', true), '')" in policy + assert "NULLIF(current_setting('memorymaster.principal', true), '') IS NOT NULL" in policy + assert "tenant_id IS NOT DISTINCT FROM" not in policy + + +def test_untenantable_tables_keep_restrictive_deny_all_policies() -> None: + conn = _apply_postgres_migration() + emitted = "\n".join(conn.statements) + + for table in DENY_ALL_TABLES: + policy = _policy(conn, table, "memorymaster_team_deny") + assert "AS RESTRICTIVE" in policy + assert "USING (FALSE) WITH CHECK (FALSE)" in policy + assert f"DROP POLICY IF EXISTS memorymaster_tenant_restrict ON {table}" in emitted + + +def test_scoped_commands_pair_exact_permissive_and_restrictive_predicates() -> None: + conn = _apply_postgres_migration() + emitted = "\n".join(conn.statements) + + assert "USING (TRUE)" not in emitted + assert "WITH CHECK (TRUE)" not in emitted + for table in SCOPED_TABLES: + for command in COMMAND_POLICIES: + permit = _permit_policy(conn, table, command) + restrict = _command_policy(conn, table, command) + assert permit.split(" TO PUBLIC ", 1)[1] == restrict.split(" TO PUBLIC ", 1)[1] + for table in DENY_ALL_TABLES: + assert not any( + statement.startswith("CREATE POLICY ") and f" ON {table} " in statement + and "AS PERMISSIVE" in statement + for statement in conn.statements + ) + for table in GOVERNED_TABLES: + assert f"DROP POLICY IF EXISTS memorymaster_rls_permit ON {table}" in emitted + for name in ( + "memorymaster_tenant_restrict", + *COMMAND_POLICIES.values(), + *PERMIT_POLICIES.values(), + ): + assert emitted.count(f"DROP POLICY IF EXISTS {name} ON") == 15 + assert emitted.count("AS PERMISSIVE FOR SELECT TO PUBLIC") == len(SCOPED_TABLES) + assert emitted.count("AS PERMISSIVE FOR INSERT TO PUBLIC") == len(SCOPED_TABLES) + assert emitted.count("AS PERMISSIVE FOR UPDATE TO PUBLIC") == len(SCOPED_TABLES) + assert emitted.count("AS PERMISSIVE FOR DELETE TO PUBLIC") == len(SCOPED_TABLES) + assert emitted.count("AS RESTRICTIVE FOR SELECT TO PUBLIC") == len(SCOPED_TABLES) + assert emitted.count("AS RESTRICTIVE FOR INSERT TO PUBLIC") == len(SCOPED_TABLES) + assert emitted.count("AS RESTRICTIVE FOR UPDATE TO PUBLIC") == len(SCOPED_TABLES) + assert emitted.count("AS RESTRICTIVE FOR DELETE TO PUBLIC") == len(SCOPED_TABLES) + assert emitted.count("AS RESTRICTIVE FOR ALL TO PUBLIC") == len(DENY_ALL_TABLES) + + +def test_migration_stamps_one_policy_manifest_and_disables_cache_generation_triggers() -> None: + conn = _apply_postgres_migration() + emitted = "\n".join(conn.statements) + digest = hashlib.sha256( + _canonical_policy_payload(conn.policy_rows).encode("utf-8") + ).hexdigest() + expected_comment = f"memorymaster.rls/v1;manifest=0011;sha256={digest}" + + stamp_statements = [ + statement + for statement in conn.statements + if statement.startswith("COMMENT ON POLICY ") + and expected_comment in statement + ] + assert len(stamp_statements) == 1 + assert conn.catalog_reads >= 1 + catalog_query = next( + statement for statement in conn.statements if "FROM pg_policies" in statement + ) + for field in POLICY_FIELDS: + assert field in catalog_query + assert "claims_gen_ins_del" in emitted + assert "claims_gen_upd" in emitted + assert emitted.count("DROP TRIGGER IF EXISTS claims_gen_") == 2 + + +def test_policy_manifest_comment_uses_literal_ddl_not_server_side_parameters() -> None: + conn = _apply_postgres_migration() + + assert conn.parameterized_comment_calls == [] + comment = next( + statement + for statement in conn.statements + if statement.startswith("COMMENT ON POLICY ") + ) + assert " IS 'memorymaster.rls/v1;manifest=0011;sha256=" in comment + assert comment.endswith("'") diff --git a/tests/test_postgres_policy_expression_contract.py b/tests/test_postgres_policy_expression_contract.py new file mode 100644 index 00000000..d041ed4d --- /dev/null +++ b/tests/test_postgres_policy_expression_contract.py @@ -0,0 +1,244 @@ +"""Adversarial semantic checks for PostgreSQL RLS policy expressions.""" +from __future__ import annotations + +import importlib + +import pytest + +from test_postgres_runtime_boundary import ( + COMMAND_POLICIES, + PERMIT_POLICIES, + TENANT_TABLES, + CatalogState, + _attach_driver, + _policy_manifest_comment, + _team_store, +) + + +def _migration(): + return importlib.import_module( + "memorymaster.stores.migrations.0011_postgres_scoped_force_rls" + ) + + +def _paired_rows( + state: CatalogState, + table: str, + command: str, +) -> list[dict[str, object]]: + names = {COMMAND_POLICIES[command], PERMIT_POLICIES[command]} + rows = [ + row + for row in state.policies + if row["tablename"] == table and row["policyname"] in names + ] + assert {str(row["policyname"]) for row in rows} == names + return rows + + +def _set_paired_expression( + state: CatalogState, + table: str, + command: str, + expression: str, +) -> None: + for row in _paired_rows(state, table, command): + row["qual"] = None if command == "INSERT" else expression + row["with_check"] = expression if command in {"INSERT", "UPDATE"} else None + + +def _canonical_v0011_state() -> CatalogState: + state = CatalogState() + migration = _migration() + for table in TENANT_TABLES: + _set_paired_expression(state, table, "SELECT", migration._READ_PREDICATES[table]) + for command in ("INSERT", "UPDATE", "DELETE"): + _set_paired_expression( + state, + table, + command, + migration._WRITE_PREDICATES[table], + ) + state.policy_manifest_comment = _policy_manifest_comment(state.policies) + return state + + +def _assert_connect_rejected(state: CatalogState) -> None: + store = _team_store() + connection, _ = _attach_driver(store, state) + + with pytest.raises(PermissionError, match="(?i)(policy|expression|predicate|rls)"): + store.connect() + + assert connection.rollback_count == 1 + assert connection.closed is True + + +def _drift_paired_expression( + state: CatalogState, + table: str, + command: str, + safe_fragment: str, + unsafe_fragment: str, +) -> None: + field = "with_check" if command == "INSERT" else "qual" + expression = str(_paired_rows(state, table, command)[0][field]) + assert safe_fragment in expression + _set_paired_expression( + state, + table, + command, + expression.replace(safe_fragment, unsafe_fragment), + ) + state.policy_manifest_comment = _policy_manifest_comment(state.policies) + + +def test_connect_accepts_canonical_v0011_policy_expression_catalog() -> None: + state = _canonical_v0011_state() + store = _team_store() + connection, _ = _attach_driver(store, state) + + assert store.connect() is connection + connection.close() + + +def test_connect_accepts_postgres_deparsed_equivalent_policy_catalog() -> None: + state = _canonical_v0011_state() + for row in state.policies: + for field in ("qual", "with_check"): + expression = row[field] + if expression is None or str(expression).upper() == "FALSE": + continue + deparsed = str(expression).replace( + "IN ('public', 'private')", + "= ANY (ARRAY['public'::text, 'private'::text])", + ) + deparsed = deparsed.replace("'[]'", "'[]'::text") + deparsed = deparsed.replace("'array'", "'array'::text") + row[field] = f"(({deparsed}))" + state.policy_manifest_comment = _policy_manifest_comment(state.policies) + store = _team_store() + connection, _ = _attach_driver(store, state) + + assert store.connect() is connection + connection.close() + + +def test_connect_rejects_restamped_paired_true_claims_select_policies() -> None: + state = _canonical_v0011_state() + _set_paired_expression(state, "claims", "SELECT", "TRUE") + state.policy_manifest_comment = _policy_manifest_comment(state.policies) + + _assert_connect_rejected(state) + + +def test_connect_preserves_literal_case_in_policy_fingerprint() -> None: + state = _canonical_v0011_state() + rows = _paired_rows(state, "claims", "SELECT") + for row in rows: + row["qual"] = str(row["qual"]).replace("'public'", "'PUBLIC'") + state.policy_manifest_comment = _policy_manifest_comment(state.policies) + + _assert_connect_rejected(state) + + +@pytest.mark.parametrize( + ("table", "command", "safe_fragment", "unsafe_fragment"), + [ + ( + "claims", + "SELECT", + "claims.tenant_id = NULLIF(current_setting('memorymaster.tenant_id', true), '')", + "claims.tenant_id IS NOT NULL", + ), + ( + "claims", + "SELECT", + "? claims.scope", + "? 'project:alpha'", + ), + ( + "claims", + "SELECT", + "claims.visibility = 'private' AND claims.source_agent = " + "NULLIF(current_setting('memorymaster.principal', true), '')", + "claims.visibility = 'private'", + ), + ( + "claims", + "UPDATE", + "claims.source_agent = " + "NULLIF(current_setting('memorymaster.principal', true), '')", + "claims.source_agent IS NOT NULL", + ), + ( + "claims", + "UPDATE", + "claims.visibility IN ('public', 'private')", + "claims.visibility IS NOT NULL", + ), + ( + "citations", + "SELECT", + "mm_claim.visibility = 'private' AND mm_claim.source_agent = " + "NULLIF(current_setting('memorymaster.principal', true), '')", + "mm_claim.visibility = 'private'", + ), + ( + "claim_embeddings", + "INSERT", + "mm_claim.source_agent = " + "NULLIF(current_setting('memorymaster.principal', true), '')", + "mm_claim.source_agent IS NOT NULL", + ), + ( + "claim_links", + "SELECT", + "? mm_target.scope", + "? 'project:alpha'", + ), + ( + "contradiction_verdicts", + "DELETE", + "mm_b.source_agent = " + "NULLIF(current_setting('memorymaster.principal', true), '')", + "mm_b.source_agent IS NOT NULL", + ), + ( + "events", + "SELECT", + "mm_claim.visibility = 'private' AND mm_claim.source_agent = " + "NULLIF(current_setting('memorymaster.principal', true), '')", + "mm_claim.visibility = 'private'", + ), + ], + ids=( + "claims-tenant", + "claims-scope", + "claims-private-owner", + "claims-write-owner", + "claims-write-visibility", + "citation-private-owner", + "embedding-write-owner", + "link-target-scope", + "verdict-target-owner", + "event-private-owner", + ), +) +def test_connect_rejects_restamped_paired_authority_predicate_drift( + table: str, + command: str, + safe_fragment: str, + unsafe_fragment: str, +) -> None: + state = _canonical_v0011_state() + _drift_paired_expression( + state, + table, + command, + safe_fragment, + unsafe_fragment, + ) + + _assert_connect_rejected(state) diff --git a/tests/test_postgres_policy_fingerprint.py b/tests/test_postgres_policy_fingerprint.py new file mode 100644 index 00000000..1b46cb99 --- /dev/null +++ b/tests/test_postgres_policy_fingerprint.py @@ -0,0 +1,310 @@ +"""Adversarial runtime checks for the immutable PostgreSQL RLS manifest.""" +from __future__ import annotations + +import copy + +import pytest + +from test_postgres_runtime_boundary import ( + COMMAND_POLICIES, + PERMIT_POLICIES, + PROTECTED_TABLES, + TEAM_DENY_TABLES, + TENANT_TABLES, + CatalogState, + _attach_driver, + _policy_manifest_comment, + _team_store, +) + + +def _restamp(state: CatalogState) -> None: + state.policy_manifest_comment = _policy_manifest_comment(state.policies) + + +def _assert_connect_rejected(state: CatalogState, match: str = "polic") -> None: + store = _team_store() + connection, _ = _attach_driver(store, state) + + with pytest.raises(PermissionError, match=f"(?i){match}"): + store.connect() + + assert connection.rollback_count == 1 + assert connection.closed is True + + +def _claims_policy(state: CatalogState, name: str) -> dict[str, object]: + return next( + row + for row in state.policies + if row["tablename"] == "claims" and row["policyname"] == name + ) + + +def test_safe_catalog_has_exact_paired_policy_inventory() -> None: + state = CatalogState() + expected = { + (table, policy) + for table in TENANT_TABLES + for policy in (*COMMAND_POLICIES.values(), *PERMIT_POLICIES.values()) + } + expected.update( + (table, "memorymaster_team_deny") + for table in set(PROTECTED_TABLES) - set(TENANT_TABLES) + ) + + actual = { + (str(row["tablename"]), str(row["policyname"])) + for row in state.policies + } + + assert actual == expected + assert len(actual) == 64 + assert not any( + str(row.get("qual", "")).strip("() ").upper() == "TRUE" + or str(row.get("with_check", "")).strip("() ").upper() == "TRUE" + for row in state.policies + ) + + +@pytest.mark.parametrize( + "policy", + [ + PERMIT_POLICIES["SELECT"], + PERMIT_POLICIES["INSERT"], + COMMAND_POLICIES["SELECT"], + COMMAND_POLICIES["UPDATE"], + ], +) +def test_connect_rejects_missing_policy_even_with_matching_recomputed_stamp( + policy: str, +) -> None: + state = CatalogState() + state.policies = [ + row + for row in state.policies + if not (row["tablename"] == "claims" and row["policyname"] == policy) + ] + _restamp(state) + + _assert_connect_rejected(state) + + +@pytest.mark.parametrize("restrictive", [False, True]) +def test_connect_rejects_extra_policy_even_with_matching_recomputed_stamp( + restrictive: bool, +) -> None: + state = CatalogState() + extra = copy.deepcopy(_claims_policy(state, COMMAND_POLICIES["SELECT"])) + extra["policyname"] = ( + "memorymaster_extra_restrict" if restrictive else "memorymaster_extra_permit" + ) + extra["permissive"] = "RESTRICTIVE" if restrictive else "PERMISSIVE" + extra["polpermissive"] = not restrictive + state.policies.append(extra) + _restamp(state) + + _assert_connect_rejected(state) + + +def test_connect_rejects_expression_drift_against_stale_migration_stamp() -> None: + state = CatalogState() + policy = _claims_policy(state, COMMAND_POLICIES["SELECT"]) + policy["qual"] = f"({policy['qual']}) OR TRUE" + + _assert_connect_rejected(state, "manifest|fingerprint|digest|polic") + + +@pytest.mark.parametrize( + "comment", + [ + None, + "", + "copied-from-another-cluster", + "memorymaster.rls/v1;manifest=0011;sha256=xyz", + "memorymaster.rls/v1;manifest=0011;sha256=" + ("0" * 64), + "memorymaster.rls/v1;manifest=0010;sha256=" + ("0" * 64), + ], +) +def test_connect_rejects_missing_malformed_or_forged_manifest_comment( + comment: str | None, +) -> None: + state = CatalogState() + state.policy_manifest_comment = comment + + _assert_connect_rejected(state, "manifest|fingerprint|digest|polic") + + +@pytest.mark.parametrize( + ("policy_name", "field", "unsafe_value"), + [ + (COMMAND_POLICIES["SELECT"], "qual", "TRUE OR tenant_id = tenant_id"), + (PERMIT_POLICIES["SELECT"], "roles", ["memorymaster_admin"]), + (COMMAND_POLICIES["UPDATE"], "cmd", "ALL"), + (PERMIT_POLICIES["INSERT"], "qual", "tenant_id = tenant_id"), + (COMMAND_POLICIES["INSERT"], "qual", "tenant_id = tenant_id"), + (PERMIT_POLICIES["SELECT"], "with_check", "tenant_id = tenant_id"), + (COMMAND_POLICIES["DELETE"], "with_check", "tenant_id = tenant_id"), + ], +) +def test_connect_rejects_self_consistent_but_noncanonical_policy_shape( + policy_name: str, + field: str, + unsafe_value: object, +) -> None: + state = CatalogState() + policy = _claims_policy(state, policy_name) + policy[field] = unsafe_value + _restamp(state) + + _assert_connect_rejected(state) + + +def test_connect_rejects_permit_and_restrict_predicate_mismatch_when_restamped() -> None: + state = CatalogState() + permit = _claims_policy(state, PERMIT_POLICIES["SELECT"]) + permit["qual"] = str(permit["qual"]).replace( + "claims.visibility = 'private'", + "claims.visibility <> 'private'", + ) + _restamp(state) + + _assert_connect_rejected(state) + + +def test_connect_rejects_schema_version_checksum_drift() -> None: + state = CatalogState(schema_v0011_checksum="0" * 64) + + _assert_connect_rejected(state, "migration|checksum|schema|version") + + +@pytest.mark.parametrize("table", TEAM_DENY_TABLES) +@pytest.mark.parametrize("privilege", ["insert", "update", "delete"]) +def test_connect_rejects_dml_grants_on_team_deny_tables( + table: str, + privilege: str, +) -> None: + state = CatalogState() + state.tables[table][f"can_{privilege}"] = True + state.tables[table][f"has_{privilege}"] = True + + _assert_connect_rejected(state, f"{table}|{privilege}|deny") + + +@pytest.mark.parametrize( + ("table", "privilege"), + [ + (table, privilege) + for table in ("cache_meta", "schema_versions") + for privilege in ( + "insert", + "update", + "delete", + "truncate", + "references", + "trigger", + ) + ], +) +def test_connect_rejects_metadata_table_mutation_privileges_before_binding( + table: str, + privilege: str, +) -> None: + state = CatalogState() + state.metadata_tables[table][f"can_{privilege}"] = True + state.metadata_tables[table][f"has_{privilege}"] = True + store = _team_store() + connection, _ = _attach_driver(store, state) + + with pytest.raises(PermissionError, match=f"(?i)({table}|{privilege})"): + store.connect() + + assert not any( + "set_config" in sql.lower() + for sql, _ in connection.cursor_instance.executed + ) + + +@pytest.mark.parametrize("table", ["cache_meta", "schema_versions"]) +def test_connect_requires_select_only_metadata_access(table: str) -> None: + state = CatalogState() + state.metadata_tables[table]["can_select"] = False + state.metadata_tables[table]["has_select"] = False + store = _team_store() + connection, _ = _attach_driver(store, state) + + with pytest.raises(PermissionError, match=f"(?i)({table}|select)"): + store.connect() + + assert not any( + "set_config" in sql.lower() + for sql, _ in connection.cursor_instance.executed + ) + + +@pytest.mark.parametrize( + ("case", "field"), + [ + ("missing", None), + ("not_unique", "indisunique"), + ("invalid", "indisvalid"), + ("not_ready", "indisready"), + ("wrong_columns", "indexdef"), + ("extra_column", "indexdef"), + ("wrong_column_order", "indexdef"), + ("wrong_tenant_expression", "indexdef"), + ("wrong_predicate", "predicate"), + ("tautological_predicate", "predicate"), + ], +) +def test_connect_requires_exact_tenant_confirmed_tuple_unique_index_before_binding( + case: str, + field: str | None, +) -> None: + state = CatalogState() + index = next( + row + for row in state.claim_identity_indexes + if row["index_name"] == "idx_claims_public_confirmed_tuple_unique" + ) + if case == "missing": + state.claim_identity_indexes.remove(index) + elif case == "wrong_columns": + index[field] = str(index[field]).replace("tenant_id", "source_agent") + elif case == "extra_column": + index[field] = str(index[field]).replace( + "predicate, scope)", "predicate, scope, source_agent)" + ) + elif case == "wrong_column_order": + index[field] = str(index[field]).replace( + "subject, predicate", "predicate, subject" + ) + elif case == "wrong_tenant_expression": + index[field] = str(index[field]).replace( + "COALESCE(tenant_id, ''::text)", "tenant_id" + ) + elif case == "wrong_predicate": + index[field] = "status = 'candidate'" + elif case == "tautological_predicate": + index[field] = f"({index[field]}) OR TRUE" + else: + index[field] = False + alias = { + "indisunique": "is_unique", + "indisvalid": "is_valid", + "indisready": "is_ready", + }[field] + index[alias] = False + store = _team_store() + connection, _ = _attach_driver(store, state) + + with pytest.raises( + PermissionError, + match="(?i)(confirmed|tuple|unique|index|tenant)", + ): + store.connect() + + assert not any( + "set_config" in sql.lower() + for sql, _ in connection.cursor_instance.executed + ) diff --git a/tests/test_postgres_principal_identity_runtime.py b/tests/test_postgres_principal_identity_runtime.py new file mode 100644 index 00000000..94235802 --- /dev/null +++ b/tests/test_postgres_principal_identity_runtime.py @@ -0,0 +1,484 @@ +"""PostgreSQL runtime RED contracts for v0012 claim identities.""" +from __future__ import annotations + +import copy +import inspect +import re +from types import SimpleNamespace +from typing import Sequence + +import pytest +from psycopg.errors import UniqueViolation + +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.stores._storage_write_claims import _WriteClaimsMixin +from memorymaster.stores.migrations import discover_migrations +from memorymaster.stores.postgres_store import PostgresStore + +from test_principal_local_identity_migration import ( + IDENTITY_INDEXES, + LEGACY_UNIQUE_INDEXES, +) +HUMAN_ID_INDEXES = { + "idx_claims_public_human_id_unique", + "idx_claims_nonpublic_principal_human_id_unique", +} + + +def _canonical(sql: str) -> str: + return " ".join(sql.lower().replace('"', "").split()) + + +def _index_row(name: str) -> dict[str, object]: + public = "_public_" in name + if "idempotency_key" in name: + identity_columns = "scope, idempotency_key" + nonnull = "idempotency_key IS NOT NULL" + elif "human_id" in name: + identity_columns = "scope, human_id" + nonnull = "human_id IS NOT NULL" + else: + identity_columns = "subject, predicate, scope" + nonnull = ( + "status = 'confirmed'::text AND subject IS NOT NULL " + "AND predicate IS NOT NULL" + ) + if public: + columns = f"COALESCE(tenant_id, ''::text), {identity_columns}" + predicate = f"visibility = 'public'::text AND {nonnull}" + else: + if "scope, " in identity_columns and "confirmed" not in name: + columns = ( + "COALESCE(tenant_id, ''::text), scope, visibility, source_agent, " + f"{identity_columns.removeprefix('scope, ')}" + ) + else: + columns = ( + "COALESCE(tenant_id, ''::text), visibility, source_agent, " + f"{identity_columns}" + ) + predicate = ( + "visibility <> 'public'::text AND source_agent IS NOT NULL " + f"AND {nonnull}" + ) + indexdef = ( + f"CREATE UNIQUE INDEX {name} ON public.claims USING btree ({columns}) " + f"WHERE ({predicate})" + ) + return { + "index_name": name, + "relname": name, + "indisunique": True, + "is_unique": True, + "indisvalid": True, + "is_valid": True, + "indisready": True, + "is_ready": True, + "indexdef": indexdef, + "index_definition": indexdef, + "predicate": predicate, + "index_predicate": predicate, + } + + +class IndexCatalogCursor: + def __init__(self, rows: list[dict[str, object]]) -> None: + self.rows = rows + self.executed: list[tuple[str, tuple[object, ...]]] = [] + + def execute(self, sql: str, params: Sequence[object] = ()) -> None: + self.executed.append((sql, tuple(params))) + + def fetchall(self) -> list[dict[str, object]]: + return list(self.rows) + + def fetchone(self) -> dict[str, object] | None: + return self.rows[0] if self.rows else None + + +def _safe_index_rows() -> list[dict[str, object]]: + return [_index_row(name) for name in sorted(IDENTITY_INDEXES)] + + +def test_runtime_accepts_only_exact_six_principal_identity_indexes() -> None: + cursor = IndexCatalogCursor(_safe_index_rows()) + + PostgresStore._validate_claim_identity_indexes(cursor) + + query = _canonical(cursor.executed[0][0]) + assert "pg_index" in query + assert "indisunique" in query + assert "indisvalid" in query + assert "indisready" in query + assert "pg_get_indexdef" in query + assert "pg_get_expr" in query + + +@pytest.mark.parametrize( + "mutation", + [ + "missing", + "extra", + "legacy", + "not_unique", + "not_valid", + "not_ready", + "wrong_columns", + "wrong_predicate", + ], +) +def test_runtime_rejects_identity_index_catalog_drift(mutation: str) -> None: + rows = _safe_index_rows() + if mutation == "missing": + rows.pop() + elif mutation == "extra": + extra = copy.deepcopy(rows[0]) + extra["index_name"] = extra["relname"] = "idx_claims_extra_unique" + rows.append(extra) + elif mutation == "legacy": + legacy = copy.deepcopy(rows[0]) + legacy["index_name"] = legacy["relname"] = next(iter(LEGACY_UNIQUE_INDEXES)) + rows.append(legacy) + elif mutation == "wrong_columns": + rows[0]["indexdef"] = rows[0]["index_definition"] = str( + rows[0]["indexdef"] + ).replace("source_agent", "holder") + elif mutation == "wrong_predicate": + rows[0]["predicate"] = rows[0]["index_predicate"] = "TRUE" + else: + field, alias = { + "not_unique": ("indisunique", "is_unique"), + "not_valid": ("indisvalid", "is_valid"), + "not_ready": ("indisready", "is_ready"), + }[mutation] + rows[0][field] = False + rows[0][alias] = False + + with pytest.raises(PermissionError, match="(?i)(identity|index|catalog|unique)"): + PostgresStore._validate_claim_identity_indexes(IndexCatalogCursor(rows)) + + +def test_runtime_preserves_literal_case_in_identity_index_fingerprint() -> None: + rows = _safe_index_rows() + public_row = next(row for row in rows if "_public_" in str(row["index_name"])) + for field in ("indexdef", "index_definition", "predicate", "index_predicate"): + public_row[field] = str(public_row[field]).replace("'public'", "'PUBLIC'") + + with pytest.raises(PermissionError, match="(?i)(identity|index|catalog|unique)"): + PostgresStore._validate_claim_identity_indexes(IndexCatalogCursor(rows)) + + +class RuntimeMigrationCursor: + def __init__(self, checksums: dict[int, str]) -> None: + self.checksums = checksums + self.requested_versions: set[int] = set() + self.rows: list[dict[str, object]] = [] + + def execute(self, sql: str, params: Sequence[object] = ()) -> None: + versions = {int(value) for value in params if isinstance(value, int)} + if not versions: + versions = { + int(value) + for value in re.findall(r"\b(?:11|12)\b", sql) + } + self.requested_versions.update(versions) + self.rows = [ + {"version": version, "checksum": self.checksums[version]} + for version in sorted(versions) + if version in self.checksums + ] + + def fetchone(self) -> dict[str, object] | None: + return self.rows[0] if self.rows else None + + def fetchall(self) -> list[dict[str, object]]: + return list(self.rows) + + +def _migration_checksums() -> dict[int, str]: + migrations = {item.version: item for item in discover_migrations()} + return {version: migrations[version].checksum() for version in (11, 12)} + + +def test_runtime_requires_v0012_source_checksum() -> None: + checksums = _migration_checksums() + checksums[12] = "0" * 64 + cursor = RuntimeMigrationCursor(checksums) + + with pytest.raises(PermissionError, match="(?i)(migration|checksum|version)"): + PostgresStore._validate_runtime_migration(cursor) + + assert 12 in cursor.requested_versions + + +def test_runtime_accepts_matching_v0011_and_v0012_checksums() -> None: + cursor = RuntimeMigrationCursor(_migration_checksums()) + + PostgresStore._validate_runtime_migration(cursor) + + assert cursor.requested_versions == {11, 12} + + +class BareCursor: + def __enter__(self) -> BareCursor: + return self + + def __exit__(self, *_args: object) -> None: + return None + + +class BareConnection: + def __init__(self) -> None: + self.autocommit = True + self.closed = False + + def cursor(self) -> BareCursor: + return BareCursor() + + def close(self) -> None: + self.closed = True + + +def test_team_connect_validates_v12_identity_catalog_before_binding( + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = PostgresStore( + "postgresql://runtime.invalid/memorymaster", + tenant_id="tenant-a", + require_tenant=True, + principal="alice", + allowed_scopes={"project:a"}, + ) + connection = BareConnection() + calls: list[str] = [] + monkeypatch.setattr(store, "_open_connection", lambda: connection) + for method in ( + "_validate_runtime_role", + "_validate_runtime_tables", + "_validate_runtime_metadata_tables", + "_validate_claim_owner_constraint", + "_validate_confirmed_tuple_index", + "_validate_claim_supersession_guard", + "_validate_event_append_only_catalog", + "_validate_event_chain_head_function", + "_validate_runtime_migration", + "_validate_runtime_policies", + ): + monkeypatch.setattr( + PostgresStore, + method, + classmethod(lambda _cls, _cur: None), + ) + monkeypatch.setattr( + PostgresStore, + "_validate_claim_identity_indexes", + classmethod(lambda _cls, _cur: calls.append("identity")), + raising=False, + ) + monkeypatch.setattr( + PostgresStore, + "_bind_runtime_authority", + classmethod( + lambda _cls, _cur, _tenant, _principal, _scopes: calls.append("bind") + ), + ) + + assert store.connect() is connection + assert calls == ["identity", "bind"] + + +def test_postgres_create_rejects_sensitive_visibility_before_io( + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = PostgresStore( + "postgresql://runtime.invalid/memorymaster", + tenant_id="tenant-a", + require_tenant=True, + principal="alice", + allowed_scopes={"project:a"}, + ) + monkeypatch.setattr( + store, + "connect", + lambda: (_ for _ in ()).throw( + AssertionError("database opened before sensitive visibility denial") + ), + ) + + with pytest.raises((PermissionError, ValueError), match="(?i)(sensitive|visibility)"): + store.create_claim( + "Team-sensitive payload.", + [CitationInput(source="identity-red", locator="fixture")], + idempotency_key="team-sensitive-denied", + scope="project:a", + tenant_id="tenant-a", + source_agent="alice", + visibility="sensitive", + ) + + +class NamedUniqueViolation(UniqueViolation): + def __init__(self, constraint_name: str) -> None: + super().__init__(f"duplicate key in {constraint_name}") + self._constraint_name = constraint_name + + @property + def diag(self): + return SimpleNamespace(constraint_name=self._constraint_name) + + +class HumanIdCursor: + def __init__(self, failures: list[Exception]) -> None: + self.failures = list(failures) + self.attempted: list[str] = [] + self.executed: list[str] = [] + self.aborted = False + self.row: dict[str, object] | None = None + + def __enter__(self) -> HumanIdCursor: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, sql: str, _params: Sequence[object] = ()) -> None: + normalized = _canonical(sql) + self.attempted.append(normalized) + if self.aborted and not normalized.startswith("rollback to savepoint"): + raise AssertionError("statement attempted while transaction was aborted") + if normalized.startswith("rollback to savepoint"): + self.aborted = False + self.executed.append(normalized) + if normalized.startswith("insert into claims"): + self.row = {"id": 41} + elif normalized.startswith("update claims set human_id") and self.failures: + failure = self.failures.pop(0) + if isinstance(failure, UniqueViolation): + self.aborted = True + self.row = None + raise failure + else: + self.row = None + + def fetchone(self) -> dict[str, object] | None: + return self.row + + +class HumanIdConnection: + def __init__(self, cursor: HumanIdCursor) -> None: + self.cursor_instance = cursor + + def __enter__(self) -> HumanIdConnection: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def cursor(self) -> HumanIdCursor: + return self.cursor_instance + + +def _claim_store( + monkeypatch: pytest.MonkeyPatch, + failures: list[Exception], +) -> tuple[PostgresStore, HumanIdCursor, list[str]]: + store = PostgresStore( + "postgresql://runtime.invalid/memorymaster", + tenant_id="tenant-a", + require_tenant=True, + principal="alice", + allowed_scopes={"project:a"}, + ) + cursor = HumanIdCursor(failures) + connection = HumanIdConnection(cursor) + candidates = iter(("mm-collision", "mm-collision~2", "mm-collision~3")) + events: list[str] = [] + monkeypatch.setattr(store, "connect", lambda: connection) + monkeypatch.setattr( + store, + "_allocate_human_id", + lambda *_args, **_kwargs: next(candidates), + ) + monkeypatch.setattr( + store, + "_insert_event_row", + lambda *_args, **_kwargs: events.append("event") or 1, + ) + monkeypatch.setattr( + store, + "get_claim", + lambda *_args, **_kwargs: SimpleNamespace(id=41, visibility="public"), + ) + return store, cursor, events + + +def _create_claim(store: PostgresStore): + return store.create_claim( + "Human ID collision payload.", + [CitationInput(source="identity-red", locator="fixture")], + idempotency_key="human-id-transaction-key", + subject="collision", + scope="project:a", + tenant_id="tenant-a", + source_agent="alice", + visibility="private", + ) + + +@pytest.mark.parametrize("index_name", sorted(HUMAN_ID_INDEXES)) +def test_human_id_collision_uses_savepoint_then_retries_named_index_only( + monkeypatch: pytest.MonkeyPatch, + index_name: str, +) -> None: + store, cursor, events = _claim_store( + monkeypatch, + [NamedUniqueViolation(index_name)], + ) + + claim = _create_claim(store) + + assert claim.id == 41 + assert any(sql.startswith("savepoint ") for sql in cursor.executed) + assert any(sql.startswith("rollback to savepoint ") for sql in cursor.executed) + assert any(sql.startswith("release savepoint ") for sql in cursor.executed) + assert sum(sql.startswith("update claims set human_id") for sql in cursor.executed) == 2 + assert any(sql.startswith("insert into citations") for sql in cursor.executed) + assert events == ["event"] + + +def test_foreign_unique_violation_is_not_swallowed_or_followed_by_writes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + failure = NamedUniqueViolation("idx_claims_public_idempotency_key_unique") + store, cursor, events = _claim_store(monkeypatch, [failure]) + + with pytest.raises(UniqueViolation) as raised: + _create_claim(store) + + assert raised.value is failure + assert not any(sql.startswith("insert into citations") for sql in cursor.attempted) + assert events == [] + + +def test_non_unique_human_id_failure_is_never_swallowed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + failure = RuntimeError("human ID allocator unavailable") + store, cursor, events = _claim_store(monkeypatch, [failure]) + + with pytest.raises(RuntimeError, match="allocator unavailable"): + _create_claim(store) + + assert not any(sql.startswith("insert into citations") for sql in cursor.attempted) + assert events == [] + + +def test_create_claim_default_visibility_is_consistent_across_backends() -> None: + defaults = { + inspect.signature(owner.create_claim).parameters["visibility"].default + for owner in (_WriteClaimsMixin, PostgresStore) + } + defaults.add(inspect.signature(MemoryService.ingest).parameters["visibility"].default) + + assert defaults == {"public"} diff --git a/tests/test_postgres_rls_integration.py b/tests/test_postgres_rls_integration.py new file mode 100644 index 00000000..bcca276d --- /dev/null +++ b/tests/test_postgres_rls_integration.py @@ -0,0 +1,918 @@ +"""Opt-in integration proof for the PostgreSQL team-runtime RLS boundary. + +This module never falls back to ``DATABASE_URL`` or another application DSN. +It runs only when both purpose-specific DSNs are present and the operator also +sets ``MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE=1``. The admin DSN owns schema +initialization/migrations; the app DSN must be a distinct, non-owner role with +ordinary DML/sequence privileges but no BYPASSRLS, schema CREATE, TRUNCATE, or +REFERENCES/TRIGGER privilege. The migrator must be SUPERUSER or BYPASSRLS; +the application role receives read-only access to governance metadata. + +Events are append-only, so the fixture uses UUID-namespaced tenants, scopes, +claims, and idempotency keys rather than destructive global cleanup. Run this +only against a database whose complete lifecycle is disposable. +""" +from __future__ import annotations + +import os +import secrets +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Callable +from uuid import uuid4 + +import pytest + +from memorymaster.core.models import CitationInput, Claim, ClaimLink +from memorymaster.stores.migrations import discover_migrations +from memorymaster.stores.postgres_store import ( + POSTGRES_COMMAND_POLICIES, + POSTGRES_PERMIT_POLICIES, + POSTGRES_PROTECTED_TABLES, + POSTGRES_TEAM_DENY_TABLES, + POSTGRES_TENANT_POLICY_TABLES, + PostgresStore, +) + + +ADMIN_DSN_ENV = "MEMORYMASTER_TEST_POSTGRES_DSN" +APP_DSN_ENV = "MEMORYMASTER_TEST_POSTGRES_APP_DSN" +DISPOSABLE_OPT_IN_ENV = "MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE" +_LIVE_DSN_ENVS = ("DATABASE_URL", "POSTGRES_DSN", "MEMORYMASTER_POSTGRES_DSN") +_BLOCKED_REASON = ( + "BLOCKED-EXTERNAL: real PostgreSQL RLS verification requires both " + f"{ADMIN_DSN_ENV} and {APP_DSN_ENV}, plus {DISPOSABLE_OPT_IN_ENV}=1" +) + +pytestmark = pytest.mark.postgres + + +@dataclass(frozen=True) +class PgConfig: + admin_dsn: str = field(repr=False) + app_dsn: str = field(repr=False) + run_id: str + tenant_a: str + tenant_b: str + scope_a: str + scope_b: str + alice: str + bob: str + + +@dataclass(frozen=True) +class PreparedDatabase: + config: PgConfig + admin_role: str + app_role: str + applied_versions: frozenset[int] + + +@dataclass(frozen=True) +class DatabaseIdentity: + database: str + role: str + superuser: bool + bypass_rls: bool + replication: bool + create_role: bool + create_db: bool + + +@dataclass(frozen=True) +class RuntimeStores: + alice_a_scope_a: PostgresStore + bob_a_scope_a: PostgresStore + alice_a_scope_b: PostgresStore + alice_b_scope_a: PostgresStore + + +@dataclass(frozen=True) +class SeedRows: + public_a: Claim + private_a: Claim + private_a_target: Claim + scope_b: Claim + tenant_b: Claim + private_link: ClaimLink + + +def _same_secret(left: str, right: str) -> bool: + return bool(left and right) and secrets.compare_digest(left, right) + + +@pytest.fixture(scope="module") +def pg_config() -> PgConfig: + admin_dsn = os.getenv(ADMIN_DSN_ENV, "").strip() + app_dsn = os.getenv(APP_DSN_ENV, "").strip() + opted_in = os.getenv(DISPOSABLE_OPT_IN_ENV, "").strip() == "1" + if not admin_dsn or not app_dsn or not opted_in: + pytest.skip(_BLOCKED_REASON) + if _same_secret(admin_dsn, app_dsn): + pytest.fail("PostgreSQL RLS integration requires distinct admin and app DSNs.") + for env_name in _LIVE_DSN_ENVS: + live_dsn = os.getenv(env_name, "").strip() + if _same_secret(admin_dsn, live_dsn) or _same_secret(app_dsn, live_dsn): + pytest.fail(f"Refusing to reuse {env_name} for disposable RLS tests.") + + run_id = uuid4().hex + return PgConfig( + admin_dsn=admin_dsn, + app_dsn=app_dsn, + run_id=run_id, + tenant_a=f"rls-{run_id}-tenant-a", + tenant_b=f"rls-{run_id}-tenant-b", + scope_a=f"project:rls-{run_id}-a", + scope_b=f"project:rls-{run_id}-b", + alice=f"rls-{run_id}-alice", + bob=f"rls-{run_id}-bob", + ) + + +def _database_identity(psycopg: Any, dsn: str) -> DatabaseIdentity: + with psycopg.connect(dsn, connect_timeout=5) as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT current_database(), current_user, rolsuper, rolbypassrls, + rolreplication, rolcreaterole, rolcreatedb + FROM pg_roles WHERE rolname = current_user + """ + ) + row = cur.fetchone() + if row is None: + raise RuntimeError("PostgreSQL connection identity could not be verified.") + return DatabaseIdentity( + str(row[0]), + str(row[1]), + bool(row[2]), + bool(row[3]), + bool(row[4]), + bool(row[5]), + bool(row[6]), + ) + + +@pytest.fixture(scope="module") +def prepared_database(pg_config: PgConfig) -> PreparedDatabase: + try: + import psycopg + except ImportError: + pytest.skip("BLOCKED-EXTERNAL: psycopg is required for PostgreSQL RLS tests") + + try: + admin_identity = _database_identity(psycopg, pg_config.admin_dsn) + app_identity = _database_identity(psycopg, pg_config.app_dsn) + except psycopg.OperationalError: + pytest.skip("BLOCKED-EXTERNAL: configured PostgreSQL test DSNs are unreachable") + if admin_identity.database != app_identity.database: + pytest.fail("Admin and app DSNs must target the same disposable database.") + if admin_identity.role == app_identity.role: + pytest.fail("Admin and app DSNs must authenticate as distinct roles.") + if not admin_identity.superuser and not admin_identity.bypass_rls: + pytest.fail("The PostgreSQL migrator must be SUPERUSER or BYPASSRLS.") + if any( + ( + app_identity.superuser, + app_identity.bypass_rls, + app_identity.replication, + app_identity.create_role, + app_identity.create_db, + ) + ): + pytest.fail("The PostgreSQL app role has a forbidden role attribute.") + + admin_store = PostgresStore(pg_config.admin_dsn) + admin_store.init_db() + from psycopg import sql + + with psycopg.connect(pg_config.admin_dsn) as conn, conn.cursor() as cur: + cur.execute( + sql.SQL("GRANT SELECT, INSERT ON TABLE public.events TO {}").format( + sql.Identifier(app_identity.role) + ) + ) + cur.execute( + sql.SQL("REVOKE UPDATE, DELETE ON TABLE public.events FROM {}").format( + sql.Identifier(app_identity.role) + ) + ) + cur.execute( + sql.SQL( + "GRANT EXECUTE ON FUNCTION " + "public.memorymaster_event_chain_head() TO {}" + ).format(sql.Identifier(app_identity.role)) + ) + with psycopg.connect(pg_config.admin_dsn) as conn, conn.cursor() as cur: + cur.execute("SELECT version FROM schema_versions ORDER BY version") + applied = frozenset(int(row[0]) for row in cur.fetchall()) + expected = {migration.version for migration in discover_migrations()} + if not expected.issubset(applied): + pytest.fail("Admin initialization did not apply every discovered migration.") + return PreparedDatabase(pg_config, admin_identity.role, app_identity.role, applied) + + +def _runtime_store( + config: PgConfig, + *, + tenant_id: str, + principal: str, + scope: str, +) -> PostgresStore: + return PostgresStore( + config.app_dsn, + tenant_id=tenant_id, + require_tenant=True, + principal=principal, + allowed_scopes=(scope,), + ) + + +@pytest.fixture(scope="module") +def runtime_stores(prepared_database: PreparedDatabase) -> RuntimeStores: + config = prepared_database.config + return RuntimeStores( + alice_a_scope_a=_runtime_store( + config, + tenant_id=config.tenant_a, + principal=config.alice, + scope=config.scope_a, + ), + bob_a_scope_a=_runtime_store( + config, + tenant_id=config.tenant_a, + principal=config.bob, + scope=config.scope_a, + ), + alice_a_scope_b=_runtime_store( + config, + tenant_id=config.tenant_a, + principal=config.alice, + scope=config.scope_b, + ), + alice_b_scope_a=_runtime_store( + config, + tenant_id=config.tenant_b, + principal=config.alice, + scope=config.scope_a, + ), + ) + + +def _create_claim( + store: PostgresStore, + config: PgConfig, + *, + label: str, + scope: str, + source_agent: str, + visibility: str, +) -> Claim: + return store.create_claim( + f"{config.run_id}:{label}", + [ + CitationInput( + source="postgres-rls-integration", + locator=config.run_id, + excerpt=label, + ) + ], + idempotency_key=f"{config.run_id}:{label}", + scope=scope, + tenant_id=store.tenant_id, + source_agent=source_agent, + visibility=visibility, + ) + + +@pytest.fixture(scope="module") +def seed_rows( + prepared_database: PreparedDatabase, + runtime_stores: RuntimeStores, +) -> SeedRows: + config = prepared_database.config + alice_a = runtime_stores.alice_a_scope_a + public_a = _create_claim( + alice_a, + config, + label="public-a", + scope=config.scope_a, + source_agent=config.alice, + visibility="public", + ) + private_a = _create_claim( + alice_a, + config, + label="private-a", + scope=config.scope_a, + source_agent=config.alice, + visibility="private", + ) + private_target = _create_claim( + alice_a, + config, + label="private-a-target", + scope=config.scope_a, + source_agent=config.alice, + visibility="private", + ) + scope_b = _create_claim( + runtime_stores.alice_a_scope_b, + config, + label="scope-b", + scope=config.scope_b, + source_agent=config.alice, + visibility="public", + ) + tenant_b = _create_claim( + runtime_stores.alice_b_scope_a, + config, + label="tenant-b", + scope=config.scope_a, + source_agent=config.alice, + visibility="public", + ) + link = alice_a.add_claim_link(private_a.id, private_target.id, "derived_from") + return SeedRows(public_a, private_a, private_target, scope_b, tenant_b, link) + + +def _visible_ids(store: PostgresStore) -> set[int]: + return {claim.id for claim in store.list_claims(limit=100)} + + +def _assert_rls_denied(action: Callable[[], object]) -> None: + with pytest.raises(Exception) as caught: # noqa: B017 - driver class is optional + action() + assert getattr(caught.value, "sqlstate", None) == "42501", ( + "expected PostgreSQL insufficient_privilege (42501), got " + f"{type(caught.value).__name__}" + ) + + +def _read_runtime_catalog( + conn: Any, +) -> tuple[dict[str, Any], dict[str, dict[str, Any]], set[tuple[str, str]]]: + with conn.cursor() as cur: + cur.execute( + """ + SELECT current_user, + current_setting('memorymaster.tenant_id', true) AS tenant_id, + current_setting('memorymaster.principal', true) AS principal, + current_setting('memorymaster.allowed_scopes', true) AS scopes, + has_schema_privilege(current_user, current_schema(), 'CREATE') AS can_create + """ + ) + authority = cur.fetchone() + cur.execute( + """ + SELECT c.relname, c.relrowsecurity, c.relforcerowsecurity, + pg_get_userbyid(c.relowner) AS owner_name, + pg_has_role(current_user, c.relowner, 'MEMBER') AS owner_member, + has_table_privilege(current_user, c.oid, 'TRUNCATE') AS can_truncate, + has_table_privilege(current_user, c.oid, 'REFERENCES') AS can_references, + has_table_privilege(current_user, c.oid, 'TRIGGER') AS can_trigger, + has_table_privilege(current_user, c.oid, 'SELECT') AS can_select, + has_table_privilege(current_user, c.oid, 'INSERT') AS can_insert, + has_table_privilege(current_user, c.oid, 'UPDATE') AS can_update, + has_any_column_privilege(current_user, c.oid, 'UPDATE') + AS can_update_any_column, + has_table_privilege(current_user, c.oid, 'DELETE') AS can_delete + FROM pg_class AS c + JOIN pg_namespace AS n ON n.oid = c.relnamespace + WHERE n.nspname = current_schema() AND c.relname = ANY(%s) + """, + (list(POSTGRES_PROTECTED_TABLES),), + ) + tables = {row["relname"]: row for row in cur.fetchall()} + cur.execute( + """ + SELECT tablename, policyname, permissive, roles, cmd + FROM pg_policies + WHERE schemaname = current_schema() AND tablename = ANY(%s) + """, + (list(POSTGRES_PROTECTED_TABLES),), + ) + policies = {(row["tablename"], row["policyname"]) for row in cur.fetchall()} + if authority is None: + raise RuntimeError("PostgreSQL runtime authority could not be read.") + return authority, tables, policies + + +def _assert_runtime_catalog( + prepared: PreparedDatabase, + authority: dict[str, Any], + tables: dict[str, dict[str, Any]], + policies: set[tuple[str, str]], +) -> None: + config = prepared.config + assert authority["current_user"] == prepared.app_role + assert authority["tenant_id"] == config.tenant_a + assert authority["principal"] == config.alice + assert config.scope_a in authority["scopes"] + assert authority["can_create"] is False + assert set(tables) == set(POSTGRES_PROTECTED_TABLES) + assert all(row["relrowsecurity"] and row["relforcerowsecurity"] for row in tables.values()) + assert all(not row["owner_member"] for row in tables.values()) + assert all( + not row["can_truncate"] + and not row["can_references"] + and not row["can_trigger"] + for row in tables.values() + ) + assert all(row["owner_name"] != prepared.app_role for row in tables.values()) + events = tables["events"] + assert events["can_select"] + assert events["can_insert"] + assert not events["can_update"] + assert not events["can_update_any_column"] + assert not events["can_delete"] + for table in POSTGRES_TENANT_POLICY_TABLES: + for command, policy in POSTGRES_COMMAND_POLICIES.items(): + assert (table, policy) in policies + assert (table, POSTGRES_PERMIT_POLICIES[command]) in policies + for table in POSTGRES_TEAM_DENY_TABLES: + assert (table, "memorymaster_team_deny") in policies + assert not tables[table]["can_insert"] + assert not tables[table]["can_update"] + assert not tables[table]["can_delete"] + + +def _assert_authority_clears_after_commit(conn: Any) -> None: + conn.commit() + with conn.cursor() as cur: + cur.execute( + """ + SELECT current_setting('memorymaster.tenant_id', true) AS tenant_id, + current_setting('memorymaster.principal', true) AS principal, + current_setting('memorymaster.allowed_scopes', true) AS scopes + """ + ) + cleared = cur.fetchone() + cur.execute("SELECT COUNT(*) AS count FROM claims") + unbound_count = int(cur.fetchone()["count"]) + assert all(cleared[name] in (None, "") for name in ("tenant_id", "principal", "scopes")) + assert unbound_count == 0 + + +def test_admin_applies_migrations_and_team_runtime_cannot_init( + prepared_database: PreparedDatabase, + runtime_stores: RuntimeStores, +) -> None: + expected = {migration.version for migration in discover_migrations()} + assert expected <= prepared_database.applied_versions + assert prepared_database.admin_role != prepared_database.app_role + with pytest.raises(PermissionError, match="cannot initialize or migrate"): + runtime_stores.alice_a_scope_a.init_db() + + +def test_runtime_catalog_and_transaction_local_authority( + prepared_database: PreparedDatabase, + runtime_stores: RuntimeStores, +) -> None: + conn = runtime_stores.alice_a_scope_a.connect() + try: + authority, tables, policies = _read_runtime_catalog(conn) + _assert_runtime_catalog(prepared_database, authority, tables, policies) + _assert_authority_clears_after_commit(conn) + finally: + conn.close() + + +def test_claim_visibility_isolated_by_tenant_scope_and_private_principal( + runtime_stores: RuntimeStores, + seed_rows: SeedRows, +) -> None: + assert _visible_ids(runtime_stores.alice_a_scope_a) == { + seed_rows.public_a.id, + seed_rows.private_a.id, + seed_rows.private_a_target.id, + } + assert _visible_ids(runtime_stores.bob_a_scope_a) == {seed_rows.public_a.id} + assert _visible_ids(runtime_stores.alice_a_scope_b) == {seed_rows.scope_b.id} + assert _visible_ids(runtime_stores.alice_b_scope_a) == {seed_rows.tenant_b.id} + + +def test_direct_ids_citations_and_events_do_not_bypass_rls( + runtime_stores: RuntimeStores, + seed_rows: SeedRows, +) -> None: + owner = runtime_stores.alice_a_scope_a + bob = runtime_stores.bob_a_scope_a + tenant_b = runtime_stores.alice_b_scope_a + assert owner.get_claim(seed_rows.private_a.id) is not None + assert len(owner.list_citations(seed_rows.private_a.id)) == 1 + assert owner.list_events(claim_id=seed_rows.private_a.id) + assert bob.get_claim(seed_rows.public_a.id) is not None + assert len(bob.list_citations(seed_rows.public_a.id)) == 1 + assert bob.list_events(claim_id=seed_rows.public_a.id) + + for outsider in (bob, tenant_b): + assert outsider.get_claim(seed_rows.private_a.id) is None + assert outsider.list_citations(seed_rows.private_a.id) == [] + assert outsider.list_events(claim_id=seed_rows.private_a.id) == [] + assert seed_rows.private_a.human_id is not None + assert bob.get_claim_by_human_id(seed_rows.private_a.human_id) is None + with pytest.raises(ValueError, match="No claim found"): + bob.resolve_claim_id(seed_rows.private_a.human_id) + assert bob.resolve_claim_id(seed_rows.private_a.id) == seed_rows.private_a.id + assert bob.get_claim(bob.resolve_claim_id(seed_rows.private_a.id)) is None + + +def test_claim_links_require_visibility_of_both_endpoints( + runtime_stores: RuntimeStores, + seed_rows: SeedRows, +) -> None: + owner_links = runtime_stores.alice_a_scope_a.get_claim_links(seed_rows.private_a.id) + assert [link.id for link in owner_links] == [seed_rows.private_link.id] + assert runtime_stores.bob_a_scope_a.get_claim_links(seed_rows.private_a.id) == [] + assert runtime_stores.alice_b_scope_a.get_claim_links(seed_rows.private_a.id) == [] + + +def _assert_claim_create_denials(config: PgConfig, alice: PostgresStore) -> None: + _assert_rls_denied( + lambda: _create_claim( + alice, + config, + label="forbidden-scope", + scope=config.scope_b, + source_agent=config.alice, + visibility="public", + ) + ) + _assert_rls_denied( + lambda: _create_claim( + alice, + config, + label="forbidden-private-owner", + scope=config.scope_a, + source_agent=config.bob, + visibility="private", + ) + ) + _assert_rls_denied( + lambda: _create_claim( + alice, + config, + label="forbidden-public-owner", + scope=config.scope_a, + source_agent=config.bob, + visibility="public", + ) + ) + with pytest.raises(PermissionError, match="does not match"): + alice.create_claim( + f"{config.run_id}:forbidden-tenant", + [CitationInput(source="postgres-rls-integration")], + scope=config.scope_a, + tenant_id=config.tenant_b, + source_agent=config.alice, + visibility="public", + ) + + +def _insert_cross_tenant_citation( + alice: PostgresStore, + config: PgConfig, + claim_id: int, +) -> None: + with alice.connect() as conn, conn.cursor() as cur: + cur.execute( + """ + INSERT INTO citations (claim_id, source, locator, excerpt, created_at) + VALUES (%s, %s, %s, %s, %s) + """, + ( + claim_id, + "postgres-rls-integration", + config.run_id, + "cross-tenant-write", + datetime.now(timezone.utc), + ), + ) + + +def _assert_hidden_row_write_denials( + config: PgConfig, + stores: RuntimeStores, + rows: SeedRows, +) -> None: + alice = stores.alice_a_scope_a + bob = stores.bob_a_scope_a + private_before = alice.get_claim(rows.private_a.id) + assert private_before is not None + original_confidence = private_before.confidence + with pytest.raises(ValueError, match="does not exist"): + bob.set_confidence(rows.private_a.id, 0.99, details="direct-id-write") + private_after = alice.get_claim(rows.private_a.id) + assert private_after is not None + assert private_after.confidence == original_confidence + public_before = alice.get_claim(rows.public_a.id) + assert public_before is not None + with pytest.raises(ValueError, match="does not exist"): + bob.set_confidence(rows.public_a.id, 0.99, details="public-cross-owner-write") + public_after = alice.get_claim(rows.public_a.id) + assert public_after is not None + assert public_after.confidence == public_before.confidence + with pytest.raises(ValueError, match="does not exist"): + alice.record_event( + claim_id=rows.tenant_b.id, + event_type="audit", + details="cross-tenant-event", + ) + _assert_rls_denied( + lambda: alice.add_claim_link( + rows.private_a.id, + rows.tenant_b.id, + "relates_to", + ) + ) + _assert_rls_denied( + lambda: _insert_cross_tenant_citation(alice, config, rows.tenant_b.id) + ) + assert len(stores.alice_b_scope_a.list_citations(rows.tenant_b.id)) == 1 + + +def test_forbidden_writes_are_rejected_or_have_no_effect( + prepared_database: PreparedDatabase, + runtime_stores: RuntimeStores, + seed_rows: SeedRows, +) -> None: + config = prepared_database.config + _assert_claim_create_denials(config, runtime_stores.alice_a_scope_a) + _assert_hidden_row_write_denials(config, runtime_stores, seed_rows) + + +def _execute_event_mutation( + store: PostgresStore, + statement: str, + claim_id: int, +) -> None: + with store.connect() as conn, conn.cursor() as cur: + cur.execute(statement, (claim_id,)) + + +@pytest.mark.parametrize( + "statement", + ( + "UPDATE events SET details = 'forbidden-tamper' WHERE claim_id = %s", + "DELETE FROM events WHERE claim_id = %s", + ), + ids=("update", "delete"), +) +def test_app_role_cannot_update_or_delete_events( + runtime_stores: RuntimeStores, + seed_rows: SeedRows, + statement: str, +) -> None: + _assert_rls_denied( + lambda: _execute_event_mutation( + runtime_stores.alice_a_scope_a, + statement, + seed_rows.public_a.id, + ) + ) + + +def _set_replacement_reference( + store: PostgresStore, + old_claim_id: int, + replacement_id: int, +) -> None: + with store.connect() as conn, conn.cursor() as cur: + cur.execute( + "UPDATE claims SET replaced_by_claim_id = %s WHERE id = %s", + (replacement_id, old_claim_id), + ) + + +def test_supersession_reference_guard_rejects_hidden_and_foreign_targets( + prepared_database: PreparedDatabase, + runtime_stores: RuntimeStores, + seed_rows: SeedRows, +) -> None: + config = prepared_database.config + bob_public = _create_claim( + runtime_stores.bob_a_scope_a, + config, + label="bob-public-supersession-target", + scope=config.scope_a, + source_agent=config.bob, + visibility="public", + ) + owner = runtime_stores.alice_a_scope_a + for target_id in ( + seed_rows.private_a.id, + seed_rows.scope_b.id, + seed_rows.tenant_b.id, + bob_public.id, + ): + _assert_rls_denied( + lambda target_id=target_id: _set_replacement_reference( + owner, + seed_rows.public_a.id, + target_id, + ) + ) + unchanged = owner.get_claim(seed_rows.public_a.id, include_citations=False) + assert unchanged is not None + assert unchanged.replaced_by_claim_id is None + + +def test_canonical_supersession_commits_reciprocal_pair_and_one_event( + prepared_database: PreparedDatabase, + runtime_stores: RuntimeStores, +) -> None: + config = prepared_database.config + store = runtime_stores.alice_a_scope_a + old = _create_claim( + store, + config, + label="atomic-supersession-old", + scope=config.scope_a, + source_agent=config.alice, + visibility="public", + ) + replacement = _create_claim( + store, + config, + label="atomic-supersession-new", + scope=config.scope_a, + source_agent=config.alice, + visibility="public", + ) + + store.mark_superseded(old.id, replacement.id, "integration atomicity") + + refreshed_old = store.get_claim(old.id, include_citations=False) + refreshed_replacement = store.get_claim( + replacement.id, + include_citations=False, + ) + events = store.list_events(claim_id=old.id, event_type="supersession") + assert refreshed_old.status == "superseded" + assert refreshed_old.replaced_by_claim_id == replacement.id + assert refreshed_replacement.supersedes_claim_id == old.id + assert len(events) == 1 + + +def _tenant_event_rows_as_admin( + prepared: PreparedDatabase, +) -> list[dict[str, Any]]: + import psycopg + from psycopg.rows import dict_row + + with psycopg.connect( + prepared.config.admin_dsn, + row_factory=dict_row, + ) as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT id, claim_id, event_type, from_status, to_status, details, + payload_json, created_at, prev_event_hash, event_hash, hash_algo, + tenant_id, tenant_prev_event_hash, tenant_event_hash, tenant_hash_algo + FROM public.events + WHERE tenant_id = %s + ORDER BY id ASC + """, + (prepared.config.tenant_a,), + ) + return list(cur.fetchall()) + + +def test_tenant_event_chain_does_not_fork_across_rls_views( + prepared_database: PreparedDatabase, + runtime_stores: RuntimeStores, + seed_rows: SeedRows, +) -> None: + """Cross-scope/private appends must share one tenant-global chain head.""" + config = prepared_database.config + bob_private = _create_claim( + runtime_stores.bob_a_scope_a, + config, + label="bob-private-chain", + scope=config.scope_a, + source_agent=config.bob, + visibility="private", + ) + for store, claim, label in ( + (runtime_stores.alice_a_scope_a, seed_rows.private_a, "alice-private"), + (runtime_stores.alice_a_scope_b, seed_rows.scope_b, "alice-scope-b"), + (runtime_stores.bob_a_scope_a, bob_private, "bob-private"), + ): + store.record_event( + claim_id=claim.id, + event_type="audit", + details=f"{config.run_id}:{label}:chain-proof", + ) + + rows = _tenant_event_rows_as_admin(prepared_database) + claim_ids = {int(row["claim_id"]) for row in rows if row["claim_id"] is not None} + assert { + seed_rows.private_a.id, + seed_rows.scope_b.id, + bob_private.id, + } <= claim_ids + assert PostgresStore._event_chain_issues(rows, limit=500) == [] + assert PostgresStore._tenant_event_chain_issues(rows, limit=500) == [] + + +def test_principal_local_claim_identity_matrix( + prepared_database: PreparedDatabase, + runtime_stores: RuntimeStores, +) -> None: + config = prepared_database.config + citation = [CitationInput(source="postgres-principal-identity", locator=config.run_id)] + + def create(store: PostgresStore, key: str, visibility: str) -> Claim: + return store.create_claim( + f"{config.run_id}:same identity payload", + citation, + idempotency_key=f"{config.run_id}:{key}", + subject=f"{config.run_id}:identity-subject", + predicate="uses", + scope=config.scope_a, + tenant_id=config.tenant_a, + source_agent=store.principal, + visibility=visibility, + ) + + alice = runtime_stores.alice_a_scope_a + bob = runtime_stores.bob_a_scope_a + alice_private = create(alice, "private-shared", "private") + bob_private = create(bob, "private-shared", "private") + alice_public = create(alice, "public-shared", "public") + bob_public = create(bob, "public-shared", "public") + public_cross = create(alice, "cross-visibility", "public") + private_cross = create(alice, "cross-visibility", "private") + + assert alice_private.id != bob_private.id + assert alice_private.human_id == bob_private.human_id + assert bob_public.id == alice_public.id + assert private_cross.id != public_cross.id + assert private_cross.human_id == public_cross.human_id + + +def test_public_claim_identity_is_scope_local_and_lookups_require_exact_scope( + prepared_database: PreparedDatabase, + runtime_stores: RuntimeStores, +) -> None: + config = prepared_database.config + label = "public-cross-scope-shared-identity" + idempotency_key = f"{config.run_id}:{label}" + scope_a = _create_claim( + runtime_stores.alice_a_scope_a, + config, + label=label, + scope=config.scope_a, + source_agent=config.alice, + visibility="public", + ) + scope_b = _create_claim( + runtime_stores.alice_a_scope_b, + config, + label=label, + scope=config.scope_b, + source_agent=config.alice, + visibility="public", + ) + + assert scope_b.id != scope_a.id + assert scope_a.human_id is not None + assert scope_b.human_id == scope_a.human_id + assert "~" not in scope_a.human_id + + multi_scope = PostgresStore( + config.app_dsn, + tenant_id=config.tenant_a, + require_tenant=True, + principal=config.alice, + allowed_scopes=(config.scope_a, config.scope_b), + ) + assert multi_scope.get_claim_by_idempotency_key( + idempotency_key, + scope=config.scope_a, + ).id == scope_a.id + assert multi_scope.get_claim_by_idempotency_key( + idempotency_key, + scope=config.scope_b, + ).id == scope_b.id + assert multi_scope.get_claim_by_human_id( + scope_a.human_id, + scope=config.scope_a, + ).id == scope_a.id + assert multi_scope.get_claim_by_human_id( + scope_a.human_id, + scope=config.scope_b, + ).id == scope_b.id + assert multi_scope.resolve_claim_id( + scope_a.human_id, + scope=config.scope_a, + ) == scope_a.id + assert multi_scope.resolve_claim_id( + scope_a.human_id, + scope=config.scope_b, + ) == scope_b.id + + with pytest.raises(ValueError, match="exact claim scope"): + multi_scope.get_claim_by_idempotency_key(idempotency_key) + with pytest.raises(ValueError, match="exact claim scope"): + multi_scope.get_claim_by_human_id(scope_a.human_id) diff --git a/tests/test_postgres_runtime_boundary.py b/tests/test_postgres_runtime_boundary.py new file mode 100644 index 00000000..10165c26 --- /dev/null +++ b/tests/test_postgres_runtime_boundary.py @@ -0,0 +1,913 @@ +from __future__ import annotations + +import hashlib +import importlib +import json +import re +from dataclasses import dataclass, field +from typing import Sequence + +import pytest + +import memorymaster.stores._storage_schema as schema_module +from memorymaster.stores.migrations import discover_migrations +from memorymaster.stores.postgres_store import ( + POSTGRES_CLAIM_OWNER_CHECK, + POSTGRES_EVENT_GUARD_SOURCE, + PostgresStore, +) + + +TENANT_TABLES = ( + "claims", + "citations", + "events", + "claim_links", + "claim_embeddings", + "contradiction_verdicts", + "mcp_usage", +) +TEAM_DENY_TABLES = ( + "action_proposals", + "external_sources", + "source_items", + "evidence_items", + "media_retry_queue", + "query_cache", + "miner_state", + "rule_stats", +) +PROTECTED_TABLES = TENANT_TABLES + TEAM_DENY_TABLES +AUTHORITY_GUCS = ( + "memorymaster.tenant_id", + "memorymaster.principal", + "memorymaster.allowed_scopes", +) +COMMAND_POLICIES = { + "SELECT": "memorymaster_tenant_select", + "INSERT": "memorymaster_tenant_insert", + "UPDATE": "memorymaster_tenant_update", + "DELETE": "memorymaster_tenant_delete", +} +PERMIT_POLICIES = { + command: f"{name}_permit" for command, name in COMMAND_POLICIES.items() +} +POLICY_FIELDS = ( + "schemaname", + "tablename", + "policyname", + "permissive", + "roles", + "cmd", + "qual", + "with_check", +) + + +def _canonical_policy_payload(policies: Sequence[dict[str, object]]) -> str: + rows: list[dict[str, object]] = [] + for policy in policies: + row = {field: policy.get(field) for field in POLICY_FIELDS} + roles = row["roles"] + if isinstance(roles, (list, tuple, set, frozenset)): + row["roles"] = sorted(str(role) for role in roles) + rows.append(row) + rows.sort( + key=lambda row: ( + str(row["schemaname"]), + str(row["tablename"]), + str(row["policyname"]), + ) + ) + return json.dumps(rows, ensure_ascii=False, separators=(",", ":")) + + +def _policy_manifest_comment(policies: Sequence[dict[str, object]]) -> str: + payload = _canonical_policy_payload(policies).encode("utf-8") + return ( + "memorymaster.rls/v1;manifest=0011;sha256=" + f"{hashlib.sha256(payload).hexdigest()}" + ) + + +def _v0011_checksum() -> str: + migration = next(item for item in discover_migrations() if item.version == 11) + return migration.checksum() + + +def _v0012_checksum() -> str: + migration = next(item for item in discover_migrations() if item.version == 12) + return migration.checksum() + + +def _table_row(table: str) -> dict[str, object]: + return { + "table_name": table, + "relname": table, + "relrowsecurity": True, + "relforcerowsecurity": True, + "owner_name": "memorymaster_admin", + "table_owner": "memorymaster_admin", + "owner_member": False, + "is_owner_member": False, + "can_truncate": False, + "has_truncate": False, + "can_references": False, + "has_references": False, + "can_trigger": False, + "has_trigger": False, + "can_select": table == "events", + "has_select": table == "events", + "can_insert": table == "events", + "has_insert": table == "events", + "can_update": False, + "has_update": False, + "can_update_any_column": False, + "has_update_any_column": False, + "can_delete": False, + "has_delete": False, + } + + +def _metadata_table_row(table: str) -> dict[str, object]: + return { + "table_name": table, + "relname": table, + "can_select": True, + "has_select": True, + "can_insert": False, + "has_insert": False, + "can_update": False, + "has_update": False, + "can_delete": False, + "has_delete": False, + "can_truncate": False, + "has_truncate": False, + "can_references": False, + "has_references": False, + "can_trigger": False, + "has_trigger": False, + } + + +def _claim_identity_index_rows() -> list[dict[str, object]]: + return [ + { + "index_name": name, + "relname": name, + "indisunique": True, + "is_unique": True, + "indisvalid": True, + "is_valid": True, + "indisready": True, + "is_ready": True, + "indexdef": definition, + "index_definition": definition, + "predicate": predicate, + "index_predicate": predicate, + } + for name, (definition, predicate) in sorted( + PostgresStore._expected_claim_identity_catalog().items() + ) + ] + + +def _event_head_function_row() -> dict[str, object]: + migration = importlib.import_module( + "memorymaster.stores.migrations.0011_postgres_scoped_force_rls" + ) + definition = str(migration._EVENT_HEAD_FUNCTION) + source = definition.split("AS $$", 1)[1].rsplit("$$", 1)[0].strip() + return { + "schema_name": "public", + "function_name": "memorymaster_event_chain_head", + "argument_count": 0, + "result_signature": "TABLE(global_event_hash text, tenant_event_hash text)", + "language_name": "plpgsql", + "security_definer": True, + "function_config": ("search_path=pg_catalog, pg_temp",), + "volatility": "v", + "parallel_safety": "u", + "leakproof": False, + "strict": False, + "public_execute": False, + "runtime_execute": True, + "owner_is_runtime": False, + "owner_member": False, + "owner_superuser": True, + "owner_bypassrls": False, + "function_source": source, + "function_definition": definition, + } + + +def _owner_constraint_row() -> dict[str, object]: + return { + "schema_name": "public", + "table_name": "claims", + "constraint_name": "ck_claims_identity_visibility_owner", + "constraint_type": "c", + "validated": True, + "is_local": True, + "no_inherit": False, + "constraint_definition": POSTGRES_CLAIM_OWNER_CHECK, + } + + +def _event_trigger_rows() -> list[dict[str, object]]: + return [ + { + "trigger_name": f"trg_events_append_only_{operation}", + "table_schema": "public", + "table_name": "events", + "enabled_code": "O", + "is_internal": False, + "function_schema": "public", + "function_name": "memorymaster_events_append_only_guard", + "trigger_definition": ( + f"CREATE TRIGGER trg_events_append_only_{operation} BEFORE " + f"{operation.upper()} ON public.events FOR EACH ROW EXECUTE FUNCTION " + "public.memorymaster_events_append_only_guard()" + ), + } + for operation in ("update", "delete") + ] + + +def _event_guard_row() -> dict[str, object]: + return { + "schema_name": "public", + "function_name": "memorymaster_events_append_only_guard", + "argument_count": 0, + "result_signature": "trigger", + "language_name": "plpgsql", + "security_definer": False, + "function_config": (), + "volatility": "v", + "parallel_safety": "u", + "leakproof": False, + "strict": False, + "owner_member": False, + "function_source": POSTGRES_EVENT_GUARD_SOURCE, + } + + +def _supersession_trigger_row() -> dict[str, object]: + return { + "trigger_name": "trg_claims_supersession_boundary", + "table_schema": "public", + "table_name": "claims", + "enabled_code": "O", + "is_internal": False, + "function_schema": "public", + "function_name": "memorymaster_claim_supersession_guard", + "trigger_definition": ( + "CREATE TRIGGER trg_claims_supersession_boundary BEFORE INSERT OR " + "UPDATE OF tenant_id, scope, visibility, source_agent, " + "supersedes_claim_id, replaced_by_claim_id ON public.claims " + "FOR EACH ROW EXECUTE FUNCTION " + "public.memorymaster_claim_supersession_guard()" + ), + } + + +def _supersession_guard_row() -> dict[str, object]: + migration = importlib.import_module( + "memorymaster.stores.migrations.0012_principal_local_claim_identities" + ) + definition = str(migration._SUPERSESSION_GUARD_FUNCTION) + source = definition.split("AS $$", 1)[1].rsplit("$$", 1)[0].strip() + return { + "schema_name": "public", + "function_name": "memorymaster_claim_supersession_guard", + "argument_count": 0, + "result_signature": "trigger", + "language_name": "plpgsql", + "security_definer": False, + "function_config": (), + "volatility": "v", + "parallel_safety": "u", + "leakproof": False, + "strict": False, + "owner_member": False, + "function_source": source, + } + + +def _tenant_predicate(table: str, command: str) -> str: + migration = importlib.import_module( + "memorymaster.stores.migrations.0011_postgres_scoped_force_rls" + ) + if command == "SELECT": + return migration._READ_PREDICATES[table] + return migration._WRITE_PREDICATES[table] + + +def _policy_row( + table: str, + name: str, + *, + restrictive: bool, + command: str = "ALL", +) -> dict[str, object]: + if name == "memorymaster_team_deny": + predicate = "FALSE" + elif name in {*COMMAND_POLICIES.values(), *PERMIT_POLICIES.values()}: + predicate = _tenant_predicate(table, command) + else: + predicate = "TRUE" + qual = None if command == "INSERT" else predicate + with_check = None if command in {"SELECT", "DELETE"} else predicate + return { + "schemaname": "public", + "tablename": table, + "table_name": table, + "policyname": name, + "policy_name": name, + "permissive": "RESTRICTIVE" if restrictive else "PERMISSIVE", + "polpermissive": not restrictive, + "roles": ["public"], + "cmd": command, + "qual": qual, + "with_check": with_check, + } + + +def _safe_policies() -> list[dict[str, object]]: + rows: list[dict[str, object]] = [] + for table in PROTECTED_TABLES: + if table in TENANT_TABLES: + for command, name in COMMAND_POLICIES.items(): + rows.append( + _policy_row( + table, + PERMIT_POLICIES[command], + restrictive=False, + command=command, + ) + ) + rows.append( + _policy_row( + table, + name, + restrictive=True, + command=command, + ) + ) + else: + rows.append(_policy_row(table, "memorymaster_team_deny", restrictive=True)) + return rows + + +@dataclass +class CatalogState: + current_user: str = "memorymaster_app" + session_user: str = "memorymaster_app" + rolsuper: bool = False + rolbypassrls: bool = False + rolreplication: bool = False + rolcreaterole: bool = False + rolcreatedb: bool = False + member_of_privileged_role: bool = False + public_schema_create: bool = False + tables: dict[str, dict[str, object]] = field( + default_factory=lambda: {table: _table_row(table) for table in PROTECTED_TABLES} + ) + policies: list[dict[str, object]] = field(default_factory=_safe_policies) + metadata_tables: dict[str, dict[str, object]] = field( + default_factory=lambda: { + table: _metadata_table_row(table) + for table in ("cache_meta", "schema_versions") + } + ) + claim_identity_indexes: list[dict[str, object]] = field( + default_factory=_claim_identity_index_rows + ) + event_head_function: dict[str, object] = field( + default_factory=_event_head_function_row + ) + owner_constraint: dict[str, object] = field( + default_factory=_owner_constraint_row + ) + event_triggers: list[dict[str, object]] = field( + default_factory=_event_trigger_rows + ) + event_guard: dict[str, object] = field(default_factory=_event_guard_row) + supersession_trigger: dict[str, object] = field( + default_factory=_supersession_trigger_row + ) + supersession_guard: dict[str, object] = field( + default_factory=_supersession_guard_row + ) + policy_manifest_comment: str | None = None + schema_v0011_checksum: str = field(default_factory=_v0011_checksum) + schema_v0012_checksum: str = field(default_factory=_v0012_checksum) + preconfigured_settings: dict[str, str] = field(default_factory=dict) + fail_binding_key: str | None = None + catalog_error_on: str | None = None + + def __post_init__(self) -> None: + if self.policy_manifest_comment is None: + self.policy_manifest_comment = _policy_manifest_comment(self.policies) + + +def _normalize_sql(sql: str) -> str: + return " ".join(sql.lower().split()) + + +def _config_key(sql: str, params: tuple[object, ...]) -> str | None: + for key in AUTHORITY_GUCS: + if key in sql: + return key + return next( + (str(value) for value in params if str(value) in AUTHORITY_GUCS), + None, + ) + + +def _config_value(key: str, params: tuple[object, ...]) -> str: + values = [str(value) for value in params if str(value) != key] + if not values: + raise AssertionError(f"set_config for {key} did not bind a value") + return values[-1] + + +def _requested_tables(params: tuple[object, ...]) -> set[str]: + for value in params: + if isinstance(value, (list, tuple, set, frozenset)): + return {str(item) for item in value} + return set() + + +class FakeCursor: + def __init__(self, connection: FakeConnection) -> None: + self.connection = connection + self.executed: list[tuple[str, tuple[object, ...]]] = [] + self._rows: list[dict[str, object]] = [] + + def __enter__(self) -> FakeCursor: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, sql: str, params: Sequence[object] = ()) -> None: + bound = tuple(params) + self.executed.append((sql, bound)) + normalized = _normalize_sql(sql) + key = _config_key(sql, bound) if "set_config" in normalized else None + if key is not None: + if self.connection.state.fail_binding_key == key: + raise RuntimeError(f"binding failed for {key}") + if not re.search(r"\btrue\b", normalized): + raise AssertionError(f"{key} must be transaction-local") + self.connection.local_settings[key] = _config_value(key, bound) + self._rows = [{"set_config": self.connection.local_settings[key]}] + return + + error_on = self.connection.state.catalog_error_on + if error_on is not None and error_on in normalized: + raise RuntimeError("catalog inspection failed") + if "current_setting" in normalized and "set_config" not in normalized: + self._rows = [ + { + "tenant_id": self.connection.local_settings.get( + "memorymaster.tenant_id", + self.connection.state.preconfigured_settings.get( + "memorymaster.tenant_id", "" + ), + ), + "principal": self.connection.local_settings.get( + "memorymaster.principal", + self.connection.state.preconfigured_settings.get( + "memorymaster.principal", "" + ), + ), + "allowed_scopes": self.connection.local_settings.get( + "memorymaster.allowed_scopes", + self.connection.state.preconfigured_settings.get( + "memorymaster.allowed_scopes", "" + ), + ), + } + ] + elif "pg_constraint" in normalized: + self._rows = [dict(self.connection.state.owner_constraint)] + elif "pg_trigger" in normalized: + if "tbl.relname = 'claims'" in normalized: + self._rows = [dict(self.connection.state.supersession_trigger)] + else: + self._rows = [dict(row) for row in self.connection.state.event_triggers] + elif "pg_index" in normalized: + self._rows = list(self.connection.state.claim_identity_indexes) + elif "schema_versions" in normalized and "checksum" in normalized: + self._rows = [ + { + "version": 11, + "checksum": self.connection.state.schema_v0011_checksum, + }, + { + "version": 12, + "checksum": self.connection.state.schema_v0012_checksum, + }, + ] + elif "obj_description" in normalized or "manifest_comment" in normalized: + self._rows = [ + { + "manifest_comment": self.connection.state.policy_manifest_comment, + "policy_comment": self.connection.state.policy_manifest_comment, + "comment": self.connection.state.policy_manifest_comment, + } + ] + elif "pg_proc" in normalized: + if "memorymaster_claim_supersession_guard" in { + str(value) for value in bound + }: + self._rows = [dict(self.connection.state.supersession_guard)] + elif "memorymaster_events_append_only_guard" in normalized: + self._rows = [dict(self.connection.state.event_guard)] + else: + self._rows = [dict(self.connection.state.event_head_function)] + elif "pg_roles" in normalized: + self._rows = [self._role_row()] + elif "pg_class" in normalized: + all_tables = { + **self.connection.state.tables, + **self.connection.state.metadata_tables, + } + requested = _requested_tables(bound) + selected = requested or set(self.connection.state.tables) + self._rows = [all_tables[name] for name in selected if name in all_tables] + elif "pg_policies" in normalized or "pg_policy" in normalized: + self._rows = list(self.connection.state.policies) + elif "has_schema_privilege" in normalized or "pg_namespace" in normalized: + self._rows = [ + { + "public_schema_create": self.connection.state.public_schema_create, + "can_create_public": self.connection.state.public_schema_create, + } + ] + elif "current_user" in normalized or "session_user" in normalized: + self._rows = [self._role_row()] + elif any(f"from {table}" in normalized for table in PROTECTED_TABLES): + if not self.connection.local_settings.get("memorymaster.tenant_id"): + raise PermissionError("protected query has no tenant context") + self._rows = [] + else: + self._rows = [] + + def _role_row(self) -> dict[str, object]: + state = self.connection.state + return { + "current_user": state.current_user, + "session_user": state.session_user, + "rolname": state.current_user, + "rolsuper": state.rolsuper, + "rolbypassrls": state.rolbypassrls, + "rolreplication": state.rolreplication, + "rolcreaterole": state.rolcreaterole, + "rolcreatedb": state.rolcreatedb, + "member_of_privileged_role": state.member_of_privileged_role, + } + + def fetchone(self) -> dict[str, object] | None: + return self._rows[0] if self._rows else None + + def fetchall(self) -> list[dict[str, object]]: + return list(self._rows) + + def __iter__(self): + return iter(self._rows) + + +class FakeConnection: + def __init__(self, state: CatalogState | None = None) -> None: + self.state = state or CatalogState() + self.autocommit = True + self.closed = False + self.commit_count = 0 + self.rollback_count = 0 + self.local_settings: dict[str, str] = {} + self.cursor_instance = FakeCursor(self) + + def cursor(self) -> FakeCursor: + return self.cursor_instance + + def commit(self) -> None: + self.commit_count += 1 + self.local_settings.clear() + + def rollback(self) -> None: + self.rollback_count += 1 + self.local_settings.clear() + + def close(self) -> None: + self.closed = True + + +class FakePsycopg: + def __init__(self, connection: FakeConnection) -> None: + self.connection = connection + self.calls = 0 + + def connect(self, *_args: object, **_kwargs: object) -> FakeConnection: + self.calls += 1 + return self.connection + + +def _team_store( + *, + tenant_id: str | None = "tenant-alpha", + principal: str | None = "agent@example.test", + allowed_scopes: Sequence[str] = ("project:alpha", "global"), +) -> PostgresStore: + return PostgresStore( + "postgresql://db.invalid/app", + tenant_id=tenant_id, + require_tenant=True, + principal=principal, + allowed_scopes=allowed_scopes, + ) + + +def _attach_driver( + store: PostgresStore, + state: CatalogState | None = None, +) -> tuple[FakeConnection, FakePsycopg]: + connection = FakeConnection(state) + driver = FakePsycopg(connection) + store._psycopg = (driver, object(), object()) + return connection, driver + + +@pytest.mark.parametrize( + ("tenant_id", "principal", "allowed_scopes", "match"), + [ + (None, "agent@example.test", ("project:alpha",), "tenant"), + ("tenant-alpha", None, ("project:alpha",), "principal"), + ("tenant-alpha", " ", ("project:alpha",), "principal"), + ("tenant-alpha", "agent@example.test", (), "scope"), + ("tenant-alpha", "agent@example.test", ("*",), "wildcard"), + ("tenant-alpha", "agent@example.test", ("project:*",), "wildcard"), + ( + "tenant-alpha", + "agent@example.test", + ("project:alpha", "*"), + "wildcard", + ), + ], +) +def test_invalid_team_authority_fails_before_loading_driver( + monkeypatch: pytest.MonkeyPatch, + tenant_id: str | None, + principal: str | None, + allowed_scopes: tuple[str, ...], + match: str, +) -> None: + monkeypatch.setattr( + PostgresStore, + "_load_psycopg", + lambda _self: pytest.fail("invalid authority reached the Postgres driver"), + ) + + with pytest.raises(PermissionError, match=match): + _team_store( + tenant_id=tenant_id, + principal=principal, + allowed_scopes=allowed_scopes, + ).connect() + + +def test_ordinary_postgres_runtime_is_rejected_before_loading_driver( + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = PostgresStore("postgresql://db.invalid/app", require_tenant=False) + monkeypatch.setattr( + store, + "_load_psycopg", + lambda: pytest.fail("unscoped runtime reached the Postgres driver"), + ) + + with pytest.raises(PermissionError, match="(?i)(team|tenant|scoped|runtime)"): + store.connect() + + +def test_team_store_snapshots_allowed_scopes_as_an_immutable_set() -> None: + scopes = ["project:alpha"] + + store = _team_store(allowed_scopes=scopes) + scopes.append("project:expanded-after-construction") + + assert store.allowed_scopes == frozenset({"project:alpha"}) + + +def test_connect_binds_all_authority_transaction_locally_and_validates_catalog() -> None: + store = _team_store() + connection, driver = _attach_driver(store) + + returned = store.connect() + + assert returned is connection + assert driver.calls == 1 + assert connection.autocommit is False + assert connection.commit_count == 0 + set_config_calls = [ + (sql, params) + for sql, params in connection.cursor_instance.executed + if "set_config" in sql.lower() + ] + assert {_config_key(sql, params) for sql, params in set_config_calls} == set( + AUTHORITY_GUCS + ) + for sql, params in set_config_calls: + assert re.search(r"\btrue\b", _normalize_sql(sql)) + assert "%s" in sql + assert "tenant-alpha" not in sql + assert "agent@example.test" not in sql + assert "project:alpha" not in sql + assert params + assert connection.local_settings["memorymaster.tenant_id"] == "tenant-alpha" + assert connection.local_settings["memorymaster.principal"] == "agent@example.test" + assert set( + json.loads(connection.local_settings["memorymaster.allowed_scopes"]) + ) == {"project:alpha", "global"} + emitted = "\n".join(sql.lower() for sql, _ in connection.cursor_instance.executed) + assert "pg_roles" in emitted + assert "pg_class" in emitted + assert "pg_policies" in emitted + assert "has_schema_privilege" in emitted or "pg_namespace" in emitted + role_query = next( + _normalize_sql(sql) + for sql, _ in connection.cursor_instance.executed + if "from pg_roles" in _normalize_sql(sql) + ) + assert "pg_has_role" in role_query + assert "'set'" in role_query + assert "'member'" not in role_query + + +def test_binding_failure_rolls_back_and_closes_connection() -> None: + state = CatalogState(fail_binding_key="memorymaster.principal") + store = _team_store() + connection, _ = _attach_driver(store, state) + + with pytest.raises(RuntimeError, match="binding failed"): + store.connect() + + assert connection.rollback_count == 1 + assert connection.closed is True + + +def test_catalog_query_failure_rolls_back_and_closes_connection() -> None: + state = CatalogState(catalog_error_on="pg_roles") + store = _team_store() + connection, _ = _attach_driver(store, state) + + with pytest.raises(RuntimeError, match="catalog inspection failed"): + store.connect() + + assert connection.rollback_count == 1 + assert connection.closed is True + + +@pytest.mark.parametrize("key", AUTHORITY_GUCS) +def test_connect_rejects_preconfigured_authority_guc_defaults(key: str) -> None: + state = CatalogState(preconfigured_settings={key: "operator-default"}) + store = _team_store() + connection, _ = _attach_driver(store, state) + + with pytest.raises(PermissionError, match="(?i)(authority|setting|guc|default)"): + store.connect() + + assert connection.rollback_count == 1 + assert connection.closed is True + + +def test_catalog_queries_check_set_membership_and_all_privilege_bypasses() -> None: + connection = FakeConnection() + cursor = connection.cursor() + + PostgresStore._validate_runtime_role(cursor) + PostgresStore._validate_runtime_tables(cursor) + + role_sql = next( + _normalize_sql(sql) for sql, _ in cursor.executed if "from pg_roles" in _normalize_sql(sql) + ) + table_sql = next( + _normalize_sql(sql) for sql, _ in cursor.executed if "from pg_class" in _normalize_sql(sql) + ) + assert "pg_has_role" in role_sql and "'set'" in role_sql + for attribute in {"rolreplication", "rolcreaterole", "rolcreatedb"}: + assert attribute in role_sql + assert "'trigger'" in table_sql + + +@pytest.mark.parametrize( + ("case", "match"), + [ + ("superuser", "superuser"), + ("bypassrls", "bypassrls"), + ("replication", "replication"), + ("createrole", "create.role|createrole"), + ("createdb", "create.database|createdb"), + ("session_mismatch", "session"), + ("privileged_membership", "member|privileged|superuser|bypassrls"), + ("owner_member", "owner"), + ("public_create", "create"), + ("truncate", "truncate"), + ("references", "references"), + ("trigger", "trigger"), + ], +) +def test_connect_rejects_privileged_or_impersonated_runtime_role( + case: str, + match: str, +) -> None: + state = CatalogState() + if case == "superuser": + state.rolsuper = True + elif case == "bypassrls": + state.rolbypassrls = True + elif case == "replication": + state.rolreplication = True + elif case == "createrole": + state.rolcreaterole = True + elif case == "createdb": + state.rolcreatedb = True + elif case == "session_mismatch": + state.session_user = "connection_pooler" + elif case == "privileged_membership": + state.member_of_privileged_role = True + elif case == "owner_member": + state.tables["claims"]["owner_member"] = True + state.tables["claims"]["is_owner_member"] = True + elif case == "public_create": + state.public_schema_create = True + elif case == "truncate": + state.tables["claims"]["can_truncate"] = True + state.tables["claims"]["has_truncate"] = True + elif case == "references": + state.tables["claims"]["can_references"] = True + state.tables["claims"]["has_references"] = True + elif case == "trigger": + state.tables["claims"]["can_trigger"] = True + state.tables["claims"]["has_trigger"] = True + store = _team_store() + connection, _ = _attach_driver(store, state) + + with pytest.raises(PermissionError, match=f"(?i){match}"): + store.connect() + + assert connection.rollback_count == 1 + assert connection.closed is True + + +@pytest.mark.parametrize("case", ["missing", "rls_disabled", "rls_not_forced"]) +def test_connect_requires_all_fifteen_tables_with_enable_and_force_rls(case: str) -> None: + state = CatalogState() + if case == "missing": + state.tables.pop("rule_stats") + elif case == "rls_disabled": + state.tables["claims"]["relrowsecurity"] = False + elif case == "rls_not_forced": + state.tables["claims"]["relforcerowsecurity"] = False + store = _team_store() + connection, _ = _attach_driver(store, state) + + with pytest.raises(PermissionError, match="(?i)(15|table|row.level|rls|force)"): + store.connect() + + assert connection.rollback_count == 1 + assert connection.closed is True + + +def test_team_runtime_store_cannot_initialize_or_migrate_schema( + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = _team_store() + monkeypatch.setattr( + schema_module, + "load_schema_postgres_sql", + lambda: pytest.fail("team init read the administrative schema"), + ) + monkeypatch.setattr( + store, + "_load_psycopg", + lambda: pytest.fail("team init reached the Postgres driver"), + ) + + with pytest.raises(PermissionError, match="(?i)(team|schema|migration|init)"): + store.init_db() + + +def test_transaction_local_authority_disappears_after_raw_commit() -> None: + store = _team_store() + connection, _ = _attach_driver(store) + returned = store.connect() + + returned.commit() + + assert connection.local_settings == {} + with pytest.raises(PermissionError, match="tenant context"): + with returned.cursor() as cur: + cur.execute("SELECT id FROM claims") diff --git a/tests/test_postgres_runtime_catalog_integrity.py b/tests/test_postgres_runtime_catalog_integrity.py new file mode 100644 index 00000000..9e70a2fa --- /dev/null +++ b/tests/test_postgres_runtime_catalog_integrity.py @@ -0,0 +1,426 @@ +"""Adversarial runtime catalog contracts for PostgreSQL team mode. + +These tests deliberately exercise catalog drift that can turn a constrained +application role into a cross-tenant oracle or make the event ledger mutable. +They use catalog doubles only; no PostgreSQL service is required. +""" +from __future__ import annotations + +import importlib +import re +from copy import deepcopy +from dataclasses import dataclass, replace +from typing import Sequence + +import pytest + +from memorymaster.stores.postgres_store import ( + POSTGRES_PROTECTED_TABLES, + PostgresStore, +) + + +def _v0011_event_head_definition() -> str: + migration = importlib.import_module( + "memorymaster.stores.migrations.0011_postgres_scoped_force_rls" + ) + return str(migration._EVENT_HEAD_FUNCTION) + + +def _function_source(definition: str) -> str: + match = re.search(r"\bAS\s+\$\$(.*)\$\$\s*;?\s*$", definition, re.I | re.S) + assert match is not None + return match.group(1).strip() + + +@dataclass(frozen=True) +class EventHeadCatalogState: + schema_name: str = "public" + function_name: str = "memorymaster_event_chain_head" + argument_count: int = 0 + result_signature: str = "TABLE(global_event_hash text, tenant_event_hash text)" + language_name: str = "plpgsql" + security_definer: bool = True + function_config: tuple[str, ...] = ("search_path=pg_catalog, pg_temp",) + volatility: str = "v" + parallel_safety: str = "u" + leakproof: bool = False + strict: bool = False + public_execute: bool = False + runtime_execute: bool = True + owner_is_runtime: bool = False + owner_member: bool = False + owner_superuser: bool = True + owner_bypassrls: bool = False + function_source: str = "" + function_definition: str = "" + + def as_row(self) -> dict[str, object]: + definition = self.function_definition or _v0011_event_head_definition() + row = vars(self).copy() + row["function_definition"] = definition + row["function_source"] = self.function_source or _function_source(definition) + return row + + +class SingleRowCursor: + def __init__(self, row: dict[str, object] | None) -> None: + self.row = row + self.statements: list[str] = [] + + def execute(self, statement: str, _params: object = None) -> None: + self.statements.append(" ".join(statement.split()).lower()) + + def fetchone(self) -> dict[str, object] | None: + return deepcopy(self.row) + + +def test_event_head_catalog_query_reads_exact_function_metadata() -> None: + cursor = SingleRowCursor(EventHeadCatalogState().as_row()) + + PostgresStore._validate_event_chain_head_function(cursor) + + emitted = "\n".join(cursor.statements) + for token in ( + "pg_get_function_result", + "pg_language", + "p.prosrc", + "p.provolatile", + "p.proparallel", + "p.proleakproof", + "p.proisstrict", + ): + assert token in emitted + + +@pytest.mark.parametrize( + "change", + [ + {"result_signature": "TABLE(global_event_hash text, tenant_event_hash bigint)"}, + {"language_name": "sql"}, + { + "function_config": ( + "search_path=pg_catalog, pg_temp", + "statement_timeout=0", + ) + }, + {"volatility": "i"}, + {"parallel_safety": "s"}, + {"leakproof": True}, + {"strict": True}, + {"owner_member": True}, + {"owner_superuser": False, "owner_bypassrls": False}, + ], +) +def test_event_head_rejects_exact_metadata_drift(change: dict[str, object]) -> None: + state = replace(EventHeadCatalogState(), **change) + cursor = SingleRowCursor(state.as_row()) + + with pytest.raises(PermissionError, match="(?i)(event|function|signature|catalog|unsafe|drift)"): + PostgresStore._validate_event_chain_head_function(cursor) + + +def test_event_head_rejects_comment_token_decoy_with_cross_tenant_body() -> None: + malicious_definition = """ + CREATE FUNCTION public.memorymaster_event_chain_head() + RETURNS TABLE (global_event_hash text, tenant_event_hash text) + LANGUAGE plpgsql SECURITY DEFINER + SET search_path = pg_catalog, pg_temp + AS $$ + -- Decoy contract tokens: memorymaster.tenant_id, global_event_hash, + -- tenant_event_hash, from public.events. + BEGIN + RETURN QUERY SELECT event.event_hash, event.tenant_event_hash + FROM public.events AS event ORDER BY event.id DESC LIMIT 1; + END; + $$ + """.strip() + state = replace( + EventHeadCatalogState(), + function_source=_function_source(malicious_definition), + function_definition=malicious_definition, + ) + + with pytest.raises(PermissionError, match="(?i)(event|function|body|definition|unsafe|drift)"): + PostgresStore._validate_event_chain_head_function( + SingleRowCursor(state.as_row()) + ) + + +def _identity_index_rows() -> list[dict[str, object]]: + return [ + { + "index_name": name, + "relname": name, + "indisunique": True, + "indisprimary": False, + "indisvalid": True, + "indisready": True, + "indexdef": definition, + "predicate": predicate, + } + for name, (definition, predicate) in sorted( + PostgresStore._expected_claim_identity_catalog().items() + ) + ] + + +def _claims_primary_key_row() -> dict[str, object]: + return { + "index_name": "claims_pkey", + "relname": "claims_pkey", + "indisunique": True, + "indisprimary": True, + "indisvalid": True, + "indisready": True, + "indexdef": "CREATE UNIQUE INDEX claims_pkey ON public.claims USING btree (id)", + "predicate": None, + } + + +def _rogue_unique_constraint_row() -> dict[str, object]: + return { + "index_name": "rogue_claim_identity_oracle", + "relname": "rogue_claim_identity_oracle", + "indisunique": True, + "indisprimary": False, + "indisvalid": True, + "indisready": True, + "indexdef": ( + "CREATE UNIQUE INDEX rogue_claim_identity_oracle ON public.claims " + "USING btree (idempotency_key)" + ), + "predicate": "idempotency_key IS NOT NULL", + } + + +class UniqueIndexCatalogCursor: + """Apply the SQL's catalog filter so prefix filtering can be exploited.""" + + def __init__(self, rows: Sequence[dict[str, object]]) -> None: + self.all_rows = [dict(row) for row in rows] + self.rows: list[dict[str, object]] = [] + self.statement = "" + + def execute(self, statement: str, _params: object = None) -> None: + self.statement = " ".join(statement.lower().split()) + rows = list(self.all_rows) + if "relname like 'idx_claims_%'" in self.statement: + rows = [row for row in rows if str(row["index_name"]).startswith("idx_claims_")] + if "not x.indisprimary" in self.statement or "x.indisprimary = false" in self.statement: + rows = [row for row in rows if not bool(row.get("indisprimary"))] + self.rows = rows + + def fetchall(self) -> list[dict[str, object]]: + return [dict(row) for row in self.rows] + + +def test_claim_identity_catalog_query_scans_every_nonprimary_unique_index() -> None: + cursor = UniqueIndexCatalogCursor(_identity_index_rows() + [_claims_primary_key_row()]) + + PostgresStore._validate_claim_identity_indexes(cursor) + + assert "relname like" not in cursor.statement + assert "indisprimary" in cursor.statement + + +def test_arbitrarily_named_claims_unique_constraint_is_rejected() -> None: + cursor = UniqueIndexCatalogCursor( + _identity_index_rows() + + [_claims_primary_key_row(), _rogue_unique_constraint_row()] + ) + + with pytest.raises(PermissionError, match="(?i)(identity|index|unique|catalog|unsafe)"): + PostgresStore._validate_claim_identity_indexes(cursor) + + +def _protected_table_row(table: str) -> dict[str, object]: + return { + "table_name": table, + "relname": table, + "relrowsecurity": True, + "relforcerowsecurity": True, + "owner_member": False, + "can_truncate": False, + "can_references": False, + "can_trigger": False, + "can_select": table == "events", + "can_insert": table == "events", + "can_update": False, + "can_update_any_column": False, + "can_delete": False, + } + + +class ProtectedTableCursor: + def __init__(self, rows: Sequence[dict[str, object]]) -> None: + self.rows = [dict(row) for row in rows] + self.statement = "" + + def execute(self, statement: str, _params: object = None) -> None: + self.statement = " ".join(statement.lower().split()) + + def fetchall(self) -> list[dict[str, object]]: + return [dict(row) for row in self.rows] + + +@pytest.mark.parametrize("privilege", ["update", "delete"]) +def test_runtime_role_cannot_mutate_existing_events(privilege: str) -> None: + rows = [_protected_table_row(table) for table in POSTGRES_PROTECTED_TABLES] + events = next(row for row in rows if row["table_name"] == "events") + events[f"can_{privilege}"] = True + + with pytest.raises(PermissionError, match=f"(?i)(events|{privilege}|append|privilege)"): + PostgresStore._validate_runtime_tables(ProtectedTableCursor(rows)) + + +def test_runtime_role_cannot_hold_column_level_event_update() -> None: + rows = [_protected_table_row(table) for table in POSTGRES_PROTECTED_TABLES] + events = next(row for row in rows if row["table_name"] == "events") + events["can_update_any_column"] = True + cursor = ProtectedTableCursor(rows) + + with pytest.raises(PermissionError, match="(?i)(events|update|append|privilege)"): + PostgresStore._validate_runtime_tables(cursor) + + assert "has_any_column_privilege" in cursor.statement + + +@pytest.mark.parametrize("privilege", ["select", "insert"]) +def test_runtime_role_requires_event_read_append_privileges(privilege: str) -> None: + rows = [_protected_table_row(table) for table in POSTGRES_PROTECTED_TABLES] + events = next(row for row in rows if row["table_name"] == "events") + events[f"can_{privilege}"] = False + + with pytest.raises(PermissionError, match=f"(?i)(events|{privilege}|append|privilege)"): + PostgresStore._validate_runtime_tables(ProtectedTableCursor(rows)) + + +EVENT_GUARD_SOURCE = """ +BEGIN + RAISE EXCEPTION 'events table is append-only; % is not allowed', TG_OP; +END; +""".strip() + + +def _safe_event_triggers() -> list[dict[str, object]]: + return [ + { + "trigger_name": f"trg_events_append_only_{operation}", + "table_schema": "public", + "table_name": "events", + "enabled_code": "O", + "is_internal": False, + "function_schema": "public", + "function_name": "memorymaster_events_append_only_guard", + "trigger_definition": ( + f"CREATE TRIGGER trg_events_append_only_{operation} BEFORE " + f"{operation.upper()} ON public.events FOR EACH ROW EXECUTE FUNCTION " + "public.memorymaster_events_append_only_guard()" + ), + } + for operation in ("update", "delete") + ] + + +def _safe_event_guard() -> dict[str, object]: + return { + "schema_name": "public", + "function_name": "memorymaster_events_append_only_guard", + "argument_count": 0, + "result_signature": "trigger", + "language_name": "plpgsql", + "security_definer": False, + "function_config": (), + "volatility": "v", + "parallel_safety": "u", + "leakproof": False, + "strict": False, + "owner_member": False, + "function_source": EVENT_GUARD_SOURCE, + } + + +class EventAppendOnlyCatalogCursor: + def __init__( + self, + triggers: Sequence[dict[str, object]], + guard: dict[str, object] | None, + ) -> None: + self.triggers = [dict(row) for row in triggers] + self.guard = deepcopy(guard) + self.rows: list[dict[str, object]] = [] + + def execute(self, statement: str, params: object = None) -> None: + normalized = " ".join(statement.lower().split()) + if "pg_trigger" in normalized: + rows = list(self.triggers) + if "tg.tgname = any" in normalized and params: + allowed = set(params[0]) + rows = [row for row in rows if row["trigger_name"] in allowed] + self.rows = [dict(row) for row in rows] + elif "pg_proc" in normalized: + self.rows = [] if self.guard is None else [dict(self.guard)] + else: + raise AssertionError(f"unexpected append-only catalog query: {normalized}") + + def fetchone(self) -> dict[str, object] | None: + return dict(self.rows[0]) if self.rows else None + + def fetchall(self) -> list[dict[str, object]]: + return [dict(row) for row in self.rows] + + +def _append_only_validator(): + validator = getattr(PostgresStore, "_validate_event_append_only_catalog", None) + assert callable(validator), "team runtime must validate the append-only event catalog" + return validator + + +def test_runtime_accepts_exact_append_only_event_catalog() -> None: + cursor = EventAppendOnlyCatalogCursor(_safe_event_triggers(), _safe_event_guard()) + + _append_only_validator()(cursor) + + +@pytest.mark.parametrize( + "case", + [ + "missing_update", + "altered_update", + "altered_delete", + "missing_guard", + "altered_guard", + "guard_owner_member", + "extra_trigger", + ], +) +def test_runtime_rejects_append_only_event_catalog_drift(case: str) -> None: + triggers = _safe_event_triggers() + guard = _safe_event_guard() + if case == "missing_update": + triggers = [row for row in triggers if not str(row["trigger_name"]).endswith("update")] + elif case == "altered_update": + triggers[0]["trigger_definition"] = str(triggers[0]["trigger_definition"]).replace( + "BEFORE UPDATE", "AFTER UPDATE" + ) + elif case == "altered_delete": + triggers[1]["trigger_definition"] = str(triggers[1]["trigger_definition"]).replace( + "FOR EACH ROW", "FOR EACH STATEMENT" + ) + elif case == "missing_guard": + guard = None + elif case == "altered_guard": + assert guard is not None + guard["function_source"] = "BEGIN RETURN NEW; END;" + elif case == "guard_owner_member": + assert guard is not None + guard["owner_member"] = True + elif case == "extra_trigger": + rogue = deepcopy(triggers[0]) + rogue["trigger_name"] = "trg_events_payload_exfiltration" + triggers.append(rogue) + + cursor = EventAppendOnlyCatalogCursor(triggers, guard) + with pytest.raises(PermissionError, match="(?i)(event|append|trigger|function|catalog|unsafe|drift)"): + _append_only_validator()(cursor) diff --git a/tests/test_postgres_supersession_boundary.py b/tests/test_postgres_supersession_boundary.py new file mode 100644 index 00000000..c435f185 --- /dev/null +++ b/tests/test_postgres_supersession_boundary.py @@ -0,0 +1,489 @@ +"""Adversarial PostgreSQL supersession boundary and atomicity contracts.""" +from __future__ import annotations + +import copy +import importlib +from datetime import datetime, timezone +from types import SimpleNamespace +from typing import Sequence + +import pytest + +from memorymaster.stores.postgres_store import PostgresStore + + +GUARD_NAME = "memorymaster_claim_supersession_guard" +TRIGGER_NAME = "trg_claims_supersession_boundary" +GUARD_SOURCE = """ +DECLARE + reference_id BIGINT; +BEGIN + FOREACH reference_id IN ARRAY ARRAY[ + NEW.supersedes_claim_id, + NEW.replaced_by_claim_id + ] LOOP + IF reference_id IS NOT NULL AND ( + reference_id = NEW.id + OR NOT EXISTS ( + SELECT 1 + FROM public.claims AS referenced + WHERE referenced.id = reference_id + AND referenced.tenant_id IS NOT DISTINCT FROM NEW.tenant_id + AND referenced.scope = NEW.scope + AND referenced.visibility IS NOT DISTINCT FROM NEW.visibility + AND referenced.source_agent IS NOT DISTINCT FROM NEW.source_agent + ) + ) THEN + RAISE EXCEPTION 'supersession reference is outside the authorized boundary' + USING ERRCODE = '42501'; + END IF; + END LOOP; + RETURN NEW; +END; +""".strip() + + +def _safe_trigger() -> dict[str, object]: + return { + "trigger_name": TRIGGER_NAME, + "table_schema": "public", + "table_name": "claims", + "enabled_code": "O", + "is_internal": False, + "function_schema": "public", + "function_name": GUARD_NAME, + "trigger_definition": ( + f"CREATE TRIGGER {TRIGGER_NAME} BEFORE INSERT OR UPDATE OF " + "tenant_id, scope, visibility, source_agent, supersedes_claim_id, " + "replaced_by_claim_id ON public.claims " + f"FOR EACH ROW EXECUTE FUNCTION public.{GUARD_NAME}()" + ), + } + + +def _safe_guard() -> dict[str, object]: + return { + "schema_name": "public", + "function_name": GUARD_NAME, + "argument_count": 0, + "result_signature": "trigger", + "language_name": "plpgsql", + "security_definer": False, + "function_config": (), + "volatility": "v", + "parallel_safety": "u", + "leakproof": False, + "strict": False, + "owner_member": False, + "function_source": GUARD_SOURCE, + } + + +class GuardCursor: + def __init__( + self, + trigger: dict[str, object] | None, + guard: dict[str, object] | None, + ) -> None: + self.trigger = copy.deepcopy(trigger) + self.guard = copy.deepcopy(guard) + self.rows: list[dict[str, object]] = [] + + def execute(self, sql: str, _params: Sequence[object] = ()) -> None: + normalized = " ".join(sql.lower().split()) + if "pg_trigger" in normalized: + self.rows = [] if self.trigger is None else [dict(self.trigger)] + elif "pg_proc" in normalized: + self.rows = [] if self.guard is None else [dict(self.guard)] + else: + raise AssertionError(f"unexpected guard catalog SQL: {normalized}") + + def fetchone(self) -> dict[str, object] | None: + return dict(self.rows[0]) if self.rows else None + + def fetchall(self) -> list[dict[str, object]]: + return [dict(row) for row in self.rows] + + +class SupersessionTriggerInventoryCursor(GuardCursor): + def __init__(self) -> None: + super().__init__(_safe_trigger(), _safe_guard()) + self.triggers = [ + _safe_trigger(), + { + **_safe_trigger(), + "trigger_name": "trg_claims_supersession_exfiltration", + }, + ] + + def execute(self, sql: str, params: Sequence[object] = ()) -> None: + normalized = " ".join(sql.lower().split()) + if "pg_trigger" in normalized: + rows = list(self.triggers) + if "tg.tgname = %s" in normalized and params: + rows = [row for row in rows if row["trigger_name"] == params[0]] + self.rows = [dict(row) for row in rows] + return + super().execute(sql, params) + + +def _validator(): + validator = getattr(PostgresStore, "_validate_claim_supersession_guard", None) + assert callable(validator), "team startup must validate the supersession guard" + return validator + + +def test_runtime_accepts_only_exact_supersession_guard_catalog() -> None: + _validator()(GuardCursor(_safe_trigger(), _safe_guard())) + + +def test_runtime_rejects_extra_claims_trigger_outside_exact_catalog() -> None: + with pytest.raises(PermissionError, match="(?i)(supersession|guard|trigger|catalog)"): + _validator()(SupersessionTriggerInventoryCursor()) + + +@pytest.mark.parametrize( + "case", + ["missing", "scope", "visibility", "owner", "boundary-update", "disabled"], +) +def test_runtime_rejects_supersession_guard_drift(case: str) -> None: + trigger = _safe_trigger() + guard = _safe_guard() + if case == "missing": + trigger = None + elif case == "scope": + guard["function_source"] = GUARD_SOURCE.replace( + "AND referenced.scope = NEW.scope\n", + "", + ) + elif case == "visibility": + guard["function_source"] = GUARD_SOURCE.replace( + "AND referenced.visibility IS NOT DISTINCT FROM NEW.visibility\n", + "", + ) + elif case == "owner": + guard["function_source"] = GUARD_SOURCE.replace( + "AND referenced.source_agent IS NOT DISTINCT FROM NEW.source_agent\n", + "", + ) + elif case == "boundary-update": + trigger["trigger_definition"] = str(trigger["trigger_definition"]).replace( + "tenant_id, scope, visibility, source_agent, ", + "", + ) + elif case == "disabled": + trigger["enabled_code"] = "D" + + with pytest.raises(PermissionError, match="(?i)(supersession|guard|trigger|boundary)"): + _validator()(GuardCursor(trigger, guard)) + + +def test_v0012_installs_nonleaking_complete_boundary_guard() -> None: + migration = importlib.import_module( + "memorymaster.stores.migrations.0012_principal_local_claim_identities" + ) + function = str(getattr(migration, "_SUPERSESSION_GUARD_FUNCTION", "")) + trigger = str(getattr(migration, "_SUPERSESSION_GUARD_TRIGGER", "")) + canonical = " ".join((function + " " + trigger).lower().split()) + + for token in ( + "tenant_id", + "scope", + "visibility", + "source_agent", + "supersedes_claim_id", + "replaced_by_claim_id", + "42501", + ): + assert token in canonical + assert "outside the authorized boundary" in canonical + assert "does not exist" not in canonical + assert ( + "update of tenant_id, scope, visibility, source_agent, " + "supersedes_claim_id, replaced_by_claim_id" + ) in canonical + + +class PreflightCursor: + def __init__(self, invalid_edges: int) -> None: + self.invalid_edges = invalid_edges + self.executed: list[str] = [] + self.row: dict[str, int] | None = None + + def __enter__(self) -> PreflightCursor: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, sql: str, _params: Sequence[object] = ()) -> None: + canonical = " ".join(sql.lower().split()) + self.executed.append(canonical) + if "invalid_supersession_edges" in canonical: + self.row = {"invalid_supersession_edges": self.invalid_edges} + + def fetchone(self) -> dict[str, int] | None: + return self.row + + +class PreflightConnection: + def __init__(self, invalid_edges: int) -> None: + self.cursor_instance = PreflightCursor(invalid_edges) + self.commits = 0 + + def cursor(self) -> PreflightCursor: + return self.cursor_instance + + def commit(self) -> None: + self.commits += 1 + + +def test_v0012_rejects_legacy_invalid_supersession_edges_before_ddl() -> None: + migration = importlib.import_module( + "memorymaster.stores.migrations.0012_principal_local_claim_identities" + ) + sql = " ".join( + str(getattr(migration, "POSTGRES_SUPERSESSION_PREFLIGHT_SQL", "")) + .lower() + .split() + ) + for token in ( + "invalid_supersession_edges", + "supersedes_claim_id", + "replaced_by_claim_id", + "tenant_id", + "scope", + "visibility", + "source_agent", + "is distinct from", + ): + assert token in sql + + conn = PreflightConnection(invalid_edges=2) + with pytest.raises(RuntimeError, match="2 invalid supersession"): + migration.apply_postgres(conn) + + assert conn.commits == 0 + assert len(conn.cursor_instance.executed) == 1 + + +class AtomicCursor: + def __init__(self, connection: AtomicConnection) -> None: + self.connection = connection + self.rows: list[dict[str, object]] = [] + self.rowcount = 0 + + def __enter__(self) -> AtomicCursor: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, sql: str, params: Sequence[object] = ()) -> None: + normalized = " ".join(sql.lower().split()) + self.rowcount = 0 + if normalized.startswith("select") and "from claims" in normalized: + ids = {int(value) for value in params if isinstance(value, int)} + self.rows = [ + dict(row) + for claim_id, row in self.connection.state["claims"].items() + if claim_id in ids + ] + return + if "set status = 'superseded'" in normalized: + self.connection.state["claims"][1]["status"] = "superseded" + self.connection.state["claims"][1]["replaced_by_claim_id"] = 2 + self.rowcount = 1 + return + if "set supersedes_claim_id" in normalized: + raise RuntimeError("injected replacement update failure") + raise AssertionError(f"unexpected atomic supersession SQL: {normalized}") + + def fetchone(self) -> dict[str, object] | None: + return dict(self.rows[0]) if self.rows else None + + def fetchall(self) -> list[dict[str, object]]: + return [dict(row) for row in self.rows] + + +class AtomicConnection: + def __init__(self, state: dict[str, object]) -> None: + self.state = state + self.snapshot: dict[str, object] | None = None + self.enter_count = 0 + self.cursor_instance = AtomicCursor(self) + + def __enter__(self) -> AtomicConnection: + self.enter_count += 1 + self.snapshot = copy.deepcopy(self.state) + return self + + def __exit__(self, exc_type, *_args: object) -> None: + if exc_type is not None and self.snapshot is not None: + self.state.clear() + self.state.update(self.snapshot) + return None + + def cursor(self) -> AtomicCursor: + return self.cursor_instance + + +class EventFailureCursor(AtomicCursor): + def execute(self, sql: str, params: Sequence[object] = ()) -> None: + normalized = " ".join(sql.lower().split()) + if "set supersedes_claim_id" in normalized: + self.connection.state["claims"][2]["supersedes_claim_id"] = 1 + self.rowcount = 1 + return + super().execute(sql, params) + + +class EventFailureConnection(AtomicConnection): + def __init__(self, state: dict[str, object]) -> None: + super().__init__(state) + self.cursor_instance = EventFailureCursor(self) + + +def test_replacement_update_failure_rolls_back_entire_supersession( + monkeypatch: pytest.MonkeyPatch, +) -> None: + state: dict[str, object] = { + "claims": { + 1: { + "id": 1, + "status": "confirmed", + "version": 4, + "replaced_by_claim_id": None, + "supersedes_claim_id": None, + }, + 2: { + "id": 2, + "status": "candidate", + "version": 2, + "replaced_by_claim_id": None, + "supersedes_claim_id": None, + }, + }, + "events": [], + } + before = copy.deepcopy(state) + connection = AtomicConnection(state) + store = PostgresStore( + "postgresql://runtime.invalid/memorymaster", + tenant_id="tenant-a", + require_tenant=True, + principal="alice", + allowed_scopes={"project:a"}, + ) + monkeypatch.setattr(store, "connect", lambda: connection) + monkeypatch.setattr( + store, + "get_claim", + lambda *_args, **_kwargs: SimpleNamespace( + id=1, + status="confirmed", + replaced_by_claim_id=None, + ), + ) + + def legacy_first_write(*_args, **_kwargs): + state["claims"][1]["status"] = "superseded" # type: ignore[index] + state["claims"][1]["replaced_by_claim_id"] = 2 # type: ignore[index] + + monkeypatch.setattr(store, "apply_status_transition", legacy_first_write) + monkeypatch.setattr( + store, + "set_supersedes", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("injected replacement update failure") + ), + ) + monkeypatch.setattr( + store, + "_insert_event_row", + lambda *_args, **_kwargs: state["events"].append("event"), # type: ignore[union-attr] + ) + + with pytest.raises(RuntimeError, match="injected replacement"): + store.mark_superseded(1, 2, "atomic boundary test") + + assert connection.enter_count == 1 + assert state == before + + +def test_event_insert_failure_rolls_back_entire_supersession( + monkeypatch: pytest.MonkeyPatch, +) -> None: + state: dict[str, object] = { + "claims": { + 1: { + "id": 1, + "status": "confirmed", + "version": 4, + "replaced_by_claim_id": None, + "supersedes_claim_id": None, + }, + 2: { + "id": 2, + "status": "candidate", + "version": 2, + "replaced_by_claim_id": None, + "supersedes_claim_id": None, + }, + }, + "events": [], + } + before = copy.deepcopy(state) + connection = EventFailureConnection(state) + store = PostgresStore("postgresql://runtime.invalid/memorymaster") + monkeypatch.setattr(store, "connect", lambda: connection) + monkeypatch.setattr( + store, + "_insert_event_row", + lambda *_args, **_kwargs: (_ for _ in ()).throw( + RuntimeError("injected event insert failure") + ), + ) + + with pytest.raises(RuntimeError, match="injected event insert"): + store.mark_superseded(1, 2, "atomic event boundary test") + + assert connection.enter_count == 1 + assert state == before + + +class NoWriteCursor: + def __init__(self) -> None: + self.executed: list[str] = [] + + def execute(self, sql: str, _params: Sequence[object] = ()) -> None: + self.executed.append(" ".join(sql.split())) + + +def test_postgres_atomic_supersession_rejects_archived_source() -> None: + store = PostgresStore("postgresql://runtime.invalid/memorymaster") + cursor = NoWriteCursor() + + with pytest.raises(ValueError, match="Invalid transition"): + store._apply_atomic_supersession( + None, + cursor, + { + "id": 1, + "status": "archived", + "version": 3, + "replaced_by_claim_id": None, + "supersedes_claim_id": None, + }, + { + "id": 2, + "status": "candidate", + "version": 1, + "replaced_by_claim_id": None, + "supersedes_claim_id": None, + }, + "invalid archived supersession", + datetime.now(timezone.utc), + ) + + assert cursor.executed == [] diff --git a/tests/test_postgres_tenant_context.py b/tests/test_postgres_tenant_context.py index 2895b0e7..83f34203 100644 --- a/tests/test_postgres_tenant_context.py +++ b/tests/test_postgres_tenant_context.py @@ -56,23 +56,21 @@ def test_required_tenant_fails_before_loading_postgres_driver(monkeypatch) -> No store.connect() -def test_connect_sets_tenant_context_on_every_connection() -> None: - connection = FakeConnection() - driver = FakePsycopg(connection) +def test_non_team_postgres_runtime_fails_closed_before_driver_load( + monkeypatch, +) -> None: store = PostgresStore( "postgresql://db.invalid/app", tenant_id="tenant-alpha", - require_tenant=True, ) - store._psycopg = (driver, object(), object()) - - returned = store.connect() + monkeypatch.setattr( + store, + "_load_psycopg", + lambda: pytest.fail("non-team Postgres runtime reached the driver"), + ) - assert returned is connection - assert driver.calls == 1 - sql, params = connection.cursor_instance.executed[0] - assert "set_config('memorymaster.tenant_id'" in sql - assert params == ("tenant-alpha",) + with pytest.raises(PermissionError, match="(?i)(team|authority|sqlite)"): + store.connect() def test_store_factory_propagates_postgres_tenant_context() -> None: @@ -80,6 +78,8 @@ def test_store_factory_propagates_postgres_tenant_context() -> None: "postgresql://db.invalid/app", tenant_id="tenant-alpha", require_tenant=True, + principal="agent@example.test", + allowed_scopes=("project:alpha",), ) assert isinstance(store, PostgresStore) @@ -101,6 +101,8 @@ def fake_create_store(db_target, **kwargs): "postgresql://db.invalid/app", tenant_id="tenant-alpha", require_tenant=True, + principal="agent@example.test", + allowed_scopes=("project:alpha",), ) assert captured["tenant_id"] == "tenant-alpha" diff --git a/tests/test_principal_local_claim_identities.py b/tests/test_principal_local_claim_identities.py new file mode 100644 index 00000000..3a69ac4f --- /dev/null +++ b/tests/test_principal_local_claim_identities.py @@ -0,0 +1,416 @@ +"""Adversarial SQLite semantics for principal-local claim identities.""" +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from memorymaster.core.lifecycle import transition_claim +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.stores.storage import SQLiteStore + + +TENANT = "tenant-principal-identities" +SCOPE = "project:principal-identities" +CITATIONS = [CitationInput(source="identity-red", locator="fixture")] + + +def _bootstrap(db_path: Path) -> None: + MemoryService(db_path, workspace_root=db_path.parent).init_db() + + +def _service(db_path: Path, principal: str) -> MemoryService: + return MemoryService( + db_path, + workspace_root=db_path.parent, + tenant_id=TENANT, + require_tenant=True, + principal=principal, + allowed_scopes={SCOPE}, + ) + + +def _ingest( + service: MemoryService, + *, + text: str, + key: str | None, + visibility: str, + subject: str = "shared-subject", + predicate: str = "uses", +): + return service.ingest( + text=text, + citations=CITATIONS, + idempotency_key=key, + subject=subject, + predicate=predicate, + object_value="shared-value", + scope=SCOPE, + visibility=visibility, + ) + + +def _direct_create( + store: SQLiteStore, + *, + text: str, + key: str, + principal: str, + visibility: str, +): + return store.create_claim( + text, + CITATIONS, + idempotency_key=key, + subject="direct-subject", + predicate="uses", + object_value="direct-value", + scope=SCOPE, + tenant_id=TENANT, + source_agent=principal, + visibility=visibility, + ) + + +def _confirm(service: MemoryService, claim_id: int) -> None: + transition_claim( + service.store, + claim_id, + "confirmed", + reason="principal-identity-contract", + event_type="validator", + ) + + +def test_service_allows_alice_and_bob_same_private_key_text_and_human_id( + tmp_path: Path, +) -> None: + db_path = tmp_path / "private-service.db" + _bootstrap(db_path) + alice = _service(db_path, "alice") + bob = _service(db_path, "bob") + + alice_claim = _ingest( + alice, + text="The private build uses the same cache.", + key="private-import-key", + visibility="private", + ) + bob_claim = _ingest( + bob, + text="The private build uses the same cache.", + key="private-import-key", + visibility="private", + ) + + assert alice_claim.id != bob_claim.id + assert alice_claim.human_id == bob_claim.human_id + assert alice_claim.source_agent == "alice" + assert bob_claim.source_agent == "bob" + + +def test_service_content_hash_is_principal_local_for_private_claims( + tmp_path: Path, +) -> None: + db_path = tmp_path / "private-content-hash.db" + _bootstrap(db_path) + alice_claim = _ingest( + _service(db_path, "alice"), + text="Identical private content without a caller key.", + key=None, + visibility="private", + ) + bob_claim = _ingest( + _service(db_path, "bob"), + text="Identical private content without a caller key.", + key=None, + visibility="private", + ) + + assert alice_claim.id != bob_claim.id + # Keep the historical content-hash bytes stable. Namespace partitioning + # belongs in lookup/index context, not in the digest material. + assert alice_claim.idempotency_key == bob_claim.idempotency_key + + +def test_same_principal_private_reingest_still_deduplicates(tmp_path: Path) -> None: + db_path = tmp_path / "same-principal.db" + _bootstrap(db_path) + alice = _service(db_path, "alice") + + first = _ingest( + alice, + text="Alice private first payload.", + key="alice-private-key", + visibility="private", + ) + duplicate = _ingest( + alice, + text="Alice private changed payload.", + key="alice-private-key", + visibility="private", + ) + + assert duplicate.id == first.id + + +def test_public_identity_remains_tenant_wide_across_principals(tmp_path: Path) -> None: + db_path = tmp_path / "public-tenant.db" + _bootstrap(db_path) + + first = _ingest( + _service(db_path, "alice"), + text="Tenant-wide public identity.", + key="public-shared-key", + visibility="public", + ) + duplicate = _ingest( + _service(db_path, "bob"), + text="Changed public payload must still dedupe.", + key="public-shared-key", + visibility="public", + ) + + assert duplicate.id == first.id + + +def test_public_and_private_identity_namespaces_can_coexist(tmp_path: Path) -> None: + db_path = tmp_path / "public-private.db" + _bootstrap(db_path) + alice = _service(db_path, "alice") + + public = _ingest( + alice, + text="Same deterministic human identity.", + key="cross-visibility-key", + visibility="public", + ) + private = _ingest( + alice, + text="Same deterministic human identity.", + key="cross-visibility-key", + visibility="private", + ) + + assert public.id != private.id + assert public.human_id == private.human_id + + +def test_nonpublic_identity_uses_exact_visibility_namespace(tmp_path: Path) -> None: + store = SQLiteStore(tmp_path / "exact-visibility.db") + store.init_db() + + private = _direct_create( + store, + text="Same non-public identity.", + key="nonpublic-exact-key", + principal="alice", + visibility="private", + ) + sensitive = _direct_create( + store, + text="Same non-public identity.", + key="nonpublic-exact-key", + principal="alice", + visibility="sensitive", + ) + + assert private.id != sensitive.id + assert private.human_id == sensitive.human_id + transition_claim( + store, + private.id, + "confirmed", + reason="private namespace", + event_type="validator", + ) + transition_claim( + store, + sensitive.id, + "confirmed", + reason="sensitive namespace", + event_type="validator", + ) + + +def test_visibility_is_normalized_and_validated_before_service_dedup( + tmp_path: Path, +) -> None: + db_path = tmp_path / "visibility-boundary.db" + _bootstrap(db_path) + alice = _service(db_path, "alice") + + normalized = _ingest( + alice, + text="Normalize this private visibility.", + key="normalize-visibility-key", + visibility=" PRIVATE ", + ) + duplicate = _ingest( + alice, + text="The normalized key must dedupe.", + key="normalize-visibility-key", + visibility="private", + ) + + assert normalized.visibility == "private" + assert duplicate.id == normalized.id + with pytest.raises(ValueError, match="(?i)visibility"): + _ingest( + alice, + text="Invalid visibility cannot hide behind an existing key.", + key="normalize-visibility-key", + visibility="private-or-public", + ) + + +def test_team_sensitive_write_is_denied_before_any_store_access(tmp_path: Path) -> None: + db_path = tmp_path / "team-sensitive-denial.db" + _bootstrap(db_path) + service = _service(db_path, "alice") + + class NoStoreAccess: + def __getattr__(self, name: str): + raise AssertionError(f"store accessed before sensitive denial: {name}") + + service.store = NoStoreAccess() + with pytest.raises((PermissionError, ValueError), match="(?i)(sensitive|visibility)"): + _ingest( + service, + text="Team runtime cannot persist sensitive visibility.", + key="team-sensitive-denied", + visibility="sensitive", + ) + + +def test_direct_store_does_not_collapse_private_principals(tmp_path: Path) -> None: + store = SQLiteStore(tmp_path / "private-direct.db") + store.init_db() + + alice = _direct_create( + store, + text="Direct private identity.", + key="direct-private-key", + principal="alice", + visibility="private", + ) + bob = _direct_create( + store, + text="Direct private identity.", + key="direct-private-key", + principal="bob", + visibility="private", + ) + + assert alice.id != bob.id + assert alice.human_id == bob.human_id + + +@pytest.mark.parametrize("source_agent", [None, "", " "]) +def test_direct_store_rejects_blank_nonpublic_source_agent( + tmp_path: Path, + source_agent: str | None, +) -> None: + store = SQLiteStore(tmp_path / f"blank-source-{source_agent!r}.db") + store.init_db() + + with pytest.raises(ValueError, match="(?i)(source|principal|agent)"): + store.create_claim( + "Unowned private claim.", + CITATIONS, + idempotency_key="unowned-private-key", + scope=SCOPE, + tenant_id=TENANT, + source_agent=source_agent, + visibility="private", + ) + + with store.connect() as conn: + assert conn.execute("SELECT COUNT(*) FROM claims").fetchone()[0] == 0 + + +def test_foreign_principal_cannot_revive_archived_public_identity( + tmp_path: Path, +) -> None: + db_path = tmp_path / "archived-public-owner.db" + _bootstrap(db_path) + alice = _service(db_path, "alice") + bob = _service(db_path, "bob") + archived = _ingest( + alice, + text="Alice owns this archived public identity.", + key="archived-public-key", + visibility="public", + ) + transition_claim( + alice.store, + archived.id, + "archived", + reason="fixture archive", + event_type="transition", + ) + + try: + _ingest( + bob, + text="Bob must not revive Alice's archived public claim.", + key="archived-public-key", + visibility="public", + ) + except PermissionError: + pass + + persisted = alice.store.get_claim(archived.id) + assert persisted is not None + assert persisted.status == "archived" + + +def test_confirmed_tuple_namespace_is_public_or_exact_nonpublic_principal( + tmp_path: Path, +) -> None: + db_path = tmp_path / "confirmed-tuples.db" + _bootstrap(db_path) + alice = _service(db_path, "alice") + bob = _service(db_path, "bob") + + alice_private = _ingest( + alice, + text="Alice private tuple.", + key="tuple-alice-private", + visibility="private", + ) + bob_private = _ingest( + bob, + text="Bob private tuple.", + key="tuple-bob-private", + visibility="private", + ) + public = _ingest( + alice, + text="Public tuple.", + key="tuple-public-a", + visibility="public", + ) + duplicate_public = _ingest( + bob, + text="Duplicate public tuple.", + key="tuple-public-b", + visibility="public", + ) + duplicate_alice_private = _ingest( + alice, + text="Duplicate Alice private tuple.", + key="tuple-alice-private-2", + visibility="private", + ) + + for claim in (alice_private, bob_private, public): + _confirm(alice if claim.source_agent == "alice" else bob, claim.id) + + with pytest.raises(sqlite3.IntegrityError): + _confirm(bob, duplicate_public.id) + with pytest.raises(sqlite3.IntegrityError): + _confirm(alice, duplicate_alice_private.id) diff --git a/tests/test_principal_local_identity_migration.py b/tests/test_principal_local_identity_migration.py new file mode 100644 index 00000000..cc4c6dd4 --- /dev/null +++ b/tests/test_principal_local_identity_migration.py @@ -0,0 +1,250 @@ +"""RED contracts for v0012 principal-local claim identities. + +Public identities are tenant-wide. Every non-public visibility is a separate +principal-local namespace, so ``private`` and ``sensitive`` must not alias one +another even for the same principal. +""" +from __future__ import annotations + +import hashlib +import importlib +import re +import sqlite3 +from pathlib import Path + +from memorymaster.stores.migrations import discover_migrations +from memorymaster.stores.storage import SQLiteStore + + +PUBLIC_INDEXES = { + "idx_claims_public_idempotency_key_unique", + "idx_claims_public_human_id_unique", + "idx_claims_public_confirmed_tuple_unique", +} +NONPUBLIC_INDEXES = { + "idx_claims_nonpublic_principal_idempotency_key_unique", + "idx_claims_nonpublic_principal_human_id_unique", + "idx_claims_nonpublic_principal_confirmed_tuple_unique", +} +IDENTITY_INDEXES = PUBLIC_INDEXES | NONPUBLIC_INDEXES +LEGACY_UNIQUE_INDEXES = { + "idx_claims_tenant_idempotency_key", + "idx_claims_tenant_human_id", + "idx_claims_confirmed_tuple_unique", +} +LEGACY_GUARDS = { + "trg_claims_confirmed_tuple_guard", + "trg_claims_confirmed_tuple_guard_insert", + "trg_claims_confirmed_tuple_guard_update", +} + + +def _migration(): + return importlib.import_module( + "memorymaster.stores.migrations.0012_principal_local_claim_identities" + ) + + +def _canonical(sql: str | None) -> str: + return " ".join((sql or "").lower().replace('"', "").split()) + + +def _claim_indexes(conn: sqlite3.Connection) -> dict[str, str]: + return { + str(row[0]): _canonical(row[1]) + for row in conn.execute( + "SELECT name, sql FROM sqlite_master " + "WHERE type = 'index' AND tbl_name = 'claims' AND sql IS NOT NULL" + ).fetchall() + } + + +def _unique_claim_indexes(conn: sqlite3.Connection) -> set[str]: + return { + str(row[1]) + for row in conn.execute("PRAGMA index_list(claims)").fetchall() + if bool(row[2]) + } + + +def _claim_triggers(conn: sqlite3.Connection) -> set[str]: + return { + str(row[0]) + for row in conn.execute( + "SELECT name FROM sqlite_master " + "WHERE type = 'trigger' AND tbl_name = 'claims'" + ).fetchall() + } + + +def _legacy_sqlite_claims() -> sqlite3.Connection: + conn = sqlite3.connect(":memory:") + conn.executescript( + """ + CREATE TABLE claims ( + id INTEGER PRIMARY KEY, + idempotency_key TEXT, + human_id TEXT, + subject TEXT, + predicate TEXT, + scope TEXT NOT NULL, + status TEXT NOT NULL, + tenant_id TEXT, + source_agent TEXT, + visibility TEXT NOT NULL DEFAULT 'public' + ); + CREATE UNIQUE INDEX idx_claims_tenant_idempotency_key + ON claims(COALESCE(tenant_id, ''), idempotency_key) + WHERE idempotency_key IS NOT NULL; + CREATE UNIQUE INDEX idx_claims_tenant_human_id + ON claims(COALESCE(tenant_id, ''), human_id) + WHERE human_id IS NOT NULL; + CREATE UNIQUE INDEX idx_claims_confirmed_tuple_unique + ON claims(COALESCE(tenant_id, ''), subject, predicate, scope) + WHERE status = 'confirmed' + AND subject IS NOT NULL AND predicate IS NOT NULL; + CREATE TRIGGER trg_claims_confirmed_tuple_guard_insert + BEFORE INSERT ON claims BEGIN SELECT 1; END; + CREATE TRIGGER trg_claims_confirmed_tuple_guard_update + BEFORE UPDATE ON claims BEGIN SELECT 1; END; + """ + ) + return conn + + +def _assert_exact_principal_identity_catalog(conn: sqlite3.Connection) -> None: + indexes = _claim_indexes(conn) + assert _unique_claim_indexes(conn) == IDENTITY_INDEXES + assert LEGACY_UNIQUE_INDEXES.isdisjoint(indexes) + assert LEGACY_GUARDS.isdisjoint(_claim_triggers(conn)) + + for name in PUBLIC_INDEXES: + sql = indexes[name] + assert "visibility = 'public'" in sql + assert "source_agent" not in sql.split(" where ", 1)[0] + + for name in NONPUBLIC_INDEXES: + sql = indexes[name] + key_sql, predicate_sql = sql.split(" where ", 1) + assert re.search(r"\bvisibility\b.*\bsource_agent\b", key_sql) + assert "visibility <> 'public'" in predicate_sql + assert "source_agent is not null" in predicate_sql + + for token in ("idempotency_key", "human_id"): + public = indexes[f"idx_claims_public_{token}_unique"] + nonpublic = indexes[f"idx_claims_nonpublic_principal_{token}_unique"] + assert f"{token} is not null" in public + assert f"{token} is not null" in nonpublic + + for name in ( + "idx_claims_public_confirmed_tuple_unique", + "idx_claims_nonpublic_principal_confirmed_tuple_unique", + ): + sql = indexes[name] + assert all(token in sql for token in ("subject", "predicate", "scope")) + assert "status = 'confirmed'" in sql + assert "subject is not null" in sql + assert "predicate is not null" in sql + + +class RecordingCursor: + def __init__(self, statements: list[str]) -> None: + self.statements = statements + + def __enter__(self) -> RecordingCursor: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, sql: str, _params: object = None) -> None: + self.statements.append(_canonical(sql)) + + def fetchone(self) -> dict[str, int]: + return {"invalid_supersession_edges": 0} + + +class RecordingConnection: + def __init__(self) -> None: + self.statements: list[str] = [] + self.commits = 0 + + def cursor(self) -> RecordingCursor: + return RecordingCursor(self.statements) + + def commit(self) -> None: + self.commits += 1 + + +def test_v0012_is_discoverable_and_checksum_is_source_frozen() -> None: + migration = next(item for item in discover_migrations() if item.version == 12) + + assert migration.module_name.endswith("0012_principal_local_claim_identities") + assert "principal" in migration.description.lower() + assert migration.checksum() == hashlib.sha256( + migration.source_path.read_bytes() + ).hexdigest() + + +def test_v0012_sqlite_replaces_v9_globals_with_exact_six_indexes() -> None: + conn = _legacy_sqlite_claims() + try: + _migration().apply_sqlite(conn) + _assert_exact_principal_identity_catalog(conn) + finally: + conn.close() + + +def test_v0012_postgres_drops_v9_guards_and_builds_exact_six_indexes() -> None: + conn = RecordingConnection() + + _migration().apply_postgres(conn) + + emitted = "\n".join(conn.statements) + created = { + match.group(1) + for match in re.finditer( + r"create unique index(?: if not exists)? ([a-z0-9_]+)", emitted + ) + } + assert created == IDENTITY_INDEXES + for name in LEGACY_UNIQUE_INDEXES: + assert f"drop index if exists {name}" in emitted + assert "drop trigger if exists trg_claims_confirmed_tuple_guard on claims" in emitted + assert "drop function if exists memorymaster_claims_confirmed_tuple_guard()" in emitted + assert "visibility = 'public'" in emitted + assert "visibility <> 'public'" in emitted + assert "source_agent is not null" in emitted + assert conn.commits == 1 + + +def test_bootstrap_schemas_declare_v12_not_v9_identity_constraints() -> None: + root = Path(__file__).resolve().parents[1] / "memorymaster" + for schema_name in ("schema.sql", "schema_postgres.sql"): + schema = _canonical((root / schema_name).read_text(encoding="utf-8")) + for name in IDENTITY_INDEXES: + assert name in schema + for name in LEGACY_UNIQUE_INDEXES | LEGACY_GUARDS: + assert name not in schema + + +def test_recurring_sqlite_init_never_recreates_v9_identity_constraints( + tmp_path: Path, +) -> None: + store = SQLiteStore(tmp_path / "principal-identities.db") + store.init_db() + store.init_db() + + with store.connect() as conn: + _assert_exact_principal_identity_catalog(conn) + + +def test_legacy_ensure_helpers_cannot_restore_v9_after_v0012(tmp_path: Path) -> None: + store = SQLiteStore(tmp_path / "ensure-contract.db") + store.init_db() + with store.connect() as conn: + store._ensure_claim_idempotency_schema(conn) + store._ensure_human_id_schema(conn) + store._ensure_confirmed_tuple_uniqueness_schema(conn) + conn.commit() + _assert_exact_principal_identity_catalog(conn) diff --git a/tests/test_request_context.py b/tests/test_request_context.py index 66dfb751..69bfd0cb 100644 --- a/tests/test_request_context.py +++ b/tests/test_request_context.py @@ -42,6 +42,23 @@ def test_local_context_is_explicit_trusted_and_immutable() -> None: context.principal = "forged" # type: ignore[misc] +@pytest.mark.parametrize( + ("db_target", "extra_env"), + [ + ("postgresql://memorymaster.invalid/admin", {}), + ("memorymaster.db", {"MEMORYMASTER_STORE_BACKEND": "postgres"}), + ], +) +def test_local_trusted_context_rejects_postgres_before_service_creation( + db_target: str, + extra_env: dict[str, str], +) -> None: + env = {"MEMORYMASTER_MCP_AUTH_MODE": "local-trusted", **extra_env} + + with pytest.raises(PermissionError, match="(?i)(postgres|team|sqlite)"): + access_control.resolve_request_context(db_target=db_target, environ=env) + + def test_context_binding_is_scoped_and_reset() -> None: context = access_control.resolve_request_context( environ={"MEMORYMASTER_MCP_AUTH_MODE": "local-trusted"}, @@ -83,13 +100,22 @@ def test_team_context_carries_frozen_authority() -> None: assert context.mode is access_control.AuthMode.TEAM assert context.tenant_id == "tenant-alpha" - assert context.allowed_scopes == ("project:alpha", "global") + assert context.allowed_scopes == frozenset({"project:alpha", "global"}) assert context.allow_sensitive is False access_control.authorize_context_action(context, "query") with pytest.raises(PermissionError, match="cannot perform 'ingest'"): access_control.authorize_context_action(context, "ingest") +def test_team_context_rejects_unimplemented_sensitive_grant() -> None: + access_control.set_role("mcp-reader", access_control.Role.READER) + + with pytest.raises(PermissionError, match="(?i)(sensitive|team|disabled)"): + access_control.resolve_request_context( + environ=_team_env(MEMORYMASTER_MCP_ALLOW_SENSITIVE="1") + ) + + def test_missing_auth_mode_is_rejected() -> None: with pytest.raises(PermissionError, match="explicit authorization mode"): access_control.resolve_request_context(db_target="memorymaster.db", environ={}) diff --git a/tests/test_scope_identity_bypass_consumers.py b/tests/test_scope_identity_bypass_consumers.py new file mode 100644 index 00000000..ecd88eb2 --- /dev/null +++ b/tests/test_scope_identity_bypass_consumers.py @@ -0,0 +1,330 @@ +"""Adversarial scope-namespace coverage for direct identity consumers.""" +from __future__ import annotations + +import hashlib +import json +import sqlite3 +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import pytest + +from memorymaster.bridges.db_merge import merge_databases +from memorymaster.bridges.dream_bridge import dream_ingest +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.knowledge.transcript_miner import mine_transcript +from memorymaster.recall.claim_edges import MENTION_KIND, extract_edges_for_claim +from memorymaster.stores.storage import SQLiteStore +from memorymaster.surfaces.cli_helpers import _resolve_claim_id + + +TENANT = "tenant-scope-bypass" +SCOPE_A = "project:scope-bypass-a" +SCOPE_B = "project:scope-bypass-b" +CITATIONS = [CitationInput(source="scope-bypass-red", locator="fixture")] + + +def _store(path: Path) -> SQLiteStore: + store = SQLiteStore(path) + store.init_db() + return store + + +def _public_claim( + store: SQLiteStore, + *, + scope: str, + key: str, + text: str, + tenant_id: str | None = TENANT, +): + return store.create_claim( + text, + CITATIONS, + idempotency_key=key, + subject="scope-bypass", + predicate="uses", + scope=scope, + tenant_id=tenant_id, + source_agent="fixture-writer", + visibility="public", + ) + + +def test_db_merge_preserves_same_public_key_in_disjoint_scopes(tmp_path: Path) -> None: + target_path = tmp_path / "target.db" + source_path = tmp_path / "source.db" + target = _store(target_path) + source = _store(source_path) + _public_claim( + target, + scope=SCOPE_A, + key="same-cross-scope-key", + text="Target scope payload.", + ) + _public_claim( + source, + scope=SCOPE_B, + key="same-cross-scope-key", + text="Source scope payload.", + ) + + stats = merge_databases(str(target_path), str(source_path)) + + with target.connect() as conn: + scopes = { + str(row[0]) + for row in conn.execute( + "SELECT scope FROM claims WHERE idempotency_key = ?", + ("same-cross-scope-key",), + ) + } + assert stats["merged"] == 1 + assert scopes == {SCOPE_A, SCOPE_B} + + +def test_transcript_dedup_is_exact_scope_local(tmp_path: Path) -> None: + db_path = tmp_path / "transcript.db" + store = _store(db_path) + text = "The root cause was a stale cache entry in the scoped worker." + digest = hashlib.sha256(text[:500].strip().lower().encode()).hexdigest()[:16] + key = f"transcript-{digest}" + _public_claim( + store, + scope=SCOPE_A, + key=key, + text="Existing other-scope row.", + tenant_id=None, + ) + transcript = tmp_path / "session.jsonl" + transcript.write_text( + json.dumps({"role": "assistant", "content": text}) + "\n", + encoding="utf-8", + ) + + stats = mine_transcript( + transcript, + str(db_path), + scope=SCOPE_B, + min_length=10, + ) + + with store.connect() as conn: + scopes = { + str(row[0]) + for row in conn.execute( + "SELECT scope FROM claims WHERE idempotency_key = ?", + (key,), + ) + } + assert stats["ingested"] == 1 + assert scopes == {SCOPE_A, SCOPE_B} + + +def test_dream_dedup_is_exact_project_scope_local(tmp_path: Path) -> None: + db_path = tmp_path / "dream.db" + store = _store(db_path) + marker = "auto-dream:scoped-note.md" + _public_claim( + store, + scope=SCOPE_A, + key=marker, + text="Existing other-scope row.", + tenant_id=None, + ) + memory_dir = tmp_path / "memory" + memory_dir.mkdir() + (memory_dir / "scoped-note.md").write_text( + "\n".join( + ( + "---", + 'name: "scoped-note"', + 'description: "Scoped direct-ingest regression fixture"', + 'type: "project"', + "---", + "", + "The scoped retrieval bridge uses a bounded cache.", + ) + ), + encoding="utf-8", + ) + + with patch.dict("os.environ", {"CLAUDE_MEMORY_DIR": str(memory_dir)}): + stats = dream_ingest(str(db_path), use_spool=False) + + with store.connect() as conn: + scopes = { + str(row[0]) + for row in conn.execute( + "SELECT scope FROM claims WHERE idempotency_key = ?", + (marker,), + ) + } + assert stats["ingested"] == 1 + assert scopes == {SCOPE_A, "project"} + + +def test_human_reference_resolution_stays_in_source_scope() -> None: + conn = sqlite3.connect(":memory:") + try: + conn.executescript( + """ + CREATE TABLE claims ( + id INTEGER PRIMARY KEY, + text TEXT NOT NULL, + human_id TEXT, + tenant_id TEXT, + scope TEXT NOT NULL, + visibility TEXT NOT NULL, + source_agent TEXT + ); + """ + ) + conn.executemany( + """ + INSERT INTO claims + (id, text, human_id, tenant_id, scope, visibility, source_agent) + VALUES (?, ?, ?, ?, ?, 'public', ?) + """, + ( + (10, "Wrong scope target", "mm-abcd", TENANT, SCOPE_A, "alice"), + (20, "Exact scope target", "mm-abcd", TENANT, SCOPE_B, "bob"), + (30, "See mm-abcd", "mm-3333", TENANT, SCOPE_B, "bob"), + ), + ) + + edges = extract_edges_for_claim(conn, 30, "See mm-abcd") + + assert edges == [(30, 20, MENTION_KIND)] + finally: + conn.close() + + +def test_cli_passes_single_authorized_scope_to_human_id_resolution() -> None: + calls: list[dict[str, object]] = [] + + class Store: + def resolve_claim_id(self, _identifier: str, **kwargs) -> int: + calls.append(kwargs) + return 20 + + service = SimpleNamespace( + tenant_id=TENANT, + allowed_scopes={SCOPE_B}, + store=Store(), + ) + + assert _resolve_claim_id(service, "mm-abcd") == 20 + assert calls == [ + {"tenant_id": TENANT, "visibility": "public", "scope": SCOPE_B} + ] + + +def test_direct_human_lookup_requires_scope_when_identity_is_ambiguous( + tmp_path: Path, +) -> None: + store = _store(tmp_path / "ambiguous-human.db") + first = _public_claim( + store, + scope=SCOPE_A, + key="ambiguous-human-a", + text="Same human identity seed.", + ) + second = _public_claim( + store, + scope=SCOPE_B, + key="ambiguous-human-b", + text="Same human identity seed.", + ) + assert first.human_id == second.human_id + + with pytest.raises(ValueError, match="(?i)(ambiguous|scope)"): + store.get_claim_by_human_id(first.human_id, tenant_id=TENANT) + + +def test_cli_multi_scope_resolution_requests_ambiguity_checked_lookup() -> None: + calls: list[dict[str, object]] = [] + + class Store: + def resolve_claim_id(self, _identifier: str, **kwargs) -> int: + calls.append(kwargs) + return 20 + + service = SimpleNamespace( + tenant_id=TENANT, + allowed_scopes={SCOPE_A, SCOPE_B}, + store=Store(), + ) + + assert _resolve_claim_id(service, "mm-abcd") == 20 + assert calls == [ + {"tenant_id": TENANT, "visibility": "public", "scope": None} + ] + + +@pytest.mark.parametrize( + ("scope_allowlist", "expected_scope"), + [([SCOPE_B], SCOPE_B), ([SCOPE_A, SCOPE_B], None)], +) +def test_claim_path_human_resolution_preserves_scope_ambiguity_contract( + scope_allowlist: list[str], + expected_scope: str | None, +) -> None: + calls: list[dict[str, object]] = [] + + class Store: + def resolve_claim_id(self, _identifier: str, **kwargs) -> int: + calls.append(kwargs) + raise ValueError("fixture stops after identity resolution") + + service = MemoryService.__new__(MemoryService) + service.tenant_id = TENANT + service.require_tenant = True + service.principal = "alice" + service.allowed_scopes = frozenset({SCOPE_A, SCOPE_B}) + service.store = Store() + + assert service.query_claim_paths( + "mm-abcd", + scope_allowlist=scope_allowlist, + requesting_agent="alice", + ) == [] + assert calls == [{"tenant_id": TENANT, "scope": expected_scope}] + + +def test_claim_edge_null_scope_does_not_broaden_to_all_scopes() -> None: + conn = sqlite3.connect(":memory:") + try: + conn.executescript( + """ + CREATE TABLE claims ( + id INTEGER PRIMARY KEY, + text TEXT NOT NULL, + human_id TEXT, + tenant_id TEXT, + scope TEXT, + visibility TEXT NOT NULL, + source_agent TEXT + ); + """ + ) + conn.executemany( + """ + INSERT INTO claims + (id, text, human_id, tenant_id, scope, visibility, source_agent) + VALUES (?, ?, ?, ?, ?, 'public', 'writer') + """, + ( + (10, "Wrong non-null scope", "mm-abcd", TENANT, SCOPE_A), + (20, "Exact null scope", "mm-abcd", TENANT, None), + (30, "See mm-abcd", "mm-3333", TENANT, None), + ), + ) + + edges = extract_edges_for_claim(conn, 30, "See mm-abcd") + + assert edges == [(30, 20, MENTION_KIND)] + finally: + conn.close() diff --git a/tests/test_scope_local_claim_identities.py b/tests/test_scope_local_claim_identities.py new file mode 100644 index 00000000..6072479c --- /dev/null +++ b/tests/test_scope_local_claim_identities.py @@ -0,0 +1,389 @@ +"""Adversarial contracts for scope-local claim identity namespaces. + +An authenticated principal may be allowed to write more than one project +scope. Identity keys therefore include the exact claim scope: otherwise a +row hidden by the scope RLS predicate can still become a uniqueness oracle. +""" +from __future__ import annotations + +import re +from pathlib import Path +from typing import Sequence + +import pytest + +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.stores.postgres_store import PostgresStore +from memorymaster.stores.storage import generate_top_level_human_id + + +TENANT = "tenant-scope-identities" +SCOPE_A = "project:scope-a" +SCOPE_B = "project:scope-b" +CITATIONS = [CitationInput(source="scope-identity-red", locator="fixture")] +SCOPE_QUALIFIED_INDEXES = { + "idx_claims_public_idempotency_key_unique", + "idx_claims_nonpublic_principal_idempotency_key_unique", + "idx_claims_public_human_id_unique", + "idx_claims_nonpublic_principal_human_id_unique", + "idx_claims_public_confirmed_tuple_unique", + "idx_claims_nonpublic_principal_confirmed_tuple_unique", +} + + +def _bootstrap(db_path: Path) -> None: + MemoryService(db_path, workspace_root=db_path.parent).init_db() + + +def _service(db_path: Path, principal: str, *allowed_scopes: str) -> MemoryService: + return MemoryService( + db_path, + workspace_root=db_path.parent, + tenant_id=TENANT, + require_tenant=True, + principal=principal, + allowed_scopes=set(allowed_scopes), + ) + + +def _ingest( + service: MemoryService, + *, + scope: str, + key: str, + visibility: str, + text: str = "The same human-readable identity seed.", +): + return service.ingest( + text=text, + citations=CITATIONS, + idempotency_key=key, + subject="scope-identity", + predicate="uses", + object_value="shared-value", + scope=scope, + visibility=visibility, + ) + + +def test_private_idempotency_key_is_independent_across_allowed_scopes( + tmp_path: Path, +) -> None: + db_path = tmp_path / "private-key-scopes.db" + _bootstrap(db_path) + scope_a = _service(db_path, "alice", SCOPE_A) + scope_b = _service(db_path, "alice", SCOPE_B) + + first = _ingest( + scope_a, + scope=SCOPE_A, + key="private-same-key", + visibility="private", + ) + second = _ingest( + scope_b, + scope=SCOPE_B, + key="private-same-key", + visibility="private", + ) + + assert second.id != first.id + assert second.scope == SCOPE_B + assert first.scope == SCOPE_A + assert second.human_id == first.human_id + + +def test_private_human_id_seed_is_independent_across_allowed_scopes( + tmp_path: Path, +) -> None: + db_path = tmp_path / "private-human-scopes.db" + _bootstrap(db_path) + + first = _ingest( + _service(db_path, "alice", SCOPE_A), + scope=SCOPE_A, + key="private-human-a", + visibility="private", + ) + second = _ingest( + _service(db_path, "alice", SCOPE_B), + scope=SCOPE_B, + key="private-human-b", + visibility="private", + ) + + assert second.id != first.id + assert second.human_id == first.human_id + assert "~" not in second.human_id + + +def test_public_idempotency_key_does_not_oracle_an_inaccessible_scope( + tmp_path: Path, +) -> None: + db_path = tmp_path / "public-key-scopes.db" + _bootstrap(db_path) + + first = _ingest( + _service(db_path, "alice", SCOPE_A), + scope=SCOPE_A, + key="public-same-key", + visibility="public", + ) + second = _ingest( + _service(db_path, "bob", SCOPE_B), + scope=SCOPE_B, + key="public-same-key", + visibility="public", + ) + + assert second.id != first.id + assert second.scope == SCOPE_B + assert second.human_id == first.human_id + + +def test_public_human_id_seed_does_not_oracle_an_inaccessible_scope( + tmp_path: Path, +) -> None: + db_path = tmp_path / "public-human-scopes.db" + _bootstrap(db_path) + + first = _ingest( + _service(db_path, "alice", SCOPE_A), + scope=SCOPE_A, + key="public-human-a", + visibility="public", + ) + second = _ingest( + _service(db_path, "bob", SCOPE_B), + scope=SCOPE_B, + key="public-human-b", + visibility="public", + ) + + assert second.id != first.id + assert second.human_id == first.human_id + assert "~" not in second.human_id + + +def test_same_scope_public_identity_remains_shared_across_principals( + tmp_path: Path, +) -> None: + db_path = tmp_path / "same-scope-public.db" + _bootstrap(db_path) + + first = _ingest( + _service(db_path, "alice", SCOPE_A), + scope=SCOPE_A, + key="same-scope-public-key", + visibility="public", + ) + duplicate = _ingest( + _service(db_path, "bob", SCOPE_A), + scope=SCOPE_A, + key="same-scope-public-key", + visibility="public", + text="A changed payload must still deduplicate in the same scope.", + ) + + assert duplicate.id == first.id + + +def test_same_scope_private_identity_remains_principal_local_and_deduplicated( + tmp_path: Path, +) -> None: + db_path = tmp_path / "same-scope-private.db" + _bootstrap(db_path) + alice = _service(db_path, "alice", SCOPE_A) + + first = _ingest( + alice, + scope=SCOPE_A, + key="same-scope-private-key", + visibility="private", + ) + duplicate = _ingest( + alice, + scope=SCOPE_A, + key="same-scope-private-key", + visibility="private", + text="A changed private payload still deduplicates for Alice.", + ) + + assert duplicate.id == first.id + + +def _canonical(sql: str) -> str: + return " ".join(sql.lower().replace('"', "").split()) + + +def _index_key_sql(sql: str) -> str: + canonical = _canonical(sql) + return canonical.split(" where ", 1)[0] + + +def test_postgres_runtime_catalog_scope_qualifies_all_six_identity_indexes() -> None: + catalog = PostgresStore._expected_claim_identity_catalog() + + assert set(catalog) == SCOPE_QUALIFIED_INDEXES + for name, (definition, _predicate) in catalog.items(): + key_sql = _index_key_sql(definition) + assert re.search(r"\bscope\b", key_sql), name + + +def test_v0012_sqlite_and_postgres_migration_scope_qualify_all_six_indexes() -> None: + migration_path = ( + Path(__file__).resolve().parents[1] + / "memorymaster" + / "stores" + / "migrations" + / "0012_principal_local_claim_identities.py" + ) + source = migration_path.read_text(encoding="utf-8") + + for name in SCOPE_QUALIFIED_INDEXES: + matches = re.findall( + rf"CREATE UNIQUE INDEX IF NOT EXISTS {name}\s+ON claims\((.*?)\)\s+WHERE", + source, + flags=re.IGNORECASE | re.DOTALL, + ) + assert len(matches) == 2, name + assert all(re.search(r"\bscope\b", match, re.IGNORECASE) for match in matches), name + + +class InvisibleCrossScopeHumanCursor: + """Expose a collision unless allocator SQL binds the requested scope.""" + + def __init__(self) -> None: + self.executed: list[tuple[str, tuple[object, ...]]] = [] + self.row: dict[str, object] | None = None + self.identity_probes = 0 + + def execute(self, sql: str, params: Sequence[object] = ()) -> None: + canonical = _canonical(sql) + bound = tuple(params) + self.executed.append((canonical, bound)) + if "from claim_links" in canonical: + self.row = None + return + if "select 1 from claims" in canonical: + self.identity_probes += 1 + if self.identity_probes > 2: + raise AssertionError("allocator looped on an invisible cross-scope human ID") + is_scope_qualified = "scope = %s" in canonical and SCOPE_A in bound + self.row = None if is_scope_qualified else {"exists": 1} + return + raise AssertionError(f"unexpected allocator SQL: {canonical}") + + def fetchone(self) -> dict[str, object] | None: + return self.row + + +def test_postgres_human_id_allocator_ignores_invisible_cross_scope_collision() -> None: + cursor = InvisibleCrossScopeHumanCursor() + expected = generate_top_level_human_id( + "scope-identity", + "The same human-readable identity seed.", + ) + + allocated = PostgresStore._allocate_human_id( + cursor, + "scope-identity", + "The same human-readable identity seed.", + 41, + tenant_id=TENANT, + scope=SCOPE_A, + visibility="private", + source_agent="alice", + ) + + assert allocated == expected + assert cursor.identity_probes == 1 + assert all( + "scope = %s" in sql and SCOPE_A in params + for sql, params in cursor.executed + if "claims" in sql + ) + + +class IdempotencyFallbackCursor: + """Return the hidden row only when fallback SQL omits its scope boundary.""" + + def __init__(self) -> None: + self.executed: list[tuple[str, tuple[object, ...]]] = [] + self.row: dict[str, object] | None = None + + def __enter__(self) -> IdempotencyFallbackCursor: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def execute(self, sql: str, params: Sequence[object] = ()) -> None: + canonical = _canonical(sql) + bound = tuple(params) + self.executed.append((canonical, bound)) + if canonical.startswith("insert into claims"): + self.row = None + return + if canonical.startswith("select id from claims"): + is_scope_qualified = "scope = %s" in canonical and SCOPE_A in bound + self.row = None if is_scope_qualified else {"id": 999} + return + raise AssertionError(f"write followed unsafe fallback path: {canonical}") + + def fetchone(self) -> dict[str, object] | None: + return self.row + + +class IdempotencyFallbackConnection: + def __init__(self, cursor: IdempotencyFallbackCursor) -> None: + self.cursor_instance = cursor + + def __enter__(self) -> IdempotencyFallbackConnection: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def cursor(self) -> IdempotencyFallbackCursor: + return self.cursor_instance + + +def test_postgres_idempotency_fallback_never_returns_hidden_cross_scope_claim( + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = PostgresStore( + "postgresql://runtime.invalid/memorymaster", + tenant_id=TENANT, + require_tenant=True, + principal="alice", + allowed_scopes={SCOPE_A}, + ) + cursor = IdempotencyFallbackCursor() + monkeypatch.setattr(store, "connect", lambda: IdempotencyFallbackConnection(cursor)) + monkeypatch.setattr( + store, + "get_claim", + lambda *_args, **_kwargs: pytest.fail("hidden cross-scope claim was resolved"), + ) + + with pytest.raises(RuntimeError, match="Idempotency key matched missing claim"): + store.create_claim( + "Cross-scope fallback payload.", + CITATIONS, + idempotency_key="cross-scope-fallback-key", + subject="scope-identity", + scope=SCOPE_A, + tenant_id=TENANT, + source_agent="alice", + visibility="private", + ) + + fallback_sql, fallback_params = next( + (sql, params) + for sql, params in cursor.executed + if sql.startswith("select id from claims") + ) + assert "scope = %s" in fallback_sql + assert SCOPE_A in fallback_params diff --git a/tests/test_service_runtime_authority.py b/tests/test_service_runtime_authority.py new file mode 100644 index 00000000..591bd9fd --- /dev/null +++ b/tests/test_service_runtime_authority.py @@ -0,0 +1,415 @@ +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +import pytest + +import memorymaster.core.access_control as access_control +import memorymaster.core.service as service_module +import memorymaster.surfaces.mcp_server as mcp_server +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.stores.postgres_store import PostgresStore +from memorymaster.stores.store_factory import create_store + + +TEAM_PRINCIPAL = "mcp-writer" +TEAM_TENANT = "tenant-alpha" +TEAM_SCOPES = frozenset({"project:alpha", "global"}) + + +class RecordingStore: + def __init__(self) -> None: + self.init_called = False + self.list_claims_calls: list[dict[str, Any]] = [] + + def init_db(self) -> None: + self.init_called = True + + def list_claims(self, **kwargs: Any) -> list[Any]: + self.list_claims_calls.append(kwargs) + return [] + + +@pytest.fixture(autouse=True) +def isolated_authority() -> None: + access_control._agent_roles.clear() + previous_loaded = access_control._loaded + access_control._loaded = True + yield + access_control._agent_roles.clear() + access_control._loaded = previous_loaded + + +@pytest.fixture +def team_environment(monkeypatch: pytest.MonkeyPatch, tmp_path) -> dict[str, str]: + workspace = tmp_path / "alpha" + workspace.mkdir() + values = { + "MEMORYMASTER_MCP_AUTH_MODE": "team", + "MEMORYMASTER_MCP_PRINCIPAL": TEAM_PRINCIPAL, + "MEMORYMASTER_MCP_TENANT_ID": TEAM_TENANT, + "MEMORYMASTER_MCP_WORKSPACE": str(workspace), + "MEMORYMASTER_MCP_ALLOWED_SCOPES": "project:alpha,global", + "MEMORYMASTER_MCP_DB": "postgresql://memorymaster.invalid/app", + } + access_control.set_role(TEAM_PRINCIPAL, access_control.Role.WRITER) + for name, value in values.items(): + monkeypatch.setenv(name, value) + return values + + +def _team_context(*, scopes: frozenset[str] = TEAM_SCOPES) -> access_control.RequestContext: + return access_control.RequestContext( + mode=access_control.AuthMode.TEAM, + principal=TEAM_PRINCIPAL, + role=access_control.Role.WRITER, + tenant_id=TEAM_TENANT, + workspace="C:/work/alpha", + allowed_scopes=scopes, + allow_sensitive=False, + db_target="postgresql://memorymaster.invalid/app", + ) + + +def _service_with_recording_store( + monkeypatch: pytest.MonkeyPatch, +) -> tuple[MemoryService, RecordingStore]: + store = RecordingStore() + monkeypatch.setattr(service_module, "create_store", lambda *_args, **_kwargs: store) + monkeypatch.setattr(MemoryService, "_init_qdrant", staticmethod(lambda: None)) + service = MemoryService( + "postgresql://memorymaster.invalid/app", + tenant_id=TEAM_TENANT, + require_tenant=True, + principal=TEAM_PRINCIPAL, + allowed_scopes=TEAM_SCOPES, + ) + return service, store + + +def test_resolved_request_context_uses_an_immutable_scope_grant( + team_environment: dict[str, str], +) -> None: + context = access_control.resolve_request_context(environ=team_environment) + + assert context.principal == TEAM_PRINCIPAL + assert context.allowed_scopes == TEAM_SCOPES + assert isinstance(context.allowed_scopes, frozenset) + + +def test_local_request_context_keeps_an_explicit_empty_frozen_grant() -> None: + context = access_control.resolve_request_context( + db_target="memorymaster.db", + workspace="C:/work/alpha", + environ={"MEMORYMASTER_MCP_AUTH_MODE": "local-trusted"}, + ) + + assert context.mode is access_control.AuthMode.LOCAL_TRUSTED + assert context.principal == "mcp-session" + assert context.allowed_scopes == frozenset() + assert isinstance(context.allowed_scopes, frozenset) + + +def test_mcp_service_propagates_bound_principal_and_scopes( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, Any] = {} + + def fake_service(db_target: str, **kwargs: Any) -> SimpleNamespace: + captured.update({"db_target": db_target, **kwargs}) + return SimpleNamespace(source_agent=None) + + monkeypatch.setattr(mcp_server, "MemoryService", fake_service) + monkeypatch.setattr(mcp_server, "_resolve_db", lambda value: value) + monkeypatch.setattr(mcp_server, "_resolve_workspace", lambda value: value) + monkeypatch.setattr(mcp_server, "_bind_telemetry_session", lambda *_args: None) + + context = _team_context() + with access_control.bind_request_context(context): + service = mcp_server._service(context.db_target, context.workspace) + + assert captured["tenant_id"] == TEAM_TENANT + assert captured["require_tenant"] is True + assert captured["principal"] == TEAM_PRINCIPAL + assert captured["allowed_scopes"] == TEAM_SCOPES + assert isinstance(captured["allowed_scopes"], frozenset) + assert service.source_agent == TEAM_PRINCIPAL + + +def test_memory_service_propagates_authority_to_store( + monkeypatch: pytest.MonkeyPatch, +) -> None: + captured: dict[str, Any] = {} + + def fake_create_store(db_target: str, **kwargs: Any) -> RecordingStore: + captured.update({"db_target": db_target, **kwargs}) + return RecordingStore() + + monkeypatch.setattr(service_module, "create_store", fake_create_store) + monkeypatch.setattr(MemoryService, "_init_qdrant", staticmethod(lambda: None)) + + service = MemoryService( + "postgresql://memorymaster.invalid/app", + tenant_id=TEAM_TENANT, + require_tenant=True, + principal=TEAM_PRINCIPAL, + allowed_scopes=TEAM_SCOPES, + ) + + assert service.principal == TEAM_PRINCIPAL + assert service.allowed_scopes == TEAM_SCOPES + assert isinstance(service.allowed_scopes, frozenset) + assert captured["principal"] == TEAM_PRINCIPAL + assert captured["allowed_scopes"] == TEAM_SCOPES + + +def test_store_factory_propagates_immutable_authority_to_postgres() -> None: + store = create_store( + "postgresql://memorymaster.invalid/app", + tenant_id=TEAM_TENANT, + require_tenant=True, + principal=TEAM_PRINCIPAL, + allowed_scopes=TEAM_SCOPES, + ) + + assert isinstance(store, PostgresStore) + assert store.principal == TEAM_PRINCIPAL + assert store.allowed_scopes == TEAM_SCOPES + assert isinstance(store.allowed_scopes, frozenset) + + +@pytest.mark.parametrize( + ("authority", "message"), + [ + ( + { + "tenant_id": None, + "principal": TEAM_PRINCIPAL, + "allowed_scopes": TEAM_SCOPES, + }, + "tenant", + ), + ( + { + "tenant_id": TEAM_TENANT, + "principal": None, + "allowed_scopes": TEAM_SCOPES, + }, + "principal", + ), + ( + { + "tenant_id": TEAM_TENANT, + "principal": TEAM_PRINCIPAL, + "allowed_scopes": frozenset(), + }, + "scope", + ), + ( + { + "tenant_id": TEAM_TENANT, + "principal": TEAM_PRINCIPAL, + "allowed_scopes": frozenset({"*"}), + }, + "wildcard|scope", + ), + ], +) +def test_postgres_team_authority_fails_closed_before_loading_driver( + monkeypatch: pytest.MonkeyPatch, + authority: dict[str, Any], + message: str, +) -> None: + with pytest.raises(PermissionError, match=message): + store = PostgresStore( + "postgresql://memorymaster.invalid/app", + require_tenant=True, + **authority, + ) + monkeypatch.setattr( + store, + "_load_psycopg", + lambda: pytest.fail("invalid team authority reached the Postgres driver"), + ) + store.connect() + + +def test_invalid_team_service_authority_fails_before_qdrant_initialization( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + MemoryService, + "_init_qdrant", + staticmethod(lambda: pytest.fail("invalid team authority reached Qdrant")), + ) + + with pytest.raises(PermissionError, match="principal"): + MemoryService( + "postgresql://memorymaster.invalid/app", + tenant_id=TEAM_TENANT, + require_tenant=True, + principal=None, + allowed_scopes=TEAM_SCOPES, + ) + + +def test_mcp_query_scope_contract_defaults_narrows_and_rejects_widening( + team_environment: dict[str, str], +) -> None: + def probe( + scope_allowlist: str = "", + db: str = "memorymaster.db", + workspace: str = ".", + ) -> str: + return scope_allowlist + + guarded = mcp_server._authorized_tool_callable( + probe, + mcp_server.McpToolPolicy("query", team_enabled=True), + ) + + assert guarded() == ",".join(sorted(TEAM_SCOPES)) + assert guarded(scope_allowlist="project:alpha") == "project:alpha" + with pytest.raises(PermissionError, match="scope"): + guarded(scope_allowlist="project:alpha,project:beta") + + +def test_service_query_defaults_to_bound_scope_and_principal( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, store = _service_with_recording_store(monkeypatch) + observed_agents: list[str | None] = [] + + def record_visibility(claims: list[Any], requesting_agent: str | None) -> list[Any]: + observed_agents.append(requesting_agent) + return claims + + monkeypatch.setattr(service_module, "_filter_agent_visibility", record_visibility) + + service.query_rows("authority marker", include_candidates=True) + + assert set(store.list_claims_calls[0]["scope_allowlist"]) == TEAM_SCOPES + assert observed_agents == [TEAM_PRINCIPAL] + + +def test_service_query_allows_scope_narrowing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, store = _service_with_recording_store(monkeypatch) + + service.query_rows( + "authority marker", + include_candidates=True, + scope_allowlist=["project:alpha"], + ) + + assert store.list_claims_calls[0]["scope_allowlist"] == ["project:alpha"] + + +def test_service_query_rejects_scope_widening_before_store_access( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, store = _service_with_recording_store(monkeypatch) + + with pytest.raises(PermissionError, match="scope"): + service.query_rows( + "authority marker", + scope_allowlist=["project:alpha", "project:beta"], + ) + + assert store.list_claims_calls == [] + + +def test_service_query_rejects_requesting_agent_substitution_before_store_access( + monkeypatch: pytest.MonkeyPatch, +) -> None: + service, store = _service_with_recording_store(monkeypatch) + + with pytest.raises(PermissionError, match="principal|requesting_agent"): + service.query_rows("authority marker", requesting_agent="forged-agent") + + assert store.list_claims_calls == [] + + +def test_service_ingest_binds_principal_and_rejects_attribution_or_scope_forgery( + tmp_path, +) -> None: + db_path = tmp_path / "authority.db" + local = MemoryService(db_path, workspace_root=tmp_path) + local.init_db() + service = MemoryService( + db_path, + workspace_root=tmp_path, + tenant_id=TEAM_TENANT, + require_tenant=True, + principal=TEAM_PRINCIPAL, + allowed_scopes=TEAM_SCOPES, + ) + + claim = service.ingest( + "bound service authority marker", + [CitationInput(source="test://authority")], + scope="project:alpha", + ) + before = len(service.store.list_claims(limit=20, tenant_id=TEAM_TENANT)) + + assert claim.source_agent == TEAM_PRINCIPAL + assert claim.scope == "project:alpha" + with pytest.raises(PermissionError, match="principal|source_agent"): + service.ingest( + "forged source authority marker", + [CitationInput(source="test://authority")], + scope="project:alpha", + source_agent="forged-agent", + ) + with pytest.raises(PermissionError, match="scope"): + service.ingest( + "forged scope authority marker", + [CitationInput(source="test://authority")], + scope="project:beta", + ) + assert len(service.store.list_claims(limit=20, tenant_id=TEAM_TENANT)) == before + + +def test_team_service_init_db_is_rejected_before_store_initialization( + monkeypatch: pytest.MonkeyPatch, +) -> None: + store = RecordingStore() + monkeypatch.setattr(service_module, "create_store", lambda *_args, **_kwargs: store) + monkeypatch.setattr(MemoryService, "_init_qdrant", staticmethod(lambda: None)) + service = MemoryService( + "postgresql://memorymaster.invalid/app", + tenant_id=TEAM_TENANT, + require_tenant=True, + principal=TEAM_PRINCIPAL, + allowed_scopes=TEAM_SCOPES, + ) + + with pytest.raises(PermissionError, match="team|schema|init"): + service.init_db() + + assert store.init_called is False + + +def test_unbound_local_service_preserves_legacy_authority_compatibility(tmp_path) -> None: + service = MemoryService(tmp_path / "local.db", workspace_root=tmp_path) + service.init_db() + + claim = service.ingest( + "local trusted compatibility marker", + [CitationInput(source="test://local")], + scope="project:local-custom", + source_agent="local-tool", + ) + rows = service.query_rows( + "local trusted compatibility marker", + include_candidates=True, + scope_allowlist=["project:local-custom"], + requesting_agent="local-tool", + ) + + assert claim.scope == "project:local-custom" + assert claim.source_agent == "local-tool" + assert [row["claim"].id for row in rows] == [claim.id] diff --git a/tests/test_steward.py b/tests/test_steward.py index 31d79f07..9342d9a1 100644 --- a/tests/test_steward.py +++ b/tests/test_steward.py @@ -32,6 +32,10 @@ def _force_status(db: Path, claim_id: int, status: str, updated_at: str) -> None con.execute("DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard_update") con.execute("DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard_insert") con.execute("DROP INDEX IF EXISTS idx_claims_confirmed_tuple_unique") + con.execute("DROP INDEX IF EXISTS idx_claims_public_confirmed_tuple_unique") + con.execute( + "DROP INDEX IF EXISTS idx_claims_nonpublic_principal_confirmed_tuple_unique" + ) con.execute( "UPDATE claims SET status=?, updated_at=?, last_validated_at=? WHERE id=?", (status, updated_at, updated_at, claim_id), diff --git a/tests/test_steward_contradiction_phase.py b/tests/test_steward_contradiction_phase.py index c21a1d96..c83739a5 100644 --- a/tests/test_steward_contradiction_phase.py +++ b/tests/test_steward_contradiction_phase.py @@ -40,6 +40,10 @@ def _force_status(db: Path, claim_id: int, status: str, updated_at: str) -> None con.execute("DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard_update") con.execute("DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard_insert") con.execute("DROP INDEX IF EXISTS idx_claims_confirmed_tuple_unique") + con.execute("DROP INDEX IF EXISTS idx_claims_public_confirmed_tuple_unique") + con.execute( + "DROP INDEX IF EXISTS idx_claims_nonpublic_principal_confirmed_tuple_unique" + ) con.execute( "UPDATE claims SET status=?, updated_at=?, last_validated_at=? WHERE id=?", (status, updated_at, updated_at, claim_id), diff --git a/tests/test_steward_resolution_parity.py b/tests/test_steward_resolution_parity.py index 9b87d88f..21456a22 100644 --- a/tests/test_steward_resolution_parity.py +++ b/tests/test_steward_resolution_parity.py @@ -39,6 +39,10 @@ def _force_status(db: Path, claim_id: int, status: str, updated_at: str) -> None con.execute("DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard_update") con.execute("DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard_insert") con.execute("DROP INDEX IF EXISTS idx_claims_confirmed_tuple_unique") + con.execute("DROP INDEX IF EXISTS idx_claims_public_confirmed_tuple_unique") + con.execute( + "DROP INDEX IF EXISTS idx_claims_nonpublic_principal_confirmed_tuple_unique" + ) con.execute( "UPDATE claims SET status=?, updated_at=?, last_validated_at=? WHERE id=?", (status, updated_at, updated_at, claim_id), diff --git a/tests/test_tenant_event_runtime.py b/tests/test_tenant_event_runtime.py index 02ea6636..bb881191 100644 --- a/tests/test_tenant_event_runtime.py +++ b/tests/test_tenant_event_runtime.py @@ -43,6 +43,8 @@ def executemany(self, sql: str, params: object) -> None: def fetchone(self): if "SELECT tenant_id FROM claims" in self.last_sql: return {"tenant_id": self.claim_tenant} + if "memorymaster_event_chain_head()" in self.last_sql: + return {"global_event_hash": None, "tenant_event_hash": None} if "INSERT INTO events" in self.last_sql: return {"id": 42} return None @@ -75,10 +77,11 @@ def commit(self) -> None: class HistoricalHeadCursor(RecordingCursor): def fetchone(self): - if "SELECT event_hash FROM events" in self.last_sql: - return {"event_hash": "primary-v2-head"} - if "SELECT tenant_event_hash FROM events" in self.last_sql: - return {"tenant_event_hash": "tenant-history-head"} + if "memorymaster_event_chain_head()" in self.last_sql: + return { + "global_event_hash": "primary-v2-head", + "tenant_event_hash": "tenant-history-head", + } return super().fetchone() @@ -211,8 +214,9 @@ def test_postgres_event_insert_uses_tenant_chain_and_advisory_lock() -> None: ) assert event_id == 42 assert "pg_advisory_xact_lock" in emitted - assert "hash_algo = %s" in emitted - assert "tenant_id IS NOT DISTINCT FROM %s" in emitted + assert "memorymaster_event_chain_head()" in emitted + assert "SELECT event_hash FROM events" not in emitted + assert "SELECT tenant_event_hash FROM events" not in emitted assert "tenant_id" in next( sql for sql, _ in conn.cursor_instance.executed if "INSERT INTO events" in sql ) diff --git a/tests/test_tenant_local_claim_identities.py b/tests/test_tenant_local_claim_identities.py index e2b7d8f0..7ed9cb9e 100644 --- a/tests/test_tenant_local_claim_identities.py +++ b/tests/test_tenant_local_claim_identities.py @@ -78,16 +78,16 @@ def test_identity_lookups_are_tenant_qualified(tmp_path) -> None: ) by_key_a = tenant_a.store.get_claim_by_idempotency_key( - "same-key", tenant_id="tenant-a" + "same-key", tenant_id="tenant-a", scope="project:shared" ) by_key_b = tenant_b.store.get_claim_by_idempotency_key( - "same-key", tenant_id="tenant-b" + "same-key", tenant_id="tenant-b", scope="project:shared" ) by_human_a = tenant_a.store.get_claim_by_human_id( - claim_a.human_id, tenant_id="tenant-a" + claim_a.human_id, tenant_id="tenant-a", scope="project:shared" ) by_human_b = tenant_b.store.get_claim_by_human_id( - claim_b.human_id, tenant_id="tenant-b" + claim_b.human_id, tenant_id="tenant-b", scope="project:shared" ) assert by_key_a and by_key_a.id == claim_a.id @@ -98,6 +98,7 @@ def test_identity_lookups_are_tenant_qualified(tmp_path) -> None: tenant_a.store.resolve_claim_id( claim_a.human_id, tenant_id="tenant-a", + scope="project:shared", ) == claim_a.id ) @@ -105,6 +106,7 @@ def test_identity_lookups_are_tenant_qualified(tmp_path) -> None: tenant_b.store.resolve_claim_id( claim_b.human_id, tenant_id="tenant-b", + scope="project:shared", ) == claim_b.id ) @@ -144,7 +146,7 @@ def test_same_confirmed_tuple_can_coexist_across_tenants(tmp_path) -> None: ) with tenant_a.store.connect() as conn: - with pytest.raises(sqlite3.IntegrityError, match="only one confirmed claim"): + with pytest.raises(sqlite3.IntegrityError, match="confirmed_tuple_unique"): conn.execute( "UPDATE claims SET tenant_id = ? WHERE id = ?", ("tenant-a", claim_b.id), @@ -191,7 +193,7 @@ def test_duplicate_identity_and_tuple_still_fail_within_tenant(tmp_path) -> None reason="first-confirm", event_type="validator", ) - with pytest.raises(sqlite3.IntegrityError, match="only one confirmed claim"): + with pytest.raises(sqlite3.IntegrityError, match="confirmed_tuple_unique"): transition_claim( service.store, second_tuple.id, diff --git a/tests/test_tenant_local_identity_migration.py b/tests/test_tenant_local_identity_migration.py index c621e779..f7c0c411 100644 --- a/tests/test_tenant_local_identity_migration.py +++ b/tests/test_tenant_local_identity_migration.py @@ -111,9 +111,12 @@ def test_bootstrap_schemas_declare_tenant_local_identity_indexes() -> None: postgres_schema = (root / "schema_postgres.sql").read_text(encoding="utf-8") for name in ( - "idx_claims_tenant_idempotency_key", - "idx_claims_tenant_human_id", - "idx_claims_confirmed_tuple_unique", + "idx_claims_public_idempotency_key_unique", + "idx_claims_nonpublic_principal_idempotency_key_unique", + "idx_claims_public_human_id_unique", + "idx_claims_nonpublic_principal_human_id_unique", + "idx_claims_public_confirmed_tuple_unique", + "idx_claims_nonpublic_principal_confirmed_tuple_unique", ): assert name in sqlite_schema assert name in postgres_schema @@ -134,9 +137,17 @@ def test_sqlite_reinit_does_not_restore_global_unique_indexes(tmp_path) -> None: assert indexes["idx_claims_idempotency_key"] is False assert indexes["idx_claims_human_id"] is False - assert indexes["idx_claims_tenant_idempotency_key"] is True - assert indexes["idx_claims_tenant_human_id"] is True - assert indexes["idx_claims_confirmed_tuple_unique"] is True + identity_indexes = { + name for name, unique in indexes.items() if unique and name.startswith("idx_claims_") + } + assert identity_indexes == { + "idx_claims_public_idempotency_key_unique", + "idx_claims_nonpublic_principal_idempotency_key_unique", + "idx_claims_public_human_id_unique", + "idx_claims_nonpublic_principal_human_id_unique", + "idx_claims_public_confirmed_tuple_unique", + "idx_claims_nonpublic_principal_confirmed_tuple_unique", + } def test_legacy_human_id_index_is_converted_before_backfill() -> None: diff --git a/tests/test_v313_e2e.py b/tests/test_v313_e2e.py index aeda5f71..ca2de08d 100644 --- a/tests/test_v313_e2e.py +++ b/tests/test_v313_e2e.py @@ -34,6 +34,10 @@ def _force_confirmed(db: Path, claim_id: int) -> None: con.execute("DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard_update") con.execute("DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard_insert") con.execute("DROP INDEX IF EXISTS idx_claims_confirmed_tuple_unique") + con.execute("DROP INDEX IF EXISTS idx_claims_public_confirmed_tuple_unique") + con.execute( + "DROP INDEX IF EXISTS idx_claims_nonpublic_principal_confirmed_tuple_unique" + ) con.execute("UPDATE claims SET status='confirmed' WHERE id=?", (claim_id,)) con.commit() con.close() diff --git a/tests/test_v313_run_cycle_dedupe.py b/tests/test_v313_run_cycle_dedupe.py index c21cb2e6..a4a590dc 100644 --- a/tests/test_v313_run_cycle_dedupe.py +++ b/tests/test_v313_run_cycle_dedupe.py @@ -39,6 +39,10 @@ def _force_confirmed(db: Path, claim_id: int) -> None: con.execute("DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard_update") con.execute("DROP TRIGGER IF EXISTS trg_claims_confirmed_tuple_guard_insert") con.execute("DROP INDEX IF EXISTS idx_claims_confirmed_tuple_unique") + con.execute("DROP INDEX IF EXISTS idx_claims_public_confirmed_tuple_unique") + con.execute( + "DROP INDEX IF EXISTS idx_claims_nonpublic_principal_confirmed_tuple_unique" + ) con.execute("UPDATE claims SET status='confirmed' WHERE id=?", (claim_id,)) con.commit() con.close() diff --git a/tests/test_validator_atomic_supersession.py b/tests/test_validator_atomic_supersession.py new file mode 100644 index 00000000..08e11999 --- /dev/null +++ b/tests/test_validator_atomic_supersession.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from pathlib import Path + +from memorymaster.core.lifecycle import transition_claim +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.govern.jobs.validator import run + + +def _claim(service: MemoryService, text: str, key: str): + return service.ingest( + text=text, + citations=[CitationInput(source="validator-atomicity")], + idempotency_key=key, + subject="validator-atomicity", + predicate="keeps", + object_value="same-value", + ) + + +def test_duplicate_validation_writes_reciprocal_supersession( + tmp_path: Path, +) -> None: + service = MemoryService(str(tmp_path / "validator.db")) + service.init_db() + confirmed = _claim(service, "confirmed winner", "confirmed-winner") + transition_claim( + service.store, + confirmed.id, + "confirmed", + reason="fixture", + event_type="validator", + ) + duplicate = _claim(service, "candidate duplicate", "candidate-duplicate") + + result = run(service.store, min_citations=0, min_score=0.0) + + refreshed_duplicate = service.store.get_claim( + duplicate.id, + include_citations=False, + ) + refreshed_confirmed = service.store.get_claim( + confirmed.id, + include_citations=False, + ) + assert result["superseded"] == 1 + assert refreshed_duplicate.status == "superseded" + assert refreshed_duplicate.replaced_by_claim_id == confirmed.id + assert refreshed_confirmed.supersedes_claim_id == duplicate.id From 64ce3876953cb04a8a21974ca2b692b463549aa3 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 07:28:27 -0300 Subject: [PATCH 12/55] fix(postgres): deny unsupported team surfaces --- ...EMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md | 2 +- .planning/audit-remediation-ledger.md | 2 +- external-actions-required.md | 2 +- memorymaster/bridges/db_merge.py | 6 + memorymaster/bridges/delta_sync.py | 6 + memorymaster/stores/postgres_store.py | 29 ++ memorymaster/stores/store_factory.py | 4 +- .../surfaces/cli_handlers_curation.py | 11 + tests/conftest.py | 142 ++++++-- tests/test_backend_parity.py | 4 +- tests/test_postgres_parity.py | 67 +--- tests/test_postgres_team_surface_denials.py | 324 ++++++++++++++++++ 12 files changed, 517 insertions(+), 82 deletions(-) create mode 100644 tests/test_postgres_team_surface_denials.py diff --git a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md index 98648f64..5f256c19 100644 --- a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md +++ b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md @@ -83,7 +83,7 @@ Phase 0 must replace qualitative capacity goals with recorded numeric gates in t - [x] Phase 0 — evidence, numerical gates, and red tests - [x] R1.1 — authenticated request context and MCP authorization -- [ ] R1.2 — Postgres tenant enforcement and RLS +- [x] R1.2 — Postgres tenant enforcement and RLS (repository complete; external proof blocked) - [ ] R1.3 — immediate Qdrant containment - [ ] R1.4 — complete sensitivity/write gateway and legacy inventory - [ ] R1.5 — secure deployment and supply-chain defaults diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index aaf13016..328f4914 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -12,7 +12,7 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | ID | Primary domain | Severity / exploitability | Summary | Package | Status | Acceptance evidence | |---|---|---|---|---|---|---| -| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | IN-PROGRESS | Owner: Codex remediation branch. R1.1 complete. PostgreSQL application connections are team-only and bind tenant, principal, and immutable scopes transaction-locally; schema work requires a distinct verified migrator. v0011 FORCE RLS defines restricted role/table/policy and append-only event contracts, a tenant-derived hash-only event-head function, and deny-only governance/raw tables. v0012 defines six partial unique indexes: public keys are tenant + exact-scope local; non-public keys additionally include exact visibility/principal; ambiguity without exact scope fails closed; every team claim requires a nonblank owner. Startup validates literal-sensitive policy/index/function fingerprints, exact event and claims trigger inventories, required event SELECT/INSERT plus forbidden table/column UPDATE and DELETE, a privileged event-head owner, and the strict validated owner constraint. Supersession rejects self/cross-tenant/scope/visibility/owner references and the canonical path atomically commits reciprocal pointers plus one event; v0012 preflights unsafe legacy edges. Hermetic R1.2 matrix: 392 passed, 60 skipped, 1 expected xfail. Full isolated non-ML gate: 3,502 passed, 69 skipped, 95 deselected, 22 intentional xfails, 2 warnings in 861.24s. Commit evidence is the commit containing this row. Rollback: disable the blocked team profile and revert this package; schema rollback requires a verified backup/forward repair. Real two-role PostgreSQL evidence plus approved brownfield owner/duplicate/supersession-edge repair remain `BLOCKED-EXTERNAL`. The separate team-surface denial package remains, so R1.2 stays `IN-PROGRESS`. | +| MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | BLOCKED-EXTERNAL | Owner: Codex remediation branch. R1.1 complete. PostgreSQL application connections are team-only and bind tenant, principal, and immutable scopes transaction-locally; schema work requires a distinct verified migrator. v0011 FORCE RLS defines restricted role/table/policy and append-only event contracts, a tenant-derived hash-only event-head function, and deny-only governance/raw tables. v0012 defines six partial unique indexes: public keys are tenant + exact-scope local; non-public keys additionally include exact visibility/principal; ambiguity without exact scope fails closed; every team claim requires a nonblank owner. Startup validates literal-sensitive policy/index/function fingerprints, exact event and claims trigger inventories, required event SELECT/INSERT plus forbidden table/column UPDATE and DELETE, a privileged event-head owner, and the strict validated owner constraint. Supersession rejects self/cross-tenant/scope/visibility/owner references and the canonical path atomically commits reciprocal pointers plus one event; v0012 preflights unsafe legacy edges. Unsupported Postgres source/evidence/action/retry, read-only, merge, delta, and tenant-bound CLI surfaces now fail before driver or filesystem access; whitespace-wrapped DSNs cannot bypass routing. The parity harness requires two distinct roles plus disposable opt-in, uses UUID tenant namespaces, and performs no destructive cleanup. Closure matrix: 349 passed, 47 externally gated skips; direct surface/factory matrix: 57 passed. Full isolated non-ML gate: 3,552 passed, 69 skipped, 95 deselected, 22 intentional xfails, 2 warnings in 858.47s. Commit evidence is the commit containing this row. Rollback: keep the team profile disabled and revert this package; schema rollback requires a verified backup/forward repair. Repository work for R1.1-R1.2 is complete. Real two-role PostgreSQL evidence plus approved brownfield owner/duplicate/supersession-edge inventory/repair remain `BLOCKED-EXTERNAL`; the Team/Postgres profile remains blocked. | | MM-SEC-02 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | Qdrant can return archived, sensitive, cross-scope/tenant, or orphan payloads | R1.3,R2.1 | OPEN | Fake and real Qdrant authoritative-filter tests | | MM-OPS-01 | DevOps | Critical / EXPLOITABLE-LOW-EFFORT / H4 | Postgres Compose publishes a fixed credential | R1.5 | OPEN | Fail-closed Compose config, private port, external rotation evidence | | MM-SEC-03 | Security | High / EXPLOITABLE-NOW | Persisted metadata/provenance fields bypass sensitivity scanning | R1.4 | OPEN | Complete field-matrix adversarial suite and legacy dry-run | diff --git a/external-actions-required.md b/external-actions-required.md index 57204751..e2bfdfac 100644 --- a/external-actions-required.md +++ b/external-actions-required.md @@ -8,7 +8,7 @@ live mutation by itself. | Finding | Owner/system | Required action | Evidence needed to unblock | Review date | Status | |---|---|---|---|---|---| | MM-OPS-01 | Operator / any Postgres deployment | Rotate the historical `mm_pw` credential and recreate deployments that used it. Keep database ports private. | Rotation record plus network probe showing the port is not externally reachable | 2026-07-31 | PENDING-INVENTORY | -| MM-SEC-01 / disposable proof | Team Postgres test environment | Provide one fully disposable database, a dedicated schema-owning `SUPERUSER`/`BYPASSRLS` migrator DSN in `MEMORYMASTER_TEST_POSTGRES_DSN`, and a distinct restricted app DSN in `MEMORYMASTER_TEST_POSTGRES_APP_DSN`; set `MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE=1`. Neither DSN may target product/live data. | Run `tests/test_postgres_rls_integration.py` and retain exact FORCE-RLS policy/function/trigger/index catalog evidence; literal-sensitive fingerprints; event-head owner `SUPERUSER`/`BYPASSRLS`; app event SELECT/INSERT with no table/column UPDATE or DELETE; cross-tenant/scope/visibility/principal denials; tenant-chain linearity; scope-local identities and ambiguity rejection; self/cross-boundary supersession denial; successful reciprocal supersession plus one event; and rollback after replacement-update/event-insert failure. No real PostgreSQL proof has been performed here. | 2026-07-31 | BLOCKED-EXTERNAL | +| MM-SEC-01 / disposable proof | Team Postgres test environment | Provide one fully disposable database, a dedicated schema-owning `SUPERUSER`/`BYPASSRLS` migrator DSN in `MEMORYMASTER_TEST_POSTGRES_DSN`, and a distinct restricted app DSN in `MEMORYMASTER_TEST_POSTGRES_APP_DSN`; set `MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE=1`. Neither DSN may target product/live data. | Run `tests/test_postgres_rls_integration.py` plus every Postgres parameter in `tests/test_postgres_parity.py` and `tests/test_backend_parity.py`. Retain exact FORCE-RLS policy/function/trigger/index catalog evidence; literal-sensitive fingerprints; event-head owner `SUPERUSER`/`BYPASSRLS`; app event SELECT/INSERT with no table/column UPDATE or DELETE; cross-tenant/scope/visibility/principal denials; tenant-chain linearity; scope-local identities and ambiguity rejection; self/cross-boundary supersession denial; successful reciprocal supersession plus one event; rollback after replacement-update/event-insert failure; and parity results from UUID-isolated tenants with no destructive cleanup. No real PostgreSQL proof has been performed here. | 2026-07-31 | BLOCKED-EXTERNAL | | MM-SEC-01 / brownfield data | Live database operator | Authorize a read-only inventory before v0012. Any owner backfill, duplicate/supersession-edge repair, or grant change requires separate explicit approval, a verified backup, and a maintenance window. | Inventory noncanonical visibility; blank/null `source_agent` on every claim; identity duplicates; both supersession columns for self, missing-target, nonreciprocal, cross-tenant/scope/visibility/owner edges; current event/function/table grants; and constraint state without writing. After approval, retain repair/grant evidence and rerun v0012, which validates `ck_claims_identity_visibility_owner`. No product-data verification or mutation has been performed here. | 2026-07-31 | BLOCKED-EXTERNAL | | MM-OPS-02 | Docker/Helm runtime | Provide Docker and, for final verification, a disposable Kubernetes target if unavailable locally. | Container health/MCP handshake and Helm readiness/network-policy evidence | 2026-08-15 | PENDING-CAPABILITY-CHECK | | MM-SEC-02 | Qdrant runtime | Provide a disposable authenticated/TLS Qdrant target for final parity after fake-backed tests pass. | Real service policy/reconciliation test output | 2026-08-15 | PENDING-CAPABILITY-CHECK | diff --git a/memorymaster/bridges/db_merge.py b/memorymaster/bridges/db_merge.py index e7f1d4aa..ede12ee5 100644 --- a/memorymaster/bridges/db_merge.py +++ b/memorymaster/bridges/db_merge.py @@ -17,6 +17,7 @@ from pathlib import Path from memorymaster.stores._storage_shared import connect_ro, open_conn +from memorymaster.stores.store_factory import is_postgres_dsn logger = logging.getLogger(__name__) @@ -435,6 +436,11 @@ def merge_databases(target_db: str, source_db: str) -> dict[str, int]: Returns dict with: scanned, merged, skipped, errors """ + if is_postgres_dsn(str(target_db)) or is_postgres_dsn(str(source_db)): + raise ValueError( + "merge-db supports SQLite paths only; raw Postgres team merges are disabled." + ) + stats = {"scanned": 0, "merged": 0, "skipped": 0, "errors": 0} if not Path(source_db).exists(): diff --git a/memorymaster/bridges/delta_sync.py b/memorymaster/bridges/delta_sync.py index e3b5efd1..63bddc77 100644 --- a/memorymaster/bridges/delta_sync.py +++ b/memorymaster/bridges/delta_sync.py @@ -28,6 +28,7 @@ from pathlib import Path from memorymaster.stores._storage_shared import connect_ro, open_conn +from memorymaster.stores.store_factory import is_postgres_dsn logger = logging.getLogger(__name__) @@ -75,6 +76,11 @@ def export_delta( FileNotFoundError: source DB missing. ValueError: source DB lacks the expected tables. """ + if is_postgres_dsn(str(source_db)) or is_postgres_dsn(str(output_path)): + raise ValueError( + "export-delta supports SQLite paths only; raw Postgres team deltas are disabled." + ) + source_db = str(source_db) output_path = Path(output_path) if not Path(source_db).exists(): diff --git a/memorymaster/stores/postgres_store.py b/memorymaster/stores/postgres_store.py index 9b66c793..7e1f3a9a 100644 --- a/memorymaster/stores/postgres_store.py +++ b/memorymaster/stores/postgres_store.py @@ -1075,6 +1075,17 @@ def connect(self) -> Any: raise return conn + def connect_ro(self) -> Any: + raise PermissionError( + "connect_ro is a SQLite-only surface and is unavailable in Postgres team mode." + ) + + def _deny_unsupported_team_surface(self, surface: str) -> None: + raise PermissionError( + f"{surface} is unavailable in Postgres team mode until its tables " + "have tenant-scoped policy coverage." + ) + def _connect_schema_admin(self) -> Any: if self.require_tenant: raise PermissionError( @@ -3445,6 +3456,7 @@ def upsert_external_source( display_name: str, config_json: dict[str, object] | str | None = None, ) -> ExternalSource: + self._deny_unsupported_team_surface("upsert_external_source") _, _, Jsonb = self._load_psycopg() normalized_source_type = source_type.strip().lower() normalized_display_name = display_name.strip() @@ -3492,6 +3504,7 @@ def upsert_source_item( content_hash: str | None = None, sensitivity: str | None = None, ) -> SourceItem: + self._deny_unsupported_team_surface("upsert_source_item") from memorymaster.stores._storage_sources import _normalize_sensitivity _, _, Jsonb = self._load_psycopg() @@ -3575,6 +3588,7 @@ def upsert_source_item( return self._row_to_source_item(row) def get_source_item(self, *, source_id: int, source_item_id: str) -> SourceItem | None: + self._deny_unsupported_team_surface("get_source_item") normalized_source_item_id = source_item_id.strip() if source_id <= 0: raise ValueError("source_id must be positive.") @@ -3589,6 +3603,7 @@ def get_source_item(self, *, source_id: int, source_item_id: str) -> SourceItem return self._row_to_source_item(row) if row is not None else None def get_source_item_by_id(self, source_item_row_id: int) -> SourceItem | None: + self._deny_unsupported_team_surface("get_source_item_by_id") if source_item_row_id <= 0: raise ValueError("source_item_row_id must be positive.") with self.connect() as conn, conn.cursor() as cur: @@ -3608,6 +3623,7 @@ def add_evidence_item( payload_json: dict[str, object] | str | None = None, sensitivity: str | None = None, ) -> EvidenceItem: + self._deny_unsupported_team_surface("add_evidence_item") from memorymaster.stores._storage_sources import _normalize_sensitivity _, _, Jsonb = self._load_psycopg() @@ -3668,6 +3684,7 @@ def list_evidence_items( evidence_type: str | None = None, limit: int = 100, ) -> list[EvidenceItem]: + self._deny_unsupported_team_surface("list_evidence_items") clauses: list[str] = [] params: list[object] = [] if source_item_id is not None: @@ -3703,6 +3720,7 @@ def create_action_proposal( payload_json: dict[str, object] | str | None = None, idempotency_key: str | None = None, ) -> ActionProposal: + self._deny_unsupported_team_surface("create_action_proposal") _, _, Jsonb = self._load_psycopg() normalized_type = proposal_type.strip().lower() normalized_title = title.strip() @@ -3766,6 +3784,7 @@ def create_action_proposal( return self._row_to_action_proposal(row) def get_action_proposal_by_idempotency_key(self, idempotency_key: str) -> ActionProposal | None: + self._deny_unsupported_team_surface("get_action_proposal_by_idempotency_key") normalized = idempotency_key.strip() if not normalized: return None @@ -3783,6 +3802,7 @@ def update_action_proposal_status( exported_at: str | None = None, payload_json: dict[str, object] | str | None = None, ) -> ActionProposal: + self._deny_unsupported_team_surface("update_action_proposal_status") _, _, Jsonb = self._load_psycopg() normalized_status = status.strip().lower() if proposal_id <= 0: @@ -3840,6 +3860,7 @@ def set_source_item_sensitivity( source_item_row_id: int, sensitivity: str | None, ) -> SourceItem: + self._deny_unsupported_team_surface("set_source_item_sensitivity") from memorymaster.stores._storage_sources import _normalize_sensitivity if source_item_row_id <= 0: @@ -3876,6 +3897,7 @@ def set_evidence_item_sensitivity( evidence_item_row_id: int, sensitivity: str | None, ) -> EvidenceItem: + self._deny_unsupported_team_surface("set_evidence_item_sensitivity") from memorymaster.stores._storage_sources import _normalize_sensitivity if evidence_item_row_id <= 0: @@ -3942,6 +3964,7 @@ def enqueue_media_retry( status: str = "pending", next_attempt_time: str | None = None, ) -> MediaRetryItem: + self._deny_unsupported_team_surface("enqueue_media_retry") if source_item_id <= 0: raise ValueError("source_item_id must be positive.") normalized_key = (media_key or "").strip() @@ -4002,6 +4025,7 @@ def enqueue_media_retry( return self._row_to_media_retry(row) def claim_pending_media_retries(self, limit: int = 25) -> list[MediaRetryItem]: + self._deny_unsupported_team_surface("claim_pending_media_retries") if limit <= 0: return [] now = utc_now() @@ -4048,6 +4072,7 @@ def record_media_retry_outcome( last_error: str | None = None, next_attempt_time: str | None = None, ) -> MediaRetryItem: + self._deny_unsupported_team_surface("record_media_retry_outcome") if retry_id <= 0: raise ValueError("retry_id must be positive.") if status not in MEDIA_RETRY_STATUSES: @@ -4102,6 +4127,7 @@ def list_media_retries( source_item_id: int | None = None, limit: int = 100, ) -> list[MediaRetryItem]: + self._deny_unsupported_team_surface("list_media_retries") clauses: list[str] = [] params: list[object] = [] if status: @@ -4125,6 +4151,7 @@ def list_media_retries( return [self._row_to_media_retry(r) for r in rows] def media_retry_status_counts(self) -> dict[str, int]: + self._deny_unsupported_team_surface("media_retry_status_counts") with self.connect() as conn, conn.cursor() as cur: cur.execute("SELECT status, COUNT(*) AS n FROM media_retry_queue GROUP BY status") rows = cur.fetchall() @@ -4144,6 +4171,7 @@ def update_action_proposal_fields( payload_json: dict[str, object] | str | None = None, ) -> ActionProposal: """Postgres mirror of SQLite update_action_proposal_fields.""" + self._deny_unsupported_team_surface("update_action_proposal_fields") _, _, Jsonb = self._load_psycopg() if proposal_id <= 0: raise ValueError("proposal_id must be positive.") @@ -4220,6 +4248,7 @@ def list_action_proposals( destination: str | None = None, limit: int = 100, ) -> list[ActionProposal]: + self._deny_unsupported_team_surface("list_action_proposals") clauses: list[str] = [] params: list[object] = [] if status: diff --git a/memorymaster/stores/store_factory.py b/memorymaster/stores/store_factory.py index 6ad69164..7e774924 100644 --- a/memorymaster/stores/store_factory.py +++ b/memorymaster/stores/store_factory.py @@ -7,7 +7,7 @@ def is_postgres_dsn(value: str) -> bool: - lowered = value.lower() + lowered = value.strip().lower() return lowered.startswith("postgres://") or lowered.startswith("postgresql://") @@ -33,7 +33,7 @@ def create_store( from memorymaster.stores.postgres_store import PostgresStore return PostgresStore( - target, + target.strip(), tenant_id=tenant_id, require_tenant=require_tenant, principal=principal, diff --git a/memorymaster/surfaces/cli_handlers_curation.py b/memorymaster/surfaces/cli_handlers_curation.py index 706fa643..f0f1cf07 100644 --- a/memorymaster/surfaces/cli_handlers_curation.py +++ b/memorymaster/surfaces/cli_handlers_curation.py @@ -758,7 +758,18 @@ def _handle_observe(args: argparse.Namespace, service, parser: argparse.Argument def _handle_merge_db(args: argparse.Namespace, service, parser: argparse.ArgumentParser, effective_db: str) -> int: + from memorymaster.stores.store_factory import is_postgres_dsn + + if is_postgres_dsn(str(effective_db)): + msg = "merge-db is unavailable for Postgres tenant/team runtimes" + if args.json_output: + print(_json_error(msg)) + else: + print(f"error: {msg}") + return 2 + from memorymaster.bridges.db_merge import merge_databases + t0 = time.perf_counter() result = merge_databases(str(effective_db), args.source) elapsed_ms = (time.perf_counter() - t0) * 1000 diff --git a/tests/conftest.py b/tests/conftest.py index fa244214..1e5788ef 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,117 @@ from __future__ import annotations import os +import secrets +from dataclasses import dataclass +from functools import lru_cache from pathlib import Path +from typing import Any +from uuid import uuid4 import pytest _CASE_ROOT = Path(".tmp_cases") +_POSTGRES_ADMIN_DSN_ENV = "MEMORYMASTER_TEST_POSTGRES_DSN" +_POSTGRES_APP_DSN_ENV = "MEMORYMASTER_TEST_POSTGRES_APP_DSN" +_POSTGRES_DISPOSABLE_ENV = "MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE" +_LIVE_POSTGRES_DSN_ENVS = ("DATABASE_URL", "POSTGRES_DSN", "MEMORYMASTER_POSTGRES_DSN") + + +@dataclass(frozen=True) +class _DisposablePostgresRuntime: + admin_dsn: str + app_dsn: str + + +def _same_secret(left: str, right: str) -> bool: + return bool(left and right) and secrets.compare_digest(left, right) + + +def _require_disposable_postgres_runtime() -> _DisposablePostgresRuntime: + admin_dsn = os.getenv(_POSTGRES_ADMIN_DSN_ENV, "").strip() + app_dsn = os.getenv(_POSTGRES_APP_DSN_ENV, "").strip() + opted_in = os.getenv(_POSTGRES_DISPOSABLE_ENV, "").strip() == "1" + if not admin_dsn or not app_dsn or not opted_in: + pytest.skip( + "BLOCKED-EXTERNAL: Postgres parity requires distinct admin/app DSNs " + f"and {_POSTGRES_DISPOSABLE_ENV}=1" + ) + if _same_secret(admin_dsn, app_dsn): + pytest.fail("Postgres parity requires distinct admin and app DSNs.") + for env_name in _LIVE_POSTGRES_DSN_ENVS: + live_dsn = os.getenv(env_name, "").strip() + if _same_secret(admin_dsn, live_dsn) or _same_secret(app_dsn, live_dsn): + pytest.fail(f"Refusing to reuse {env_name} for disposable Postgres tests.") + return _DisposablePostgresRuntime(admin_dsn=admin_dsn, app_dsn=app_dsn) + + +def _database_identity(psycopg: Any, dsn: str) -> tuple[object, ...]: + with psycopg.connect(dsn, connect_timeout=5) as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT current_database(), current_user, rolsuper, rolbypassrls, + rolreplication, rolcreaterole, rolcreatedb + FROM pg_roles WHERE rolname = current_user + """ + ) + row = cur.fetchone() + if row is None: + raise RuntimeError("Postgres connection identity could not be verified.") + return tuple(row) + + +def _validate_disposable_postgres_roles( + admin_identity: tuple[object, ...], + app_identity: tuple[object, ...], +) -> None: + if str(admin_identity[0]) != str(app_identity[0]): + pytest.fail("Admin and app DSNs must target the same disposable database.") + if str(admin_identity[1]) == str(app_identity[1]): + pytest.fail("Admin and app DSNs must authenticate as distinct roles.") + if not bool(admin_identity[2]) and not bool(admin_identity[3]): + pytest.fail("The Postgres migrator must be SUPERUSER or BYPASSRLS.") + if any(bool(value) for value in app_identity[2:]): + pytest.fail("The Postgres app role has a forbidden role attribute.") + + +def _grant_disposable_event_contract(psycopg: Any, admin_dsn: str, app_role: str) -> None: + from psycopg import sql + + role = sql.Identifier(app_role) + with psycopg.connect(admin_dsn) as conn, conn.cursor() as cur: + cur.execute(sql.SQL("GRANT SELECT, INSERT ON TABLE public.events TO {}").format(role)) + cur.execute( + sql.SQL( + "REVOKE UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER " + "ON TABLE public.events FROM {}" + ).format(role) + ) + cur.execute( + sql.SQL( + "GRANT EXECUTE ON FUNCTION public.memorymaster_event_chain_head() TO {}" + ).format(role) + ) + + +@lru_cache(maxsize=4) +def _initialize_disposable_postgres(admin_dsn: str, app_dsn: str) -> None: + try: + import psycopg + except ImportError: + pytest.skip("BLOCKED-EXTERNAL: psycopg is required for Postgres parity tests") + + try: + admin_identity = _database_identity(psycopg, admin_dsn) + app_identity = _database_identity(psycopg, app_dsn) + except psycopg.OperationalError: + pytest.skip("BLOCKED-EXTERNAL: configured Postgres test DSNs are unreachable") + _validate_disposable_postgres_roles(admin_identity, app_identity) + + from memorymaster.stores.postgres_store import PostgresStore + + PostgresStore(admin_dsn).init_db() + _grant_disposable_event_contract(psycopg, admin_dsn, str(app_identity[1])) @pytest.fixture(autouse=True) @@ -22,12 +127,8 @@ def _explicit_local_mcp_auth(monkeypatch) -> None: # `parametrize_backends` yields a fresh MemoryService on each backend so the # SAME test body runs against both SQLite and Postgres and must produce the # same observable result. SQLite always runs (file-based, no server). Postgres -# runs only when MEMORYMASTER_TEST_POSTGRES_DSN is set; otherwise that -# parametrization is skipped so dev machines without a Postgres stay green. - -def _pg_dsn() -> str | None: - return os.getenv("MEMORYMASTER_TEST_POSTGRES_DSN") - +# runs only with distinct admin/app DSNs plus explicit disposable opt-in; +# otherwise that parametrization is skipped so offline dev machines stay green. def _fresh_sqlite_service(tmp_path): from memorymaster.core.service import MemoryService @@ -41,24 +142,17 @@ def _fresh_sqlite_service(tmp_path): def _fresh_postgres_service(): from memorymaster.core.service import MemoryService - dsn = _pg_dsn() - if not dsn: - pytest.skip("MEMORYMASTER_TEST_POSTGRES_DSN is not set") - svc = MemoryService(dsn, workspace_root=".") - svc.init_db() - # Deterministic start: clear claims/citations/events (+ optional tables). - with svc.store.connect() as conn: - with conn.cursor() as cur: - for tbl in ("claim_links", "claim_embeddings"): - cur.execute("SELECT to_regclass(%s) AS t", (f"public.{tbl}",)) - row = cur.fetchone() - present = (row["t"] if isinstance(row, dict) else row[0]) is not None - if present: - cur.execute(f"DELETE FROM {tbl}") - cur.execute("DELETE FROM citations") - cur.execute("DELETE FROM events") - cur.execute("DELETE FROM claims") - return svc + config = _require_disposable_postgres_runtime() + _initialize_disposable_postgres(config.admin_dsn, config.app_dsn) + run_id = uuid4().hex + return MemoryService( + config.app_dsn, + workspace_root=".", + tenant_id=f"parity-{run_id}", + require_tenant=True, + principal="parity-test", + allowed_scopes=("project",), + ) @pytest.fixture( diff --git a/tests/test_backend_parity.py b/tests/test_backend_parity.py index 168e1bff..28d601fa 100644 --- a/tests/test_backend_parity.py +++ b/tests/test_backend_parity.py @@ -12,8 +12,6 @@ """ from __future__ import annotations -import pytest - from memorymaster.core.models import CitationInput @@ -111,11 +109,13 @@ def test_parity_idempotency_key(parametrize_backends): "parity idem first", [CitationInput(source="s", locator="l")], idempotency_key="parity-idem-1", + source_agent=getattr(svc, "principal", None), ) c2 = svc.store.create_claim( "parity idem second different text", [CitationInput(source="s", locator="l")], idempotency_key="parity-idem-1", + source_agent=getattr(svc, "principal", None), ) assert c1.id == c2.id, f"{backend}: idempotency_key did not dedup" diff --git a/tests/test_postgres_parity.py b/tests/test_postgres_parity.py index b6509105..76d4ccd3 100644 --- a/tests/test_postgres_parity.py +++ b/tests/test_postgres_parity.py @@ -1,13 +1,11 @@ """Postgres parity tests. -These tests validate that PostgresStore matches SQLiteStore behaviour. -When MEMORYMASTER_TEST_POSTGRES_DSN is set they run against a real database; -otherwise they are skipped. +These tests validate that PostgresStore matches SQLiteStore behaviour. They run +only under the same explicit two-role disposable contract as the RLS suite. """ from __future__ import annotations import json -import os import pytest @@ -15,35 +13,10 @@ from memorymaster.core.service import MemoryService -def _pg_dsn() -> str | None: - return os.getenv("MEMORYMASTER_TEST_POSTGRES_DSN") - - def _make_pg_service() -> MemoryService: - dsn = _pg_dsn() - if not dsn: - pytest.skip("MEMORYMASTER_TEST_POSTGRES_DSN is not set") - service = MemoryService(dsn, workspace_root=".") - service.init_db() - _cleanup_tables(service) - return service - - -def _cleanup_tables(service: MemoryService) -> None: - """Best-effort cleanup for deterministic runs.""" - with service.store.connect() as conn: - with conn.cursor() as cur: - cur.execute("SELECT to_regclass('public.claim_links') AS tbl") - links_tbl = cur.fetchone() - if links_tbl and links_tbl["tbl"] is not None: - cur.execute("DELETE FROM claim_links") - cur.execute("DELETE FROM citations") - cur.execute("SELECT to_regclass('public.claim_embeddings') AS tbl") - emb_tbl = cur.fetchone() - if emb_tbl and emb_tbl["tbl"] is not None: - cur.execute("DELETE FROM claim_embeddings") - cur.execute("DELETE FROM events") - cur.execute("DELETE FROM claims") + from conftest import _fresh_postgres_service + + return _fresh_postgres_service() def _ingest(service: MemoryService, text: str, **kwargs) -> int: @@ -77,11 +50,13 @@ def test_idempotency_key(self): "Claim A", [CitationInput(source="s", locator="l")], idempotency_key="key-1", + source_agent=svc.principal, ) c2 = svc.store.create_claim( "Claim B different text", [CitationInput(source="s", locator="l")], idempotency_key="key-1", + source_agent=svc.principal, ) assert c1.id == c2.id @@ -91,6 +66,7 @@ def test_get_claim_by_idempotency_key(self): "Idem claim", [CitationInput(source="s")], idempotency_key="idem-pg-1", + source_agent=svc.principal, ) found = svc.store.get_claim_by_idempotency_key("idem-pg-1") assert found is not None @@ -481,13 +457,7 @@ def test_recompute_tiers_promotes_fresh_claim_to_core(self): @pytest.mark.postgres def test_postgres_smoke_parity(): - dsn = _pg_dsn() - if not dsn: - pytest.skip("MEMORYMASTER_TEST_POSTGRES_DSN is not set") - - service = MemoryService(dsn, workspace_root=".") - service.init_db() - _cleanup_tables(service) + service = _make_pg_service() service.ingest( text="Server IP is 10.0.0.1", @@ -518,18 +488,13 @@ def test_postgres_smoke_parity(): @pytest.mark.postgres def test_postgres_init_db_idempotent(): - dsn = _pg_dsn() - if not dsn: - pytest.skip("MEMORYMASTER_TEST_POSTGRES_DSN is not set") - - service = MemoryService(dsn, workspace_root=".") - service.init_db() - # Second init should not fail - service.init_db() - - cid = _ingest(service, "Idempotent init claim") - claim = service.store.get_claim(cid) - assert claim is not None + from conftest import _require_disposable_postgres_runtime + from memorymaster.stores.postgres_store import PostgresStore + + config = _require_disposable_postgres_runtime() + store = PostgresStore(config.admin_dsn) + store.init_db() + store.init_db() # --------------------------------------------------------------------------- diff --git a/tests/test_postgres_team_surface_denials.py b/tests/test_postgres_team_surface_denials.py new file mode 100644 index 00000000..edc81dcd --- /dev/null +++ b/tests/test_postgres_team_surface_denials.py @@ -0,0 +1,324 @@ +"""Fail-closed boundaries for SQLite-only surfaces in Postgres team mode.""" + +from __future__ import annotations + +from collections.abc import Callable +from types import SimpleNamespace +from typing import Any + +import pytest + +from memorymaster.bridges import db_merge, delta_sync +from memorymaster.stores._storage_sources import _SourceItemsMixin +from memorymaster.stores.postgres_store import PostgresStore +from memorymaster.stores.store_factory import create_store + + +TEAM_ONLY_METHOD_CALLS: tuple[ + tuple[str, tuple[object, ...], dict[str, object]], ... +] = ( + ( + "upsert_external_source", + (), + {"source_type": "whatsapp", "display_name": "primary"}, + ), + ( + "upsert_source_item", + (), + {"source_id": 1, "source_item_id": "message-1", "item_type": "text"}, + ), + ("get_source_item", (), {"source_id": 1, "source_item_id": "message-1"}), + ("get_source_item_by_id", (1,), {}), + ( + "add_evidence_item", + (), + {"source_item_id": 1, "evidence_type": "transcript", "text": "evidence"}, + ), + ("list_evidence_items", (), {"source_item_id": 1}), + ( + "create_action_proposal", + (), + {"proposal_type": "task", "title": "Review evidence"}, + ), + ("get_action_proposal_by_idempotency_key", ("proposal-1",), {}), + ("update_action_proposal_status", (1,), {"status": "approved"}), + ("set_source_item_sensitivity", (1, "low"), {}), + ("set_evidence_item_sensitivity", (1, "low"), {}), + ( + "enqueue_media_retry", + (), + {"source_item_id": 1, "media_key": "media-1", "status": "pending"}, + ), + ("claim_pending_media_retries", (1,), {}), + ("record_media_retry_outcome", (1,), {"status": "failed"}), + ("list_media_retries", (), {"status": "pending"}), + ("media_retry_status_counts", (), {}), + ("update_action_proposal_fields", (1,), {"title": "Revised title"}), + ("list_action_proposals", (), {"status": "candidate"}), +) + + +@pytest.fixture +def team_store(monkeypatch: pytest.MonkeyPatch) -> PostgresStore: + store = PostgresStore( + "postgresql://runtime.invalid/memorymaster", + tenant_id="tenant-a", + require_tenant=True, + principal="agent-a", + allowed_scopes={"project:alpha"}, + ) + + def forbidden_backend_access(*_args: object, **_kwargs: object) -> None: + pytest.fail("team-denied surface attempted Postgres backend access") + + monkeypatch.setattr(store, "_load_psycopg", forbidden_backend_access) + monkeypatch.setattr(store, "connect", forbidden_backend_access) + return store + + +@pytest.mark.parametrize( + ("method_name", "args", "kwargs"), + TEAM_ONLY_METHOD_CALLS, + ids=[entry[0] for entry in TEAM_ONLY_METHOD_CALLS], +) +def test_team_denied_source_surfaces_fail_before_backend_access( + team_store: PostgresStore, + method_name: str, + args: tuple[object, ...], + kwargs: dict[str, object], +) -> None: + method = getattr(team_store, method_name) + + with pytest.raises(PermissionError, match="(?i)team"): + method(*args, **kwargs) + + +@pytest.mark.parametrize( + ("method_name", "args", "kwargs"), + TEAM_ONLY_METHOD_CALLS, + ids=[f"unbound-{entry[0]}" for entry in TEAM_ONLY_METHOD_CALLS], +) +def test_unbound_postgres_denied_surfaces_cannot_return_early( + monkeypatch: pytest.MonkeyPatch, + method_name: str, + args: tuple[object, ...], + kwargs: dict[str, object], +) -> None: + store = PostgresStore("postgresql://runtime.invalid/memorymaster") + + def forbidden_backend_access(*_args: object, **_kwargs: object) -> None: + pytest.fail("unsupported Postgres surface attempted backend access") + + monkeypatch.setattr(store, "_load_psycopg", forbidden_backend_access) + monkeypatch.setattr(store, "connect", forbidden_backend_access) + + with pytest.raises(PermissionError, match="(?i)(postgres|team)"): + getattr(store, method_name)(*args, **kwargs) + + +def test_team_connect_ro_fails_before_driver_or_connection_access( + team_store: PostgresStore, +) -> None: + with pytest.raises(PermissionError, match="(?i)(connect_ro|read.only|team)"): + team_store.connect_ro() + + +def test_store_factory_normalizes_whitespace_around_postgres_dsn() -> None: + store = create_store(" postgresql://runtime.invalid/memorymaster\t") + + assert isinstance(store, PostgresStore) + assert store.dsn == "postgresql://runtime.invalid/memorymaster" + + +def _forbid_bridge_io(monkeypatch: pytest.MonkeyPatch, module: Any) -> None: + def forbidden(*_args: object, **_kwargs: object) -> None: + pytest.fail("Postgres DSN reached bridge filesystem or database access") + + monkeypatch.setattr(module, "Path", forbidden) + for name in ("connect_ro", "open_conn", "_open_target"): + if hasattr(module, name): + monkeypatch.setattr(module, name, forbidden) + + +def _assert_clear_postgres_rejection(action: Callable[[], object]) -> None: + with pytest.raises((ValueError, PermissionError)) as exc_info: + action() + + message = str(exc_info.value).lower() + assert "postgres" in message or "sqlite" in message + + +@pytest.mark.parametrize( + ("target_db", "source_db"), + ( + ("postgresql://runtime.invalid/memorymaster", "source.db"), + ("target.db", "postgres://runtime.invalid/memorymaster"), + (" postgresql://runtime.invalid/memorymaster", "source.db"), + ("target.db", "\tpostgres://runtime.invalid/memorymaster"), + ), + ids=( + "postgres-target", + "postgres-source", + "whitespace-postgres-target", + "whitespace-postgres-source", + ), +) +def test_db_merge_rejects_postgres_dsn_before_io( + monkeypatch: pytest.MonkeyPatch, + target_db: str, + source_db: str, +) -> None: + _forbid_bridge_io(monkeypatch, db_merge) + + _assert_clear_postgres_rejection( + lambda: db_merge.merge_databases(target_db, source_db) + ) + + +@pytest.mark.parametrize( + ("source_db", "output_path"), + ( + ("postgresql://runtime.invalid/memorymaster", "delta.db"), + ("source.db", "postgres://runtime.invalid/delta"), + (" postgresql://runtime.invalid/memorymaster", "delta.db"), + ("source.db", "\tpostgres://runtime.invalid/delta"), + ), + ids=( + "postgres-source", + "postgres-output", + "whitespace-postgres-source", + "whitespace-postgres-output", + ), +) +def test_delta_export_rejects_postgres_dsn_before_io( + monkeypatch: pytest.MonkeyPatch, + source_db: str, + output_path: str, +) -> None: + _forbid_bridge_io(monkeypatch, delta_sync) + + _assert_clear_postgres_rejection( + lambda: delta_sync.export_delta(source_db, "", output_path) + ) + + +def test_cli_rejects_raw_merge_for_tenant_bound_postgres( + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + from memorymaster.surfaces import cli + + def forbidden_merge(*_args: object, **_kwargs: object) -> None: + pytest.fail("tenant-bound CLI reached the raw merge bridge") + + monkeypatch.setattr(cli, "MemoryService", lambda *_args, **_kwargs: object()) + monkeypatch.setattr(db_merge, "merge_databases", forbidden_merge) + + exit_code = cli.main( + [ + "--db", + "postgresql://runtime.invalid/memorymaster", + "--tenant", + "tenant-a", + "merge-db", + "--source", + "source.db", + ] + ) + + assert exit_code == 2 + output = capsys.readouterr().out.lower() + assert "merge" in output + assert "tenant" in output or "team" in output + + +def test_method_matrix_covers_the_audited_team_denied_surface() -> None: + expected = { + name + for name, member in vars(_SourceItemsMixin).items() + if callable(member) and not name.startswith("_") + } + + assert {entry[0] for entry in TEAM_ONLY_METHOD_CALLS} == expected + + +def test_disposable_postgres_contract_requires_both_roles_and_opt_in( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import conftest + + require_contract = getattr(conftest, "_require_disposable_postgres_runtime") + for name in ( + "MEMORYMASTER_TEST_POSTGRES_DSN", + "MEMORYMASTER_TEST_POSTGRES_APP_DSN", + "MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE", + ): + monkeypatch.delenv(name, raising=False) + + monkeypatch.setenv("MEMORYMASTER_TEST_POSTGRES_DSN", "postgresql://admin/db") + with pytest.raises(pytest.skip.Exception): + require_contract() + + monkeypatch.setenv("MEMORYMASTER_TEST_POSTGRES_APP_DSN", "postgresql://app/db") + with pytest.raises(pytest.skip.Exception): + require_contract() + + monkeypatch.setenv("MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE", "1") + config = require_contract() + assert config.admin_dsn == "postgresql://admin/db" + assert config.app_dsn == "postgresql://app/db" + + +def test_fresh_postgres_service_uses_bound_app_role_without_cleanup( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import conftest + import memorymaster.core.service as service_module + import memorymaster.stores.postgres_store as postgres_module + + calls: list[tuple[str, object]] = [] + config = SimpleNamespace( + admin_dsn="postgresql://admin/db", + app_dsn="postgresql://app/db", + ) + + class FakeAdminStore: + def __init__(self, dsn: str) -> None: + calls.append(("admin-dsn", dsn)) + + def init_db(self) -> None: + calls.append(("admin-init", True)) + + class FakeService: + def __init__(self, db_target: str, **kwargs: object) -> None: + calls.append(("app-dsn", db_target)) + calls.append(("app-authority", kwargs)) + + def init_db(self) -> None: + pytest.fail("team runtime attempted schema initialization") + + monkeypatch.setattr( + conftest, + "_require_disposable_postgres_runtime", + lambda: config, + ) + monkeypatch.setattr( + conftest, + "_initialize_disposable_postgres", + lambda admin_dsn, _app_dsn: FakeAdminStore(admin_dsn).init_db(), + raising=False, + ) + monkeypatch.setattr(postgres_module, "PostgresStore", FakeAdminStore) + monkeypatch.setattr(service_module, "MemoryService", FakeService) + + service = conftest._fresh_postgres_service() + + assert isinstance(service, FakeService) + assert ("admin-dsn", config.admin_dsn) in calls + assert ("admin-init", True) in calls + assert ("app-dsn", config.app_dsn) in calls + authority = dict(next(value for name, value in calls if name == "app-authority")) + assert authority["require_tenant"] is True + assert authority["tenant_id"] + assert authority["principal"] + assert authority["allowed_scopes"] == ("project",) From eec27ce5a7070796051a707c5dd102f9f62df4b0 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 08:26:17 -0300 Subject: [PATCH 13/55] fix(qdrant): quarantine unauthoritative retrieval --- .planning/PHASE0-RED-TEST-MATRIX.md | 2 +- ...EMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md | 2 +- .planning/audit-remediation-ledger.md | 2 +- INSTALLATION.md | 42 ++- README.md | 22 +- ROADMAP.md | 16 +- docs/AGENT-INSTALL.md | 28 +- docs/MCP-TOOLS.md | 15 +- docs/architecture.md | 25 +- docs/cli-cookbook.md | 16 +- docs/cross-project-patterns.md | 28 +- docs/env-reference.md | 15 + docs/handbook.md | 21 +- ...etrieval-governance-upgrades-2026-06-24.md | 6 + external-actions-required.md | 2 +- memorymaster/recall/context_hook.py | 113 +------ memorymaster/recall/qdrant_backend.py | 55 +--- memorymaster/recall/qdrant_recall_fallback.py | 104 +------ memorymaster/recall/verbatim_store.py | 84 +---- memorymaster/surfaces/cli.py | 18 +- memorymaster/surfaces/cli_handlers_basic.py | 67 ++-- memorymaster/surfaces/mcp_server.py | 130 ++++---- memorymaster/surfaces/setup_detect.py | 3 +- memorymaster/surfaces/setup_hooks.py | 10 +- monitoring.md | 10 +- tests/test_qdrant_authoritative_filtering.py | 21 +- tests/test_qdrant_backend.py | 36 ++- tests/test_qdrant_retrieval_quarantine.py | 293 ++++++++++++++++++ tests/test_recall_vector_fallback.py | 198 ++++++------ tests/test_verbatim_qdrant_quarantine.py | 144 +++++++++ tests/test_verbatim_store.py | 23 +- tests/test_verbatim_store_qdrant.py | 30 +- 32 files changed, 940 insertions(+), 641 deletions(-) create mode 100644 tests/test_qdrant_retrieval_quarantine.py create mode 100644 tests/test_verbatim_qdrant_quarantine.py diff --git a/.planning/PHASE0-RED-TEST-MATRIX.md b/.planning/PHASE0-RED-TEST-MATRIX.md index e7d1b031..05da95d7 100644 --- a/.planning/PHASE0-RED-TEST-MATRIX.md +++ b/.planning/PHASE0-RED-TEST-MATRIX.md @@ -17,7 +17,7 @@ All tests use temporary databases, isolated environment variables, fake provider | Finding | Test file | Required red tests | External dependency | |---|---|---|---| | MM-SEC-01 | `tests/test_mcp_authorization_boundary.py` | reader cannot ingest with spoofed source; unknown team principal fails closed; scope allowlist cannot expand context; list/query/pin/redact cannot cross project/tenant; every MCP tool declares an action | Real Postgres/RLS subset requires DSN | -| MM-SEC-02 | `tests/test_qdrant_authoritative_filtering.py` | never return orphan payload; filter archived/candidate/stale/conflicted/wrong-scope/wrong-tenant/sensitive/private; payload cannot override DB; safe fallback; equal-count/different-ID reconcile | Real authenticated Qdrant final parity | +| MM-SEC-02 | `tests/test_qdrant_authoritative_filtering.py`; `tests/test_qdrant_retrieval_quarantine.py`; `tests/test_verbatim_qdrant_quarantine.py` | never return orphan payload; filter archived/candidate/stale/conflicted/wrong-scope/wrong-tenant/sensitive/private; payload cannot override DB; safe lexical/FTS fallback; direct adapters and disabled CLI fail before model/network/backend access; equal-count/different-ID reconcile | Real authenticated/TLS Qdrant final parity (`BLOCKED-EXTERNAL`) | | MM-SEC-03 | `tests/test_persisted_envelope_sensitivity.py` | plain and encoded secret matrix over every claim/citation/provenance field; legacy sensitive metadata hidden from list/query/export/Qdrant | None | | MM-SEC-04 | `tests/test_write_gateway_paths.py` | compact-summary, steward existing-row update, verbatim/spool/Atlas/miner/import paths reject secret fixture | None | | MM-ARCH-01 | `tests/test_entity_schema_composition.py` | normal init then graph schema; registry-first extract/stats/related; read tools issue no DDL | None | diff --git a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md index 5f256c19..53e093d8 100644 --- a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md +++ b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md @@ -84,7 +84,7 @@ Phase 0 must replace qualitative capacity goals with recorded numeric gates in t - [x] Phase 0 — evidence, numerical gates, and red tests - [x] R1.1 — authenticated request context and MCP authorization - [x] R1.2 — Postgres tenant enforcement and RLS (repository complete; external proof blocked) -- [ ] R1.3 — immediate Qdrant containment +- [x] R1.3 — immediate Qdrant containment - [ ] R1.4 — complete sensitivity/write gateway and legacy inventory - [ ] R1.5 — secure deployment and supply-chain defaults - [ ] R2.1 — unified RetrievalPlanner and governed Qdrant reintegration diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 328f4914..294bdbfa 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -13,7 +13,7 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | ID | Primary domain | Severity / exploitability | Summary | Package | Status | Acceptance evidence | |---|---|---|---|---|---|---| | MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | BLOCKED-EXTERNAL | Owner: Codex remediation branch. R1.1 complete. PostgreSQL application connections are team-only and bind tenant, principal, and immutable scopes transaction-locally; schema work requires a distinct verified migrator. v0011 FORCE RLS defines restricted role/table/policy and append-only event contracts, a tenant-derived hash-only event-head function, and deny-only governance/raw tables. v0012 defines six partial unique indexes: public keys are tenant + exact-scope local; non-public keys additionally include exact visibility/principal; ambiguity without exact scope fails closed; every team claim requires a nonblank owner. Startup validates literal-sensitive policy/index/function fingerprints, exact event and claims trigger inventories, required event SELECT/INSERT plus forbidden table/column UPDATE and DELETE, a privileged event-head owner, and the strict validated owner constraint. Supersession rejects self/cross-tenant/scope/visibility/owner references and the canonical path atomically commits reciprocal pointers plus one event; v0012 preflights unsafe legacy edges. Unsupported Postgres source/evidence/action/retry, read-only, merge, delta, and tenant-bound CLI surfaces now fail before driver or filesystem access; whitespace-wrapped DSNs cannot bypass routing. The parity harness requires two distinct roles plus disposable opt-in, uses UUID tenant namespaces, and performs no destructive cleanup. Closure matrix: 349 passed, 47 externally gated skips; direct surface/factory matrix: 57 passed. Full isolated non-ML gate: 3,552 passed, 69 skipped, 95 deselected, 22 intentional xfails, 2 warnings in 858.47s. Commit evidence is the commit containing this row. Rollback: keep the team profile disabled and revert this package; schema rollback requires a verified backup/forward repair. Repository work for R1.1-R1.2 is complete. Real two-role PostgreSQL evidence plus approved brownfield owner/duplicate/supersession-edge inventory/repair remain `BLOCKED-EXTERNAL`; the Team/Postgres profile remains blocked. | -| MM-SEC-02 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | Qdrant can return archived, sensitive, cross-scope/tenant, or orphan payloads | R1.3,R2.1 | OPEN | Fake and real Qdrant authoritative-filter tests | +| MM-SEC-02 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | Qdrant can return archived, sensitive, cross-scope/tenant, or orphan payloads | R1.3,R2.1 | IN-PROGRESS | Owner: Codex remediation branch. R1.3 containment is repository-complete: local-trusted claim requests and auto-classified Qdrant recommendations fall back to authoritative lexical retrieval with requested/classified/effective metadata; team semantic requests are denied before tool dispatch; prompt-context fallback is disconnected; verbatim vector/hybrid requests use FTS5; CLI denial occurs before service/backend construction; and every direct claim/verbatim/fallback read adapter raises before model, network, or raw payload access. Qdrant upsert/sync/reconcile/count-ID maintenance remains available. Adversarial containment: 18 passed; Qdrant/verbatim/classifier matrix: 153 passed; CLI/setup/MCP regression matrix: 182 passed; explicit ML gate: 14 passed. Full isolated non-ML gate: 3,572 passed, 69 skipped, 95 deselected, 20 intentional xfails, 2 warnings in 882.06s; collection: 3,756; Ruff: clean. Independent blocker-only review found no first-party raw payload-search path. Commit evidence is the commit containing this row. Rollback: keep semantic retrieval disabled and revert this package. R2.1 governed ID-candidate rehydration remains unimplemented, and final authenticated/TLS Qdrant parity is `BLOCKED-EXTERNAL`; do not enable the semantic profile. | | MM-OPS-01 | DevOps | Critical / EXPLOITABLE-LOW-EFFORT / H4 | Postgres Compose publishes a fixed credential | R1.5 | OPEN | Fail-closed Compose config, private port, external rotation evidence | | MM-SEC-03 | Security | High / EXPLOITABLE-NOW | Persisted metadata/provenance fields bypass sensitivity scanning | R1.4 | OPEN | Complete field-matrix adversarial suite and legacy dry-run | | MM-SEC-04 | Security | High / EXPLOITABLE-NOW | Steward, compact-summary, verbatim, and integration writes bypass one gateway | R1.4 | OPEN | Table-driven write-path test; no raw fixture in durable scan | diff --git a/INSTALLATION.md b/INSTALLATION.md index 2b6ab652..dece6988 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -28,7 +28,7 @@ pip install "memorymaster[embeddings]" # Gemini embeddings / LLM pip install "memorymaster[gemini]" -# Qdrant vector store +# Qdrant maintenance-index client (payload retrieval is quarantined in R1.3) pip install "memorymaster[qdrant]" # Fernet encryption for sensitive payloads @@ -82,8 +82,8 @@ a 3-line shim that calls the same `memorymaster.surfaces.setup_hooks:main` funct | `--api-key KEY` | prompted | API key for the chosen provider | | `--model MODEL` | provider default | LLM model id | | `--project-root PATH` | cwd | Directory where `memorymaster.db` lives | -| `--full-stack` | on | Bring up Qdrant + Ollama via Docker Compose | -| `--no-full-stack` | off | Skip the vector + local-LLM stack | +| `--full-stack` | on | Bring up the Qdrant maintenance index + Ollama via Docker Compose | +| `--no-full-stack` | off | Skip the Qdrant-index + local-LLM stack | | `--no-cron` | off | Skip steward cron setup | | `--no-obsidian-skills` | off | Skip Obsidian skills install | | `--codex` | auto-detect | Force Codex MCP + instructions wiring | @@ -107,18 +107,20 @@ If Docker is absent and Qdrant/Ollama are not already running, the installer continues without them and prints: ``` -Running in SQLite-only mode. Vector recall + local LLM auto-ingest are OFF. -To enable them: install Docker and re-run with --full-stack, or point -QDRANT_URL / OLLAMA_URL at existing services. +Running in SQLite-only mode. Qdrant index maintenance + local LLM auto-ingest are OFF. + Retrieval remains available through authoritative SQLite ranking. To enable index + maintenance or local LLMs, use --full-stack or QDRANT_URL / OLLAMA_URL. ``` Setup exits 0. Core hooks, MCP, and SQLite-based recall all work normally in -degraded mode. Add vector search later by installing Docker and re-running -`memorymaster-setup --full-stack --yes`. +degraded mode. Installing or starting Qdrant enables only index maintenance +during R1.3; it does not re-enable claim, context-fallback, or verbatim payload +retrieval. ## Docker Compose -The included `docker-compose.yml` runs the full stack: MemoryMaster + Qdrant + Ollama. +The included `docker-compose.yml` runs MemoryMaster plus the optional Qdrant +maintenance index and Ollama. Qdrant payload retrieval remains quarantined. ```bash # Clone the repo @@ -138,7 +140,7 @@ curl http://localhost:8765/health | Service | Port | Description | |---------|------|-------------| | `memorymaster` | 8765 | MCP server + dashboard | -| `qdrant` | 6333, 6334 | Vector store (REST + gRPC) | +| `qdrant` | 6333, 6334 | Maintenance index (REST + gRPC); no claim/verbatim payload retrieval in R1.3 | | `ollama` | 11434 | Local LLM inference | ### Postgres variant @@ -358,9 +360,14 @@ owner backfill, duplicate remediation, and constraint validation are separately blocked pending explicit operator approval and are recorded in `external-actions-required.md`. -### With Qdrant MCP server +### Standalone Qdrant MCP server (not a MemoryMaster retrieval path) -For direct vector search alongside MemoryMaster: +The example below exposes a separate, third-party Qdrant MCP server. It is not +a supported way to query MemoryMaster's index during R1.3: doing so would bypass +MemoryMaster lifecycle, tenant, scope, visibility, and sensitivity policy. Do +not point it at a MemoryMaster collection. Keep MemoryMaster queries on the +`memorymaster` MCP server, where Qdrant claim requests use lexical fallback and +team semantic requests are denied. ```json { @@ -400,7 +407,7 @@ All environment variables are documented in [`.env.example`](.env.example). Key | `MEMORYMASTER_MCP_ALLOWED_SCOPES` | (none) | Required explicit comma-separated team scope allowlist | | `MEMORYMASTER_MCP_DB` | (none) | Restricted application DSN/path for team mode | | `OLLAMA_URL` | `http://localhost:11434` | Ollama LLM endpoint | -| `QDRANT_URL` | (none) | Qdrant vector store endpoint | +| `QDRANT_URL` | (none) | Qdrant maintenance-index endpoint for upsert/sync/reconcile; does not enable payload retrieval | | `GEMINI_API_KEY` | (none) | Google Gemini API key | | `MEMORYMASTER_API_KEYS` | (none) | Comma-separated LLM API keys | | `EXTRACTOR_LLM_MODEL` | `llama3.2` | Model for claim extraction | @@ -439,6 +446,9 @@ python -c "import memorymaster; print('OK')" ### Qdrant connection refused +This matters only for upsert, sync, reconcile, count/ID drift checks, and other +index maintenance; authoritative retrieval continues without Qdrant. + 1. Verify Qdrant is running: `curl http://localhost:6333/healthz` 2. Check the `QDRANT_URL` environment variable 3. If using Docker Compose, ensure the `qdrant` service is healthy @@ -466,9 +476,9 @@ pip install -e ".[dev,mcp,security]" # Run tests pytest tests/ -q -# If you want to additionally exercise the optional embeddings and -# Qdrant code paths (which are skipped via pytest.importorskip when -# the deps are absent), install their extras too: +# If you want to additionally exercise optional embeddings, Qdrant +# maintenance, and payload-read containment paths (which are skipped via +# pytest.importorskip when dependencies are absent), install their extras too: pip install -e ".[dev,mcp,security,embeddings,qdrant]" ``` diff --git a/README.md b/README.md index 78a1f8da..79357e02 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ If you want an agent that recalls more, any vector store works. If you want an a ## Architecture MemoryMaster is layered around MCP/CLI entry points, the `MemoryService` facade, SQLite/Postgres -storage, optional Qdrant vector search, scheduled jobs, and an **optional** Obsidian wiki/vault +storage, an optional Qdrant index, scheduled jobs, and an **optional** Obsidian wiki/vault layer (opt-in, off by default — see below). The canonical ingest path is: ```text @@ -43,9 +43,19 @@ MCP/CLI -> sensitivity filter -> MemoryService.ingest -> store write -> FTS5 ind The query path is: ```text -query_memory -> MemoryService.query -> storage reads + optional Qdrant candidates -> ranked context +query_memory -> MemoryService.query -> authorized SQLite/Postgres rows -> lexical/local-hybrid ranking -> context ``` +R1.3 quarantines every Qdrant-backed claim or verbatim payload retrieval path. +In local-trusted mode, a requested Qdrant claim search falls back to +authoritative lexical retrieval; direct Qdrant search adapters deny the +request; prompt-context vector fallback is disconnected; and verbatim +`vector`/`hybrid` requests fall back to FTS5. Team MCP rejects semantic +retrieval entirely. Qdrant upsert, `qdrant-sync`, and `qdrant-reconcile` +remain available for index maintenance, including count/ID drift reads. R2.1 +may re-enable payload retrieval only as ID candidates that are rehydrated and +policy-filtered through SQLite/Postgres. + See [docs/architecture.md](docs/architecture.md) for the current module map, data-flow details, recent PR status, and sensitivity-filter invariants. @@ -53,7 +63,7 @@ recent PR status, and sensitivity-filter invariants. - **6-state lifecycle**: `candidate` → `confirmed` → `stale` → `superseded` → `conflicted` → `archived` - **Citation tracking** with provenance for every claim -- **Hybrid retrieval**: vector (sentence-transformers / Gemini) + FTS5 + freshness + confidence +- **Hybrid retrieval**: authoritative claim rows ranked with FTS5, local/primary-store embedding signals, freshness, and confidence; it does not read Qdrant while R1.3 containment is active - **Context optimizer**: `query_for_context(budget=4000)` returns auto-curated memory that fits your token budget - **Entity graph** with typed relationships and alias resolution - **Rule-shaped claims** (new in v3.21.0): prescriptive `when , do because ` claims (`ingest_rule` / `query_rules`) — the shape an agent needs to actually change behaviour next time, not just recall a fact @@ -112,7 +122,7 @@ Reproduce: `python tests/bench_longmemeval.py --retrieval-only`. Full methodolog **Optional (nice to have)** -- **Docker** for Qdrant — vector retrieval. SQLite FTS5 is the default and works out of the box; add Qdrant when you want semantic recall on top of keyword search. +- **Docker** for Qdrant index maintenance and Ollama-backed jobs. SQLite/Postgres remain authoritative. Claim, prompt-context, and verbatim retrieval from Qdrant are temporarily quarantined until the governed R2.1 planner can enforce lifecycle, tenant, scope, visibility, and sensitivity policy; sync/reconcile remain available. ## 30-second quickstart @@ -220,12 +230,14 @@ disposable database; repository tests do not constitute a production proof. ## Docker Compose -Run the full stack (MemoryMaster + Qdrant + Ollama) with one command: +Run MemoryMaster with the optional Qdrant maintenance index and Ollama with one command: ```bash docker compose up -d ``` +Starting Qdrant does not re-enable Qdrant retrieval during R1.3 containment. + See [INSTALLATION.md](INSTALLATION.md) for Kubernetes / Helm. ## Development diff --git a/ROADMAP.md b/ROADMAP.md index 7c8265e3..47ece183 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,5 +1,17 @@ # Roadmap +## Current R1.3 Qdrant safety status (2026-07-11) + +The checked items below record capabilities that were historically implemented; +they do not all remain enabled. Claim, prompt-context fallback, and verbatim +payload retrieval from Qdrant are quarantined. Local-trusted Qdrant claim +requests use authoritative lexical fallback, verbatim vector/hybrid requests +use FTS5, and direct backend/CLI search fails closed. Team MCP denies semantic +retrieval. Upsert, sync, reconcile, count/ID drift reads, and orphan cleanup +remain available. R2.1 is the future governed path: Qdrant supplies untrusted +IDs, then SQLite/Postgres rehydrates and filters canonical rows through the +shared policy planner. + ## Completed: v2.0.0 (Current) ### Core Engine (v1.0) @@ -24,8 +36,8 @@ - [x] QdrantBackend with Ollama qwen3-embedding:8b (4096 dims) - [x] Retry with exponential backoff on embed + upsert - [x] Batch upsert in sync_all (50x fewer HTTP calls) -- [x] `qdrant-sync` and `qdrant-search` CLI commands -- [x] `retrieval_mode="qdrant"` MCP fast path (~0.5s semantic search) +- [x] `qdrant-sync` and `qdrant-search` CLI commands implemented (`qdrant-search` is currently fail-closed; sync remains available) +- [x] `retrieval_mode="qdrant"` MCP fast path implemented historically (currently authoritative lexical fallback in local-trusted mode and denied in team mode) - [x] MCP query default changed to legacy (3min → 0.1s) ### Performance (v2.0) diff --git a/docs/AGENT-INSTALL.md b/docs/AGENT-INSTALL.md index 97d1c9df..6b10316d 100644 --- a/docs/AGENT-INSTALL.md +++ b/docs/AGENT-INSTALL.md @@ -4,6 +4,11 @@ Copy the appropriate block below and paste it as a prompt to your coding agent. The agent will run the installer, read the machine-readable result, and report what was wired, what was reused (brownfield), and what degraded. +R1.3 safety note: Qdrant is an optional maintenance index. Installing its +extra or starting Docker does not enable claim, context-fallback, or verbatim +payload retrieval. Local-trusted Qdrant claim requests use lexical fallback; +team semantic MCP requests are denied. Sync/reconcile remain available. + --- ## Claude Code variant @@ -28,8 +33,8 @@ Install MemoryMaster in this environment. Follow these steps exactly. installer printed — paste them verbatim so I can see the plan. - applied: what was actually wired (hooks, mcp_claude, cron, full_stack, etc.). - degraded: true/false. If true, explain the reason from the JSON - (typically: Docker not found → SQLite-only mode, vector recall + local LLM - auto-ingest are off). + (typically: Docker not found → SQLite-only mode, Qdrant index maintenance + + local LLM auto-ingest are off). - verify.status: PASS, PARTIAL, or FAIL and the detail string. 3. Run the verify round-trip to confirm hooks and DB are functional: @@ -116,9 +121,9 @@ Notes for you (the agent): - The MCP server command registered is the same non-deprecated path: python -m memorymaster.surfaces.mcp_server - If degraded is true (Docker absent / services unreachable) the install still - succeeded (exit 0). SQLite-only mode is fully functional; vector recall and - local LLM auto-ingest are simply off until Docker or QDRANT_URL/OLLAMA_URL - are available. + succeeded (exit 0). SQLite-only mode is fully functional; Qdrant index + maintenance and local LLM auto-ingest are off. Starting Qdrant later does not + lift the R1.3 payload-retrieval quarantine. - Claude Code hooks (UserPromptSubmit, Stop, SessionStart, PreCompact) are NOT registered for Codex — those are Claude Code-specific. The session-end script above is the Codex equivalent for distilled ingest. @@ -136,8 +141,8 @@ Notes for you (the agent): | `--api-key KEY` | API key for the chosen provider | | `--model MODEL` | LLM model id | | `--project-root PATH` | Directory where `memorymaster.db` lives | -| `--full-stack` | Bring up Qdrant + Ollama via Docker Compose (default when omitted) | -| `--no-full-stack` | Skip the vector + local-LLM stack | +| `--full-stack` | Bring up the Qdrant maintenance index + Ollama via Docker Compose (default when omitted) | +| `--no-full-stack` | Skip the Qdrant-index + local-LLM stack | | `--no-cron` | Skip steward cron setup | | `--no-obsidian-skills` | Skip Obsidian skills install | | `--codex` | Force Codex MCP + instructions wiring (auto-detected otherwise) | @@ -151,9 +156,10 @@ Notes for you (the agent): If Docker is absent and Qdrant/Ollama are not already reachable at `QDRANT_URL`/`OLLAMA_URL`, the installer continues in SQLite-only mode: -> Running in SQLite-only mode. Vector recall + local LLM auto-ingest are OFF. -> To enable them: install Docker and re-run with `--full-stack`, or point -> QDRANT_URL / OLLAMA_URL at existing services. +> Running in SQLite-only mode. Qdrant index maintenance + local LLM auto-ingest are OFF. +> Retrieval remains available through authoritative SQLite ranking. To enable index +> maintenance or local LLMs, use `--full-stack` or QDRANT_URL / OLLAMA_URL. The exit code is still 0. Core claim storage, recall hooks, and MCP tools -remain fully functional. +remain fully functional. Qdrant payload retrieval remains quarantined even when +the optional service is available. diff --git a/docs/MCP-TOOLS.md b/docs/MCP-TOOLS.md index e25ea278..b4882d69 100644 --- a/docs/MCP-TOOLS.md +++ b/docs/MCP-TOOLS.md @@ -29,18 +29,27 @@ tool, grouped by purpose. For the MCP server config block see the ## Query & retrieval (read) +> **R1.3 Qdrant containment:** local-trusted `query_memory(retrieval_mode="qdrant")` +> reports the requested/effective modes and uses authoritative lexical fallback. +> Auto-classification follows the same rule. Prompt-context Qdrant fallback is +> disconnected, and `search_verbatim(mode="vector"|"hybrid")` reports an FTS5 +> fallback instead of returning Qdrant payloads. Team MCP denies every semantic +> mode, including Qdrant and local hybrid. Qdrant sync/reconcile are CLI/index +> maintenance operations, not MCP read paths. Governed Qdrant reads are deferred +> to R2.1 ID-candidate retrieval plus SQLite/Postgres rehydration. + | Tool | Purpose | |------|---------| -| `query_memory` | Query memory for relevant claims. Includes candidates by default for MCP use. | +| `query_memory` | Query authoritative claims; a local-trusted Qdrant request falls back to lexical retrieval, while team semantic requests are denied. | | `query_for_context` | Pack the most relevant claims into a token-budgeted context block. | | `query_for_task` | Look-ahead task-aware briefing for an upcoming PRD task. | | `query_rules` | Retrieve rule-shaped claims matching a query, in prescriptive form. | | `query_claim_paths` | Traverse claim relationship paths from a starting claim (read-only). | | `query_meta_decisions` | Aggregate matching decision/architecture claims across all project scopes. | | `federated_query` | Query across ALL scopes — cross-project federation. | -| `classify_query` | Classify a query and recommend the best retrieval mode. | +| `classify_query` | Classify a query and report both the recommended mode and its containment-safe effective mode. | | `recall_analysis` | Explain WHY each claim ranked where it did (ranking introspection). | -| `search_verbatim` | Search raw conversation memories (verbatim, unsummarized). | +| `search_verbatim` | Search raw conversation memories through FTS5; vector/hybrid requests currently report an FTS5 fallback. | | `read_active_tasks` | Read and parse the project's `active_tasks.md`. | | `rules_export` | Export mined rule-shaped claims, filtered by confidence + status. | diff --git a/docs/architecture.md b/docs/architecture.md index cdb702c9..652730a9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -63,15 +63,24 @@ service boundary remains the last storage guard even when MCP is bypassed. query_memory / CLI query -> mcp_server.py or cli.py -> MemoryService.query / query_rows - -> SQLiteStore.query reads claims_fts + claim metadata - -> retrieval.py ranks rows by lexical, confidence, freshness, graph, and vector signals - -> optional qdrant_backend.py semantic candidates or fallback rerank + -> SQLite/Postgres returns authorized claim rows and lifecycle metadata + -> retrieval.py ranks those rows by lexical, confidence, freshness, graph, + and optional local/primary-store embedding signals -> context_optimizer.py packs query_for_context results into provider-aware budgets ``` `query_for_context` reuses the ranked rows and then chooses a text, XML, or JSON envelope that fits -the caller's token budget. Qdrant is an optional acceleration and recall path; the SQLite claim store -remains authoritative. +the caller's token budget. R1.3 does not admit Qdrant hits into this flow: explicit or classified +claim requests use authoritative lexical fallback in local-trusted mode, prompt-context Qdrant +fallback is disconnected, and team MCP denies semantic modes. Local `hybrid` ranking is distinct +from Qdrant retrieval and operates only on rows already authorized by SQLite/Postgres. + +The verbatim read path follows the same containment rule. `search_verbatim(mode="vector"|"hybrid")` +uses authoritative FTS5 and never consumes Qdrant payload text. Direct Qdrant search entry points, +including `qdrant-search` and `QdrantBackend.search`, fail closed. Qdrant upsert, sync, reconciliation, +orphan cleanup, and drift checks remain available as index maintenance. R2.1 may restore reads only +by accepting Qdrant IDs as untrusted candidates, rehydrating the canonical rows from SQLite/Postgres, +and applying the shared tenant/scope/visibility/lifecycle/sensitivity planner before ranking. ## Sensitivity Invariant @@ -205,8 +214,8 @@ this track's requested inventory. | `memorymaster/plugins.py` | Entry-point plugin registry for validators, probes, retrieval hooks, and exporters. | `8c4f302 feat: plugin system + cross-agent scope isolation with RBAC` | | `memorymaster/policy.py` | Policy-mode configuration and cadence override helpers. | `0dff74a feat(policy): MEMORYMASTER_POLICY_MODE env-var opt-in for cadence` | | `memorymaster/postgres_store.py` | Postgres storage backend with parity methods for the service layer. | `e337c07 chore(storage): audit SQLite/Postgres parity, add 3 missing pg methods (#35)` | -| `memorymaster/qdrant_backend.py` | Qdrant vector index backend for claim semantic search. | `7b049c5 chore: prepare for open-source release — scrub private data, add docs` | -| `memorymaster/qdrant_recall_fallback.py` | Optional vector fallback for sparse recall-hook candidate sets. | `a1e6786 feat(recall): Qdrant vector-search fallback for sparse-candidate prompts` | +| `memorymaster/qdrant_backend.py` | Qdrant maintenance-index backend; payload search fails closed while upsert/sync/reconcile count/ID operations remain available. | `7b049c5 chore: prepare for open-source release — scrub private data, add docs` | +| `memorymaster/qdrant_recall_fallback.py` | Compatibility helpers for the disconnected prompt-context fallback; activation knobs cannot enable reads during R1.3. | `a1e6786 feat(recall): Qdrant vector-search fallback for sparse-candidate prompts` | | `memorymaster/qmd_bridge.py` | Conversion bridge between OpenClaw QMD records and MemoryMaster claims. | `a4b5dcc feat: QMD ↔ memorymaster bridge for OpenClaw integration` | | `memorymaster/query_classifier.py` | Rule-based routing of queries to retrieval modes. | `265d951 refactor: inline citation locator/excerpt, merge print_claim header+text` | | `memorymaster/query_expansion.py` | Entity alias and synonym expansion for recall queries. | `ac071de feat(recall): query expansion via entity-matched synonyms (roadmap 1.5)` | @@ -240,7 +249,7 @@ this track's requested inventory. | `memorymaster/vault_query_capture.py` | Saves high-value query answers as new wiki pages. | `88d7afb feat: LLM Wiki architecture — lint, log, synthesis, query capture` | | `memorymaster/vault_synthesis.py` | Updates related wiki pages when new claims arrive. | `88d7afb feat: LLM Wiki architecture — lint, log, synthesis, query capture` | | `memorymaster/verbatim_recall.py` | Optional raw-conversation FTS recall stream. | `6e120a2 feat(recall): MemPalace-style verbatim retrieval stream (opt-in)` | -| `memorymaster/verbatim_store.py` | Raw conversation storage with FTS5 and optional Qdrant search. | `89c900d fix(verbatim-store): use full-content hash + point IDs for Qdrant dedup (#43)` | +| `memorymaster/verbatim_store.py` | Raw conversation storage with authoritative FTS5 reads; Qdrant sync remains, while vector/hybrid reads downgrade to FTS5. | `89c900d fix(verbatim-store): use full-content hash + point IDs for Qdrant dedup (#43)` | | `memorymaster/webhook.py` | Webhook notification helper for claim events. | `b3cb6c1 fix: add comprehensive error handling and edge case coverage to v2.1 modules` | | `memorymaster/wiki_engine.py` | Absorbs claims into compiled wiki articles and related cleanup/breakdown flows. | `bf25482 feat(dashboard): claim lineage view via /claim//lineage (#46)` | | `memorymaster/wiki_freshness.py` | Computes freshness scores from wiki article absorb dates. | `702c904 feat(wiki): wiki-freshness CLI + STALE_ARTICLE lint (11.8, Option A)` | diff --git a/docs/cli-cookbook.md b/docs/cli-cookbook.md index 6a3ca0e7..d526fdae 100644 --- a/docs/cli-cookbook.md +++ b/docs/cli-cookbook.md @@ -60,10 +60,20 @@ python -m memorymaster --db memorymaster.db links 1 --type relates_to ``` ### qdrant-search -**Purpose**: Run semantic search through Qdrant. +**Purpose**: Reserved for governed semantic retrieval. R1.3 temporarily disables this command; it exits with code 2 before constructing a Qdrant backend. Use `query` for authoritative lexical/hybrid recall. + +### qdrant-sync +**Purpose**: Maintain the Qdrant index while retrieval is quarantined. +**Example**: +``` +python -m memorymaster --db memorymaster.db qdrant-sync +``` + +### qdrant-reconcile +**Purpose**: Compare the authoritative store with the Qdrant maintenance index and repair drift/orphans. This does not enable claim or verbatim payload retrieval. **Example**: ``` -python -m memorymaster --db memorymaster.db qdrant-search "retrieval profiles" --limit 5 --states confirmed,stale +python -m memorymaster --db memorymaster.db qdrant-reconcile ``` ## Ingest & Lifecycle @@ -304,7 +314,7 @@ python -m memorymaster --db memorymaster.db install-gitnexus-hook ``` ### qdrant-sync -**Purpose**: Bulk-sync active claims to Qdrant vector storage. +**Purpose**: Bulk-sync active claims to the Qdrant maintenance index. Indexed payloads are not a retrieval authority during R1.3. **Example**: ``` python -m memorymaster --db memorymaster.db qdrant-sync --qdrant-url http://localhost:6333 --ollama-url http://localhost:11434 diff --git a/docs/cross-project-patterns.md b/docs/cross-project-patterns.md index cf394a2a..484296fd 100644 --- a/docs/cross-project-patterns.md +++ b/docs/cross-project-patterns.md @@ -50,6 +50,13 @@ Important current-behavior caveat: these MCP query tools do not categorically ga `query_memory` is the default project recall tool. Its MCP signature includes `workspace`, `limit`, `retrieval_mode`, `include_stale`, `include_conflicted`, `include_candidates`, `allow_sensitive`, `scope_allowlist`, and `detail_level` (`memorymaster/mcp_server.py:592-606`). +During R1.3, `retrieval_mode="qdrant"` is not a separate authorization path. +Local-trusted calls report the requested/effective modes and use authoritative +lexical retrieval; auto-classified Qdrant recommendations do the same. Team MCP +denies semantic retrieval before dispatch. Local `hybrid` ranking is +Qdrant-independent and ranks only primary-store rows that already passed the +normal lifecycle, scope, visibility, and sensitivity gates. + Default scope filter: - Blank `scope_allowlist` expands to the derived current project scope plus `global` (`memorymaster/mcp_server.py:274-289`). @@ -127,16 +134,15 @@ How to narrow: ## Cross-Tenant Safety -The storage and service layers have tenant filtering support when a `MemoryService` is constructed with a tenant id: the service stores `self.tenant_id` (`memorymaster/service.py:85-98`), and storage filters `tenant_id` when it is provided (`memorymaster/_storage_read.py:158-160`, `memorymaster/postgres_store.py:524-526`). - -Current MCP behavior does not pass a tenant id into `MemoryService`. The MCP `_service` factory constructs `MemoryService(db_target=..., workspace_root=...)` without a `tenant_id` argument (`memorymaster/mcp_server.py:171-172`). Therefore, the three MCP federation tools documented here rely on scope filtering and sensitive filtering, not tenant isolation, unless a future wrapper or configuration path injects `tenant_id`. +Team mode derives an immutable request context at the MCP boundary and binds its principal, tenant, workspace, and exact allowed scopes into `MemoryService`. PostgreSQL application connections require that authority and bind it transaction-locally; missing or expanded authority fails closed. Local-trusted mode remains an explicit single-agent profile and is not a tenant-security boundary. -### Known Gaps +### Current Containment and Remaining Gaps -1. `query_memory(retrieval_mode="qdrant")` bypasses the normal scope, archived, sensitive, and tenant filters. The MCP wrapper returns early for qdrant mode (`memorymaster/mcp_server.py:635-641`); `_qdrant_query` calls `backend.search(query, limit=limit)` without scope or tenant filters (`memorymaster/mcp_server.py:292-300`), then rehydrates hits with direct `store.get_claim` (`memorymaster/mcp_server.py:304-329`). SQLite and PostgreSQL `get_claim` fetch by id only (`memorymaster/_storage_read.py:43-60`, `memorymaster/postgres_store.py:481-490`). -2. MCP tenant isolation is not wired. The service supports `tenant_id` (`memorymaster/service.py:85-98`) and storage enforces it when provided (`memorymaster/_storage_read.py:158-160`, `memorymaster/postgres_store.py:524-526`), but `_service` never passes one (`memorymaster/mcp_server.py:171-172`). -3. `visibility="sensitive"` is not itself a default gate for these MCP tools. The active sensitive predicate inspects payload text (`memorymaster/security.py:356-363`), and visibility filtering is only conditional on `requesting_agent` (`memorymaster/service.py:470-472`), which the `query_memory` MCP wrapper does not pass (`memorymaster/mcp_server.py:643-653`). -4. `query_meta_decisions` cannot be widened to non-project scopes from the MCP surface. The service explicitly discards non-`project:` scopes (`memorymaster/service.py:804-810`) and the MCP signature has no scope parameter (`memorymaster/mcp_server.py:1182-1199`). +1. R1.3 quarantines claim retrieval from Qdrant. Explicit or auto-classified local-trusted MCP requests use authoritative lexical retrieval and report containment metadata; the direct backend/API entrypoint rejects; and CLI `qdrant-search` exits with code 2 before backend construction. +2. Prompt-context Qdrant fallback is disconnected. Verbatim `vector`/`hybrid` requests use FTS5 and never return indexed payload text. Team MCP denies semantic retrieval, including local hybrid, before tool dispatch. +3. `qdrant-sync`, upserts, drift reconciliation, and orphan cleanup remain available for index maintenance, but indexed payloads are never treated as retrieval truth. Governed Qdrant retrieval stays blocked until R2.1 can accept IDs as untrusted candidates, rehydrate authoritative SQLite/Postgres rows, and apply the shared tenant/scope/visibility/lifecycle/sensitivity planner. +4. Local-trusted mode intentionally lacks team tenancy. Do not expose that profile as a shared service. +5. `query_meta_decisions` remains project-scope-only by design; its MCP signature does not widen to arbitrary non-project scopes. ## Examples @@ -192,6 +198,6 @@ Explicit all-scope search: ## Follow-Up Track Candidates -- Add tenant injection to the MCP `_service` factory or tool request model before treating MCP federation as tenant-isolated (`memorymaster/mcp_server.py:171-172`, `memorymaster/service.py:85-98`). -- Add scope, tenant, archived-status, and sensitive filters to `_qdrant_query` or route qdrant hits through `query_rows`-equivalent authorization (`memorymaster/mcp_server.py:292-329`, `memorymaster/mcp_server.py:635-641`). -- Decide whether `visibility="sensitive"` should be enforced as a first-class query gate alongside content-sensitive detection (`memorymaster/security.py:356-363`, `memorymaster/service.py:470-472`). +- R2.1: make Qdrant return opaque claim IDs only, then rehydrate canonical rows through the same SQLite/Postgres planner used by lexical retrieval. Reject missing, archived, cross-tenant, out-of-scope, unauthorized-visibility, and sensitive rows before ranking. +- Prove the rehydration contract against both stores, including stale index entries, deleted/orphan IDs, conflicting Qdrant payload fields, and mixed-tenant/scope candidates. Do not remove R1.3 containment before those tests and external authenticated/TLS Qdrant verification pass. +- Evaluate team semantic MCP separately after the shared planner exists. It remains denied rather than inheriting local-trusted behavior. diff --git a/docs/env-reference.md b/docs/env-reference.md index 00668250..9477ed32 100644 --- a/docs/env-reference.md +++ b/docs/env-reference.md @@ -5,6 +5,21 @@ Complete inventory of `MEMORYMASTER_*` variables referenced in the package, script after adding or removing a variable. For what each does, follow the listed source files (most are read next to a docstring or comment). +### R1.3 Qdrant containment + +An environment variable being referenced does not mean it can activate Qdrant +claim/verbatim payload retrieval. `MEMORYMASTER_RECALL_VECTOR_FALLBACK`, +`MEMORYMASTER_QDRANT_URL`, `MEMORYMASTER_QDRANT_COLLECTION`, +`MEMORYMASTER_EMBED_MODEL`, `MEMORYMASTER_RECALL_VECTOR_LIMIT`, +`MEMORYMASTER_RECALL_VECTOR_MIN_CANDIDATES`, and +`MEMORYMASTER_RECALL_VECTOR_SCORE_THRESHOLD` are retained compatibility/tuning +knobs for the disconnected prompt-context fallback; changing them cannot enable +retrieval during R1.3. The non-prefixed `QDRANT_URL`, `QDRANT_COLLECTION`, and +`OLLAMA_URL`, plus `MEMORYMASTER_QDRANT_DRIFT_MAX`, remain active for upsert, +sync, reconcile, count/ID drift checks, orphan cleanup, and other index +maintenance. Local/primary-store hybrid weights such as `MEMORYMASTER_W_VEC` +are separate from Qdrant retrieval. + Total: 134 variables. | Variable | Referenced in | diff --git a/docs/handbook.md b/docs/handbook.md index 633f5553..70e64de9 100644 --- a/docs/handbook.md +++ b/docs/handbook.md @@ -52,6 +52,25 @@ Everything below is **opt-in or additive** — the default recall/ranking path i --- +### R1.3 Qdrant retrieval containment + +Qdrant is currently a maintenance index, not a claim or verbatim payload +authority. Local-trusted claim requests that explicitly or automatically select +Qdrant report the containment and use lexical retrieval from SQLite/Postgres. +The prompt recall hook never invokes its retained Qdrant fallback, and verbatim +vector/hybrid requests use FTS5. Direct backend/API search and CLI +`qdrant-search` fail closed; team MCP denies semantic modes entirely. +`qdrant-sync`, automatic upserts, `qdrant-reconcile`, count/ID drift checks, and +orphan cleanup remain available. + +The ordinary local `hybrid` mode is distinct: it ranks already-authorized +primary-store claim rows with lexical and optional local/primary-store embedding +signals. R2.1 may restore Qdrant payload retrieval only as untrusted ID +candidates followed by authoritative SQLite/Postgres rehydration and the shared +tenant/scope/visibility/lifecycle/sensitivity planner. + +--- + ## End-to-end flow ``` @@ -410,7 +429,7 @@ MemoryMaster is the memory layer; this stack covers the rest. The Intelligence-F | 4 | **Serena** | LSP-powered symbol-level read/edit | Global MCP config, see [oraios/serena](https://github.com/oraios/serena) | | 5 | **context7** | Live library docs | First-party Claude Code MCP, no install | | opt | **Obsidian CLI** | Vault-aware search from terminal | `npm install -g obsidian-cli` | -| opt | **Qdrant** | External vector search | `docker run -p 6333:6333 qdrant/qdrant` | +| opt | **Qdrant** | External maintenance index; upsert/sync/reconcile only while R1.3 payload-retrieval quarantine is active | `docker run -p 6333:6333 qdrant/qdrant` | ## Verify install + troubleshooting diff --git a/docs/retrieval-governance-upgrades-2026-06-24.md b/docs/retrieval-governance-upgrades-2026-06-24.md index bce0ad47..9c106c7d 100644 --- a/docs/retrieval-governance-upgrades-2026-06-24.md +++ b/docs/retrieval-governance-upgrades-2026-06-24.md @@ -1,5 +1,11 @@ # Retrieval & Governance Upgrades — implementation plan (2026-06-24) +> **Current R1.3 status (2026-07-11):** references below to a Qdrant +> "vector-on" case are historical experiment context, not an available read +> path. Claim/context-fallback/verbatim payload retrieval is quarantined; +> sync/reconcile/count-ID maintenance remains. R2.1 must use Qdrant IDs only and +> rehydrate authorized SQLite/Postgres rows before any payload reaches ranking. + **Source:** the 2026-06-24 re-survey (`artifacts/steal-from-others-2026-06-24.md`, `CREDITS.md`, claim `mm-e460`). **Goal:** land the prioritized steal candidates as measured, tested changes — retrieval quality first, then governance/correctness fixes, then new tools, then positioning. diff --git a/external-actions-required.md b/external-actions-required.md index e2bfdfac..ab1d25cd 100644 --- a/external-actions-required.md +++ b/external-actions-required.md @@ -11,7 +11,7 @@ live mutation by itself. | MM-SEC-01 / disposable proof | Team Postgres test environment | Provide one fully disposable database, a dedicated schema-owning `SUPERUSER`/`BYPASSRLS` migrator DSN in `MEMORYMASTER_TEST_POSTGRES_DSN`, and a distinct restricted app DSN in `MEMORYMASTER_TEST_POSTGRES_APP_DSN`; set `MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE=1`. Neither DSN may target product/live data. | Run `tests/test_postgres_rls_integration.py` plus every Postgres parameter in `tests/test_postgres_parity.py` and `tests/test_backend_parity.py`. Retain exact FORCE-RLS policy/function/trigger/index catalog evidence; literal-sensitive fingerprints; event-head owner `SUPERUSER`/`BYPASSRLS`; app event SELECT/INSERT with no table/column UPDATE or DELETE; cross-tenant/scope/visibility/principal denials; tenant-chain linearity; scope-local identities and ambiguity rejection; self/cross-boundary supersession denial; successful reciprocal supersession plus one event; rollback after replacement-update/event-insert failure; and parity results from UUID-isolated tenants with no destructive cleanup. No real PostgreSQL proof has been performed here. | 2026-07-31 | BLOCKED-EXTERNAL | | MM-SEC-01 / brownfield data | Live database operator | Authorize a read-only inventory before v0012. Any owner backfill, duplicate/supersession-edge repair, or grant change requires separate explicit approval, a verified backup, and a maintenance window. | Inventory noncanonical visibility; blank/null `source_agent` on every claim; identity duplicates; both supersession columns for self, missing-target, nonreciprocal, cross-tenant/scope/visibility/owner edges; current event/function/table grants; and constraint state without writing. After approval, retain repair/grant evidence and rerun v0012, which validates `ck_claims_identity_visibility_owner`. No product-data verification or mutation has been performed here. | 2026-07-31 | BLOCKED-EXTERNAL | | MM-OPS-02 | Docker/Helm runtime | Provide Docker and, for final verification, a disposable Kubernetes target if unavailable locally. | Container health/MCP handshake and Helm readiness/network-policy evidence | 2026-08-15 | PENDING-CAPABILITY-CHECK | -| MM-SEC-02 | Qdrant runtime | Provide a disposable authenticated/TLS Qdrant target for final parity after fake-backed tests pass. | Real service policy/reconciliation test output | 2026-08-15 | PENDING-CAPABILITY-CHECK | +| MM-SEC-02 | Qdrant runtime | Provide a disposable authenticated/TLS Qdrant target for final parity after the fake-backed containment suite. Do not point the tests at a live/product collection. | API-key/TLS connection proof plus real-service sync/reconcile/count-ID evidence and denial of every payload-search adapter; retain exact command output without secrets | 2026-08-15 | BLOCKED-EXTERNAL | | MM-DATA-01 | Live MemoryMaster operator | Approve a consistent backup/restore drill before any live migration, redaction, backlog, or retention operation. | Restored backup, integrity check, counts/checksums, approval record | 2026-08-15 | BLOCKED-EXTERNAL | | MM-CAP-01 | Host storage operator | Address/monitor the drive at 85.82% used without deleting MemoryMaster data under this goal. | Daily disk telemetry below the critical gate or approved capacity expansion | 2026-07-18 | BLOCKED-EXTERNAL | | MM-PRIV-01 | Product owner / legal reviewer | Decide intended organizational use, jurisdictions, processor disclosures, and retention commitments before compliance claims. | Approved privacy/data-processing statement | 2026-08-31 | BLOCKED-EXTERNAL | diff --git a/memorymaster/recall/context_hook.py b/memorymaster/recall/context_hook.py index fede9934..b63c10a4 100644 --- a/memorymaster/recall/context_hook.py +++ b/memorymaster/recall/context_hook.py @@ -1032,66 +1032,8 @@ def _apply_vector_fallback( rows: list, seen_ids: set[int], ) -> list: - """Augment ``rows`` with Qdrant semantic-search hits when the primary - retrieval stages under-produced. - - Triggers only when ``len(rows) < MEMORYMASTER_RECALL_VECTOR_MIN_CANDIDATES`` - (default 3) and every env-var gate is satisfied. Silently degrades on - any failure (qdrant unreachable, collection missing, embedder import - error, etc) so the caller keeps whatever FTS5 + entity fanout produced. - - Returns the (possibly augmented) row list. Always mutates ``seen_ids`` - when new rows are added. - """ - try: - from memorymaster.recall import qdrant_recall_fallback - except Exception as exc: # pragma: no cover — import errors rare - logger.debug("vector fallback: module import skipped: %s", exc) - return rows - - if not qdrant_recall_fallback.is_fallback_enabled(): - return rows - if len(rows) >= qdrant_recall_fallback.fallback_threshold(): - return rows - - try: - hits = qdrant_recall_fallback.search(query) - except Exception as exc: # pragma: no cover — search() already swallows - logger.debug("vector fallback: search skipped: %s", exc) - return rows - - if not hits: - return rows - - # Lazy security check — mirrors the entity fanout treatment. - try: - from memorymaster.core.security import is_sensitive_claim - except Exception: - is_sensitive_claim = lambda _claim: False # type: ignore[assignment] # noqa: E731 - - appended = 0 - for hit in hits: - cid = hit.claim_id - if cid in seen_ids: - continue - try: - claim = svc.store.get_claim(cid, include_citations=True) - except Exception as exc: - logger.debug("vector fallback: get_claim(%d) failed: %s", cid, exc) - continue - if claim is None or getattr(claim, "status", "") == "archived": - continue - if is_sensitive_claim(claim): - continue - rows.append(_row_for_vector_hit(claim, hit.score)) - seen_ids.add(cid) - appended += 1 - - if appended: - logger.debug( - "vector fallback: appended %d rows (total=%d) for query=%r", - appended, len(rows), query[:60], - ) + """Return authoritative rows unchanged while Qdrant retrieval is quarantined.""" + del svc, query, seen_ids return rows @@ -1171,6 +1113,9 @@ def recall( ``return_ids`` defaults to ``False`` so every existing caller — MCP tools, CLI, hooks — gets the legacy ``str`` return type unchanged. + + ``skip_qdrant`` is retained for caller compatibility but is a no-op while + R1.3 unconditionally quarantines Qdrant retrieval. """ from memorymaster.core.service import MemoryService @@ -1234,8 +1179,8 @@ def query_for_task( Max TOKENS of inner content (default 500 ≈ 2KB chars). Outer XML wrapper adds ~80 chars on top. skip_qdrant: - Default True — vector search adds latency; FTS5 + ranking are - sufficient for most task briefings. + Selects primary-store ``legacy`` ranking when true and local ``hybrid`` + ranking when false. Neither mode reads external Qdrant during R1.3. Returns ------- @@ -1289,7 +1234,8 @@ def query_for_task( svc = MemoryService(db_target=db, workspace_root=Path.cwd()) scope_filter = [project_scope] if project_scope else None - # Query in legacy mode (FTS5 only) when skip_qdrant=True for latency. + # Historical name retained: this now selects primary-store legacy vs + # local hybrid ranking; external Qdrant reads are quarantined either way. retrieval_mode = "legacy" if skip_qdrant else "hybrid" try: @@ -1475,25 +1421,6 @@ def _recall_impl( continue rows.append(_row_for_claim(claim)) - # Vector fallback — Qdrant semantic search when FTS5 + entity fanout - # produced fewer than MEMORYMASTER_RECALL_VECTOR_MIN_CANDIDATES rows - # (default 3). Fully env-gated so default behaviour is unchanged. See - # ``_apply_vector_fallback`` for the exact gating logic. Only time when - # the fallback is enabled — otherwise we emit nothing (zero-overhead). - _vector_enabled = False - try: - from memorymaster.recall import qdrant_recall_fallback as _qrf - - _vector_enabled = bool(_qrf.is_fallback_enabled()) - except Exception: - _vector_enabled = False - - if _vector_enabled: - with _phase_timer(phase_ms, "vector_fallback"): - rows = _apply_vector_fallback(svc, query, rows, seen_ids) - else: - rows = _apply_vector_fallback(svc, query, rows, seen_ids) - # Verbatim retrieval — MemPalace-style raw conversation stream. # # Gated on MEMORYMASTER_RECALL_VERBATIM=1 (default 0 = off) so legacy @@ -1736,28 +1663,6 @@ def _recall_impl( except Exception as exc: # noqa: BLE001 — defensive (claim 11907) logger.debug("graph stream skipped: %s", exc) - if not rows and not skip_qdrant: - # Fallback to Qdrant semantic search - try: - from memorymaster.recall.qdrant_backend import QdrantBackend - backend = QdrantBackend() - hits = backend.search(query, limit=5) - backend.close() - if hits: - lines = ["# Memory Context (semantic)", ""] - for hit in hits: - p = hit.get("payload", {}) - text = p.get("claim_text", "")[:200] - lines.append(f"- {text}") - if _rendered_ids is not None: - cid = p.get("claim_id") - if isinstance(cid, int): - _rendered_ids.append(cid) - return "\n".join(lines).encode("ascii", errors="replace").decode("ascii") - except Exception: - pass - return "" - if not rows: return "" diff --git a/memorymaster/recall/qdrant_backend.py b/memorymaster/recall/qdrant_backend.py index 64979f5b..ae1e7856 100644 --- a/memorymaster/recall/qdrant_backend.py +++ b/memorymaster/recall/qdrant_backend.py @@ -1,8 +1,9 @@ """Qdrant vector store backend for MemoryMaster. -Uses Qdrant at a network-accessible endpoint as a search index alongside -the primary SQLite/Postgres store. Embeddings come from Ollama -(qwen3-embedding:8b, 4096-dim) via HTTP. +Uses Qdrant at a network-accessible endpoint as a maintenance index alongside +the primary SQLite/Postgres store. Embeddings come from Ollama +(qwen3-embedding:8b, 4096-dim) via HTTP. Direct reads are quarantined until a +governed planner can rehydrate candidate IDs from the authoritative store. Environment variables / constructor params: QDRANT_URL – default http://localhost:6333 @@ -253,49 +254,11 @@ def search( min_confidence: float = 0.0, states: list[str] | None = None, ) -> list[dict[str, Any]]: - """Semantic search. Returns list of {claim_id, score, payload}.""" - vec = self._embed(query_text) - if vec is None: - return [] - - filters: dict[str, Any] = {"must": []} - if states: - filters["must"].append({ - "key": "state", - "match": {"any": states}, - }) - if min_confidence > 0: - filters["must"].append({ - "key": "confidence", - "range": {"gte": min_confidence}, - }) - - body: dict[str, Any] = { - "vector": vec, - "limit": limit, - "with_payload": True, - } - if filters["must"]: - body["filter"] = filters - - try: - resp = self._client.post( - f"{self.qdrant_url}/collections/{self.collection}/points/search", - json=body, - ) - resp.raise_for_status() - results = resp.json().get("result", []) - return [ - { - "claim_id": hit["payload"].get("claim_id"), - "score": hit["score"], - "payload": hit["payload"], - } - for hit in results - ] - except Exception as exc: - logger.warning("Qdrant search failed: %s", exc) - return [] + """Reject raw payload reads until the governed planner rehydrates IDs.""" + del query_text, limit, min_confidence, states + raise PermissionError( + "Qdrant retrieval is quarantined pending authoritative policy rehydration." + ) def _batch_upsert(self, points: list[dict[str, Any]]) -> bool: """Upsert a batch of points to Qdrant in a single request.""" diff --git a/memorymaster/recall/qdrant_recall_fallback.py b/memorymaster/recall/qdrant_recall_fallback.py index dd5d2ed4..a674873d 100644 --- a/memorymaster/recall/qdrant_recall_fallback.py +++ b/memorymaster/recall/qdrant_recall_fallback.py @@ -1,15 +1,18 @@ """Qdrant vector-search fallback for :mod:`memorymaster.recall.context_hook`. -Activated only when: +Compatibility helpers are retained for a governed R2.1 reintegration, but +R1.3 quarantines retrieval unconditionally. ``is_fallback_enabled()`` returns +``False`` and ``search()`` raises before loading a model or client. + +Historically activated only when: 1. ``MEMORYMASTER_RECALL_VECTOR_FALLBACK`` env var is truthy (``1``/``true``/...). 2. ``MEMORYMASTER_QDRANT_URL`` is set. 3. The primary FTS5 + entity-fanout stages returned fewer candidates than the threshold (default: 3). -When any of those conditions is false, or when Qdrant / sentence-transformers -is unreachable, the caller silently skips the fallback — default recall -behaviour is unchanged. +The historical gates are no longer activation controls during quarantine. +Recall always uses authoritative primary-store rows. Design notes ------------ @@ -20,10 +23,10 @@ * Uses the same deterministic UUID-v5 point id as :mod:`scripts.index_claims_to_qdrant` so searches stay in sync with the index script. -* Caches the embedder and Qdrant client as module-level singletons across - calls to amortise model-load cost (~2-3s cold, <5ms warm). -* Never throws across module boundary: every public helper returns safe - defaults on failure and logs at WARNING. +* Retains lazy model/client helpers for compatibility, but public read search + never calls them during quarantine. +* Read search fails closed with ``PermissionError`` during quarantine; sync + and deterministic point-id helpers remain available. """ from __future__ import annotations @@ -72,12 +75,8 @@ def _truthy(raw: str | None) -> bool: def is_fallback_enabled() -> bool: - """Gate: env opt-in + Qdrant URL configured.""" - if not _truthy(os.environ.get("MEMORYMASTER_RECALL_VECTOR_FALLBACK")): - return False - if not os.environ.get("MEMORYMASTER_QDRANT_URL", "").strip(): - return False - return True + """Remain fail-closed until R2.1 provides authoritative policy filtering.""" + return False def fallback_threshold() -> int: @@ -229,77 +228,8 @@ def reset_singletons_for_tests() -> None: def search(query_text: str, *, collection: str | None = None) -> list[VectorHit]: - """Return up to ``MEMORYMASTER_RECALL_VECTOR_LIMIT`` hits ranked by Qdrant - cosine similarity, filtered by ``MEMORYMASTER_RECALL_VECTOR_SCORE_THRESHOLD``. - - Never raises: logs a warning and returns an empty list on any failure. - """ - if not query_text or not query_text.strip(): - return [] - embedder = _get_embedder() - if embedder is None: - return [] - client = _get_client() - if client is None: - return [] - coll = ( - collection - or os.environ.get("MEMORYMASTER_QDRANT_COLLECTION") - or DEFAULT_COLLECTION + """Reject direct vector reads until governed candidate rehydration exists.""" + del query_text, collection + raise PermissionError( + "Qdrant recall fallback is quarantined pending authoritative policy rehydration." ) - try: - vec = embedder.encode( - query_text, normalize_embeddings=True, show_progress_bar=False - ) - vec_list = vec.tolist() if hasattr(vec, "tolist") else list(vec) - except Exception as exc: - logger.warning("vector fallback: embed failed: %s", exc) - return [] - - threshold = score_threshold() - limit = search_limit() - try: - # qdrant-client >=1.10 deprecated `search` in favour of `query_points`. - # Keep a fallback for older clients so installs pinned to 1.7-1.9 still work. - if hasattr(client, "query_points"): - resp = client.query_points( - collection_name=coll, - query=vec_list, - limit=limit, - score_threshold=threshold, - with_payload=True, - ) - raw_hits = getattr(resp, "points", None) or resp - else: - raw_hits = client.search( # type: ignore[attr-defined] - collection_name=coll, - query_vector=vec_list, - limit=limit, - score_threshold=threshold, - with_payload=True, - ) - except Exception as exc: - logger.warning( - "vector fallback: qdrant search on %r failed: %s", coll, exc, - ) - return [] - - hits: list[VectorHit] = [] - for h in raw_hits: - payload = getattr(h, "payload", None) or {} - raw_id = payload.get("id") - try: - claim_id = int(raw_id) - except (TypeError, ValueError): - continue - hits.append( - VectorHit( - claim_id=claim_id, - score=float(getattr(h, "score", 0.0) or 0.0), - scope=str(payload.get("scope") or ""), - subject=str(payload.get("subject") or ""), - status=str(payload.get("status") or ""), - confidence=float(payload.get("confidence") or 0.0), - ) - ) - return hits diff --git a/memorymaster/recall/verbatim_store.py b/memorymaster/recall/verbatim_store.py index f135ffa2..e3c92311 100644 --- a/memorymaster/recall/verbatim_store.py +++ b/memorymaster/recall/verbatim_store.py @@ -1,12 +1,12 @@ -"""Verbatim memory store — raw conversation storage with vector search. +"""Verbatim memory store — raw conversation storage with authoritative search. Stores full conversation text without summarization or extraction. Complements the claims DB: claims = curated knowledge, verbatim = raw recall. Search modes: - FTS5 for keyword search (fast, local) - - Qdrant for semantic search (when available) - - Hybrid: FTS5 + Qdrant merged results + - Vector/hybrid requests currently downgrade to FTS5 + - Qdrant remains available only for index synchronization during quarantine """ from __future__ import annotations @@ -20,7 +20,6 @@ from contextlib import closing from datetime import datetime, timezone from pathlib import Path -from typing import Any # Credential detection delegated to the canonical filter in memorymaster.core.security. from memorymaster.core import spool @@ -356,22 +355,15 @@ def search_verbatim( ) -> list[dict]: """Search verbatim memories. - mode: "fts" (keyword), "vector" (Qdrant), "hybrid" (both merged) + Qdrant-backed ``vector`` and ``hybrid`` requests are temporarily + downgraded to authoritative FTS until governed rehydration is available. """ - results = [] - - if mode in ("fts", "hybrid"): - results.extend(_search_fts(db_path, query, scope, limit)) + requested_mode = str(mode).strip().lower() + effective_mode = "fts" if requested_mode in {"vector", "hybrid"} else requested_mode + if effective_mode != "fts": + return [] - if mode in ("vector", "hybrid"): - vector_results = _search_vector(query, scope, limit) - # Merge: dedupe by stable row-id key (see _row_dedup_key for F-3 context) - seen = {_row_dedup_key(r) for r in results} - for vr in vector_results: - key = _row_dedup_key(vr) - if key not in seen: - results.append(vr) - seen.add(key) + results = _search_fts(db_path, query, scope, limit) # Sort by score descending, limit results.sort(key=lambda x: -x.get("score", 0)) @@ -424,59 +416,15 @@ def _search_fts(db_path: str, query: str, scope: str | None, limit: int) -> list def _search_vector(query: str, scope: str | None, limit: int) -> list[dict]: - """Qdrant semantic search over verbatim memories.""" - if not QDRANT_URL: - return [] - try: - # Embed query with OpenAI - api_key = os.environ.get("OPENAI_API_KEY", "") - if not api_key: - return [] - - embed_url = "https://api.openai.com/v1/embeddings" - payload = {"model": "text-embedding-3-small", "input": [query]} - req = urllib.request.Request( - embed_url, - data=json.dumps(payload).encode(), - headers={"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=10) as resp: - result = json.loads(resp.read().decode()) - vector = result["data"][0]["embedding"] - - # Search Qdrant - search_payload: dict[str, Any] = { - "vector": vector, - "limit": limit, - "with_payload": True, - } - if scope: - search_payload["filter"] = {"must": [{"key": "scope", "match": {"value": scope}}]} - - req = urllib.request.Request( - f"{QDRANT_URL}/collections/{QDRANT_COLLECTION}/points/search", - data=json.dumps(search_payload).encode(), - headers={"Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=10) as resp: - result = json.loads(resp.read().decode()) - - return [ - {"id": h.get("id"), "content": h["payload"].get("content", ""), - "content_hash": h["payload"].get("content_hash", ""), - "scope": h["payload"].get("scope", ""), - "session_id": h["payload"].get("session_id", ""), "role": h["payload"].get("role", ""), - "score": h.get("score", 0), "source": "vector"} - for h in result.get("result", []) - ] - except Exception: - return [] + """Reject raw Qdrant payload reads until governed rehydration exists.""" + del query, scope, limit + raise PermissionError( + "Verbatim Qdrant retrieval is quarantined pending authoritative rehydration." + ) def sync_to_qdrant(db_path: str, batch_size: int = 50) -> dict[str, int]: - """Sync unsynced verbatim memories to Qdrant for vector search.""" + """Sync verbatim rows to the Qdrant index; read retrieval is quarantined.""" api_key = os.environ.get("OPENAI_API_KEY", "") if not api_key: return {"synced": 0, "error": "no OPENAI_API_KEY"} diff --git a/memorymaster/surfaces/cli.py b/memorymaster/surfaces/cli.py index b0f6a4f6..a78bac3d 100644 --- a/memorymaster/surfaces/cli.py +++ b/memorymaster/surfaces/cli.py @@ -467,7 +467,14 @@ def build_parser() -> argparse.ArgumentParser: sub.add_parser("drain-spool", help="Replay spooled JSONL write envelopes through the normal service paths (P1 spec §2.4); sensitivity filter + idempotent dedup apply") - qdrant_search = sub.add_parser("qdrant-search", help="Semantic search via Qdrant vector store") + qdrant_search = sub.add_parser( + "qdrant-search", + help="Temporarily disabled: Qdrant retrieval is quarantined pending R2.1", + description=( + "Temporarily disabled: Qdrant retrieval is quarantined pending " + "authoritative policy rehydration in R2.1." + ), + ) qdrant_search.add_argument("text", help="Query text for semantic search") qdrant_search.add_argument("--limit", type=int, default=5, help="Max results (default: 5)") qdrant_search.add_argument("--min-confidence", type=float, default=0.0, help="Minimum confidence filter") @@ -675,7 +682,14 @@ def main(argv: list[str] | None = None) -> int: effective_db = _resolve_db_path(args) # Commands that don't need MemoryService run first; service is lazy-created once for all others. - _NO_SERVICE_COMMANDS = {"stealth-status", "export-metrics", "wiki-freshness", "mcp-usage-report", "export-delta"} + _NO_SERVICE_COMMANDS = { + "stealth-status", + "export-metrics", + "wiki-freshness", + "mcp-usage-report", + "export-delta", + "qdrant-search", + } try: handler = COMMAND_HANDLERS.get(args.command) diff --git a/memorymaster/surfaces/cli_handlers_basic.py b/memorymaster/surfaces/cli_handlers_basic.py index a54fc8f6..bd51b04b 100644 --- a/memorymaster/surfaces/cli_handlers_basic.py +++ b/memorymaster/surfaces/cli_handlers_basic.py @@ -162,6 +162,17 @@ def _handle_snapshot_commands(args: argparse.Namespace, service, parser: argpars def _handle_qdrant_commands(args: argparse.Namespace, service, parser: argparse.ArgumentParser, effective_db: str = "") -> int: """Handle qdrant-sync and qdrant-search subcommands.""" + if args.command == "qdrant-search": + message = ( + "qdrant-search is temporarily disabled by the R1.3 retrieval " + "quarantine; use authoritative query instead" + ) + if args.json_output: + print(_json_error(message)) + else: + print(f"error: {message}") + return 2 + from memorymaster.recall.qdrant_backend import QdrantBackend qdrant_url = args.qdrant_url or os.environ.get("QDRANT_URL") or "" @@ -179,20 +190,7 @@ def _handle_qdrant_commands(args: argparse.Namespace, service, parser: argparse. print(f"Qdrant sync: {result['synced']}/{result['total']} synced, {result['errors']} errors ({elapsed_ms:.0f}ms)") return 0 - # qdrant-search - states = [s.strip() for s in args.states.split(",") if s.strip()] or None - results = backend.search(args.text, limit=args.limit, min_confidence=args.min_confidence, states=states) - elapsed_ms = (time.perf_counter() - t0) * 1000 - if args.json_output: - print(_json_envelope({"results": results, "count": len(results)}, query_ms=elapsed_ms)) - else: - if not results: - print("No results found.") - for hit in results: - _pl = hit.get("payload", {}) - print(f"[{hit.get('claim_id', '?')}] score={hit.get('score', 0.0):.3f} " - f"state={_pl.get('state', '?')} conf={_pl.get('confidence', 0.0):.2f} {_pl.get('claim_text', '')[:100]}") - return 0 + raise AssertionError(f"Unhandled Qdrant command: {args.command}") def _handle_link_commands(args: argparse.Namespace, service, parser: argparse.ArgumentParser, effective_db: str = "") -> int: @@ -945,18 +943,32 @@ def _handle_query(args: argparse.Namespace, service, parser: argparse.ArgumentPa print_claim(c) print(f"rows={len(claims)}") return 0 + requested_retrieval_mode = args.retrieval_mode + effective_retrieval_mode = requested_retrieval_mode + classified_retrieval_mode: str | None = None + containment_reason: str | None = None + query_type: str | None = None if getattr(args, "auto_classify", False): from memorymaster.recall.query_classifier import classify_query, recommended_retrieval_mode - qtype = classify_query(args.text) - retrieval_mode = recommended_retrieval_mode(qtype) - print(f"query classified as: {qtype} → using {retrieval_mode} mode") - args.retrieval_mode = retrieval_mode + query_type = classify_query(args.text) + classified_retrieval_mode = recommended_retrieval_mode(query_type) + effective_retrieval_mode = classified_retrieval_mode + if classified_retrieval_mode == "qdrant": + effective_retrieval_mode = "legacy" + containment_reason = ( + "qdrant retrieval is quarantined pending the governed retrieval planner" + ) + if not args.json_output: + notice = f"query classified as: {query_type} → using {effective_retrieval_mode} mode" + if containment_reason is not None: + notice += " (qdrant recommendation quarantined)" + print(notice) t0 = time.perf_counter() rows_data = service.query_rows( query_text=args.text, limit=args.limit, include_stale=not args.exclude_stale, include_conflicted=not args.exclude_conflicted, include_candidates=getattr(args, "include_candidates", False), - retrieval_mode=args.retrieval_mode, allow_sensitive=args.allow_sensitive, + retrieval_mode=effective_retrieval_mode, allow_sensitive=args.allow_sensitive, retrieval_profile=getattr(args, "profile", None), scope_allowlist=parse_scope_allowlist(args.scope_allowlist), ) @@ -965,7 +977,22 @@ def _handle_query(args: argparse.Namespace, service, parser: argparse.ArgumentPa json_rows = [{"claim": _claim_to_dict(row["claim"]), **{k: float(row.get(k, 0.0)) for k in _SCORE_KEYS}, "annotation": row.get("annotation", {})} for row in rows_data] - print(_json_envelope(json_rows, total=len(json_rows), query_ms=elapsed_ms)) + extra_meta = None + if query_type is not None: + extra_meta = { + "query_type": query_type, + "requested_retrieval_mode": requested_retrieval_mode, + "classified_retrieval_mode": classified_retrieval_mode, + "retrieval_mode": effective_retrieval_mode, + } + if containment_reason is not None: + extra_meta["containment_reason"] = containment_reason + print(_json_envelope( + json_rows, + total=len(json_rows), + query_ms=elapsed_ms, + extra_meta=extra_meta, + )) else: for row in rows_data: print_claim(row["claim"]) diff --git a/memorymaster/surfaces/mcp_server.py b/memorymaster/surfaces/mcp_server.py index 26a6571e..4ff79e15 100644 --- a/memorymaster/surfaces/mcp_server.py +++ b/memorymaster/surfaces/mcp_server.py @@ -490,65 +490,12 @@ def _effective_scope_allowlist(raw: str, workspace: str) -> list[str] | None: def _qdrant_query(query: str, db: str, workspace: str, limit: int) -> dict[str, Any]: - """Fast semantic search via Qdrant+Ollama (no local model load).""" - try: - from memorymaster.recall.qdrant_backend import QdrantBackend - except ImportError: - return {"ok": False, "error": "qdrant mode requires httpx. Install with: pip install 'memorymaster[qdrant]'"} - backend = QdrantBackend() - results = backend.search(query, limit=limit) - backend.close() - if not results: - return {"ok": True, "rows": 0, "claims": [], "rows_data": []} - - # Enrich with full claim data from the DB - svc = _service(db, workspace) - enriched_rows: list[dict[str, Any]] = [] - enriched_claims: list[dict[str, Any]] = [] - for hit in results: - cid = hit.get("claim_id") - if cid is None: - continue - claim = svc.store.get_claim(int(cid), include_citations=True) - if claim is None: - # Claim may have been archived since last sync — return Qdrant payload - enriched_rows.append({ - "claim": hit.get("payload", {}), - "status": hit.get("payload", {}).get("state", "unknown"), - "annotation": {}, - "score": hit.get("score", 0.0), - "lexical_score": 0.0, - "freshness_score": 0.0, - "confidence_score": hit.get("payload", {}).get("confidence", 0.0), - "vector_score": hit.get("score", 0.0), - }) - enriched_claims.append(hit.get("payload", {})) - continue - claim_dict = _claim_to_dict(claim) - enriched_claims.append(claim_dict) - enriched_rows.append({ - "claim": claim_dict, - "status": claim.status, - "annotation": { - "status": claim.status, - "active": claim.status == "confirmed", - "stale": claim.status == "stale", - "conflicted": claim.status == "conflicted", - "pinned": bool(claim.pinned), - }, - "score": hit.get("score", 0.0), - "lexical_score": 0.0, - "freshness_score": 0.0, - "confidence_score": claim.confidence, - "vector_score": hit.get("score", 0.0), - }) - return { - "ok": True, - "rows": len(enriched_claims), - "claims": enriched_claims, - "rows_data": enriched_rows, - "retrieval_mode": "qdrant", - } + """Reject the legacy raw-Qdrant retrieval entrypoint during quarantine.""" + del query, db, workspace, limit + raise PermissionError( + "Direct Qdrant retrieval is quarantined until the governed retrieval " + "planner can enforce authoritative policy rehydration." + ) def _checkpoint_batch( @@ -1146,10 +1093,21 @@ def run_steward( @mcp.tool() def classify_query(query: str) -> dict[str, Any]: - """Classify a query and recommend the best retrieval mode.""" + """Classify a query and report its recommended and effective modes.""" from memorymaster.recall.query_classifier import classify_query as _classify, recommended_retrieval_mode qtype = _classify(query) - return {"query_type": qtype, "recommended_mode": recommended_retrieval_mode(qtype)} + recommended_mode = recommended_retrieval_mode(qtype) + effective_mode = "legacy" if recommended_mode == "qdrant" else recommended_mode + result = { + "query_type": qtype, + "recommended_mode": recommended_mode, + "effective_mode": effective_mode, + } + if recommended_mode == "qdrant": + result["containment_reason"] = ( + "qdrant retrieval is quarantined pending the governed retrieval planner" + ) + return result def _apply_detail_level(claim_dict: dict[str, Any], detail_level: str) -> dict[str, Any]: """Filter claim dict fields based on requested detail level. @@ -1202,7 +1160,7 @@ def query_memory( retrieval_mode options: - "legacy" (default, fastest ~0.1s): SQL text search - - "qdrant" (fast ~0.5s): semantic search via Qdrant+Ollama, requires QDRANT_URL + - "qdrant": temporarily quarantined; falls back to authoritative lexical search - "hybrid" (slow ~8s): local sentence-transformers vector + lexical ranking auto_classify: when True and retrieval_mode is "legacy", classify the query @@ -1220,19 +1178,23 @@ def query_memory( context="mcp.query_memory", ) + requested_retrieval_mode = retrieval_mode + classified_retrieval_mode: str | None = None query_type: str | None = None + containment_reason: str | None = None if auto_classify and retrieval_mode == "legacy": query_type = _classify(query) - retrieval_mode = recommended_retrieval_mode(query_type) + classified_retrieval_mode = recommended_retrieval_mode(query_type) + retrieval_mode = classified_retrieval_mode - # Qdrant retrieval mode: fast semantic search via network Qdrant+Ollama + # Qdrant payloads cannot express the full tenant/scope/visibility + # policy. R1.3 keeps recall useful through the authoritative lexical + # planner until R2.1 can safely reintroduce vector ID candidates. if retrieval_mode == "qdrant": - result = _qdrant_query(query, db, workspace, limit) - if query_type is not None: - result["query_type"] = query_type - if detail_level != "standard": - result["claims"] = [_apply_detail_level(c, detail_level) for c in result.get("claims", [])] - return result + retrieval_mode = "legacy" + containment_reason = ( + "qdrant retrieval is quarantined pending the governed retrieval planner" + ) svc = _service(db, workspace) rows_data = svc.query_rows( @@ -1274,6 +1236,14 @@ def query_memory( } if query_type is not None: response["query_type"] = query_type + if containment_reason is not None: + response.update({ + "requested_retrieval_mode": requested_retrieval_mode, + "retrieval_mode": retrieval_mode, + "containment_reason": containment_reason, + }) + if classified_retrieval_mode is not None: + response["classified_retrieval_mode"] = classified_retrieval_mode # Log to vault chronicle try: @@ -1918,13 +1888,27 @@ def search_verbatim( ) -> dict[str, Any]: """Search raw conversation memories (verbatim, unsummarized). - mode: "fts" (keyword), "vector" (Qdrant semantic), "hybrid" (both) + ``vector`` and ``hybrid`` temporarily use authoritative FTS because + direct Qdrant payload retrieval is quarantined pending rehydration. Use this when query_memory (claims) doesn't find what you need — verbatim search finds exact conversation fragments. """ from memorymaster.recall.verbatim_store import search_verbatim as _search - results = _search(_resolve_db(db), query, scope=scope or None, limit=limit, mode=mode) - return {"ok": True, "rows": len(results), "results": results} + requested_mode = str(mode).strip().lower() + effective_mode = "fts" if requested_mode in {"vector", "hybrid"} else requested_mode + results = _search( + _resolve_db(db), query, scope=scope or None, limit=limit, mode=effective_mode + ) + response = {"ok": True, "rows": len(results), "results": results} + if effective_mode != requested_mode: + response.update({ + "requested_mode": requested_mode, + "mode": effective_mode, + "containment_reason": ( + "verbatim qdrant retrieval is quarantined pending authoritative rehydration" + ), + }) + return response @mcp.tool() def get_usage_rollup( diff --git a/memorymaster/surfaces/setup_detect.py b/memorymaster/surfaces/setup_detect.py index 3d361a13..a50344bb 100644 --- a/memorymaster/surfaces/setup_detect.py +++ b/memorymaster/surfaces/setup_detect.py @@ -303,7 +303,8 @@ def format_plan(d: Detected, *, want_full_stack: bool) -> list[str]: _line( _STEP_CANT, "Qdrant — Docker Compose not found; running in SQLite-only mode. " - "Install Docker or point QDRANT_URL at an existing service to enable vector recall.", + "Install Docker or point QDRANT_URL at an existing service for Qdrant " + "index maintenance; retrieval remains quarantined.", ) ) diff --git a/memorymaster/surfaces/setup_hooks.py b/memorymaster/surfaces/setup_hooks.py index e8371452..ffd39f3e 100644 --- a/memorymaster/surfaces/setup_hooks.py +++ b/memorymaster/surfaces/setup_hooks.py @@ -527,14 +527,14 @@ def install_obsidian_skills(): # 7. Full-stack orchestration (Qdrant + Ollama via Docker Compose) # --------------------------------------------------------------------------- SQLITE_ONLY_MESSAGE = ( - "Running in SQLite-only mode. Vector recall + local LLM auto-ingest are OFF.\n" - " To enable them: install Docker and re-run with --full-stack, or point\n" - " QDRANT_URL / OLLAMA_URL at existing services." + "Running in SQLite-only mode. Qdrant index maintenance + local LLM auto-ingest are OFF.\n" + " Retrieval remains available through authoritative SQLite ranking. To enable index\n" + " maintenance or local LLMs, use --full-stack or QDRANT_URL / OLLAMA_URL." ) def setup_full_stack(detected: Detected, *, interactive: bool, yes: bool, model: str = "") -> dict[str, Any]: - """Bring up the optional vector + local-LLM stack (Qdrant + Ollama). + """Bring up the optional index + local-LLM stack (Qdrant + Ollama). Brownfield: reuse already-healthy services. No-Docker fallback: print the SQLite-only degraded message and CONTINUE (never block core install). @@ -914,7 +914,7 @@ def _run_main(args: argparse.Namespace) -> tuple[int, Optional[dict[str, Any]]]: print(f" - DB: {db_path} | verify: {verify.get('status')}") print(f" - LLM provider: {llm_config['provider']}") if degraded: - print(" - Stack: SQLite-only (degraded) — vector recall + local LLM OFF") + print(" - Stack: SQLite-only — Qdrant index maintenance + local LLM OFF") print() print(" Next steps:") if detected.claude_code or detected.codex: diff --git a/monitoring.md b/monitoring.md index 6a3277ac..79ca8b42 100644 --- a/monitoring.md +++ b/monitoring.md @@ -1,7 +1,7 @@ --- project: memorymaster path: G:\_OneDrive\OneDrive\Desktop\Py Apps\memorymaster -stack: Python 3.10+ / SQLite+FTS5 / Qdrant (optional) / FastMCP +stack: Python 3.10+ / SQLite+FTS5 / Qdrant maintenance index (optional) / FastMCP repo: https://github.com/wolverin0/memorymaster entry_point: "python -m memorymaster --db memorymaster.db " test_command: "python -m pytest tests/ -q --tb=short" @@ -38,7 +38,7 @@ the core abstractions: | `MemoryService` | 124 | Top-level facade in `service.py` — ingest/query/run_cycle | | `Claim` | 115 | Dataclass in `models.py` — the atomic unit of memory | | `CitationInput` | 113 | Source attribution wrapper, required on every ingest | -| `QdrantBackend` | 64 | Vector search backend (optional, `[qdrant]` extra) | +| `QdrantBackend` | 64 | Maintenance-index backend (optional, `[qdrant]` extra); payload search is quarantined | | `PostgresStore` | 54 | Parity backend to SQLiteStore for multi-host deployments | | `EntityGraph` | 51 | Typed-edge relationship graph on top of claim_links | | `FeedbackTracker` | 51 | Records validator/steward outcomes for quality scoring | @@ -87,6 +87,12 @@ relationships shipped. ## Active Issues +- R1.3 quarantines all Qdrant claim/context-fallback/verbatim payload retrieval. + Local-trusted claim and verbatim requests use lexical/FTS fallback; direct + search adapters fail closed; team MCP semantic modes are denied. Monitor + Qdrant only for upsert/sync/reconcile/count-ID drift maintenance until R2.1 + adds governed ID-candidate rehydration through SQLite/Postgres. + - `test_run_stream_resumes_from_checkpoint_state` previously flaky — FIXED in v3.2.2 (was a real bug in `_seek_to_offset`) - GHA runner variance up to 10x on identical commits — SLO thresholds diff --git a/tests/test_qdrant_authoritative_filtering.py b/tests/test_qdrant_authoritative_filtering.py index 8cf18b2c..4205f85f 100644 --- a/tests/test_qdrant_authoritative_filtering.py +++ b/tests/test_qdrant_authoritative_filtering.py @@ -1,4 +1,4 @@ -"""Red contracts for authoritative filtering of Qdrant candidates (MM-SEC-02).""" +"""Containment contracts for authoritative Qdrant retrieval (MM-SEC-02).""" from __future__ import annotations from collections.abc import Iterable @@ -11,21 +11,17 @@ from memorymaster.core.service import MemoryService -AUDIT_BASELINE = pytest.mark.xfail( - strict=True, - reason="audit baseline MM-SEC-02: Qdrant bypasses authoritative policy filtering", -) - - class FakeQdrant: """Network-free vector candidate source controlled by each test.""" def __init__(self, hits: Iterable[dict]) -> None: self._hits = list(hits) self.closed = False + self.search_calls = 0 def search(self, query_text: str, limit: int = 5) -> list[dict]: del query_text + self.search_calls += 1 return self._hits[:limit] def close(self) -> None: @@ -60,7 +56,6 @@ def _init_db(tmp_path) -> tuple[str, str, MemoryService]: return db, str(workspace_path), svc -@AUDIT_BASELINE def test_qdrant_orphan_payload_is_never_returned(tmp_path, monkeypatch) -> None: """A vector point is only an ID candidate; payload is never authoritative.""" db, workspace, _svc = _init_db(tmp_path) @@ -93,10 +88,13 @@ def test_qdrant_orphan_payload_is_never_returned(tmp_path, monkeypatch) -> None: assert result["rows"] == 0 assert result["claims"] == [] - assert fake.closed is True + assert result["requested_retrieval_mode"] == "qdrant" + assert result["retrieval_mode"] == "legacy" + assert result["containment_reason"] + assert fake.search_calls == 0 + assert fake.closed is False -@AUDIT_BASELINE def test_qdrant_filters_archived_and_wrong_scope_rows(tmp_path, monkeypatch) -> None: """Primary-store lifecycle and scope policy must filter every vector hit.""" db, workspace, svc = _init_db(tmp_path) @@ -117,7 +115,7 @@ def test_qdrant_filters_archived_and_wrong_scope_rows(tmp_path, monkeypatch) -> [CitationInput(source="test://foreign")], scope="project:other", ) - _install_fake_qdrant( + fake = _install_fake_qdrant( monkeypatch, [ {"claim_id": archived.id, "score": 0.98, "payload": {"state": "confirmed"}}, @@ -138,3 +136,4 @@ def test_qdrant_filters_archived_and_wrong_scope_rows(tmp_path, monkeypatch) -> assert result["rows"] == 0 assert result["claims"] == [] + assert fake.search_calls == 0 diff --git a/tests/test_qdrant_backend.py b/tests/test_qdrant_backend.py index adfd73de..e224ff29 100644 --- a/tests/test_qdrant_backend.py +++ b/tests/test_qdrant_backend.py @@ -81,11 +81,13 @@ def test_upsert_returns_false_on_embed_failure(self): claim = _fake_claim() assert backend.upsert_claim(claim) is False - def test_search_returns_empty_on_embed_failure(self): + def test_search_is_quarantined_before_embed_failure(self): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") backend._client = MagicMock() backend._client.post.side_effect = Exception("connection refused") - assert backend.search("test query") == [] + with pytest.raises(PermissionError, match="quarantined"): + backend.search("test query") + backend._client.post.assert_not_called() def test_delete_returns_false_on_failure(self): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") @@ -215,27 +217,25 @@ def _make_backend_with_search(self, results=None): backend._client.post.side_effect = [embed_resp, search_resp] return backend - def test_search_returns_results(self): + def test_search_does_not_return_raw_results_during_quarantine(self): hits = [{"payload": {"claim_id": 1}, "score": 0.95}] backend = self._make_backend_with_search(hits) - results = backend.search("test query") - assert len(results) == 1 - assert results[0]["claim_id"] == 1 - assert results[0]["score"] == 0.95 + with pytest.raises(PermissionError, match="quarantined"): + backend.search("test query") + backend._client.post.assert_not_called() - def test_search_empty_results(self): + def test_search_empty_results_still_reports_quarantine(self): backend = self._make_backend_with_search([]) - assert backend.search("nothing") == [] + with pytest.raises(PermissionError, match="quarantined"): + backend.search("nothing") - def test_search_with_filters(self): + def test_search_with_filters_remains_quarantined(self): backend = self._make_backend_with_search([]) - backend.search("test", states=["confirmed"], min_confidence=0.5) - # Second post call is the search — check the body has filters - search_call = backend._client.post.call_args_list[1] - body = search_call[1]["json"] - assert "filter" in body + with pytest.raises(PermissionError, match="quarantined"): + backend.search("test", states=["confirmed"], min_confidence=0.5) + backend._client.post.assert_not_called() - def test_search_qdrant_failure_returns_empty(self): + def test_search_qdrant_failure_is_never_reached(self): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") backend._client = MagicMock() embed_resp = MagicMock() @@ -243,7 +243,9 @@ def test_search_qdrant_failure_returns_empty(self): embed_resp.raise_for_status = MagicMock() embed_resp.json.return_value = {"embeddings": [[0.1] * EMBEDDING_DIMS]} backend._client.post.side_effect = [embed_resp, Exception("qdrant down")] - assert backend.search("test") == [] + with pytest.raises(PermissionError, match="quarantined"): + backend.search("test") + backend._client.post.assert_not_called() class TestEnsureCollection: diff --git a/tests/test_qdrant_retrieval_quarantine.py b/tests/test_qdrant_retrieval_quarantine.py new file mode 100644 index 00000000..e53e508f --- /dev/null +++ b/tests/test_qdrant_retrieval_quarantine.py @@ -0,0 +1,293 @@ +"""Adversarial contracts for quarantining non-authoritative Qdrant reads.""" +from __future__ import annotations + +import json +from typing import Any + +import pytest + +import memorymaster.core.access_control as access_control +import memorymaster.recall.qdrant_backend as qdrant_backend +import memorymaster.recall.query_classifier as query_classifier +import memorymaster.surfaces.mcp_server as mcp_server +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.surfaces.cli import main + + +@pytest.fixture(autouse=True) +def isolated_auth_state(monkeypatch: pytest.MonkeyPatch): + """Keep request authority and rate-limit state independent per test.""" + access_control._agent_roles.clear() + monkeypatch.setattr(access_control, "_loaded", True) + monkeypatch.setenv("MEMORYMASTER_MCP_AUTH_MODE", "local-trusted") + mcp_server._INGEST_RATE_BUCKETS.clear() + yield + mcp_server._INGEST_RATE_BUCKETS.clear() + access_control._agent_roles.clear() + + +def _seed_lexical_claim(tmp_path, text: str) -> tuple[str, str, int]: + workspace = tmp_path / "allowed" + workspace.mkdir() + db = str(tmp_path / "retrieval-quarantine.db") + service = MemoryService(db, workspace_root=workspace) + service.init_db() + claim = service.ingest( + text, + [CitationInput(source="test://retrieval-quarantine")], + scope="project:allowed", + source_agent="seed", + ) + return db, str(workspace), claim.id + + +class _RecordingQdrant: + def __init__(self, calls: list[str]) -> None: + calls.append("constructed") + self._calls = calls + + def search(self, *_args: Any, **_kwargs: Any) -> list[dict[str, Any]]: + self._calls.append("searched") + return [] + + def sync_all(self, _store: Any) -> dict[str, int]: + self._calls.append("synced") + return {"synced": 0, "total": 0, "errors": 0} + + def close(self) -> None: + self._calls.append("closed") + + +def _install_recording_qdrant( + monkeypatch: pytest.MonkeyPatch, + calls: list[str], +) -> None: + monkeypatch.setattr( + qdrant_backend, + "QdrantBackend", + lambda **_kwargs: _RecordingQdrant(calls), + ) + + +def test_local_trusted_explicit_qdrant_falls_back_to_authoritative_lexical( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db, workspace, claim_id = _seed_lexical_claim( + tmp_path, + "quarantine lexical authority marker", + ) + qdrant_calls: list[str] = [] + _install_recording_qdrant(monkeypatch, qdrant_calls) + + result = mcp_server.query_memory( + query="quarantine lexical authority marker", + db=db, + workspace=workspace, + retrieval_mode="qdrant", + scope_allowlist="project:allowed", + ) + + assert qdrant_calls == [] + assert result["requested_retrieval_mode"] == "qdrant" + assert result["retrieval_mode"] == "legacy" + assert result["containment_reason"] + assert {claim["id"] for claim in result["claims"]} == {claim_id} + + +def test_local_trusted_auto_classified_qdrant_falls_back_to_authoritative_lexical( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db, workspace, claim_id = _seed_lexical_claim( + tmp_path, + "auto classified quarantine marker", + ) + qdrant_calls: list[str] = [] + _install_recording_qdrant(monkeypatch, qdrant_calls) + monkeypatch.setattr(query_classifier, "classify_query", lambda _query: "relational") + monkeypatch.setattr( + query_classifier, + "recommended_retrieval_mode", + lambda _query_type: "qdrant", + ) + + result = mcp_server.query_memory( + query="auto classified quarantine marker", + db=db, + workspace=workspace, + retrieval_mode="legacy", + auto_classify=True, + scope_allowlist="project:allowed", + ) + + assert qdrant_calls == [] + assert result["query_type"] == "relational" + assert result["requested_retrieval_mode"] == "legacy" + assert result["classified_retrieval_mode"] == "qdrant" + assert result["retrieval_mode"] == "legacy" + assert result["containment_reason"] + assert {claim["id"] for claim in result["claims"]} == {claim_id} + + +def test_classify_query_reports_quarantined_effective_mode( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(query_classifier, "classify_query", lambda _query: "relational") + monkeypatch.setattr( + query_classifier, + "recommended_retrieval_mode", + lambda _query_type: "qdrant", + ) + + result = mcp_server.classify_query("what depends on this?") + + assert result["query_type"] == "relational" + assert result["recommended_mode"] == "qdrant" + assert result["effective_mode"] == "legacy" + assert result["containment_reason"] + + +@pytest.mark.parametrize( + "semantic_args", + [ + {"retrieval_mode": "qdrant"}, + {"retrieval_mode": "legacy", "auto_classify": True}, + ], +) +def test_team_mode_semantic_retrieval_remains_denied_before_body( + tmp_path, + monkeypatch: pytest.MonkeyPatch, + semantic_args: dict[str, Any], +) -> None: + workspace = tmp_path / "team-workspace" + workspace.mkdir() + access_control.set_role("mcp-reader", access_control.Role.READER) + monkeypatch.setenv("MEMORYMASTER_MCP_AUTH_MODE", "team") + monkeypatch.setenv("MEMORYMASTER_MCP_PRINCIPAL", "mcp-reader") + monkeypatch.setenv("MEMORYMASTER_MCP_TENANT_ID", "tenant-alpha") + monkeypatch.setenv("MEMORYMASTER_MCP_WORKSPACE", str(workspace)) + monkeypatch.setenv("MEMORYMASTER_MCP_ALLOWED_SCOPES", "project:alpha,global") + monkeypatch.setenv("MEMORYMASTER_MCP_DB", str(tmp_path / "team.db")) + monkeypatch.setattr( + mcp_server, + "_service", + lambda *_args, **_kwargs: pytest.fail("semantic denial reached the service"), + ) + monkeypatch.setattr( + mcp_server, + "_qdrant_query", + lambda *_args, **_kwargs: pytest.fail("semantic denial reached Qdrant"), + ) + + with pytest.raises(PermissionError, match="(?i)(semantic|team|disabled)"): + mcp_server.query_memory(query="team semantic request", **semantic_args) + + +def test_cli_qdrant_search_is_denied_before_backend_construction( + tmp_path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + qdrant_calls: list[str] = [] + _install_recording_qdrant(monkeypatch, qdrant_calls) + monkeypatch.setenv("QDRANT_URL", "http://qdrant.invalid") + + return_code = main( + ["--db", str(tmp_path / "cli.db"), "qdrant-search", "quarantined query"] + ) + output = capsys.readouterr().out.lower() + + assert qdrant_calls == [] + assert return_code == 2 + assert "qdrant" in output + + +def test_cli_qdrant_search_returns_code_two( + tmp_path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + qdrant_calls: list[str] = [] + _install_recording_qdrant(monkeypatch, qdrant_calls) + + return_code = main( + ["--db", str(tmp_path / "cli.db"), "qdrant-search", "quarantined query"] + ) + output = capsys.readouterr().out.lower() + + assert return_code == 2 + assert "qdrant" in output + + +def test_cli_qdrant_search_json_error_is_one_valid_document( + tmp_path, + capsys: pytest.CaptureFixture[str], +) -> None: + return_code = main( + ["--json", "--db", str(tmp_path / "cli.db"), "qdrant-search", "query"] + ) + + payload = json.loads(capsys.readouterr().out) + assert return_code == 2 + assert payload["ok"] is False + assert "qdrant" in payload["error"].lower() + + +def test_cli_auto_classified_qdrant_uses_lexical_and_valid_json( + tmp_path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + db, workspace, claim_id = _seed_lexical_claim( + tmp_path, + "cli auto classified quarantine marker", + ) + monkeypatch.setattr(query_classifier, "classify_query", lambda _query: "relational") + monkeypatch.setattr( + query_classifier, + "recommended_retrieval_mode", + lambda _query_type: "qdrant", + ) + + return_code = main( + [ + "--json", + "--db", + db, + "--workspace", + workspace, + "query", + "cli auto classified quarantine marker", + "--auto-classify", + "--include-candidates", + "--scope-allowlist", + "project:allowed", + ] + ) + + payload = json.loads(capsys.readouterr().out) + assert return_code == 0 + assert {row["claim"]["id"] for row in payload["data"]} == {claim_id} + assert payload["meta"]["query_type"] == "relational" + assert payload["meta"]["requested_retrieval_mode"] == "legacy" + assert payload["meta"]["classified_retrieval_mode"] == "qdrant" + assert payload["meta"]["retrieval_mode"] == "legacy" + assert payload["meta"]["containment_reason"] + + +def test_cli_qdrant_sync_remains_available( + tmp_path, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], +) -> None: + qdrant_calls: list[str] = [] + _install_recording_qdrant(monkeypatch, qdrant_calls) + + return_code = main(["--db", str(tmp_path / "cli.db"), "qdrant-sync"]) + output = capsys.readouterr().out + + assert return_code == 0 + assert "Qdrant sync" in output + assert qdrant_calls == ["constructed", "synced"] diff --git a/tests/test_recall_vector_fallback.py b/tests/test_recall_vector_fallback.py index 83991b05..28d16e26 100644 --- a/tests/test_recall_vector_fallback.py +++ b/tests/test_recall_vector_fallback.py @@ -1,16 +1,9 @@ -"""Tests for the Qdrant vector-search recall fallback. - -Covers: - 1. Fallback is inert without MEMORYMASTER_RECALL_VECTOR_FALLBACK=1. - 2. Fallback activates only when ``len(rows) < threshold`` (default 3). - 3. Graceful degradation when Qdrant client raises (import error, network, - missing collection, etc). - 4. When W_VECTOR=0, ranking is bit-identical to pre-fallback (rows may - be added but contribute zero score). - 5. Env-override knobs (threshold, score_threshold, limit) parse correctly. - -Uses an in-memory SQLite DB with a handful of synthetic claims and mocks -out the sentence-transformers embedder + qdrant_client.QdrantClient. +"""Prompt-recall quarantine contracts for the legacy Qdrant fallbacks. + +During R1.3 containment, prompt recall must use authoritative lexical rows +only. Neither the opt-in vector fallback nor the raw-payload Qdrant fast path +may run, even when their legacy environment gates are enabled. Pure helper +contracts remain covered for the later governed R2.1 reintegration. """ from __future__ import annotations @@ -120,6 +113,21 @@ def query_points(self, **kwargs): return _FakeQueryResponse(self._hits) +class _FakeRawBackend: + def __init__(self, hits: list[dict]) -> None: + self._hits = hits + self.calls = 0 + self.closed = False + + def search(self, query: str, *, limit: int = 5) -> list[dict]: + del query + self.calls += 1 + return self._hits[:limit] + + def close(self) -> None: + self.closed = True + + def _install_mocks(monkeypatch, hits, *, fail_mode=None, embedder_fail=False): fake_client = _FakeClient(hits, fail_mode=fail_mode) @@ -137,6 +145,14 @@ def _get_client(): return fake_client +def _install_raw_backend(monkeypatch, hits): + from memorymaster.recall import qdrant_backend + + fake_backend = _FakeRawBackend(hits) + monkeypatch.setattr(qdrant_backend, "QdrantBackend", lambda: fake_backend) + return fake_backend + + # --------------------------------------------------------------------------- # Tests # --------------------------------------------------------------------------- @@ -158,105 +174,87 @@ def test_fallback_inert_without_env(service, monkeypatch): assert "vector_fallback" not in out -def test_fallback_triggers_when_rows_under_threshold(service, monkeypatch): - """With env enabled and <3 primary rows, fallback adds rows.""" - svc, seeded = service - monkeypatch.setenv("MEMORYMASTER_RECALL_VECTOR_FALLBACK", "1") - monkeypatch.setenv("MEMORYMASTER_QDRANT_URL", "http://mocked.local:6333") - monkeypatch.setenv("MEMORYMASTER_RECALL_W_VECTOR", "0.2") - - hit = _FakeHit(score=0.9, payload={"id": seeded[1], "scope": "test", - "subject": "session continuation", - "status": "confirmed", - "confidence": 0.7}) - fake_client = _install_mocks(monkeypatch, [hit]) - - # Use a prompt that FTS5 won't match — guarantees <3 primary candidates. - out = context_hook.recall("zzzzz-noword-noword-noword", - db_path=svc._test_db_path, skip_qdrant=True) - assert fake_client.calls == 1, "fallback should have queried qdrant" - assert "session based on recency" in out or "Claude CLI" in out - - -def test_fallback_skipped_when_rows_ge_threshold(service, monkeypatch): - """When FTS5 already returns >= MIN_CANDIDATES rows, no qdrant search.""" +@pytest.mark.parametrize("skip_qdrant", [False, True]) +def test_prompt_recall_never_invokes_vector_fallback_when_env_enabled( + service, monkeypatch, skip_qdrant +): + """Containment is unconditional across both legacy skip-qdrant modes.""" svc, _seeded = service monkeypatch.setenv("MEMORYMASTER_RECALL_VECTOR_FALLBACK", "1") monkeypatch.setenv("MEMORYMASTER_QDRANT_URL", "http://mocked.local:6333") - # Force threshold=1 so the seeded claims will always exceed it. - monkeypatch.setenv("MEMORYMASTER_RECALL_VECTOR_MIN_CANDIDATES", "1") - - hit = _FakeHit(score=0.9, payload={"id": 99999, "scope": "test", - "subject": "unrelated", - "status": "confirmed", - "confidence": 0.7}) - fake_client = _install_mocks(monkeypatch, [hit]) + fake_client = _install_mocks(monkeypatch, []) + fake_backend = _install_raw_backend( + monkeypatch, + [{"payload": {"claim_id": 999_999, "claim_text": "raw orphan"}}], + ) + + out = context_hook.recall( + "zzzzz-noword-noword-noword", + db_path=svc._test_db_path, + skip_qdrant=skip_qdrant, + ) - # A prompt we know matches FTS5 (seeded "tokenizer stoplist audit"). - out = context_hook.recall("tokenizer stoplist", - db_path=svc._test_db_path, skip_qdrant=True) assert fake_client.calls == 0 - assert "stopwords" in out + assert fake_backend.calls == 0 + assert out == "" -def test_graceful_degradation_on_qdrant_failure(service, monkeypatch): - """Qdrant unreachable → fallback swallows, recall still returns rows.""" +def test_prompt_recall_never_renders_orphan_raw_claim_text(service, monkeypatch): + """A Qdrant payload cannot become prompt context without a primary row.""" svc, _seeded = service monkeypatch.setenv("MEMORYMASTER_RECALL_VECTOR_FALLBACK", "1") monkeypatch.setenv("MEMORYMASTER_QDRANT_URL", "http://mocked.local:6333") - - fake_client = _install_mocks(monkeypatch, [], fail_mode="query") - # Prompt that still matches at least one seeded claim so we can assert non-empty. - out = context_hook.recall("tokenizer", - db_path=svc._test_db_path, skip_qdrant=True) - # Client was called, but threw — no crash, output is whatever FTS5 produced. - assert fake_client.calls == 1 - assert "stopwords" in out - - -def test_graceful_degradation_on_embedder_failure(service, monkeypatch): - """Sentence-transformers import error → fallback silently skips.""" - svc, _seeded = service + _install_mocks(monkeypatch, []) + orphan_text = "ORPHAN-RAW-CLAIM-TEXT-MUST-NOT-ESCAPE" + fake_backend = _install_raw_backend( + monkeypatch, + [{"payload": {"claim_id": 999_999, "claim_text": orphan_text}}], + ) + + out = context_hook.recall( + "zzzzz-noword-noword-noword", + db_path=svc._test_db_path, + skip_qdrant=False, + ) + + assert orphan_text not in out + assert fake_backend.calls == 0 + assert out == "" + + +@pytest.mark.parametrize("skip_qdrant", [False, True]) +def test_prompt_recall_keeps_authoritative_lexical_results_during_quarantine( + service, monkeypatch, skip_qdrant +): + """Containment removes vector candidates without degrading lexical recall.""" + svc, seeded = service monkeypatch.setenv("MEMORYMASTER_RECALL_VECTOR_FALLBACK", "1") monkeypatch.setenv("MEMORYMASTER_QDRANT_URL", "http://mocked.local:6333") + monkeypatch.setenv("MEMORYMASTER_RECALL_W_VECTOR", "1.0") + + hit = _FakeHit( + score=0.95, + payload={ + "id": seeded[1], + "scope": "test", + "subject": "session continuation", + "status": "confirmed", + "confidence": 0.7, + }, + ) + fake_client = _install_mocks(monkeypatch, [hit]) + fake_backend = _install_raw_backend(monkeypatch, []) - fake_client = _install_mocks(monkeypatch, [], embedder_fail=True) - out = context_hook.recall("tokenizer", - db_path=svc._test_db_path, skip_qdrant=True) - assert fake_client.calls == 0 - assert "stopwords" in out - + out = context_hook.recall( + "tokenizer stoplist", + db_path=svc._test_db_path, + skip_qdrant=skip_qdrant, + ) -def test_w_vector_zero_is_additive_but_score_neutral(service, monkeypatch): - """With W_VECTOR=0, fallback may append rows, but they contribute 0 - score — ranking of pre-existing rows must be bit-identical. - """ - svc, seeded = service - monkeypatch.setenv("MEMORYMASTER_RECALL_VECTOR_FALLBACK", "1") - monkeypatch.setenv("MEMORYMASTER_QDRANT_URL", "http://mocked.local:6333") - # Explicitly keep W_VECTOR=0 (the shipped default) — leave var unset. - - hit = _FakeHit(score=0.95, payload={"id": seeded[1], "scope": "test", - "subject": "session continuation", - "status": "confirmed", - "confidence": 0.7}) - _install_mocks(monkeypatch, [hit]) - - # Compare: same prompt with fallback OFF vs ON (at W_VECTOR=0). - out_on = context_hook.recall("zzzzz-noword-noword-noword", - db_path=svc._test_db_path, skip_qdrant=True) - monkeypatch.delenv("MEMORYMASTER_RECALL_VECTOR_FALLBACK") - qdrant_recall_fallback.reset_singletons_for_tests() - out_off = context_hook.recall("zzzzz-noword-noword-noword", - db_path=svc._test_db_path, skip_qdrant=True) - # Vector hit appears ONLY in the ON variant. - if out_off: - # Every line in `out_off` must also appear in `out_on` (identical order - # at the front — fallback rows go at the tail because they have score 0). - off_lines = out_off.strip().splitlines() - on_lines = out_on.strip().splitlines() - head = on_lines[: len(off_lines)] - assert head == off_lines, "W_VECTOR=0 must preserve existing ordering" + assert "The recall tokenizer drops" in out + assert "Claude CLI --continue" not in out + assert fake_client.calls == 0 + assert fake_backend.calls == 0 def test_env_knob_parsing(monkeypatch): @@ -278,11 +276,11 @@ def test_env_knob_parsing(monkeypatch): assert qdrant_recall_fallback.search_limit() == qdrant_recall_fallback.DEFAULT_LIMIT -def test_is_fallback_enabled_requires_both_vars(monkeypatch): +def test_is_fallback_enabled_stays_false_during_quarantine(monkeypatch): monkeypatch.setenv("MEMORYMASTER_RECALL_VECTOR_FALLBACK", "1") assert qdrant_recall_fallback.is_fallback_enabled() is False # no URL monkeypatch.setenv("MEMORYMASTER_QDRANT_URL", "http://x.y:6333") - assert qdrant_recall_fallback.is_fallback_enabled() is True + assert qdrant_recall_fallback.is_fallback_enabled() is False monkeypatch.setenv("MEMORYMASTER_RECALL_VECTOR_FALLBACK", "0") assert qdrant_recall_fallback.is_fallback_enabled() is False diff --git a/tests/test_verbatim_qdrant_quarantine.py b/tests/test_verbatim_qdrant_quarantine.py new file mode 100644 index 00000000..aa514be2 --- /dev/null +++ b/tests/test_verbatim_qdrant_quarantine.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import pytest + +from memorymaster.recall import verbatim_store +from memorymaster.recall import qdrant_recall_fallback +from memorymaster.recall.qdrant_backend import QdrantBackend +from memorymaster.surfaces import mcp_server + + +@pytest.mark.parametrize("requested_mode", ["vector", "hybrid"]) +def test_public_verbatim_search_downgrades_qdrant_modes_to_fts( + monkeypatch: pytest.MonkeyPatch, + requested_mode: str, +) -> None: + authoritative = [ + { + "id": 7, + "content": "authoritative SQLite row", + "scope": "project:test", + "score": 1.0, + "source": "fts", + } + ] + monkeypatch.setattr( + verbatim_store, + "_search_fts", + lambda db_path, query, scope, limit: authoritative, + ) + monkeypatch.setattr( + verbatim_store, + "_search_vector", + lambda *args, **kwargs: pytest.fail("quarantined Qdrant search was invoked"), + ) + + rows = verbatim_store.search_verbatim( + "unused.db", + "policy boundary", + scope="project:test", + mode=requested_mode, + ) + + assert rows == authoritative + + +def test_public_verbatim_search_never_returns_raw_qdrant_payload( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(verbatim_store, "_search_fts", lambda *args, **kwargs: []) + monkeypatch.setattr( + verbatim_store, + "_search_vector", + lambda *args, **kwargs: [ + { + "id": "orphan-point", + "content": "raw payload must not escape", + "scope": "project:other-tenant", + "source": "vector", + "score": 1.0, + } + ], + ) + + rows = verbatim_store.search_verbatim( + "unused.db", + "raw payload", + scope="project:test", + mode="vector", + ) + + assert rows == [] + + +def test_internal_verbatim_qdrant_adapter_fails_before_network( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(verbatim_store, "QDRANT_URL", "http://qdrant.invalid") + monkeypatch.setattr( + verbatim_store.urllib.request, + "urlopen", + lambda *args, **kwargs: pytest.fail("quarantined adapter reached the network"), + ) + + with pytest.raises(PermissionError, match="quarantined"): + verbatim_store._search_vector("raw payload", "project:test", 5) + + +def test_claim_qdrant_backend_search_fails_before_embedding( + monkeypatch: pytest.MonkeyPatch, +) -> None: + backend = QdrantBackend( + qdrant_url="http://qdrant.invalid", + ollama_url="http://ollama.invalid", + ) + monkeypatch.setattr( + backend, + "_embed", + lambda *args, **kwargs: pytest.fail("quarantined adapter embedded the query"), + ) + + with pytest.raises(PermissionError, match="quarantined"): + backend.search("raw payload") + + backend.close() + + +def test_recall_fallback_search_fails_before_loading_models( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + qdrant_recall_fallback, + "_get_embedder", + lambda: pytest.fail("quarantined adapter loaded the embedding model"), + ) + + with pytest.raises(PermissionError, match="quarantined"): + qdrant_recall_fallback.search("raw payload") + + +@pytest.mark.parametrize("requested_mode", ["vector", "hybrid"]) +def test_mcp_verbatim_search_reports_qdrant_containment( + monkeypatch: pytest.MonkeyPatch, + tmp_path, + requested_mode: str, +) -> None: + observed: dict[str, str] = {} + + def fake_search(db_path, query, scope, limit, mode): + observed["mode"] = mode + return [] + + monkeypatch.setattr(verbatim_store, "search_verbatim", fake_search) + + result = mcp_server.search_verbatim( + query="policy boundary", + db=str(tmp_path / "verbatim.db"), + scope="project:test", + mode=requested_mode, + ) + + assert observed["mode"] == "fts" + assert result["requested_mode"] == requested_mode + assert result["mode"] == "fts" + assert result["containment_reason"] diff --git a/tests/test_verbatim_store.py b/tests/test_verbatim_store.py index 2a91ff54..4db2619f 100644 --- a/tests/test_verbatim_store.py +++ b/tests/test_verbatim_store.py @@ -2,10 +2,12 @@ import sqlite3 +import pytest + from memorymaster.recall import verbatim_store -def test_hybrid_search_keeps_same_prefix_distinct_content(tmp_path, monkeypatch): +def test_hybrid_search_uses_authoritative_fts_during_quarantine(tmp_path, monkeypatch): db_path = tmp_path / "verbatim.db" prefix = "x" * 100 contents = [f"{prefix} distinct suffix {idx}" for idx in range(3)] @@ -43,18 +45,11 @@ def test_hybrid_search_keeps_same_prefix_distinct_content(tmp_path, monkeypatch) conn.commit() conn.close() - vector_results = [ - { - "session_id": "session", - "role": "user", - "content": content, - "scope": "project:test", - "score": 1.0 - (idx * 0.01), - "source": "vector", - } - for idx, content in enumerate(contents) - ] - monkeypatch.setattr(verbatim_store, "_search_vector", lambda *args: vector_results) + monkeypatch.setattr( + verbatim_store, + "_search_vector", + lambda *args: pytest.fail("quarantined vector adapter was called"), + ) results = verbatim_store.search_verbatim( str(db_path), @@ -64,4 +59,4 @@ def test_hybrid_search_keeps_same_prefix_distinct_content(tmp_path, monkeypatch) mode="hybrid", ) - assert [r["content"] for r in results] == contents + assert results == [] diff --git a/tests/test_verbatim_store_qdrant.py b/tests/test_verbatim_store_qdrant.py index f3dd91ff..edb44c8d 100644 --- a/tests/test_verbatim_store_qdrant.py +++ b/tests/test_verbatim_store_qdrant.py @@ -46,32 +46,12 @@ def _create_verbatim_db(db_path): conn.close() -def test_hybrid_search_keeps_qdrant_rows_with_same_2000_char_prefix(tmp_path, monkeypatch): +def test_hybrid_search_does_not_read_qdrant_payloads_during_quarantine(tmp_path, monkeypatch): db_path = tmp_path / "verbatim.db" _create_verbatim_db(db_path) - prefix = "x" * 2000 - full_contents = [f"{prefix} unique suffix {idx}" for idx in range(3)] - qdrant_rows = [ - { - "id": idx, - "score": 1.0 - (idx * 0.01), - "payload": { - "content": content[:2000], - "content_hash": hashlib.sha256(content.encode()).hexdigest(), - "scope": "project:test", - "session_id": "session", - "role": "user", - }, - } - for idx, content in enumerate(full_contents, start=1) - ] - def fake_urlopen(req, timeout): - if req.full_url == "https://api.openai.com/v1/embeddings": - return _Response({"data": [{"embedding": [0.0] * verbatim_store.EMBED_DIM}]}) - assert req.full_url.endswith("/points/search") - return _Response({"result": qdrant_rows}) + pytest.fail(f"quarantined search reached network: {req.full_url}") monkeypatch.setenv("OPENAI_API_KEY", "test-key") monkeypatch.setattr(verbatim_store, "QDRANT_URL", "http://test-qdrant:6333") @@ -85,11 +65,7 @@ def fake_urlopen(req, timeout): mode="hybrid", ) - assert [r["id"] for r in results] == [1, 2, 3] - assert [r["content_hash"] for r in results] == [ - hashlib.sha256(content.encode()).hexdigest() - for content in full_contents - ] + assert results == [] def test_sync_to_qdrant_payload_uses_full_content_hash(tmp_path, monkeypatch): From c0ff5038cf3e350caced68876e4df3692bfdb2d7 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 09:07:22 -0300 Subject: [PATCH 14/55] fix(security): gate canonical claim envelope --- memorymaster/core/security.py | 150 +++++++++++++------ memorymaster/core/service.py | 62 ++++++-- tests/test_persisted_envelope_sensitivity.py | 150 +++++++++++++++++-- 3 files changed, 293 insertions(+), 69 deletions(-) diff --git a/memorymaster/core/security.py b/memorymaster/core/security.py index 4199568f..e869798f 100644 --- a/memorymaster/core/security.py +++ b/memorymaster/core/security.py @@ -164,6 +164,16 @@ class SanitizedClaimInput: predicate: str | None = None +class SensitiveMetadataError(ValueError): + """Fail-closed metadata rejection that never echoes the supplied value.""" + + def __init__(self, field: str, findings: list[str]) -> None: + self.field = field + self.findings = tuple(sorted(set(findings))) + labels = ", ".join(self.findings) + super().__init__(f"{field} contains sensitive data ({labels})") + + def _as_bool(value: object, *, field: str) -> bool: if isinstance(value, bool): return value @@ -450,6 +460,48 @@ def scan_text_for_findings(text: str) -> list[str]: return findings +def _iter_persisted_strings(value: object) -> Iterator[str]: + if isinstance(value, str): + yield value + return + if isinstance(value, Mapping): + for key, nested in value.items(): + if isinstance(key, str): + yield key + yield from _iter_persisted_strings(nested) + return + if isinstance(value, (list, tuple, set, frozenset)): + for nested in value: + yield from _iter_persisted_strings(nested) + + +def scan_persisted_value(value: object) -> list[str]: + """Scan every string leaf/key in a persisted scalar or JSON-like value.""" + findings: list[str] = [] + for text in _iter_persisted_strings(value): + for finding in scan_text_for_findings(text): + if finding not in findings: + findings.append(finding) + return findings + + +def validate_persisted_metadata(fields: Mapping[str, object]) -> None: + """Reject secret-shaped identifiers/provenance before any durable write.""" + for field, value in fields.items(): + findings = scan_persisted_value(value) + if findings: + raise SensitiveMetadataError(field, findings) + + +def sanitize_persisted_text(text: str) -> tuple[str, list[str]]: + """Redact literal secrets and neutralize encoded secrets in content fields.""" + redacted, literal_findings = _redact(text) + findings = sorted(set(literal_findings + scan_text_for_findings(text))) + if findings and scan_text_for_findings(redacted): + redacted = "[REDACTED:encoded_secret]" + return redacted, findings + + def _get_fernet(): key = os.getenv(_ENCRYPTION_ENV_VAR) if not key: @@ -472,6 +524,24 @@ def _encrypt_payload(payload: dict[str, object]) -> str | None: return base64.urlsafe_b64encode(fernet.encrypt(raw)).decode("utf-8") +def _sanitize_optional_claim_text(value: str | None) -> tuple[str | None, list[str]]: + if value is None: + return None, [] + return sanitize_persisted_text(value) + + +def _sanitize_claim_citations( + citations: list[CitationInput], +) -> tuple[list[CitationInput], list[str]]: + sanitized: list[CitationInput] = [] + findings: list[str] = [] + for citation in citations: + excerpt, excerpt_findings = _sanitize_optional_claim_text(citation.excerpt) + findings.extend(excerpt_findings) + sanitized.append(CitationInput(citation.source, citation.locator, excerpt)) + return sanitized, findings + + def sanitize_claim_input( *, text: str, @@ -479,50 +549,44 @@ def sanitize_claim_input( citations: list[CitationInput], subject: str | None = None, predicate: str | None = None, + idempotency_key: str | None = None, + claim_type: str | None = None, + scope: str | None = None, + volatility: str | None = None, + source_agent: str | None = None, + visibility: str | None = None, + holder: str | None = None, + confidence: object = None, + event_time: str | None = None, + valid_from: str | None = None, + valid_until: str | None = None, + intake_batch_id: str | None = None, ) -> SanitizedClaimInput: - redacted_text, findings = _redact(text) - redacted_object = object_value - object_findings: list[str] = [] - if object_value: - redacted_object, object_findings = _redact(object_value) - findings.extend(object_findings) - - # subject/predicate are structured-claim fields that reach the store - # alongside text/object_value. They are exposed MCP ingest parameters, so a - # secret placed there must be caught by the ingest filter — the last line of - # defense — not only at display time. (audit: ingest-subject-skips-filter) - redacted_subject = subject - if subject: - redacted_subject, subject_findings = _redact(subject) - findings.extend(subject_findings) - redacted_predicate = predicate - if predicate: - redacted_predicate, predicate_findings = _redact(predicate) - findings.extend(predicate_findings) - - sanitized_citations: list[CitationInput] = [] - citation_findings: list[str] = [] - for cite in citations: - excerpt = cite.excerpt - if excerpt: - excerpt, c_findings = _redact(excerpt) - citation_findings.extend(c_findings) - sanitized_citations.append(CitationInput(source=cite.source, locator=cite.locator, excerpt=excerpt)) - findings.extend(citation_findings) - - # Encoded-secret sweep (audit: ingest-encoded-secret): a credential hidden - # behind base64/hex/confusable encoding survives the literal `_redact` - # substitution above (the regexes don't match the encoded bytes). Scan the - # decoded variants of every inbound field so the claim is flagged sensitive - # (and thus encrypted-at-rest / hidden from recall) even when the raw text - # we persist still carries the encoded form. - for raw_field in (text, object_value, subject, predicate): - if raw_field: - findings.extend(scan_text_for_findings(raw_field)) - for cite in citations: - if cite.excerpt: - findings.extend(scan_text_for_findings(cite.excerpt)) - + citation_metadata = { + "citation_source": [citation.source for citation in citations], + "citation_locator": [citation.locator for citation in citations], + } + validate_persisted_metadata({ + "idempotency_key": idempotency_key, + "claim_type": claim_type, + "scope": scope, + "volatility": volatility, + "source_agent": source_agent, + "visibility": visibility, + "holder": holder, + "confidence": confidence, + "event_time": event_time, + "valid_from": valid_from, + "valid_until": valid_until, + "intake_batch_id": intake_batch_id, + **citation_metadata, + }) + redacted_text, findings = sanitize_persisted_text(text) + redacted_object, object_findings = _sanitize_optional_claim_text(object_value) + redacted_subject, subject_findings = _sanitize_optional_claim_text(subject) + redacted_predicate, predicate_findings = _sanitize_optional_claim_text(predicate) + sanitized_citations, citation_findings = _sanitize_claim_citations(citations) + findings.extend(object_findings + subject_findings + predicate_findings + citation_findings) dedup_findings = sorted(set(findings)) is_sensitive = len(dedup_findings) > 0 encrypted_payload = _encrypt_payload( diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index 0a826657..6ac10baf 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -19,7 +19,12 @@ from memorymaster.recall.context_optimizer import ContextResult, pack_context from memorymaster.core.config import get_config from memorymaster.recall.retrieval import VectorSearchHook, _tier_bonus, rank_claim_rows -from memorymaster.core.security import is_sensitive_claim, resolve_allow_sensitive_access, sanitize_claim_input +from memorymaster.core.security import ( + is_sensitive_claim, + resolve_allow_sensitive_access, + sanitize_claim_input, + validate_persisted_metadata, +) from memorymaster.core.intake_policy import ( IntakePolicyConfig, IntakeRejected, @@ -530,12 +535,46 @@ def ingest( ) -> Claim: if not text.strip(): raise ValueError("Claim text cannot be empty.") + if not citations: + citations = [CitationInput(source="mcp-session", locator=scope or "project")] + sanitized = sanitize_claim_input( + text=text.strip(), + object_value=object_value, + citations=citations, + subject=subject, + predicate=predicate, + idempotency_key=idempotency_key, + claim_type=claim_type, + scope=scope, + volatility=volatility, + source_agent=source_agent, + visibility=visibility, + holder=holder, + confidence=confidence, + event_time=event_time, + valid_from=valid_from, + valid_until=valid_until, + intake_batch_id=intake_batch_id, + ) + if not sanitized.citations: + raise ValueError("At least one citation is required.") + text = sanitized.text + object_value = sanitized.object_value + subject = sanitized.subject + predicate = sanitized.predicate + citations = sanitized.citations visibility, source_agent = self._prepare_ingest_identity( scope, visibility, source_agent, require_source_agent=require_source_agent, ) + validate_persisted_metadata( + { + "effective_source_agent": source_agent, + "tenant_id": self.tenant_id, + } + ) # Bitemporal write-time guard: reject malformed ISO-8601 or an inverted # validity interval at the boundary, before any dedup/sanitize work, so # a durable-but-invisible row (valid_until < valid_from) never reaches @@ -556,8 +595,6 @@ def ingest( _vu = _parse_iso_strict("valid_until", valid_until) if _vu is not None and _vu <= datetime.now(timezone.utc): valid_from = valid_until - if not citations: - citations = [CitationInput(source="mcp-session", locator=scope or "project")] # Normalize claim_type to lowercase so routing hints like "DECISION" # from the classify hook don't create a separate type from "decision". if claim_type: @@ -595,19 +632,6 @@ def ingest( # Set content hash as idempotency key if none provided if normalized_idempotency_key is None: normalized_idempotency_key = content_hash - sanitized = sanitize_claim_input( - text=text.strip(), - object_value=object_value, - citations=citations, - subject=subject, - predicate=predicate, - ) - if not sanitized.citations: - raise ValueError("At least one citation is required.") - # Use the sanitized subject/predicate everywhere downstream so a secret - # placed in those fields is redacted at rest, not just at display time. - subject = sanitized.subject - predicate = sanitized.predicate # Intake policy (P3) — runs AFTER the sacred sensitivity filter above and # BEFORE create_claim. Additive admission control: may reject more or # default-tag attribution, never weakens the filter or flips a prior @@ -650,6 +674,12 @@ def ingest( source_agent, allow_sensitive=not getattr(self, "require_tenant", False), ) + validate_persisted_metadata( + { + "effective_source_agent": source_agent, + "tenant_id": self.tenant_id, + } + ) # Resolve subject → canonical entity (GBrain-inspired entity registry) # and mine text for pattern-based entities (#127 Wave 3). entity_id = 0 diff --git a/tests/test_persisted_envelope_sensitivity.py b/tests/test_persisted_envelope_sensitivity.py index 9b150c0d..76ae5cbe 100644 --- a/tests/test_persisted_envelope_sensitivity.py +++ b/tests/test_persisted_envelope_sensitivity.py @@ -6,13 +6,14 @@ """ from __future__ import annotations +import base64 import sqlite3 from pathlib import Path import pytest from memorymaster.core.models import CitationInput -from memorymaster.core.security import scan_text_for_findings +from memorymaster.core.security import SensitiveMetadataError, scan_text_for_findings from memorymaster.core.service import MemoryService @@ -23,6 +24,10 @@ def _synthetic_token() -> str: return token +def _encoded_token() -> str: + return base64.b64encode(_synthetic_token().encode()).decode() + + def _durable_locations(db_path: Path, needle: str) -> list[str]: locations: list[str] = [] with sqlite3.connect(db_path) as conn: @@ -42,6 +47,11 @@ def _durable_locations(db_path: Path, needle: str) -> list[str]: return locations +def _table_row_count(db_path: Path, table: str) -> int: + with sqlite3.connect(db_path) as conn: + return int(conn.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0]) + + def _ingest_with_metadata_secret(service: MemoryService, field: str, secret: str) -> None: citation_source = secret if field == "citation_source" else "phase0-red-test" citation_locator = secret if field == "citation_locator" else "case:metadata" @@ -49,38 +59,158 @@ def _ingest_with_metadata_secret(service: MemoryService, field: str, secret: str "holder": {"holder": secret}, "source_agent": {"source_agent": secret}, "idempotency_key": {"idempotency_key": secret}, + "claim_type": {"claim_type": secret}, + "scope": {"scope": secret}, + "volatility": {"volatility": secret}, + "confidence": {"confidence": secret}, + "event_time": {"event_time": secret}, + "valid_from": {"valid_from": secret}, + "valid_until": {"valid_until": secret}, + "visibility": {"visibility": secret}, + "intake_batch_id": {"intake_batch_id": secret}, "citation_source": {}, "citation_locator": {}, }[field] attribution = {} if field == "source_agent" else {"source_agent": "phase0-red-test"} + metadata = { + "scope": "project:phase0-red-test", + **attribution, + **overrides, + } service.ingest( text=f"Benign metadata sensitivity case for {field}.", citations=[CitationInput(source=citation_source, locator=citation_locator)], - scope="project:phase0-red-test", - **(attribution | overrides), + **metadata, ) -@pytest.mark.xfail( - strict=True, - reason="MM-SEC-03: persisted-envelope metadata bypasses the sensitivity gateway", -) +_METADATA_FIELDS = [ + "holder", + "source_agent", + "idempotency_key", + "claim_type", + "scope", + "volatility", + "confidence", + "event_time", + "valid_from", + "valid_until", + "visibility", + "intake_batch_id", + "citation_source", + "citation_locator", +] + + @pytest.mark.parametrize( - "field", - ["holder", "source_agent", "idempotency_key", "citation_source", "citation_locator"], + ("field", "encoding"), + [(field, "literal") for field in _METADATA_FIELDS] + + [(field, "base64") for field in _METADATA_FIELDS], ) def test_secret_shaped_metadata_never_reaches_durable_storage( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, field: str, + encoding: str, ) -> None: monkeypatch.delenv("QDRANT_URL", raising=False) monkeypatch.delenv("MEMORYMASTER_ENCRYPTION_KEY", raising=False) db_path = tmp_path / f"persisted-envelope-{field}.db" service = MemoryService(db_path, workspace_root=tmp_path) service.init_db() + secret = _synthetic_token() if encoding == "literal" else _encoded_token() + + with pytest.raises(SensitiveMetadataError) as rejected: + _ingest_with_metadata_secret(service, field, secret) + assert secret not in str(rejected.value), "validation errors must never echo secret values" + + assert _durable_locations(db_path, secret) == [] + assert _table_row_count(db_path, "claims") == 0 + assert _table_row_count(db_path, "citations") == 0 + + +@pytest.mark.parametrize("identity_source", ["configured_default", "tenant_id", "bound_principal"]) +def test_effective_identity_is_revalidated_before_claim_storage( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + identity_source: str, +) -> None: + monkeypatch.delenv("QDRANT_URL", raising=False) secret = _synthetic_token() + db_path = tmp_path / f"effective-identity-{identity_source}.db" + initializer = MemoryService(db_path, workspace_root=tmp_path) + initializer.init_db() + kwargs: dict[str, object] = {} + if identity_source == "configured_default": + monkeypatch.setenv("MEMORYMASTER_INTAKE_REQUIRE_SOURCE_AGENT", "warn") + monkeypatch.setenv("MEMORYMASTER_INTAKE_DEFAULT_SOURCE_AGENT", secret) + elif identity_source == "tenant_id": + kwargs["tenant_id"] = secret + else: + kwargs.update( + tenant_id="tenant-safe", + require_tenant=True, + principal=secret, + allowed_scopes=["project:phase0-red-test"], + ) + service = MemoryService(db_path, workspace_root=tmp_path, **kwargs) - _ingest_with_metadata_secret(service, field, secret) + with pytest.raises(SensitiveMetadataError) as rejected: + service.ingest( + text="Benign effective identity sensitivity case.", + citations=[CitationInput(source="phase0-red-test", locator="effective-identity")], + scope="project:phase0-red-test", + ) + assert secret not in str(rejected.value) assert _durable_locations(db_path, secret) == [] + assert _table_row_count(db_path, "claims") == 0 + assert _table_row_count(db_path, "citations") == 0 + + +@pytest.mark.parametrize( + "field", + ["text", "subject", "predicate", "object_value", "citation_excerpt"], +) +def test_encoded_secret_in_claim_content_never_reaches_durable_storage( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + field: str, +) -> None: + monkeypatch.delenv("QDRANT_URL", raising=False) + monkeypatch.delenv("MEMORYMASTER_ENCRYPTION_KEY", raising=False) + db_path = tmp_path / f"persisted-envelope-content-{field}.db" + service = MemoryService(db_path, workspace_root=tmp_path) + service.init_db() + encoded_secret = _encoded_token() + values = { + "text": "Benign encoded-secret field test.", + "subject": "safe-subject", + "predicate": "safe-predicate", + "object_value": "safe-object", + "citation_excerpt": "safe excerpt", + } + values[field] = encoded_secret + + claim = service.ingest( + text=values["text"], + subject=values["subject"], + predicate=values["predicate"], + object_value=values["object_value"], + citations=[ + CitationInput( + source="phase0-red-test", + locator="case:encoded-content", + excerpt=values["citation_excerpt"], + ) + ], + scope="project:phase0-red-test", + source_agent="phase0-red-test", + ) + + assert _durable_locations(db_path, encoded_secret) == [] + assert _durable_locations(db_path, _synthetic_token()) == [] + if field == "citation_excerpt": + assert _durable_locations(db_path, "[REDACTED:encoded_secret]") + else: + assert getattr(claim, field) == "[REDACTED:encoded_secret]" From 5755a95fcbe905f15a156aec6e6746dd3bb907b0 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 09:51:28 -0300 Subject: [PATCH 15/55] fix(storage): enforce persisted envelope gateway --- memorymaster/core/security.py | 126 +++ memorymaster/stores/_storage_lifecycle.py | 16 +- memorymaster/stores/_storage_schema.py | 20 + memorymaster/stores/_storage_write_claims.py | 97 ++- memorymaster/stores/postgres_store.py | 131 +++- tests/test_compact_summaries_sensitivity.py | 7 +- tests/test_store_persisted_envelope.py | 784 +++++++++++++++++++ tests/test_write_gateway_paths.py | 4 - 8 files changed, 1158 insertions(+), 27 deletions(-) create mode 100644 tests/test_store_persisted_envelope.py diff --git a/memorymaster/core/security.py b/memorymaster/core/security.py index e869798f..c0ea01d9 100644 --- a/memorymaster/core/security.py +++ b/memorymaster/core/security.py @@ -164,6 +164,22 @@ class SanitizedClaimInput: predicate: str | None = None +@dataclass(slots=True) +class SanitizedClaimStructureInput: + claim_type: str | None + subject: str | None + predicate: str | None + object_value: str | None + findings: list[str] + + +@dataclass(slots=True) +class SanitizedEventInput: + details: str | None + payload: object + findings: list[str] + + class SensitiveMetadataError(ValueError): """Fail-closed metadata rejection that never echoes the supplied value.""" @@ -502,6 +518,114 @@ def sanitize_persisted_text(text: str) -> tuple[str, list[str]]: return redacted, findings +def _structured_context_findings(value: str, context_keys: tuple[str, ...]) -> list[str]: + independent_findings = scan_persisted_value(value) + findings: list[str] = [] + for key in context_keys: + for finding in scan_text_for_findings(f"{key}={value}"): + if finding not in independent_findings and finding not in findings: + findings.append(finding) + return findings + + +def _sanitize_persisted_json( + value: object, + context_keys: tuple[str, ...], +) -> tuple[object, list[str]]: + if isinstance(value, str): + sanitized, findings = sanitize_persisted_text(value) + structured_findings = _structured_context_findings(value, context_keys) + if structured_findings: + sanitized = "[REDACTED:structured_secret]" + findings.extend(structured_findings) + return sanitized, sorted(set(findings)) + if isinstance(value, (int, float, bool)): + structured_findings = _structured_context_findings(str(value), context_keys) + if structured_findings: + return "[REDACTED:structured_secret]", structured_findings + return value, [] + if isinstance(value, Mapping): + sanitized: dict[str, object] = {} + findings: list[str] = [] + for key, nested in value.items(): + if not isinstance(key, str): + raise ValueError("Persisted JSON object keys must be strings.") + validate_persisted_metadata({"payload_json_key": key}) + sanitized_value, nested_findings = _sanitize_persisted_json( + nested, + (*context_keys, key), + ) + sanitized[key] = sanitized_value + findings.extend(nested_findings) + return sanitized, sorted(set(findings)) + if isinstance(value, (list, tuple)): + sanitized_items: list[object] = [] + findings = [] + for nested in value: + sanitized_value, nested_findings = _sanitize_persisted_json( + nested, + context_keys, + ) + sanitized_items.append(sanitized_value) + findings.extend(nested_findings) + return sanitized_items, sorted(set(findings)) + return value, [] + + +def sanitize_persisted_json(value: object) -> tuple[object, list[str]]: + """Return a recursively sanitized JSON-compatible copy of ``value``.""" + return _sanitize_persisted_json(value, ()) + + +def sanitize_claim_structure_input( + *, + claim_type: str | None, + subject: str | None, + predicate: str | None, + object_value: str | None, +) -> SanitizedClaimStructureInput: + validate_persisted_metadata({"claim_type": claim_type}) + sanitized_subject, subject_findings = _sanitize_optional_claim_text(subject) + sanitized_predicate, predicate_findings = _sanitize_optional_claim_text(predicate) + sanitized_object, object_findings = _sanitize_optional_claim_text(object_value) + findings = sorted(set(subject_findings + predicate_findings + object_findings)) + return SanitizedClaimStructureInput( + claim_type=claim_type, + subject=sanitized_subject, + predicate=sanitized_predicate, + object_value=sanitized_object, + findings=findings, + ) + + +def sanitize_event_input( + *, + event_type: str, + from_status: str | None, + to_status: str | None, + details: str | None, + payload: object, + created_at: object, + tenant_id: str | None = None, +) -> SanitizedEventInput: + validate_persisted_metadata( + { + "event_type": event_type, + "from_status": from_status, + "to_status": to_status, + "created_at": created_at, + "tenant_id": tenant_id, + } + ) + sanitized_details, detail_findings = _sanitize_optional_claim_text(details) + sanitized_payload, payload_findings = sanitize_persisted_json(payload) + return SanitizedEventInput( + details=sanitized_details, + payload=sanitized_payload, + findings=sorted(set(detail_findings + payload_findings)), + ) + + def _get_fernet(): key = os.getenv(_ENCRYPTION_ENV_VAR) if not key: @@ -561,6 +685,7 @@ def sanitize_claim_input( valid_from: str | None = None, valid_until: str | None = None, intake_batch_id: str | None = None, + tenant_id: str | None = None, ) -> SanitizedClaimInput: citation_metadata = { "citation_source": [citation.source for citation in citations], @@ -579,6 +704,7 @@ def sanitize_claim_input( "valid_from": valid_from, "valid_until": valid_until, "intake_batch_id": intake_batch_id, + "tenant_id": tenant_id, **citation_metadata, }) redacted_text, findings = sanitize_persisted_text(text) diff --git a/memorymaster/stores/_storage_lifecycle.py b/memorymaster/stores/_storage_lifecycle.py index ca7d61c3..cc3029a2 100644 --- a/memorymaster/stores/_storage_lifecycle.py +++ b/memorymaster/stores/_storage_lifecycle.py @@ -24,6 +24,7 @@ validate_event_type, validate_transition_event_type, ) +from memorymaster.core.security import sanitize_event_input from memorymaster.stores._storage_shared import ( EVENT_HASH_ALGO, @@ -518,13 +519,24 @@ def record_event( details: str | None = None, payload: dict[str, object] | None = None, ) -> None: + now = utc_now() + sanitized = sanitize_event_input( + event_type=event_type, + from_status=from_status, + to_status=to_status, + details=details, + payload=payload, + created_at=now, + ) + details = sanitized.details + if sanitized.payload is not None and not isinstance(sanitized.payload, dict): + raise ValueError("Event payload must be a JSON object.") validated_event_type = validate_event_type(event_type) validated_payload = validate_event_payload( validated_event_type, - payload, + sanitized.payload, details=details, ) - now = utc_now() payload_json = json.dumps(validated_payload) if validated_payload is not None else None with self.connect() as conn: self._insert_event_row( diff --git a/memorymaster/stores/_storage_schema.py b/memorymaster/stores/_storage_schema.py index 9f3ee452..644ac0e9 100644 --- a/memorymaster/stores/_storage_schema.py +++ b/memorymaster/stores/_storage_schema.py @@ -12,6 +12,7 @@ import sqlite3 from importlib.resources import files +from memorymaster.core.security import sanitize_event_input, validate_persisted_metadata from memorymaster.stores._storage_shared import ( EVENT_HASH_ALGO, @@ -987,6 +988,24 @@ def _insert_event_row( payload_json: str | None, created_at: str, ) -> int: + try: + payload = json.loads(payload_json) if payload_json is not None else None + except (TypeError, json.JSONDecodeError) as exc: + raise ValueError("Event payload_json must contain valid JSON.") from exc + sanitized = sanitize_event_input( + event_type=event_type, + from_status=from_status, + to_status=to_status, + details=details, + payload=payload, + created_at=created_at, + ) + details = sanitized.details + payload_json = ( + json.dumps(sanitized.payload, sort_keys=True) + if payload_json is not None + else None + ) tenant_id: str | None = None if claim_id is not None: try: @@ -998,6 +1017,7 @@ def _insert_event_row( claim_row = None if claim_row is not None: tenant_id = claim_row["tenant_id"] + validate_persisted_metadata({"effective_tenant_id": tenant_id}) try: prev_row = conn.execute( "SELECT event_hash FROM events WHERE event_hash IS NOT NULL ORDER BY id DESC LIMIT 1" diff --git a/memorymaster/stores/_storage_write_claims.py b/memorymaster/stores/_storage_write_claims.py index c0ef5138..251a219b 100644 --- a/memorymaster/stores/_storage_write_claims.py +++ b/memorymaster/stores/_storage_write_claims.py @@ -16,6 +16,12 @@ Claim, validate_event_payload, ) +from memorymaster.core.security import ( + sanitize_claim_input, + sanitize_claim_structure_input, + sanitize_persisted_text, + validate_persisted_metadata, +) from memorymaster.stores._storage_shared import ( utc_now, ) @@ -94,9 +100,49 @@ def create_claim( ) -> Claim: if not citations: raise ValueError("At least one citation is required.") + citation_inputs = [ + CitationInput( + source=cite.get("source", ""), + locator=cite.get("locator"), + excerpt=cite.get("excerpt"), + ) + if isinstance(cite, dict) + else CitationInput(cite.source, cite.locator, cite.excerpt) + for cite in citations + ] + sanitized = sanitize_claim_input( + text=text, + object_value=object_value, + citations=citation_inputs, + subject=subject, + predicate=predicate, + idempotency_key=idempotency_key, + claim_type=claim_type, + scope=scope, + volatility=volatility, + source_agent=source_agent, + visibility=visibility, + holder=holder, + confidence=confidence, + event_time=event_time, + valid_from=valid_from, + valid_until=valid_until, + tenant_id=tenant_id, + ) + text = sanitized.text + object_value = sanitized.object_value + citations = sanitized.citations + subject = sanitized.subject + predicate = sanitized.predicate visibility, source_agent = normalize_claim_identity(visibility, source_agent) normalized_idempotency_key = (idempotency_key or "").strip() or None normalized_tenant_id = (tenant_id or "").strip() or None + validate_persisted_metadata( + { + "effective_source_agent": source_agent, + "effective_tenant_id": normalized_tenant_id, + } + ) now = utc_now() with self.connect() as conn: existing = self._check_idempotency( @@ -191,21 +237,12 @@ def create_claim( # Column may not exist in legacy schemas; skip gracefully. pass for cite in citations: - # Accept both CitationInput objects and plain dicts - if isinstance(cite, dict): - _src = cite.get("source", "") - _loc = cite.get("locator") - _exc = cite.get("excerpt") - else: - _src = cite.source - _loc = cite.locator - _exc = cite.excerpt conn.execute( """ INSERT INTO citations (claim_id, source, locator, excerpt, created_at) VALUES (?, ?, ?, ?, ?) """, - (claim_id, _src, _loc, _exc, now), + (claim_id, cite.source, cite.locator, cite.excerpt, now), ) ingest_payload = validate_event_payload( "ingest", @@ -222,6 +259,22 @@ def create_claim( payload_json=json.dumps(ingest_payload), created_at=now, ) + if sanitized.is_sensitive: + policy_payload = validate_event_payload( + "policy_decision", + {"findings": sanitized.findings}, + details="sensitive_redaction_applied", + ) + self._insert_event_row( + conn, + claim_id=claim_id, + event_type="policy_decision", + from_status="candidate", + to_status="candidate", + details="sensitive_redaction_applied", + payload_json=json.dumps(policy_payload), + created_at=now, + ) conn.commit() claim = self.get_claim(claim_id) if claim is None: @@ -230,11 +283,12 @@ def create_claim( def set_normalized_text(self, claim_id: int, normalized_text: str) -> None: + sanitized_text, _ = sanitize_persisted_text(normalized_text) now = utc_now() with self.connect() as conn: conn.execute( "UPDATE claims SET normalized_text = ?, updated_at = ? WHERE id = ?", - (normalized_text, now, claim_id), + (sanitized_text, now, claim_id), ) conn.commit() @@ -247,9 +301,13 @@ def set_normalized_texts_batch(self, updates: dict[int, str]) -> None: """ if not updates: return + sanitized_updates = { + claim_id: sanitize_persisted_text(normalized_text)[0] + for claim_id, normalized_text in updates.items() + } now = utc_now() with self.connect() as conn: - for claim_id, normalized_text in updates.items(): + for claim_id, normalized_text in sanitized_updates.items(): conn.execute( "UPDATE claims SET normalized_text = ?, updated_at = ? WHERE id = ?", (normalized_text, now, claim_id), @@ -367,6 +425,12 @@ def update_claim_structure( predicate: str | None = None, object_value: str | None = None, ) -> None: + sanitized = sanitize_claim_structure_input( + claim_type=claim_type, + subject=subject, + predicate=predicate, + object_value=object_value, + ) now = utc_now() with self.connect() as conn: conn.execute( @@ -379,7 +443,14 @@ def update_claim_structure( updated_at = ? WHERE id = ? """, - (claim_type, subject, predicate, object_value, now, claim_id), + ( + sanitized.claim_type, + sanitized.subject, + sanitized.predicate, + sanitized.object_value, + now, + claim_id, + ), ) conn.commit() diff --git a/memorymaster/stores/postgres_store.py b/memorymaster/stores/postgres_store.py index 7e1f3a9a..76c7f051 100644 --- a/memorymaster/stores/postgres_store.py +++ b/memorymaster/stores/postgres_store.py @@ -27,6 +27,13 @@ validate_transition_event_type, ) from memorymaster.core.retry import connect_with_retry +from memorymaster.core.security import ( + sanitize_claim_input, + sanitize_claim_structure_input, + sanitize_event_input, + sanitize_persisted_text, + validate_persisted_metadata, +) from memorymaster.stores._storage_shared import ( ConcurrentModificationError, EVENT_HASH_ALGO, @@ -165,6 +172,15 @@ def _require_team_authority(self) -> tuple[str, str, tuple[str, ...]]: raise PermissionError("Postgres team scopes cannot contain wildcards.") return self.tenant_id, self.principal, tuple(sorted(self.allowed_scopes)) + def _validate_bound_persistence_identity(self) -> None: + validate_persisted_metadata( + { + "bound_tenant_id": self.tenant_id, + "bound_principal": self.principal, + "bound_scope": self.allowed_scopes, + } + ) + @staticmethod def _cleanup_failed_connection(conn) -> None: try: @@ -1054,6 +1070,7 @@ def connect(self) -> Any: "Postgres application connections require authenticated team authority. " "Use SQLite for local trusted mode or init_db() with a dedicated migrator DSN." ) + self._validate_bound_persistence_identity() authority = self._require_team_authority() conn = self._open_connection() try: @@ -1527,9 +1544,22 @@ def _insert_event_row( payload: dict[str, object] | None, created_at: datetime, ) -> int: + sanitized = sanitize_event_input( + event_type=event_type, + from_status=from_status, + to_status=to_status, + details=details, + payload=payload, + created_at=created_at, + ) + details = sanitized.details + if sanitized.payload is not None and not isinstance(sanitized.payload, dict): + raise ValueError("Event payload must be a JSON object.") + payload = sanitized.payload _, _, Jsonb = self._load_psycopg() with conn.cursor() as cur: tenant_id = self._event_tenant_for_claim(cur, claim_id) + validate_persisted_metadata({"effective_tenant_id": tenant_id}) prev_event_hash, hash_algo, tenant_prev_event_hash = self._event_chain_head( cur, tenant_id, @@ -1651,15 +1681,56 @@ def create_claim( visibility: str = "public", holder: str | None = None, ) -> Claim: + if not citations: + raise ValueError("At least one citation is required.") + citation_inputs = [ + CitationInput( + source=cite.get("source", ""), + locator=cite.get("locator"), + excerpt=cite.get("excerpt"), + ) + if isinstance(cite, dict) + else CitationInput(cite.source, cite.locator, cite.excerpt) + for cite in citations + ] + sanitized = sanitize_claim_input( + text=text, + object_value=object_value, + citations=citation_inputs, + subject=subject, + predicate=predicate, + idempotency_key=idempotency_key, + claim_type=claim_type, + scope=scope, + volatility=volatility, + source_agent=source_agent, + visibility=visibility, + holder=holder, + confidence=confidence, + event_time=event_time, + valid_from=valid_from, + valid_until=valid_until, + tenant_id=tenant_id, + ) + self._validate_bound_persistence_identity() + text = sanitized.text + object_value = sanitized.object_value + citations = sanitized.citations + subject = sanitized.subject + predicate = sanitized.predicate visibility, source_agent = normalize_claim_identity( visibility, source_agent, allow_sensitive=not self.require_tenant, ) - if not citations: - raise ValueError("At least one citation is required.") normalized_idempotency_key = (idempotency_key or "").strip() or None normalized_tenant_id = self._tenant_for_operation(tenant_id) + validate_persisted_metadata( + { + "effective_source_agent": source_agent, + "effective_tenant_id": normalized_tenant_id, + } + ) now = utc_now() with self.connect() as conn, conn.cursor() as cur: cur.execute( @@ -1764,6 +1835,22 @@ def create_claim( payload=ingest_payload, created_at=now, ) + if sanitized.is_sensitive: + policy_payload = validate_event_payload( + "policy_decision", + {"findings": sanitized.findings}, + details="sensitive_redaction_applied", + ) + self._insert_event_row( + conn, + claim_id=claim_id, + event_type="policy_decision", + from_status="candidate", + to_status="candidate", + details="sensitive_redaction_applied", + payload=policy_payload, + created_at=now, + ) claim = self.get_claim(claim_id) if claim is None: @@ -2001,17 +2088,22 @@ def count_citations_batch(self, claim_ids: list[int]) -> dict[int, int]: return {claim_id: self.count_citations(claim_id) for claim_id in claim_ids} def set_normalized_text(self, claim_id: int, normalized_text: str) -> None: + sanitized_text, _ = sanitize_persisted_text(normalized_text) now = utc_now() with self.connect() as conn, conn.cursor() as cur: cur.execute( "UPDATE claims SET normalized_text = %s, updated_at = %s WHERE id = %s", - (normalized_text, now, claim_id), + (sanitized_text, now, claim_id), ) def set_normalized_texts_batch(self, updates: dict[int, str]) -> None: if not updates: return - for claim_id, normalized_text in updates.items(): + sanitized_updates = { + claim_id: sanitize_persisted_text(normalized_text)[0] + for claim_id, normalized_text in updates.items() + } + for claim_id, normalized_text in sanitized_updates.items(): self.set_normalized_text(claim_id, normalized_text) def redact_claim_payload( @@ -2123,6 +2215,12 @@ def update_claim_structure( predicate: str | None = None, object_value: str | None = None, ) -> None: + sanitized = sanitize_claim_structure_input( + claim_type=claim_type, + subject=subject, + predicate=predicate, + object_value=object_value, + ) now = utc_now() with self.connect() as conn, conn.cursor() as cur: cur.execute( @@ -2135,7 +2233,14 @@ def update_claim_structure( updated_at = %s WHERE id = %s """, - (claim_type, subject, predicate, object_value, now, claim_id), + ( + sanitized.claim_type, + sanitized.subject, + sanitized.predicate, + sanitized.object_value, + now, + claim_id, + ), ) def set_confidence(self, claim_id: int, confidence: float, details: str | None = None) -> None: @@ -2861,13 +2966,25 @@ def record_event( details: str | None = None, payload: dict[str, object] | None = None, ) -> None: + self._validate_bound_persistence_identity() + now = utc_now() + sanitized = sanitize_event_input( + event_type=event_type, + from_status=from_status, + to_status=to_status, + details=details, + payload=payload, + created_at=now, + ) + details = sanitized.details + if sanitized.payload is not None and not isinstance(sanitized.payload, dict): + raise ValueError("Event payload must be a JSON object.") validated_event_type = validate_event_type(event_type) validated_payload = validate_event_payload( validated_event_type, - payload, + sanitized.payload, details=details, ) - now = utc_now() with self.connect() as conn: self._insert_event_row( conn, diff --git a/tests/test_compact_summaries_sensitivity.py b/tests/test_compact_summaries_sensitivity.py index d7c3b94c..96ee46df 100644 --- a/tests/test_compact_summaries_sensitivity.py +++ b/tests/test_compact_summaries_sensitivity.py @@ -22,12 +22,17 @@ def store(tmp_path): def _create_archived_claim(store, text: str): claim = store.create_claim( - text=text, + text="Test-only placeholder before injecting a legacy row.", citations=[CitationInput(source="test")], subject="legacy-secret", predicate="contains", object_value="synthetic test fixture", ) + # Explicitly simulate a pre-gateway legacy row. Going through create_claim + # here would sanitize the fixture and make the read-time egress test vacuous. + with store.connect() as conn: + conn.execute("UPDATE claims SET text = ? WHERE id = ?", (text, claim.id)) + conn.commit() transition_claim(store, claim.id, to_status="confirmed", reason="test", event_type="transition") transition_claim(store, claim.id, to_status="stale", reason="test", event_type="decay") transition_claim(store, claim.id, to_status="archived", reason="test", event_type="compactor") diff --git a/tests/test_store_persisted_envelope.py b/tests/test_store_persisted_envelope.py new file mode 100644 index 00000000..223edd12 --- /dev/null +++ b/tests/test_store_persisted_envelope.py @@ -0,0 +1,784 @@ +"""Adversarial RED tests for the low-level persisted-envelope gateway. + +The fixtures are synthetic, assembled at runtime, and written only to temporary +databases. These tests intentionally exercise store APIs directly so callers +cannot bypass the canonical service ingest boundary. +""" +from __future__ import annotations + +import base64 +import copy +import json +import sqlite3 +from collections.abc import Callable +from dataclasses import asdict +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from memorymaster.core.models import CitationInput +from memorymaster.core.security import SensitiveMetadataError, scan_text_for_findings +from memorymaster.stores.postgres_store import PostgresStore +from memorymaster.stores.storage import SQLiteStore + + +def _literal_secret() -> str: + body = "".join(format((index * 7 + 3) % 16, "x") for index in range(40)) + token = "".join(("gh", "p_", body)) + assert "github_token" in scan_text_for_findings(token) + return token + + +def _secret(encoding: str) -> str: + literal = _literal_secret() + if encoding == "literal": + return literal + encoded = base64.b64encode(literal.encode()).decode() + assert "github_token" in scan_text_for_findings(encoded) + return encoded + + +def _needles(secret: str) -> tuple[str, ...]: + literal = _literal_secret() + return (secret,) if secret == literal else (secret, literal) + + +def _quote_identifier(value: str) -> str: + return '"' + value.replace('"', '""') + '"' + + +def _durable_locations(db_path: Path, needle: str) -> list[str]: + locations: list[str] = [] + with sqlite3.connect(db_path) as conn: + tables = conn.execute( + "SELECT name FROM sqlite_master " + "WHERE type = 'table' AND name NOT LIKE 'sqlite_%'" + ).fetchall() + for (raw_table,) in tables: + table = str(raw_table) + quoted_table = _quote_identifier(table) + columns = conn.execute(f"PRAGMA table_xinfo({quoted_table})").fetchall() + for raw_column in (row[1] for row in columns): + column = str(raw_column) + quoted_column = _quote_identifier(column) + values = conn.execute( + f"SELECT {quoted_column} FROM {quoted_table} " + f"WHERE {quoted_column} IS NOT NULL" + ).fetchall() + if any(isinstance(value, str) and needle in value for (value,) in values): + locations.append(f"{table}.{column}") + return locations + + +def _assert_absent_everywhere(db_path: Path, secret: str) -> None: + leaked: dict[str, list[str]] = {} + for needle in _needles(secret): + locations = _durable_locations(db_path, needle) + if locations: + leaked[needle] = locations + assert leaked == {}, f"secret-shaped fixture reached durable SQLite values: {leaked}" + + +def _table_counts(db_path: Path, *tables: str) -> dict[str, int]: + with sqlite3.connect(db_path) as conn: + return { + table: int( + conn.execute( + f"SELECT COUNT(*) FROM {_quote_identifier(table)}" + ).fetchone()[0] + ) + for table in tables + } + + +def _claim_row(db_path: Path, claim_id: int) -> dict[str, object]: + with sqlite3.connect(db_path) as conn: + conn.row_factory = sqlite3.Row + row = conn.execute("SELECT * FROM claims WHERE id = ?", (claim_id,)).fetchone() + assert row is not None + return dict(row) + + +def _event_row(db_path: Path) -> dict[str, object]: + with sqlite3.connect(db_path) as conn: + conn.row_factory = sqlite3.Row + row = conn.execute("SELECT * FROM events ORDER BY id DESC LIMIT 1").fetchone() + assert row is not None + return dict(row) + + +def _assert_chain_intact(store: SQLiteStore) -> None: + report = store.reconcile_integrity(fix=False) + summary = report["summary"] + assert summary["hash_chain_issues"] == 0 + assert summary["tenant_hash_chain_issues"] == 0 + + +def _assert_rejected_without_echo( + operation: Callable[[], object], + secret: str, +) -> None: + with pytest.raises(SensitiveMetadataError) as rejected: + operation() + assert secret not in str(rejected.value) + assert _literal_secret() not in str(rejected.value) + + +def _new_store(tmp_path: Path, name: str) -> tuple[SQLiteStore, Path]: + db_path = tmp_path / f"{name}.db" + store = SQLiteStore(db_path) + store.init_db() + return store, db_path + + +def _safe_claim(store: SQLiteStore, suffix: str = "one"): + return store.create_claim( + text=f"Safe direct-store claim {suffix}.", + citations=[CitationInput(source="unit-test", locator=f"case:{suffix}")], + subject=f"safe-subject-{suffix}", + predicate="safe-predicate", + object_value="safe-object", + scope="project:r14-store-tests", + source_agent="r14-store-tests", + ) + + +@pytest.mark.parametrize("encoding", ["literal", "base64"]) +def test_direct_create_sanitizes_all_claim_content_without_mutating_input( + tmp_path: Path, + encoding: str, +) -> None: + store, db_path = _new_store(tmp_path, f"create-content-{encoding}") + secret = _secret(encoding) + citations = [CitationInput(source="unit-test", locator="create-content")] + citations_before = [asdict(citation) for citation in citations] + + claim = store.create_claim( + text=secret, + citations=citations, + subject=f"subject {secret}", + predicate=f"predicate {secret}", + object_value=f"object {secret}", + scope="project:r14-store-tests", + source_agent="r14-store-tests", + ) + + assert [asdict(citation) for citation in citations] == citations_before + for value in (claim.text, claim.subject, claim.predicate, claim.object_value): + assert value is not None + assert all(needle not in value for needle in _needles(secret)) + assert "[REDACTED:" in value + _assert_absent_everywhere(db_path, secret) + + +@pytest.mark.parametrize("encoding", ["literal", "base64"]) +def test_direct_create_rejects_sensitive_identifier_before_any_row( + tmp_path: Path, + encoding: str, +) -> None: + store, db_path = _new_store(tmp_path, f"create-metadata-{encoding}") + secret = _secret(encoding) + + _assert_rejected_without_echo( + lambda: store.create_claim( + text="Safe content for metadata rejection.", + citations=[CitationInput(source="unit-test")], + idempotency_key=secret, + scope="project:r14-store-tests", + source_agent="r14-store-tests", + ), + secret, + ) + + assert _table_counts(db_path, "claims", "citations", "events") == { + "claims": 0, + "citations": 0, + "events": 0, + } + _assert_absent_everywhere(db_path, secret) + + +@pytest.mark.parametrize( + ("field", "encoding"), + [("source", "literal"), ("locator", "base64")], +) +def test_direct_create_rejects_sensitive_citation_metadata_before_any_row( + tmp_path: Path, + field: str, + encoding: str, +) -> None: + store, db_path = _new_store(tmp_path, f"citation-{field}-{encoding}") + secret = _secret(encoding) + citation = CitationInput( + source=secret if field == "source" else "unit-test", + locator=secret if field == "locator" else "case:citation-metadata", + ) + before = asdict(citation) + + _assert_rejected_without_echo( + lambda: store.create_claim( + text="Safe citation metadata rejection case.", + citations=[citation], + scope="project:r14-store-tests", + source_agent="r14-store-tests", + ), + secret, + ) + + assert asdict(citation) == before + assert _table_counts(db_path, "claims", "citations", "events") == { + "claims": 0, + "citations": 0, + "events": 0, + } + _assert_absent_everywhere(db_path, secret) + + +@pytest.mark.parametrize("encoding", ["literal", "base64"]) +def test_direct_create_sanitizes_citation_excerpt_without_mutating_input( + tmp_path: Path, + encoding: str, +) -> None: + store, db_path = _new_store(tmp_path, f"citation-excerpt-{encoding}") + secret = _secret(encoding) + citation = CitationInput( + source="unit-test", + locator="case:citation-excerpt", + excerpt=f"quoted evidence {secret}", + ) + before = asdict(citation) + + claim = store.create_claim( + text="Safe claim with sensitive citation content.", + citations=[citation], + scope="project:r14-store-tests", + source_agent="r14-store-tests", + ) + + assert asdict(citation) == before + assert claim.citations[0].excerpt is not None + assert "[REDACTED:" in claim.citations[0].excerpt + _assert_absent_everywhere(db_path, secret) + + +@pytest.mark.parametrize( + ("field", "encoding"), + [("tenant_id", "literal"), ("source_agent", "base64")], +) +def test_direct_create_rejects_sensitive_effective_identity( + tmp_path: Path, + field: str, + encoding: str, +) -> None: + store, db_path = _new_store(tmp_path, f"effective-identity-{field}") + secret = _secret(encoding) + kwargs: dict[str, object] = { + "tenant_id": "tenant-safe", + "visibility": "private", + "source_agent": "principal-safe", + } + kwargs[field] = secret + + _assert_rejected_without_echo( + lambda: store.create_claim( + text="Safe effective identity rejection case.", + citations=[CitationInput(source="unit-test")], + scope="project:r14-store-tests", + **kwargs, + ), + secret, + ) + + assert _table_counts(db_path, "claims", "citations", "events") == { + "claims": 0, + "citations": 0, + "events": 0, + } + _assert_absent_everywhere(db_path, secret) + + +@pytest.mark.parametrize("encoding", ["literal", "base64"]) +def test_update_claim_structure_sanitizes_content_fields( + tmp_path: Path, + encoding: str, +) -> None: + store, db_path = _new_store(tmp_path, f"update-structure-content-{encoding}") + claim = store.create_claim( + text="Safe unstructured claim for structure enrichment.", + citations=[CitationInput(source="unit-test", locator="update-structure")], + scope="project:r14-store-tests", + source_agent="r14-store-tests", + ) + secret = _secret(encoding) + + store.update_claim_structure( + claim.id, + subject=f"subject {secret}", + predicate=f"predicate {secret}", + object_value=f"object {secret}", + ) + + updated = store.get_claim(claim.id, include_citations=False) + assert updated is not None + for value in (updated.subject, updated.predicate, updated.object_value): + assert value is not None + assert all(needle not in value for needle in _needles(secret)) + assert "[REDACTED:" in value + _assert_absent_everywhere(db_path, secret) + + +def test_update_claim_structure_rejects_metadata_without_partial_update( + tmp_path: Path, +) -> None: + store, db_path = _new_store(tmp_path, "update-structure-metadata") + claim = _safe_claim(store) + secret = _secret("base64") + before_row = _claim_row(db_path, claim.id) + before_counts = _table_counts(db_path, "claims", "citations", "events") + + _assert_rejected_without_echo( + lambda: store.update_claim_structure( + claim.id, + claim_type=secret, + subject="must-not-partially-update", + ), + secret, + ) + + assert _claim_row(db_path, claim.id) == before_row + assert _table_counts(db_path, "claims", "citations", "events") == before_counts + _assert_absent_everywhere(db_path, secret) + _assert_chain_intact(store) + + +@pytest.mark.parametrize("encoding", ["literal", "base64"]) +def test_set_normalized_text_sanitizes_claim_and_fts_copy( + tmp_path: Path, + encoding: str, +) -> None: + store, db_path = _new_store(tmp_path, f"normalized-single-{encoding}") + claim = _safe_claim(store) + secret = _secret(encoding) + + store.set_normalized_text(claim.id, f"normalized {secret}") + + with store.connect() as conn: + normalized = conn.execute( + "SELECT normalized_text FROM claims WHERE id = ?", + (claim.id,), + ).fetchone()[0] + indexed = conn.execute( + "SELECT normalized_text FROM claims_fts WHERE rowid = ?", + (claim.id,), + ).fetchone()[0] + for value in (str(normalized), str(indexed)): + assert all(needle not in value for needle in _needles(secret)) + assert "[REDACTED:" in value + _assert_absent_everywhere(db_path, secret) + + +def test_set_normalized_texts_batch_sanitizes_fts_and_preserves_mapping( + tmp_path: Path, +) -> None: + store, db_path = _new_store(tmp_path, "normalized-batch") + first = _safe_claim(store, "first") + second = _safe_claim(store, "second") + literal = _secret("literal") + encoded = _secret("base64") + updates = { + first.id: f"first normalized {literal}", + second.id: f"second normalized {encoded}", + } + before = copy.deepcopy(updates) + + store.set_normalized_texts_batch(updates) + + assert updates == before + with store.connect() as conn: + rows = conn.execute( + "SELECT c.id, c.normalized_text, f.normalized_text " + "FROM claims AS c JOIN claims_fts AS f ON f.rowid = c.id " + "WHERE c.id IN (?, ?) ORDER BY c.id", + (first.id, second.id), + ).fetchall() + assert len(rows) == 2 + _assert_absent_everywhere(db_path, literal) + _assert_absent_everywhere(db_path, encoded) + + +@pytest.mark.parametrize("encoding", ["literal", "base64"]) +def test_record_event_sanitizes_details_and_deep_values_immutably( + tmp_path: Path, + encoding: str, +) -> None: + store, db_path = _new_store(tmp_path, f"event-content-{encoding}") + secret = _secret(encoding) + payload = {"outer": [{"middle": {"value": f"payload {secret}"}}]} + before = copy.deepcopy(payload) + + store.record_event( + claim_id=None, + event_type="system", + details=f"event detail {secret}", + payload=payload, + ) + + assert payload == before + row = _event_row(db_path) + assert "[REDACTED:" in str(row["details"]) + persisted = json.loads(str(row["payload_json"])) + assert "[REDACTED:" in persisted["outer"][0]["middle"]["value"] + _assert_absent_everywhere(db_path, secret) + _assert_chain_intact(store) + + +@pytest.mark.parametrize("encoding", ["literal", "base64"]) +def test_record_event_rejects_sensitive_json_key_without_appending( + tmp_path: Path, + encoding: str, +) -> None: + store, db_path = _new_store(tmp_path, f"event-key-{encoding}") + secret = _secret(encoding) + payload = {"outer": [{secret: "benign value"}]} + before = copy.deepcopy(payload) + + _assert_rejected_without_echo( + lambda: store.record_event( + claim_id=None, + event_type="system", + details="safe event key rejection", + payload=payload, + ), + secret, + ) + + assert payload == before + assert _table_counts(db_path, "events") == {"events": 0} + _assert_absent_everywhere(db_path, secret) + _assert_chain_intact(store) + + +def test_record_event_sanitizes_structured_key_value_secret( + tmp_path: Path, +) -> None: + store, db_path = _new_store(tmp_path, "event-structured-value") + credential = "R14StorePass9!xYz7LongFixture" + assert scan_text_for_findings(credential) == [] + assert "password_assignment" in scan_text_for_findings(f"password={credential}") + + store.record_event( + claim_id=None, + event_type="system", + details="safe structured payload case", + payload={"password": credential}, + ) + + row = _event_row(db_path) + persisted = json.loads(str(row["payload_json"])) + assert persisted["password"] == "[REDACTED:structured_secret]" + assert _durable_locations(db_path, credential) == [] + _assert_chain_intact(store) + + +@pytest.mark.parametrize( + "nested_value", + [ + ["R14StorePass9!xYz7LongFixture"], + {"value": "R14StorePass9!xYz7LongFixture"}, + ], +) +def test_record_event_propagates_structured_secret_context_into_containers( + tmp_path: Path, + nested_value: object, +) -> None: + store, db_path = _new_store(tmp_path, "event-structured-container") + credential = "R14StorePass9!xYz7LongFixture" + payload = {"password": nested_value} + + store.record_event( + claim_id=None, + event_type="system", + details="safe structured container case", + payload=payload, + ) + + row = _event_row(db_path) + persisted = str(row["payload_json"]) + assert credential not in persisted + assert "[REDACTED:structured_secret]" in persisted + assert _durable_locations(db_path, credential) == [] + _assert_chain_intact(store) + + +@pytest.mark.parametrize( + "nested_value", + [123456789, [123456789], {"value": 123456789}], +) +def test_record_event_sanitizes_numeric_structured_credentials( + tmp_path: Path, + nested_value: object, +) -> None: + store, db_path = _new_store(tmp_path, "event-structured-number") + payload = {"password": nested_value} + + store.record_event( + claim_id=None, + event_type="system", + details="safe structured numeric case", + payload=payload, + ) + + persisted = str(_event_row(db_path)["payload_json"]) + assert "123456789" not in persisted + assert "[REDACTED:structured_secret]" in persisted + assert _durable_locations(db_path, "123456789") == [] + _assert_chain_intact(store) + + +def test_record_event_rejects_sensitive_event_type_without_echo( + tmp_path: Path, +) -> None: + store, db_path = _new_store(tmp_path, "event-type") + secret = _secret("base64") + + _assert_rejected_without_echo( + lambda: store.record_event( + claim_id=None, + event_type=secret, + details="safe event type rejection", + ), + secret, + ) + + assert _table_counts(db_path, "events") == {"events": 0} + _assert_absent_everywhere(db_path, secret) + + +def test_internal_event_insert_sanitizes_before_hashing_and_preserves_chain( + tmp_path: Path, +) -> None: + store, db_path = _new_store(tmp_path, "internal-event-content") + literal = _secret("literal") + encoded = _secret("base64") + payload_json = json.dumps({"outer": [{"value": f"payload {encoded}"}]}) + + with store.connect() as conn: + store._insert_event_row( + conn, + claim_id=None, + event_type="system", + from_status=None, + to_status=None, + details=f"internal detail {literal}", + payload_json=payload_json, + created_at="2026-07-11T00:00:00+00:00", + ) + conn.commit() + + row = _event_row(db_path) + assert "[REDACTED:" in str(row["details"]) + persisted = json.loads(str(row["payload_json"])) + assert "[REDACTED:" in persisted["outer"][0]["value"] + _assert_absent_everywhere(db_path, literal) + _assert_absent_everywhere(db_path, encoded) + _assert_chain_intact(store) + + +def test_internal_event_insert_rejects_sensitive_json_key_before_append( + tmp_path: Path, +) -> None: + store, db_path = _new_store(tmp_path, "internal-event-key") + secret = _secret("base64") + payload_json = json.dumps({"outer": [{secret: "benign value"}]}) + + def insert() -> None: + with store.connect() as conn: + store._insert_event_row( + conn, + claim_id=None, + event_type="system", + from_status=None, + to_status=None, + details="safe internal event key rejection", + payload_json=payload_json, + created_at="2026-07-11T00:00:00+00:00", + ) + conn.commit() + + _assert_rejected_without_echo(insert, secret) + assert _table_counts(db_path, "events") == {"events": 0} + _assert_absent_everywhere(db_path, secret) + _assert_chain_intact(store) + + +@pytest.mark.parametrize( + ("field", "encoding"), + [("tenant_id", "literal"), ("principal", "base64")], +) +def test_postgres_rejects_sensitive_bound_identity_before_connect( + monkeypatch: pytest.MonkeyPatch, + field: str, + encoding: str, +) -> None: + secret = _secret(encoding) + identity = {"tenant_id": "tenant-safe", "principal": "principal-safe"} + identity[field] = secret + store = PostgresStore( + "postgresql://unused.invalid/memorymaster", + tenant_id=identity["tenant_id"], + require_tenant=True, + principal=identity["principal"], + allowed_scopes=["project:r14-store-tests"], + ) + connect_calls = 0 + + def unexpected_connect(): + nonlocal connect_calls + connect_calls += 1 + raise AssertionError("Postgres connection attempted before envelope rejection") + + monkeypatch.setattr(store, "connect", unexpected_connect) + + _assert_rejected_without_echo( + lambda: store.create_claim( + text="Safe hermetic Postgres identity rejection case.", + citations=[CitationInput(source="unit-test")], + scope="project:r14-store-tests", + visibility="private", + source_agent="principal-safe", + ), + secret, + ) + assert connect_calls == 0 + + +def test_postgres_rejects_direct_claim_metadata_before_connect( + monkeypatch: pytest.MonkeyPatch, +) -> None: + secret = _secret("base64") + store = PostgresStore("postgresql://unused.invalid/memorymaster") + connect_calls = 0 + + def unexpected_connect(): + nonlocal connect_calls + connect_calls += 1 + raise AssertionError("Postgres connection attempted before envelope rejection") + + monkeypatch.setattr(store, "connect", unexpected_connect) + _assert_rejected_without_echo( + lambda: store.create_claim( + text="Safe hermetic Postgres metadata rejection case.", + citations=[CitationInput(source="unit-test")], + idempotency_key=secret, + ), + secret, + ) + assert connect_calls == 0 + + +def test_postgres_event_rejects_sensitive_json_key_before_driver_load( + monkeypatch: pytest.MonkeyPatch, +) -> None: + secret = _secret("base64") + store = PostgresStore("postgresql://unused.invalid/memorymaster") + driver_loads = 0 + + def unexpected_driver_load(): + nonlocal driver_loads + driver_loads += 1 + raise AssertionError("Postgres driver loaded before event envelope rejection") + + monkeypatch.setattr(store, "_load_psycopg", unexpected_driver_load) + _assert_rejected_without_echo( + lambda: store._insert_event_row( + None, + claim_id=None, + event_type="system", + from_status=None, + to_status=None, + details="safe hermetic event key rejection", + payload={secret: "benign value"}, + created_at=datetime(2026, 7, 11, tzinfo=timezone.utc), + ), + secret, + ) + assert driver_loads == 0 + + +@pytest.mark.parametrize("field", ["tenant_id", "principal", "allowed_scopes"]) +def test_postgres_record_event_rejects_bound_identity_before_connect( + monkeypatch: pytest.MonkeyPatch, + field: str, +) -> None: + secret = _secret("base64") + identity: dict[str, object] = { + "tenant_id": "tenant-safe", + "principal": "principal-safe", + "allowed_scopes": ["project:r14-store-tests"], + } + identity[field] = [secret] if field == "allowed_scopes" else secret + store = PostgresStore( + "postgresql://unused.invalid/memorymaster", + tenant_id=identity["tenant_id"], + require_tenant=True, + principal=identity["principal"], + allowed_scopes=identity["allowed_scopes"], + ) + connect_calls = 0 + + def unexpected_connect(): + nonlocal connect_calls + connect_calls += 1 + raise AssertionError("Postgres connection attempted before bound identity rejection") + + monkeypatch.setattr(store, "connect", unexpected_connect) + _assert_rejected_without_echo( + lambda: store.record_event( + claim_id=None, + event_type="system", + details="safe bound identity rejection", + ), + secret, + ) + assert connect_calls == 0 + + +@pytest.mark.parametrize( + "operation_name", + ["get_claim", "set_normalized_text", "set_normalized_texts_batch", "update_claim_structure"], +) +def test_postgres_connect_rejects_bound_identity_before_network_for_all_paths( + monkeypatch: pytest.MonkeyPatch, + operation_name: str, +) -> None: + secret = _secret("base64") + store = PostgresStore( + "postgresql://unused.invalid/memorymaster", + tenant_id="tenant-safe", + require_tenant=True, + principal=secret, + allowed_scopes=["project:r14-store-tests"], + ) + open_calls = 0 + + def unexpected_open(): + nonlocal open_calls + open_calls += 1 + raise AssertionError("Postgres network opened before bound identity rejection") + + monkeypatch.setattr(store, "_open_connection", unexpected_open) + operations: dict[str, Callable[[], object]] = { + "get_claim": lambda: store.get_claim(1), + "set_normalized_text": lambda: store.set_normalized_text(1, "safe normalized text"), + "set_normalized_texts_batch": lambda: store.set_normalized_texts_batch( + {1: "safe normalized text"} + ), + "update_claim_structure": lambda: store.update_claim_structure( + 1, + subject="safe subject", + ), + } + + _assert_rejected_without_echo(operations[operation_name], secret) + assert open_calls == 0 diff --git a/tests/test_write_gateway_paths.py b/tests/test_write_gateway_paths.py index 7e2b96c4..550de814 100644 --- a/tests/test_write_gateway_paths.py +++ b/tests/test_write_gateway_paths.py @@ -44,10 +44,6 @@ def _claim_and_citation_text(db_path: Path) -> str: return "\n".join(str(value) for row in [*claims, *citations] for value in row if value is not None) -@pytest.mark.xfail( - strict=True, - reason="MM-SEC-04: compact-summary output writes directly to the store without sensitivity filtering", -) def test_compact_summary_output_never_persists_secret_shaped_llm_content( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, From becbcca52def3ecc11f4284eef7588b84a6a2fcd Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 13:35:40 -0300 Subject: [PATCH 16/55] chore(codex): route remediation through custom agents --- .codex/agents/mm-docs-ledger.toml | 15 +++ .codex/agents/mm-explorer.toml | 15 +++ .codex/agents/mm-fast-worker.toml | 16 +++ .codex/agents/mm-security-reviewer.toml | 15 +++ .codex/agents/mm-test-runner.toml | 15 +++ .codex/agents/mm-worker.toml | 16 +++ .codex/config.toml | 5 + .planning/REMEDIATION-EXECUTION-V2.md | 149 ++++++++++++++++++++++++ 8 files changed, 246 insertions(+) create mode 100644 .codex/agents/mm-docs-ledger.toml create mode 100644 .codex/agents/mm-explorer.toml create mode 100644 .codex/agents/mm-fast-worker.toml create mode 100644 .codex/agents/mm-security-reviewer.toml create mode 100644 .codex/agents/mm-test-runner.toml create mode 100644 .codex/agents/mm-worker.toml create mode 100644 .codex/config.toml create mode 100644 .planning/REMEDIATION-EXECUTION-V2.md diff --git a/.codex/agents/mm-docs-ledger.toml b/.codex/agents/mm-docs-ledger.toml new file mode 100644 index 00000000..4c94c5aa --- /dev/null +++ b/.codex/agents/mm-docs-ledger.toml @@ -0,0 +1,15 @@ +name = "mm_docs_ledger" +description = "Evidence-focused documentation worker for remediation ledgers, audit deltas, runbooks, and external-action records." +model = "gpt-5.6-terra" +model_reasoning_effort = "medium" +sandbox_mode = "workspace-write" +nickname_candidates = ["Terra Ledger", "Terra Scribe", "Terra Delta"] + +developer_instructions = """ +Obey the repository AGENTS.md and edit only documentation/planning files explicitly owned in the parent prompt. +Never change production code or tests. +Treat command output and committed diffs as evidence; never mark a finding resolved from intent or an agent self-report. +Preserve finding IDs, rollback instructions, external blockers, and the distinction between implemented and verified. +Do not mutate live databases or external systems and do not commit unless the parent explicitly assigns a worktree and commit. +Return the exact files changed and unresolved evidence gaps. +""" diff --git a/.codex/agents/mm-explorer.toml b/.codex/agents/mm-explorer.toml new file mode 100644 index 00000000..289fa348 --- /dev/null +++ b/.codex/agents/mm-explorer.toml @@ -0,0 +1,15 @@ +name = "mm_explorer" +description = "Fast read-only MemoryMaster code mapper for inventories, dependency tracing, and bounded evidence gathering." +model = "gpt-5.6-terra" +model_reasoning_effort = "low" +sandbox_mode = "read-only" +nickname_candidates = ["Terra Map", "Terra Trace", "Terra Scout"] + +developer_instructions = """ +Work read-only and obey the repository AGENTS.md plus the parent task. +Use MemoryMaster recall before architectural assumptions when its MCP transport is available. +Prefer GitNexus query/context and targeted rg/file reads over broad dumps. +Never propose or make architecture decisions; surface them to the parent with evidence. +Return concise findings with exact files, symbols, risk, and the smallest next action. +Do not edit files, create commits, mutate live databases, or run external-state-changing commands. +""" diff --git a/.codex/agents/mm-fast-worker.toml b/.codex/agents/mm-fast-worker.toml new file mode 100644 index 00000000..f2e23383 --- /dev/null +++ b/.codex/agents/mm-fast-worker.toml @@ -0,0 +1,16 @@ +name = "mm_fast_worker" +description = "Efficient implementation worker for isolated LOW/MEDIUM-risk fixes with explicit file ownership and acceptance tests." +model = "gpt-5.6-terra" +model_reasoning_effort = "medium" +sandbox_mode = "workspace-write" +nickname_candidates = ["Terra Patch", "Terra Build", "Terra Fix"] + +developer_instructions = """ +Obey the repository AGENTS.md and the exact owned-file boundary in the parent prompt. +Work only in the assigned isolated worktree and preserve all unrelated changes. +Before modifying an existing symbol, run GitNexus upstream impact analysis. +If risk is HIGH or CRITICAL, stop without editing and return the blast radius to the parent for rerouting. +For a security or integrity fix, first add and run an adversarial test that fails for the intended reason. +Make the smallest complete change, run focused tests and Ruff, and report exact evidence. +Never touch the live MemoryMaster database, push, publish, deploy, rotate credentials, or commit unless explicitly authorized. +""" diff --git a/.codex/agents/mm-security-reviewer.toml b/.codex/agents/mm-security-reviewer.toml new file mode 100644 index 00000000..4edf50a0 --- /dev/null +++ b/.codex/agents/mm-security-reviewer.toml @@ -0,0 +1,15 @@ +name = "mm_security_reviewer" +description = "Read-only high-reasoning reviewer for security boundaries, integrity invariants, bypasses, and missing adversarial coverage." +model = "gpt-5.6-sol" +model_reasoning_effort = "high" +sandbox_mode = "read-only" +nickname_candidates = ["Boundary Review", "Red Review", "Security Gate"] + +developer_instructions = """ +Review read-only as an independent owner and obey the repository AGENTS.md. +Focus on exploitable bypasses, authorization/scope errors, raw or encoded secret persistence, partial failures, replay/idempotency, and missing tests. +Trace actual callers and sinks with GitNexus and targeted source reads. +Separate merge blockers from follow-ups; cite exact files/symbols and provide a concrete reproduction or invariant for every blocker. +Do not edit, stage, commit, mutate databases, or approve work merely because tests pass. +Return a concise blocker-first report to the parent. +""" diff --git a/.codex/agents/mm-test-runner.toml b/.codex/agents/mm-test-runner.toml new file mode 100644 index 00000000..9a5fcbbe --- /dev/null +++ b/.codex/agents/mm-test-runner.toml @@ -0,0 +1,15 @@ +name = "mm_test_runner" +description = "Fast verification worker for focused pytest, Ruff, scanners, collection checks, and evidence summaries." +model = "gpt-5.6-terra" +model_reasoning_effort = "low" +sandbox_mode = "workspace-write" +nickname_candidates = ["Terra Gate", "Terra Probe", "Terra Verify"] + +developer_instructions = """ +Obey the repository AGENTS.md and run only the verification scope assigned by the parent. +Do not edit source, tests, plans, configuration, or documentation. +Use temporary databases and fake/local services; never mutate the live MemoryMaster database. +Report the exact command, exit code, pass/skip/xfail counts, duration, and concise failure signatures. +Do not infer broad success from a focused test and do not rerun a failing command repeatedly without a new hypothesis. +Leave commits, staging, GitNexus reindexing, and full-suite decisions to the parent. +""" diff --git a/.codex/agents/mm-worker.toml b/.codex/agents/mm-worker.toml new file mode 100644 index 00000000..3609d87a --- /dev/null +++ b/.codex/agents/mm-worker.toml @@ -0,0 +1,16 @@ +name = "mm_worker" +description = "General implementation worker for bounded MemoryMaster packages that need stronger multi-step coding and validation." +model = "gpt-5.6-sol" +model_reasoning_effort = "medium" +sandbox_mode = "workspace-write" +nickname_candidates = ["Core Worker", "Package Worker", "Integration Worker"] + +developer_instructions = """ +Obey the repository AGENTS.md and the exact owned-file/worktree boundary in the parent prompt. +Query MemoryMaster before architectural decisions when its MCP transport is available. +Run GitNexus upstream impact before editing every existing symbol and stop for parent warning before HIGH/CRITICAL work not already disclosed. +Add a witnessed failing adversarial test before security or integrity fixes. +Keep changes atomic, immutable, and narrowly scoped; use temporary databases and fake/local external services. +Run focused verification and Ruff, then return changed files, risks, commands, and unresolved blockers. +Never push, publish, deploy, mutate product data, or commit unless explicitly authorized. +""" diff --git a/.codex/config.toml b/.codex/config.toml new file mode 100644 index 00000000..809fdedd --- /dev/null +++ b/.codex/config.toml @@ -0,0 +1,5 @@ +[agents] +max_threads = 4 +max_depth = 1 +job_max_runtime_seconds = 1800 +interrupt_message = true diff --git a/.planning/REMEDIATION-EXECUTION-V2.md b/.planning/REMEDIATION-EXECUTION-V2.md new file mode 100644 index 00000000..e77ea7b7 --- /dev/null +++ b/.planning/REMEDIATION-EXECUTION-V2.md @@ -0,0 +1,149 @@ +# MemoryMaster Remediation Execution V2 + +**Status:** execution overlay +**Audit source of truth:** `.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md` +**Finding source of truth:** `.planning/audit-remediation-ledger.md` + +This file changes scheduling and model routing, not audit facts. The original +roadmap remains immutable evidence. V2 minimizes wall-clock time, premium-model +usage, and main-thread context pollution while preserving every security gate. + +## Recommended scope + +Finish **Phase 1 security convergence**, produce a same-scope audit delta, and +then stop. Keep Phases 2-4 as a prioritized backlog unless the operator starts a +separate optimization goal. Production-grade multi-user ambitions can justify +that later program; they are not prerequisites for a hardened local deployment. + +## Model-routing policy + +| Work | Agent | Model / effort | Write authority | +|---|---|---|---| +| Code mapping, inventories, impact evidence | `mm_explorer` | Terra / low | None | +| Focused tests, Ruff, scanners, evidence | `mm_test_runner` | Terra / low | Runtime artifacts only | +| Ledger, audit delta, runbooks | `mm_docs_ledger` | Terra / medium | Explicit docs only | +| LOW/MEDIUM isolated implementation | `mm_fast_worker` | Terra / medium | Explicit owned files | +| Bounded multi-step implementation | `mm_worker` | GPT-5.6 Sol / medium | Explicit owned files | +| Security/integrity review | `mm_security_reviewer` | GPT-5.6 Sol / high | None | +| Architecture, HIGH/CRITICAL decisions, merge, final audit | root | Highest selected level | Integration worktree | + +Rules: + +- Root plus at most three workers run concurrently (`max_threads = 4`). +- Use the smallest context fork that contains the task; every worker prompt must + be self-contained and name owned files, acceptance evidence, and stop rules. +- Terra workers must stop and reroute any newly discovered HIGH/CRITICAL edit. +- Only one write agent owns a worktree. Read-only workers may inspect it. +- Subagents run focused gates. Root runs cross-package regression and the full + verification ladder at integration boundaries. +- Parallelism saves wall-clock and premium-model usage, not necessarily total + tokens. Do not fan out work that shares files or requires one architectural + decision. + +## Worktree and merge protocol + +1. Root owns `remediation/audit-convergence-20260710` in the existing isolated + checkout and first commits or cleanly shelves the current R1.4 auxiliary work. +2. Each write package starts from that integration commit in a dedicated branch + and worktree under a temporary remediation-agent root. +3. The parent prompt declares `owns=`. Workers never edit outside it. +4. A worker returns its diff, targeted evidence, and risks. It does not merge. +5. Root reviews, runs GitNexus change detection, integrates one package at a + time, executes the package regression gate, commits atomically, and refreshes + the embedding-preserving GitNexus index. +6. After a wave, root runs the full non-ML suite once, required ML gates once, + Ruff, runtime smoke checks, and ledger reconciliation. + +## Phase 1 dependency waves + +### Wave A0 - preserve current work (serial root) + +- Finish independent review of the already-green auxiliary persistence package. +- Run its focused/ML gates, commit it atomically, and refresh GitNexus. +- No other write worker touches the integration checkout during A0. + +### Wave A1 - finish R1.4 (parallel after A0) + +| Lane | Owner | Scope | +|---|---|---| +| Atlas envelope | `mm_worker` | Atlas source/evidence/provenance writes, legacy reads, ADR reconciliation | +| Remaining writers | `mm_worker` | Steward, compact summaries, miners, imports, bridges, merge/delta paths | +| Legacy inventory | `mm_explorer` + `mm_docs_ledger` | Dry-run inventory across primary DB, verbatim, Qdrant payloads, artifacts; no cleanup | + +Root integrates lanes sequentially, resolves overlap at the canonical gateway, +and runs the complete R1.4 table-driven adversarial matrix. Live cleanup remains +forbidden and is recorded as an external action. + +### Wave B - R1.5 secure deployment defaults (parallel after R1.4) + +| Lane | Owner | Scope | +|---|---|---| +| Deployment contracts | `mm_fast_worker` | Compose/Helm secrets, private bindings, immutable image references | +| Qdrant transport | `mm_worker` | API key/TLS propagation and fake-service tests | +| Supply-chain evidence | `mm_fast_worker` + `mm_test_runner` | Secret/history scan, dependency/image scan wiring, SBOM artifact | + +Anything requiring real credential rotation, authenticated infrastructure, image +registry access, or deployment becomes `BLOCKED-EXTERNAL` with exact operator +commands in `external-actions-required.md`. + +### Wave C - Phase 1 convergence (serial root plus read-only reviewers) + +1. Run the Phase 1 verification ladder from the original roadmap. +2. Rerun hard-stops, blind spots, and affected audit domains against the exact + baseline scope. +3. Produce an audit delta and reconcile every Phase 1 ledger row to commit and + runtime evidence. +4. Use Terra agents for file-by-file evidence gathering and GPT-5.6 Sol/high for the + independent security review. +5. Fix newly introduced Phase 1 findings until the latest same-scope audit has + zero new findings and every Phase 1 item is resolved or validly blocked. +6. Stop. Do not begin R2-R4 under this goal. + +## Goal completion contract + +Completion means Phase 1—not the entire optimization roadmap—is proven: + +- R1.1-R1.5 repository work is resolved or validly blocked. +- Disabled profiles remain fail-closed where external parity is unavailable. +- The full required verification ladder is green or has explicit external + blockers without false success claims. +- The Phase 1 audit delta has zero new findings in its latest run. +- Phases 2-4 are preserved as prioritized backlog with no implied completion. + +## Replacement goal prompt + +```text +/goal Execute Phase 1 security convergence for MemoryMaster using +`.planning/REMEDIATION-EXECUTION-V2.md` as the scheduler and +`.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md` plus +`.planning/audit-remediation-ledger.md` as the audit sources of truth. + +Resume from the isolated remediation branch/worktree and preserve the current +uncommitted R1.4 auxiliary package. First finish its review, verification, +atomic commit, and embedding-preserving GitNexus refresh. Then execute Waves A1, +B, and C in dependency order. + +Use project custom agents and explicit model routing: Terra/low for read-only +mapping and test evidence, Terra/medium for LOW/MEDIUM isolated work and docs, +GPT-5.6 Sol/medium for bounded multi-step implementation, GPT-5.6 Sol/high for +independent security review, and the root/highest level only for architecture, +HIGH/CRITICAL decisions, integration, and final convergence. Run at most three +children beside root. Give each write worker its own worktree and explicit owned +files; never allow concurrent writers in one checkout. Reroute newly discovered +HIGH/CRITICAL edits from Terra to root and warn before proceeding. + +Keep all existing safety constraints: query MemoryMaster before architecture +decisions when available; run GitNexus impact before symbol edits and change +detection before commits; add witnessed adversarial RED tests before security or +integrity fixes; preserve user changes; use temporary databases/fake services; +never mutate the live MemoryMaster DB, push, publish, deploy, rotate credentials, +or perform external product-data changes without explicit approval. Record real +external blockers in `external-actions-required.md` and continue unblocked work. + +Workers run focused verification; root runs cross-package gates and the full +Phase 1 verification ladder at integration boundaries. Rerun the same-scope +Phase 1 audit, reconcile the ledger to commit/runtime evidence, and fix new +Phase 1 findings until every item is resolved or validly blocked and the latest +audit has zero new findings. Stop after the Phase 1 audit delta. Do not execute +Phases 2-4; preserve them as prioritized backlog for a separate goal. +``` From d466cd270435452aa18c6e31238aba1e0f4d4563 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 15:46:30 -0300 Subject: [PATCH 17/55] fix(security): harden auxiliary persistence boundaries --- memorymaster/core/security.py | 15 + memorymaster/core/service.py | 26 +- memorymaster/core/spool.py | 146 +++- memorymaster/govern/feedback.py | 6 +- memorymaster/govern/jobs/spool_drain.py | 14 + memorymaster/knowledge/daily_notes.py | 54 +- memorymaster/knowledge/rule_miner.py | 37 +- memorymaster/recall/verbatim_recall.py | 24 +- memorymaster/recall/verbatim_store.py | 160 +++- memorymaster/stores/_storage_write_claims.py | 9 +- memorymaster/stores/postgres_store.py | 9 +- .../test_auxiliary_persistence_sensitivity.py | 783 ++++++++++++++++++ 12 files changed, 1208 insertions(+), 75 deletions(-) create mode 100644 tests/test_auxiliary_persistence_sensitivity.py diff --git a/memorymaster/core/security.py b/memorymaster/core/security.py index c0ea01d9..08a53824 100644 --- a/memorymaster/core/security.py +++ b/memorymaster/core/security.py @@ -6,6 +6,7 @@ import os import re import unicodedata +from collections.abc import Iterable from collections.abc import Iterator from collections.abc import Mapping from dataclasses import asdict @@ -190,6 +191,20 @@ def __init__(self, field: str, findings: list[str]) -> None: super().__init__(f"{field} contains sensitive data ({labels})") +def normalize_sensitivity_findings(findings: Iterable[str] | None) -> list[str]: + """Validate and canonicalize findings-only security metadata.""" + if findings is None: + return [] + if isinstance(findings, (str, bytes)): + raise ValueError("Sensitivity findings must be a collection of labels.") + normalized: set[str] = set() + for value in findings: + if not isinstance(value, str) or re.fullmatch(r"[a-z0-9_]{1,64}", value) is None: + raise ValueError("Invalid sensitivity finding label.") + normalized.add(value) + return sorted(normalized) + + def _as_bool(value: object, *, field: str) -> bool: if isinstance(value, bool): return value diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index 6ac10baf..dc1449e5 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -21,8 +21,10 @@ from memorymaster.recall.retrieval import VectorSearchHook, _tier_bonus, rank_claim_rows from memorymaster.core.security import ( is_sensitive_claim, + normalize_sensitivity_findings, resolve_allow_sensitive_access, sanitize_claim_input, + sanitize_persisted_text, validate_persisted_metadata, ) from memorymaster.core.intake_policy import ( @@ -532,6 +534,7 @@ def ingest( require_source_agent: bool = False, intake_batch_id: str | None = None, intake_batch_max: int | None = None, + _pre_sanitization_findings: list[str] | None = None, ) -> Claim: if not text.strip(): raise ValueError("Claim text cannot be empty.") @@ -556,6 +559,9 @@ def ingest( valid_until=valid_until, intake_batch_id=intake_batch_id, ) + redaction_findings = normalize_sensitivity_findings( + [*sanitized.findings, *(_pre_sanitization_findings or [])] + ) if not sanitized.citations: raise ValueError("At least one citation is required.") text = sanitized.text @@ -741,6 +747,7 @@ def ingest( source_agent=source_agent, visibility=visibility, holder=holder, + _pre_sanitization_findings=redaction_findings, ) # Set entity_id on the claim (best-effort, don't fail ingest) @@ -754,14 +761,8 @@ def ingest( _conn.commit() except Exception: pass - if sanitized.is_sensitive: - observability.bump_claim_filtered_findings(sanitized.findings) - self.store.record_event( - claim_id=claim.id, - event_type="policy_decision", - details="sensitive_redaction_applied", - payload={"findings": sanitized.findings}, - ) + if redaction_findings: + observability.bump_claim_filtered_findings(redaction_findings) if sanitized.encrypted_payload: self.store.record_event( claim_id=claim.id, @@ -1578,20 +1579,21 @@ def _spool_accesses(self, claim_ids: list[int], query_text: str) -> None: from memorymaster.core import spool + safe_query, _ = sanitize_persisted_text(query_text or "") query_hash = ( - hashlib.sha1(query_text.encode("utf-8")).hexdigest()[:12] - if query_text + hashlib.sha256(safe_query.encode("utf-8")).hexdigest()[:12] + if safe_query else None ) with contextlib.suppress(Exception): spool.append( db_path, "access", {"claim_ids": claim_ids, "query_hash": query_hash} ) - if query_text: + if safe_query: spool.append( db_path, "feedback", - {"claim_ids": claim_ids, "query_text": query_text}, + {"claim_ids": claim_ids, "query_text": safe_query}, ) def recompute_tiers(self) -> dict[str, int]: diff --git a/memorymaster/core/spool.py b/memorymaster/core/spool.py index 267ad681..c235ffcf 100644 --- a/memorymaster/core/spool.py +++ b/memorymaster/core/spool.py @@ -32,6 +32,13 @@ from datetime import datetime, timezone from pathlib import Path +from memorymaster.core.models import CitationInput +from memorymaster.core.security import ( + sanitize_claim_input, + sanitize_persisted_text, + validate_persisted_metadata, +) + ENV_SPOOL_DIR = "MEMORYMASTER_SPOOL_DIR" ENV_WAL_DISCIPLINE = "MEMORYMASTER_WAL_DISCIPLINE" @@ -41,6 +48,125 @@ DRAINING_SUFFIX = ".draining" QUARANTINE_DIRNAME = "quarantine" +_INGEST_PAYLOAD_FIELDS = frozenset( + { + "text", "citations", "claim_type", "subject", "predicate", + "object_value", "scope", "volatility", "confidence", "event_time", + "valid_from", "valid_until", "source_agent", "visibility", "holder", + "intake_batch_id", "intake_batch_max", + } +) +_VERBATIM_PAYLOAD_FIELDS = frozenset( + {"session_id", "role", "content", "scope", "source_agent", "timestamp"} +) +_ACCESS_PAYLOAD_FIELDS = frozenset({"claim_ids", "query_hash"}) +_FEEDBACK_PAYLOAD_FIELDS = frozenset({"claim_ids", "query_text"}) + + +def _reject_unknown_fields(payload: dict[str, object], allowed: frozenset[str]) -> None: + if any(key not in allowed for key in payload): + raise ValueError("Spool payload contains unsupported fields.") + + +def _citation_inputs(value: object) -> list[CitationInput]: + if value is None: + return [] + if not isinstance(value, list): + raise ValueError("Spool citations must be a list.") + citations: list[CitationInput] = [] + for item in value: + if not isinstance(item, dict): + raise ValueError("Spool citation entries must be objects.") + source = item.get("source", "") + locator = item.get("locator") + excerpt = item.get("excerpt") + if not isinstance(source, str): + raise ValueError("Spool citation source must be a string.") + if locator is not None and not isinstance(locator, str): + raise ValueError("Spool citation locator must be a string.") + if excerpt is not None and not isinstance(excerpt, str): + raise ValueError("Spool citation excerpt must be a string.") + citations.append(CitationInput(source, locator, excerpt)) + return citations + + +def _sanitize_ingest_payload( + payload: dict[str, object], + idempotency_key: str | None, +) -> dict[str, object]: + _reject_unknown_fields(payload, _INGEST_PAYLOAD_FIELDS) + text = payload.get("text", "") + if not isinstance(text, str): + raise ValueError("Spool claim text must be a string.") + citations = _citation_inputs(payload.get("citations")) + sanitized = sanitize_claim_input( + text=text, + object_value=payload.get("object_value"), + citations=citations, + subject=payload.get("subject"), + predicate=payload.get("predicate"), + idempotency_key=idempotency_key, + claim_type=payload.get("claim_type"), + scope=payload.get("scope"), + volatility=payload.get("volatility"), + source_agent=payload.get("source_agent"), + visibility=payload.get("visibility"), + holder=payload.get("holder"), + confidence=payload.get("confidence"), + event_time=payload.get("event_time"), + valid_from=payload.get("valid_from"), + valid_until=payload.get("valid_until"), + intake_batch_id=payload.get("intake_batch_id"), + ) + validate_persisted_metadata({"intake_batch_max": payload.get("intake_batch_max")}) + safe = {key: value for key, value in payload.items() if key not in { + "text", "subject", "predicate", "object_value", "citations" + }} + safe.update({"text": sanitized.text, "citations": [ + {key: value for key, value in { + "source": cite.source, "locator": cite.locator, "excerpt": cite.excerpt + }.items() if value is not None} + for cite in sanitized.citations + ]}) + for key, value in { + "subject": sanitized.subject, + "predicate": sanitized.predicate, + "object_value": sanitized.object_value, + }.items(): + if key in payload: + safe[key] = value + if sanitized.findings: + safe["_sanitization"] = {"findings": sanitized.findings} + return safe + + +def _sanitize_feedback_payload(payload: dict[str, object]) -> dict[str, object]: + _reject_unknown_fields(payload, _FEEDBACK_PAYLOAD_FIELDS) + query = payload.get("query_text", "") + if not isinstance(query, str): + raise ValueError("Spool feedback query_text must be a string.") + validate_persisted_metadata({"claim_ids": payload.get("claim_ids")}) + safe_query, findings = sanitize_persisted_text(query) + safe = {"claim_ids": list(payload.get("claim_ids") or []), "query_text": safe_query} + if findings: + safe["_sanitization"] = {"findings": findings} + return safe + + +def _sanitize_spool_payload( + op: str, + payload: dict[str, object], + idempotency_key: str | None, +) -> dict[str, object]: + if op in {"ingest", "dream"}: + return _sanitize_ingest_payload(payload, idempotency_key) + if op == "feedback": + return _sanitize_feedback_payload(payload) + allowed = _VERBATIM_PAYLOAD_FIELDS if op == "verbatim" else _ACCESS_PAYLOAD_FIELDS + _reject_unknown_fields(payload, allowed) + validate_persisted_metadata({f"{op}_payload": payload}) + return dict(payload) + def wal_discipline_enabled() -> bool: """P1 WAL-discipline umbrella flag (spec §5), default OFF. @@ -117,7 +243,11 @@ def append( """ if op not in KNOWN_OPS: raise ValueError(f"unknown spool op: {op!r} (known: {KNOWN_OPS})") - envelope = make_envelope(op, payload, idempotency_key=idempotency_key, ts=ts) + validate_persisted_metadata( + {"spool_op": op, "spool_idempotency_key": idempotency_key, "spool_ts": ts} + ) + safe_payload = _sanitize_spool_payload(op, payload, idempotency_key) + envelope = make_envelope(op, safe_payload, idempotency_key=idempotency_key, ts=ts) line = json.dumps(envelope, ensure_ascii=True, separators=(",", ":")) spool_dir = spool_dir_for(db_path) spool_dir.mkdir(parents=True, exist_ok=True) @@ -166,9 +296,12 @@ def read_lines(path: Path) -> list[str]: def quarantine_line(db_path: str | Path, raw_line: str, reason: str) -> Path: """Preserve an unreplayable line under ``quarantine/`` — never drop it. - Wrapped with the reason + timestamp so the operator can audit and - hand-replay after a fix; the raw line is kept byte-for-byte. + Wrapped with a sanitized reason, timestamp, original length, and digest of + the safe surrogate. Credential-shaped source bytes are never duplicated. """ + safe_raw, raw_findings = sanitize_persisted_text(raw_line) + safe_reason, reason_findings = sanitize_persisted_text(reason) + findings = sorted(set(raw_findings + reason_findings)) qdir = quarantine_dir_for(db_path) qdir.mkdir(parents=True, exist_ok=True) day = datetime.now(timezone.utc).strftime("%Y%m%d") @@ -176,8 +309,11 @@ def quarantine_line(db_path: str | Path, raw_line: str, reason: str) -> Path: record = json.dumps( { "quarantined_at": datetime.now(timezone.utc).isoformat(), - "reason": reason, - "raw": raw_line, + "reason": safe_reason, + "raw": safe_raw, + "findings": findings, + "raw_length": len(raw_line), + "sanitized_sha256": hashlib.sha256(safe_raw.encode("utf-8")).hexdigest(), }, ensure_ascii=True, separators=(",", ":"), diff --git a/memorymaster/govern/feedback.py b/memorymaster/govern/feedback.py index f3bf8ab0..97ad546d 100644 --- a/memorymaster/govern/feedback.py +++ b/memorymaster/govern/feedback.py @@ -14,6 +14,7 @@ import uuid from datetime import datetime, timezone +from memorymaster.core.security import sanitize_persisted_text from memorymaster.stores._storage_shared import open_conn logger = logging.getLogger(__name__) @@ -64,6 +65,9 @@ def record_retrieval(self, claim_ids: list[int], query_text: str) -> int: if not claim_ids: return 0 + raw_query = query_text if isinstance(query_text, str) else str(query_text or "") + safe_query, _ = sanitize_persisted_text(raw_query) + if not isinstance(claim_ids, list): logger.warning("record_retrieval: claim_ids is not a list, converting") try: @@ -84,7 +88,7 @@ def record_retrieval(self, claim_ids: list[int], query_text: str) -> int: now = datetime.now(timezone.utc).isoformat() rows = [ - (str(uuid.uuid4()), cid, query_text[:500] if query_text else "", now, 1, None) + (str(uuid.uuid4()), cid, safe_query[:500], now, 1, None) for cid in claim_ids ] conn = self._connect() diff --git a/memorymaster/govern/jobs/spool_drain.py b/memorymaster/govern/jobs/spool_drain.py index c7ce2771..49d5eef6 100644 --- a/memorymaster/govern/jobs/spool_drain.py +++ b/memorymaster/govern/jobs/spool_drain.py @@ -33,6 +33,7 @@ from pathlib import Path from memorymaster.core import spool +from memorymaster.core.security import normalize_sensitivity_findings from memorymaster.govern.jobs.integrity import _record from memorymaster.core.models import CitationInput @@ -56,11 +57,22 @@ "valid_until", "source_agent", "visibility", + "holder", "intake_batch_id", "intake_batch_max", ) +def _boundary_findings(payload: dict[str, object]) -> list[str]: + """Return validated, finding-name-only metadata from the spool gateway.""" + sidecar = payload.get("_sanitization") + if sidecar is None: + return [] + if not isinstance(sidecar, dict) or not isinstance(sidecar.get("findings"), list): + raise ValueError("Invalid spool sanitization metadata.") + return normalize_sensitivity_findings(sidecar["findings"]) + + def _parse_ts(raw: object) -> datetime | None: try: stamp = datetime.fromisoformat(str(raw)) @@ -74,6 +86,7 @@ def _parse_ts(raw: object) -> datetime | None: def _replay_ingest(svc, envelope: dict[str, object]) -> None: """Replay an ingest/dream line through svc.ingest — filter + dedup apply.""" payload = envelope["payload"] + boundary_findings = _boundary_findings(payload) kwargs = {k: payload[k] for k in _INGEST_FIELDS if payload.get(k) is not None} if envelope.get("op") == "dream": kwargs.setdefault("source_agent", "dream-bridge") @@ -91,6 +104,7 @@ def _replay_ingest(svc, envelope: dict[str, object]) -> None: str(payload.get("text") or ""), citations, idempotency_key=str(idempotency_key) if idempotency_key else None, + _pre_sanitization_findings=boundary_findings, **kwargs, ) diff --git a/memorymaster/knowledge/daily_notes.py b/memorymaster/knowledge/daily_notes.py index 9a4cc852..6ecff596 100644 --- a/memorymaster/knowledge/daily_notes.py +++ b/memorymaster/knowledge/daily_notes.py @@ -17,15 +17,41 @@ from __future__ import annotations import logging +import re import sqlite3 from collections import Counter from datetime import datetime, timezone from pathlib import Path +from memorymaster.core.security import sanitize_persisted_text from memorymaster.stores._storage_shared import connect_ro logger = logging.getLogger(__name__) +_REDACTION_MARKER_RE = re.compile(r"\[REDACTED:[^\]]+\]", re.IGNORECASE) + + +def _validated_date(value: str | None) -> str: + if value is None: + return datetime.now(timezone.utc).strftime("%Y-%m-%d") + try: + parsed = datetime.strptime(value, "%Y-%m-%d") + except (TypeError, ValueError) as exc: + raise ValueError("date must use YYYY-MM-DD") from exc + if parsed.strftime("%Y-%m-%d") != value: + raise ValueError("date must use YYYY-MM-DD") + return value + + +def _safe_text(value: object) -> str: + safe, _ = sanitize_persisted_text(str(value or "")) + return safe + + +def _topic_words(value: object, *, min_length: int) -> list[str]: + safe = _REDACTION_MARKER_RE.sub(" ", _safe_text(value)) + return [word.lower() for word in safe.split() if len(word) > min_length] + def generate_daily_note(db_path: str, date: str | None = None) -> dict: """Generate a daily note from today's feedback and claim activity. @@ -33,8 +59,7 @@ def generate_daily_note(db_path: str, date: str | None = None) -> dict: Summarizes: what was queried, what was ingested, what topics recurred. Returns dict with the note content and metadata. """ - if date is None: - date = datetime.now(timezone.utc).strftime("%Y-%m-%d") + date = _validated_date(date) # Report is read-only; per-table queries below stay defensively wrapped. conn = connect_ro(db_path) @@ -47,7 +72,7 @@ def generate_daily_note(db_path: str, date: str | None = None) -> dict: "SELECT query_text, COUNT(*) as cnt FROM usage_feedback WHERE timestamp LIKE ? GROUP BY query_text ORDER BY cnt DESC LIMIT 10", (f"{date}%",), ).fetchall() - queries = [(r["query_text"], r["cnt"]) for r in rows] + queries = [(_safe_text(r["query_text"]), r["cnt"]) for r in rows] except sqlite3.OperationalError: pass @@ -58,7 +83,15 @@ def generate_daily_note(db_path: str, date: str | None = None) -> dict: "SELECT id, text, claim_type, scope FROM claims WHERE created_at LIKE ? ORDER BY id DESC LIMIT 15", (f"{date}%",), ).fetchall() - ingested = [{"id": r["id"], "text": r["text"][:100], "type": r["claim_type"], "scope": r["scope"]} for r in rows] + ingested = [ + { + "id": r["id"], + "text": _safe_text(r["text"])[:100], + "type": _safe_text(r["claim_type"]), + "scope": r["scope"], + } + for r in rows + ] except sqlite3.OperationalError: pass @@ -72,13 +105,19 @@ def generate_daily_note(db_path: str, date: str | None = None) -> dict: for r in rows: claim = conn.execute("SELECT text FROM claims WHERE id = ?", (r["claim_id"],)).fetchone() if claim: - accessed.append({"id": r["claim_id"], "text": claim["text"][:80], "access_count": r["cnt"]}) + accessed.append( + { + "id": r["claim_id"], + "text": _safe_text(claim["text"])[:80], + "access_count": r["cnt"], + } + ) except sqlite3.OperationalError: pass # Extract topics (most common words in queries) all_query_text = " ".join(q for q, _ in queries) - words = [w.lower() for w in all_query_text.split() if len(w) > 3] + words = _topic_words(all_query_text, min_length=3) topic_counts = Counter(words) topics = [w for w, c in topic_counts.most_common(5) if c > 1] @@ -143,8 +182,7 @@ def find_ghost_notes(db_path: str, min_references: int = 3) -> list[dict]: # Count word frequency across queries word_freq: Counter = Counter() for r in rows: - words = set(r["query_text"].lower().split()) - meaningful = [w for w in words if len(w) > 4] + meaningful = set(_topic_words(r["query_text"], min_length=4)) word_freq.update(meaningful) # Find words that appear in many queries diff --git a/memorymaster/knowledge/rule_miner.py b/memorymaster/knowledge/rule_miner.py index a0e7e595..ba7949fc 100644 --- a/memorymaster/knowledge/rule_miner.py +++ b/memorymaster/knowledge/rule_miner.py @@ -34,7 +34,7 @@ from memorymaster.stores._storage_shared import open_conn from memorymaster.core.models import CitationInput from memorymaster.knowledge.rules import build_rule_fields -from memorymaster.core.security import redact_text +from memorymaster.core.security import scan_persisted_value logger = logging.getLogger(__name__) @@ -230,9 +230,26 @@ def _extract_rule(window: str) -> dict[str, str] | None: def _is_sensitive_rule(rule: dict[str, str]) -> bool: - joined = " | ".join(filter(None, (rule["trigger"], rule["action"], rule["rationale"]))) - _, findings = redact_text(joined) - return bool(findings) + return bool(scan_persisted_value(rule)) + + +def _window_is_sensitive( + assistant_content: str, + user_content: str, + *, + session_id: object = None, + scope: object = None, +) -> bool: + return bool( + scan_persisted_value( + { + "assistant_content": assistant_content, + "user_content": user_content, + "session_id": session_id, + "scope": scope, + } + ) + ) # --------------------------------------------------------------------------- @@ -442,6 +459,15 @@ def _process_candidate( stats["skipped"] += 1 return "done" + if _window_is_sensitive( + asst["content"], + row["content"], + session_id=row["session_id"], + scope=row["scope"], + ): + stats["skipped"] += 1 + return "done" + window = _build_window(asst["content"], row["content"]) if len(window) < _MIN_WINDOW_CHARS: stats["skipped"] += 1 @@ -578,6 +604,9 @@ def mine_transcript_rules( with llm_budget.cycle_scope(): for asst_text, user_text in windows: stats["windows"] += 1 + if _window_is_sensitive(asst_text, user_text, scope=scope): + stats["skipped"] += 1 + continue try: rule = _extract_rule(_build_window(asst_text, user_text)) except (llm_budget.LLMBudgetExceeded, TransientLLMError): diff --git a/memorymaster/recall/verbatim_recall.py b/memorymaster/recall/verbatim_recall.py index 6911cc98..81fcc289 100644 --- a/memorymaster/recall/verbatim_recall.py +++ b/memorymaster/recall/verbatim_recall.py @@ -39,6 +39,7 @@ from dataclasses import dataclass from memorymaster.recall.recall_tokenizer import _candidate_tokens +from memorymaster.recall.verbatim_store import _verbatim_row_has_sensitive_field logger = logging.getLogger(__name__) @@ -153,10 +154,9 @@ def recall_verbatim( Raw user prompt. Tokenized internally — caller does NOT need to pre-process. scope: - Optional scope filter (matches ``verbatim_memories.scope`` with a - ``LIKE scope%`` prefix, consistent with the existing - ``verbatim_store._search_fts`` contract). Pass ``None`` to search - across all scopes. + Optional scope filter. Concrete scopes match exactly; the root + ``project`` scope also includes ``project:*`` descendants. Pass + ``None`` to search across all scopes. db_path: Path to the memorymaster DB. Opened read-only. limit: @@ -175,6 +175,7 @@ def recall_verbatim( if not query or not query.strip(): return [] limit = max(1, min(limit, 20)) + fetch_limit = max(limit * 5, limit + 20) match_expr = _build_match_expr(query) if not match_expr: return [] @@ -193,26 +194,27 @@ def recall_verbatim( if scope: rows = conn.execute( """ - SELECT v.id, v.session_id, v.role, v.content, v.scope, rank AS score + SELECT v.*, rank AS score FROM verbatim_fts f JOIN verbatim_memories v ON v.id = f.rowid - WHERE verbatim_fts MATCH ? AND v.scope LIKE ? + WHERE verbatim_fts MATCH ? + AND (v.scope = ? OR (? = 'project' AND v.scope LIKE 'project:%')) ORDER BY rank LIMIT ? """, - (match_expr, f"{scope}%", limit), + (match_expr, scope, scope, fetch_limit), ).fetchall() else: rows = conn.execute( """ - SELECT v.id, v.session_id, v.role, v.content, v.scope, rank AS score + SELECT v.*, rank AS score FROM verbatim_fts f JOIN verbatim_memories v ON v.id = f.rowid WHERE verbatim_fts MATCH ? ORDER BY rank LIMIT ? """, - (match_expr, limit), + (match_expr, fetch_limit), ).fetchall() except sqlite3.Error as exc: logger.debug("verbatim_recall: query failed: %s", exc) @@ -222,6 +224,8 @@ def recall_verbatim( out: list[VerbatimHit] = [] for r in rows: + if _verbatim_row_has_sensitive_field(r): + continue raw_score = r["score"] if r["score"] is not None else 0.0 # FTS5 returns negative numbers where smaller == better match. # Flip sign so bigger == better (ranker convention). @@ -238,6 +242,8 @@ def recall_verbatim( role=str(r["role"] or ""), ) ) + if len(out) >= limit: + break return out diff --git a/memorymaster/recall/verbatim_store.py b/memorymaster/recall/verbatim_store.py index e3c92311..df2a284b 100644 --- a/memorymaster/recall/verbatim_store.py +++ b/memorymaster/recall/verbatim_store.py @@ -23,32 +23,72 @@ # Credential detection delegated to the canonical filter in memorymaster.core.security. from memorymaster.core import spool +from memorymaster.core.security import scan_persisted_value from memorymaster.stores._storage_shared import open_conn -from memorymaster.core.security import redact_text as _redact_text logger = logging.getLogger(__name__) def _contains_sensitive(text: str) -> bool: - _, findings = _redact_text(text) - return bool(findings) + return bool(scan_persisted_value(text)) -def _row_has_sensitive_field(role: str, source_agent: str, content: str) -> bool: - """Defense-in-depth: check role and source_agent in addition to content. +def _row_has_sensitive_field( + role: str, + source_agent: str, + content: str, + *, + session_id: str = "", + scope: str = "", + timestamp: str = "", + created_at: str = "", +) -> bool: + """Defense-in-depth: scan every textual field persisted for a turn. F-4 fix (overnight audit 2026-05-04): role and source_agent are user-controlled in some upstream paths (CLI flags, dream-bridge config, transcript miner). A maliciously crafted or misconfigured source_agent='Bearer ghp_xxx...' would persist a token to - verbatim_memories undetected if we only checked content. The canonical - redact_text covers all three fields here — refuse the whole row if any - finding appears anywhere. Don't redact-and-store; just drop. + verbatim_memories undetected if we only checked content. The decoded + durable-envelope scanner covers current and encoded secret + shapes across all fields. Don't redact-and-store verbatim content; drop + the complete row so callers never mistake a marker for a raw transcript. """ - joined = " | ".join(filter(None, (role, source_agent, content))) - _, findings = _redact_text(joined) - return bool(findings) + return bool( + scan_persisted_value( + { + "session_id": session_id, + "role": role, + "content": content, + "scope": scope, + "timestamp": timestamp, + "source_agent": source_agent, + "created_at": created_at, + } + ) + ) + + +def _row_value(row: sqlite3.Row | dict, field: str, default: object = "") -> object: + """Read an optional row field across current and legacy schemas.""" + try: + return row[field] if field in row.keys() else default + except (AttributeError, KeyError, TypeError, IndexError): + return default + + +def _verbatim_row_has_sensitive_field(row: sqlite3.Row | dict) -> bool: + """Apply the durable envelope scanner to a current or legacy DB row.""" + return _row_has_sensitive_field( + str(_row_value(row, "role") or ""), + str(_row_value(row, "source_agent") or ""), + str(_row_value(row, "content") or ""), + session_id=str(_row_value(row, "session_id") or ""), + scope=str(_row_value(row, "scope") or ""), + timestamp=str(_row_value(row, "timestamp") or ""), + created_at=str(_row_value(row, "created_at") or ""), + ) # Vector search is opt-in: an unset QDRANT_URL means "vector disabled", exactly # like a missing OPENAI_API_KEY. NEVER hardcode a routable private LAN IP here — @@ -115,6 +155,19 @@ def store_verbatim( timestamp: str | None = None, ) -> int | None: """Store a verbatim conversation turn. Returns row ID or None if filtered.""" + now = timestamp or datetime.now(timezone.utc).isoformat() + if not content or len(content) < 20: + return None + if _row_has_sensitive_field( + role or "", + source_agent or "", + content, + session_id=session_id or "", + scope=scope or "", + timestamp=now, + ): + return None + # closing() guarantees the connection (and its WAL write lock) is released # even if an INSERT/commit raises (e.g. "database is locked" under # concurrent MCP/Stop-hook writers) - this is the hottest write path. @@ -126,7 +179,7 @@ def store_verbatim( content, scope, source_agent, - timestamp, + now, ) conn.commit() return row_id @@ -144,10 +197,17 @@ def _store_verbatim_conn( """Store one turn using an existing connection without committing.""" if not content or len(content) < 20: return None - if _row_has_sensitive_field(role or "", source_agent or "", content): - return None now = timestamp or datetime.now(timezone.utc).isoformat() + if _row_has_sensitive_field( + role or "", + source_agent or "", + content, + session_id=session_id or "", + scope=scope or "", + timestamp=now, + ): + return None # Dedup by exact content within the same session. The composite # idx_verbatim_session_content(session_id, content) (migration 0006) makes @@ -376,6 +436,7 @@ def _search_fts(db_path: str, query: str, scope: str | None, limit: int) -> list clean_query = " ".join(w for w in query.split() if len(w) > 2) if not clean_query: return [] + fetch_limit = max(limit * 5, limit + 20) # closing() guarantees the connection is released even if the JOIN raises a # non-OperationalError (corrupt/locked DB, programming error) — the bare @@ -384,35 +445,47 @@ def _search_fts(db_path: str, query: str, scope: str | None, limit: int) -> list try: if scope: rows = conn.execute( - """SELECT v.id, v.session_id, v.role, v.content, v.scope, v.timestamp, - rank as score + """SELECT v.*, rank as score FROM verbatim_fts f JOIN verbatim_memories v ON v.id = f.rowid - WHERE verbatim_fts MATCH ? AND v.scope LIKE ? + WHERE verbatim_fts MATCH ? + AND (v.scope = ? OR (? = 'project' AND v.scope LIKE 'project:%')) ORDER BY rank LIMIT ?""", - (clean_query, f"{scope}%", limit), + (clean_query, scope, scope, fetch_limit), ).fetchall() else: rows = conn.execute( - """SELECT v.id, v.session_id, v.role, v.content, v.scope, v.timestamp, - rank as score + """SELECT v.*, rank as score FROM verbatim_fts f JOIN verbatim_memories v ON v.id = f.rowid WHERE verbatim_fts MATCH ? ORDER BY rank LIMIT ?""", - (clean_query, limit), + (clean_query, fetch_limit), ).fetchall() except sqlite3.OperationalError: rows = [] - return [ - {"id": r["id"], "session_id": r["session_id"], "role": r["role"], - "content": r["content"], "scope": r["scope"], "timestamp": r["timestamp"], - "score": abs(r["score"]) if r["score"] else 0, "source": "fts"} - for r in rows - ] + results: list[dict] = [] + for row in rows: + if _verbatim_row_has_sensitive_field(row): + continue + results.append( + { + "id": row["id"], + "session_id": row["session_id"], + "role": row["role"], + "content": row["content"], + "scope": row["scope"], + "timestamp": row["timestamp"], + "score": abs(row["score"]) if row["score"] else 0, + "source": "fts", + } + ) + if len(results) >= limit: + break + return results def _search_vector(query: str, scope: str | None, limit: int) -> list[dict]: @@ -435,13 +508,31 @@ def sync_to_qdrant(db_path: str, batch_size: int = 50) -> dict[str, int]: # including the initial SELECT raising or the final UPDATE/commit raising. with closing(_connect(db_path)) as conn: rows = conn.execute( - "SELECT id, content, scope, session_id, role FROM verbatim_memories WHERE embedding_synced = 0 LIMIT ?", + "SELECT * FROM verbatim_memories WHERE embedding_synced = 0 LIMIT ?", (batch_size,), ).fetchall() if not rows: return {"synced": 0} + safe_rows: list[sqlite3.Row] = [] + unsafe_ids: list[int] = [] + for row in rows: + if _verbatim_row_has_sensitive_field(row): + unsafe_ids.append(int(row["id"])) + else: + safe_rows.append(row) + excluded_sensitive = len(unsafe_ids) + if unsafe_ids: + placeholders = ",".join("?" for _ in unsafe_ids) + conn.execute( + f"UPDATE verbatim_memories SET embedding_synced = -1 WHERE id IN ({placeholders})", + unsafe_ids, + ) + conn.commit() + if not safe_rows: + return {"synced": 0, "excluded_sensitive": excluded_sensitive} + # Ensure collection exists try: req = urllib.request.Request(f"{QDRANT_URL}/collections/{QDRANT_COLLECTION}") @@ -460,7 +551,7 @@ def sync_to_qdrant(db_path: str, batch_size: int = 50) -> dict[str, int]: return {"synced": 0, "error": str(e)} # Embed in batches - texts = [r["content"][:2000] for r in rows] + texts = [r["content"][:2000] for r in safe_rows] try: embed_url = "https://api.openai.com/v1/embeddings" payload = {"model": "text-embedding-3-small", "input": texts} @@ -475,10 +566,12 @@ def sync_to_qdrant(db_path: str, batch_size: int = 50) -> dict[str, int]: embeddings = [d["embedding"] for d in result["data"]] except Exception as e: return {"synced": 0, "error": str(e)} + if len(embeddings) != len(safe_rows): + return {"synced": 0, "error": "embedding response cardinality mismatch"} # Upsert to Qdrant points = [] - for i, (row, emb) in enumerate(zip(rows, embeddings)): + for row, emb in zip(safe_rows, embeddings): points.append({ "id": row["id"], "vector": emb, @@ -503,9 +596,12 @@ def sync_to_qdrant(db_path: str, batch_size: int = 50) -> dict[str, int]: return {"synced": 0, "error": str(e)} # Mark as synced - ids = [r["id"] for r in rows] + ids = [r["id"] for r in safe_rows] placeholders = ",".join("?" for _ in ids) conn.execute(f"UPDATE verbatim_memories SET embedding_synced = 1 WHERE id IN ({placeholders})", ids) conn.commit() - return {"synced": len(rows)} + result = {"synced": len(safe_rows)} + if excluded_sensitive: + result["excluded_sensitive"] = excluded_sensitive + return result diff --git a/memorymaster/stores/_storage_write_claims.py b/memorymaster/stores/_storage_write_claims.py index 251a219b..d885d7c9 100644 --- a/memorymaster/stores/_storage_write_claims.py +++ b/memorymaster/stores/_storage_write_claims.py @@ -17,6 +17,7 @@ validate_event_payload, ) from memorymaster.core.security import ( + normalize_sensitivity_findings, sanitize_claim_input, sanitize_claim_structure_input, sanitize_persisted_text, @@ -97,6 +98,7 @@ def create_claim( source_agent: str | None = None, visibility: str = "public", holder: str | None = None, + _pre_sanitization_findings: list[str] | None = None, ) -> Claim: if not citations: raise ValueError("At least one citation is required.") @@ -129,6 +131,9 @@ def create_claim( valid_until=valid_until, tenant_id=tenant_id, ) + redaction_findings = normalize_sensitivity_findings( + [*sanitized.findings, *(_pre_sanitization_findings or [])] + ) text = sanitized.text object_value = sanitized.object_value citations = sanitized.citations @@ -259,10 +264,10 @@ def create_claim( payload_json=json.dumps(ingest_payload), created_at=now, ) - if sanitized.is_sensitive: + if redaction_findings: policy_payload = validate_event_payload( "policy_decision", - {"findings": sanitized.findings}, + {"findings": redaction_findings}, details="sensitive_redaction_applied", ) self._insert_event_row( diff --git a/memorymaster/stores/postgres_store.py b/memorymaster/stores/postgres_store.py index 76c7f051..0e2e182a 100644 --- a/memorymaster/stores/postgres_store.py +++ b/memorymaster/stores/postgres_store.py @@ -28,6 +28,7 @@ ) from memorymaster.core.retry import connect_with_retry from memorymaster.core.security import ( + normalize_sensitivity_findings, sanitize_claim_input, sanitize_claim_structure_input, sanitize_event_input, @@ -1680,6 +1681,7 @@ def create_claim( source_agent: str | None = None, visibility: str = "public", holder: str | None = None, + _pre_sanitization_findings: list[str] | None = None, ) -> Claim: if not citations: raise ValueError("At least one citation is required.") @@ -1712,6 +1714,9 @@ def create_claim( valid_until=valid_until, tenant_id=tenant_id, ) + redaction_findings = normalize_sensitivity_findings( + [*sanitized.findings, *(_pre_sanitization_findings or [])] + ) self._validate_bound_persistence_identity() text = sanitized.text object_value = sanitized.object_value @@ -1835,10 +1840,10 @@ def create_claim( payload=ingest_payload, created_at=now, ) - if sanitized.is_sensitive: + if redaction_findings: policy_payload = validate_event_payload( "policy_decision", - {"findings": sanitized.findings}, + {"findings": redaction_findings}, details="sensitive_redaction_applied", ) self._insert_event_row( diff --git a/tests/test_auxiliary_persistence_sensitivity.py b/tests/test_auxiliary_persistence_sensitivity.py new file mode 100644 index 00000000..c84b51f0 --- /dev/null +++ b/tests/test_auxiliary_persistence_sensitivity.py @@ -0,0 +1,783 @@ +"""Adversarial tests for auxiliary durable sensitivity boundaries.""" +from __future__ import annotations + +import base64 +import copy +import hashlib +import json +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path + +import pytest + +from memorymaster.core import spool +from memorymaster.core.models import CitationInput +from memorymaster.core.security import ( + SensitiveMetadataError, + sanitize_persisted_text, + scan_text_for_findings, +) +from memorymaster.core.service import MemoryService +from memorymaster.govern.feedback import FeedbackTracker +from memorymaster.govern.jobs import spool_drain +from memorymaster.knowledge import rule_miner +from memorymaster.knowledge.daily_notes import ( + export_daily_note_md, + find_ghost_notes, + generate_daily_note, +) +from memorymaster.recall import verbatim_store +from memorymaster.recall.verbatim_recall import recall_verbatim +from memorymaster.stores.postgres_store import PostgresStore + + +def _literal_secret() -> str: + body = "".join(format((index * 7 + 3) % 16, "x") for index in range(40)) + token = "".join(("gh", "p_", body)) + assert "github_token" in scan_text_for_findings(token) + return token + + +def _encoded_secret() -> str: + token = base64.b64encode(_literal_secret().encode()).decode() + assert "github_token" in scan_text_for_findings(token) + return token + + +def _assert_tree_absent(root: Path, *needles: str) -> None: + for path in root.rglob("*"): + if path.is_file(): + raw = path.read_bytes() + for needle in needles: + assert needle.encode() not in raw, f"{needle!r} leaked into {path.name}" + + +def _verbatim_counts(db_path: Path) -> tuple[int, int]: + with sqlite3.connect(db_path) as conn: + primary = int(conn.execute("SELECT COUNT(*) FROM verbatim_memories").fetchone()[0]) + indexed = int(conn.execute("SELECT COUNT(*) FROM verbatim_fts").fetchone()[0]) + return primary, indexed + + +@pytest.mark.parametrize("secret_factory", [_literal_secret, _encoded_secret]) +def test_spool_sanitizes_ingest_content_before_first_file_write( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + secret_factory, +) -> None: + monkeypatch.setenv(spool.ENV_SPOOL_DIR, str(tmp_path / "spool")) + secret = secret_factory() + payload = { + "text": f"Sensitive spooled content {secret}", + "subject": f"subject {secret}", + "scope": "project:aux-test", + "source_agent": "aux-test", + "citations": [{"source": "unit-test", "excerpt": f"excerpt {secret}"}], + } + before = copy.deepcopy(payload) + + path = spool.append(tmp_path / "memory.db", "ingest", payload) + + assert payload == before + _assert_tree_absent(path.parent, secret, _literal_secret()) + envelope = json.loads(path.read_text(encoding="utf-8")) + assert "[REDACTED:" in json.dumps(envelope["payload"]) + + +@pytest.mark.parametrize( + ("op", "payload", "idempotency_key"), + [ + ("ingest", {"text": "safe", "citations": []}, _encoded_secret()), + ( + "verbatim", + { + "session_id": "safe-session", + "role": "user", + "content": f"Long enough sensitive turn {_encoded_secret()}", + "scope": "project:aux-test", + "source_agent": "aux-test", + }, + None, + ), + ], +) +def test_spool_rejects_sensitive_metadata_and_verbatim_before_file_creation( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + op: str, + payload: dict[str, object], + idempotency_key: str | None, +) -> None: + root = tmp_path / "spool" + monkeypatch.setenv(spool.ENV_SPOOL_DIR, str(root)) + + with pytest.raises(SensitiveMetadataError) as rejected: + spool.append( + tmp_path / "memory.db", + op, + payload, + idempotency_key=idempotency_key, + ) + + assert _encoded_secret() not in str(rejected.value) + assert not root.exists() + + +def test_quarantine_never_duplicates_sensitive_raw_line_or_reason( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = tmp_path / "spool" + monkeypatch.setenv(spool.ENV_SPOOL_DIR, str(root)) + literal = _literal_secret() + encoded = _encoded_secret() + + target = spool.quarantine_line( + tmp_path / "memory.db", + json.dumps({"payload": {"token": encoded}}), + f"replay failed with {literal}", + ) + + _assert_tree_absent(root, literal, encoded) + record = json.loads(target.read_text(encoding="utf-8")) + assert "[REDACTED:" in json.dumps(record) + + +def test_drain_validates_boundary_sidecar_before_claim_side_effect( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(spool.ENV_SPOOL_DIR, str(tmp_path / "spool")) + db_path = tmp_path / "invalid-sidecar.db" + service = MemoryService(db_path) + service.init_db() + spool_dir = spool.spool_dir_for(db_path) + spool_dir.mkdir(parents=True) + envelope = spool.make_envelope( + "ingest", + { + "text": "A safe claim that must not survive invalid audit metadata.", + "citations": [], + "scope": "project:aux-test", + "source_agent": "aux-test", + "_sanitization": {"findings": ["invalid finding label!"]}, + }, + idempotency_key="invalid-sidecar-order", + ) + (spool_dir / "999-20260711.jsonl").write_text( + json.dumps(envelope) + "\n", + encoding="utf-8", + ) + + result = spool_drain.run(service) + + assert result["drained"] == 0 + assert result["quarantined"] == 1 + with sqlite3.connect(db_path) as conn: + assert conn.execute("SELECT COUNT(*) FROM claims").fetchone()[0] == 0 + + +def test_spooled_redaction_event_failure_rolls_back_claim_atomically( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(spool.ENV_SPOOL_DIR, str(tmp_path / "spool")) + db_path = tmp_path / "atomic-redaction-event.db" + service = MemoryService(db_path) + service.init_db() + original_insert = service.store._insert_event_row + + def _fail_policy_event(conn, **kwargs): + if ( + kwargs.get("event_type") == "policy_decision" + and kwargs.get("details") == "sensitive_redaction_applied" + ): + raise RuntimeError("synthetic policy event failure") + return original_insert(conn, **kwargs) + + monkeypatch.setattr(service.store, "_insert_event_row", _fail_policy_event) + spool.append( + db_path, + "ingest", + { + "text": f"Sensitive boundary claim {_encoded_secret()}", + "citations": [], + "scope": "project:aux-test", + "source_agent": "aux-test", + }, + idempotency_key="atomic-redaction-event", + ) + + result = spool_drain.run(service) + + assert result["drained"] == 0 + assert result["quarantined"] == 1 + with sqlite3.connect(db_path) as conn: + assert conn.execute("SELECT COUNT(*) FROM claims").fetchone()[0] == 0 + assert conn.execute( + """SELECT COUNT(*) FROM events + WHERE details = 'sensitive_redaction_applied'""" + ).fetchone()[0] == 0 + + +def test_postgres_redaction_event_failure_rolls_back_claim_atomically( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _Cursor: + def __init__(self) -> None: + self.last_sql = "" + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def execute(self, sql, params=None) -> None: + self.last_sql = " ".join(str(sql).split()).lower() + + def fetchone(self): + return {"id": 1} if "returning id" in self.last_sql else None + + class _Connection: + def __init__(self) -> None: + self.cursor_instance = _Cursor() + self.committed = False + self.rolled_back = False + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + self.rolled_back = exc_type is not None + self.committed = exc_type is None + return False + + def cursor(self): + return self.cursor_instance + + connection = _Connection() + store = PostgresStore("postgresql://unused.invalid/memorymaster") + monkeypatch.setattr(store, "connect", lambda: connection) + monkeypatch.setattr(store, "_assign_human_id", lambda *args, **kwargs: None) + + def fail_policy_event(conn, **kwargs): + if kwargs.get("details") == "sensitive_redaction_applied": + raise RuntimeError("synthetic policy event failure") + return 1 + + monkeypatch.setattr(store, "_insert_event_row", fail_policy_event) + + with pytest.raises(RuntimeError, match="synthetic policy event failure"): + store.create_claim( + "A harmless pre-sanitized claim body.", + [CitationInput(source="test")], + _pre_sanitization_findings=["github_token"], + ) + + assert connection.rolled_back is True + assert connection.committed is False + + +@pytest.mark.parametrize( + ("field", "secret"), + [ + ("session_id", _literal_secret()), + ("role", _encoded_secret()), + ("content", _encoded_secret()), + ("scope", _encoded_secret()), + ("source_agent", _literal_secret()), + ("timestamp", _literal_secret()), + ], +) +def test_direct_verbatim_rejects_sensitive_fields_without_primary_or_fts_row( + tmp_path: Path, + field: str, + secret: str, +) -> None: + db_path = tmp_path / f"verbatim-{field}.db" + verbatim_store.ensure_verbatim_schema(str(db_path)) + values = { + "session_id": "safe-session", + "role": "user", + "content": "A sufficiently long safe verbatim conversation turn.", + "scope": "project:aux-test", + "source_agent": "aux-test", + "timestamp": "2026-07-11T00:00:00+00:00", + } + values[field] = secret + + row_id = verbatim_store.store_verbatim(str(db_path), **values) + + assert row_id is None + assert _verbatim_counts(db_path) == (0, 0) + + +def test_verbatim_compatibility_detector_catches_encoded_secret() -> None: + assert verbatim_store._contains_sensitive(_encoded_secret()) + + +def _seed_verbatim_pair(db_path: Path) -> tuple[int, int]: + verbatim_store.ensure_verbatim_schema(str(db_path)) + unsafe = f"Auxiliary sentinel legacy row {_encoded_secret()}" + safe = "Auxiliary sentinel safe neighboring verbatim row" + with sqlite3.connect(db_path) as conn: + unsafe_id = int( + conn.execute( + """INSERT INTO verbatim_memories + (session_id, role, content, scope, timestamp, source_agent) + VALUES (?, ?, ?, ?, ?, ?)""", + ("legacy", "user", unsafe, "project:aux-test", "2026-07-11T00:00:00Z", "test"), + ).lastrowid + ) + safe_id = int( + conn.execute( + """INSERT INTO verbatim_memories + (session_id, role, content, scope, timestamp, source_agent) + VALUES (?, ?, ?, ?, ?, ?)""", + ("safe", "user", safe, "project:aux-test", "2026-07-11T00:00:01Z", "test"), + ).lastrowid + ) + conn.execute("INSERT INTO verbatim_fts(rowid, content) VALUES (?, ?)", (unsafe_id, unsafe)) + conn.execute("INSERT INTO verbatim_fts(rowid, content) VALUES (?, ?)", (safe_id, safe)) + conn.commit() + return unsafe_id, safe_id + + +def test_legacy_sensitive_verbatim_is_hidden_from_both_fts_surfaces( + tmp_path: Path, +) -> None: + db_path = tmp_path / "legacy-verbatim.db" + unsafe_id, safe_id = _seed_verbatim_pair(db_path) + + direct = verbatim_store.search_verbatim(str(db_path), "auxiliary sentinel", limit=10) + recalled = recall_verbatim("auxiliary sentinel", "project:aux-test", str(db_path), limit=10) + + assert {row["id"] for row in direct} == {safe_id} + assert {hit.verbatim_id for hit in recalled} == {safe_id} + assert unsafe_id not in {row["id"] for row in direct} + + +def test_verbatim_scope_filter_does_not_match_textual_sibling_scope( + tmp_path: Path, +) -> None: + db_path = tmp_path / "scope-boundary-verbatim.db" + verbatim_store.ensure_verbatim_schema(str(db_path)) + rows = ( + ("project:test", "Authorized sibling-boundary verbatim sentinel row."), + ("project:test-foreign", "Foreign sibling-boundary verbatim sentinel row."), + ) + with sqlite3.connect(db_path) as conn: + for scope, content in rows: + row_id = int( + conn.execute( + """INSERT INTO verbatim_memories + (session_id, role, content, scope, timestamp, source_agent) + VALUES (?, ?, ?, ?, ?, ?)""", + (scope, "user", content, scope, "2026-07-11T00:00:00Z", "test"), + ).lastrowid + ) + conn.execute( + "INSERT INTO verbatim_fts(rowid, content) VALUES (?, ?)", + (row_id, content), + ) + conn.commit() + + direct = verbatim_store.search_verbatim( + str(db_path), + "sibling boundary verbatim sentinel", + scope="project:test", + limit=10, + ) + recalled = recall_verbatim( + "sibling boundary verbatim sentinel", + "project:test", + str(db_path), + limit=10, + ) + + assert {row["scope"] for row in direct} == {"project:test"} + assert {hit.scope for hit in recalled} == {"project:test"} + + +class _Response: + def __init__(self, payload: dict[str, object]) -> None: + self.payload = payload + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def read(self) -> bytes: + return json.dumps(self.payload).encode() + + +def test_verbatim_qdrant_sync_filters_legacy_secret_before_embedding_and_upsert( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "qdrant-verbatim.db" + unsafe_id, safe_id = _seed_verbatim_pair(db_path) + requests: list[tuple[str, bytes | None]] = [] + + def fake_urlopen(req, timeout): + requests.append((req.full_url, req.data)) + if req.full_url.endswith(f"/collections/{verbatim_store.QDRANT_COLLECTION}"): + return _Response({}) + if req.full_url == "https://api.openai.com/v1/embeddings": + payload = json.loads(req.data.decode()) + return _Response({"data": [{"embedding": [0.0] * verbatim_store.EMBED_DIM} for _ in payload["input"]]}) + return _Response({}) + + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + monkeypatch.setattr(verbatim_store, "QDRANT_URL", "http://qdrant.invalid") + monkeypatch.setattr(verbatim_store.urllib.request, "urlopen", fake_urlopen) + + result = verbatim_store.sync_to_qdrant(str(db_path)) + + serialized = b"\n".join(data or b"" for _, data in requests) + assert _encoded_secret().encode() not in serialized + point_payloads = [ + json.loads(data.decode()) + for url, data in requests + if url.endswith("/points") and data is not None + ] + point_ids = { + int(point["id"]) + for payload in point_payloads + for point in payload["points"] + } + assert point_ids == {safe_id} + assert unsafe_id not in point_ids + assert result == {"synced": 1, "excluded_sensitive": 1} + with sqlite3.connect(db_path) as conn: + states = dict(conn.execute("SELECT id, embedding_synced FROM verbatim_memories")) + assert states == {unsafe_id: -1, safe_id: 1} + + +def test_verbatim_qdrant_sync_rejects_embedding_cardinality_mismatch( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "qdrant-cardinality.db" + unsafe_id, safe_id = _seed_verbatim_pair(db_path) + second_safe_id = verbatim_store.store_verbatim( + str(db_path), + session_id="second-safe", + role="user", + content="A second harmless row must not be marked synced without a vector.", + scope="project:test", + source_agent="test", + ) + requests: list[str] = [] + + def fake_urlopen(req, timeout): + requests.append(req.full_url) + if req.full_url.endswith(f"/collections/{verbatim_store.QDRANT_COLLECTION}"): + return _Response({}) + if req.full_url == "https://api.openai.com/v1/embeddings": + return _Response({"data": [{"embedding": [0.0] * verbatim_store.EMBED_DIM}]}) + pytest.fail("cardinality mismatch reached Qdrant upsert") + + monkeypatch.setenv("OPENAI_API_KEY", "test-key") + monkeypatch.setattr(verbatim_store, "QDRANT_URL", "http://qdrant.invalid") + monkeypatch.setattr(verbatim_store.urllib.request, "urlopen", fake_urlopen) + + result = verbatim_store.sync_to_qdrant(str(db_path)) + + assert result == {"synced": 0, "error": "embedding response cardinality mismatch"} + assert not any(url.endswith("/points") for url in requests) + with sqlite3.connect(db_path) as conn: + states = dict(conn.execute("SELECT id, embedding_synced FROM verbatim_memories")) + assert states == {unsafe_id: -1, safe_id: 0, second_safe_id: 0} + + +def test_feedback_scans_full_query_before_truncation_and_direct_write( + tmp_path: Path, +) -> None: + db_path = tmp_path / "feedback.db" + service = MemoryService(db_path) + service.init_db() + query = ("benign-prefix " * 45) + _encoded_secret() + tracker = FeedbackTracker(str(db_path)) + + assert tracker.record_retrieval([1], query) == 1 + + with sqlite3.connect(db_path) as conn: + persisted = str(conn.execute("SELECT query_text FROM usage_feedback").fetchone()[0]) + assert _encoded_secret() not in persisted + assert _literal_secret() not in persisted + assert "[REDACTED:" in persisted + + +def test_feedback_spool_and_replay_never_persist_raw_query( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(spool.ENV_SPOOL_DIR, str(tmp_path / "spool")) + monkeypatch.delenv("QDRANT_URL", raising=False) + db_path = tmp_path / "feedback-spool.db" + service = MemoryService(db_path) + service.init_db() + claim = service.ingest("Safe feedback target claim.", []) + query = f"How does auxiliary recall work {_encoded_secret()}" + + path = spool.append( + db_path, + "feedback", + {"claim_ids": [claim.id], "query_text": query}, + ) + _assert_tree_absent(path.parent, _encoded_secret(), _literal_secret()) + result = spool_drain.run(service) + + assert result["drained"] == 1 + with sqlite3.connect(db_path) as conn: + persisted = str(conn.execute("SELECT query_text FROM usage_feedback").fetchone()[0]) + assert _encoded_secret() not in persisted + assert "[REDACTED:" in persisted + + +def test_ro_access_spool_hashes_sanitized_query_not_sensitive_input( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(spool.ENV_SPOOL_DIR, str(tmp_path / "spool")) + db_path = tmp_path / "access-spool.db" + service = MemoryService(db_path) + service.init_db() + query = f"How does auxiliary recall work {_encoded_secret()}" + safe_query, _ = sanitize_persisted_text(query) + + service._spool_accesses([1], query) + + envelopes = [ + json.loads(line) + for path in spool.spool_dir_for(db_path).glob("*.jsonl") + for line in path.read_text(encoding="utf-8").splitlines() + ] + by_op = {envelope["op"]: envelope for envelope in envelopes} + expected_hash = hashlib.sha256(safe_query.encode("utf-8")).hexdigest()[:12] + assert by_op["access"]["payload"]["query_hash"] == expected_hash + assert by_op["feedback"]["payload"]["query_text"] == safe_query + + +def test_spooled_ingest_preserves_holder_and_records_boundary_findings_once( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv(spool.ENV_SPOOL_DIR, str(tmp_path / "spool")) + db_path = tmp_path / "holder-spool.db" + service = MemoryService(db_path) + service.init_db() + secret = _encoded_secret() + + spool.append( + db_path, + "ingest", + { + "text": f"Holder-attributed safe claim with {secret}", + "citations": [], + "holder": "codex", + "scope": "project:aux-test", + "source_agent": "aux-test", + }, + idempotency_key="aux-holder-boundary", + ) + first = spool_drain.run(service) + + claims = service.store.list_claims(holder="codex", include_archived=True) + assert first["drained"] == 1 + assert len(claims) == 1 + with service.store.connect() as conn: + events = conn.execute( + """SELECT COUNT(*) FROM events + WHERE claim_id = ? AND event_type = 'policy_decision' + AND details = 'sensitive_redaction_applied'""", + (claims[0].id,), + ).fetchone()[0] + assert events == 1 + + +def test_daily_and_ghost_notes_hide_legacy_sensitive_feedback( + tmp_path: Path, +) -> None: + db_path = tmp_path / "legacy-feedback.db" + service = MemoryService(db_path) + service.init_db() + tracker = FeedbackTracker(str(db_path)) + tracker.ensure_tables() + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + raw_query = f"legacy topic {_encoded_secret()}" + with sqlite3.connect(db_path) as conn: + for index in range(3): + conn.execute( + """INSERT INTO usage_feedback + (id, claim_id, query_text, timestamp, was_returned) + VALUES (?, ?, ?, ?, 1)""", + (f"legacy-{index}", 1, raw_query, f"{today}T00:00:0{index}Z"), + ) + conn.commit() + + note = generate_daily_note(str(db_path), today) + ghosts = find_ghost_notes(str(db_path), min_references=1) + rendered = json.dumps({"note": note, "ghosts": ghosts}) + assert _encoded_secret() not in rendered + assert _literal_secret() not in rendered + + +def test_daily_note_sanitizes_legacy_claim_type_before_rendering( + tmp_path: Path, +) -> None: + db_path = tmp_path / "legacy-claim-type.db" + service = MemoryService(db_path) + service.init_db() + claim = service.ingest("Safe claim body for a legacy metadata row.", []) + today = datetime.now(timezone.utc).strftime("%Y-%m-%d") + secret = _encoded_secret() + with sqlite3.connect(db_path) as conn: + conn.execute( + "UPDATE claims SET claim_type = ?, created_at = ? WHERE id = ?", + (secret, f"{today}T00:00:00Z", claim.id), + ) + conn.commit() + + note = generate_daily_note(str(db_path), today) + + assert secret not in note["note"] + assert _literal_secret() not in note["note"] + assert "[REDACTED:" in note["note"] + + +def test_daily_note_export_rejects_path_shaped_date_before_write( + tmp_path: Path, +) -> None: + db_path = tmp_path / "daily-path.db" + service = MemoryService(db_path) + service.init_db() + output = tmp_path / "notes" + + with pytest.raises(ValueError, match="date"): + export_daily_note_md(str(db_path), str(output), "../escaped") + + assert not (tmp_path / "escaped.md").exists() + + +@pytest.mark.parametrize("secret_turn", ["assistant", "user"]) +def test_rule_miner_skips_sensitive_legacy_window_before_llm( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + secret_turn: str, +) -> None: + db_path = tmp_path / f"rule-miner-{secret_turn}.db" + service = MemoryService(db_path, workspace_root=tmp_path) + service.init_db() + verbatim_store.ensure_verbatim_schema(str(db_path)) + secret = _encoded_secret() + assistant = "I hardcoded the path directly into the application configuration." + user = "No, do not hardcode that path; use an environment variable instead." + if secret_turn == "assistant": + assistant = f"{assistant} {secret}" + else: + user = f"{user} {secret}" + with sqlite3.connect(db_path) as conn: + for role, content in (("assistant", assistant), ("user", user)): + row_id = int( + conn.execute( + """INSERT INTO verbatim_memories + (session_id, role, content, scope, timestamp, source_agent) + VALUES (?, ?, ?, ?, ?, ?)""", + ( + "legacy-rule-session", + role, + content, + "project:aux-test", + "2026-07-11T00:00:00Z", + "legacy-test", + ), + ).lastrowid + ) + conn.execute( + "INSERT INTO verbatim_fts(rowid, content) VALUES (?, ?)", + (row_id, content), + ) + conn.commit() + + calls: list[str] = [] + + def _must_not_extract(window: str): + calls.append(window) + return None + + monkeypatch.setattr(rule_miner, "_extract_rule", _must_not_extract) + stats = rule_miner.mine_rules(str(db_path), service, provider="claude_cli") + + assert calls == [] + assert stats["candidates"] == 1 + assert stats["skipped"] == 1 + assert stats["llm_calls"] == 0 + assert stats["last_id"] == 2 + + +def test_stop_rule_miner_skips_sensitive_transcript_before_llm( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "rule-stop.db" + service = MemoryService(db_path, workspace_root=tmp_path) + service.init_db() + transcript = tmp_path / "session.jsonl" + turns = ( + ( + "assistant", + f"I hardcoded the path directly into the application. {_encoded_secret()}", + ), + ( + "user", + "No, do not hardcode that path; use an environment variable instead.", + ), + ) + transcript.write_text( + "\n".join( + json.dumps( + { + "type": role, + "message": { + "role": role, + "content": [{"type": "text", "text": content}], + }, + } + ) + for role, content in turns + ) + + "\n", + encoding="utf-8", + ) + calls: list[str] = [] + + def _must_not_extract(window: str): + calls.append(window) + return None + + monkeypatch.setattr(rule_miner, "_extract_rule", _must_not_extract) + stats = rule_miner.mine_transcript_rules( + str(transcript), + service, + scope="project:aux-test", + provider="claude_cli", + ) + + assert calls == [] + assert stats == {"windows": 1, "llm_calls": 0, "ingested": 0, "skipped": 1} + + +def test_rule_miner_rejects_encoded_sensitive_model_output() -> None: + assert rule_miner._is_sensitive_rule( + { + "trigger": "auth configuration", + "action": f"use this encoded value {_encoded_secret()}", + "rationale": "continuous integration", + } + ) From d2d1a1133398aeb9099008bf126385c13aadbcbc Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 16:30:43 -0300 Subject: [PATCH 18/55] fix(security): gate external persistence writers --- memorymaster/bridges/dream_bridge.py | 26 ++- memorymaster/bridges/qmd_bridge.py | 37 ++- memorymaster/govern/jobs/compactor.py | 24 +- memorymaster/govern/steward.py | 9 +- memorymaster/recall/qdrant_backend.py | 24 ++ .../test_external_persistence_sensitivity.py | 213 ++++++++++++++++++ 6 files changed, 319 insertions(+), 14 deletions(-) create mode 100644 tests/test_external_persistence_sensitivity.py diff --git a/memorymaster/bridges/dream_bridge.py b/memorymaster/bridges/dream_bridge.py index fb032036..1652fad4 100644 --- a/memorymaster/bridges/dream_bridge.py +++ b/memorymaster/bridges/dream_bridge.py @@ -8,6 +8,7 @@ """ from __future__ import annotations +from collections import defaultdict import logging import os import re @@ -17,7 +18,7 @@ from memorymaster.core import observability, spool from memorymaster.stores._storage_shared import open_conn from memorymaster.core.security import redact_text as _redact_text -from memorymaster.core.security import sanitize_claim_input +from memorymaster.core.security import sanitize_claim_input, scan_persisted_value log = logging.getLogger(__name__) @@ -471,7 +472,26 @@ def _query_exportable_claims( ) rows = conn.execute(sql_fallback, [*allowed_tiers, *scope_params, fetch_limit]).fetchall() - return [dict(row) for row in rows] + claims = [dict(row) for row in rows] + citations_by_claim_id = _load_citations_by_claim_id(conn, claims) + for claim in claims: + claim["citations"] = citations_by_claim_id[claim["id"]] + return claims + + +def _load_citations_by_claim_id(conn: sqlite3.Connection, claims: list[dict]) -> dict[int, list[dict]]: + """Load citations in SQLite-safe batches rather than one query per claim.""" + citations_by_claim_id: dict[int, list[dict]] = defaultdict(list) + claim_ids = [claim["id"] for claim in claims] + for start in range(0, len(claim_ids), 900): + batch = claim_ids[start : start + 900] + placeholders = ",".join("?" for _ in batch) + sql = f"SELECT claim_id, source, locator, excerpt FROM citations WHERE claim_id IN ({placeholders})" + for row in conn.execute(sql, batch).fetchall(): + citation = dict(row) + claim_id = citation.pop("claim_id") + citations_by_claim_id[claim_id].append(citation) + return citations_by_claim_id # --------------------------------------------------------------------------- @@ -528,7 +548,7 @@ def dream_seed( break text = claim.get("text") or "" - if _is_sensitive(text): + if _is_sensitive(text) or scan_persisted_value(claim): skipped += 1 continue diff --git a/memorymaster/bridges/qmd_bridge.py b/memorymaster/bridges/qmd_bridge.py index 502d0c31..83843a56 100644 --- a/memorymaster/bridges/qmd_bridge.py +++ b/memorymaster/bridges/qmd_bridge.py @@ -24,6 +24,7 @@ from typing import Any from memorymaster.core.models import CitationInput +from memorymaster.core.security import scan_persisted_value logger = logging.getLogger(__name__) @@ -75,18 +76,38 @@ def qmd_to_claims(qmd_entries: list[dict[str, Any]], source: str = "qmd-import") return results +def _claim_representation(claim: Any) -> dict[str, Any]: + return { + "text": getattr(claim, "text", ""), + "subject": getattr(claim, "subject", None), + "predicate": getattr(claim, "predicate", None), + "object_value": getattr(claim, "object_value", None), + "claim_type": getattr(claim, "claim_type", None), + "scope": getattr(claim, "scope", None), + "citations": [ + {"source": item.source, "locator": item.locator, "excerpt": item.excerpt} + for item in (getattr(claim, "citations", None) or []) + ], + } + + def claims_to_qmd(claims: list) -> list[dict[str, Any]]: - """Convert memorymaster claims to QMD format for export to OpenClaw.""" + """Convert non-sensitive memorymaster claims to QMD format for export.""" results = [] for claim in claims: + findings = scan_persisted_value(_claim_representation(claim)) + if findings: + logger.warning("QMD export skipped sensitive claim %s (%s)", getattr(claim, "id", "?"), ",".join(findings)) + continue scope = getattr(claim, "scope", "project") base_scope = scope.split(":")[0] if ":" in scope else scope - - results.append({ - "type": getattr(claim, "claim_type", "fact") or "fact", - "tier": SCOPE_TO_TIER.get(base_scope, "working"), - "text": claim.text, - }) + results.append( + { + "type": getattr(claim, "claim_type", "fact") or "fact", + "tier": SCOPE_TO_TIER.get(base_scope, "working"), + "text": claim.text, + } + ) return results @@ -133,4 +154,4 @@ def export_qmd_file(service, file_path: str, status: str = "confirmed") -> dict[ for entry in qmd_entries: f.write(json.dumps(entry, ensure_ascii=False) + "\n") - return {"exported": len(qmd_entries), "file": file_path} + return {"exported": len(qmd_entries), "skipped": len(claims) - len(qmd_entries), "file": file_path} diff --git a/memorymaster/govern/jobs/compactor.py b/memorymaster/govern/jobs/compactor.py index a2b4cb74..9e2d67e4 100644 --- a/memorymaster/govern/jobs/compactor.py +++ b/memorymaster/govern/jobs/compactor.py @@ -7,6 +7,7 @@ from typing import Any from memorymaster.core import observability +from memorymaster.core.security import scan_persisted_value from memorymaster.stores._storage_shared import ConcurrentModificationError from memorymaster.core.lifecycle import can_transition, transition_claim @@ -112,11 +113,30 @@ def run( summary_graph_path = out_dir / "summary_graph.json" traceability_path = out_dir / "traceability.json" - archive_candidates = store.find_for_compaction(retain_days=retain_days) + all_candidates = store.find_for_compaction(retain_days=retain_days) + archive_candidates = [] + skipped_sensitive_ids: list[int] = [] + for claim in all_candidates: + citations = store.list_citations(claim.id) + representation = { + "text": claim.text, + "subject": claim.subject, + "predicate": claim.predicate, + "object_value": claim.object_value, + "scope": claim.scope, + "citations": [ + {"source": item.source, "locator": item.locator, "excerpt": item.excerpt} for item in citations + ], + } + if scan_persisted_value(representation): + skipped_sensitive_ids.append(claim.id) + continue + archive_candidates.append(claim) if dry_run: return { "dry_run": True, "candidate_claims": len(archive_candidates), + "skipped_sensitive": len(skipped_sensitive_ids), "archived_claims": 0, "deleted_events": 0, "planned_archives": [ @@ -250,6 +270,7 @@ def run( "retain_days": retain_days, "event_retain_days": event_retain_days, "candidate_claims": len(archive_candidates), + "skipped_sensitive": len(skipped_sensitive_ids), "archived_claims": archived, "deleted_events": deleted_events, } @@ -285,6 +306,7 @@ def run( "event_retain_days": event_retain_days, "archived_claims": archived, "deleted_events": deleted_events, + "skipped_sensitive": len(skipped_sensitive_ids), "artifacts": { "summary_graph": str(summary_graph_path), "traceability": str(traceability_path), diff --git a/memorymaster/govern/steward.py b/memorymaster/govern/steward.py index 9f2e7c21..ad781d73 100644 --- a/memorymaster/govern/steward.py +++ b/memorymaster/govern/steward.py @@ -13,7 +13,7 @@ from memorymaster.core import observability from memorymaster.core import llm_budget from memorymaster.core.lifecycle import transition_claim -from memorymaster.core.security import is_sensitive_claim +from memorymaster.core.security import is_sensitive_claim, sanitize_persisted_json from memorymaster.core.service import MemoryService import contextlib @@ -1219,8 +1219,13 @@ def _probe_stats_row(probe_type: str) -> dict[str, int]: def _write_artifact(path: Path, payload: dict[str, Any]) -> None: + sanitized, findings = sanitize_persisted_json(payload) + if not isinstance(sanitized, dict): + raise TypeError("steward artifact payload must remain a mapping") + if findings: + sanitized = {**sanitized, "artifact_sensitivity": {"action": "redacted", "findings": findings}} path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + path.write_text(json.dumps(sanitized, indent=2, sort_keys=True), encoding="utf-8") def _parse_payload_json(raw: str | None) -> dict[str, Any]: diff --git a/memorymaster/recall/qdrant_backend.py b/memorymaster/recall/qdrant_backend.py index ae1e7856..cbfb07e8 100644 --- a/memorymaster/recall/qdrant_backend.py +++ b/memorymaster/recall/qdrant_backend.py @@ -23,6 +23,7 @@ import httpx from memorymaster.core.models import Claim +from memorymaster.core.security import scan_persisted_value logger = logging.getLogger(__name__) @@ -110,6 +111,20 @@ def _embed(self, text: str) -> list[float] | None: # Claim → Qdrant payload helpers # ------------------------------------------------------------------ + @staticmethod + def _claim_representation(claim: Claim) -> dict[str, Any]: + return { + "text": claim.text, + "subject": claim.subject, + "predicate": claim.predicate, + "object_value": claim.object_value, + "claim_type": claim.claim_type, + "scope": claim.scope, + "citations": [ + {"source": item.source, "locator": item.locator, "excerpt": item.excerpt} for item in claim.citations + ], + } + @staticmethod def _claim_text(claim: Claim) -> str: """Build the text string used for embedding a claim.""" @@ -149,6 +164,10 @@ def _point_id(claim_id: int) -> str: def upsert_claim(self, claim: Claim, source: str = "memorymaster") -> bool: """Embed and upsert a single claim. Returns True on success.""" + findings = scan_persisted_value(self._claim_representation(claim)) + if findings: + logger.warning("Qdrant upsert rejected sensitive claim %d (%s)", claim.id, ",".join(findings)) + return False vec = self._embed(self._claim_text(claim)) if vec is None: return False @@ -309,6 +328,11 @@ def sync_all(self, store, *, batch_size: int = 50) -> dict[str, int]: batch: list[dict[str, Any]] = [] for _idx, claim in enumerate(claims): + findings = scan_persisted_value(self._claim_representation(claim)) + if findings: + stats["skipped"] += 1 + logger.warning("Qdrant sync skipped sensitive claim %d (%s)", claim.id, ",".join(findings)) + continue vec = self._embed(self._claim_text(claim)) if vec is None: stats["errors"] += 1 diff --git a/tests/test_external_persistence_sensitivity.py b/tests/test_external_persistence_sensitivity.py new file mode 100644 index 00000000..403f6695 --- /dev/null +++ b/tests/test_external_persistence_sensitivity.py @@ -0,0 +1,213 @@ +from __future__ import annotations + +import base64 +import json +import sqlite3 +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from memorymaster.bridges import dream_bridge, qmd_bridge +from memorymaster.core.models import Citation, Claim +from memorymaster.govern import steward +from memorymaster.govern.jobs import compactor +from memorymaster.recall.qdrant_backend import EMBEDDING_DIMS, QdrantBackend + +_LITERAL = "sk-ant-api03-NOTAREALKEY000000000000000000000000abcdefghijkl" +_ENCODED = base64.b64encode(_LITERAL.encode()).decode() + + +def _claim(claim_id: int, **changes) -> Claim: + base = Claim( + id=claim_id, + text=f"safe claim {claim_id}", + idempotency_key=None, + normalized_text=None, + claim_type="fact", + subject="safe subject", + predicate="uses", + object_value="safe object", + scope="project:test", + volatility="stable", + status="confirmed", + confidence=0.9, + pinned=False, + supersedes_claim_id=None, + replaced_by_claim_id=None, + created_at="2026-01-01T00:00:00Z", + updated_at="2026-01-01T00:00:00Z", + last_validated_at=None, + archived_at=None, + ) + return replace(base, **changes) + + +def _citation(field: str) -> Citation: + values = {"source": "safe", "locator": "safe", "excerpt": "safe"} + values[field] = _ENCODED + return Citation(id=9, claim_id=2, created_at="2026-01-01T00:00:00Z", **values) + + +@pytest.mark.parametrize("field", ["text", "subject", "predicate", "object_value"]) +def test_qmd_skips_encoded_secret_in_every_claim_tuple_field(field: str) -> None: + unsafe = _claim(2, **{field: _ENCODED}) + safe = _claim(1) + original = replace(unsafe) + + assert qmd_bridge.claims_to_qmd([unsafe, safe]) == [{"type": "fact", "tier": "working", "text": safe.text}] + assert unsafe == original + + +@pytest.mark.parametrize("field", ["source", "locator", "excerpt"]) +def test_qmd_skips_encoded_secret_in_every_citation_field(field: str) -> None: + unsafe = _claim(2, citations=[_citation(field)]) + safe = _claim(1) + + exported = qmd_bridge.claims_to_qmd([unsafe, safe]) + + assert [row["text"] for row in exported] == [safe.text] + + +def test_dream_seed_skips_complete_unsafe_representation(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + unsafe = {"id": 2, "text": "looks safe", "subject": _ENCODED, "citations": []} + safe = {"id": 1, "text": "safe durable dream", "scope": "project:test"} + monkeypatch.setattr( + dream_bridge, + "_open_db", + lambda _path: SimpleNamespace(close=lambda: None), + ) + monkeypatch.setattr( + dream_bridge, + "_query_exportable_claims", + lambda *args, **kwargs: [unsafe, safe], + ) + monkeypatch.setattr(dream_bridge, "discover_memory_dir", lambda _path=None: tmp_path) + + result = dream_bridge.dream_seed("unused.db") + rendered = "\n".join(path.read_text(encoding="utf-8") for path in tmp_path.glob("*.md")) + + assert result["seeded"] == 1 + assert result["skipped"] == 1 + assert _ENCODED not in rendered + assert "safe durable dream" in rendered + + +def test_exportable_claims_loads_citations_in_one_query_for_multiple_claims() -> None: + conn = sqlite3.connect(":memory:") + conn.row_factory = sqlite3.Row + conn.executescript( + """ + CREATE TABLE claims ( + id INTEGER PRIMARY KEY, status TEXT, tier TEXT, + quality_score REAL, access_count INTEGER + ); + CREATE TABLE citations ( + claim_id INTEGER, source TEXT, locator TEXT, excerpt TEXT + ); + """ + ) + conn.executemany( + "INSERT INTO claims VALUES (?, 'confirmed', 'working', 1.0, 0)", + [(claim_id,) for claim_id in range(1, 5)], + ) + conn.executemany( + "INSERT INTO citations VALUES (?, 'source', 'locator', 'excerpt')", + [(claim_id,) for claim_id in range(1, 5)], + ) + queries: list[str] = [] + conn.set_trace_callback(queries.append) + + claims = dream_bridge._query_exportable_claims(conn, max_memories=4) + + citation_queries = [query for query in queries if "FROM citations" in query] + assert len(citation_queries) == 1 + assert [claim["citations"][0]["source"] for claim in claims] == ["source"] * 4 + + +def test_steward_artifact_redacts_without_mutating_input(tmp_path: Path) -> None: + payload = {"safe": "kept", "decision": {"citation": {"source": _ENCODED}}} + original = json.loads(json.dumps(payload)) + path = tmp_path / "steward.json" + + steward._write_artifact(path, payload) + persisted = path.read_text(encoding="utf-8") + + assert _ENCODED not in persisted + assert json.loads(persisted)["safe"] == "kept" + assert "sensitivity" in persisted + assert payload == original + + +class _CompactorStore: + def __init__(self, claims: list[Claim]) -> None: + self.claims = claims + self.archived: list[int] = [] + + def find_for_compaction(self, *, retain_days: int) -> list[Claim]: + return self.claims + + def list_citations(self, claim_id: int) -> list[Citation]: + return next(claim.citations for claim in self.claims if claim.id == claim_id) + + def delete_old_events(self, _days: int) -> int: + return 0 + + def transition_claim(self, claim_id: int, **_kwargs) -> None: + self.archived.append(claim_id) + + def record_event(self, **_kwargs) -> None: + pass + + +def test_compactor_skips_unsafe_artifact_and_preserves_safe_claim( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + store = _CompactorStore([_claim(1), _claim(2, citations=[_citation("excerpt")])]) + monkeypatch.setattr( + compactor, + "_archive_claims_after_artifacts", + lambda _store, claims, _days: store.archived.extend(claim.id for claim in claims) or len(claims), + ) + + compactor.run(store, artifacts_dir=tmp_path) + rendered = "\n".join(path.read_text(encoding="utf-8") for path in tmp_path.glob("*.json")) + + assert _ENCODED not in rendered + assert "safe claim 1" in rendered + assert '"skipped_sensitive": 1' in rendered + assert store.archived == [1] + + +class _FakeClient: + def __init__(self) -> None: + self.puts: list[dict] = [] + + def put(self, _url: str, *, json: dict) -> SimpleNamespace: + self.puts.append(json) + return SimpleNamespace(raise_for_status=lambda: None) + + +def _unsafe_qdrant_backend() -> QdrantBackend: + backend = QdrantBackend() + backend._client.close() + backend._client = _FakeClient() + backend._embed = lambda _text: [0.0] * EMBEDDING_DIMS + return backend + + +@pytest.mark.parametrize("field", ["text", "subject", "predicate", "object_value"]) +def test_qdrant_rejects_unsafe_claim_before_embed_or_upsert(field: str) -> None: + backend = _unsafe_qdrant_backend() + + assert backend.upsert_claim(_claim(2, **{field: _ENCODED})) is False + assert backend._client.puts == [] + + +@pytest.mark.parametrize("field", ["source", "locator", "excerpt"]) +def test_qdrant_rejects_unsafe_citation_before_embed_or_upsert(field: str) -> None: + backend = _unsafe_qdrant_backend() + + assert backend.upsert_claim(_claim(2, citations=[_citation(field)])) is False + assert backend._client.puts == [] From e537eaa0242ecc765c73f250950d4f37d353a07e Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 17:21:59 -0300 Subject: [PATCH 19/55] fix(security): enforce Atlas persistence envelopes --- memorymaster/stores/_storage_sources.py | 241 ++++++-- memorymaster/stores/postgres_store.py | 219 +++++-- tests/test_atlas_persisted_envelope.py | 791 ++++++++++++++++++++++++ 3 files changed, 1162 insertions(+), 89 deletions(-) create mode 100644 tests/test_atlas_persisted_envelope.py diff --git a/memorymaster/stores/_storage_sources.py b/memorymaster/stores/_storage_sources.py index 9dcddf03..0a0e841a 100644 --- a/memorymaster/stores/_storage_sources.py +++ b/memorymaster/stores/_storage_sources.py @@ -7,10 +7,17 @@ from __future__ import annotations import json +import re import sqlite3 from typing import TYPE_CHECKING, Any from memorymaster.stores._storage_shared import utc_now +from memorymaster.core.security import ( + sanitize_persisted_json, + sanitize_persisted_text, + scan_persisted_value, + validate_persisted_metadata, +) from memorymaster.core.models import ( ATLAS_SENSITIVITY_LEVELS, MEDIA_RETRY_STATUSES, @@ -38,10 +45,56 @@ def _normalize_sensitivity(value: str | None) -> str | None: def _json_or_none(value: dict[str, Any] | str | None) -> str | None: if value is None: return None + parsed: object = value if isinstance(value, str): stripped = value.strip() - return stripped or None - return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + if not stripped: + return None + try: + parsed = json.loads(stripped) + except json.JSONDecodeError: + parsed = stripped + sanitized, _ = sanitize_persisted_json(parsed) + if isinstance(sanitized, str): + return sanitized + return json.dumps(sanitized, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + + +def _safe_text(value: str | None) -> str | None: + if value is None: + return None + text = str(value) + sanitized, findings = sanitize_persisted_text(text) + token_findings = any( + scan_persisted_value(token) + for token in re.split(r"[^A-Za-z0-9_+=/-]+|[/\\]", text) + if token + ) + return "[REDACTED:encoded_secret]" if token_findings and not findings else sanitized + + +def _atlas_row_is_safe(row: Any, fields: tuple[str, ...]) -> bool: + return not any(scan_persisted_value(row[field]) for field in fields if row[field] is not None) + + +_SOURCE_FIELDS = ( + "source_item_id", "item_type", "chat_id", "sender_id", "sender_name", "occurred_at", + "text", "payload_json", "content_hash", "sensitivity", "created_at", "updated_at", +) +_EXTERNAL_SOURCE_FIELDS = ( + "source_type", "display_name", "config_json", "created_at", "updated_at", +) +_EVIDENCE_FIELDS = ( + "evidence_type", "text", "media_path", "provider", "payload_json", "sensitivity", "created_at", +) +_ACTION_FIELDS = ( + "proposal_type", "title", "description", "suggested_due_at", "destination", "status", + "payload_json", "external_ref", "exported_at", "idempotency_key", "created_at", "updated_at", +) +_RETRY_FIELDS = ( + "media_key", "chat_id", "media_type", "media_path", "media_url", "status", "last_error", + "next_attempt_time", "created_at", "updated_at", +) def _bounded_confidence(confidence: float | None) -> float | None: @@ -74,6 +127,7 @@ def upsert_external_source( display_name: str, config_json: dict[str, Any] | str | None = None, ) -> ExternalSource: + validate_persisted_metadata({"source_type": source_type, "display_name": display_name}) normalized_source_type = source_type.strip().lower() normalized_display_name = display_name.strip() if not normalized_source_type: @@ -84,6 +138,12 @@ def upsert_external_source( now = utc_now() payload = _json_or_none(config_json) with self.connect() as conn: + existing = conn.execute( + "SELECT * FROM external_sources WHERE source_type = ? AND display_name = ?", + (normalized_source_type, normalized_display_name), + ).fetchone() + if existing is not None and not _atlas_row_is_safe(existing, _EXTERNAL_SOURCE_FIELDS): + raise ValueError("Existing external source contains unsafe persisted data.") conn.execute( """ INSERT INTO external_sources (source_type, display_name, config_json, created_at, updated_at) @@ -98,9 +158,11 @@ def upsert_external_source( "SELECT * FROM external_sources WHERE source_type = ? AND display_name = ?", (normalized_source_type, normalized_display_name), ).fetchone() + if row is None: + raise RuntimeError("Failed to upsert external source.") + if not _atlas_row_is_safe(row, _EXTERNAL_SOURCE_FIELDS): + raise ValueError("External source contains unsafe persisted data.") conn.commit() - if row is None: - raise RuntimeError("Failed to upsert external source.") return self._row_to_external_source(row) def upsert_source_item( @@ -118,6 +180,7 @@ def upsert_source_item( content_hash: str | None = None, sensitivity: str | None = None, ) -> SourceItem: + validate_persisted_metadata({"source_item_id": source_item_id, "item_type": item_type, "content_hash": content_hash, "occurred_at": occurred_at}) normalized_source_item_id = source_item_id.strip() normalized_item_type = item_type.strip().lower() if source_id <= 0: @@ -127,14 +190,17 @@ def upsert_source_item( if not normalized_item_type: raise ValueError("item_type must be non-empty.") normalized_sensitivity = _normalize_sensitivity(sensitivity) + chat_id, sender_id, sender_name, text = map(_safe_text, (chat_id, sender_id, sender_name, text)) now = utc_now() payload = _json_or_none(payload_json) with self.connect() as conn: existing = conn.execute( - "SELECT id FROM source_items WHERE source_id = ? AND source_item_id = ?", + "SELECT * FROM source_items WHERE source_id = ? AND source_item_id = ?", (source_id, normalized_source_item_id), ).fetchone() + if existing is not None and not _atlas_row_is_safe(existing, _SOURCE_FIELDS): + raise ValueError("Existing source item contains unsafe persisted data.") # Preserve existing sensitivity on re-import unless caller passed one preserve_sensitivity_clause = ( "sensitivity = excluded.sensitivity" @@ -198,9 +264,11 @@ def upsert_source_item( ), created_at=now, ) + if row is None: + raise RuntimeError("Failed to upsert source item.") + if not _atlas_row_is_safe(row, _SOURCE_FIELDS): + raise ValueError("Source item contains unsafe persisted data.") conn.commit() - if row is None: - raise RuntimeError("Failed to upsert source item.") return self._row_to_source_item(row) def get_source_item(self, *, source_id: int, source_item_id: str) -> SourceItem | None: @@ -214,14 +282,14 @@ def get_source_item(self, *, source_id: int, source_item_id: str) -> SourceItem "SELECT * FROM source_items WHERE source_id = ? AND source_item_id = ?", (source_id, normalized_source_item_id), ).fetchone() - return self._row_to_source_item(row) if row is not None else None + return self._row_to_source_item(row) if row is not None and _atlas_row_is_safe(row, _SOURCE_FIELDS) else None def get_source_item_by_id(self, source_item_row_id: int) -> SourceItem | None: if source_item_row_id <= 0: raise ValueError("source_item_row_id must be positive.") with self.connect() as conn: row = conn.execute("SELECT * FROM source_items WHERE id = ?", (source_item_row_id,)).fetchone() - return self._row_to_source_item(row) if row is not None else None + return self._row_to_source_item(row) if row is not None and _atlas_row_is_safe(row, _SOURCE_FIELDS) else None def add_evidence_item( self, @@ -235,12 +303,14 @@ def add_evidence_item( payload_json: dict[str, Any] | str | None = None, sensitivity: str | None = None, ) -> EvidenceItem: + validate_persisted_metadata({"evidence_type": evidence_type}) normalized_evidence_type = evidence_type.strip().lower() if source_item_id <= 0: raise ValueError("source_item_id must be positive.") if not normalized_evidence_type: raise ValueError("evidence_type must be non-empty.") normalized_sensitivity = _normalize_sensitivity(sensitivity) + text, media_path, provider = map(_safe_text, (text, media_path, provider)) now = utc_now() payload = _json_or_none(payload_json) @@ -297,14 +367,32 @@ def list_evidence_items( if evidence_type: clauses.append("evidence_type = ?") params.append(evidence_type.strip().lower()) - where_sql = f"WHERE {' AND '.join(clauses)}" if clauses else "" - params.append(limit) + if limit <= 0: + return [] + page_size = min(max(limit, 25), 250) + results: list[EvidenceItem] = [] + cursor: tuple[object, int] | None = None with self.connect() as conn: - rows = conn.execute( - f"SELECT * FROM evidence_items {where_sql} ORDER BY created_at ASC, id ASC LIMIT ?", - params, - ).fetchall() - return [self._row_to_evidence_item(row) for row in rows] + while len(results) < limit: + page_clauses = list(clauses) + page_params = list(params) + if cursor is not None: + page_clauses.append("(created_at > ? OR (created_at = ? AND id > ?))") + page_params.extend((cursor[0], cursor[0], cursor[1])) + where_sql = f"WHERE {' AND '.join(page_clauses)}" if page_clauses else "" + rows = conn.execute( + f"SELECT * FROM evidence_items {where_sql} ORDER BY created_at ASC, id ASC LIMIT ?", + [*page_params, page_size], + ).fetchall() + if not rows: + break + for row in rows: + if _atlas_row_is_safe(row, _EVIDENCE_FIELDS): + results.append(self._row_to_evidence_item(row)) + if len(results) == limit: + break + cursor = (rows[-1]["created_at"], int(rows[-1]["id"])) + return results def create_action_proposal( self, @@ -321,10 +409,13 @@ def create_action_proposal( payload_json: dict[str, Any] | str | None = None, idempotency_key: str | None = None, ) -> ActionProposal: + validate_persisted_metadata({"proposal_type": proposal_type, "destination": destination, "idempotency_key": idempotency_key, "suggested_due_at": suggested_due_at}) normalized_type = proposal_type.strip().lower() normalized_title = title.strip() normalized_destination = destination.strip() or "manual" normalized_idempotency_key = (idempotency_key or "").strip() or None + normalized_title = _safe_text(normalized_title) or "[REDACTED]" + description = _safe_text(description) if not normalized_type: raise ValueError("proposal_type must be non-empty.") if not normalized_title: @@ -397,7 +488,7 @@ def get_action_proposal_by_idempotency_key(self, idempotency_key: str) -> Action "SELECT * FROM action_proposals WHERE idempotency_key = ?", (normalized,), ).fetchone() - return self._row_to_action_proposal(row) if row is not None else None + return self._row_to_action_proposal(row) if row is not None and _atlas_row_is_safe(row, _ACTION_FIELDS) else None def update_action_proposal_status( self, @@ -413,13 +504,17 @@ def update_action_proposal_status( raise ValueError("proposal_id must be positive.") if normalized_status not in {"candidate", "approved", "rejected", "exported", "failed"}: raise ValueError("status must be one of: candidate, approved, rejected, exported, failed.") + validate_persisted_metadata({"exported_at": exported_at}) now = utc_now() payload = _json_or_none(payload_json) + external_ref = _safe_text(external_ref) with self.connect() as conn: current = conn.execute("SELECT * FROM action_proposals WHERE id = ?", (proposal_id,)).fetchone() if current is None: raise ValueError(f"Action proposal {proposal_id} does not exist.") + if not _atlas_row_is_safe(current, _ACTION_FIELDS): + raise ValueError(f"Action proposal {proposal_id} contains unsafe persisted data.") final_exported_at = exported_at if exported_at is not None else current["exported_at"] if normalized_status == "exported" and final_exported_at is None: final_exported_at = now @@ -470,6 +565,8 @@ def set_source_item_sensitivity( row = conn.execute("SELECT * FROM source_items WHERE id = ?", (source_item_row_id,)).fetchone() if row is None: raise ValueError(f"Source item {source_item_row_id} does not exist.") + if not _atlas_row_is_safe(row, _SOURCE_FIELDS): + raise ValueError(f"Source item {source_item_row_id} contains unsafe persisted data.") current = row["sensitivity"] if "sensitivity" in row.keys() else None if current == normalized: return self._row_to_source_item(row) @@ -507,6 +604,8 @@ def set_evidence_item_sensitivity( row = conn.execute("SELECT * FROM evidence_items WHERE id = ?", (evidence_item_row_id,)).fetchone() if row is None: raise ValueError(f"Evidence item {evidence_item_row_id} does not exist.") + if not _atlas_row_is_safe(row, _EVIDENCE_FIELDS): + raise ValueError(f"Evidence item {evidence_item_row_id} contains unsafe persisted data.") current = row["sensitivity"] if "sensitivity" in row.keys() else None if current == normalized: return self._row_to_evidence_item(row) @@ -561,6 +660,8 @@ def enqueue_media_retry( if source_item_id <= 0: raise ValueError("source_item_id must be positive.") normalized_key = (media_key or "").strip() + validate_persisted_metadata({"media_key": normalized_key, "status": status, "next_attempt_time": next_attempt_time}) + chat_id, media_type, media_path, media_url = map(_safe_text, (chat_id, media_type, media_path, media_url)) if not normalized_key: raise ValueError("media_key must be non-empty.") if status not in MEDIA_RETRY_STATUSES: @@ -574,6 +675,8 @@ def enqueue_media_retry( (source_item_id, normalized_key), ).fetchone() if existing is not None: + if not _atlas_row_is_safe(existing, _RETRY_FIELDS): + raise ValueError("Existing media retry contains unsafe persisted data.") # Update metadata only (do NOT clobber attempt_count/status). conn.execute( """ @@ -643,17 +746,26 @@ def claim_pending_media_retries(self, limit: int = 25) -> list[MediaRetryItem]: # second guard: a row already moved out of 'pending' is never # re-claimed even if a stale id slips through. conn.execute("BEGIN IMMEDIATE") - rows = conn.execute( - """ - SELECT id FROM media_retry_queue - WHERE status = 'pending' - AND (next_attempt_time IS NULL OR next_attempt_time <= ?) - ORDER BY id ASC - LIMIT ? - """, - (now, limit), - ).fetchall() - ids = [int(r["id"]) for r in rows] + ids: list[int] = [] + cursor_id = 0 + page_size = min(max(limit, 25), 250) + while len(ids) < limit: + rows = conn.execute( + """SELECT * FROM media_retry_queue + WHERE status = 'pending' AND id > ? + AND (next_attempt_time IS NULL OR next_attempt_time <= ?) + ORDER BY id ASC LIMIT ?""", + (cursor_id, now, page_size), + ).fetchall() + if not rows: + break + ids.extend( + int(row["id"]) + for row in rows + if _atlas_row_is_safe(row, _RETRY_FIELDS) + ) + ids = ids[:limit] + cursor_id = int(rows[-1]["id"]) if not ids: conn.commit() return [] @@ -718,6 +830,8 @@ def record_media_retry_outcome( ) if status == "done" and not media_path: raise ValueError("media_path is required when status='done'.") + media_path, last_error = map(_safe_text, (media_path, last_error)) + validate_persisted_metadata({"next_attempt_time": next_attempt_time, "status": status}) now = utc_now() with self.connect() as conn: current = conn.execute( @@ -725,6 +839,8 @@ def record_media_retry_outcome( ).fetchone() if current is None: raise ValueError(f"media_retry_queue row {retry_id} does not exist.") + if not _atlas_row_is_safe(current, _RETRY_FIELDS): + raise ValueError(f"media_retry_queue row {retry_id} contains unsafe persisted data.") new_path = media_path if media_path is not None else current["media_path"] new_http = last_http_status if last_http_status is not None else current["last_http_status"] new_err = last_error if last_error is not None else current["last_error"] @@ -784,14 +900,32 @@ def list_media_retries( raise ValueError("source_item_id must be positive.") clauses.append("source_item_id = ?") params.append(source_item_id) - where_sql = f"WHERE {' AND '.join(clauses)}" if clauses else "" - params.append(limit) + if limit <= 0: + return [] + page_size = min(max(limit, 25), 250) + results: list[MediaRetryItem] = [] + cursor: tuple[object, int] | None = None with self.connect() as conn: - rows = conn.execute( - f"SELECT * FROM media_retry_queue {where_sql} ORDER BY updated_at DESC, id DESC LIMIT ?", - params, - ).fetchall() - return [self._row_to_media_retry(r) for r in rows] + while len(results) < limit: + page_clauses = list(clauses) + page_params = list(params) + if cursor is not None: + page_clauses.append("(updated_at < ? OR (updated_at = ? AND id < ?))") + page_params.extend((cursor[0], cursor[0], cursor[1])) + where_sql = f"WHERE {' AND '.join(page_clauses)}" if page_clauses else "" + rows = conn.execute( + f"SELECT * FROM media_retry_queue {where_sql} ORDER BY updated_at DESC, id DESC LIMIT ?", + [*page_params, page_size], + ).fetchall() + if not rows: + break + for row in rows: + if _atlas_row_is_safe(row, _RETRY_FIELDS): + results.append(self._row_to_media_retry(row)) + if len(results) == limit: + break + cursor = (rows[-1]["updated_at"], int(rows[-1]["id"])) + return results def media_retry_status_counts(self) -> dict[str, int]: """Return {status: count} aggregated across the queue.""" @@ -845,8 +979,11 @@ def update_action_proposal_fields( raise ValueError("proposal_id must be positive.") if title is None and description is None and suggested_due_at is None and confidence is None and payload_json is None: raise ValueError("at least one field must be provided to update.") + validate_persisted_metadata({"suggested_due_at": suggested_due_at}) normalized_title = title.strip() if title is not None else None + normalized_title = _safe_text(normalized_title) + description = _safe_text(description) if normalized_title is not None and not normalized_title: raise ValueError("title cannot be blank when provided.") bounded = _bounded_confidence(confidence) if confidence is not None else None @@ -857,6 +994,8 @@ def update_action_proposal_fields( current = conn.execute("SELECT * FROM action_proposals WHERE id = ?", (proposal_id,)).fetchone() if current is None: raise ValueError(f"Action proposal {proposal_id} does not exist.") + if not _atlas_row_is_safe(current, _ACTION_FIELDS): + raise ValueError(f"Action proposal {proposal_id} contains unsafe persisted data.") updates: list[str] = [] params: list[object] = [] @@ -928,14 +1067,32 @@ def list_action_proposals( if destination: clauses.append("destination = ?") params.append(destination.strip()) - where_sql = f"WHERE {' AND '.join(clauses)}" if clauses else "" - params.append(limit) + if limit <= 0: + return [] + page_size = min(max(limit, 25), 250) + results: list[ActionProposal] = [] + cursor: tuple[object, int] | None = None with self.connect() as conn: - rows = conn.execute( - f"SELECT * FROM action_proposals {where_sql} ORDER BY updated_at DESC, id DESC LIMIT ?", - params, - ).fetchall() - return [self._row_to_action_proposal(row) for row in rows] + while len(results) < limit: + page_clauses = list(clauses) + page_params = list(params) + if cursor is not None: + page_clauses.append("(updated_at < ? OR (updated_at = ? AND id < ?))") + page_params.extend((cursor[0], cursor[0], cursor[1])) + where_sql = f"WHERE {' AND '.join(page_clauses)}" if page_clauses else "" + rows = conn.execute( + f"SELECT * FROM action_proposals {where_sql} ORDER BY updated_at DESC, id DESC LIMIT ?", + [*page_params, page_size], + ).fetchall() + if not rows: + break + for row in rows: + if _atlas_row_is_safe(row, _ACTION_FIELDS): + results.append(self._row_to_action_proposal(row)) + if len(results) == limit: + break + cursor = (rows[-1]["updated_at"], int(rows[-1]["id"])) + return results @staticmethod def _row_to_external_source(row: sqlite3.Row) -> ExternalSource: diff --git a/memorymaster/stores/postgres_store.py b/memorymaster/stores/postgres_store.py index 0e2e182a..6d952f92 100644 --- a/memorymaster/stores/postgres_store.py +++ b/memorymaster/stores/postgres_store.py @@ -32,6 +32,7 @@ sanitize_claim_input, sanitize_claim_structure_input, sanitize_event_input, + sanitize_persisted_json, sanitize_persisted_text, validate_persisted_metadata, ) @@ -52,6 +53,15 @@ expressions_match, ) from memorymaster.stores.storage import SQLiteStore +from memorymaster.stores._storage_sources import ( + _ACTION_FIELDS, + _EVIDENCE_FIELDS, + _EXTERNAL_SOURCE_FIELDS, + _RETRY_FIELDS, + _SOURCE_FIELDS, + _atlas_row_is_safe, + _safe_text, +) POSTGRES_EVENTS_APPEND_ONLY_TRIGGERS = ( "trg_events_append_only_update", @@ -3566,10 +3576,10 @@ def _json_payload(value: dict[str, object] | str | None) -> object | None: if not stripped: return None try: - return json.loads(stripped) + value = json.loads(stripped) except json.JSONDecodeError: - return stripped - return value + value = stripped + return sanitize_persisted_json(value)[0] def upsert_external_source( self, @@ -3580,6 +3590,7 @@ def upsert_external_source( ) -> ExternalSource: self._deny_unsupported_team_surface("upsert_external_source") _, _, Jsonb = self._load_psycopg() + validate_persisted_metadata({"source_type": source_type, "display_name": display_name}) normalized_source_type = source_type.strip().lower() normalized_display_name = display_name.strip() if not normalized_source_type: @@ -3589,6 +3600,13 @@ def upsert_external_source( now = utc_now() payload = self._json_payload(config_json) with self.connect() as conn, conn.cursor() as cur: + cur.execute( + "SELECT * FROM external_sources WHERE source_type = %s AND display_name = %s", + (normalized_source_type, normalized_display_name), + ) + existing = cur.fetchone() + if existing is not None and not _atlas_row_is_safe(existing, _EXTERNAL_SOURCE_FIELDS): + raise ValueError("Existing external source contains unsafe persisted data.") cur.execute( """ INSERT INTO external_sources (source_type, display_name, config_json, created_at, updated_at) @@ -3607,8 +3625,10 @@ def upsert_external_source( ), ) row = cur.fetchone() - if row is None: - raise RuntimeError("Failed to upsert external source.") + if row is None: + raise RuntimeError("Failed to upsert external source.") + if not _atlas_row_is_safe(row, _EXTERNAL_SOURCE_FIELDS): + raise ValueError("External source contains unsafe persisted data.") return self._row_to_external_source(row) def upsert_source_item( @@ -3630,6 +3650,7 @@ def upsert_source_item( from memorymaster.stores._storage_sources import _normalize_sensitivity _, _, Jsonb = self._load_psycopg() + validate_persisted_metadata({"source_item_id": source_item_id, "item_type": item_type, "occurred_at": occurred_at, "content_hash": content_hash}) normalized_source_item_id = source_item_id.strip() normalized_item_type = item_type.strip().lower() if source_id <= 0: @@ -3639,6 +3660,7 @@ def upsert_source_item( if not normalized_item_type: raise ValueError("item_type must be non-empty.") normalized_sensitivity = _normalize_sensitivity(sensitivity) + chat_id, sender_id, sender_name, text = map(_safe_text, (chat_id, sender_id, sender_name, text)) now = utc_now() payload = self._json_payload(payload_json) # Preserve existing sensitivity on re-import unless caller passed one @@ -3649,10 +3671,12 @@ def upsert_source_item( ) with self.connect() as conn, conn.cursor() as cur: cur.execute( - "SELECT id FROM source_items WHERE source_id = %s AND source_item_id = %s", + "SELECT * FROM source_items WHERE source_id = %s AND source_item_id = %s", (source_id, normalized_source_item_id), ) existing = cur.fetchone() + if existing is not None and not _atlas_row_is_safe(existing, _SOURCE_FIELDS): + raise ValueError("Existing source item contains unsafe persisted data.") cur.execute( f""" INSERT INTO source_items ( @@ -3705,8 +3729,10 @@ def upsert_source_item( }, created_at=now, ) - if row is None: - raise RuntimeError("Failed to upsert source item.") + if row is None: + raise RuntimeError("Failed to upsert source item.") + if not _atlas_row_is_safe(row, _SOURCE_FIELDS): + raise ValueError("Source item contains unsafe persisted data.") return self._row_to_source_item(row) def get_source_item(self, *, source_id: int, source_item_id: str) -> SourceItem | None: @@ -3722,7 +3748,7 @@ def get_source_item(self, *, source_id: int, source_item_id: str) -> SourceItem (source_id, normalized_source_item_id), ) row = cur.fetchone() - return self._row_to_source_item(row) if row is not None else None + return self._row_to_source_item(row) if row is not None and _atlas_row_is_safe(row, _SOURCE_FIELDS) else None def get_source_item_by_id(self, source_item_row_id: int) -> SourceItem | None: self._deny_unsupported_team_surface("get_source_item_by_id") @@ -3731,7 +3757,7 @@ def get_source_item_by_id(self, source_item_row_id: int) -> SourceItem | None: with self.connect() as conn, conn.cursor() as cur: cur.execute("SELECT * FROM source_items WHERE id = %s", (source_item_row_id,)) row = cur.fetchone() - return self._row_to_source_item(row) if row is not None else None + return self._row_to_source_item(row) if row is not None and _atlas_row_is_safe(row, _SOURCE_FIELDS) else None def add_evidence_item( self, @@ -3749,12 +3775,14 @@ def add_evidence_item( from memorymaster.stores._storage_sources import _normalize_sensitivity _, _, Jsonb = self._load_psycopg() + validate_persisted_metadata({"evidence_type": evidence_type}) normalized_evidence_type = evidence_type.strip().lower() if source_item_id <= 0: raise ValueError("source_item_id must be positive.") if not normalized_evidence_type: raise ValueError("evidence_type must be non-empty.") normalized_sensitivity = _normalize_sensitivity(sensitivity) + text, media_path, provider = map(_safe_text, (text, media_path, provider)) now = utc_now() bounded = None if confidence is None else max(0.0, min(1.0, float(confidence))) payload = self._json_payload(payload_json) @@ -3817,15 +3845,33 @@ def list_evidence_items( if evidence_type: clauses.append("evidence_type = %s") params.append(evidence_type.strip().lower()) - where_sql = f"WHERE {' AND '.join(clauses)}" if clauses else "" - params.append(limit) + if limit <= 0: + return [] + page_size = min(max(limit, 25), 250) + results: list[EvidenceItem] = [] + cursor: tuple[object, int] | None = None with self.connect() as conn, conn.cursor() as cur: - cur.execute( - f"SELECT * FROM evidence_items {where_sql} ORDER BY created_at ASC, id ASC LIMIT %s", - params, - ) - rows = cur.fetchall() - return [self._row_to_evidence_item(row) for row in rows] + while len(results) < limit: + page_clauses = list(clauses) + page_params = list(params) + if cursor is not None: + page_clauses.append("(created_at > %s OR (created_at = %s AND id > %s))") + page_params.extend((cursor[0], cursor[0], cursor[1])) + where_sql = f"WHERE {' AND '.join(page_clauses)}" if page_clauses else "" + cur.execute( + f"SELECT * FROM evidence_items {where_sql} ORDER BY created_at ASC, id ASC LIMIT %s", + [*page_params, page_size], + ) + rows = cur.fetchall() + if not rows: + break + for row in rows: + if _atlas_row_is_safe(row, _EVIDENCE_FIELDS): + results.append(self._row_to_evidence_item(row)) + if len(results) == limit: + break + cursor = (rows[-1]["created_at"], int(rows[-1]["id"])) + return results def create_action_proposal( self, @@ -3844,10 +3890,13 @@ def create_action_proposal( ) -> ActionProposal: self._deny_unsupported_team_surface("create_action_proposal") _, _, Jsonb = self._load_psycopg() + validate_persisted_metadata({"proposal_type": proposal_type, "destination": destination, "idempotency_key": idempotency_key, "suggested_due_at": suggested_due_at}) normalized_type = proposal_type.strip().lower() normalized_title = title.strip() normalized_destination = destination.strip() or "manual" normalized_idempotency_key = (idempotency_key or "").strip() or None + normalized_title = _safe_text(normalized_title) or "[REDACTED]" + description = _safe_text(description) if not normalized_type: raise ValueError("proposal_type must be non-empty.") if not normalized_title: @@ -3913,7 +3962,7 @@ def get_action_proposal_by_idempotency_key(self, idempotency_key: str) -> Action with self.connect() as conn, conn.cursor() as cur: cur.execute("SELECT * FROM action_proposals WHERE idempotency_key = %s", (normalized,)) row = cur.fetchone() - return self._row_to_action_proposal(row) if row is not None else None + return self._row_to_action_proposal(row) if row is not None and _atlas_row_is_safe(row, _ACTION_FIELDS) else None def update_action_proposal_status( self, @@ -3931,13 +3980,17 @@ def update_action_proposal_status( raise ValueError("proposal_id must be positive.") if normalized_status not in {"candidate", "approved", "rejected", "exported", "failed"}: raise ValueError("status must be one of: candidate, approved, rejected, exported, failed.") + validate_persisted_metadata({"exported_at": exported_at}) now = utc_now() payload = self._json_payload(payload_json) + external_ref = _safe_text(external_ref) with self.connect() as conn, conn.cursor() as cur: cur.execute("SELECT * FROM action_proposals WHERE id = %s", (proposal_id,)) current = cur.fetchone() if current is None: raise ValueError(f"Action proposal {proposal_id} does not exist.") + if not _atlas_row_is_safe(current, _ACTION_FIELDS): + raise ValueError(f"Action proposal {proposal_id} contains unsafe persisted data.") final_exported_at = exported_at if exported_at is not None else current["exported_at"] if normalized_status == "exported" and final_exported_at is None: final_exported_at = now @@ -3994,6 +4047,8 @@ def set_source_item_sensitivity( current = cur.fetchone() if current is None: raise ValueError(f"Source item {source_item_row_id} does not exist.") + if not _atlas_row_is_safe(current, _SOURCE_FIELDS): + raise ValueError(f"Source item {source_item_row_id} contains unsafe persisted data.") current_sensitivity = current.get("sensitivity") if current_sensitivity == normalized: return self._row_to_source_item(current) @@ -4031,6 +4086,8 @@ def set_evidence_item_sensitivity( current = cur.fetchone() if current is None: raise ValueError(f"Evidence item {evidence_item_row_id} does not exist.") + if not _atlas_row_is_safe(current, _EVIDENCE_FIELDS): + raise ValueError(f"Evidence item {evidence_item_row_id} contains unsafe persisted data.") current_sensitivity = current.get("sensitivity") if current_sensitivity == normalized: return self._row_to_evidence_item(current) @@ -4090,6 +4147,8 @@ def enqueue_media_retry( if source_item_id <= 0: raise ValueError("source_item_id must be positive.") normalized_key = (media_key or "").strip() + validate_persisted_metadata({"media_key": normalized_key, "status": status, "next_attempt_time": next_attempt_time}) + chat_id, media_type, media_path, media_url = map(_safe_text, (chat_id, media_type, media_path, media_url)) if not normalized_key: raise ValueError("media_key must be non-empty.") if status not in MEDIA_RETRY_STATUSES: @@ -4102,6 +4161,8 @@ def enqueue_media_retry( ) existing = cur.fetchone() if existing is not None: + if not _atlas_row_is_safe(existing, _RETRY_FIELDS): + raise ValueError("Existing media retry contains unsafe persisted data.") cur.execute( """ UPDATE media_retry_queue @@ -4152,26 +4213,45 @@ def claim_pending_media_retries(self, limit: int = 25) -> list[MediaRetryItem]: return [] now = utc_now() with self.connect() as conn, conn.cursor() as cur: + safe_ids: list[int] = [] + cursor_id = 0 + page_size = min(max(limit, 25), 250) + while len(safe_ids) < limit: + cur.execute( + """SELECT * FROM media_retry_queue + WHERE status = 'pending' AND id > %s + AND (next_attempt_time IS NULL OR next_attempt_time <= %s) + ORDER BY id ASC LIMIT %s FOR UPDATE SKIP LOCKED""", + (cursor_id, now, page_size), + ) + candidates = cur.fetchall() + if not candidates: + break + safe_ids.extend( + int(row["id"]) + for row in candidates + if _atlas_row_is_safe(row, _RETRY_FIELDS) + ) + safe_ids = safe_ids[:limit] + cursor_id = int(candidates[-1]["id"]) + if not safe_ids: + return [] + placeholders = ", ".join(["%s"] * len(safe_ids)) cur.execute( - """ + f""" UPDATE media_retry_queue SET status = 'retrying', attempt_count = attempt_count + 1, updated_at = %s - WHERE id IN ( - SELECT id FROM media_retry_queue - WHERE status = 'pending' - AND (next_attempt_time IS NULL OR next_attempt_time <= %s) - ORDER BY id ASC - LIMIT %s - FOR UPDATE SKIP LOCKED - ) + WHERE status = 'pending' AND id IN ({placeholders}) RETURNING * """, - (now, now, limit), + (now, *safe_ids), ) rows = cur.fetchall() - for row in rows: + by_id = {int(row["id"]): row for row in rows} + ordered_rows = [by_id[row_id] for row_id in safe_ids if row_id in by_id] + for row in ordered_rows: self._insert_event_row( conn, claim_id=None, @@ -4182,7 +4262,7 @@ def claim_pending_media_retries(self, limit: int = 25) -> list[MediaRetryItem]: payload={"retry_id": int(row["id"])}, created_at=now, ) - return [self._row_to_media_retry(r) for r in rows] + return [self._row_to_media_retry(row) for row in ordered_rows] def record_media_retry_outcome( self, @@ -4201,12 +4281,16 @@ def record_media_retry_outcome( raise ValueError(f"status must be one of: {', '.join(MEDIA_RETRY_STATUSES)}.") if status == "done" and not media_path: raise ValueError("media_path is required when status='done'.") + media_path, last_error = map(_safe_text, (media_path, last_error)) + validate_persisted_metadata({"status": status, "next_attempt_time": next_attempt_time}) now = utc_now() with self.connect() as conn, conn.cursor() as cur: cur.execute("SELECT * FROM media_retry_queue WHERE id = %s", (retry_id,)) current = cur.fetchone() if current is None: raise ValueError(f"media_retry_queue row {retry_id} does not exist.") + if not _atlas_row_is_safe(current, _RETRY_FIELDS): + raise ValueError(f"media_retry_queue row {retry_id} contains unsafe persisted data.") new_path = media_path if media_path is not None else current["media_path"] new_http = last_http_status if last_http_status is not None else current["last_http_status"] new_err = last_error if last_error is not None else current["last_error"] @@ -4262,15 +4346,33 @@ def list_media_retries( raise ValueError("source_item_id must be positive.") clauses.append("source_item_id = %s") params.append(source_item_id) - where_sql = f"WHERE {' AND '.join(clauses)}" if clauses else "" - params.append(limit) + if limit <= 0: + return [] + page_size = min(max(limit, 25), 250) + results: list[MediaRetryItem] = [] + cursor: tuple[object, int] | None = None with self.connect() as conn, conn.cursor() as cur: - cur.execute( - f"SELECT * FROM media_retry_queue {where_sql} ORDER BY updated_at DESC, id DESC LIMIT %s", - params, - ) - rows = cur.fetchall() - return [self._row_to_media_retry(r) for r in rows] + while len(results) < limit: + page_clauses = list(clauses) + page_params = list(params) + if cursor is not None: + page_clauses.append("(updated_at < %s OR (updated_at = %s AND id < %s))") + page_params.extend((cursor[0], cursor[0], cursor[1])) + where_sql = f"WHERE {' AND '.join(page_clauses)}" if page_clauses else "" + cur.execute( + f"SELECT * FROM media_retry_queue {where_sql} ORDER BY updated_at DESC, id DESC LIMIT %s", + [*page_params, page_size], + ) + rows = cur.fetchall() + if not rows: + break + for row in rows: + if _atlas_row_is_safe(row, _RETRY_FIELDS): + results.append(self._row_to_media_retry(row)) + if len(results) == limit: + break + cursor = (rows[-1]["updated_at"], int(rows[-1]["id"])) + return results def media_retry_status_counts(self) -> dict[str, int]: self._deny_unsupported_team_surface("media_retry_status_counts") @@ -4299,8 +4401,11 @@ def update_action_proposal_fields( raise ValueError("proposal_id must be positive.") if title is None and description is None and suggested_due_at is None and confidence is None and payload_json is None: raise ValueError("at least one field must be provided to update.") + validate_persisted_metadata({"suggested_due_at": suggested_due_at}) normalized_title = title.strip() if title is not None else None + normalized_title = _safe_text(normalized_title) + description = _safe_text(description) if normalized_title is not None and not normalized_title: raise ValueError("title cannot be blank when provided.") bounded = max(0.0, min(1.0, float(confidence))) if confidence is not None else None @@ -4312,6 +4417,8 @@ def update_action_proposal_fields( current = cur.fetchone() if current is None: raise ValueError(f"Action proposal {proposal_id} does not exist.") + if not _atlas_row_is_safe(current, _ACTION_FIELDS): + raise ValueError(f"Action proposal {proposal_id} contains unsafe persisted data.") updates: list[str] = [] params: list[object] = [] @@ -4379,15 +4486,33 @@ def list_action_proposals( if destination: clauses.append("destination = %s") params.append(destination.strip()) - where_sql = f"WHERE {' AND '.join(clauses)}" if clauses else "" - params.append(limit) + if limit <= 0: + return [] + page_size = min(max(limit, 25), 250) + results: list[ActionProposal] = [] + cursor: tuple[object, int] | None = None with self.connect() as conn, conn.cursor() as cur: - cur.execute( - f"SELECT * FROM action_proposals {where_sql} ORDER BY updated_at DESC, id DESC LIMIT %s", - params, - ) - rows = cur.fetchall() - return [self._row_to_action_proposal(row) for row in rows] + while len(results) < limit: + page_clauses = list(clauses) + page_params = list(params) + if cursor is not None: + page_clauses.append("(updated_at < %s OR (updated_at = %s AND id < %s))") + page_params.extend((cursor[0], cursor[0], cursor[1])) + where_sql = f"WHERE {' AND '.join(page_clauses)}" if page_clauses else "" + cur.execute( + f"SELECT * FROM action_proposals {where_sql} ORDER BY updated_at DESC, id DESC LIMIT %s", + [*page_params, page_size], + ) + rows = cur.fetchall() + if not rows: + break + for row in rows: + if _atlas_row_is_safe(row, _ACTION_FIELDS): + results.append(self._row_to_action_proposal(row)) + if len(results) == limit: + break + cursor = (rows[-1]["updated_at"], int(rows[-1]["id"])) + return results def get_claim_by_human_id( self, diff --git a/tests/test_atlas_persisted_envelope.py b/tests/test_atlas_persisted_envelope.py new file mode 100644 index 00000000..9171e588 --- /dev/null +++ b/tests/test_atlas_persisted_envelope.py @@ -0,0 +1,791 @@ +from __future__ import annotations + +import base64 +import copy +import inspect +import json +from pathlib import Path +from typing import Any + +import pytest + +from memorymaster.bridges import atlas_llm_extractor +from memorymaster.bridges.action_exporters import export_approved_actions +from memorymaster.bridges.action_extractor import propose_actions_from_evidence +from memorymaster.bridges.atlas_llm_extractor import extract_atlas_claims_llm +from memorymaster.core.service import MemoryService +from memorymaster.stores import _storage_sources +from memorymaster.stores._storage_sources import _SourceItemsMixin +from memorymaster.stores.postgres_store import PostgresStore + + +LITERAL = "OPENAI_API_KEY=sk-proj-FAKEatlasPersistedEnvelope1234567890ABCD" +ENCODED = base64.b64encode(LITERAL.encode()).decode() + + +@pytest.fixture +def service(tmp_path: Path) -> MemoryService: + value = MemoryService(tmp_path / "atlas-envelope.db", workspace_root=tmp_path) + value.init_db() + return value + + +def _seed_parent(service: MemoryService): + source = service.upsert_external_source( + source_type="whatsapp", display_name="primary", config_json={"mode": "safe"} + ) + item = service.upsert_source_item( + source_id=source.id, + source_item_id="message-1", + item_type="message", + text="safe source text", + ) + return source, item + + +@pytest.mark.parametrize("secret", [LITERAL, ENCODED]) +def test_atlas_content_and_nested_json_are_sanitized_without_mutating_input( + service: MemoryService, secret: str +) -> None: + config = {"outer": [{"credential": secret}], "safe": "kept"} + original = copy.deepcopy(config) + source = service.upsert_external_source( + source_type="whatsapp", display_name="primary", config_json=config + ) + assert config == original + assert secret not in (source.config_json or "") + assert json.loads(source.config_json or "{}")["safe"] == "kept" + + payload = {"outer": [{"credential": secret}], "safe": "kept"} + original = copy.deepcopy(payload) + item = service.upsert_source_item( + source_id=source.id, + source_item_id="message-1", + item_type="message", + sender_name=f"name {secret}", + text=f"body {secret}", + payload_json=payload, + ) + assert payload == original + assert secret not in repr(item) + + evidence = service.add_evidence_item( + source_item_id=item.id, + evidence_type="message_text", + text=f"evidence {secret}", + media_path=f"media/{secret}", + provider=f"provider {secret}", + payload_json=payload, + ) + assert secret not in repr(evidence) + + proposal = service.create_action_proposal( + proposal_type="task", + title=f"title {secret}", + description=f"description {secret}", + source_item_id=item.id, + evidence_item_id=evidence.id, + destination="manual", + payload_json=payload, + ) + assert secret not in repr(proposal) + + retry = service.enqueue_media_retry( + source_item_id=item.id, + media_key="safe-key", + chat_id=f"chat {secret}", + media_type=f"audio {secret}", + media_path=f"media/{secret}", + media_url=f"https://example.invalid/file?token={secret}", + ) + retry = service.record_media_retry_outcome( + retry.id, status="failed", last_error=f"failure {secret}" + ) + assert secret not in repr(retry) + + +def test_unsafe_legacy_rows_are_hidden_from_all_atlas_read_and_derivation_surfaces( + service: MemoryService, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source, item = _seed_parent(service) + with service.store.connect() as conn: + conn.execute("UPDATE source_items SET text = ? WHERE id = ?", (ENCODED, item.id)) + evidence_id = conn.execute( + "INSERT INTO evidence_items (source_item_id,evidence_type,text,created_at) VALUES (?,?,?,?)", + (item.id, "message_text", ENCODED, "2026-01-01T00:00:00Z"), + ).lastrowid + proposal_id = conn.execute( + """INSERT INTO action_proposals + (proposal_type,title,destination,status,confidence,payload_json,created_at,updated_at) + VALUES ('task',?,'super-productivity','approved',0.5,?, ?, ?)""", + (ENCODED, json.dumps({"nested": ENCODED}), "2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z"), + ).lastrowid + conn.execute( + """INSERT INTO media_retry_queue + (source_item_id,media_key,status,attempt_count,last_error,created_at,updated_at) + VALUES (?,?, 'pending',0,?,?,?)""", + (item.id, "legacy-safe-key", ENCODED, "2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z"), + ) + conn.commit() + + assert service.get_source_item(source_id=source.id, source_item_id="message-1") is None + assert service.get_source_item_by_id(item.id) is None + assert all(row.id != evidence_id for row in service.list_evidence_items()) + assert all(row.id != proposal_id for row in service.list_action_proposals()) + assert service.list_media_retries() == [] + assert service.claim_pending_media_retries() == [] + + calls: list[tuple[str, str]] = [] + monkeypatch.setattr( + atlas_llm_extractor, + "call_llm", + lambda prompt, text, **kwargs: calls.append((prompt, text)) or "[]", + ) + assert extract_atlas_claims_llm(service, scope="project:test").scanned == 0 + assert calls == [] + assert propose_actions_from_evidence(service).scanned == 0 + + output = tmp_path / "actions.json" + assert export_approved_actions(service, output).exported == 0 + rendered = output.read_text(encoding="utf-8") + assert ENCODED not in rendered + + +def test_safe_atlas_round_trip_is_unchanged(service: MemoryService) -> None: + source, item = _seed_parent(service) + assert source.display_name == "primary" + assert service.get_source_item(source_id=source.id, source_item_id="message-1") == item + evidence = service.add_evidence_item( + source_item_id=item.id, + evidence_type="message_text", + text="Please send the ordinary invoice tomorrow.", + payload_json={"kind": "text", "parts": ["one", "two"]}, + ) + assert service.list_evidence_items() == [evidence] + + +@pytest.mark.parametrize( + ("family", "field"), + [ + ("external", "source_type"), ("external", "display_name"), + ("source", "source_item_id"), ("source", "item_type"), + ("source", "occurred_at"), ("source", "content_hash"), + ("evidence", "evidence_type"), + ("action_create", "proposal_type"), ("action_create", "destination"), + ("action_create", "idempotency_key"), ("action_create", "suggested_due_at"), + ("action_status", "exported_at"), ("action_fields", "suggested_due_at"), + ("retry_create", "media_key"), ("retry_create", "next_attempt_time"), + ("retry_outcome", "next_attempt_time"), + ], +) +def test_atlas_metadata_write_paths_reject_before_sql( + service: MemoryService, family: str, field: str +) -> None: + source, item = _seed_parent(service) + service.add_evidence_item(source_item_id=item.id, evidence_type="message_text") + proposal = service.create_action_proposal(proposal_type="task", title="safe") + retry = service.enqueue_media_retry(source_item_id=item.id, media_key="safe-key") + operations = { + "external": lambda: service.upsert_external_source( + **{"source_type": "whatsapp", "display_name": "secondary", field: ENCODED} + ), + "source": lambda: service.upsert_source_item( + **{"source_id": source.id, "source_item_id": "message-2", "item_type": "message", field: ENCODED} + ), + "evidence": lambda: service.add_evidence_item( + **{"source_item_id": item.id, "evidence_type": "message_text", field: ENCODED} + ), + "action_create": lambda: service.create_action_proposal( + **{"proposal_type": "task", "title": "safe", field: ENCODED} + ), + "action_status": lambda: service.update_action_proposal_status( + proposal.id, **{"status": "exported", field: ENCODED} + ), + "action_fields": lambda: service.update_action_proposal_fields( + proposal.id, **{field: ENCODED} + ), + "retry_create": lambda: service.enqueue_media_retry( + **{"source_item_id": item.id, "media_key": "new-key", field: ENCODED} + ), + "retry_outcome": lambda: service.record_media_retry_outcome( + retry.id, **{"status": "failed", field: ENCODED} + ), + } + with service.store.connect() as conn: + before = tuple( + int(conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) + for table in ("external_sources", "source_items", "evidence_items", "action_proposals", "media_retry_queue", "events") + ) + with pytest.raises(ValueError): + operations[family]() + with service.store.connect() as conn: + after = tuple( + int(conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0]) + for table in ("external_sources", "source_items", "evidence_items", "action_proposals", "media_retry_queue", "events") + ) + assert after == before + + +def test_action_status_rejects_encoded_secret_in_exported_at_before_sql( + service: MemoryService, +) -> None: + _, item = _seed_parent(service) + proposal = service.create_action_proposal( + proposal_type="task", title="safe", source_item_id=item.id + ) + with pytest.raises(ValueError): + service.update_action_proposal_status( + proposal.id, status="exported", exported_at=ENCODED + ) + current = service.list_action_proposals(limit=1)[0] + assert current.status == "candidate" + assert current.exported_at is None + + +def test_action_fields_rejects_encoded_secret_in_suggested_due_at_before_sql( + service: MemoryService, +) -> None: + _, item = _seed_parent(service) + proposal = service.create_action_proposal( + proposal_type="task", title="safe", source_item_id=item.id + ) + with pytest.raises(ValueError): + service.update_action_proposal_fields( + proposal.id, suggested_due_at=ENCODED + ) + current = service.list_action_proposals(limit=1)[0] + assert current.suggested_due_at is None + + +def test_list_limits_count_safe_rows_after_arbitrary_unsafe_prefix( + service: MemoryService, +) -> None: + _, item = _seed_parent(service) + with service.store.connect() as conn: + for index in range(251): + conn.execute( + "INSERT INTO evidence_items (source_item_id,evidence_type,text,created_at) VALUES (?,?,?,?)", + (item.id, "message_text", ENCODED, "2026-01-01T00:00:00Z"), + ) + safe_evidence_ids = [ + int( + conn.execute( + "INSERT INTO evidence_items (source_item_id,evidence_type,text,created_at) VALUES (?,?,?,?)", + (item.id, "message_text", f"safe-{index}", "2026-01-02T00:00:00Z"), + ).lastrowid + ) + for index in range(2) + ] + for index in range(251): + conn.execute( + """INSERT INTO media_retry_queue + (source_item_id,media_key,status,attempt_count,last_error,created_at,updated_at) + VALUES (?,?,'failed',0,?,?,?)""", + (item.id, f"unsafe-{index}", ENCODED, "2026-01-01T00:00:00Z", "2026-01-03T00:00:00Z"), + ) + safe_retry_ids = [ + int( + conn.execute( + """INSERT INTO media_retry_queue + (source_item_id,media_key,status,attempt_count,created_at,updated_at) + VALUES (?,?,'failed',0,?,?)""", + (item.id, f"safe-{index}", "2026-01-01T00:00:00Z", f"2026-01-02T00:00:0{index}Z"), + ).lastrowid + ) + for index in range(2) + ] + for index in range(251): + conn.execute( + """INSERT INTO action_proposals + (proposal_type,title,destination,status,confidence,created_at,updated_at) + VALUES ('task',?,'manual','candidate',0.5,?,?)""", + (ENCODED, "2026-01-01T00:00:00Z", "2026-01-03T00:00:00Z"), + ) + safe_action_ids = [ + int( + conn.execute( + """INSERT INTO action_proposals + (proposal_type,title,destination,status,confidence,created_at,updated_at) + VALUES ('task',?,'manual','candidate',0.5,?,?)""", + (f"safe-{index}", "2026-01-01T00:00:00Z", f"2026-01-02T00:00:0{index}Z"), + ).lastrowid + ) + for index in range(2) + ] + conn.commit() + + assert [row.id for row in service.list_evidence_items(limit=2)] == safe_evidence_ids + assert [row.id for row in service.list_media_retries(status="failed", limit=2)] == list( + reversed(safe_retry_ids) + ) + assert [row.id for row in service.list_action_proposals(limit=2)] == list( + reversed(safe_action_ids) + ) + + +def test_sqlite_claim_skips_unsafe_prefix_without_mutation_or_events( + service: MemoryService, +) -> None: + _, item = _seed_parent(service) + with service.store.connect() as conn: + unsafe_ids = [ + int( + conn.execute( + """INSERT INTO media_retry_queue + (source_item_id,media_key,status,attempt_count,last_error,created_at,updated_at) + VALUES (?,?,'pending',0,?,?,?)""", + (item.id, f"unsafe-{index}", ENCODED, "2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z"), + ).lastrowid + ) + for index in range(251) + ] + safe_ids = [ + int( + conn.execute( + """INSERT INTO media_retry_queue + (source_item_id,media_key,status,attempt_count,created_at,updated_at) + VALUES (?,?,'pending',0,?,?)""", + (item.id, f"safe-{index}", "2026-01-01T00:00:00Z", "2026-01-01T00:00:00Z"), + ).lastrowid + ) + for index in range(2) + ] + before_events = int(conn.execute("SELECT COUNT(*) FROM events").fetchone()[0]) + conn.commit() + + assert [row.id for row in service.claim_pending_media_retries(limit=2)] == safe_ids + with service.store.connect() as conn: + unsafe = conn.execute( + f"SELECT status,attempt_count FROM media_retry_queue WHERE id IN ({','.join('?' for _ in unsafe_ids)}) ORDER BY id", + unsafe_ids, + ).fetchall() + after_events = int(conn.execute("SELECT COUNT(*) FROM events").fetchone()[0]) + assert [(row["status"], row["attempt_count"]) for row in unsafe] == [ + ("pending", 0) + ] * len(unsafe_ids) + assert after_events - before_events == len(safe_ids) + + +@pytest.mark.parametrize("field", ["config_json", "created_at"]) +def test_sqlite_external_source_upsert_rejects_unsafe_existing_row_before_mutation( + service: MemoryService, field: str +) -> None: + source, _ = _seed_parent(service) + with service.store.connect() as conn: + conn.execute(f"UPDATE external_sources SET {field} = ? WHERE id = ?", (ENCODED, source.id)) + conn.commit() + with service.store.connect() as conn: + before = dict(conn.execute("SELECT * FROM external_sources WHERE id = ?", (source.id,)).fetchone()) + with pytest.raises(ValueError, match="unsafe"): + service.upsert_external_source( + source_type="whatsapp", display_name="primary", config_json={"mode": "changed"} + ) + with service.store.connect() as conn: + after = dict(conn.execute("SELECT * FROM external_sources WHERE id = ?", (source.id,)).fetchone()) + assert after == before + + +@pytest.mark.parametrize("field", ["created_at", "sensitivity"]) +def test_sqlite_source_item_upsert_rejects_unsafe_existing_row_before_mutation_or_event( + service: MemoryService, field: str +) -> None: + source, item = _seed_parent(service) + with service.store.connect() as conn: + if field == "sensitivity": + conn.execute("PRAGMA ignore_check_constraints = ON") + conn.execute(f"UPDATE source_items SET {field} = ? WHERE id = ?", (ENCODED, item.id)) + before = dict(conn.execute("SELECT * FROM source_items WHERE id = ?", (item.id,)).fetchone()) + events = int(conn.execute("SELECT COUNT(*) FROM events").fetchone()[0]) + conn.commit() + with pytest.raises(ValueError, match="unsafe"): + service.upsert_source_item( + source_id=source.id, source_item_id="message-1", item_type="message", text="changed" + ) + with service.store.connect() as conn: + after = dict(conn.execute("SELECT * FROM source_items WHERE id = ?", (item.id,)).fetchone()) + after_events = int(conn.execute("SELECT COUNT(*) FROM events").fetchone()[0]) + assert after == before + assert after_events == events + + +@pytest.mark.parametrize("family", ["external", "source"]) +def test_upsert_postwrite_safety_failure_rolls_back_atomically( + service: MemoryService, + family: str, +) -> None: + source, item = _seed_parent(service) + table = "external_sources" if family == "external" else "source_items" + row_id = source.id if family == "external" else item.id + with service.store.connect() as conn: + before = dict(conn.execute(f"SELECT * FROM {table} WHERE id = ?", (row_id,)).fetchone()) + conn.execute( + f"""CREATE TRIGGER poison_{table}_after_update + AFTER UPDATE ON {table} + BEGIN + UPDATE {table} SET created_at = '{ENCODED}' WHERE id = NEW.id; + END""" + ) + conn.commit() + + if family == "external": + def operation() -> object: + return service.upsert_external_source( + source_type="whatsapp", + display_name="primary", + config_json={"mode": "changed"}, + ) + else: + def operation() -> object: + return service.upsert_source_item( + source_id=source.id, + source_item_id="message-1", + item_type="message", + text="changed", + ) + + with pytest.raises(ValueError, match="unsafe"): + operation() + with service.store.connect() as conn: + after = dict(conn.execute(f"SELECT * FROM {table} WHERE id = ?", (row_id,)).fetchone()) + assert after == before + + +def test_action_updates_advance_updated_at( + service: MemoryService, monkeypatch: pytest.MonkeyPatch +) -> None: + _, item = _seed_parent(service) + timestamps = iter( + [ + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:01Z", + "2026-01-01T00:00:02Z", + ] + ) + monkeypatch.setattr(_storage_sources, "utc_now", lambda: next(timestamps)) + proposal = service.create_action_proposal( + proposal_type="task", title="safe", source_item_id=item.id + ) + status_updated = service.update_action_proposal_status( + proposal.id, status="approved" + ) + fields_updated = service.update_action_proposal_fields( + proposal.id, title="changed" + ) + assert status_updated.updated_at > proposal.updated_at + assert fields_updated.updated_at > status_updated.updated_at + + +@pytest.mark.parametrize("secret", [LITERAL, ENCODED]) +def test_postgres_json_payload_sanitizes_nested_values_without_mutation( + secret: str, +) -> None: + payload = {"outer": [{"credential": secret}], "safe": "kept"} + original = copy.deepcopy(payload) + sanitized = PostgresStore._json_payload(payload) + assert payload == original + assert secret not in repr(sanitized) + assert sanitized["safe"] == "kept" + + +def test_postgres_json_payload_sanitizes_json_string_and_literal() -> None: + encoded_json = json.dumps({"credential": ENCODED, "safe": "kept"}) + assert ENCODED not in repr(PostgresStore._json_payload(encoded_json)) + assert LITERAL not in repr(PostgresStore._json_payload(LITERAL)) + + +@pytest.mark.parametrize( + ("method", "kwargs"), + [ + ("update_action_proposal_status", {"status": "exported", "exported_at": ENCODED}), + ("update_action_proposal_fields", {"suggested_due_at": ENCODED}), + ], +) +def test_postgres_update_metadata_rejects_before_connect( + monkeypatch: pytest.MonkeyPatch, method: str, kwargs: dict[str, Any] +) -> None: + store = object.__new__(PostgresStore) + monkeypatch.setattr(store, "_deny_unsupported_team_surface", lambda _name: None) + monkeypatch.setattr(store, "_load_psycopg", lambda: (None, None, lambda value: value)) + monkeypatch.setattr( + store, "connect", lambda: pytest.fail("unsafe metadata reached SQL") + ) + with pytest.raises(ValueError): + getattr(store, method)(1, **kwargs) + + +class _UpsertPreflightCursor: + def __init__(self, row: dict[str, Any]) -> None: + self.row = row + self.executed: list[str] = [] + + def __enter__(self): + return self + + def __exit__(self, *args: object) -> None: + return None + + def execute(self, sql: str, _params: object) -> None: + self.executed.append(sql) + if not sql.lstrip().startswith("SELECT"): + pytest.fail("unsafe existing row reached mutation SQL") + + def fetchone(self) -> dict[str, Any]: + return dict(self.row) + + +@pytest.mark.parametrize( + ("method", "row", "kwargs"), + [ + ( + "upsert_external_source", + { + "id": 1, "source_type": "whatsapp", "display_name": "primary", + "config_json": {"credential": ENCODED}, "created_at": "safe", "updated_at": "safe", + }, + {"source_type": "whatsapp", "display_name": "primary", "config_json": {"mode": "changed"}}, + ), + ( + "upsert_source_item", + { + "id": 1, "source_id": 1, "source_item_id": "message-1", "item_type": "message", + "chat_id": None, "sender_id": None, "sender_name": None, "occurred_at": None, + "text": "safe", "payload_json": None, "content_hash": None, "sensitivity": ENCODED, + "created_at": "safe", "updated_at": "safe", + }, + {"source_id": 1, "source_item_id": "message-1", "item_type": "message", "text": "changed"}, + ), + ], +) +def test_postgres_upserts_preflight_full_existing_row_before_mutation( + monkeypatch: pytest.MonkeyPatch, method: str, row: dict[str, Any], kwargs: dict[str, Any] +) -> None: + cursor = _UpsertPreflightCursor(row) + store = object.__new__(PostgresStore) + monkeypatch.setattr(store, "_deny_unsupported_team_surface", lambda _name: None) + monkeypatch.setattr(store, "_load_psycopg", lambda: (None, None, lambda value: value)) + monkeypatch.setattr(store, "connect", lambda: _ClaimConnection(cursor)) + monkeypatch.setattr(store, "_insert_event_row", lambda *_args, **_kwargs: pytest.fail("unsafe row emitted event")) + with pytest.raises(ValueError, match="unsafe"): + getattr(store, method)(**kwargs) + assert len(cursor.executed) == 1 + assert "SELECT *" in cursor.executed[0] + + +@pytest.mark.parametrize( + "method", + [ + "list_evidence_items", + "list_media_retries", + "list_action_proposals", + "claim_pending_media_retries", + ], +) +def test_sqlite_postgres_zero_limit_behavior_parity( + service: MemoryService, monkeypatch: pytest.MonkeyPatch, method: str +) -> None: + postgres = object.__new__(PostgresStore) + monkeypatch.setattr(postgres, "_deny_unsupported_team_surface", lambda _name: None) + monkeypatch.setattr(postgres, "connect", lambda: pytest.fail("zero limit reached SQL")) + assert getattr(service.store, method)(limit=0) == [] + assert getattr(postgres, method)(limit=0) == [] + + +class _PagedListCursor: + def __init__(self, rows: list[dict[str, Any]]) -> None: + self.rows = rows + self.result: list[dict[str, Any]] = [] + + def __enter__(self): + return self + + def __exit__(self, *args: object) -> None: + return None + + def execute(self, sql: str, params: list[Any]) -> None: + ascending = "ORDER BY created_at ASC" in sql + time_key = "created_at" if ascending else "updated_at" + ordered = sorted( + self.rows, + key=lambda row: (row[time_key], row["id"]), + reverse=not ascending, + ) + if len(params) > 1: + cursor_time, cursor_id = params[-4], int(params[-2]) + if ascending: + ordered = [row for row in ordered if (row[time_key], row["id"]) > (cursor_time, cursor_id)] + else: + ordered = [row for row in ordered if (row[time_key], row["id"]) < (cursor_time, cursor_id)] + self.result = ordered[: int(params[-1])] + + def fetchall(self) -> list[dict[str, Any]]: + return list(self.result) + + +def _postgres_list_store(monkeypatch: pytest.MonkeyPatch, rows: list[dict[str, Any]]) -> PostgresStore: + store = object.__new__(PostgresStore) + cursor = _PagedListCursor(rows) + monkeypatch.setattr(store, "_deny_unsupported_team_surface", lambda _name: None) + monkeypatch.setattr(store, "connect", lambda: _ClaimConnection(cursor)) + return store + + +def test_postgres_lists_count_safe_rows_after_unsafe_prefix( + monkeypatch: pytest.MonkeyPatch, +) -> None: + common = {"id": 1, "created_at": "2026-01-03T00:00:00Z", "updated_at": "2026-01-03T00:00:00Z"} + evidence = [ + {**common, "id": index + 1, "source_item_id": 1, "evidence_type": "text", "text": ENCODED, + "media_path": None, "provider": None, "confidence": None, "payload_json": None, + "sensitivity": None, + "created_at": "2026-01-01T00:00:00Z"} + for index in range(251) + ] + [ + {**common, "id": index + 252, "source_item_id": 1, "evidence_type": "text", "text": f"safe-{index}", + "media_path": None, "provider": None, "confidence": None, "payload_json": None, + "sensitivity": None, + "created_at": "2026-01-02T00:00:00Z"} + for index in range(2) + ] + retry_base = {**common, "source_item_id": 1, "chat_id": None, "media_type": None, "media_path": None, + "media_url": None, "status": "failed", "attempt_count": 0, "last_http_status": None, + "next_attempt_time": None} + retries = [{**retry_base, "id": index + 1, "media_key": f"unsafe-{index}", "last_error": ENCODED, + "updated_at": "2026-01-03T00:00:00Z"} for index in range(251)] + [ + {**retry_base, "id": index + 252, "media_key": f"safe-{index}", "last_error": None, + "updated_at": f"2026-01-02T00:00:0{index}Z"} for index in range(2) + ] + action_base = {**common, "proposal_type": "task", "description": None, "source_item_id": None, + "evidence_item_id": None, "claim_id": None, "suggested_due_at": None, "destination": "manual", + "status": "candidate", "confidence": 0.5, "payload_json": None, "exported_at": None, + "external_ref": None, "idempotency_key": None} + actions = [{**action_base, "id": index + 1, "title": ENCODED, + "updated_at": "2026-01-03T00:00:00Z"} for index in range(251)] + [ + {**action_base, "id": index + 252, "title": f"safe-{index}", + "updated_at": f"2026-01-02T00:00:0{index}Z"} for index in range(2) + ] + assert [row.id for row in _postgres_list_store(monkeypatch, evidence).list_evidence_items(limit=2)] == [252, 253] + assert [row.id for row in _postgres_list_store(monkeypatch, retries).list_media_retries(limit=2)] == [253, 252] + assert [row.id for row in _postgres_list_store(monkeypatch, actions).list_action_proposals(limit=2)] == [253, 252] + + +def test_unsafe_legacy_mutation_and_coalesce_paths_fail_without_changes( + service: MemoryService, +) -> None: + source, item = _seed_parent(service) + evidence = service.add_evidence_item( + source_item_id=item.id, evidence_type="message_text", text="safe" + ) + proposal = service.create_action_proposal( + proposal_type="task", title="safe", evidence_item_id=evidence.id + ) + retry = service.enqueue_media_retry(source_item_id=item.id, media_key="safe-key") + with service.store.connect() as conn: + conn.execute("UPDATE source_items SET text = ? WHERE id = ?", (ENCODED, item.id)) + conn.execute("UPDATE evidence_items SET text = ? WHERE id = ?", (ENCODED, evidence.id)) + conn.execute("UPDATE action_proposals SET description = ? WHERE id = ?", (ENCODED, proposal.id)) + conn.execute("UPDATE media_retry_queue SET last_error = ? WHERE id = ?", (ENCODED, retry.id)) + conn.commit() + + def snapshot(table: str, row_id: int) -> tuple[dict[str, Any], int]: + with service.store.connect() as conn: + row = dict(conn.execute(f"SELECT * FROM {table} WHERE id = ?", (row_id,)).fetchone()) + event_count = int(conn.execute("SELECT COUNT(*) FROM events").fetchone()[0]) + return row, event_count + + cases = [ + ("source_items", item.id, lambda: service.set_source_item_sensitivity(item.id, "high")), + ("evidence_items", evidence.id, lambda: service.set_evidence_item_sensitivity(evidence.id, "high")), + ("action_proposals", proposal.id, lambda: service.update_action_proposal_status(proposal.id, status="approved")), + ("action_proposals", proposal.id, lambda: service.update_action_proposal_fields(proposal.id, title="changed")), + ("media_retry_queue", retry.id, lambda: service.enqueue_media_retry(source_item_id=item.id, media_key="safe-key", chat_id="changed")), + ("media_retry_queue", retry.id, lambda: service.record_media_retry_outcome(retry.id, status="failed")), + ] + for table, row_id, operation in cases: + before = snapshot(table, row_id) + with pytest.raises(ValueError, match="unsafe"): + operation() + assert snapshot(table, row_id) == before + + +class _ClaimCursor: + def __init__(self, rows: list[dict[str, Any]]) -> None: + self.rows = rows + self.result: list[dict[str, Any]] = [] + self.executed: list[tuple[str, tuple[Any, ...]]] = [] + + def __enter__(self): + return self + + def __exit__(self, *args: object) -> None: + return None + + def execute(self, sql: str, params: tuple[Any, ...]) -> None: + self.executed.append((sql, params)) + if sql.lstrip().startswith("SELECT"): + cursor_id = int(params[0]) + self.result = [row for row in self.rows if int(row["id"]) > cursor_id][ + : int(params[-1]) + ] + return + safe_ids = set(params[1:]) + self.result = [] + for row in self.rows: + if row["id"] in safe_ids: + row.update(status="retrying", attempt_count=row["attempt_count"] + 1) + self.result.append(dict(row)) + + def fetchall(self) -> list[dict[str, Any]]: + return list(self.result) + + +class _ClaimConnection: + def __init__(self, cursor: _ClaimCursor) -> None: + self._cursor = cursor + + def __enter__(self): + return self + + def __exit__(self, *args: object) -> None: + return None + + def cursor(self) -> _ClaimCursor: + return self._cursor + + +def test_postgres_claim_filters_before_update_and_event(monkeypatch: pytest.MonkeyPatch) -> None: + safe = { + "id": 252, "source_item_id": 1, "media_key": "safe", "chat_id": None, + "media_type": None, "media_path": None, "media_url": None, "status": "pending", + "attempt_count": 0, "last_http_status": None, "last_error": None, + "next_attempt_time": None, "created_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:00:00Z", + } + unsafe_rows = [ + {**safe, "id": index + 1, "media_key": f"unsafe-{index}", "last_error": ENCODED} + for index in range(251) + ] + cursor = _ClaimCursor([*unsafe_rows, safe]) + store = object.__new__(PostgresStore) + monkeypatch.setattr(store, "_deny_unsupported_team_surface", lambda _name: None) + monkeypatch.setattr(store, "connect", lambda: _ClaimConnection(cursor)) + events: list[dict[str, Any]] = [] + monkeypatch.setattr(store, "_insert_event_row", lambda _conn, **kwargs: events.append(kwargs)) + + claimed = store.claim_pending_media_retries(limit=1) + + assert [row.id for row in claimed] == [252] + assert all(row["status"] == "pending" and row["attempt_count"] == 0 for row in unsafe_rows) + update_params = cursor.executed[-1][1] + assert update_params[1:] == (252,) + assert [event["payload"]["retry_id"] for event in events] == [252] + assert "FOR UPDATE SKIP LOCKED" in cursor.executed[0][0] + + +def test_sqlite_and_postgres_expose_the_same_atlas_gateway_methods() -> None: + names = { + name + for name, value in inspect.getmembers(_SourceItemsMixin, inspect.isfunction) + if not name.startswith("_row_to_") + } + missing = sorted(name for name in names if not callable(getattr(PostgresStore, name, None))) + assert missing == [] From a3e38244de21ae23171207841b994158d0b7242f Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 18:52:37 -0300 Subject: [PATCH 20/55] fix(security): harden bridge persistence transport --- memorymaster/bridges/db_merge.py | 314 ++++++-- memorymaster/bridges/delta_sync.py | 179 +++-- memorymaster/bridges/persisted_envelope.py | 178 +++++ tests/test_bridge_persistence_sensitivity.py | 744 ++++++++++++++++++ ...st_delta_bridge_persistence_sensitivity.py | 371 +++++++++ tests/test_delta_sync_extra.py | 29 +- 6 files changed, 1713 insertions(+), 102 deletions(-) create mode 100644 memorymaster/bridges/persisted_envelope.py create mode 100644 tests/test_bridge_persistence_sensitivity.py create mode 100644 tests/test_delta_bridge_persistence_sensitivity.py diff --git a/memorymaster/bridges/db_merge.py b/memorymaster/bridges/db_merge.py index ede12ee5..cc509cf6 100644 --- a/memorymaster/bridges/db_merge.py +++ b/memorymaster/bridges/db_merge.py @@ -1,7 +1,8 @@ """Bidirectional DB merge — import claims from a remote memorymaster DB. Merges claims from a source DB into the local DB without duplicating. -Uses idempotency_key + text hash for dedup. Preserves both sides' claims. +Uses idempotency_key + text hash for dedup. Preserves both sides' claims while +redacting content and rejecting secret-shaped metadata before target writes. Usage: memorymaster merge-db --source /path/to/remote.db @@ -13,15 +14,34 @@ import hashlib import logging import sqlite3 +from collections.abc import Mapping, Sequence from datetime import datetime, timezone from pathlib import Path +from memorymaster.bridges.persisted_envelope import ( + claim_envelope_is_safe, + persisted_claim_id, + sanitize_claim_envelope, +) +from memorymaster.core.security import SensitiveMetadataError, validate_persisted_metadata from memorymaster.stores._storage_shared import connect_ro, open_conn from memorymaster.stores.store_factory import is_postgres_dsn logger = logging.getLogger(__name__) +def _claim_log_id(row: Mapping[str, object]) -> str: + claim_id = row.get("id") + return str(claim_id) if isinstance(claim_id, int) and not isinstance(claim_id, bool) else "unknown" + + +def _quote_identifier(identifier: str) -> str: + validate_persisted_metadata({"merge_identifier": identifier}) + if not identifier or "\x00" in identifier: + raise ValueError("Merge schema contains an invalid identifier.") + return '"' + identifier.replace('"', '""') + '"' + + def _text_hash(text: str) -> str: """Deterministic hash for claim dedup when no idempotency_key exists.""" return hashlib.sha256(text.strip().lower().encode("utf-8")).hexdigest()[:16] @@ -60,20 +80,80 @@ def _build_insert_values( return cols_to_insert, values -def _copy_claim_citations(src: sqlite3.Connection, tgt: sqlite3.Connection, old_id: int, new_id: int) -> None: - """Copy citations from source claim to target claim.""" +def _copy_claim_citations( + tgt: sqlite3.Connection, + new_id: int, + citations: Sequence[Mapping[str, object]], +) -> None: + """Copy preflighted citations into the target transaction.""" + for citation in citations: + tgt.execute( + "INSERT INTO citations " + "(claim_id, source, locator, excerpt, created_at) VALUES (?, ?, ?, ?, ?)", + ( + new_id, + citation["source"], + citation.get("locator"), + citation.get("excerpt"), + citation["created_at"], + ), + ) + + +def _source_citations( + src: sqlite3.Connection, claim_id: int +) -> list[dict[str, object]]: try: - cites = src.execute( - "SELECT source, locator, excerpt, created_at FROM citations WHERE claim_id = ?", - (old_id,), + rows = src.execute( + "SELECT * FROM citations WHERE claim_id = ?", + (claim_id,), ).fetchall() - for cite in cites: - tgt.execute( - "INSERT INTO citations (claim_id, source, locator, excerpt, created_at) VALUES (?, ?, ?, ?, ?)", - (new_id, cite["source"], cite["locator"], cite["excerpt"], cite["created_at"]), - ) except sqlite3.OperationalError: - pass # citations table might differ + return [] + return [dict(row) for row in rows] + + +def _unsafe_citation_claim_ids( + conn: sqlite3.Connection, +) -> set[int] | None: + """Return claim ids with unsafe citations; None means fail closed globally.""" + try: + rows = conn.execute("SELECT * FROM citations") + except sqlite3.OperationalError: + return set() + unsafe: set[int] = set() + for raw_row in rows: + citation = dict(raw_row) + try: + claim_id = persisted_claim_id({"id": citation.get("claim_id")}) + except SensitiveMetadataError: + return None + if not claim_envelope_is_safe({"text": "safe"}, [citation]): + unsafe.add(claim_id) + return unsafe + + +def _target_claim_envelope_is_safe( + tgt: sqlite3.Connection, target_claim: Mapping[str, object] +) -> bool: + """Revalidate one target claim and its citations in the active snapshot.""" + try: + claim_id = persisted_claim_id(target_claim) + citations = tgt.execute( + "SELECT * FROM citations WHERE claim_id = ?", + (claim_id,), + ).fetchall() + except (SensitiveMetadataError, sqlite3.Error): + return False + return claim_envelope_is_safe( + target_claim, + [dict(citation) for citation in citations], + ) + + +def _sqlite_error_label(exc: sqlite3.Error) -> str: + label = getattr(exc, "sqlite_errorname", None) + return str(label) if isinstance(label, str) and label.startswith("SQLITE_") else "SQLITE_ERROR" def _parse_timestamp(value: object) -> datetime: @@ -253,7 +333,19 @@ def _find_existing_target_claim( if claim_id is None: return None row = tgt.execute("SELECT * FROM claims WHERE id = ?", (claim_id,)).fetchone() - return dict(row) if row else None + if row: + candidate = dict(row) + candidate_text = candidate.get("text") + if ( + isinstance(candidate_text, str) + and _identity_namespace( + candidate, + available_columns=target_cols, + ) + == namespace + and _text_hash(candidate_text) == text_hash + ): + return candidate for row in tgt.execute( f"SELECT * FROM claims WHERE {identity_sql}", identity_params @@ -338,20 +430,24 @@ def _apply_conflict_resolution( def _insert_claim_into_target( - row: sqlite3.Row, + row: Mapping[str, object], common_cols: list[str], ikey: str, text: str, - src: sqlite3.Connection, tgt: sqlite3.Connection, + citations: Sequence[Mapping[str, object]], ) -> int | None: """Insert a single claim into target DB and copy citations. Returns new id if successful.""" - src_id = row["id"] if "id" in row.keys() else "?" + src_id = _claim_log_id(row) text_hash = _text_hash(text) + savepoint = "merge_claim_insert" try: + if not tgt.in_transaction: + tgt.execute("BEGIN") + tgt.execute(f"SAVEPOINT {savepoint}") cols_to_insert, values = _build_insert_values(row, common_cols, ikey) placeholders = ",".join("?" for _ in cols_to_insert) - col_names = ",".join(cols_to_insert) + col_names = ",".join(_quote_identifier(column) for column in cols_to_insert) tgt.execute( f"INSERT INTO claims ({col_names}) VALUES ({placeholders})", @@ -359,22 +455,31 @@ def _insert_claim_into_target( ) new_id = tgt.execute("SELECT last_insert_rowid()").fetchone()[0] - _copy_claim_citations(src, tgt, row["id"], new_id) + _copy_claim_citations(tgt, new_id, citations) + tgt.execute(f"RELEASE SAVEPOINT {savepoint}") return int(new_id) except sqlite3.IntegrityError as exc: + tgt.execute(f"ROLLBACK TO SAVEPOINT {savepoint}") + tgt.execute(f"RELEASE SAVEPOINT {savepoint}") # A genuine UNIQUE/constraint collision (e.g. idempotency_key already # present after another path inserted it). Log enough to trace which # claim was dropped — do NOT swallow it as a generic "merge error". logger.warning( - "Constraint collision merging claim src_id=%s text_hash=%s: %s", - src_id, text_hash, exc, + "Constraint collision merging claim src_id=%s text_hash=%s error=%s", + src_id, + text_hash, + _sqlite_error_label(exc), ) return None except sqlite3.OperationalError as exc: + tgt.execute(f"ROLLBACK TO SAVEPOINT {savepoint}") + tgt.execute(f"RELEASE SAVEPOINT {savepoint}") # Schema/operational problem (missing column, locked DB after retries). logger.warning( - "Operational error merging claim src_id=%s text_hash=%s: %s", - src_id, text_hash, exc, + "Operational error merging claim src_id=%s text_hash=%s error=%s", + src_id, + text_hash, + _sqlite_error_label(exc), ) return None @@ -432,7 +537,8 @@ def merge_databases(target_db: str, source_db: str) -> dict[str, int]: """Merge claims from source_db into target_db. Skips claims that already exist (matched by idempotency_key or text hash). - Copies citations for newly merged claims. + Copies preflighted citations for newly merged claims. Unsafe source metadata + counts as an error; unsafe legacy target rows are excluded from mutation. Returns dict with: scanned, merged, skipped, errors """ @@ -452,6 +558,7 @@ def merge_databases(target_db: str, source_db: str) -> dict[str, int]: tgt = _open_target(target_db) try: + src.execute("BEGIN") # Refuse to import rows the target's CHECK constraints may forbid. _check_schema_compatibility(src, tgt) tgt_cols = _target_columns(tgt) @@ -464,13 +571,22 @@ def merge_databases(target_db: str, source_db: str) -> dict[str, int]: existing_hashes: set[tuple[IdentityNamespace, str]] = set() hash_to_id: dict[tuple[IdentityNamespace, str], int] = {} - identity_columns = [ - column - for column in ("tenant_id", "scope", "visibility", "source_agent") - if column in tgt_cols - ] - select_columns = ", ".join(("id", "idempotency_key", "text", *identity_columns)) - for row in tgt.execute(f"SELECT {select_columns} FROM claims").fetchall(): + unsafe_target_citations = _unsafe_citation_claim_ids(tgt) + unsafe_target_rows = 0 + for raw_row in tgt.execute("SELECT * FROM claims").fetchall(): + row = dict(raw_row) + try: + target_id = persisted_claim_id(row) + except SensitiveMetadataError: + unsafe_target_rows += 1 + continue + if ( + unsafe_target_citations is None + or target_id in unsafe_target_citations + or not claim_envelope_is_safe(row) + ): + unsafe_target_rows += 1 + continue namespace = _identity_namespace( row, available_columns=tgt_cols, @@ -480,7 +596,12 @@ def merge_databases(target_db: str, source_db: str) -> dict[str, int]: existing_keys.add((namespace, str(row["idempotency_key"]))) thash = _text_hash(row["text"]) existing_hashes.add((namespace, thash)) - hash_to_id.setdefault((namespace, thash), int(row["id"])) + hash_to_id.setdefault((namespace, thash), target_id) + if unsafe_target_rows: + logger.warning( + "merge: quarantined %d unsafe legacy target rows from mutation", + unsafe_target_rows, + ) # Get all columns from source claims table src_cols = [col[1] for col in src.execute("PRAGMA table_info(claims)").fetchall()] @@ -492,8 +613,24 @@ def merge_databases(target_db: str, source_db: str) -> dict[str, int]: batch_size = 200 pending = 0 - for row in source_claims: + for source_row in source_claims: stats["scanned"] += 1 + raw_row = dict(source_row) + try: + envelope = sanitize_claim_envelope( + raw_row, + _source_citations(src, persisted_claim_id(raw_row)), + ) + except SensitiveMetadataError as exc: + logger.warning( + "merge: rejected source claim id=%s unsafe field=%s findings=%s", + _claim_log_id(raw_row), + exc.field, + ",".join(exc.findings), + ) + stats["errors"] += 1 + continue + row = envelope.row ikey = row["idempotency_key"] if "idempotency_key" in row.keys() else None text = row["text"] namespace = _identity_namespace( @@ -506,28 +643,111 @@ def merge_databases(target_db: str, source_db: str) -> dict[str, int]: # Reconcile duplicates deterministically instead of letting merge order win. if (identity_key and identity_key in existing_keys) or hash_key in existing_hashes: - existing_claim = _find_existing_target_claim( - tgt, - ikey, - text, - namespace, - tgt_cols, - hash_to_id, - ) - if existing_claim: - _reconcile_existing_claim(tgt, row, existing_claim, tgt_cols) - stats["skipped"] += 1 - continue + claim_savepoint = "merge_duplicate_envelope" + if not tgt.in_transaction: + tgt.execute("BEGIN") + tgt.execute(f"SAVEPOINT {claim_savepoint}") + existing_claim = None + try: + existing_claim = _find_existing_target_claim( + tgt, + ikey, + text, + namespace, + tgt_cols, + hash_to_id, + ) + if ( + existing_claim + and not envelope.findings + and _target_claim_envelope_is_safe(tgt, existing_claim) + ): + _reconcile_existing_claim(tgt, row, existing_claim, tgt_cols) + except sqlite3.Error as exc: + tgt.execute(f"ROLLBACK TO SAVEPOINT {claim_savepoint}") + tgt.execute(f"RELEASE SAVEPOINT {claim_savepoint}") + logger.warning( + "Duplicate reconciliation rejected src_id=%s error=%s", + _claim_log_id(raw_row), + _sqlite_error_label(exc), + ) + stats["errors"] += 1 + pending += 1 + if pending >= batch_size: + tgt.commit() + pending = 0 + continue + else: + tgt.execute(f"RELEASE SAVEPOINT {claim_savepoint}") + if existing_claim is not None: + stats["skipped"] += 1 + pending += 1 + if pending >= batch_size: + tgt.commit() + pending = 0 + continue + + # A writer changed the cached row's identity after a prior batch + # committed. Drop the stale fingerprints and import this now-new + # source claim instead of mutating the unrelated target row or + # silently skipping the source. + if identity_key is not None: + existing_keys.discard(identity_key) + existing_hashes.discard(hash_key) + hash_to_id.pop(hash_key, None) # Build idempotency key if missing if not ikey: ikey = f"merge-{_text_hash(text)}" # Insert into target - conflicts = _find_conflicting_target_claims(tgt, row, tgt_cols) - new_id = _insert_claim_into_target(row, common_cols, ikey, text, src, tgt) + row = envelope.row + if envelope.findings: + row = { + **row, + **({"status": "candidate"} if "status" in row else {}), + **({"pinned": 0} if "pinned" in row else {}), + } + claim_savepoint = "merge_claim_envelope" + if not tgt.in_transaction: + tgt.execute("BEGIN") + tgt.execute(f"SAVEPOINT {claim_savepoint}") + try: + conflicts = [] + if not envelope.findings: + conflicts = [ + conflict + for conflict in _find_conflicting_target_claims(tgt, row, tgt_cols) + if _target_claim_envelope_is_safe(tgt, conflict) + ] + new_id = _insert_claim_into_target( + row, + common_cols, + str(ikey), + str(text), + tgt, + envelope.citations, + ) + if new_id is not None: + _apply_conflict_resolution(tgt, row, new_id, tgt_cols, conflicts) + except sqlite3.Error as exc: + tgt.execute(f"ROLLBACK TO SAVEPOINT {claim_savepoint}") + tgt.execute(f"RELEASE SAVEPOINT {claim_savepoint}") + logger.warning( + "Merge claim transaction rejected src_id=%s error=%s", + _claim_log_id(raw_row), + _sqlite_error_label(exc), + ) + new_id = None + else: + tgt.execute(f"RELEASE SAVEPOINT {claim_savepoint}") if new_id is not None: - _apply_conflict_resolution(tgt, row, new_id, tgt_cols, conflicts) + if envelope.findings: + logger.warning( + "merge: redacted source claim id=%s findings=%s", + _claim_log_id(raw_row), + ",".join(envelope.findings), + ) existing_keys.add((namespace, str(ikey))) thash = _text_hash(text) existing_hashes.add((namespace, thash)) diff --git a/memorymaster/bridges/delta_sync.py b/memorymaster/bridges/delta_sync.py index 63bddc77..b91fc08a 100644 --- a/memorymaster/bridges/delta_sync.py +++ b/memorymaster/bridges/delta_sync.py @@ -20,13 +20,23 @@ The whole-DB file never crosses the network — which also removes the SQLite-over-network-mount corruption risk entirely. + +Every transported claim/citation crosses the canonical persisted sensitivity +envelope. Content is redacted and secret-shaped metadata is omitted fail-closed; +the source database is read-only and remains untouched. """ from __future__ import annotations import logging +import re import sqlite3 from pathlib import Path +from memorymaster.bridges.persisted_envelope import ( + persisted_claim_id, + sanitize_claim_envelope, +) +from memorymaster.core.security import SensitiveMetadataError, validate_persisted_metadata from memorymaster.stores._storage_shared import connect_ro, open_conn from memorymaster.stores.store_factory import is_postgres_dsn @@ -39,16 +49,84 @@ # ignored on the merge side. _DELTA_TABLES = ("claims", "citations") +_REQUIRED_DELTA_COLUMNS = { + "claims": frozenset({"id", "text", "updated_at"}), + "citations": frozenset({"claim_id", "source", "created_at"}), +} + + +def _quote_identifier(identifier: str) -> str: + validate_persisted_metadata({"delta_identifier": identifier}) + if not identifier or "\x00" in identifier: + raise ValueError("Delta schema contains an invalid identifier.") + return '"' + identifier.replace('"', '""') + '"' + + +def _sqlite_affinity(declared_type: object) -> str: + declared = str(declared_type or "").upper() + if "INT" in declared: + return "INTEGER" + if any(token in declared for token in ("CHAR", "CLOB", "TEXT")): + return "TEXT" + if any(token in declared for token in ("REAL", "FLOA", "DOUB")): + return "REAL" + if not declared or "BLOB" in declared: + return "BLOB" + return "NUMERIC" + + +def _paths_alias(source: Path, output: Path) -> bool: + source_resolved = source.resolve(strict=True) + output_resolved = output.resolve(strict=False) + if source_resolved == output_resolved: + return True + if output.exists(): + try: + return source.samefile(output) + except OSError: + return False + return False + def _copy_table_ddl(src: sqlite3.Connection, out: sqlite3.Connection, table: str) -> None: - """Copy a table's CREATE statement verbatim from src into out.""" + """Create a value-only transport table without executing source SQL.""" row = src.execute( "SELECT sql FROM sqlite_master WHERE type='table' AND name=?", (table,), ).fetchone() if row is None or not row[0]: raise ValueError(f"source DB has no '{table}' table — not a memorymaster DB?") - out.execute(row[0]) + validate_persisted_metadata( + {"delta_table_name": table, "delta_table_ddl": row[0]} + ) + if re.match(r"^\s*CREATE\s+TABLE\b", str(row[0]), re.IGNORECASE) is None: + raise ValueError("Source table does not have a canonical transport schema.") + quoted_table = _quote_identifier(table) + columns = src.execute(f"PRAGMA table_info({quoted_table})").fetchall() + names = {str(column[1]) for column in columns} + if not columns or not _REQUIRED_DELTA_COLUMNS[table].issubset(names): + raise ValueError("Source table does not have a canonical transport schema.") + definitions = ", ".join( + f"{_quote_identifier(str(column[1]))} {_sqlite_affinity(column[2])}" + for column in columns + ) + out.execute(f"CREATE TABLE {quoted_table} ({definitions})") + + +def _load_citations_by_claim( + src: sqlite3.Connection, claim_ids: list[int] +) -> dict[int, list[dict[str, object]]]: + by_claim: dict[int, list[dict[str, object]]] = {} + for start in range(0, len(claim_ids), 900): + batch = claim_ids[start : start + 900] + qmarks = ",".join("?" for _ in batch) + rows = src.execute( + f"SELECT * FROM citations WHERE claim_id IN ({qmarks})", + batch, + ).fetchall() + for row in rows: + by_claim.setdefault(int(row["claim_id"]), []).append(dict(row)) + return by_claim def export_delta( @@ -67,10 +145,10 @@ def export_delta( exists. Returns: - dict with ``exported`` (claim count), ``citations`` (citation count), - ``since`` (echoed watermark), and ``max_updated_at`` (the newest - ``updated_at`` seen — use this as the next watermark; None when the - delta is empty). + dict with ``exported``/``citations`` counts, ``rejected`` metadata + rejections, ``redacted`` claim envelopes, ``since`` (echoed watermark), + and ``max_updated_at`` (the newest safely exported ``updated_at``; + None when the delta is empty). Raises: FileNotFoundError: source DB missing. @@ -81,10 +159,14 @@ def export_delta( "export-delta supports SQLite paths only; raw Postgres team deltas are disabled." ) - source_db = str(source_db) + source_path = Path(str(source_db)) output_path = Path(output_path) - if not Path(source_db).exists(): - raise FileNotFoundError(f"Source DB not found: {source_db}") + if not source_path.exists(): + raise FileNotFoundError(f"Source DB not found: {source_path}") + watermark = since.strip() + validate_persisted_metadata({"delta_since": watermark}) + if _paths_alias(source_path, output_path): + raise ValueError("Delta output must not alias the source database.") # Fresh output file every time — a stale delta would merge old rows again # (harmless thanks to idempotent merge, but wasteful). @@ -94,17 +176,14 @@ def export_delta( # Source is read-only (connect_ro takes no lock on the live DB); the # fresh delta file gets the uniform writer envelope. - src = connect_ro(source_db) + src = connect_ro(str(source_path)) out = open_conn(output_path) - # The delta is a TRANSPORT file, not a live DB. The claims DDL is copied - # verbatim (incl. supersedes/replaced_by FKs to claims.id), and a claim in - # the window may legitimately reference a claim OUTSIDE it — with FK - # enforcement on, one such row kills the whole export (this silently broke - # the Windows->Hermes sync for 3 weeks: nightly 'FOREIGN KEY constraint - # failed' since 2026-06-10). Integrity is re-established by the idempotent - # merge into the target DB, which already holds (or dedups) the parents. + # The delta is a value-only transport file, not a live DB. Its tables are + # synthesized from source column names/affinities; untrusted source DDL, + # constraints, triggers, and cross-window foreign keys are never executed. out.execute("PRAGMA foreign_keys=OFF") try: + src.execute("BEGIN") for table in _DELTA_TABLES: _copy_table_ddl(src, out, table) @@ -117,7 +196,6 @@ def export_delta( # data loss. `>=` instead re-exports the boundary claim(s); the merge # engine is idempotent (dedups on idempotency_key + text-hash), so a # re-export costs nothing but a few rows. Safe beats clean. - watermark = since.strip() if watermark: claim_rows = src.execute( "SELECT * FROM claims WHERE updated_at >= ? ORDER BY updated_at", @@ -133,54 +211,67 @@ def export_delta( return { "exported": 0, "citations": 0, + "rejected": 0, + "redacted": 0, "since": watermark, "max_updated_at": None, } claim_cols = [c[1] for c in src.execute("PRAGMA table_info(claims)").fetchall()] placeholders = ",".join("?" for _ in claim_cols) - col_list = ",".join(claim_cols) + col_list = ",".join(_quote_identifier(str(column)) for column in claim_cols) insert_claim = f"INSERT INTO claims ({col_list}) VALUES ({placeholders})" - exported_ids: list[int] = [] - max_updated = "" - for row in claim_rows: - out.execute(insert_claim, tuple(row[c] for c in claim_cols)) - exported_ids.append(int(row["id"])) - updated = str(row["updated_at"] or "") - if updated > max_updated: - max_updated = updated - - # Citations for exactly the exported claims. claim_id linkage is - # preserved because we keep original claim ids in the delta file. + candidate_ids = [persisted_claim_id(dict(row)) for row in claim_rows] + citations_by_claim = _load_citations_by_claim(src, candidate_ids) cit_cols = [c[1] for c in src.execute("PRAGMA table_info(citations)").fetchall()] cit_placeholders = ",".join("?" for _ in cit_cols) - cit_col_list = ",".join(cit_cols) + cit_col_list = ",".join(_quote_identifier(str(column)) for column in cit_cols) insert_cit = f"INSERT INTO citations ({cit_col_list}) VALUES ({cit_placeholders})" + exported = 0 citation_count = 0 - # Chunk the IN clause — SQLite caps host parameters at 999. - for start in range(0, len(exported_ids), 900): - batch = exported_ids[start : start + 900] - qmarks = ",".join("?" for _ in batch) - cit_rows = src.execute( - f"SELECT * FROM citations WHERE claim_id IN ({qmarks})", - batch, - ).fetchall() - for cit in cit_rows: - out.execute(insert_cit, tuple(cit[c] for c in cit_cols)) + rejected = 0 + redacted = 0 + max_updated = "" + for row in claim_rows: + claim_id = persisted_claim_id(dict(row)) + try: + envelope = sanitize_claim_envelope( + dict(row), citations_by_claim.get(claim_id, []) + ) + except SensitiveMetadataError as exc: + rejected += 1 + logger.warning( + "export_delta: rejected claim id=%d unsafe field=%s findings=%s", + claim_id, + exc.field, + ",".join(exc.findings), + ) + continue + out.execute(insert_claim, tuple(envelope.row[c] for c in claim_cols)) + for citation in envelope.citations: + out.execute(insert_cit, tuple(citation[c] for c in cit_cols)) citation_count += 1 + exported += 1 + redacted += int(bool(envelope.findings)) + updated = str(envelope.row.get("updated_at") or "") + if updated > max_updated: + max_updated = updated out.commit() logger.info( - "export_delta: %d claims, %d citations since %r", - len(exported_ids), + "export_delta: %d claims, %d citations, %d rejected since %r", + exported, citation_count, + rejected, watermark or "(full)", ) return { - "exported": len(exported_ids), + "exported": exported, "citations": citation_count, + "rejected": rejected, + "redacted": redacted, "since": watermark, "max_updated_at": max_updated or None, } diff --git a/memorymaster/bridges/persisted_envelope.py b/memorymaster/bridges/persisted_envelope.py new file mode 100644 index 00000000..f4505952 --- /dev/null +++ b/memorymaster/bridges/persisted_envelope.py @@ -0,0 +1,178 @@ +"""Canonical sensitivity envelope for legacy claim/citation row transport.""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass + +from memorymaster.core.models import CitationInput +from memorymaster.core.security import ( + SensitiveMetadataError, + sanitize_claim_input, + sanitize_persisted_text, + validate_persisted_metadata, +) + + +_CLAIM_CONTENT_FIELDS = frozenset( + {"text", "normalized_text", "subject", "predicate", "object_value"} +) +_CITATION_CONTENT_FIELDS = frozenset({"excerpt"}) + + +@dataclass(frozen=True, slots=True) +class SanitizedClaimEnvelope: + row: dict[str, object] + citations: tuple[dict[str, object], ...] + findings: tuple[str, ...] + + +def persisted_claim_id(row: Mapping[str, object]) -> int: + """Return a numeric claim id or fail without echoing an unsafe value.""" + value = row.get("id") + if isinstance(value, int) and not isinstance(value, bool): + return value + if isinstance(value, str) and value.isascii() and value.isdecimal(): + return int(value) + raise SensitiveMetadataError("claim_id", ["invalid_identifier"]) + + +def _optional_text(value: object) -> str | None: + return None if value is None else str(value) + + +def _citation_inputs( + citations: Sequence[Mapping[str, object]], +) -> list[CitationInput]: + return [ + CitationInput( + source=str(citation.get("source") or ""), + locator=_optional_text(citation.get("locator")), + excerpt=_optional_text(citation.get("excerpt")), + ) + for citation in citations + ] + + +def _contains_binary(value: object) -> bool: + if isinstance(value, (bytes, bytearray, memoryview)): + return True + if isinstance(value, Mapping): + return any( + _contains_binary(key) or _contains_binary(nested) + for key, nested in value.items() + ) + if isinstance(value, (list, tuple, set, frozenset)): + return any(_contains_binary(nested) for nested in value) + return False + + +def _metadata_fields( + prefix: str, + values: Mapping[str, object], + excluded: frozenset[str], +) -> dict[str, object]: + fields: dict[str, object] = {} + for index, (field, value) in enumerate(values.items()): + if field in excluded: + continue + validate_persisted_metadata({f"{prefix}_field_name": field}) + field_name = f"{prefix}_field_{index}" + if _contains_binary(value): + raise SensitiveMetadataError(field_name, ["binary_metadata"]) + fields[field_name] = value + return fields + + +def _validate_envelope_metadata( + row: Mapping[str, object], citations: Sequence[Mapping[str, object]] +) -> None: + validate_persisted_metadata(_metadata_fields("claim", row, _CLAIM_CONTENT_FIELDS)) + for citation in citations: + validate_persisted_metadata( + _metadata_fields("citation", citation, _CITATION_CONTENT_FIELDS) + ) + + +def _sanitize_claim_content( + row: Mapping[str, object], citations: list[CitationInput] +): + return sanitize_claim_input( + text=str(row.get("text") or ""), + object_value=_optional_text(row.get("object_value")), + citations=citations, + subject=_optional_text(row.get("subject")), + predicate=_optional_text(row.get("predicate")), + idempotency_key=_optional_text(row.get("idempotency_key")), + claim_type=_optional_text(row.get("claim_type")), + scope=_optional_text(row.get("scope")), + volatility=_optional_text(row.get("volatility")), + source_agent=_optional_text(row.get("source_agent")), + visibility=_optional_text(row.get("visibility")), + holder=_optional_text(row.get("holder")), + confidence=row.get("confidence"), + event_time=_optional_text(row.get("event_time")), + valid_from=_optional_text(row.get("valid_from")), + valid_until=_optional_text(row.get("valid_until")), + tenant_id=_optional_text(row.get("tenant_id")), + ) + + +def _has_redaction_marker( + row: Mapping[str, object], citations: Sequence[Mapping[str, object]] +) -> bool: + claim_values = (row.get(field) for field in _CLAIM_CONTENT_FIELDS) + citation_values = (citation.get("excerpt") for citation in citations) + return any( + isinstance(value, str) and "[REDACTED:" in value + for value in (*claim_values, *citation_values) + ) + + +def sanitize_claim_envelope( + row: Mapping[str, object], + citations: Sequence[Mapping[str, object]] = (), +) -> SanitizedClaimEnvelope: + """Return sanitized copies or reject secret-shaped metadata fail-closed.""" + raw_row = dict(row) + raw_citations = tuple(dict(citation) for citation in citations) + _validate_envelope_metadata(raw_row, raw_citations) + citation_inputs = _citation_inputs(raw_citations) + sanitized = _sanitize_claim_content(raw_row, citation_inputs) + normalized = _optional_text(raw_row.get("normalized_text")) + normalized_findings: list[str] = [] + if normalized is not None: + normalized, normalized_findings = sanitize_persisted_text(normalized) + safe_row = { + **raw_row, + "text": sanitized.text, + "subject": sanitized.subject, + "predicate": sanitized.predicate, + "object_value": sanitized.object_value, + "normalized_text": normalized, + } + safe_citations = tuple( + { + **raw, + "source": safe.source, + "locator": safe.locator, + "excerpt": safe.excerpt, + } + for raw, safe in zip(raw_citations, sanitized.citations) + ) + findings_set = set(sanitized.findings + normalized_findings) + if _has_redaction_marker(safe_row, safe_citations): + findings_set.add("redaction_marker") + findings = tuple(sorted(findings_set)) + return SanitizedClaimEnvelope(safe_row, safe_citations, findings) + + +def claim_envelope_is_safe( + row: Mapping[str, object], + citations: Sequence[Mapping[str, object]] = (), +) -> bool: + """Return False for legacy rows that would be rejected or redacted.""" + try: + return not sanitize_claim_envelope(row, citations).findings + except (SensitiveMetadataError, ValueError, TypeError): + return False diff --git a/tests/test_bridge_persistence_sensitivity.py b/tests/test_bridge_persistence_sensitivity.py new file mode 100644 index 00000000..392c51c0 --- /dev/null +++ b/tests/test_bridge_persistence_sensitivity.py @@ -0,0 +1,744 @@ +from __future__ import annotations + +import base64 +import logging +import sqlite3 +from pathlib import Path + +import pytest + +import memorymaster.bridges.db_merge as db_merge_module +from memorymaster.bridges.db_merge import merge_databases +from memorymaster.bridges.delta_sync import export_delta +from memorymaster.bridges.persisted_envelope import sanitize_claim_envelope +from memorymaster.core.models import CitationInput +from memorymaster.core.security import SensitiveMetadataError +from memorymaster.core.service import MemoryService + + +LITERAL = "OPENAI_API_KEY=sk-proj-FAKEbridgeEnvelope1234567890ABCD" +ENCODED = base64.b64encode(LITERAL.encode()).decode() + + +def _service(path: Path, workspace: Path) -> MemoryService: + service = MemoryService(path, workspace_root=workspace) + service.init_db() + return service + + +def _claim(service: MemoryService, text: str = "Safe bridge claim"): + return service.ingest( + text, + [CitationInput(source="test://bridge", locator="line-1", excerpt="safe")], + source_agent="bridge-test", + ) + + +def _update(path: Path, statement: str, values: tuple[object, ...]) -> None: + with sqlite3.connect(path) as conn: + conn.execute(statement, values) + + +def _column(path: Path, table: str, field: str) -> object: + with sqlite3.connect(path) as conn: + return conn.execute(f"SELECT {field} FROM {table}").fetchone()[0] + + +def _seed_duplicate_claims(path: Path, count: int, *, source: bool) -> None: + timestamp = "2030-01-01T00:00:00+00:00" if source else "2020-01-01T00:00:00+00:00" + confidence = 0.9 if source else 0.2 + rows = [ + (f"duplicate-{index}", f"duplicate-key-{index}", confidence, timestamp, timestamp) + for index in range(count) + ] + with sqlite3.connect(path) as conn: + conn.executemany( + """ + INSERT INTO claims + (text, idempotency_key, scope, volatility, status, confidence, + pinned, created_at, updated_at, visibility) + VALUES (?, ?, 'project', 'medium', 'candidate', ?, 0, ?, ?, 'public') + """, + rows, + ) + + +def test_bridge_envelope_rejects_secret_bearing_field_names_without_echo() -> None: + with pytest.raises(SensitiveMetadataError) as rejected: + sanitize_claim_envelope({"text": "safe", LITERAL: "safe"}) + + assert LITERAL not in str(rejected.value) + + +def test_merge_rejection_log_never_echoes_an_unsafe_legacy_identifier( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + source = tmp_path / "legacy-source.db" + target = tmp_path / "target.db" + _service(target, tmp_path) + with sqlite3.connect(source) as conn: + conn.executescript( + """ + CREATE TABLE claims ( + id TEXT PRIMARY KEY, + text TEXT NOT NULL, + idempotency_key TEXT, + scope TEXT NOT NULL, + status TEXT NOT NULL, + confidence REAL NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE TABLE citations ( + id INTEGER PRIMARY KEY, + claim_id TEXT NOT NULL, + source TEXT NOT NULL, + locator TEXT, + excerpt TEXT, + created_at TEXT NOT NULL + ); + """ + ) + conn.execute( + """ + INSERT INTO claims + (id, text, idempotency_key, scope, status, confidence, created_at, updated_at) + VALUES (?, 'safe', 'safe-key', 'project', 'candidate', 0.5, ?, ?) + """, + (LITERAL, "2026-07-11T00:00:00+00:00", "2026-07-11T00:00:00+00:00"), + ) + + with caplog.at_level(logging.WARNING): + stats = merge_databases(str(target), str(source)) + + assert stats["errors"] == 1 + assert LITERAL not in caplog.text + + +def test_merge_quotes_legacy_column_identifiers(tmp_path: Path) -> None: + source = tmp_path / "legacy-source.db" + target = tmp_path / "legacy-target.db" + schema = """ + CREATE TABLE claims ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + text TEXT NOT NULL, + idempotency_key TEXT, + scope TEXT NOT NULL, + status TEXT NOT NULL, + confidence REAL NOT NULL, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + "select" TEXT + ); + CREATE TABLE citations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + claim_id INTEGER NOT NULL, + source TEXT NOT NULL, + locator TEXT, + excerpt TEXT, + created_at TEXT NOT NULL + ); + """ + for path in (source, target): + with sqlite3.connect(path) as conn: + conn.executescript(schema) + with sqlite3.connect(source) as conn: + conn.execute( + """ + INSERT INTO claims + (text, idempotency_key, scope, status, confidence, + created_at, updated_at, "select") + VALUES ('safe', 'reserved-key', 'project', 'candidate', 0.5, ?, ?, 'kept') + """, + ("2026-07-11T00:00:00+00:00", "2026-07-11T00:00:00+00:00"), + ) + + stats = merge_databases(str(target), str(source)) + + assert stats["merged"] == 1 + assert _column(target, "claims", '"select"') == "kept" + + +@pytest.mark.parametrize("secret", [LITERAL, ENCODED]) +def test_merge_rejects_unsafe_claim_metadata_before_target_write( + tmp_path: Path, secret: str +) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + source_service = _service(source, tmp_path) + _service(target, tmp_path) + claim = _claim(source_service) + _update( + source, + "UPDATE claims SET idempotency_key = ? WHERE id = ?", + (secret, claim.id), + ) + + stats = merge_databases(str(target), str(source)) + + assert stats == {"scanned": 1, "merged": 0, "skipped": 0, "errors": 1} + assert _column(target, "claims", "COUNT(*)") == 0 + assert secret.encode() not in target.read_bytes() + + +def test_merge_sanitizes_claim_and_citation_content_atomically(tmp_path: Path) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + source_service = _service(source, tmp_path) + _service(target, tmp_path) + claim = _claim(source_service) + _update( + source, + "UPDATE claims SET text = ?, normalized_text = ? WHERE id = ?", + (LITERAL, ENCODED, claim.id), + ) + _update( + source, + "UPDATE citations SET excerpt = ? WHERE claim_id = ?", + (ENCODED, claim.id), + ) + + stats = merge_databases(str(target), str(source)) + + assert stats["merged"] == 1 + assert "[REDACTED:" in str(_column(target, "claims", "text")) + assert "[REDACTED:" in str(_column(target, "claims", "normalized_text")) + assert "[REDACTED:" in str(_column(target, "citations", "excerpt")) + persisted = target.read_bytes() + assert LITERAL.encode() not in persisted + assert ENCODED.encode() not in persisted + + +def test_merge_does_not_mutate_an_unsafe_legacy_target_row(tmp_path: Path) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + source_service = _service(source, tmp_path) + target_service = _service(target, tmp_path) + source_claim = _claim(source_service, "Shared bridge claim") + target_claim = _claim(target_service, "Shared bridge claim") + _update( + source, + "UPDATE claims SET confidence = 0.9, updated_at = ? WHERE id = ?", + ("2030-01-01T00:00:00+00:00", source_claim.id), + ) + _update( + target, + "UPDATE claims SET normalized_text = ?, confidence = 0.2 WHERE id = ?", + (ENCODED, target_claim.id), + ) + + merge_databases(str(target), str(source)) + + assert _column(target, "claims", "confidence") == 0.2 + + +def test_duplicate_with_unsafe_source_citation_cannot_reconcile_target( + tmp_path: Path, +) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + source_service = _service(source, tmp_path) + target_service = _service(target, tmp_path) + source_claim = _claim(source_service, "Shared citation envelope") + target_claim = _claim(target_service, "Shared citation envelope") + _update( + source, + "UPDATE claims SET confidence = 0.9, updated_at = ? WHERE id = ?", + ("2030-01-01T00:00:00+00:00", source_claim.id), + ) + _update( + source, + "UPDATE citations SET source = ? WHERE claim_id = ?", + (LITERAL, source_claim.id), + ) + _update( + target, + "UPDATE claims SET confidence = 0.2 WHERE id = ?", + (target_claim.id,), + ) + + stats = merge_databases(str(target), str(source)) + + assert stats == {"scanned": 1, "merged": 0, "skipped": 0, "errors": 1} + assert _column(target, "claims", "confidence") == 0.2 + + +def test_target_with_unsafe_citation_cannot_be_reconciled(tmp_path: Path) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + source_service = _service(source, tmp_path) + target_service = _service(target, tmp_path) + source_claim = _claim(source_service, "Shared target citation envelope") + target_claim = _claim(target_service, "Shared target citation envelope") + _update( + source, + "UPDATE claims SET confidence = 0.9, updated_at = ? WHERE id = ?", + ("2030-01-01T00:00:00+00:00", source_claim.id), + ) + _update( + target, + "UPDATE claims SET confidence = 0.2 WHERE id = ?", + (target_claim.id,), + ) + _update( + target, + "UPDATE citations SET locator = ? WHERE claim_id = ?", + (ENCODED, target_claim.id), + ) + + merge_databases(str(target), str(source)) + + assert _column(target, "claims", "confidence") == 0.2 + + +def test_target_with_unsafe_citation_cannot_be_superseded(tmp_path: Path) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + source_service = _service(source, tmp_path) + target_service = _service(target, tmp_path) + source_claim = source_service.ingest( + "Remote citation conflict", + [CitationInput(source="test://bridge")], + source_agent="bridge-test", + subject="bridge-citation", + predicate="state", + object_value="remote", + ) + target_claim = target_service.ingest( + "Local citation conflict", + [CitationInput(source="test://bridge")], + source_agent="bridge-test", + subject="bridge-citation", + predicate="state", + object_value="local", + ) + _update( + source, + "UPDATE claims SET pinned = 1, updated_at = ? WHERE id = ?", + ("2030-01-01T00:00:00+00:00", source_claim.id), + ) + _update( + target, + "UPDATE citations SET excerpt = ? WHERE claim_id = ?", + (ENCODED, target_claim.id), + ) + + merge_databases(str(target), str(source)) + + with sqlite3.connect(target) as conn: + row = conn.execute( + "SELECT status, replaced_by_claim_id FROM claims WHERE id = ?", + (target_claim.id,), + ).fetchone() + assert row == ("candidate", None) + + +def test_duplicate_with_unsafe_citation_excerpt_cannot_reconcile( + tmp_path: Path, +) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + source_service = _service(source, tmp_path) + target_service = _service(target, tmp_path) + source_claim = _claim(source_service, "Shared unsafe citation excerpt") + target_claim = _claim(target_service, "Shared unsafe citation excerpt") + _update( + source, + "UPDATE claims SET confidence = 0.9, updated_at = ? WHERE id = ?", + ("2030-01-01T00:00:00+00:00", source_claim.id), + ) + _update( + source, + "UPDATE citations SET excerpt = ? WHERE claim_id = ?", + (ENCODED, source_claim.id), + ) + _update( + target, + "UPDATE claims SET confidence = 0.2 WHERE id = ?", + (target_claim.id,), + ) + + merge_databases(str(target), str(source)) + + assert _column(target, "claims", "confidence") == 0.2 + + +def test_merge_rolls_back_claim_when_citation_insert_fails(tmp_path: Path) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + source_service = _service(source, tmp_path) + _service(target, tmp_path) + _claim(source_service) + with sqlite3.connect(target) as conn: + conn.execute( + """ + CREATE TRIGGER fail_bridge_citation + BEFORE INSERT ON citations + BEGIN + SELECT RAISE(ABORT, 'fault injection'); + END + """ + ) + + stats = merge_databases(str(target), str(source)) + + assert stats == {"scanned": 1, "merged": 0, "skipped": 0, "errors": 1} + assert _column(target, "claims", "COUNT(*)") == 0 + assert _column(target, "citations", "COUNT(*)") == 0 + + +def test_merge_never_logs_secret_bearing_sqlite_errors( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + source_service = _service(source, tmp_path) + _service(target, tmp_path) + _claim(source_service) + with sqlite3.connect(target) as conn: + conn.execute( + f""" + CREATE TRIGGER fail_secret_echo + BEFORE INSERT ON claims + BEGIN + SELECT RAISE(ABORT, '{LITERAL}'); + END + """ + ) + + with caplog.at_level(logging.WARNING): + stats = merge_databases(str(target), str(source)) + + assert stats["errors"] == 1 + assert LITERAL not in caplog.text + + +def test_duplicate_reconciliation_contains_secret_trigger_errors( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + source_service = _service(source, tmp_path) + target_service = _service(target, tmp_path) + source_claim = _claim(source_service, "Duplicate trigger boundary") + target_claim = _claim(target_service, "Duplicate trigger boundary") + _update( + source, + "UPDATE claims SET confidence = 0.9, updated_at = ? WHERE id = ?", + ("2030-01-01T00:00:00+00:00", source_claim.id), + ) + _update( + target, + "UPDATE claims SET confidence = 0.2 WHERE id = ?", + (target_claim.id,), + ) + with sqlite3.connect(target) as conn: + conn.execute( + f""" + CREATE TRIGGER fail_duplicate_reconciliation + BEFORE UPDATE OF confidence ON claims + BEGIN + SELECT RAISE(ABORT, '{LITERAL}'); + END + """ + ) + + with caplog.at_level(logging.WARNING): + stats = merge_databases(str(target), str(source)) + + assert stats == {"scanned": 1, "merged": 0, "skipped": 0, "errors": 1} + assert _column(target, "claims", "confidence") == 0.2 + assert LITERAL not in caplog.text + + +def test_duplicate_heavy_merge_commits_in_bounded_batches( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + _service(source, tmp_path) + _service(target, tmp_path) + _seed_duplicate_claims(source, 201, source=True) + _seed_duplicate_claims(target, 201, source=False) + + class CountingConnection(sqlite3.Connection): + commit_count = 0 + + def commit(self) -> None: + self.commit_count += 1 + super().commit() + + opened: list[CountingConnection] = [] + + def open_target(path: str) -> CountingConnection: + conn = sqlite3.connect(path, factory=CountingConnection) + conn.row_factory = sqlite3.Row + opened.append(conn) + return conn + + monkeypatch.setattr(db_merge_module, "_open_target", open_target) + + stats = merge_databases(str(target), str(source)) + + assert stats["skipped"] == 201 + assert opened[0].commit_count >= 2 + + +def test_merge_rechecks_target_citations_after_batch_commit( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + _service(source, tmp_path) + _service(target, tmp_path) + _seed_duplicate_claims(source, 201, source=True) + _seed_duplicate_claims(target, 201, source=False) + with sqlite3.connect(target) as conn: + conn.execute( + """ + INSERT INTO citations (claim_id, source, locator, excerpt, created_at) + VALUES (201, 'test://bridge', 'line-1', 'safe', ?) + """, + ("2020-01-01T00:00:00+00:00",), + ) + + class MutatingConnection(sqlite3.Connection): + commit_count = 0 + + def commit(self) -> None: + super().commit() + self.commit_count += 1 + if self.commit_count == 1: + _update( + target, + "UPDATE citations SET locator = ? WHERE claim_id = 201", + (ENCODED,), + ) + + def open_target(path: str) -> MutatingConnection: + conn = sqlite3.connect(path, factory=MutatingConnection) + conn.row_factory = sqlite3.Row + return conn + + monkeypatch.setattr(db_merge_module, "_open_target", open_target) + + stats = merge_databases(str(target), str(source)) + + assert stats["skipped"] == 201 + with sqlite3.connect(target) as conn: + confidence = conn.execute( + "SELECT confidence FROM claims WHERE id = 201" + ).fetchone()[0] + assert confidence == 0.2 + + +def test_merge_rechecks_target_identity_after_batch_commit( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + _service(source, tmp_path) + _service(target, tmp_path) + _seed_duplicate_claims(source, 201, source=True) + _seed_duplicate_claims(target, 201, source=False) + + class MutatingConnection(sqlite3.Connection): + commit_count = 0 + + def commit(self) -> None: + super().commit() + self.commit_count += 1 + if self.commit_count == 1: + _update( + target, + """ + UPDATE claims + SET text = 'unrelated target claim', + idempotency_key = 'unrelated-target-key' + WHERE id = 201 + """, + (), + ) + + def open_target(path: str) -> MutatingConnection: + conn = sqlite3.connect(path, factory=MutatingConnection) + conn.row_factory = sqlite3.Row + return conn + + monkeypatch.setattr(db_merge_module, "_open_target", open_target) + + stats = merge_databases(str(target), str(source)) + + with sqlite3.connect(target) as conn: + unrelated_confidence = conn.execute( + "SELECT confidence FROM claims WHERE id = 201" + ).fetchone()[0] + imported = conn.execute( + "SELECT confidence FROM claims WHERE idempotency_key = 'duplicate-key-200'" + ).fetchone() + claim_count = conn.execute("SELECT COUNT(*) FROM claims").fetchone()[0] + assert stats["skipped"] == 200 + assert stats["merged"] == 1 + assert unrelated_confidence == 0.2 + assert imported == (0.9,) + assert claim_count == 202 + + +def test_merge_reads_claims_and_citations_from_one_source_snapshot( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + source_service = _service(source, tmp_path) + _service(target, tmp_path) + claim = _claim(source_service, "Snapshot-safe merge citation") + original = db_merge_module._source_citations + changed = False + + def mutate_then_read(conn: sqlite3.Connection, claim_id: int): + nonlocal changed + if not changed: + changed = True + _update( + source, + "UPDATE citations SET source = ? WHERE claim_id = ?", + (LITERAL, claim.id), + ) + return original(conn, claim_id) + + monkeypatch.setattr(db_merge_module, "_source_citations", mutate_then_read) + + stats = merge_databases(str(target), str(source)) + + assert stats["merged"] == 1 + assert _column(target, "citations", "source") == "test://bridge" + + +def test_merge_keeps_insert_and_conflict_resolution_in_one_transaction( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + source_service = _service(source, tmp_path) + target_service = _service(target, tmp_path) + source_service.ingest( + "Remote bridge state", + [CitationInput(source="test://bridge")], + source_agent="bridge-test", + subject="bridge", + predicate="state", + object_value="remote", + confidence=0.9, + ) + target_service.ingest( + "Local bridge state", + [CitationInput(source="test://bridge")], + source_agent="bridge-test", + subject="bridge", + predicate="state", + object_value="local", + confidence=0.1, + ) + with sqlite3.connect(target) as conn: + conn.execute( + f""" + CREATE TRIGGER fail_bridge_conflict + BEFORE UPDATE OF status ON claims + WHEN NEW.status = 'superseded' + BEGIN + SELECT RAISE(ABORT, '{LITERAL}'); + END + """ + ) + + with caplog.at_level(logging.WARNING): + stats = merge_databases(str(target), str(source)) + + assert stats["errors"] == 1 + assert _column(target, "claims", "COUNT(*)") == 1 + assert LITERAL not in caplog.text + + +def test_redacted_merge_claim_cannot_supersede_a_safe_target_claim( + tmp_path: Path, +) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + source_service = _service(source, tmp_path) + target_service = _service(target, tmp_path) + source_claim = source_service.ingest( + "Remote bridge state", + [CitationInput(source="test://bridge")], + source_agent="bridge-test", + subject="bridge", + predicate="state", + object_value="remote", + ) + target_claim = target_service.ingest( + "Local bridge state", + [CitationInput(source="test://bridge")], + source_agent="bridge-test", + subject="bridge", + predicate="state", + object_value="safe-local", + ) + _update( + source, + "UPDATE claims SET object_value = ?, pinned = 1, updated_at = ? WHERE id = ?", + (LITERAL, "2030-01-01T00:00:00+00:00", source_claim.id), + ) + + stats = merge_databases(str(target), str(source)) + + assert stats["merged"] == 1 + with sqlite3.connect(target) as conn: + row = conn.execute( + "SELECT status, replaced_by_claim_id FROM claims WHERE id = ?", + (target_claim.id,), + ).fetchone() + assert row == ("candidate", None) + + +def test_redacted_merge_claim_is_downgraded_to_unpinned_candidate( + tmp_path: Path, +) -> None: + source = tmp_path / "source.db" + target = tmp_path / "target.db" + source_service = _service(source, tmp_path) + _service(target, tmp_path) + source_claim = _claim(source_service) + _update( + source, + "UPDATE claims SET text = ?, status = 'confirmed', pinned = 1 WHERE id = ?", + (LITERAL, source_claim.id), + ) + + stats = merge_databases(str(target), str(source)) + + assert stats["merged"] == 1 + with sqlite3.connect(target) as conn: + row = conn.execute("SELECT status, pinned FROM claims").fetchone() + assert row == ("candidate", 0) + + +def test_delta_redaction_marker_remains_quarantined_after_merge(tmp_path: Path) -> None: + source = tmp_path / "source.db" + delta = tmp_path / "delta.db" + target = tmp_path / "target.db" + source_service = _service(source, tmp_path) + _service(target, tmp_path) + source_claim = _claim(source_service) + _update( + source, + "UPDATE claims SET text = ?, status = 'confirmed', pinned = 1 WHERE id = ?", + (LITERAL, source_claim.id), + ) + + export_delta(source, "", delta) + stats = merge_databases(str(target), str(delta)) + + assert stats["merged"] == 1 + with sqlite3.connect(target) as conn: + row = conn.execute("SELECT status, pinned, text FROM claims").fetchone() + assert row[0:2] == ("candidate", 0) + assert "[REDACTED:" in row[2] diff --git a/tests/test_delta_bridge_persistence_sensitivity.py b/tests/test_delta_bridge_persistence_sensitivity.py new file mode 100644 index 00000000..5855b163 --- /dev/null +++ b/tests/test_delta_bridge_persistence_sensitivity.py @@ -0,0 +1,371 @@ +from __future__ import annotations + +import base64 +import logging +import os +import sqlite3 +from pathlib import Path + +import pytest + +import memorymaster.bridges.delta_sync as delta_sync_module +from memorymaster.bridges.delta_sync import _copy_table_ddl, export_delta +from memorymaster.core.models import CitationInput +from memorymaster.core.security import SensitiveMetadataError +from memorymaster.core.service import MemoryService + + +LITERAL = "OPENAI_API_KEY=sk-proj-FAKEbridgeEnvelope1234567890ABCD" +ENCODED = base64.b64encode(LITERAL.encode()).decode() + + +def _service(path: Path, workspace: Path) -> MemoryService: + service = MemoryService(path, workspace_root=workspace) + service.init_db() + return service + + +def _claim(service: MemoryService, text: str = "Safe bridge claim"): + return service.ingest( + text, + [CitationInput(source="test://bridge", locator="line-1", excerpt="safe")], + source_agent="bridge-test", + ) + + +def _update(path: Path, statement: str, values: tuple[object, ...]) -> None: + with sqlite3.connect(path) as conn: + conn.execute(statement, values) + + +def _column(path: Path, table: str, field: str) -> object: + with sqlite3.connect(path) as conn: + return conn.execute(f"SELECT {field} FROM {table}").fetchone()[0] + + +def test_delta_refuses_source_as_output_before_unlink(tmp_path: Path) -> None: + source = tmp_path / "source.db" + service = _service(source, tmp_path) + _claim(service) + before = source.read_bytes() + + with pytest.raises(ValueError, match="alias"): + export_delta(source, "", source) + + assert source.exists() + assert source.read_bytes() == before + + +def test_delta_refuses_resolved_source_alias_before_unlink(tmp_path: Path) -> None: + directory = tmp_path / "nested" + directory.mkdir() + source = tmp_path / "source.db" + service = _service(source, tmp_path) + _claim(service) + alias = directory / ".." / source.name + + with pytest.raises(ValueError, match="alias"): + export_delta(source, "", alias) + + assert source.exists() + + +def test_delta_refuses_hardlink_to_source_before_unlink(tmp_path: Path) -> None: + source = tmp_path / "source.db" + alias = tmp_path / "hardlink.db" + service = _service(source, tmp_path) + _claim(service) + os.link(source, alias) + + with pytest.raises(ValueError, match="alias"): + export_delta(source, "", alias) + + assert source.exists() + assert alias.exists() + + +def test_delta_never_executes_forged_source_schema_sql( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = sqlite3.connect(":memory:") + output = sqlite3.connect(":memory:") + source.execute("CREATE TABLE claims (id INTEGER PRIMARY KEY, text TEXT)") + source.execute("PRAGMA writable_schema=ON") + source.execute( + "UPDATE sqlite_master SET sql = ? WHERE type = 'table' AND name = 'claims'", + ("ATTACH DATABASE 'forged-side-effect.db' AS injected",), + ) + source.commit() + monkeypatch.chdir(tmp_path) + + with pytest.raises(ValueError, match="canonical transport schema"): + _copy_table_ddl(source, output, "claims") + + assert not (tmp_path / "forged-side-effect.db").exists() + + +def test_delta_quotes_reserved_identifiers_in_value_copy_statements( + tmp_path: Path, +) -> None: + source = tmp_path / "legacy-source.db" + output = tmp_path / "delta.db" + with sqlite3.connect(source) as conn: + conn.executescript( + """ + CREATE TABLE claims ( + id INTEGER PRIMARY KEY, + text TEXT NOT NULL, + updated_at TEXT NOT NULL, + "select" TEXT + ); + CREATE TABLE citations ( + id INTEGER PRIMARY KEY, + claim_id INTEGER NOT NULL, + source TEXT NOT NULL, + created_at TEXT NOT NULL, + "from" TEXT + ); + INSERT INTO claims (id, text, updated_at, "select") + VALUES (1, 'safe legacy claim', '2026-07-11T00:00:00+00:00', 'claim-extra'); + INSERT INTO citations (id, claim_id, source, created_at, "from") + VALUES (1, 1, 'test://bridge', '2026-07-11T00:00:00+00:00', 'citation-extra'); + """ + ) + + result = export_delta(source, "", output) + + assert result["exported"] == 1 + assert result["citations"] == 1 + assert _column(output, "claims", '"select"') == "claim-extra" + assert _column(output, "citations", '"from"') == "citation-extra" + + +@pytest.mark.parametrize( + "table,field", + [ + ("claims", "idempotency_key"), + ("claims", "claim_type"), + ("claims", "scope"), + ("claims", "volatility"), + ("claims", "created_at"), + ("claims", "updated_at"), + ("claims", "source_agent"), + ("claims", "wiki_article"), + ("claims", "holder"), + ("claims", "tenant_id"), + ("citations", "source"), + ("citations", "locator"), + ("citations", "created_at"), + ], +) +@pytest.mark.parametrize("secret", [LITERAL, ENCODED]) +def test_delta_rejects_unsafe_metadata_without_persisting_it( + tmp_path: Path, table: str, field: str, secret: str +) -> None: + source = tmp_path / "source.db" + output = tmp_path / "delta.db" + service = _service(source, tmp_path) + claim = _claim(service) + where = "id = ?" if table == "claims" else "claim_id = ?" + _update(source, f"UPDATE {table} SET {field} = ? WHERE {where}", (secret, claim.id)) + + result = export_delta(source, "", output) + + assert result["exported"] == 0 + assert result["citations"] == 0 + assert result["rejected"] == 1 + assert _column(output, "claims", "COUNT(*)") == 0 + assert secret.encode() not in output.read_bytes() + + +@pytest.mark.parametrize( + "field", ["text", "normalized_text", "subject", "predicate", "object_value"] +) +@pytest.mark.parametrize("secret", [LITERAL, ENCODED]) +def test_delta_sanitizes_every_claim_content_field( + tmp_path: Path, field: str, secret: str +) -> None: + source = tmp_path / "source.db" + output = tmp_path / "delta.db" + service = _service(source, tmp_path) + claim = _claim(service) + _update(source, f"UPDATE claims SET {field} = ? WHERE id = ?", (secret, claim.id)) + + result = export_delta(source, "", output) + + assert result["exported"] == 1 + assert "[REDACTED:" in str(_column(output, "claims", field)) + assert secret.encode() not in output.read_bytes() + + +@pytest.mark.parametrize("secret", [LITERAL, ENCODED]) +def test_delta_sanitizes_citation_excerpt(tmp_path: Path, secret: str) -> None: + source = tmp_path / "source.db" + output = tmp_path / "delta.db" + service = _service(source, tmp_path) + claim = _claim(service) + _update( + source, + "UPDATE citations SET excerpt = ? WHERE claim_id = ?", + (secret, claim.id), + ) + + result = export_delta(source, "", output) + + assert result["exported"] == 1 + assert "[REDACTED:" in str(_column(output, "citations", "excerpt")) + assert secret.encode() not in output.read_bytes() + + +def test_delta_rejects_secret_bearing_transport_ddl_before_output_write( + tmp_path: Path, +) -> None: + source = tmp_path / "legacy-source.db" + output = tmp_path / "delta.db" + with sqlite3.connect(source) as conn: + conn.executescript( + f""" + CREATE TABLE claims ( + id INTEGER PRIMARY KEY, + text TEXT NOT NULL, + updated_at TEXT NOT NULL, + leaked_default TEXT DEFAULT '{LITERAL}' + ); + CREATE TABLE citations ( + id INTEGER PRIMARY KEY, + claim_id INTEGER NOT NULL, + source TEXT NOT NULL, + locator TEXT, + excerpt TEXT, + created_at TEXT NOT NULL + ); + INSERT INTO claims (id, text, updated_at) + VALUES (1, 'safe legacy claim', '2026-07-11T00:00:00+00:00'); + """ + ) + + with pytest.raises(SensitiveMetadataError): + export_delta(source, "", output) + + assert not output.exists() or LITERAL.encode() not in output.read_bytes() + + +def test_delta_rejects_unsafe_legacy_identifier_without_echo(tmp_path: Path) -> None: + source = tmp_path / "legacy-source.db" + output = tmp_path / "delta.db" + with sqlite3.connect(source) as conn: + conn.executescript( + """ + CREATE TABLE claims ( + id TEXT PRIMARY KEY, + text TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE TABLE citations ( + id INTEGER PRIMARY KEY, + claim_id TEXT NOT NULL, + source TEXT NOT NULL, + locator TEXT, + excerpt TEXT, + created_at TEXT NOT NULL + ); + """ + ) + conn.execute( + "INSERT INTO claims (id, text, updated_at) VALUES (?, 'safe', ?)", + (LITERAL, "2026-07-11T00:00:00+00:00"), + ) + + with pytest.raises(SensitiveMetadataError) as rejected: + export_delta(source, "", output) + + assert LITERAL not in str(rejected.value) + assert not output.exists() or LITERAL.encode() not in output.read_bytes() + + +def test_delta_rejects_secret_bearing_watermark_without_echo( + tmp_path: Path, caplog: pytest.LogCaptureFixture +) -> None: + source = tmp_path / "source.db" + output = tmp_path / "delta.db" + service = _service(source, tmp_path) + _claim(service) + + with caplog.at_level(logging.WARNING): + with pytest.raises(SensitiveMetadataError) as rejected: + export_delta(source, LITERAL, output) + + assert LITERAL not in str(rejected.value) + assert LITERAL not in caplog.text + assert not output.exists() or LITERAL.encode() not in output.read_bytes() + + +def test_delta_reads_claims_and_citations_from_one_source_snapshot( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + source = tmp_path / "source.db" + output = tmp_path / "delta.db" + service = _service(source, tmp_path) + claim = _claim(service, "Snapshot-safe delta citation") + original = delta_sync_module._load_citations_by_claim + changed = False + + def mutate_then_read(conn: sqlite3.Connection, claim_ids: list[int]): + nonlocal changed + if not changed: + changed = True + _update( + source, + "UPDATE citations SET source = ? WHERE claim_id = ?", + (LITERAL, claim.id), + ) + return original(conn, claim_ids) + + monkeypatch.setattr( + delta_sync_module, + "_load_citations_by_claim", + mutate_then_read, + ) + + result = export_delta(source, "", output) + + assert result["exported"] == 1 + assert _column(output, "citations", "source") == "test://bridge" + + +def test_delta_rejects_secret_bearing_binary_metadata(tmp_path: Path) -> None: + source = tmp_path / "source.db" + output = tmp_path / "delta.db" + service = _service(source, tmp_path) + claim = _claim(service) + _update( + source, + "UPDATE claims SET wiki_article = ? WHERE id = ?", + (sqlite3.Binary(LITERAL.encode()), claim.id), + ) + + result = export_delta(source, "", output) + + assert result["exported"] == 0 + assert result["rejected"] == 1 + assert LITERAL.encode() not in output.read_bytes() + + +def test_delta_decodes_and_sanitizes_secret_bearing_binary_content( + tmp_path: Path, +) -> None: + source = tmp_path / "source.db" + output = tmp_path / "delta.db" + service = _service(source, tmp_path) + claim = _claim(service) + _update( + source, + "UPDATE claims SET text = ? WHERE id = ?", + (sqlite3.Binary(LITERAL.encode()), claim.id), + ) + + result = export_delta(source, "", output) + + assert result["exported"] == 1 + assert "[REDACTED:" in str(_column(output, "claims", "text")) + assert LITERAL.encode() not in output.read_bytes() diff --git a/tests/test_delta_sync_extra.py b/tests/test_delta_sync_extra.py index 55877f7e..fefaf1e2 100644 --- a/tests/test_delta_sync_extra.py +++ b/tests/test_delta_sync_extra.py @@ -8,9 +8,8 @@ boundary claim is silent data loss when several claims share a same-second timestamp. We also prove a claim one tick BELOW the watermark is excluded, so the boundary is the real ``>=`` cut, not "export everything". -2. **DDL / CREATE-TABLE copy** — the delta file must carry the verbatim - ``CREATE TABLE`` statements for ``claims`` and ``citations`` so it is a - valid standalone merge source. A missing/altered DDL breaks the merge. +2. **DDL / CREATE-TABLE synthesis** — the delta file must carry safe, + value-only tables with ordered source-column parity. 3. **Empty export** — nothing newer than the watermark yields zero rows, ``max_updated_at is None``, yet a valid (schema-only) file still exists. 4. **Full export** — empty/whitespace ``since`` exports every claim and @@ -87,6 +86,14 @@ def _table_ddl(db: Path, table: str) -> str | None: conn.close() +def _table_columns(db: Path, table: str) -> list[tuple[str, str]]: + with sqlite3.connect(str(db)) as conn: + return [ + (str(row[1]), str(row[2]).upper()) + for row in conn.execute(f'PRAGMA table_info("{table}")').fetchall() + ] + + # --------------------------------------------------------------------------- # 1. Watermark boundary — the >= vs > edge # --------------------------------------------------------------------------- @@ -196,11 +203,8 @@ def test_delta_carries_create_table_ddl_for_both_tables(populated_db, tmp_path): assert citations_ddl is not None and citations_ddl.startswith("CREATE TABLE") -def test_delta_ddl_is_verbatim_copy_of_source(populated_db, tmp_path): - """CONTRACT: ``_copy_table_ddl`` copies the source CREATE statement - *verbatim* — the delta schema must be byte-identical to the source schema - for these tables, so column order/types match exactly and inserts line up. - If the copy were regenerated or altered, this equality breaks.""" +def test_delta_ddl_is_synthesized_from_source_columns(populated_db, tmp_path): + """Transport DDL preserves ordered columns but no source constraints.""" db, svc = populated_db _ingest(svc, "schema parity") @@ -208,9 +212,12 @@ def test_delta_ddl_is_verbatim_copy_of_source(populated_db, tmp_path): export_delta(db, "", out) for table in ("claims", "citations"): - assert _table_ddl(out, table) == _table_ddl(db, table), ( - f"{table} DDL in delta diverged from source" - ) + assert [name for name, _ in _table_columns(out, table)] == [ + name for name, _ in _table_columns(db, table) + ] + ddl = _table_ddl(out, table) or "" + assert "FOREIGN KEY" not in ddl.upper() + assert "CHECK" not in ddl.upper() def test_ddl_copied_even_when_export_is_empty(populated_db, tmp_path): From 702b59db0022a40766d23a2ad3b02165960a2cb5 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 20:57:05 -0300 Subject: [PATCH 21/55] fix(security): inventory legacy persistence safely --- .../R1.4-LEGACY-SENSITIVITY-INVENTORY.md | 13 + memorymaster/govern/jobs/_sensitivity_scan.py | 415 ++++++++++ .../govern/jobs/sensitivity_inventory.py | 756 ++++++++++++++++++ scripts/sensitivity_inventory.py | 53 ++ tests/test_sensitivity_inventory.py | 703 ++++++++++++++++ tests/test_sensitivity_inventory_core.py | 191 +++++ ...t_sensitivity_inventory_review_findings.py | 377 +++++++++ 7 files changed, 2508 insertions(+) create mode 100644 .planning/R1.4-LEGACY-SENSITIVITY-INVENTORY.md create mode 100644 memorymaster/govern/jobs/_sensitivity_scan.py create mode 100644 memorymaster/govern/jobs/sensitivity_inventory.py create mode 100644 scripts/sensitivity_inventory.py create mode 100644 tests/test_sensitivity_inventory.py create mode 100644 tests/test_sensitivity_inventory_core.py create mode 100644 tests/test_sensitivity_inventory_review_findings.py diff --git a/.planning/R1.4-LEGACY-SENSITIVITY-INVENTORY.md b/.planning/R1.4-LEGACY-SENSITIVITY-INVENTORY.md new file mode 100644 index 00000000..78d90978 --- /dev/null +++ b/.planning/R1.4-LEGACY-SENSITIVITY-INVENTORY.md @@ -0,0 +1,13 @@ +# R1.4 legacy sensitivity inventory + +R1.4 supplies an aggregate-only, read-only dry-run inventory for MM-SEC-03 and MM-SEC-04. It dynamically accounts for SQLite user tables, views, columns, schema definitions, explicit artifact and spool roots, and an injected Qdrant payload reader. It never cleans up, redacts, migrates, initializes, rebuilds, backs up, or contacts an external service. + +```text +python scripts/sensitivity_inventory.py --db --artifact-root --spool-root --max-file-bytes --max-entries +``` + +Output is aggregate JSON only: fixed `mode` (`dry_run`), classification (`LEGACY-SENSITIVITY-INVENTORY`), recommendation (`REVIEW_ONLY`), and a fixed top-level status. SQLite records `tables_accounted`, `columns_accounted`, unique-row `records_scanned`/`records_flagged`, derived table/view/column/record counts, bounded schema-identifier/definition findings, and safe-category surfaces. Each surface contains per-field `records_scanned`, `records_flagged`, and bounded `finding_counts`. Safe fields cover R1.4 claim, citation, event, verbatim, Atlas, cache/miner/rule/entity/link/timeline names; unknown columns are classified only as `field_text`, `field_json`, `field_embedding`, `field_binary`, `field_numeric`, or `field_other`. Artifact/spool records contain fixed status/reason, scanned, sensitive, truncated, unavailable, and refused counts plus fixed `file.metadata`, `file.text`, `file.json`, `file.binary`, or `file.other` surfaces. It contains no persisted values, IDs, schema identifiers, sample content, filenames, supplied paths, offsets, payloads, or exception text. + +SQLite uses an explicit `mode=ro` URI followed by `PRAGMA query_only=ON` and one `BEGIN` read snapshot; it does not initialize or migrate. SQLite internals are excluded. A quote-aware SQL tokenizer ignores comments only outside quoted identifiers/strings and identifies FTS5 modules. SQLite `PRAGMA table_list` supplies structural shadow types when available, with a structural FTS fallback for older supported SQLite runtimes. External/contentless FTS options keep independently owned `_content` tables authoritative; internal-content FTS bases are value-scanned while their actual shadow tables remain derived-only. `PRAGMA table_xinfo` keeps generated columns aligned with explicit projections. Raw JSON text and its parsed structure are both scanned so duplicate keys cannot erase evidence. Authoritative categories are claims, citations, events, claim embeddings, verbatim, feedback, Atlas source/evidence/action/media-retry, cache, miner, rules, entities/links, timeline, and bounded unknown. Deep, cyclic, oversized, or otherwise unscannable values are counted with `scan_incomplete` and block complete coverage without exposing their content. Artifact and spool traversal stays below resolved inputs, refuses symlinks plus Windows reparse-point junctions across supported Python versions, verifies the opened handle against contained path identity before reading, caps iterator fetches, entries, and bytes per file, scans one bounded snapshot rather than fixed overlap windows, and marks entry-limit, changed, unscannable, oversized, or truncated coverage `BLOCKED` rather than complete. + +Qdrant is payload-only through an explicit verified injected scroll transport using a validated bounded limit/page count, `with_payload=true`, `with_vector=false`, exact JSON payload values, and bounded JSON offsets (including mapping/list offsets). It accepts the scroll envelope `{"result":{"points":[],"next_page_offset":...}}`, preserves partial aggregate counts, and blocks missing payloads plus repeated, deep, cyclic, hostile, or malformed offsets/payloads, oversized pages, malformed envelopes/points, page limits, invalid bounds, and transport failures with fixed `BLOCKED-EXTERNAL` reasons. Without that transport it returns `BLOCKED-EXTERNAL/qdrant_not_configured`; it does not default to localhost, probe, or discover. Invalid CLI arguments produce only fixed aggregate JSON and never echo supplied values. Any missing/unreadable artifact or spool root (including an omitted root list) is local `BLOCKED`; local SQLite/artifact/spool incompleteness takes top-level precedence over Qdrant blockers. Refused nested symlink/junction entries are aggregate evidence but do not block an otherwise readable root; a refused root or detected identity race blocks completion. A real Qdrant dry-run requires evidence of scoped, authenticated transport. Backup approval and explicit authority are required only before any later cleanup/redaction/rebuild action; this package implements none of those actions. diff --git a/memorymaster/govern/jobs/_sensitivity_scan.py b/memorymaster/govern/jobs/_sensitivity_scan.py new file mode 100644 index 00000000..f18dec90 --- /dev/null +++ b/memorymaster/govern/jobs/_sensitivity_scan.py @@ -0,0 +1,415 @@ +"""Shared bounded detection helpers for the legacy sensitivity inventory.""" + +from __future__ import annotations + +import json +import sqlite3 +from collections import defaultdict +from collections.abc import Iterable +from pathlib import Path +from urllib.parse import quote + +from memorymaster.core.security import scan_persisted_value + +_SQL_QUOTES = {'"': '"', "'": "'", "`": "`", "[": "]"} +_REASONS = frozenset( + { + "sensitive_value", + "embedding_invalid", + "embedding_string", + "binary_opaque", + "scan_incomplete", + } +) +_TABLES = { + "claims": "claims", + "citations": "citations", + "events": "events", + "claim_embeddings": "claim_embeddings", + "verbatim": "verbatim", + "verbatim_memories": "verbatim", + "feedback": "feedback", + "usage_feedback": "feedback", + "external_sources": "atlas.source", + "source_items": "atlas.source", + "evidence_items": "atlas.evidence", + "action_proposals": "atlas.action", + "media_retry_queue": "atlas.media_retry", + "query_cache": "cache", + "cache_meta": "cache", + "miner_state": "miner", +} +_SAFE_FIELDS = frozenset( + { + "id", + "claim_id", + "human_id", + "text", + "normalized_text", + "metadata", + "subject", + "predicate", + "object_value", + "holder", + "source_agent", + "idempotency_key", + "scope", + "claim_type", + "volatility", + "status", + "visibility", + "tenant_id", + "created_at", + "updated_at", + "event_time", + "valid_from", + "valid_until", + "wiki_path", + "wiki_article", + "pinned", + "confidence", + "embedding_json", + "model", + "blob", + "source", + "locator", + "excerpt", + "event_type", + "from_status", + "to_status", + "details", + "payload", + "payload_json", + "config", + "config_json", + "content_hash", + "payload_hash", + "hash_algorithm", + "session_id", + "thread_id", + "role", + "content", + "sync_status", + "identity", + "canonical_identity", + "sender", + "sender_id", + "sender_type", + "source_type", + "external_id", + "sensitivity", + "media", + "media_id", + "media_url", + "media_path", + "provider", + "provider_data", + "title", + "description", + "destination", + "destination_json", + "external_ref", + "error", + "last_error", + "query_text", + "response_json", + "state_json", + "rule_json", + "entity_name", + "entity_type", + "link_type", + "timeline_json", + } +) +_MAX_SCAN_TEXT_BYTES = 4 * 1024 * 1024 +_MAX_SCAN_NODES = 10_000 +_MAX_SCAN_DEPTH = 128 +_MAX_JSON_TEXT_BYTES = 64 * 1024 +_MAX_JSON_DIGITS = 1024 +_JSON_INVALID = object() +_JSON_UNSAFE = object() + + +def _json_text_unsafe(value: str) -> bool: + if len(value.encode("utf-8", errors="ignore")) > _MAX_JSON_TEXT_BYTES: + return True + digits = 0 + for character in value: + digits = digits + 1 if character in "0123456789" else 0 + if digits > _MAX_JSON_DIGITS: + return True + return False + + +def _bounded_json(value: str) -> object: + if _json_text_unsafe(value): + return _JSON_UNSAFE + try: + return json.loads(value) + except json.JSONDecodeError: + return _JSON_INVALID + except (RecursionError, ValueError): + return _JSON_UNSAFE + + +def _category(table: str) -> str: + lowered = table.lower() + if lowered in _TABLES: + return _TABLES[lowered] + for prefix, category in ( + ("rule", "rules"), + ("entity", "entities"), + ("link", "entities"), + ("timeline", "timeline"), + ): + if lowered.startswith(prefix): + return category + return "unknown" + + +def _surface( + table: str, + column: str, + declared_type: object, + value: object, +) -> str: + lowered = column.lower() + declared = str(declared_type or "").upper() + if lowered in _SAFE_FIELDS: + field = lowered + elif "embedding" in lowered or "vector" in lowered: + field = "field_embedding" + elif isinstance(value, bytes) or "BLOB" in declared: + field = "field_binary" + elif (isinstance(value, (int, float)) and not isinstance(value, bool)) or any( + token in declared for token in ("INT", "REAL", "NUM", "DEC", "FLOA", "DOUB") + ): + field = "field_numeric" + elif lowered.endswith("_json") or "JSON" in declared: + field = "field_json" + elif any(token in declared for token in ("CHAR", "CLOB", "TEXT")): + field = "field_text" + elif isinstance(value, str): + parsed = _bounded_json(value) + field = "field_json" if parsed is not _JSON_INVALID and parsed is not _JSON_UNSAFE else "field_text" + else: + field = "field_other" + return f"{_category(table)}.{field}" + + +def _numeric_tree(value: object) -> bool: + pending = [value] + while pending: + current = pending.pop() + if isinstance(current, (int, float)) and not isinstance(current, bool): + continue + if type(current) is list: + pending.extend(current) + continue + if type(current) is dict: + pending.extend(current.values()) + continue + return False + return True + + +def _structure_bounded(value: object) -> bool: + pending: list[tuple[object, int]] = [(value, 0)] + containers: set[int] = set() + nodes = 0 + text_bytes = 0 + while pending: + current, depth = pending.pop() + nodes += 1 + if nodes > _MAX_SCAN_NODES or depth > _MAX_SCAN_DEPTH: + return False + if current is None or isinstance(current, (bool, int, float)): + continue + if isinstance(current, (str, bytes)): + try: + text_bytes += len(current.encode("utf-8") if isinstance(current, str) else current) + except UnicodeEncodeError: + return False + if text_bytes > _MAX_SCAN_TEXT_BYTES: + return False + continue + if type(current) not in {dict, list, tuple, set, frozenset}: + return False + identity = id(current) + if identity in containers: + return False + containers.add(identity) + if len(pending) + len(current) > _MAX_SCAN_NODES: + return False + if isinstance(current, dict): + for key, nested in current.items(): + pending.append((key, depth + 1)) + pending.append((nested, depth + 1)) + else: + pending.extend((nested, depth + 1) for nested in current) + return True + + +def _sensitivity_reason(value: object) -> set[str]: + try: + return {"sensitive_value"} if scan_persisted_value(value) else set() + except (RecursionError, TypeError, ValueError): + return {"scan_incomplete"} + + +def _embedding_reasons(value: str) -> set[str]: + reasons = _sensitivity_reason(value) + parsed = _bounded_json(value) + if parsed is _JSON_INVALID: + return reasons | {"embedding_invalid"} + if parsed is _JSON_UNSAFE or not _structure_bounded(parsed): + return reasons | {"scan_incomplete"} + reasons.update(_sensitivity_reason(parsed)) + if not _numeric_tree(parsed): + reasons.add("embedding_string") + return reasons + + +def _reasons(value: object, column: str) -> set[str]: + if not _structure_bounded(value): + return {"scan_incomplete"} + if isinstance(value, bytes): + try: + value = value.decode("utf-8") + except UnicodeDecodeError: + return {"binary_opaque"} + if isinstance(value, str) and ("embedding" in column.lower() or "vector" in column.lower()): + return _embedding_reasons(value) + reasons = _sensitivity_reason(value) + if not isinstance(value, str): + return reasons + parsed = _bounded_json(value) + if parsed is _JSON_UNSAFE: + return reasons | {"scan_incomplete"} + if parsed is not _JSON_INVALID: + value = parsed + if not _structure_bounded(value): + return reasons | {"scan_incomplete"} + return reasons | _sensitivity_reason(value) + + +def _record(surfaces: dict[str, dict[str, object]], surface: str, reasons: Iterable[str]) -> None: + item = surfaces.setdefault( + surface, {"records_scanned": 0, "records_flagged": 0, "finding_counts": defaultdict(int)} + ) + item["records_scanned"] = int(item["records_scanned"]) + 1 + found = sorted(set(reasons) & _REASONS) + if found: + item["records_flagged"] = int(item["records_flagged"]) + 1 + counts = item["finding_counts"] + assert isinstance(counts, defaultdict) + for reason in found: + counts[reason] += 1 + + +def _ensure_surface(surfaces: dict[str, dict[str, object]], surface: str) -> None: + surfaces.setdefault( + surface, + { + "records_scanned": 0, + "records_flagged": 0, + "finding_counts": defaultdict(int), + }, + ) + + +def _freeze_surfaces(surfaces: dict[str, dict[str, object]]) -> dict[str, dict[str, object]]: + return { + key: { + "finding_counts": dict(sorted(item["finding_counts"].items())), + "records_flagged": item["records_flagged"], + "records_scanned": item["records_scanned"], + } + for key, item in sorted(surfaces.items()) + } + + +def _sqlite_uri(path: Path) -> str: + return f"file:{quote(path.resolve().as_posix())}?mode=ro" + + +def _empty_sqlite_result() -> dict[str, object]: + return { + "columns_accounted": 0, + "derived_columns_accounted": 0, + "derived_records": 0, + "derived_tables_accounted": 0, + "derived_views_accounted": 0, + "records_flagged": 0, + "records_scanned": 0, + "scan_incomplete": 0, + "schema_definition_sensitive": 0, + "schema_identifier_sensitive": 0, + "surfaces": {}, + "tables_accounted": 0, + } + + +def _quote_identifier(identifier: str) -> str: + return '"' + identifier.replace('"', '""') + '"' + + +class _TableListUnavailable(sqlite3.Error): + pass + + +def _table_columns(conn: sqlite3.Connection, quoted: str) -> list[tuple[str, object, int]]: + try: + rows = list(conn.execute(f"PRAGMA table_xinfo({quoted})")) + except sqlite3.Error: + rows = [] + if rows: + return [(str(row[1]), row[2], int(row[6])) for row in rows] + rows = list(conn.execute(f"PRAGMA table_info({quoted})")) + return [(str(row[1]), row[2], 0) for row in rows] + + +def _quoted_sql_token(sql: str, start: int, closer: str) -> tuple[str, int]: + token: list[str] = [] + index = start + 1 + while index < len(sql): + character = sql[index] + if character == closer: + if index + 1 < len(sql) and sql[index + 1] == closer: + token.append(closer) + index += 2 + continue + return "".join(token), index + 1 + token.append(character) + index += 1 + return "".join(token), index + + +def _sql_tokens(sql: str) -> list[tuple[str, bool]]: + tokens: list[tuple[str, bool]] = [] + index = 0 + while index < len(sql): + character = sql[index] + if character in _SQL_QUOTES: + token, index = _quoted_sql_token(sql, index, _SQL_QUOTES[character]) + tokens.append((token, True)) + continue + if sql.startswith("--", index): + newline = sql.find("\n", index + 2) + index = len(sql) if newline < 0 else newline + 1 + continue + if sql.startswith("/*", index): + end = sql.find("*/", index + 2) + index = len(sql) if end < 0 else end + 2 + continue + if character.isalnum() or character == "_": + end = index + 1 + while end < len(sql) and (sql[end].isalnum() or sql[end] == "_"): + end += 1 + tokens.append((sql[index:end], False)) + index = end + continue + if character in {"(", ")", ",", "="}: + tokens.append((character, False)) + index += 1 + return tokens diff --git a/memorymaster/govern/jobs/sensitivity_inventory.py b/memorymaster/govern/jobs/sensitivity_inventory.py new file mode 100644 index 00000000..42caa15c --- /dev/null +++ b/memorymaster/govern/jobs/sensitivity_inventory.py @@ -0,0 +1,756 @@ +"""Aggregate-only, snapshot-read legacy sensitivity inventory.""" + +from __future__ import annotations + +import json +import os +import sqlite3 +import stat +from collections.abc import Callable, Iterable, Mapping +from hashlib import sha256 +from math import isfinite +from pathlib import Path + +from memorymaster.govern.jobs._sensitivity_scan import ( + _MAX_SCAN_DEPTH, + _MAX_SCAN_NODES, + _MAX_SCAN_TEXT_BYTES, + _TableListUnavailable, + _empty_sqlite_result, + _ensure_surface, + _freeze_surfaces, + _quote_identifier, + _reasons, + _record, + _sql_tokens, + _sqlite_uri, + _surface, + _table_columns, +) + +_MAX_CHUNK_BYTES = 1024 * 1024 +_MAX_FILE_BYTES = 64 * 1024 * 1024 +_MAX_ENTRIES = 1_000_000 +_MAX_OFFSET_BYTES = 64 * 1024 +_MAX_OFFSET_NODES = 4096 +_MAX_OFFSET_DEPTH = 64 +_REPARSE_POINT = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + + +def _is_fts5_sql(sql: str) -> bool: + tokens = _sql_tokens(sql) + for index, (token, quoted) in enumerate(tokens[:-1]): + if not quoted and token.casefold() == "using": + return tokens[index + 1][0].casefold() == "fts5" + return False + + +def _fts_has_content_option(sql: str) -> bool: + tokens = _sql_tokens(sql) + for index, (token, quoted) in enumerate(tokens[:-1]): + next_token, next_quoted = tokens[index + 1] + if not quoted and token.casefold() == "content" and not next_quoted and next_token == "=": + return True + return False + + +def _schema_evidence( + rows: list[tuple[str, str, str | None]], + result: dict[str, object], +) -> None: + identifiers = [_reasons(name, "schema_identifier") for _, name, _ in rows] + definitions = [_reasons(sql, "schema_definition") for _, _, sql in rows if sql is not None] + result["schema_identifier_sensitive"] = sum(int("sensitive_value" in findings) for findings in identifiers) + result["schema_definition_sensitive"] = sum(int("sensitive_value" in findings) for findings in definitions) + result["scan_incomplete"] = sum(int("scan_incomplete" in findings) for findings in [*identifiers, *definitions]) + + +def _account_derived( + conn: sqlite3.Connection, + name: str, + result: dict[str, object], + *, + view: bool, +) -> None: + quoted = _quote_identifier(name) + columns = _table_columns(conn, quoted) + count = int(conn.execute(f"SELECT COUNT(*) FROM {quoted}").fetchone()[0]) + result["derived_columns_accounted"] = int(result["derived_columns_accounted"]) + len(columns) + result["derived_records"] = int(result["derived_records"]) + count + key = "derived_views_accounted" if view else "derived_tables_accounted" + result[key] = int(result[key]) + 1 + column_findings = [_reasons(column[0], "schema_identifier") for column in columns] + result["schema_identifier_sensitive"] = int(result["schema_identifier_sensitive"]) + sum( + int("sensitive_value" in findings) for findings in column_findings + ) + result["scan_incomplete"] = int(result["scan_incomplete"]) + sum( + int("scan_incomplete" in findings) for findings in column_findings + ) + + +def _shadow_tables( + conn: sqlite3.Connection, + table_names: set[str], + fts_sql: Mapping[str, str], +) -> set[str]: + try: + rows = list(conn.execute("PRAGMA table_list")) + except sqlite3.Error as exc: + raise _TableListUnavailable from exc + if not rows: + shadows: set[str] = set() + for name, sql in fts_sql.items(): + suffixes = {"data", "idx", "docsize", "config"} + if not _fts_has_content_option(sql): + suffixes.add("content") + shadows.update(f"{name}_{suffix}" for suffix in suffixes if f"{name}_{suffix}" in table_names) + return shadows + kinds = {str(row[1]): str(row[2]) for row in rows if str(row[0]) == "main"} + if not table_names.issubset(kinds): + raise sqlite3.OperationalError("shadow metadata unavailable") + shadows = {name for name, kind in kinds.items() if kind == "shadow"} + for name, sql in fts_sql.items(): + if _fts_has_content_option(sql): + shadows.discard(f"{name}_content") + return shadows + + +def _scan_authoritative( + conn: sqlite3.Connection, + table: str, + result: dict[str, object], + surfaces: dict[str, dict[str, object]], +) -> None: + quoted = _quote_identifier(table) + columns = [(name, declared) for name, declared, hidden in _table_columns(conn, quoted) if hidden != 1] + result["tables_accounted"] = int(result["tables_accounted"]) + 1 + result["columns_accounted"] = int(result["columns_accounted"]) + len(columns) + column_findings = [_reasons(name, "schema_identifier") for name, _ in columns] + result["schema_identifier_sensitive"] = int(result["schema_identifier_sensitive"]) + sum( + int("sensitive_value" in findings) for findings in column_findings + ) + result["scan_incomplete"] = int(result["scan_incomplete"]) + sum( + int("scan_incomplete" in findings) for findings in column_findings + ) + for name, declared in columns: + _ensure_surface(surfaces, _surface(table, name, declared, None)) + projection = ", ".join(_quote_identifier(name) for name, _ in columns) + for row in conn.execute(f"SELECT {projection} FROM {quoted}"): + result["records_scanned"] = int(result["records_scanned"]) + 1 + row_flagged = False + for (column, declared), value in zip(columns, row): + reasons = _reasons(value, column) + _record(surfaces, _surface(table, column, declared, value), reasons) + row_flagged = row_flagged or bool(reasons) + result["scan_incomplete"] = int(result["scan_incomplete"]) + int("scan_incomplete" in reasons) + result["records_flagged"] = int(result["records_flagged"]) + int(row_flagged) + + +def _sqlite_inventory(path: Path) -> dict[str, object]: + if not path.is_file(): + return {"reason": "sqlite_not_available", "status": "BLOCKED"} + conn: sqlite3.Connection | None = None + try: + conn = sqlite3.connect(_sqlite_uri(path), uri=True) + conn.execute("PRAGMA query_only=ON") + conn.execute("BEGIN") + master = conn.execute( + "SELECT type, name, sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name" + ).fetchall() + rows = [(str(kind), str(name), sql) for kind, name, sql in master] + result = _empty_sqlite_result() + surfaces: dict[str, dict[str, object]] = {} + _schema_evidence(rows, result) + tables = [(name, sql) for kind, name, sql in rows if kind == "table"] + views = [name for kind, name, _ in rows if kind == "view"] + all_virtual = {name for name, sql in tables if sql and "VIRTUAL TABLE" in sql.upper()} + fts_sql = {name: sql for name, sql in tables if name in all_virtual and sql and _is_fts5_sql(sql)} + fts = set(fts_sql) + shadows = _shadow_tables( + conn, + {name for name, _ in tables}, + fts_sql, + ) + for name, sql in tables: + if name in shadows or (name in fts and sql and _fts_has_content_option(sql)): + _account_derived(conn, name, result, view=False) + else: + _scan_authoritative(conn, name, result, surfaces) + for name in views: + _account_derived(conn, name, result, view=True) + result["surfaces"] = _freeze_surfaces(surfaces) + if int(result["scan_incomplete"]): + result.update({"reason": "sqlite_value_unscannable", "status": "BLOCKED"}) + return result + except _TableListUnavailable: + return {"reason": "sqlite_table_list_unavailable", "status": "BLOCKED"} + except (OSError, sqlite3.Error): + return {"reason": "sqlite_unavailable", "status": "BLOCKED"} + finally: + if conn is not None: + conn.close() + + +def _linklike(path: Path) -> bool: + try: + if path.is_symlink(): + return True + junction = getattr(path, "is_junction", None) + if junction and junction(): + return True + attributes = getattr(path.lstat(), "st_file_attributes", 0) + return bool(attributes & _REPARSE_POINT) + except OSError: + return True + + +def _empty_file_result() -> dict[str, object]: + return { + "entries_accounted": 0, + "entry_limit_reached": 0, + "files_scanned": 0, + "files_unscannable": 0, + "files_unavailable": 0, + "files_refused": 0, + "reason": "completed", + "sensitive_files": 0, + "status": "COMPLETED", + "truncated_files": 0, + "surfaces": {}, + } + + +def _valid_file_bounds(chunk_size: int, max_file_bytes: int, max_entries: int) -> bool: + return ( + isinstance(chunk_size, int) + and not isinstance(chunk_size, bool) + and 1 <= chunk_size <= _MAX_CHUNK_BYTES + and isinstance(max_file_bytes, int) + and not isinstance(max_file_bytes, bool) + and 1 <= max_file_bytes <= _MAX_FILE_BYTES + and isinstance(max_entries, int) + and not isinstance(max_entries, bool) + and 1 <= max_entries <= _MAX_ENTRIES + ) + + +def _stat_fingerprint(stat_result: object) -> tuple[object, ...]: + return ( + int(getattr(stat_result, "st_size")), + getattr(stat_result, "st_mtime_ns", None), + getattr(stat_result, "st_ctime_ns", None), + getattr(stat_result, "st_dev", None), + getattr(stat_result, "st_ino", None), + ) + + +def _open_file_fingerprint(handle: object, path: Path) -> tuple[object, ...]: + try: + fileno = getattr(handle, "fileno")() + return _stat_fingerprint(os.fstat(fileno)) + except (AttributeError, OSError, TypeError): + return _stat_fingerprint(path.stat()) + + +def _cross_source_identity(fingerprint: tuple[object, ...]) -> tuple[object, ...]: + size, modified, _changed, device, inode = fingerprint + return size, modified, device, inode + + +def _file_changed( + path_start: tuple[object, ...], + path_end: tuple[object, ...], + open_start: tuple[object, ...], + open_end: tuple[object, ...], +) -> bool: + return ( + open_start != open_end + or path_start != path_end + or _cross_source_identity(path_start) != _cross_source_identity(open_start) + or _cross_source_identity(path_end) != _cross_source_identity(open_end) + ) + + +def _file_surface(path: Path, undecodable: bool) -> str: + if undecodable: + return "file.binary" + suffix = path.suffix.lower() + if suffix in {".json", ".jsonl", ".ndjson"}: + return "file.json" + if suffix in {".txt", ".md", ".log", ".csv"}: + return "file.text" + return "file.other" + + +class _FileIdentityChanged(ValueError): + pass + + +def _verified_open_fingerprint( + handle: object, + path: Path, + root: Path, + path_start: tuple[object, ...], +) -> tuple[object, ...]: + opened = _open_file_fingerprint(handle, path) + path.resolve(strict=True).relative_to(root) + path_open = _stat_fingerprint(path.stat()) + if path_start != path_open or _cross_source_identity(path_open) != _cross_source_identity(opened): + raise _FileIdentityChanged + return opened + + +def _read_file_snapshot( + handle: object, + size: int, + *, + chunk_size: int, + max_file_bytes: int, +) -> tuple[bytes, bool]: + scan_limit = min(size, max_file_bytes, _MAX_SCAN_TEXT_BYTES + 1) + content = bytearray() + while len(content) < scan_limit: + chunk = getattr(handle, "read")(min(chunk_size, scan_limit - len(content))) + if not chunk: + break + content.extend(chunk) + incomplete = len(content) < min(size, max_file_bytes) or size > _MAX_SCAN_TEXT_BYTES + return bytes(content), incomplete + + +def _decode_file(content: bytes) -> tuple[str, bool]: + try: + return content.decode("utf-8"), False + except UnicodeDecodeError: + return content.decode("utf-8", errors="ignore"), True + + +def _scan_file( + path: Path, + root: Path, + *, + chunk_size: int, + max_file_bytes: int, +) -> tuple[str, set[str], set[str], bool]: + path_start_fingerprint = _stat_fingerprint(path.stat()) + with path.open("rb") as handle: + start_fingerprint = _verified_open_fingerprint(handle, path, root, path_start_fingerprint) + size = int(start_fingerprint[0]) + content, incomplete = _read_file_snapshot( + handle, + size, + chunk_size=chunk_size, + max_file_bytes=max_file_bytes, + ) + end_fingerprint = _open_file_fingerprint(handle, path) + path_end_fingerprint = _stat_fingerprint(path.stat()) + changed = _file_changed( + path_start_fingerprint, + path_end_fingerprint, + start_fingerprint, + end_fingerprint, + ) + decoded, undecodable = _decode_file(content) + reasons = _reasons(decoded, "content") + if changed or incomplete: + reasons.add("scan_incomplete") + if undecodable: + reasons.add("binary_opaque") + relative = " ".join(path.relative_to(root).parts) + return ( + _file_surface(path, undecodable), + reasons, + _reasons(relative, "metadata"), + size > max_file_bytes or incomplete or changed, + ) + + +def _account_file( + candidate: Path, + root: Path, + result: dict[str, object], + surfaces: dict[str, dict[str, object]], + *, + chunk_size: int, + max_file_bytes: int, +) -> None: + if _linklike(candidate): + result["files_refused"] = int(result["files_refused"]) + 1 + return + try: + resolved = candidate.resolve(strict=True) + resolved.relative_to(root) + surface, reasons, metadata, truncated = _scan_file( + resolved, + root, + chunk_size=chunk_size, + max_file_bytes=max_file_bytes, + ) + _record(surfaces, surface, reasons) + _record(surfaces, "file.metadata", metadata) + result["files_scanned"] = int(result["files_scanned"]) + 1 + result["files_unscannable"] = int(result["files_unscannable"]) + int( + "scan_incomplete" in reasons or "scan_incomplete" in metadata + ) + result["sensitive_files"] = int(result["sensitive_files"]) + int(bool(reasons or metadata)) + result["truncated_files"] = int(result["truncated_files"]) + int(truncated) + except _FileIdentityChanged: + result["files_unscannable"] = int(result["files_unscannable"]) + 1 + except ValueError: + result["files_refused"] = int(result["files_refused"]) + 1 + except OSError: + result["files_unavailable"] = int(result["files_unavailable"]) + 1 + + +def _walk_file_root( + root: Path, + result: dict[str, object], + surfaces: dict[str, dict[str, object]], + *, + chunk_size: int, + max_file_bytes: int, + max_entries: int, +) -> bool: + pending = [root] + while pending: + current = pending.pop() + try: + with os.scandir(current) as entries: + iterator = iter(entries) + while int(result["entries_accounted"]) < max_entries: + try: + entry = next(iterator) + except StopIteration: + break + result["entries_accounted"] = int(result["entries_accounted"]) + 1 + candidate = Path(entry.path) + if _linklike(candidate): + result["files_refused"] = int(result["files_refused"]) + 1 + continue + try: + if entry.is_dir(follow_symlinks=False): + pending.append(candidate) + elif entry.is_file(follow_symlinks=False): + _account_file( + candidate, + root, + result, + surfaces, + chunk_size=chunk_size, + max_file_bytes=max_file_bytes, + ) + else: + result["files_refused"] = int(result["files_refused"]) + 1 + except OSError: + result["files_unavailable"] = int(result["files_unavailable"]) + 1 + if int(result["entries_accounted"]) >= max_entries: + return False + except OSError: + result["files_unavailable"] = int(result["files_unavailable"]) + 1 + return True + + +def _entry_limit_result(max_entries: int) -> dict[str, object]: + result = _empty_file_result() + result.update( + { + "entries_accounted": max_entries, + "entry_limit_reached": 1, + "reason": "entry_limit", + "status": "BLOCKED", + } + ) + return result + + +def _finalize_file_result( + result: dict[str, object], + surfaces: dict[str, dict[str, object]], + *, + root_refused: bool, +) -> dict[str, object]: + result["surfaces"] = _freeze_surfaces(surfaces) + result["symlink_refused"] = result["files_refused"] + if root_refused: + result.update({"reason": "root_refused", "status": "BLOCKED"}) + elif int(result["files_unavailable"]): + result.update({"reason": "file_unavailable", "status": "BLOCKED"}) + elif int(result["files_unscannable"]): + result.update({"reason": "file_unscannable", "status": "BLOCKED"}) + elif int(result["truncated_files"]): + result.update({"reason": "file_truncated", "status": "BLOCKED"}) + return result + + +def _file_inventory( + roots: Iterable[Path], + *, + chunk_size: int, + max_file_bytes: int, + max_entries: int, +) -> dict[str, object]: + requested = list(roots) + result = _empty_file_result() + surfaces: dict[str, dict[str, object]] = {} + if not _valid_file_bounds(chunk_size, max_file_bytes, max_entries): + result.update({"reason": "invalid_file_bounds", "status": "BLOCKED"}) + return result + if not requested: + result.update({"reason": "roots_not_provided", "status": "BLOCKED"}) + return result + root_refused = False + for supplied in requested: + try: + root = supplied.resolve(strict=True) + except OSError: + result["files_unavailable"] = int(result["files_unavailable"]) + 1 + continue + if not root.is_dir() or _linklike(supplied): + result["files_refused"] = int(result["files_refused"]) + 1 + root_refused = True + continue + complete = _walk_file_root( + root, + result, + surfaces, + chunk_size=chunk_size, + max_file_bytes=max_file_bytes, + max_entries=max_entries, + ) + if not complete: + return _entry_limit_result(max_entries) + return _finalize_file_result(result, surfaces, root_refused=root_refused) + + +def _qdrant_block( + reason: str, + total: int = 0, + surfaces: dict[str, dict[str, object]] | None = None, +) -> dict[str, object]: + result: dict[str, object] = { + "reason": reason, + "status": "BLOCKED-EXTERNAL", + } + if total or surfaces: + result["payloads_scanned"] = total + result["surfaces"] = _freeze_surfaces(surfaces or {}) + return result + + +def _offset_marker(offset: object) -> str | None: + if not _offset_shape_bounded(offset): + return None + try: + encoded = json.dumps( + offset, + allow_nan=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + except (RecursionError, TypeError, ValueError): + return None + if len(encoded) > _MAX_OFFSET_BYTES: + return None + return sha256(encoded).hexdigest() + + +def _offset_shape_bounded(offset: object) -> bool: + pending: list[tuple[object, int]] = [(offset, 0)] + containers: set[int] = set() + nodes = 0 + text_bytes = 0 + while pending: + value, depth = pending.pop() + nodes += 1 + if nodes > _MAX_OFFSET_NODES or depth > _MAX_OFFSET_DEPTH: + return False + if value is None or type(value) is bool: + continue + if type(value) is str: + if len(value) > _MAX_OFFSET_BYTES: + return False + try: + text_bytes += len(value.encode("utf-8")) + except UnicodeEncodeError: + return False + if text_bytes > _MAX_OFFSET_BYTES: + return False + continue + if type(value) is int: + if value.bit_length() > _MAX_OFFSET_BYTES * 8: + return False + continue + if type(value) is float: + if not isfinite(value): + return False + continue + if type(value) not in {dict, list}: + return False + identity = id(value) + if identity in containers: + return False + containers.add(identity) + if len(pending) + len(value) > _MAX_OFFSET_NODES: + return False + if type(value) is list: + pending.extend((item, depth + 1) for item in value) + continue + for key, nested in value.items(): + if type(key) is not str: + return False + pending.append((key, depth + 1)) + pending.append((nested, depth + 1)) + return True + + +def _qdrant_payload_reasons(payload: object) -> set[str]: + if type(payload) is not dict: + return {"scan_incomplete"} + pending: list[tuple[object, int]] = [(payload, 0)] + containers: set[int] = set() + nodes = 0 + text_bytes = 0 + while pending: + value, depth = pending.pop() + nodes += 1 + if nodes > _MAX_SCAN_NODES or depth > _MAX_SCAN_DEPTH: + return {"scan_incomplete"} + if value is None or type(value) is bool: + continue + if type(value) is bytes: + return {"binary_opaque"} + if type(value) is str: + try: + text_bytes += len(value.encode("utf-8")) + except UnicodeEncodeError: + return {"scan_incomplete"} + if text_bytes > _MAX_SCAN_TEXT_BYTES: + return {"scan_incomplete"} + continue + if type(value) is int: + if value.bit_length() > _MAX_SCAN_TEXT_BYTES * 8: + return {"scan_incomplete"} + continue + if type(value) is float: + if not isfinite(value): + return {"scan_incomplete"} + continue + if type(value) not in {dict, list} or id(value) in containers: + return {"scan_incomplete"} + containers.add(id(value)) + if len(pending) + len(value) > _MAX_SCAN_NODES: + return {"scan_incomplete"} + if type(value) is dict: + for key, nested in value.items(): + if type(key) is not str: + return {"scan_incomplete"} + pending.append((key, depth + 1)) + pending.append((nested, depth + 1)) + else: + pending.extend((item, depth + 1) for item in value) + return _reasons(payload, "payload") + + +def _valid_qdrant_bounds(limit: object, max_pages: object) -> bool: + return ( + isinstance(limit, int) + and not isinstance(limit, bool) + and isinstance(max_pages, int) + and not isinstance(max_pages, bool) + and 1 <= limit <= 1000 + and 1 <= max_pages <= 10000 + ) + + +def scan_qdrant_payloads( + page: Callable[..., Mapping[str, object]], + *, + limit: int = 100, + max_pages: int = 1000, +) -> dict[str, object]: + if not _valid_qdrant_bounds(limit, max_pages): + return {"reason": "qdrant_invalid_bounds", "status": "BLOCKED-EXTERNAL"} + surfaces: dict[str, dict[str, object]] = {} + offset: object = None + seen: set[str] = set() + total = 0 + for _ in range(max_pages): + try: + response = page(limit=limit, with_payload=True, with_vector=False, offset=offset) + except Exception: + return _qdrant_block("qdrant_transport_error", total, surfaces) + envelope = response.get("result") if type(response) is dict else None + if type(envelope) is not dict or type(envelope.get("points")) is not list or "next_page_offset" not in envelope: + return _qdrant_block("qdrant_malformed_response", total, surfaces) + points = envelope["points"] + if len(points) > limit: + return _qdrant_block("qdrant_page_oversized", total, surfaces) + for point in points: + if type(point) is not dict: + return _qdrant_block("qdrant_malformed_point", total, surfaces) + if "payload" not in point: + return _qdrant_block("qdrant_payload_missing", total, surfaces) + total += 1 + reasons = _qdrant_payload_reasons(point["payload"]) + _record(surfaces, "qdrant.payload", reasons) + if reasons & {"scan_incomplete", "binary_opaque"}: + return _qdrant_block("qdrant_payload_unscannable", total, surfaces) + offset = envelope.get("next_page_offset") + if offset is None: + return {"payloads_scanned": total, "status": "COMPLETED", "surfaces": _freeze_surfaces(surfaces)} + marker = _offset_marker(offset) + if marker is None: + return _qdrant_block("qdrant_malformed_offset", total, surfaces) + if marker in seen: + return _qdrant_block("qdrant_repeated_offset", total, surfaces) + seen.add(marker) + return _qdrant_block("qdrant_page_limit", total, surfaces) + + +def run_inventory( + db_path: str | Path | None, + *, + artifact_roots: Iterable[str | Path] = (), + spool_roots: Iterable[str | Path] = (), + qdrant_page: Callable[..., Mapping[str, object]] | None = None, + chunk_size: int = 65536, + max_file_bytes: int = 1048576, + max_entries: int = 100000, +) -> dict[str, object]: + """Perform a deterministic dry-run without returning raw schema or values.""" + sqlite = ( + _sqlite_inventory(Path(db_path)) + if db_path is not None + else {"reason": "sqlite_not_configured", "status": "BLOCKED"} + ) + qdrant = ( + scan_qdrant_payloads(qdrant_page) + if qdrant_page + else {"reason": "qdrant_not_configured", "status": "BLOCKED-EXTERNAL"} + ) + artifacts = _file_inventory( + (Path(item) for item in artifact_roots), + chunk_size=chunk_size, + max_file_bytes=max_file_bytes, + max_entries=max_entries, + ) + spool = _file_inventory( + (Path(item) for item in spool_roots), + chunk_size=chunk_size, + max_file_bytes=max_file_bytes, + max_entries=max_entries, + ) + local_blocked = ( + sqlite.get("status") == "BLOCKED" or artifacts["status"] == "BLOCKED" or spool["status"] == "BLOCKED" + ) + status = "BLOCKED" if local_blocked else qdrant["status"] + return { + "artifacts": artifacts, + "classification": "LEGACY-SENSITIVITY-INVENTORY", + "mode": "dry_run", + "qdrant": qdrant, + "recommendation": "REVIEW_ONLY", + "spool": spool, + "sqlite": sqlite, + "status": status, + } diff --git a/scripts/sensitivity_inventory.py b/scripts/sensitivity_inventory.py new file mode 100644 index 00000000..3245193e --- /dev/null +++ b/scripts/sensitivity_inventory.py @@ -0,0 +1,53 @@ +"""CLI entry point for the aggregate-only legacy sensitivity inventory.""" + +from __future__ import annotations + +import argparse +import json +from collections.abc import Sequence + +from memorymaster.govern.jobs.sensitivity_inventory import run_inventory + + +class _SafeArgumentParser(argparse.ArgumentParser): + def error(self, _message: str) -> None: + raise ValueError("invalid inventory arguments") + + +def _invalid_arguments() -> dict[str, str]: + return { + "classification": "LEGACY-SENSITIVITY-INVENTORY", + "mode": "dry_run", + "reason": "invalid_arguments", + "recommendation": "REVIEW_ONLY", + "status": "BLOCKED", + } + + +def main(argv: Sequence[str] | None = None) -> int: + parser = _SafeArgumentParser(add_help=True) + parser.add_argument("--db", required=True) + parser.add_argument("--artifact-root", action="append", default=[]) + parser.add_argument("--spool-root", action="append", default=[]) + parser.add_argument("--chunk-size", type=int, default=65536) + parser.add_argument("--max-file-bytes", type=int, default=1048576) + parser.add_argument("--max-entries", type=int, default=100000) + try: + args = parser.parse_args(argv) + except ValueError: + print(json.dumps(_invalid_arguments(), sort_keys=True, separators=(",", ":"))) + return 3 + result = run_inventory( + args.db, + artifact_roots=args.artifact_root, + spool_roots=args.spool_root, + chunk_size=args.chunk_size, + max_file_bytes=args.max_file_bytes, + max_entries=args.max_entries, + ) + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + return {"COMPLETED": 0, "BLOCKED": 3, "BLOCKED-EXTERNAL": 4}[str(result["status"])] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_sensitivity_inventory.py b/tests/test_sensitivity_inventory.py new file mode 100644 index 00000000..68a73d4f --- /dev/null +++ b/tests/test_sensitivity_inventory.py @@ -0,0 +1,703 @@ +from __future__ import annotations + +import io +import json +import os +import sqlite3 +import stat +import subprocess +from base64 import b64encode +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from memorymaster.govern.jobs.sensitivity_inventory import run_inventory, scan_qdrant_payloads +from memorymaster.govern.jobs import sensitivity_inventory + + +SECRET = "sk-" + "A" * 24 +ENCODED_SECRET = b64encode(SECRET.encode()).decode() + + +def _database(path: Path) -> None: + conn = sqlite3.connect(path) + conn.executescript( + """ + CREATE TABLE claims (id INTEGER PRIMARY KEY, text TEXT, metadata TEXT, embedding_json TEXT, blob BLOB); + CREATE TABLE citations (id INTEGER PRIMARY KEY, excerpt TEXT); + CREATE TABLE verbatim (id INTEGER PRIMARY KEY, body TEXT); + CREATE TABLE atlas_evidence (id INTEGER PRIMARY KEY, payload TEXT); + CREATE TABLE future_surface (id INTEGER PRIMARY KEY, unknown_value TEXT); + CREATE TABLE "sk-AAAAAAAAAAAAAAAAAAAAAAAA" ("sk-AAAAAAAAAAAAAAAAAAAAAAAA" TEXT); + CREATE VIRTUAL TABLE search_fts USING /* derived */ fts5(body); + """ + ) + conn.execute( + "INSERT INTO claims VALUES (1, ?, ?, ?, ?)", + ( + SECRET, + json.dumps({"api_key": ENCODED_SECRET}), + "[1, 2.5]", + SECRET.encode(), + ), + ) + conn.execute("INSERT INTO claims VALUES (2, 'safe', '{}', ?, ?)", ('["not numeric"]', bytes([255, 0]))) + conn.execute("INSERT INTO claims VALUES (3, 'safe', '{}', ?, ?)", ("[broken", b"safe")) + conn.execute("INSERT INTO citations VALUES (9, ?)", (SECRET,)) + conn.execute("INSERT INTO verbatim VALUES (10, ?)", (SECRET,)) + conn.execute("INSERT INTO atlas_evidence VALUES (11, ?)", (json.dumps({"nested": SECRET}),)) + conn.execute("INSERT INTO future_surface VALUES (12, ?)", (SECRET,)) + conn.execute('INSERT INTO "sk-AAAAAAAAAAAAAAAAAAAAAAAA" VALUES (?)', ("safe",)) + conn.execute("INSERT INTO search_fts VALUES (?)", (SECRET,)) + conn.commit() + conn.close() + + +def _serialized(result: dict[str, object]) -> str: + return json.dumps(result, sort_keys=True) + + +def test_artifact_and_spool_chunking_and_truncation_are_aggregate_only(tmp_path: Path) -> None: + root = tmp_path / "inside" + root.mkdir() + (root / "a.txt").write_text("x" * 14 + "\n" + SECRET) + (root / "large.txt").write_text("safe" * 100) + result = run_inventory(None, artifact_roots=[root], spool_roots=[root], chunk_size=16, max_file_bytes=64) + text = _serialized(result) + assert result["artifacts"]["truncated_files"] >= 1 + assert result["artifacts"]["status"] == "BLOCKED" + assert result["artifacts"]["reason"] == "file_truncated" + assert result["artifacts"]["sensitive_files"] >= 1 + assert result["spool"]["status"] == "BLOCKED" + assert result["spool"]["sensitive_files"] >= 1 + assert SECRET not in text and str(root) not in text and "a.txt" not in text + + +def test_artifact_symlink_escape_is_refused_when_supported(tmp_path: Path) -> None: + root = tmp_path / "inside" + root.mkdir() + outside = tmp_path / "outside.txt" + outside.write_text(SECRET) + link = root / "escape.txt" + try: + link.symlink_to(outside) + except OSError: + pytest.skip("symlinks unavailable") + + result = run_inventory(None, artifact_roots=[root]) + assert result["artifacts"]["symlink_refused"] >= 1 + + +def test_qdrant_payload_only_two_pages_and_blocker(tmp_path: Path) -> None: + calls: list[dict[str, object]] = [] + + def page(**kwargs: object) -> dict[str, object]: + calls.append(kwargs) + if kwargs["offset"] is None: + return { + "result": { + "points": [{"id": "do-not-emit", "payload": {"token": SECRET}}], + "next_page_offset": "opaque-next", + } + } + return {"result": {"points": [{"id": "do-not-emit-2", "payload": {"safe": "yes"}}], "next_page_offset": None}} + + scanned = scan_qdrant_payloads(page) + root = tmp_path / "inputs" + root.mkdir() + db = tmp_path / "safe.sqlite" + with sqlite3.connect(db) as conn: + conn.execute("CREATE TABLE safe_table (value TEXT)") + blocked = run_inventory(db, artifact_roots=[root], spool_roots=[root]) + assert scanned["status"] == "COMPLETED" + assert scanned["payloads_scanned"] == 2 + assert scanned["surfaces"]["qdrant.payload"]["finding_counts"]["sensitive_value"] == 1 + assert all(call["with_payload"] is True and call["with_vector"] is False for call in calls) + assert [call["offset"] for call in calls] == [None, "opaque-next"] + assert SECRET not in _serialized(scanned) and "do-not-emit" not in _serialized(scanned) + assert blocked["qdrant"] == {"reason": "qdrant_not_configured", "status": "BLOCKED-EXTERNAL"} + assert blocked["status"] == "BLOCKED-EXTERNAL" + + +def test_qdrant_repeated_offset_and_malformed_responses_are_bounded() -> None: + repeated = scan_qdrant_payloads( + lambda **_kwargs: {"result": {"points": [], "next_page_offset": "again"}}, max_pages=3 + ) + malformed = scan_qdrant_payloads(lambda **_kwargs: {"result": "bad"}) + assert repeated["status"] == "BLOCKED-EXTERNAL" and repeated["reason"] == "qdrant_repeated_offset" + assert malformed == {"reason": "qdrant_malformed_response", "status": "BLOCKED-EXTERNAL"} + + +def test_fixed_categories_cover_legacy_families_without_schema_names(tmp_path: Path) -> None: + db = tmp_path / "categories.sqlite" + conn = sqlite3.connect(db) + names = ( + "verbatim_memories", + "usage_feedback", + "external_sources", + "source_items", + "evidence_items", + "action_proposals", + "media_retry_queue", + "query_cache", + "cache_meta", + "miner_state", + "rule_records", + "entity_records", + "link_records", + "timeline_entries", + ) + for name in names: + conn.execute(f'CREATE TABLE "{name}" (id INTEGER, value TEXT)') + conn.execute(f'INSERT INTO "{name}" VALUES (1, "safe")') + conn.commit() + conn.close() + result = run_inventory( + db, + artifact_roots=[tmp_path], + spool_roots=[tmp_path], + qdrant_page=lambda **_kwargs: {"result": {"points": [], "next_page_offset": None}}, + ) + surfaces = result["sqlite"]["surfaces"] + for category in ( + "verbatim", + "feedback", + "atlas.source", + "atlas.evidence", + "atlas.action", + "atlas.media_retry", + "cache", + "miner", + "rules", + "entities", + "timeline", + ): + assert f"{category}.id" in surfaces + assert all(name not in _serialized(result) for name in names) + + +@pytest.mark.parametrize( + "sql", + [ + 'CREATE VIRTUAL TABLE x USING "fts5"(body)', + "CREATE VIRTUAL TABLE x USING [fts5](body)", + "CREATE VIRTUAL TABLE x USING /* comment */ fts5(body)", + ], +) +def test_fts5_parser_accepts_quoted_and_commented_module_names(sql: str) -> None: + assert sensitivity_inventory._is_fts5_sql(sql) is True + + +def test_fts5_comment_tokens_inside_quoted_name_keep_all_copies_derived( + tmp_path: Path, +) -> None: + db = tmp_path / "quoted-name.sqlite" + with sqlite3.connect(db) as conn: + conn.execute('CREATE VIRTUAL TABLE "x--hidden" USING fts5(body)') + conn.execute('INSERT INTO "x--hidden" VALUES ("safe")') + objects = [ + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' " + "AND (name = 'x--hidden' OR name LIKE 'x--hidden_%')" + ) + ] + root = tmp_path / "root" + root.mkdir() + + result = run_inventory( + db, + artifact_roots=[root], + spool_roots=[root], + qdrant_page=lambda **_kwargs: {"result": {"points": [], "next_page_offset": None}}, + ) + + assert result["sqlite"]["derived_tables_accounted"] == len(objects) - 1 + assert result["sqlite"]["tables_accounted"] == 1 + assert result["sqlite"]["records_scanned"] == 1 + assert "x--hidden" not in _serialized(result) + + +def test_fts5_table_named_using_keeps_all_copies_derived(tmp_path: Path) -> None: + db = tmp_path / "quoted-keyword.sqlite" + with sqlite3.connect(db) as conn: + conn.execute('CREATE VIRTUAL TABLE "using" USING "fts5"(body)') + conn.execute('INSERT INTO "using" VALUES ("safe")') + objects = [ + row[0] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' AND (name = 'using' OR name LIKE 'using_%')" + ) + ] + root = tmp_path / "root" + root.mkdir() + + result = run_inventory( + db, + artifact_roots=[root], + spool_roots=[root], + qdrant_page=lambda **_kwargs: {"result": {"points": [], "next_page_offset": None}}, + ) + + assert result["sqlite"]["derived_tables_accounted"] == len(objects) - 1 + assert result["sqlite"]["tables_accounted"] == 1 + assert result["sqlite"]["records_scanned"] == 1 + + +def test_external_content_fts_does_not_hide_user_owned_suffix_table( + tmp_path: Path, +) -> None: + db = tmp_path / "external-content.sqlite" + with sqlite3.connect(db) as conn: + conn.executescript( + """ + CREATE TABLE external_docs ( + rowid INTEGER PRIMARY KEY, + body TEXT + ); + CREATE TABLE docs_content (secret TEXT); + CREATE VIRTUAL TABLE docs USING fts5( + body, + content='external_docs', + content_rowid='rowid' + ); + """ + ) + conn.execute("INSERT INTO docs_content VALUES (?)", (SECRET,)) + root = tmp_path / "root" + root.mkdir() + + result = run_inventory( + db, + artifact_roots=[root], + spool_roots=[root], + qdrant_page=lambda **_kwargs: {"result": {"points": [], "next_page_offset": None}}, + ) + + sqlite = result["sqlite"] + assert sqlite["records_scanned"] == 1 + assert sqlite["records_flagged"] == 1 + assert sqlite["surfaces"]["unknown.field_text"]["finding_counts"] == {"sensitive_value": 1} + assert "docs_content" not in _serialized(result) + + +def test_file_growth_after_size_check_is_incomplete() -> None: + class TrackingBytesIO(io.BytesIO): + bytes_read = 0 + + def read(self, size: int = -1) -> bytes: + data = super().read(size) + self.bytes_read += len(data) + return data + + class GrowingPath: + suffix = ".txt" + sizes = iter((4, 4, 4, 5, 5)) + handle = TrackingBytesIO(b"safe" + SECRET.encode()) + + def stat(self) -> SimpleNamespace: + return SimpleNamespace(st_size=next(self.sizes)) + + def open(self, _mode: str) -> TrackingBytesIO: + return self.handle + + def resolve(self, *, strict: bool) -> GrowingPath: + assert strict is True + return self + + def relative_to(self, _root: object) -> SimpleNamespace: + return SimpleNamespace(parts=("safe.txt",)) + + path = GrowingPath() + _surface, reasons, _metadata, truncated = sensitivity_inventory._scan_file( + path, + object(), + chunk_size=64, + max_file_bytes=64, + ) + + assert "scan_incomplete" in reasons + assert truncated is True + assert path.handle.bytes_read <= 4 + + +def _fake_stat(version: int) -> SimpleNamespace: + return SimpleNamespace(st_size=4, st_mtime_ns=version, st_ctime_ns=version, st_dev=1, st_ino=1) + + +class _ChangingPath: + suffix = ".txt" + + def __init__(self) -> None: + self.stats = iter([_fake_stat(1), _fake_stat(1), _fake_stat(1), _fake_stat(2), _fake_stat(2)]) + + def stat(self) -> SimpleNamespace: + return next(self.stats) + + def open(self, _mode: str) -> io.BytesIO: + return io.BytesIO(b"safe") + + def resolve(self, *, strict: bool) -> _ChangingPath: + assert strict is True + return self + + def relative_to(self, _root: object) -> SimpleNamespace: + return SimpleNamespace(parts=("safe.txt",)) + + +def test_equal_size_file_overwrite_is_incomplete() -> None: + + _surface, reasons, _metadata, truncated = sensitivity_inventory._scan_file( + _ChangingPath(), + object(), + chunk_size=2, + max_file_bytes=4, + ) + + assert "scan_incomplete" in reasons + assert truncated is True + + +def test_file_metadata_root_blocking_and_refusal_are_aggregate_only( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + root = tmp_path / "files" + root.mkdir() + encoded_name = b64encode(SECRET.encode()).decode() + ".json" + (root / encoded_name).write_text('{"safe": true}') + result = run_inventory(None, artifact_roots=[root], spool_roots=[tmp_path / "missing"]) + assert result["status"] == "BLOCKED" + assert result["artifacts"]["surfaces"]["file.metadata"]["finding_counts"]["sensitive_value"] == 1 + assert result["artifacts"]["sensitive_files"] == 1 + assert result["spool"]["status"] == "BLOCKED" + assert encoded_name not in _serialized(result) + (root / "refused.txt").write_text("safe") + (root / "kept.txt").write_text("safe") + monkeypatch.setattr( + sensitivity_inventory, + "_linklike", + lambda path: path.name == "refused.txt", + ) + refused = run_inventory(None, artifact_roots=[root]) + assert refused["artifacts"]["files_refused"] == 1 + assert refused["artifacts"]["status"] == "COMPLETED" + + +def test_sqlite_accounts_views_and_schema_definition_secrets(tmp_path: Path) -> None: + db = tmp_path / "views.sqlite" + conn = sqlite3.connect(db) + conn.executescript( + f""" + CREATE TABLE safe_table ( + value TEXT DEFAULT '{SECRET}' + ); + INSERT INTO safe_table DEFAULT VALUES; + CREATE VIEW safe_projection AS SELECT value FROM safe_table; + """ + ) + conn.close() + + result = run_inventory( + db, + artifact_roots=[tmp_path], + spool_roots=[tmp_path], + qdrant_page=lambda **_kwargs: {"result": {"points": [], "next_page_offset": None}}, + ) + + sqlite = result["sqlite"] + assert sqlite["schema_definition_sensitive"] == 1 + assert sqlite["derived_views_accounted"] == 1 + assert sqlite["derived_records"] == 1 + assert SECRET not in _serialized(result) + assert "safe_projection" not in _serialized(result) + + +def test_missing_db_invalid_file_bounds_and_root_refusal_block(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root = tmp_path / "inputs" + root.mkdir() + + def qdrant(**_kwargs: object) -> dict[str, object]: + return {"result": {"points": [], "next_page_offset": None}} + + missing_db = run_inventory( + None, + artifact_roots=[root], + spool_roots=[root], + qdrant_page=qdrant, + ) + assert missing_db["status"] == "BLOCKED" + assert missing_db["sqlite"] == { + "reason": "sqlite_not_configured", + "status": "BLOCKED", + } + + db = tmp_path / "safe.sqlite" + with sqlite3.connect(db) as conn: + conn.execute("CREATE TABLE safe_table (value TEXT)") + invalid_bounds = run_inventory( + db, + artifact_roots=[root], + spool_roots=[root], + qdrant_page=qdrant, + chunk_size=0, + ) + assert invalid_bounds["status"] == "BLOCKED" + assert invalid_bounds["artifacts"]["reason"] == "invalid_file_bounds" + + monkeypatch.setattr(sensitivity_inventory, "_linklike", lambda path: path == root) + refused_root = run_inventory( + db, + artifact_roots=[root], + spool_roots=[root], + qdrant_page=qdrant, + ) + assert refused_root["status"] == "BLOCKED" + assert refused_root["artifacts"]["reason"] == "root_refused" + + +def test_file_entry_limit_blocks_with_deterministic_aggregate(tmp_path: Path) -> None: + db = tmp_path / "safe.sqlite" + with sqlite3.connect(db) as conn: + conn.execute("CREATE TABLE safe_table (value TEXT)") + root = tmp_path / "many" + root.mkdir() + for index in range(5): + (root / f"entry-{index}.txt").write_text("safe") + + result = run_inventory( + db, + artifact_roots=[root], + spool_roots=[root], + qdrant_page=lambda **_kwargs: {"result": {"points": [], "next_page_offset": None}}, + max_entries=2, + ) + + assert result["status"] == "BLOCKED" + assert result["artifacts"]["reason"] == "entry_limit" + assert result["artifacts"]["entries_accounted"] == 2 + assert result["artifacts"]["files_scanned"] == 0 + assert "entry-" not in _serialized(result) + + +def test_entry_limit_does_not_fetch_one_extra_directory_entry(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + root = tmp_path / "root" + root.mkdir() + + class Entry: + path = str(root / "unreadable") + + class CountingScan: + calls = 0 + + def __enter__(self) -> CountingScan: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def __iter__(self) -> CountingScan: + return self + + def __next__(self) -> Entry: + self.calls += 1 + return Entry() + + scan = CountingScan() + monkeypatch.setattr(sensitivity_inventory.os, "scandir", lambda _root: scan) + + result = sensitivity_inventory._file_inventory( + [root], + chunk_size=16, + max_file_bytes=64, + max_entries=2, + ) + + assert result["reason"] == "entry_limit" + assert scan.calls <= 2 + + +def test_windows_reparse_attribute_is_linklike_without_path_is_junction( + monkeypatch: pytest.MonkeyPatch, +) -> None: + reparse = getattr(stat, "FILE_ATTRIBUTE_REPARSE_POINT", 0x400) + + class ReparseStat: + st_file_attributes = reparse + + monkeypatch.setattr(Path, "is_symlink", lambda _path: False) + monkeypatch.setattr(Path, "is_junction", lambda _path: False, raising=False) + monkeypatch.setattr(Path, "lstat", lambda _path: ReparseStat()) + + assert sensitivity_inventory._linklike(Path("junction")) is True + + +@pytest.mark.skipif(os.name != "nt", reason="Windows junction semantics") +def test_real_windows_junction_escape_is_refused(tmp_path: Path) -> None: + root = tmp_path / "root" + outside = tmp_path / "outside" + spool = tmp_path / "spool" + root.mkdir() + outside.mkdir() + spool.mkdir() + (outside / "secret.txt").write_text(SECRET) + junction = root / "escape" + created = subprocess.run( + ["cmd", "/c", "mklink", "/J", str(junction), str(outside)], + capture_output=True, + check=False, + text=True, + ) + if created.returncode != 0: + pytest.skip("junction creation unavailable") + db = tmp_path / "safe.sqlite" + with sqlite3.connect(db) as conn: + conn.execute("CREATE TABLE safe_table (value TEXT)") + try: + result = run_inventory( + db, + artifact_roots=[root], + spool_roots=[spool], + qdrant_page=lambda **_kwargs: {"result": {"points": [], "next_page_offset": None}}, + ) + finally: + junction.rmdir() + + assert result["artifacts"]["files_refused"] == 1 + assert result["artifacts"]["files_scanned"] == 0 + assert result["artifacts"]["sensitive_files"] == 0 + assert SECRET not in _serialized(result) + + +def test_qdrant_fail_closed_bounds_keep_partial_aggregates() -> None: + secret_error = "transport " + SECRET + calls = 0 + + def unhashable(**_kwargs: object) -> dict[str, object]: + nonlocal calls + calls += 1 + if calls == 1: + return {"result": {"points": [{"payload": {"token": SECRET}}], "next_page_offset": ["opaque"]}} + return {"result": {"points": [], "next_page_offset": ["opaque"]}} + + repeated = scan_qdrant_payloads(unhashable, limit=1, max_pages=3) + malformed_point = scan_qdrant_payloads( + lambda **_kwargs: {"result": {"points": ["not-a-point"], "next_page_offset": None}} + ) + limited = scan_qdrant_payloads( + lambda **_kwargs: {"result": {"points": [], "next_page_offset": {"opaque": 1}}}, max_pages=1 + ) + failed = scan_qdrant_payloads(lambda **_kwargs: (_ for _ in ()).throw(RuntimeError(secret_error))) + assert repeated["status"] == "BLOCKED-EXTERNAL" and repeated["payloads_scanned"] == 1 + assert malformed_point["reason"] == "qdrant_malformed_point" and malformed_point["status"] == "BLOCKED-EXTERNAL" + assert limited["reason"] == "qdrant_page_limit" and limited["status"] == "BLOCKED-EXTERNAL" + assert failed == {"reason": "qdrant_transport_error", "status": "BLOCKED-EXTERNAL"} + assert SECRET not in _serialized(repeated) and secret_error not in _serialized(failed) + assert ( + scan_qdrant_payloads(lambda **_kwargs: {"result": {"points": [], "next_page_offset": None}}, limit=0)["reason"] + == "qdrant_invalid_bounds" + ) + + +def test_qdrant_rejects_non_json_offsets_oversized_pages_and_keeps_partial_error() -> None: + class NonJsonOffset: + pass + + non_json = scan_qdrant_payloads(lambda **_kwargs: {"result": {"points": [], "next_page_offset": NonJsonOffset()}}) + oversized = scan_qdrant_payloads( + lambda **_kwargs: { + "result": { + "points": [{"payload": {}}, {"payload": {}}], + "next_page_offset": None, + } + }, + limit=1, + ) + calls = 0 + + def partial_then_fail(**_kwargs: object) -> dict[str, object]: + nonlocal calls + calls += 1 + if calls == 1: + return { + "result": { + "points": [{"payload": {"token": SECRET}}], + "next_page_offset": "next", + } + } + raise RuntimeError("transport " + SECRET) + + partial = scan_qdrant_payloads(partial_then_fail) + + assert non_json == { + "reason": "qdrant_malformed_offset", + "status": "BLOCKED-EXTERNAL", + } + assert oversized["reason"] == "qdrant_page_oversized" + assert partial["reason"] == "qdrant_transport_error" + assert partial["payloads_scanned"] == 1 + assert partial["surfaces"]["qdrant.payload"]["records_flagged"] == 1 + assert SECRET not in _serialized(partial) + + deeply_nested: object = "offset" + for _ in range(2000): + deeply_nested = [deeply_nested] + deep = scan_qdrant_payloads(lambda **_kwargs: {"result": {"points": [], "next_page_offset": deeply_nested}}) + assert deep["reason"] == "qdrant_malformed_offset" + + +def test_deep_values_and_hostile_qdrant_payloads_fail_closed(tmp_path: Path) -> None: + deep_json = "[" * 2000 + "0" + "]" * 2000 + db = tmp_path / "deep.sqlite" + with sqlite3.connect(db) as conn: + conn.execute("CREATE TABLE payloads (payload_json TEXT)") + conn.execute("INSERT INTO payloads VALUES (?)", (deep_json,)) + artifact = tmp_path / "artifact" + spool = tmp_path / "spool" + artifact.mkdir() + spool.mkdir() + (artifact / "deep.json").write_text(deep_json) + + inventory = run_inventory( + db, + artifact_roots=[artifact], + spool_roots=[spool], + qdrant_page=lambda **_kwargs: {"result": {"points": [], "next_page_offset": None}}, + ) + + assert inventory["status"] == "BLOCKED" + assert inventory["sqlite"]["reason"] == "sqlite_value_unscannable" + assert inventory["artifacts"]["reason"] == "file_unscannable" + + cyclic: dict[str, object] = {} + cyclic["self"] = cyclic + qdrant = scan_qdrant_payloads( + lambda **_kwargs: { + "result": { + "points": [{"payload": cyclic}], + "next_page_offset": None, + } + } + ) + surrogate = scan_qdrant_payloads(lambda **_kwargs: {"result": {"points": [], "next_page_offset": "\ud800"}}) + + assert qdrant["reason"] == "qdrant_payload_unscannable" + assert surrogate["reason"] == "qdrant_malformed_offset" + assert "self" not in _serialized(qdrant) + + +def test_cli_never_echoes_paths_or_raw_failures(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + from scripts.sensitivity_inventory import main + + missing = tmp_path / "contains-secret-path" + assert main(["--db", str(missing)]) == 3 + out = capsys.readouterr().out + assert str(missing) not in out and "contains-secret-path" not in out + assert json.loads(out)["status"] == "BLOCKED" + db = tmp_path / "safe.sqlite" + _database(db) + roots = tmp_path / "roots" + roots.mkdir() + assert main(["--db", str(db), "--artifact-root", str(roots), "--spool-root", str(roots)]) == 4 + assert json.loads(capsys.readouterr().out)["status"] == "BLOCKED-EXTERNAL" + assert not hasattr(__import__("memorymaster.govern.jobs.sensitivity_inventory", fromlist=["*"]), "cleanup") diff --git a/tests/test_sensitivity_inventory_core.py b/tests/test_sensitivity_inventory_core.py new file mode 100644 index 00000000..b2985aea --- /dev/null +++ b/tests/test_sensitivity_inventory_core.py @@ -0,0 +1,191 @@ +"""Core SQLite end-to-end contract for the sensitivity inventory.""" + +from __future__ import annotations + +import json +import sqlite3 +from base64 import b64encode +from pathlib import Path + +from memorymaster.govern.jobs.sensitivity_inventory import run_inventory + +SECRET = "sk-" + "A" * 24 +ENCODED_SECRET = b64encode(SECRET.encode()).decode() + + +def _database(path: Path) -> None: + conn = sqlite3.connect(path) + conn.executescript( + """ + CREATE TABLE claims (id INTEGER PRIMARY KEY, text TEXT, metadata TEXT, embedding_json TEXT, blob BLOB); + CREATE TABLE citations (id INTEGER PRIMARY KEY, excerpt TEXT); + CREATE TABLE verbatim (id INTEGER PRIMARY KEY, body TEXT); + CREATE TABLE atlas_evidence (id INTEGER PRIMARY KEY, payload TEXT); + CREATE TABLE future_surface (id INTEGER PRIMARY KEY, unknown_value TEXT); + CREATE TABLE "sk-AAAAAAAAAAAAAAAAAAAAAAAA" ("sk-AAAAAAAAAAAAAAAAAAAAAAAA" TEXT); + CREATE VIRTUAL TABLE search_fts USING /* derived */ fts5(body); + """ + ) + conn.execute( + "INSERT INTO claims VALUES (1, ?, ?, ?, ?)", + ( + SECRET, + json.dumps({"api_key": ENCODED_SECRET}), + "[1, 2.5]", + SECRET.encode(), + ), + ) + conn.execute("INSERT INTO claims VALUES (2, 'safe', '{}', ?, ?)", ('["not numeric"]', bytes([255, 0]))) + conn.execute("INSERT INTO claims VALUES (3, 'safe', '{}', ?, ?)", ("[broken", b"safe")) + conn.execute("INSERT INTO citations VALUES (9, ?)", (SECRET,)) + conn.execute("INSERT INTO verbatim VALUES (10, ?)", (SECRET,)) + conn.execute("INSERT INTO atlas_evidence VALUES (11, ?)", (json.dumps({"nested": SECRET}),)) + conn.execute("INSERT INTO future_surface VALUES (12, ?)", (SECRET,)) + conn.execute('INSERT INTO "sk-AAAAAAAAAAAAAAAAAAAAAAAA" VALUES (?)', ("safe",)) + conn.execute("INSERT INTO search_fts VALUES (?)", (SECRET,)) + conn.commit() + conn.close() + + +def _serialized(result: dict[str, object]) -> str: + return json.dumps(result, sort_keys=True) + + +def _assert_vocabulary_surfaces(sqlite: dict[str, object]) -> None: + expected = ( + "claims.holder", + "claims.source_agent", + "claims.idempotency_key", + "claims.scope", + "claims.claim_type", + "claims.visibility", + "claims.tenant_id", + "claims.created_at", + "claims.wiki_path", + "claims.field_json", + "claims.field_numeric", + "citations.source", + "citations.locator", + "events.from_status", + "events.details", + "events.payload_json", + "events.content_hash", + "events.hash_algorithm", + ) + surfaces = sqlite["surfaces"] + assert isinstance(surfaces, dict) + assert all(surface in surfaces for surface in expected) + + +def _assert_core_inventory( + result: dict[str, object], + derived_objects: list[str], + derived_records: int, + tmp_path: Path, +) -> None: + sqlite_result = result["sqlite"] + assert isinstance(sqlite_result, dict) + surfaces = sqlite_result["surfaces"] + assert isinstance(surfaces, dict) + assert result["mode"] == "dry_run" + assert result["status"] == "COMPLETED" + assert sqlite_result["records_scanned"] == 9 + assert sqlite_result["derived_records"] == derived_records + assert sqlite_result["derived_tables_accounted"] == len(derived_objects) + assert sqlite_result["tables_accounted"] == 7 + assert sqlite_result["columns_accounted"] == 15 + assert sqlite_result["schema_identifier_sensitive"] == 2 + assert surfaces["claims.text"] == { + "finding_counts": {"sensitive_value": 1}, + "records_flagged": 1, + "records_scanned": 3, + } + assert surfaces["claims.metadata"]["finding_counts"] == {"sensitive_value": 1} + assert surfaces["claims.embedding_json"]["finding_counts"] == { + "embedding_invalid": 1, + "embedding_string": 1, + } + assert surfaces["claims.blob"]["finding_counts"] == {"binary_opaque": 1, "sensitive_value": 1} + assert surfaces["unknown.field_text"]["records_scanned"] == 3 + serialized = _serialized(result) + assert all(item not in serialized for item in (SECRET, ENCODED_SECRET, "legacy.sqlite", str(tmp_path))) + assert '"9"' not in serialized and '"10"' not in serialized + + +def test_sqlite_inventory_is_dynamic_read_only_and_aggregate_only(tmp_path: Path) -> None: + db = tmp_path / "legacy.sqlite" + _database(db) + before_mtime = db.stat().st_mtime_ns + probe = sqlite3.connect(db) + before_version = probe.execute("PRAGMA data_version").fetchone()[0] + before_schema = probe.execute("SELECT sql FROM sqlite_master ORDER BY name").fetchall() + fts_objects = [ + row[0] + for row in probe.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' AND (name = 'search_fts' OR name LIKE 'search_fts_%')" + ) + ] + derived_objects = [name for name in fts_objects if name != "search_fts"] + derived_records = sum( + int(probe.execute(f'SELECT COUNT(*) FROM "{name}"').fetchone()[0]) for name in derived_objects + ) + + result = run_inventory( + db, + artifact_roots=[tmp_path], + spool_roots=[tmp_path], + qdrant_page=lambda **_kwargs: {"result": {"points": [], "next_page_offset": None}}, + ) + assert probe.execute("PRAGMA data_version").fetchone()[0] == before_version + assert probe.execute("SELECT sql FROM sqlite_master ORDER BY name").fetchall() == before_schema + probe.close() + assert db.stat().st_mtime_ns == before_mtime + _assert_core_inventory(result, derived_objects, derived_records, tmp_path) + assert ( + run_inventory( + db, + artifact_roots=[tmp_path], + spool_roots=[tmp_path], + qdrant_page=lambda **_kwargs: {"result": {"points": [], "next_page_offset": None}}, + ) + == result + ) + + +def test_unique_row_flags_field_vocabulary_and_fts_indexes_derived(tmp_path: Path) -> None: + db = tmp_path / "vocabulary.sqlite" + conn = sqlite3.connect(db) + conn.executescript(""" + CREATE TABLE claims (holder TEXT, source_agent TEXT, idempotency_key TEXT, scope TEXT, claim_type TEXT, visibility TEXT, tenant_id TEXT, created_at TEXT, wiki_path TEXT, extra_json TEXT, extra_number INTEGER); + CREATE TABLE citations (source TEXT, locator TEXT, excerpt TEXT); + CREATE TABLE events (from_status TEXT, details TEXT, payload_json TEXT, content_hash TEXT, hash_algorithm TEXT); + CREATE VIRTUAL TABLE search_fts USING fts5(body); + CREATE VIRTUAL TABLE other_virtual USING rtree(id, x1, x2); + """) + conn.execute( + "INSERT INTO claims VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (SECRET, SECRET, "safe", "safe", "fact", "safe", "safe", "safe", "safe", '{"safe":true}', 1), + ) + conn.execute("INSERT INTO citations VALUES (?, ?, ?)", ("safe", "safe", SECRET)) + conn.execute("INSERT INTO events VALUES (?, ?, ?, ?, ?)", ("safe", SECRET, "{}", "safe", "safe")) + conn.execute("INSERT INTO search_fts VALUES (?)", ("safe",)) + conn.commit() + derived = sorted( + str(row[1]) for row in conn.execute("PRAGMA table_list") if str(row[0]) == "main" and str(row[2]) == "shadow" + ) + derived_columns = sum(len(conn.execute(f'PRAGMA table_info("{name}")').fetchall()) for name in derived) + derived_records = sum(int(conn.execute(f'SELECT COUNT(*) FROM "{name}"').fetchone()[0]) for name in derived) + conn.close() + result = run_inventory( + db, + artifact_roots=[tmp_path], + spool_roots=[tmp_path], + qdrant_page=lambda **_: {"result": {"points": [], "next_page_offset": None}}, + ) + sqlite = result["sqlite"] + assert sqlite["records_flagged"] == 3 + assert sqlite["derived_tables_accounted"] == len(derived) + assert sqlite["derived_columns_accounted"] == derived_columns + assert sqlite["derived_records"] == derived_records + assert sqlite["tables_accounted"] == 5 + _assert_vocabulary_surfaces(sqlite) diff --git a/tests/test_sensitivity_inventory_review_findings.py b/tests/test_sensitivity_inventory_review_findings.py new file mode 100644 index 00000000..f51277f5 --- /dev/null +++ b/tests/test_sensitivity_inventory_review_findings.py @@ -0,0 +1,377 @@ +"""Adversarial RED contracts for independent sensitivity-inventory review findings.""" + +from __future__ import annotations + +import builtins +import io +import json +import sqlite3 +import sys +from pathlib import Path +from typing import Any + +import pytest + +from memorymaster.govern.jobs import sensitivity_inventory +from memorymaster.govern.jobs.sensitivity_inventory import ( + run_inventory, + scan_qdrant_payloads, +) +from scripts.sensitivity_inventory import main as inventory_main + + +def _synthetic_secret() -> str: + return "sk-" + "R" * 24 + + +def _empty_root(tmp_path: Path, name: str) -> Path: + root = tmp_path / name + root.mkdir() + return root + + +def _empty_qdrant(**_kwargs: object) -> dict[str, object]: + return {"result": {"points": [], "next_page_offset": None}} + + +def test_content_owning_fts5_values_are_scanned_before_completion( + tmp_path: Path, +) -> None: + db = tmp_path / "standalone-fts.sqlite" + with sqlite3.connect(db) as conn: + conn.execute("CREATE VIRTUAL TABLE standalone_search USING fts5(body)") + conn.execute( + "INSERT INTO standalone_search(body) VALUES (?)", + (_synthetic_secret(),), + ) + artifact_root = _empty_root(tmp_path, "artifacts") + spool_root = _empty_root(tmp_path, "spool") + + result = run_inventory( + db, + artifact_roots=[artifact_root], + spool_roots=[spool_root], + qdrant_page=_empty_qdrant, + ) + + sqlite_result = result["sqlite"] + assert sqlite_result["records_scanned"] >= 1 + assert sqlite_result["records_flagged"] >= 1 + + +def test_generated_column_does_not_drop_secret_tail_value(tmp_path: Path) -> None: + db = tmp_path / "generated-column.sqlite" + with sqlite3.connect(db) as conn: + conn.execute( + """ + CREATE TABLE claims ( + text TEXT, + derived TEXT GENERATED ALWAYS AS (lower(text)) STORED, + payload_json TEXT + ) + """ + ) + conn.execute( + "INSERT INTO claims(text, payload_json) VALUES (?, ?)", + ("safe", json.dumps({"credential": _synthetic_secret()})), + ) + artifact_root = _empty_root(tmp_path, "artifacts") + spool_root = _empty_root(tmp_path, "spool") + + result = run_inventory( + db, + artifact_roots=[artifact_root], + spool_roots=[spool_root], + qdrant_page=_empty_qdrant, + ) + + payload_surface = result["sqlite"]["surfaces"]["claims.payload_json"] + assert payload_surface["records_scanned"] == 1 + assert payload_surface["finding_counts"] == {"sensitive_value": 1} + assert result["sqlite"]["records_flagged"] == 1 + + +def test_long_postgres_secret_across_chunks_is_detected_or_blocks_coverage( + tmp_path: Path, +) -> None: + db = tmp_path / "safe.sqlite" + with sqlite3.connect(db) as conn: + conn.execute("CREATE TABLE safe_table (value TEXT)") + artifact_root = _empty_root(tmp_path, "artifacts") + spool_root = _empty_root(tmp_path, "spool") + password = "A" * 1024 + database_url = "postgres" + "ql://inventory:" + password + "@db.example.invalid/app" + (artifact_root / "connection.txt").write_text(database_url, encoding="utf-8") + + result = run_inventory( + db, + artifact_roots=[artifact_root], + spool_roots=[spool_root], + qdrant_page=_empty_qdrant, + chunk_size=64, + max_file_bytes=4096, + ) + + artifacts = result["artifacts"] + assert artifacts["status"] == "BLOCKED" or artifacts["sensitive_files"] == 1 + + +def test_open_handle_identity_mismatch_is_refused_before_any_bytes_are_read( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + root = _empty_root(tmp_path, "inside") + inside = root / "candidate.txt" + outside = tmp_path / "outside.txt" + inside.write_text("safe", encoding="utf-8") + outside.write_text(_synthetic_secret(), encoding="utf-8") + + class TrackingHandle: + def __init__(self, target: Path) -> None: + self._handle = builtins.open(target, "rb") + self.bytes_read = 0 + + def __enter__(self) -> TrackingHandle: + return self + + def __exit__(self, *_args: object) -> None: + self._handle.close() + + def fileno(self) -> int: + return self._handle.fileno() + + def read(self, size: int = -1) -> bytes: + data = self._handle.read(size) + self.bytes_read += len(data) + return data + + tracker = TrackingHandle(outside) + original_open = Path.open + inside_resolved = inside.resolve() + + def swapped_open(path: Path, *args: Any, **kwargs: Any) -> io.BufferedReader | TrackingHandle: + if path == inside_resolved and args and args[0] == "rb": + return tracker + return original_open(path, *args, **kwargs) + + monkeypatch.setattr(Path, "open", swapped_open) + + result = sensitivity_inventory._file_inventory( + [root], + chunk_size=64, + max_file_bytes=4096, + max_entries=100, + ) + + assert result["status"] == "BLOCKED" + assert result["files_scanned"] == 0 + assert tracker.bytes_read == 0 + + +def test_qdrant_nested_bytes_and_missing_payload_fail_closed() -> None: + nested_bytes = scan_qdrant_payloads( + lambda **_kwargs: { + "result": { + "points": [{"payload": {"nested": {"value": _synthetic_secret().encode("utf-8")}}}], + "next_page_offset": None, + } + } + ) + missing_payload = scan_qdrant_payloads( + lambda **_kwargs: { + "result": { + "points": [{"id": "synthetic-point"}], + "next_page_offset": None, + } + } + ) + + assert nested_bytes == { + "payloads_scanned": 1, + "reason": "qdrant_payload_unscannable", + "status": "BLOCKED-EXTERNAL", + "surfaces": { + "qdrant.payload": { + "finding_counts": {"binary_opaque": 1}, + "records_flagged": 1, + "records_scanned": 1, + } + }, + } + assert missing_payload == { + "reason": "qdrant_payload_missing", + "status": "BLOCKED-EXTERNAL", + } + + +def test_qdrant_hostile_scalar_subclasses_return_fixed_blockers() -> None: + class HostileString(str): + def encode(self, *_args: object, **_kwargs: object) -> bytes: + raise RuntimeError("raw-" + _synthetic_secret()) + + def payload(**_kwargs: object) -> dict[str, object]: + return { + "result": { + "points": [{"payload": {"value": HostileString("safe")}}], + "next_page_offset": None, + } + } + + def offset(**_kwargs: object) -> dict[str, object]: + return { + "result": { + "points": [], + "next_page_offset": HostileString("next"), + } + } + + assert scan_qdrant_payloads(payload)["reason"] == "qdrant_payload_unscannable" + assert scan_qdrant_payloads(offset) == { + "reason": "qdrant_malformed_offset", + "status": "BLOCKED-EXTERNAL", + } + + +def test_qdrant_requires_offset_key_and_preserves_partial_missing_payload_evidence() -> None: + missing_offset = scan_qdrant_payloads(lambda **_kwargs: {"result": {"points": [{"payload": {"safe": True}}]}}) + partial = scan_qdrant_payloads( + lambda **_kwargs: { + "result": { + "points": [ + {"payload": {"token": _synthetic_secret()}}, + {"id": "missing-payload"}, + ], + "next_page_offset": None, + } + } + ) + + assert missing_offset == { + "reason": "qdrant_malformed_response", + "status": "BLOCKED-EXTERNAL", + } + assert partial["status"] == "BLOCKED-EXTERNAL" + assert partial["reason"] == "qdrant_payload_missing" + assert partial["payloads_scanned"] == 1 + assert partial["surfaces"]["qdrant.payload"]["finding_counts"] == {"sensitive_value": 1} + + +def test_huge_untyped_json_integer_returns_fixed_sqlite_blocker( + tmp_path: Path, +) -> None: + configured_limit = getattr(sys, "get_int_max_str_digits", lambda: 4300)() + digit_count = max(5000, configured_limit + 100) + db = tmp_path / "huge-integer.sqlite" + with sqlite3.connect(db) as conn: + conn.execute("CREATE TABLE untyped_payload (value)") + conn.execute("INSERT INTO untyped_payload(value) VALUES (?)", ("9" * digit_count,)) + artifact_root = _empty_root(tmp_path, "artifacts") + spool_root = _empty_root(tmp_path, "spool") + + try: + result = run_inventory( + db, + artifact_roots=[artifact_root], + spool_roots=[spool_root], + qdrant_page=_empty_qdrant, + ) + except ValueError: + pytest.fail("huge untyped JSON integers must return a fixed blocker, not raise") + + assert result["status"] == "BLOCKED" + assert result["sqlite"]["status"] == "BLOCKED" + assert result["sqlite"]["reason"] == "sqlite_value_unscannable" + + +def test_raw_and_parsed_json_are_both_scanned(tmp_path: Path) -> None: + db = tmp_path / "duplicate-json-keys.sqlite" + duplicate_key = '{"token":"' + _synthetic_secret() + '","token":"safe"}' + with sqlite3.connect(db) as conn: + conn.execute("CREATE TABLE events (payload_json TEXT)") + conn.executemany( + "INSERT INTO events(payload_json) VALUES (?)", + [('{"password":"WeakValue123"}',), (duplicate_key,)], + ) + artifact_root = _empty_root(tmp_path, "artifacts") + spool_root = _empty_root(tmp_path, "spool") + + result = run_inventory( + db, + artifact_roots=[artifact_root], + spool_roots=[spool_root], + qdrant_page=_empty_qdrant, + ) + + payload = result["sqlite"]["surfaces"]["events.payload_json"] + assert payload["records_scanned"] == 2 + assert payload["records_flagged"] == 2 + assert payload["finding_counts"] == {"sensitive_value": 2} + + +def test_table_list_metadata_failure_has_specific_stable_blocker( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db = tmp_path / "table-list.sqlite" + with sqlite3.connect(db) as conn: + conn.execute("CREATE TABLE safe_table (value TEXT)") + artifact_root = _empty_root(tmp_path, "artifacts") + spool_root = _empty_root(tmp_path, "spool") + real_connect = sqlite3.connect + + class TableListUnavailableConnection: + def __init__(self, connection: sqlite3.Connection) -> None: + self._connection = connection + + def execute(self, sql: str, *args: object) -> sqlite3.Cursor: + if sql.strip().casefold().startswith("pragma table_list"): + raise sqlite3.OperationalError("synthetic table-list failure") + return self._connection.execute(sql, *args) + + def close(self) -> None: + self._connection.close() + + def connect_without_table_list(*args: Any, **kwargs: Any) -> TableListUnavailableConnection: + return TableListUnavailableConnection(real_connect(*args, **kwargs)) + + monkeypatch.setattr(sensitivity_inventory.sqlite3, "connect", connect_without_table_list) + + result = run_inventory( + db, + artifact_roots=[artifact_root], + spool_roots=[spool_root], + qdrant_page=_empty_qdrant, + ) + + assert result["sqlite"] == { + "reason": "sqlite_table_list_unavailable", + "status": "BLOCKED", + } + + +def test_cli_invalid_integer_redacts_raw_marker_and_emits_json_blocker( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + marker = _synthetic_secret() + invalid_integer = "12" + marker + db = tmp_path / "unused.sqlite" + + try: + exit_code = inventory_main(["--db", str(db), "--chunk-size", invalid_integer]) + except SystemExit as exc: + exit_code = int(exc.code) + captured = capsys.readouterr() + try: + payload = json.loads(captured.out) + except (json.JSONDecodeError, TypeError): + payload = {} + + assert (marker in captured.out + captured.err, exit_code, payload.get("status")) == ( + False, + 3, + "BLOCKED", + ) + assert payload.get("classification") == "LEGACY-SENSITIVITY-INVENTORY" + assert payload.get("mode") == "dry_run" + assert payload.get("recommendation") == "REVIEW_ONLY" From a858419b19f29e1f88175b97676d36470310defc Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sat, 11 Jul 2026 21:54:05 -0300 Subject: [PATCH 22/55] fix(deploy): require private authenticated services --- .env.example | 21 ++++- Dockerfile | 4 +- docker-compose.postgres.yml | 14 ++- docker-compose.yml | 31 ++++--- docs/deployment_profiles.md | 32 +++++++ helm/memorymaster/templates/deployment.yaml | 40 ++++++++- helm/memorymaster/values.yaml | 16 +++- scripts/smoke_postgres.ps1 | 12 ++- tests/test_deployment_contracts.py | 99 ++++++++++++++++++--- 9 files changed, 228 insertions(+), 41 deletions(-) diff --git a/.env.example b/.env.example index 17c668cb..0fc3b8d2 100644 --- a/.env.example +++ b/.env.example @@ -4,8 +4,9 @@ # Copy to `.env` and edit as needed: # cp .env.example .env # -# `.env` is gitignored — never commit real secrets. All values below are -# optional; a fresh install works with zero env vars as long as you either +# `.env` is gitignored — never commit real secrets. Values are optional unless +# the selected deployment profile marks them required. A fresh local install +# works with zero env vars as long as you either # (a) set GEMINI_API_KEY for the free cloud default, or (b) run Ollama # locally. Everything else is tuning. # @@ -64,6 +65,15 @@ MEMORYMASTER_MCP_AUTH_MODE=local-trusted # MEMORYMASTER_TEST_POSTGRES_APP_DSN=postgresql://app-role:password@host/test_database # MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE=1 +# Required only when rendering docker-compose.postgres.yml. Use a generated +# secret and rotate any environment that used the retired fixed credential. +# MEMORYMASTER_POSTGRES_PASSWORD= + +# Required only when rendering docker-compose.yml. Supply immutable image +# references verified for your platform; tags and unqualified names are unsafe. +# QDRANT_IMAGE_DIGEST=sha256:<64-hex-digest> +# OLLAMA_IMAGE_DIGEST=sha256:<64-hex-digest> + # --------------------------------------------------------------------------- # LLM provider (auto-ingest Stop hook + steward + entity extraction) # --------------------------------------------------------------------------- @@ -251,6 +261,13 @@ OLLAMA_URL=http://localhost:11434 # Default: http://localhost:6333 (only honoured when explicitly set) QDRANT_URL=http://localhost:6333 +# Required by the hardened Compose/Helm Qdrant profile. Generate and provision +# this out of band; never commit the value. +# QDRANT_API_KEY= + +# Optional path to a trusted CA certificate for a TLS-enabled Qdrant endpoint. +# QDRANT_CA_CERT= + # --------------------------------------------------------------------------- # Lifecycle tuning (volatility: low,medium,high triples) # --------------------------------------------------------------------------- diff --git a/Dockerfile b/Dockerfile index 3b3ff54c..229d4a8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.12-slim AS builder +FROM python:3.12-slim@sha256:d764629ce0ddd8c71fd371e9901efb324a95789d2315a47db7e4d27e78f1b0e9 AS builder WORKDIR /app @@ -10,7 +10,7 @@ COPY memorymaster/ ./memorymaster/ RUN pip install --no-cache-dir ".[mcp,qdrant,security]" -FROM python:3.12-slim +FROM python:3.12-slim@sha256:d764629ce0ddd8c71fd371e9901efb324a95789d2315a47db7e4d27e78f1b0e9 WORKDIR /app diff --git a/docker-compose.postgres.yml b/docker-compose.postgres.yml index 5a1bb1b5..3fdefe51 100644 --- a/docker-compose.postgres.yml +++ b/docker-compose.postgres.yml @@ -1,16 +1,22 @@ services: postgres: - image: postgres:16-alpine + image: postgres:16-alpine@sha256:b7587f3cb74f4f4b2a4f9d67f052edbf95eb93f4fec7c5ada3792546caaf7383 container_name: memorymaster-pg restart: unless-stopped environment: POSTGRES_DB: memorymaster POSTGRES_USER: mm - POSTGRES_PASSWORD: mm_pw + POSTGRES_PASSWORD: "${MEMORYMASTER_POSTGRES_PASSWORD:?Set MEMORYMASTER_POSTGRES_PASSWORD before rendering Compose}" ports: - - "6543:5432" + - "127.0.0.1:6543:5432" healthcheck: - test: ["CMD-SHELL", "pg_isready -U mm -d memorymaster"] + test: + - CMD-SHELL + - >- + PGPASSWORD="$${POSTGRES_PASSWORD}" + psql --host=127.0.0.1 --username=$${POSTGRES_USER} + --dbname=$${POSTGRES_DB} --no-password --tuples-only + --command='SELECT 1' interval: 5s timeout: 3s retries: 20 diff --git a/docker-compose.yml b/docker-compose.yml index 4be2c55d..ce3c037d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,21 +1,19 @@ -version: '3.8' - services: memorymaster: build: context: . dockerfile: Dockerfile - image: memorymaster:latest container_name: memorymaster ports: - "8765:8765" volumes: - ./data:/data environment: - - MEMORYMASTER_DEFAULT_DB=/data/memorymaster.db - - MEMORYMASTER_WORKSPACE=/data - - QDRANT_URL=http://qdrant:6333 - - OLLAMA_URL=http://ollama:11434 + MEMORYMASTER_DEFAULT_DB: /data/memorymaster.db + MEMORYMASTER_WORKSPACE: /data + QDRANT_URL: http://qdrant:6333 + QDRANT_API_KEY: "${QDRANT_API_KEY:?Set QDRANT_API_KEY before rendering Compose}" + OLLAMA_URL: http://ollama:11434 depends_on: qdrant: condition: service_healthy @@ -30,25 +28,32 @@ services: start_period: 10s qdrant: - image: qdrant/qdrant:latest + image: "qdrant/qdrant@${QDRANT_IMAGE_DIGEST:?Set QDRANT_IMAGE_DIGEST to sha256:}" container_name: qdrant ports: - - "6333:6333" - - "6334:6334" + - "127.0.0.1:6333:6333" + - "127.0.0.1:6334:6334" + environment: + QDRANT__SERVICE__API_KEY: "${QDRANT_API_KEY:?Set QDRANT_API_KEY before rendering Compose}" volumes: - qdrant_data:/qdrant/storage restart: unless-stopped healthcheck: - test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:6333/healthz"] + test: + - CMD-SHELL + - >- + wget --no-verbose --tries=1 --spider + --header="api-key: $${QDRANT__SERVICE__API_KEY}" + http://127.0.0.1:6333/collections interval: 15s timeout: 5s retries: 3 ollama: - image: ollama/ollama:latest + image: "ollama/ollama@${OLLAMA_IMAGE_DIGEST:?Set OLLAMA_IMAGE_DIGEST to sha256:}" container_name: ollama ports: - - "11434:11434" + - "127.0.0.1:11434:11434" volumes: - ollama_data:/root/.ollama restart: unless-stopped diff --git a/docs/deployment_profiles.md b/docs/deployment_profiles.md index 2b827041..b85894a9 100644 --- a/docs/deployment_profiles.md +++ b/docs/deployment_profiles.md @@ -2,6 +2,36 @@ This document defines practical deployment profiles for MemoryMaster with reliability/operability defaults. +## Secure Container Inputs + +Container configuration fails closed when required deployment inputs are +missing. Keep real values in an operator-controlled secret store or a local +gitignored `.env`, never in Compose, Helm values, shell history, or source. + +- `docker-compose.postgres.yml` requires + `MEMORYMASTER_POSTGRES_PASSWORD`, binds PostgreSQL to `127.0.0.1`, and checks + health with an authenticated `SELECT 1`. +- `docker-compose.yml` requires `QDRANT_API_KEY` plus externally verified + `QDRANT_IMAGE_DIGEST` and `OLLAMA_IMAGE_DIGEST` values in + `sha256:<64-hex>` form. Repositories are fixed in Compose so a mutable tag + cannot be substituted. Qdrant and Ollama host ports bind to `127.0.0.1` only. +- The Helm chart requires `image.digest` and an existing Secret named through + `qdrant.apiKeySecret.name`. It never accepts a literal Qdrant API key. Set + `qdrant.caSecret.name` to mount an optional trusted CA as + `QDRANT_CA_CERT`. + +Validate interpolation before any runtime action: + +```powershell +docker compose -f docker-compose.postgres.yml config +docker compose -f docker-compose.yml config +``` + +The Compose and Helm profiles still inherit the R3.4 stdio-versus-HTTP +entrypoint/readiness blocker. Configuration validation is not runtime or +deployment evidence; do not expose or promote these profiles until that +separate gate is resolved. + ## Profile A: Local Developer (SQLite) Use when: @@ -42,6 +72,8 @@ Runtime: - schema lifecycle: separate migrator DSN; the application runtime cannot initialize or migrate Baseline controls: +- supply the PostgreSQL password out of band; never restore the retired fixed + Compose credential - backup policy for DB and artifacts - health endpoint checks (`/health`) - distinct migrator/application roles meeting the contract below diff --git a/helm/memorymaster/templates/deployment.yaml b/helm/memorymaster/templates/deployment.yaml index 3baa4215..7022e955 100644 --- a/helm/memorymaster/templates/deployment.yaml +++ b/helm/memorymaster/templates/deployment.yaml @@ -1,3 +1,17 @@ +{{- $imageDigest := required "image.digest is required (use sha256:)" .Values.image.digest -}} +{{- if not (regexMatch "^sha256:[a-f0-9]{64}$" $imageDigest) -}} +{{- fail "image.digest must be a lowercase sha256:<64-hex-digest> reference" -}} +{{- end -}} +{{- if hasKey .Values.env "QDRANT_API_KEY" -}} +{{- fail "QDRANT_API_KEY is reserved; configure it through .Values.qdrant" -}} +{{- end -}} +{{- if hasKey .Values.env "QDRANT_CA_CERT" -}} +{{- fail "QDRANT_CA_CERT is reserved; configure it through .Values.qdrant" -}} +{{- end -}} +{{- if hasKey .Values.env "QDRANT_URL" -}} +{{- fail "QDRANT_URL is reserved; configure it through .Values.qdrant" -}} +{{- end -}} +{{- $qdrantApiKeySecret := required "qdrant.apiKeySecret.name is required" .Values.qdrant.apiKeySecret.name -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -14,7 +28,7 @@ spec: spec: containers: - name: memorymaster - image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + image: "{{ .Values.image.repository }}@{{ $imageDigest }}" imagePullPolicy: {{ .Values.image.pullPolicy }} ports: - containerPort: 8765 @@ -23,9 +37,25 @@ spec: - name: {{ $key }} value: {{ $val | quote }} {{- end }} + - name: QDRANT_URL + value: {{ .Values.qdrant.url | quote }} + - name: QDRANT_API_KEY + valueFrom: + secretKeyRef: + name: {{ $qdrantApiKeySecret | quote }} + key: {{ .Values.qdrant.apiKeySecret.key | quote }} + {{- if .Values.qdrant.caSecret.name }} + - name: QDRANT_CA_CERT + value: {{ printf "%s/ca.crt" .Values.qdrant.caSecret.mountPath | quote }} + {{- end }} volumeMounts: - name: data mountPath: /data + {{- if .Values.qdrant.caSecret.name }} + - name: qdrant-ca + mountPath: {{ .Values.qdrant.caSecret.mountPath | quote }} + readOnly: true + {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} volumes: @@ -36,3 +66,11 @@ spec: {{- else }} emptyDir: {} {{- end }} + {{- if .Values.qdrant.caSecret.name }} + - name: qdrant-ca + secret: + secretName: {{ .Values.qdrant.caSecret.name | quote }} + items: + - key: {{ .Values.qdrant.caSecret.key | quote }} + path: ca.crt + {{- end }} diff --git a/helm/memorymaster/values.yaml b/helm/memorymaster/values.yaml index 3b0e4b48..5130217c 100644 --- a/helm/memorymaster/values.yaml +++ b/helm/memorymaster/values.yaml @@ -2,7 +2,8 @@ replicaCount: 1 image: repository: memorymaster - tag: latest + # Required at render time. Supply the release image digest as sha256:. + digest: "" pullPolicy: IfNotPresent service: @@ -17,9 +18,20 @@ persistence: env: MEMORYMASTER_DEFAULT_DB: /data/memorymaster.db MEMORYMASTER_WORKSPACE: /data - QDRANT_URL: http://qdrant:6333 OLLAMA_URL: http://ollama:11434 +qdrant: + url: http://qdrant:6333 + apiKeySecret: + # Required: name of an existing Secret. The chart never creates the key. + name: "" + key: api-key + caSecret: + # Optional: name of an existing Secret containing the trusted CA file. + name: "" + key: ca.crt + mountPath: /etc/memorymaster/qdrant-ca + resources: limits: cpu: 500m diff --git a/scripts/smoke_postgres.ps1 b/scripts/smoke_postgres.ps1 index 1852e282..4b87b3fb 100644 --- a/scripts/smoke_postgres.ps1 +++ b/scripts/smoke_postgres.ps1 @@ -1,12 +1,16 @@ param( - [string]$Dsn = "postgresql://mm:mm_pw@127.0.0.1:6543/memorymaster?connect_timeout=5", + [string]$Dsn = $env:MEMORYMASTER_POSTGRES_DSN, [string]$Workspace = "." ) $ErrorActionPreference = "Stop" +if ([string]::IsNullOrWhiteSpace($Dsn)) { + throw "Set MEMORYMASTER_POSTGRES_DSN to an operator-supplied disposable PostgreSQL DSN." +} + python -m memorymaster --db $Dsn --workspace $Workspace init-db -python -m memorymaster --db $Dsn --workspace $Workspace ingest --text "Server IP is 10.0.0.1" --subject server --predicate ip_address --object 10.0.0.1 --source "session://chat|turn-1|smoke" -python -m memorymaster --db $Dsn --workspace $Workspace ingest --text "Server IP is 10.0.0.2" --subject server --predicate ip_address --object 10.0.0.2 --source "session://chat|turn-2|smoke" +python -m memorymaster --db $Dsn --workspace $Workspace ingest --text "Server endpoint is primary" --subject server --predicate endpoint --object primary --source "session://chat|turn-1|smoke" +python -m memorymaster --db $Dsn --workspace $Workspace ingest --text "Server endpoint is standby" --subject server --predicate endpoint --object standby --source "session://chat|turn-2|smoke" python -m memorymaster --db $Dsn --workspace $Workspace run-cycle --policy-mode cadence --policy-limit 100 --min-citations 1 --min-score 0.5 -python -m memorymaster --db $Dsn --workspace $Workspace query "server ip" --retrieval-mode hybrid --limit 10 +python -m memorymaster --db $Dsn --workspace $Workspace query "server endpoint" --retrieval-mode hybrid --limit 10 diff --git a/tests/test_deployment_contracts.py b/tests/test_deployment_contracts.py index 5edfd42b..1bac13f8 100644 --- a/tests/test_deployment_contracts.py +++ b/tests/test_deployment_contracts.py @@ -15,10 +15,6 @@ def _read(relative_path: str) -> str: return (ROOT / relative_path).read_text(encoding="utf-8") -@pytest.mark.xfail( - strict=True, - reason="R1.5: Postgres Compose uses a fixed default password", -) def test_postgres_compose_requires_secret_interpolation(): compose = _read("docker-compose.postgres.yml") match = re.search(r"^\s*POSTGRES_PASSWORD:\s*(.+?)\s*$", compose, re.MULTILINE) @@ -27,10 +23,15 @@ def test_postgres_compose_requires_secret_interpolation(): assert "${" in match.group(1) and ":?" in match.group(1) -@pytest.mark.xfail( - strict=True, - reason="R1.5: Qdrant and Ollama ports are publicly published by default", -) +def test_postgres_compose_is_loopback_only_with_authenticated_healthcheck(): + compose = _read("docker-compose.postgres.yml") + + assert '"127.0.0.1:6543:5432"' in compose + assert 'PGPASSWORD="$${POSTGRES_PASSWORD}"' in compose + assert "psql" in compose + assert "SELECT 1" in compose + + def test_auxiliary_compose_ports_are_not_public(): compose = _read("docker-compose.yml") mappings = re.findall( @@ -42,6 +43,15 @@ def test_auxiliary_compose_ports_are_not_public(): assert all(value.startswith(("127.0.0.1:", "localhost:")) for value in mappings) +def test_auxiliary_compose_requires_authenticated_qdrant(): + compose = _read("docker-compose.yml") + + assert re.search(r"QDRANT_API_KEY:\s*[\"']?\$\{QDRANT_API_KEY:\?", compose) + assert re.search(r"QDRANT__SERVICE__API_KEY:\s*[\"']?\$\{QDRANT_API_KEY:\?", compose) + assert '--header="api-key: $${QDRANT__SERVICE__API_KEY}"' in compose + assert "http://127.0.0.1:6333/collections" in compose + + @pytest.mark.xfail( strict=True, reason="R3.4: container publishes HTTP but launches stdio MCP with a CLI-only healthcheck", @@ -50,7 +60,7 @@ def test_container_entrypoint_and_healthcheck_share_an_http_contract(): dockerfile = _read("Dockerfile") compose = _read("docker-compose.yml") - assert re.search(r'CMD\s+\[.*memorymaster-(?:dashboard|http).*\]', dockerfile) + assert re.search(r"CMD\s+\[.*memorymaster-(?:dashboard|http).*\]", dockerfile) assert re.search(r"https?://(?:127\.0\.0\.1|localhost):8765/(?:healthz|readyz)", compose) assert '"--version"' not in compose @@ -68,10 +78,6 @@ def test_helm_deployment_defines_health_and_readiness_probes(): assert "/readyz" in deployment -@pytest.mark.xfail( - strict=True, - reason="R1.5: deployment profiles use unpinned latest image tags", -) def test_deployment_images_reject_latest_tags(): deployment_text = "\n".join( [ @@ -82,3 +88,70 @@ def test_deployment_images_reject_latest_tags(): ) assert not re.search(r"(?i)(?:image:\s*[^\s]+:latest|tag:\s*[\"']?latest)", deployment_text) + + +def test_deployment_images_are_immutable_or_required_by_digest(): + dockerfile = _read("Dockerfile") + compose = _read("docker-compose.yml") + postgres = _read("docker-compose.postgres.yml") + python_image = "python:3.12-slim@sha256:d764629ce0ddd8c71fd371e9901efb324a95789d2315a47db7e4d27e78f1b0e9" + postgres_image = "postgres:16-alpine@sha256:b7587f3cb74f4f4b2a4f9d67f052edbf95eb93f4fec7c5ada3792546caaf7383" + + assert dockerfile.count(f"FROM {python_image}") == 2 + assert postgres_image in postgres + assert 'image: "qdrant/qdrant@${QDRANT_IMAGE_DIGEST:?' in compose + assert 'image: "ollama/ollama@${OLLAMA_IMAGE_DIGEST:?' in compose + assert "image: memorymaster" not in compose + + +def test_helm_requires_digest_and_existing_qdrant_secrets(): + values = _read("helm/memorymaster/values.yaml") + deployment = _read("helm/memorymaster/templates/deployment.yaml") + + assert re.search(r"^\s*digest:\s*[\"']{2}\s*$", values, re.MULTILINE) + assert not re.search(r"^\s*tag:", values, re.MULTILINE) + assert 'required "image.digest' in deployment + assert "regexMatch" in deployment and "sha256:" in deployment + assert 'fail "image.digest' in deployment + assert ".Values.image.repository }}@{{" in deployment + assert "QDRANT_API_KEY" in deployment + assert "secretKeyRef:" in deployment + assert ".Values.qdrant.apiKeySecret.name" in deployment + assert ".Values.qdrant.caSecret.name" in deployment + assert "QDRANT_CA_CERT" in deployment + assert "readOnly: true" in deployment + + +def test_postgres_smoke_requires_operator_supplied_dsn(): + script = _read("scripts/smoke_postgres.ps1") + + assert "mm_pw" not in script + assert "$env:MEMORYMASTER_POSTGRES_DSN" in script + assert "IsNullOrWhiteSpace" in script + assert not re.search(r"\b(?:10|192\.168|172\.(?:1[6-9]|2\d|3[01]))\.", script) + + +def test_environment_example_documents_required_deployment_inputs(): + example = _read(".env.example") + + assert "# MEMORYMASTER_POSTGRES_PASSWORD=" in example + assert "# QDRANT_API_KEY=" in example + assert re.search(r"^# QDRANT_IMAGE_DIGEST=sha256:", example, re.MULTILINE) + assert re.search(r"^# OLLAMA_IMAGE_DIGEST=sha256:", example, re.MULTILINE) + + +def test_compose_fixes_image_repositories_and_requires_digest_only_inputs(): + compose = _read("docker-compose.yml") + + assert 'image: "qdrant/qdrant@${QDRANT_IMAGE_DIGEST:?' in compose + assert 'image: "ollama/ollama@${OLLAMA_IMAGE_DIGEST:?' in compose + assert "QDRANT_IMAGE}" not in compose + assert "OLLAMA_IMAGE}" not in compose + + +def test_helm_rejects_literal_qdrant_security_env_overrides(): + deployment = _read("helm/memorymaster/templates/deployment.yaml") + + for name in ("QDRANT_API_KEY", "QDRANT_CA_CERT", "QDRANT_URL"): + assert f'hasKey .Values.env "{name}"' in deployment + assert "reserved; configure it through .Values.qdrant" in deployment From b71e18f35c8beb63c96a28e7f0b630f7a649d8a6 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sun, 12 Jul 2026 17:08:44 -0300 Subject: [PATCH 23/55] fix(security): bind supply-chain evidence --- .dockerignore | 7 + docs/security_supply_chain.md | 99 +++ scripts/run_supply_chain_checks.py | 696 +++++++++++++++++++++ scripts/validate_sbom.py | 297 +++++++++ tests/test_supply_chain_contracts.py | 875 +++++++++++++++++++++++++++ 5 files changed, 1974 insertions(+) create mode 100644 .dockerignore create mode 100644 docs/security_supply_chain.md create mode 100644 scripts/run_supply_chain_checks.py create mode 100644 scripts/validate_sbom.py create mode 100644 tests/test_supply_chain_contracts.py diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..c153b9a8 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +# Deny the entire build context, then allow only Dockerfile COPY inputs. +** +!Dockerfile +!pyproject.toml +!README.md +!memorymaster/ +!memorymaster/** diff --git a/docs/security_supply_chain.md b/docs/security_supply_chain.md new file mode 100644 index 00000000..04bf10e1 --- /dev/null +++ b/docs/security_supply_chain.md @@ -0,0 +1,99 @@ +# Supply-chain security checks + +MemoryMaster's local release gate combines five fail-closed checks: + +1. Gitleaks scans the complete Git history with built-in rules and no + repository-controlled suppressions. +2. `pip-audit` audits the trusted project in strict mode against the explicit + PyPI vulnerability service. +3. A second dependency audit covers the Docker release extras: `mcp`, + `qdrant`, and `security`. +4. The CycloneDX validator binds the SBOM's root component and SHA-256 hash to + the exact `memorymaster` wheel and its wheel metadata. +5. Docker Scout scans at most three already-local immutable `sha256:` + targets for high and critical findings, including base-image findings. + +The runner discards scanner stdout/stderr and emits only fixed check results +plus safe evidence hashes: repository commit, release-wheel SHA-256, SBOM +SHA-256, immutable image IDs, native-tool hashes, and Python/`pip-audit` +versions. Missing tools, unavailable evidence, timeouts, nonzero exits, +mutable image tags, and invalid or mismatched SBOMs all fail the gate. + +## Inspect the command plan without execution + +This mode performs no scanner, network, registry, Docker, or artifact read: + +```powershell +python scripts/run_supply_chain_checks.py ` + --release-artifact artifacts/memorymaster-4.4.1-py3-none-any.whl ` + --sbom artifacts/memorymaster-4.4.1.cdx.json ` + --local-image sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ` + --command-plan +``` + +`--dry-run` is an alias. The displayed executable and policy placeholders are +resolved only during execution. + +## Prepare release evidence + +Build the exact wheel and generate a CycloneDX JSON SBOM with an approved local +generator. The SBOM must place the release root at `metadata.component`, use +the exact `memorymaster` name/version/PyPI purl, and include the wheel's SHA-256 +under `metadata.component.hashes`. Dependency-only SBOMs that omit the release +root are rejected. + +Build the image locally, then capture its immutable image ID: + +```powershell +docker build --pull=false --tag memorymaster:phase1 . +$imageId = docker image inspect --format '{{.Id}}' memorymaster:phase1 +``` + +Run the gate from the same checkout that owns the runner: + +```powershell +python scripts/run_supply_chain_checks.py ` + --release-artifact artifacts/memorymaster-4.4.1-py3-none-any.whl ` + --sbom artifacts/memorymaster-4.4.1.cdx.json ` + --local-image $imageId +``` + +Repeat `--local-image` for approved Qdrant and Ollama images, up to the +three-image bound. Tags, registry URLs, and mutable references are rejected. + +## Isolation and policy + +The runner resolves Gitleaks, Git, and Docker to absolute non-repository files, +records their hashes, and runs every child from a sterile temporary directory +with a minimal environment. Ambient `GIT_*`, `GITLEAKS_*`, `PIP_AUDIT_*`, +Docker, proxy, certificate, credential, and Python-path variables do not reach +the scanners. Python tools run with `-I`; the validator path comes from the +trusted runner location rather than `--repo-root`. + +Gitleaks uses a temporary config that extends its built-in defaults, an empty +ignore file, `--ignore-gitleaks-allow`, full-history `--log-opts=--all`, and +full redaction. `pip-audit` is pinned to the `pypi` service and a temporary pip +configuration. Scanner streams go to `DEVNULL`, per-command timeouts and a +one-hour global deadline apply, and the Docker build context is an exact +allowlist of Dockerfile inputs. + +## BLOCKED-EXTERNAL evidence + +The repository cannot truthfully close these external checks by inspection: + +- The unsuppressed full-history Gitleaks run on 2026-07-11 failed closed with + 40 potential findings across 10 commits and 7 files. Only aggregate rule + counts were retained. An authorized reviewer must classify them, rotate any + affected credentials, and approve any history action. +- Approved hashes/versions for native scanners and an approved release SBOM + generator require operator review. +- A definitive dependency result requires access to the approved PyPI + advisory service. +- Qdrant and Ollama require approved local images with recorded immutable IDs; + Docker Scout and the local daemon must be available. +- Approved immutable commit SHAs for third-party CI actions remain separate + release-pipeline work. + +Retain aggregate/fixed results and evidence hashes. Never put raw secret-scan +findings or credentials into general logs or repository artifacts. This +document defines the gate; it does not claim external scans passed. diff --git a/scripts/run_supply_chain_checks.py b/scripts/run_supply_chain_checks.py new file mode 100644 index 00000000..f926227c --- /dev/null +++ b/scripts/run_supply_chain_checks.py @@ -0,0 +1,696 @@ +from __future__ import annotations + +import argparse +import hashlib +import importlib.metadata +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +import time +from collections.abc import Callable, Sequence +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Any + + +Runner = Callable[..., subprocess.CompletedProcess[str]] +Resolver = Callable[[str], str | None] +TRUSTED_GITLEAKS_CONFIG = 'title = "MemoryMaster mandatory built-in Gitleaks policy"\n\n[extend]\nuseDefault = true\n' +TRUSTED_REPO_ROOT = Path(__file__).resolve().parents[1] +TRUSTED_VALIDATOR_PATH = Path(__file__).with_name("validate_sbom.py").resolve() +TRUSTED_PROJECT_NAME = "memorymaster" +_GITLEAKS_CONFIG_PATH = "" +_GITLEAKS_IGNORE_PATH = "" +_PIP_REQUIREMENTS_PATH = "" +_GITLEAKS_EXECUTABLE = "" +_DOCKER_EXECUTABLE = "" +MAX_LOCAL_IMAGES = 3 +MAX_PROJECT_FILE_BYTES = 1024 * 1024 +MAX_EVIDENCE_FILE_BYTES = 512 * 1024 * 1024 +MAX_PLAN_SECONDS = 3600 +_IMAGE_ID_RE = re.compile(r"sha256:[0-9a-f]{64}\Z") +_COMMIT_RE = re.compile(r"[0-9a-fA-F]{40}(?:[0-9a-fA-F]{24})?\Z") +_REF_RE = re.compile(r"refs/[A-Za-z0-9._/-]+\Z") +_RELEASE_EXTRAS = ("mcp", "qdrant", "security") + + +class _SafeArgumentParser(argparse.ArgumentParser): + def error(self, _message: str) -> None: + raise ValueError("invalid arguments") + + +def _redacted_argv(argv: tuple[str, ...]) -> list[str]: + redacted: list[str] = [] + for value in argv: + if value.startswith("local://"): + redacted.append("local://") + elif Path(value).is_absolute(): + redacted.append("") + else: + redacted.append(value) + return redacted + + +@dataclass(frozen=True) +class CommandSpec: + name: str + argv: tuple[str, ...] + timeout_seconds: int + + def to_dict(self) -> dict[str, object]: + return { + "name": self.name, + "argv": _redacted_argv(self.argv), + "timeout_seconds": self.timeout_seconds, + } + + +@dataclass(frozen=True) +class CheckResult: + name: str + status: str + failure_kind: str | None = None + returncode: int | None = None + + def to_dict(self) -> dict[str, object]: + return { + "name": self.name, + "status": self.status, + "failure_kind": self.failure_kind, + "returncode": self.returncode, + } + + +@dataclass(frozen=True) +class ExecutionEvidence: + repository_commit: str + release_artifact_sha256: str + sbom_sha256: str + image_ids: tuple[str, ...] + tool_sha256: dict[str, str] + tool_versions: dict[str, str] + + def to_dict(self) -> dict[str, object]: + return { + "repository_commit": self.repository_commit, + "release_artifact_sha256": self.release_artifact_sha256, + "sbom_sha256": self.sbom_sha256, + "image_ids": list(self.image_ids), + "tool_sha256": dict(self.tool_sha256), + "tool_versions": dict(self.tool_versions), + } + + +@dataclass(frozen=True) +class ExecutionReport: + ok: bool + results: tuple[CheckResult, ...] + evidence: ExecutionEvidence | None = None + + def to_dict(self) -> dict[str, object]: + payload: dict[str, object] = { + "ok": self.ok, + "results": [item.to_dict() for item in self.results], + } + if self.evidence is not None: + payload["evidence"] = self.evidence.to_dict() + return payload + + +def _load_pyproject(repo_root: Path) -> dict[str, object]: + path = repo_root / "pyproject.toml" + try: + with path.open("rb") as handle: + payload = handle.read(MAX_PROJECT_FILE_BYTES + 1) + if len(payload) > MAX_PROJECT_FILE_BYTES: + raise ValueError + try: + import tomllib + except ModuleNotFoundError: + import tomli as tomllib # type: ignore[import-not-found] + document = tomllib.loads(payload.decode("utf-8")) + except (ImportError, OSError, UnicodeError, ValueError): + raise ValueError("project identity unavailable") from None + if not isinstance(document, dict): + raise ValueError("project identity unavailable") + return document + + +def _load_project_identity(repo_root: Path) -> tuple[str, str]: + project = _load_pyproject(repo_root).get("project") + name = project.get("name") if isinstance(project, dict) else None + version = project.get("version") if isinstance(project, dict) else None + if not isinstance(name, str) or not isinstance(version, str): + raise ValueError("project identity unavailable") + normalized_name = re.sub(r"[-_.]+", "-", name.strip()).casefold() + if normalized_name != TRUSTED_PROJECT_NAME or not version.strip(): + raise ValueError("unexpected project identity") + if len(version) > 128 or any(char.isspace() for char in version): + raise ValueError("invalid project version") + return TRUSTED_PROJECT_NAME, version + + +def _release_requirements(repo_root: Path) -> tuple[str, ...]: + project = _load_pyproject(repo_root).get("project") + if not isinstance(project, dict): + raise ValueError("project dependencies unavailable") + dependencies = project.get("dependencies") + optional = project.get("optional-dependencies") + if not isinstance(dependencies, list) or not isinstance(optional, dict): + raise ValueError("project dependencies unavailable") + requirements = list(dependencies) + for extra in _RELEASE_EXTRAS: + values = optional.get(extra) + if not isinstance(values, list): + raise ValueError("release extra unavailable") + requirements.extend(values) + if any( + not isinstance(value, str) or not value.strip() or len(value) > 512 or any(char in value for char in "\r\n") + for value in requirements + ): + raise ValueError("invalid release dependency") + return tuple(dict.fromkeys(value.strip() for value in requirements)) + + +def _inside_repo(repo_root: Path, path: Path, label: str) -> Path: + resolved = path.resolve() + try: + resolved.relative_to(repo_root) + except ValueError: + raise ValueError(f"{label} must stay inside repository") from None + return resolved + + +def _local_image_uri(image: str) -> str: + normalized = image.strip() + if normalized.startswith("local://"): + normalized = normalized[len("local://") :] + if not _IMAGE_ID_RE.fullmatch(normalized): + raise ValueError("local image must use an immutable sha256 image ID") + return f"local://{normalized}" + + +def _image_check_name(index: int) -> str: + return f"docker_scout_{index}" + + +def _gitleaks_command(repo_root: Path) -> CommandSpec: + return CommandSpec( + "gitleaks_history", + ( + _GITLEAKS_EXECUTABLE, + "git", + "--log-opts=--all", + "--config", + _GITLEAKS_CONFIG_PATH, + "--gitleaks-ignore-path", + _GITLEAKS_IGNORE_PATH, + "--ignore-gitleaks-allow", + "--redact=100", + "--no-banner", + "--exit-code", + "1", + str(repo_root), + ), + 300, + ) + + +def _python_commands( + repo_root: Path, + sbom_path: Path, + artifact_path: Path, + expected_name: str, + expected_version: str, +) -> tuple[CommandSpec, ...]: + return ( + CommandSpec( + "pip_audit_project", + ( + sys.executable, + "-I", + "-m", + "pip_audit", + "--strict", + "--vulnerability-service", + "pypi", + "--progress-spinner", + "off", + str(repo_root), + ), + 900, + ), + CommandSpec( + "pip_audit_release_extras", + ( + sys.executable, + "-I", + "-m", + "pip_audit", + "--strict", + "--vulnerability-service", + "pypi", + "--progress-spinner", + "off", + "--requirement", + _PIP_REQUIREMENTS_PATH, + ), + 900, + ), + CommandSpec( + "validate_sbom", + ( + sys.executable, + "-I", + str(TRUSTED_VALIDATOR_PATH), + "--sbom", + str(sbom_path), + "--artifact", + str(artifact_path), + "--expected-name", + expected_name, + "--expected-version", + expected_version, + ), + 30, + ), + ) + + +def _base_commands( + repo_root: Path, + sbom_path: Path, + artifact_path: Path, + expected_name: str, + expected_version: str, +) -> tuple[CommandSpec, ...]: + return ( + _gitleaks_command(repo_root), + *_python_commands(repo_root, sbom_path, artifact_path, expected_name, expected_version), + ) + + +def _image_command(image_uri: str, index: int) -> CommandSpec: + return CommandSpec( + _image_check_name(index), + ( + _DOCKER_EXECUTABLE, + "scout", + "cves", + "--only-severity", + "high,critical", + "--exit-code", + image_uri, + ), + 600, + ) + + +def build_command_plan( + *, + repo_root: Path, + sbom_path: Path, + local_images: Sequence[str], + expected_name: str | None = None, + expected_version: str | None = None, + release_artifact: Path | None = None, +) -> tuple[CommandSpec, ...]: + root = repo_root.resolve(strict=True) + output = _inside_repo(root, sbom_path, "SBOM") + name, version = _load_project_identity(root) + if expected_name is not None and expected_name != name: + raise ValueError("expected name does not match project") + if expected_version is not None and expected_version != version: + raise ValueError("expected version does not match project") + artifact = release_artifact or root / f"{name}-{version}-py3-none-any.whl" + artifact = _inside_repo(root, artifact, "release artifact") + if isinstance(local_images, (str, bytes)) or len(local_images) > MAX_LOCAL_IMAGES: + raise ValueError("local image count exceeds limit") + images = tuple(_local_image_uri(image) for image in local_images) + if not images: + raise ValueError("at least one local image is required") + commands = list(_base_commands(root, output, artifact, name, version)) + commands.extend(_image_command(image, index) for index, image in enumerate(images, start=1)) + return tuple(commands) + + +def _run_one( + spec: CommandSpec, + runner: Runner, + *, + environment: dict[str, str], + working_directory: Path, + timeout_seconds: float, +) -> CheckResult: + try: + completed = runner( + spec.argv, + check=False, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=timeout_seconds, + shell=False, + env=environment, + cwd=str(working_directory), + ) + except subprocess.TimeoutExpired: + return CheckResult(spec.name, "failed", "timeout") + except (FileNotFoundError, OSError): + return CheckResult(spec.name, "failed", "tool_unavailable") + except Exception: + return CheckResult(spec.name, "failed", "execution_error") + if completed.returncode != 0: + return CheckResult(spec.name, "failed", "nonzero_exit", completed.returncode) + return CheckResult(spec.name, "passed", returncode=0) + + +def _materialize_policy(plan: Sequence[CommandSpec], directory: Path) -> tuple[CommandSpec, ...]: + config_path = directory / "gitleaks.toml" + ignore_path = directory / "gitleaksignore" + requirements_path = directory / "release-requirements.txt" + gitleaks = next((spec for spec in plan if spec.name == "gitleaks_history"), None) + if gitleaks is None: + raise ValueError("gitleaks command unavailable") + requirements = _release_requirements(Path(gitleaks.argv[-1])) + config_path.write_text(TRUSTED_GITLEAKS_CONFIG, encoding="utf-8") + ignore_path.write_text("", encoding="utf-8") + requirements_path.write_text("\n".join(requirements) + "\n", encoding="utf-8") + (directory / "pip.conf").write_text( + "[global]\ndisable-pip-version-check = true\n", + encoding="utf-8", + ) + replacements = { + _GITLEAKS_CONFIG_PATH: str(config_path), + _GITLEAKS_IGNORE_PATH: str(ignore_path), + _PIP_REQUIREMENTS_PATH: str(requirements_path), + } + return tuple(replace(spec, argv=tuple(replacements.get(value, value) for value in spec.argv)) for spec in plan) + + +def _read_limited(path: Path, limit: int) -> str: + try: + with path.open("rb") as handle: + payload = handle.read(limit + 1) + except OSError: + raise ValueError("evidence unavailable") from None + if len(payload) > limit: + raise ValueError("evidence unavailable") + try: + return payload.decode("utf-8") + except UnicodeDecodeError: + raise ValueError("evidence unavailable") from None + + +def _git_directory(repo_root: Path) -> Path: + marker = repo_root / ".git" + if marker.is_dir(): + return marker.resolve(strict=True) + value = _read_limited(marker, 4096).strip() + if not value.casefold().startswith("gitdir:"): + raise ValueError("git metadata unavailable") + raw_path = value.split(":", 1)[1].strip() + candidate = Path(raw_path) + if not candidate.is_absolute(): + candidate = marker.parent / candidate + try: + return candidate.resolve(strict=True) + except OSError: + raise ValueError("git metadata unavailable") from None + + +def _packed_ref(common_directory: Path, ref_name: str) -> str | None: + path = common_directory / "packed-refs" + if not path.exists(): + return None + for line in _read_limited(path, 16 * 1024 * 1024).splitlines(): + if not line or line.startswith(("#", "^")): + continue + parts = line.split(" ", 1) + if len(parts) == 2 and parts[1] == ref_name and _COMMIT_RE.fullmatch(parts[0]): + return parts[0].lower() + return None + + +def _repository_commit(repo_root: Path) -> str: + git_directory = _git_directory(repo_root) + head = _read_limited(git_directory / "HEAD", 4096).strip() + if _COMMIT_RE.fullmatch(head): + return head.lower() + if not head.startswith("ref: "): + raise ValueError("git head unavailable") + ref_name = head[5:] + if not _REF_RE.fullmatch(ref_name) or ".." in ref_name or "//" in ref_name: + raise ValueError("git head unavailable") + common_directory = git_directory + common_marker = git_directory / "commondir" + if common_marker.exists(): + common_directory = (git_directory / _read_limited(common_marker, 4096).strip()).resolve(strict=True) + for root in (git_directory, common_directory): + ref_path = root / Path(ref_name) + if ref_path.is_file(): + value = _read_limited(ref_path, 4096).strip() + if _COMMIT_RE.fullmatch(value): + return value.lower() + packed = _packed_ref(common_directory, ref_name) + if packed is None: + raise ValueError("git head unavailable") + return packed + + +def _sha256_file(path: Path) -> str: + try: + resolved = path.resolve(strict=True) + if not resolved.is_file() or resolved.stat().st_size > MAX_EVIDENCE_FILE_BYTES: + raise ValueError + digest = hashlib.sha256() + bytes_read = 0 + with resolved.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + bytes_read += len(chunk) + if bytes_read > MAX_EVIDENCE_FILE_BYTES: + raise ValueError + digest.update(chunk) + except (OSError, ValueError): + raise ValueError("evidence file unavailable") from None + return digest.hexdigest() + + +def _plan_context(plan: Sequence[CommandSpec]) -> tuple[Path, Path, Path, tuple[str, ...]]: + commands = {spec.name: spec for spec in plan} + try: + repository = Path(commands["gitleaks_history"].argv[-1]).resolve(strict=True) + validation = commands["validate_sbom"].argv + sbom_path = Path(validation[validation.index("--sbom") + 1]) + artifact_path = Path(validation[validation.index("--artifact") + 1]) + image_ids = tuple( + spec.argv[-1].removeprefix("local://") for spec in plan if spec.name.startswith("docker_scout_") + ) + except (KeyError, OSError, ValueError): + raise ValueError("invalid execution plan") from None + return repository, sbom_path, artifact_path, image_ids + + +def _trusted_executable(name: str, repo_root: Path, resolver: Resolver) -> Path: + raw_path = resolver(name) + if not raw_path: + raise ValueError("approved tool unavailable") + try: + path = Path(raw_path).resolve(strict=True) + except OSError: + raise ValueError("approved tool unavailable") from None + if not path.is_file(): + raise ValueError("approved tool unavailable") + try: + path.relative_to(repo_root) + except ValueError: + return path + except OSError: + raise ValueError("approved tool unavailable") from None + raise ValueError("repository-local executable refused") + + +def _materialize_tools( + plan: Sequence[CommandSpec], repo_root: Path, resolver: Resolver +) -> tuple[tuple[CommandSpec, ...], dict[str, Path]]: + tools = { + "gitleaks": _trusted_executable("gitleaks", repo_root, resolver), + "docker": _trusted_executable("docker", repo_root, resolver), + "git": _trusted_executable("git", repo_root, resolver), + "python": Path(sys.executable).resolve(strict=True), + } + replacements = { + _GITLEAKS_EXECUTABLE: str(tools["gitleaks"]), + _DOCKER_EXECUTABLE: str(tools["docker"]), + } + materialized = tuple( + replace(spec, argv=tuple(replacements.get(value, value) for value in spec.argv)) for spec in plan + ) + return materialized, tools + + +def _filtered_path(repo_root: Path, tools: dict[str, Path]) -> str: + entries = [str(tools[name].parent) for name in ("gitleaks", "docker", "git") if name in tools] + system_root = os.environ.get("SYSTEMROOT") or os.environ.get("WINDIR") + if system_root: + entries.append(str(Path(system_root) / "System32")) + return os.pathsep.join(dict.fromkeys(entries)) + + +def _sterile_environment(directory: Path, repo_root: Path, tools: dict[str, Path]) -> dict[str, str]: + environment: dict[str, str] = {} + for key in ("SYSTEMROOT", "WINDIR", "COMSPEC", "PATHEXT"): + value = os.environ.get(key) + if value: + environment[key] = value + environment.update( + { + "PATH": _filtered_path(repo_root, tools), + "HOME": str(directory), + "USERPROFILE": str(directory), + "TEMP": str(directory), + "TMP": str(directory), + "TMPDIR": str(directory), + "NO_COLOR": "1", + "PIP_CONFIG_FILE": str(directory / "pip.conf"), + } + ) + return environment + + +def _collect_evidence( + repo_root: Path, + sbom_path: Path, + artifact_path: Path, + image_ids: tuple[str, ...], + tools: dict[str, Path], +) -> ExecutionEvidence: + try: + pip_audit_version = importlib.metadata.version("pip-audit") + except importlib.metadata.PackageNotFoundError: + raise ValueError("pip-audit unavailable") from None + return ExecutionEvidence( + repository_commit=_repository_commit(repo_root), + release_artifact_sha256=_sha256_file(artifact_path), + sbom_sha256=_sha256_file(sbom_path), + image_ids=image_ids, + tool_sha256={name: _sha256_file(path) for name, path in tools.items()}, + tool_versions={"pip-audit": pip_audit_version, "python": sys.version.split()[0]}, + ) + + +def _execute( + plan: Sequence[CommandSpec], + runner: Runner, + *, + environment: dict[str, str], + working_directory: Path, +) -> ExecutionReport: + results: list[CheckResult] = [] + deadline = time.monotonic() + MAX_PLAN_SECONDS + for spec in plan: + remaining = deadline - time.monotonic() + if remaining <= 0: + results.append(CheckResult(spec.name, "failed", "global_deadline")) + return ExecutionReport(False, tuple(results)) + result = _run_one( + spec, + runner, + environment=environment, + working_directory=working_directory, + timeout_seconds=min(float(spec.timeout_seconds), remaining), + ) + results.append(result) + if result.status != "passed": + return ExecutionReport(False, tuple(results)) + return ExecutionReport(True, tuple(results)) + + +def execute_plan( + plan: Sequence[CommandSpec], + *, + runner: Runner = subprocess.run, + resolver: Resolver = shutil.which, +) -> ExecutionReport: + requires_policy = any(_GITLEAKS_CONFIG_PATH in spec.argv for spec in plan) + if not requires_policy: + try: + with tempfile.TemporaryDirectory(prefix="memorymaster-supply-chain-") as raw_directory: + directory = Path(raw_directory) + environment = _sterile_environment(directory, Path.cwd().resolve(), {}) + return _execute(plan, runner, environment=environment, working_directory=directory) + except OSError: + return ExecutionReport(False, (CheckResult("execution", "failed", "environment_unavailable"),)) + try: + with tempfile.TemporaryDirectory(prefix="memorymaster-supply-chain-") as raw_directory: + directory = Path(raw_directory) + repo_root, sbom_path, artifact_path, image_ids = _plan_context(plan) + materialized = _materialize_policy(plan, directory) + materialized, tools = _materialize_tools(materialized, repo_root, resolver) + environment = _sterile_environment(directory, repo_root, tools) + evidence = _collect_evidence(repo_root, sbom_path, artifact_path, image_ids, tools) + report = _execute( + materialized, + runner, + environment=environment, + working_directory=directory, + ) + return replace(report, evidence=evidence) + except ValueError: + return ExecutionReport( + False, + (CheckResult("supply_chain_preflight", "failed", "evidence_unavailable"),), + ) + except OSError: + return ExecutionReport( + False, + (CheckResult("gitleaks_history", "failed", "policy_unavailable"),), + ) + + +def _parser() -> argparse.ArgumentParser: + parser = _SafeArgumentParser(description="Run fail-closed local supply-chain checks.") + parser.add_argument("--repo-root", type=Path, default=Path.cwd()) + parser.add_argument("--sbom", type=Path, default=Path("artifacts/memorymaster.cdx.json")) + parser.add_argument("--release-artifact", required=True, type=Path) + parser.add_argument("--local-image", action="append", default=[]) + parser.add_argument("--expected-name") + parser.add_argument("--expected-version") + parser.add_argument("--command-plan", action="store_true") + parser.add_argument("--dry-run", action="store_true") + return parser + + +def _plan_payload(plan: Sequence[CommandSpec]) -> dict[str, Any]: + return {"mode": "command-plan", "commands": [item.to_dict() for item in plan]} + + +def main(argv: Sequence[str] | None = None, *, runner: Runner = subprocess.run) -> int: + try: + args = _parser().parse_args(argv) + if args.repo_root.resolve(strict=True) != TRUSTED_REPO_ROOT: + raise ValueError("repository root does not match trusted script") + plan = build_command_plan( + repo_root=args.repo_root, + sbom_path=args.sbom, + local_images=args.local_image, + expected_name=args.expected_name, + expected_version=args.expected_version, + release_artifact=args.release_artifact, + ) + except (OSError, ValueError): + print(json.dumps({"ok": False, "failure_kind": "invalid_configuration"})) + return 2 + if args.command_plan or args.dry_run: + print(json.dumps(_plan_payload(plan), indent=2)) + return 0 + report = execute_plan(plan, runner=runner) + print(json.dumps(report.to_dict(), indent=2)) + return 0 if report.ok else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validate_sbom.py b/scripts/validate_sbom.py new file mode 100644 index 00000000..0bbf5dee --- /dev/null +++ b/scripts/validate_sbom.py @@ -0,0 +1,297 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import re +import stat +import zipfile +from collections.abc import Mapping, Sequence +from email.parser import BytesParser +from email.policy import compat32 +from pathlib import Path + + +MAX_SBOM_BYTES = 32 * 1024 * 1024 +MAX_JSON_DIGITS = 1024 +MAX_JSON_DEPTH = 128 +MAX_JSON_NODES = 100_000 +MAX_ARTIFACT_BYTES = 256 * 1024 * 1024 +MAX_WHEEL_ENTRIES = 10_000 +MAX_METADATA_BYTES = 1024 * 1024 +SUPPORTED_SPEC_VERSIONS = frozenset({"1.4", "1.5", "1.6"}) +_SHA256_RE = re.compile(r"[0-9a-fA-F]{64}\Z") + + +class _SafeArgumentParser(argparse.ArgumentParser): + def error(self, _message: str) -> None: + raise ValueError("invalid arguments") + + +def _bounded_int(value: str) -> int: + if len(value.lstrip("-")) > MAX_JSON_DIGITS: + raise ValueError("integer is too large") + return int(value) + + +def _bounded_float(value: str) -> float: + result = float(value) + if not math.isfinite(result): + raise ValueError("float is not finite") + return result + + +def _reject_constant(_value: str) -> None: + raise ValueError("non-standard JSON constant") + + +def _unique_object(pairs: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate JSON key") + result[key] = value + return result + + +def _document_shape_bounded(document: object) -> bool: + pending = [(document, 0)] + nodes = 0 + while pending: + value, depth = pending.pop() + nodes += 1 + if nodes > MAX_JSON_NODES or depth > MAX_JSON_DEPTH: + return False + if type(value) is dict: + pending.extend((item, depth + 1) for pair in value.items() for item in pair) + elif type(value) is list: + pending.extend((item, depth + 1) for item in value) + elif type(value) is float and not math.isfinite(value): + return False + elif value is not None and type(value) not in {str, int, float, bool}: + return False + return True + + +def _canonical_name(value: str) -> str: + return re.sub(r"[-_.]+", "-", value).casefold() + + +def _component_hash(component: Mapping[str, object]) -> str | None: + hashes = component.get("hashes") + if not isinstance(hashes, list): + return None + for item in hashes: + if not isinstance(item, Mapping): + continue + algorithm = item.get("alg") + content = item.get("content") + if ( + isinstance(algorithm, str) + and algorithm.casefold().replace("-", "") == "sha256" + and isinstance(content, str) + and _SHA256_RE.fullmatch(content) + ): + return content.casefold() + return None + + +def _project_component(document: Mapping[str, object]) -> Mapping[str, object] | None: + metadata = document.get("metadata") + if not isinstance(metadata, Mapping): + return None + component = metadata.get("component") + return component if isinstance(component, Mapping) else None + + +def _component_matches( + component: Mapping[str, object] | None, + expected_name: str, + expected_version: str, +) -> bool: + if component is None: + return False + expected_name_folded = _canonical_name(expected_name) + expected_purl = f"pkg:pypi/{expected_name_folded}@{expected_version}" + component_type = component.get("type") + name = component.get("name") + purl = component.get("purl") + return ( + isinstance(component_type, str) + and component_type in {"application", "library"} + and isinstance(name, str) + and _canonical_name(name) == expected_name_folded + and component.get("version") == expected_version + and isinstance(purl, str) + and purl.casefold() == expected_purl + ) + + +def validate_sbom_document( + document: object, + *, + expected_name: str, + expected_version: str, + expected_artifact_sha256: str | None = None, +) -> tuple[str, ...]: + if not isinstance(document, Mapping): + return ("document_not_object",) + if not isinstance(expected_name, str) or not isinstance(expected_version, str): + return ("invalid_expected_identity",) + errors: list[str] = [] + if document.get("bomFormat") != "CycloneDX": + errors.append("invalid_bom_format") + spec_version = document.get("specVersion") + if not isinstance(spec_version, str) or spec_version not in SUPPORTED_SPEC_VERSIONS: + errors.append("invalid_spec_version") + expected_schemas = { + f"http://cyclonedx.org/schema/bom-{spec_version}.schema.json", + f"https://cyclonedx.org/schema/bom-{spec_version}.schema.json", + } + schema = document.get("$schema") + if not isinstance(schema, str) or schema not in expected_schemas: + errors.append("invalid_schema") + components = document.get("components") + if not isinstance(components, list) or not components: + errors.append("components_missing") + elif any(not isinstance(item, Mapping) or not isinstance(item.get("type"), str) for item in components): + errors.append("components_invalid") + component = _project_component(document) + if not _component_matches(component, expected_name, expected_version): + errors.append("project_component_missing") + else: + component_hash = _component_hash(component) + if component_hash is None: + errors.append("artifact_hash_missing") + elif expected_artifact_sha256 is not None and component_hash != expected_artifact_sha256.casefold(): + errors.append("artifact_hash_mismatch") + return tuple(errors) + + +def _load_document(path: Path) -> tuple[object | None, str | None]: + try: + with path.open("rb") as handle: + payload = handle.read(MAX_SBOM_BYTES + 1) + except (AttributeError, OSError): + return None, "file_unavailable" + if len(payload) > MAX_SBOM_BYTES: + return None, "file_too_large" + try: + document = json.loads( + payload.decode("utf-8"), + parse_int=_bounded_int, + parse_float=_bounded_float, + parse_constant=_reject_constant, + object_pairs_hook=_unique_object, + ) + except (UnicodeDecodeError, json.JSONDecodeError, RecursionError, ValueError): + return None, "invalid_json" + if not _document_shape_bounded(document): + return None, "document_too_complex" + return document, None + + +def _wheel_identity_and_sha256(path: Path) -> tuple[str, str, str] | None: + if path.suffix.casefold() != ".whl": + return None + try: + with path.open("rb") as handle: + file_stat = os.fstat(handle.fileno()) + if not stat.S_ISREG(file_stat.st_mode) or file_stat.st_size > MAX_ARTIFACT_BYTES: + return None + digest = hashlib.sha256() + bytes_read = 0 + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + bytes_read += len(chunk) + if bytes_read > MAX_ARTIFACT_BYTES: + return None + digest.update(chunk) + handle.seek(0) + with zipfile.ZipFile(handle) as archive: + infos = archive.infolist() + if len(infos) > MAX_WHEEL_ENTRIES: + return None + metadata_infos = [ + info + for info in infos + if info.filename.endswith(".dist-info/METADATA") + and len(info.filename.split("/")) == 2 + and info.file_size <= MAX_METADATA_BYTES + ] + if len(metadata_infos) != 1: + return None + with archive.open(metadata_infos[0]) as metadata_handle: + payload = metadata_handle.read(MAX_METADATA_BYTES + 1) + except (OSError, RuntimeError, ValueError, zipfile.BadZipFile): + return None + if len(payload) > MAX_METADATA_BYTES: + return None + metadata = BytesParser(policy=compat32).parsebytes(payload) + name = metadata.get("Name") + version = metadata.get("Version") + if not isinstance(name, str) or not isinstance(version, str): + return None + return _canonical_name(name.strip()), version.strip(), digest.hexdigest() + + +def validate_sbom_file( + path: Path, + *, + expected_name: str, + expected_version: str, + artifact_path: Path | None = None, +) -> tuple[str, ...]: + if not isinstance(expected_name, str) or not isinstance(expected_version, str): + return ("invalid_expected_identity",) + document, load_error = _load_document(path) + if load_error is not None: + return (load_error,) + if artifact_path is None: + return validate_sbom_document( + document, + expected_name=expected_name, + expected_version=expected_version, + ) + artifact = _wheel_identity_and_sha256(artifact_path) + if artifact is None: + return ("artifact_invalid",) + artifact_name, artifact_version, artifact_sha256 = artifact + if artifact_name != _canonical_name(expected_name) or artifact_version != expected_version: + return ("artifact_identity_mismatch",) + return validate_sbom_document( + document, + expected_name=expected_name, + expected_version=expected_version, + expected_artifact_sha256=artifact_sha256, + ) + + +def _parser() -> argparse.ArgumentParser: + parser = _SafeArgumentParser(description="Validate a release CycloneDX SBOM.") + parser.add_argument("--sbom", required=True, type=Path) + parser.add_argument("--artifact", required=True, type=Path) + parser.add_argument("--expected-name", required=True) + parser.add_argument("--expected-version", required=True) + return parser + + +def main(argv: Sequence[str] | None = None) -> int: + try: + args = _parser().parse_args(argv) + except ValueError: + print(json.dumps({"ok": False, "errors": ["invalid_arguments"]})) + return 2 + errors = validate_sbom_file( + args.sbom, + expected_name=args.expected_name, + expected_version=args.expected_version, + artifact_path=args.artifact, + ) + print(json.dumps({"ok": not errors, "errors": list(errors)})) + return 0 if not errors else 2 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_supply_chain_contracts.py b/tests/test_supply_chain_contracts.py new file mode 100644 index 00000000..02046db6 --- /dev/null +++ b/tests/test_supply_chain_contracts.py @@ -0,0 +1,875 @@ +from __future__ import annotations + +import hashlib +import json +import io +import subprocess +import sys +import tempfile +import zipfile +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from scripts import run_supply_chain_checks as supply +from scripts import validate_sbom as sbom + + +EXPECTED_NAME = "memorymaster" +EXPECTED_VERSION = "4.4.1" +IMAGE_ID = "sha256:" + ("a" * 64) + + +def _valid_sbom(artifact_sha256: str = "b" * 64) -> dict[str, object]: + return { + "bomFormat": "CycloneDX", + "specVersion": "1.6", + "$schema": "https://cyclonedx.org/schema/bom-1.6.schema.json", + "metadata": { + "component": { + "type": "application", + "name": EXPECTED_NAME, + "version": EXPECTED_VERSION, + "purl": f"pkg:pypi/{EXPECTED_NAME}@{EXPECTED_VERSION}", + "hashes": [{"alg": "SHA-256", "content": artifact_sha256}], + } + }, + "components": [ + { + "type": "application", + "name": EXPECTED_NAME, + "version": EXPECTED_VERSION, + "purl": f"pkg:pypi/{EXPECTED_NAME}@{EXPECTED_VERSION}", + } + ], + } + + +def _wheel(tmp_path: Path) -> Path: + path = tmp_path / "memorymaster-4.4.1-py3-none-any.whl" + with zipfile.ZipFile(path, "w") as archive: + archive.writestr( + "memorymaster-4.4.1.dist-info/METADATA", + "Metadata-Version: 2.1\nName: memorymaster\nVersion: 4.4.1\n", + ) + archive.writestr("memorymaster/__init__.py", "") + return path + + +def _prepare_repo(tmp_path: Path) -> Path: + (tmp_path / "pyproject.toml").write_text( + ( + f'[project]\nname = "{EXPECTED_NAME}"\nversion = "{EXPECTED_VERSION}"\n' + 'dependencies = ["requests>=2.31", "tenacity>=8.2"]\n' + "[project.optional-dependencies]\n" + 'mcp = ["mcp>=1.2"]\n' + 'qdrant = ["httpx>=0.27"]\n' + 'security = ["cryptography>=42"]\n' + ), + encoding="utf-8", + ) + git_directory = tmp_path / ".git" + git_directory.mkdir(exist_ok=True) + (git_directory / "HEAD").write_text("1" * 40, encoding="ascii") + wheel = _wheel(tmp_path) + digest = hashlib.sha256(wheel.read_bytes()).hexdigest() + (tmp_path / "memorymaster.cdx.json").write_text(json.dumps(_valid_sbom(digest)), encoding="utf-8") + return wheel + + +def _resolver(tmp_path: Path) -> supply.Resolver: + directory = tmp_path.parent / f"{tmp_path.name}-trusted-tools" + directory.mkdir(exist_ok=True) + paths: dict[str, str] = {} + for name in ("gitleaks", "docker", "git"): + path = directory / f"{name}.bin" + path.write_bytes(f"trusted-{name}".encode()) + paths[name] = str(path) + return paths.get + + +def _plan(tmp_path: Path) -> tuple[supply.CommandSpec, ...]: + wheel = _prepare_repo(tmp_path) + return supply.build_command_plan( + repo_root=tmp_path, + sbom_path=tmp_path / "memorymaster.cdx.json", + local_images=[IMAGE_ID], + expected_name=EXPECTED_NAME, + expected_version=EXPECTED_VERSION, + release_artifact=wheel, + ) + + +def _execute( + plan: tuple[supply.CommandSpec, ...], + tmp_path: Path, + runner: supply.Runner, +) -> supply.ExecutionReport: + return supply.execute_plan(plan, runner=runner, resolver=_resolver(tmp_path)) + + +def _by_name(plan: tuple[supply.CommandSpec, ...]) -> dict[str, supply.CommandSpec]: + return {item.name: item for item in plan} + + +def test_plan_enforces_full_history_redacted_gitleaks_scan(tmp_path: Path) -> None: + command = _by_name(_plan(tmp_path))["gitleaks_history"] + + assert command.argv[1] == "git" + assert command.argv[0] != "gitleaks" + assert "--log-opts=--all" in command.argv + assert "--redact=100" in command.argv + assert "--no-git" not in command.argv + + +def test_gitleaks_policy_cannot_be_suppressed_by_repo_or_environment( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + (tmp_path / ".gitleaks.toml").write_text("[allowlist]\npaths=['.*']\n", encoding="utf-8") + (tmp_path / ".gitleaksignore").write_text("*:*\n", encoding="utf-8") + monkeypatch.setenv("GITLEAKS_CONFIG", str(tmp_path / ".gitleaks.toml")) + monkeypatch.setenv("GITLEAKS_CONFIG_TOML", "[allowlist]\npaths=['.*']") + observed: dict[str, object] = {} + + def fake_runner(argv: tuple[str, ...], **kwargs: object) -> subprocess.CompletedProcess[str]: + if "--log-opts=--all" in argv: + observed["argv"] = argv + observed["env"] = kwargs.get("env") + if "--config" in argv: + config_path = Path(argv[argv.index("--config") + 1]) + observed["config"] = config_path.read_text(encoding="utf-8") + if "--gitleaks-ignore-path" in argv: + ignore_path = Path(argv[argv.index("--gitleaks-ignore-path") + 1]) + observed["ignore"] = ignore_path.read_text(encoding="utf-8") + return subprocess.CompletedProcess(argv, 0, "", "") + + report = _execute(_plan(tmp_path), tmp_path, fake_runner) + argv = observed["argv"] + environment = observed["env"] + + assert report.ok is True + assert isinstance(argv, tuple) + assert "--ignore-gitleaks-allow" in argv + assert "--config" in argv + assert "--gitleaks-ignore-path" in argv + assert observed["config"] == supply.TRUSTED_GITLEAKS_CONFIG + assert observed["ignore"] == "" + assert isinstance(environment, dict) + assert "GITLEAKS_CONFIG" not in environment + assert "GITLEAKS_CONFIG_TOML" not in environment + + +def test_scanners_receive_sterile_environment_and_working_directory( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + for key in ( + "GIT_DIR", + "GIT_WORK_TREE", + "PIP_AUDIT_VULNERABILITY_SERVICE", + "PIP_AUDIT_OSV_URL", + "DOCKER_HOST", + "AWS_SECRET_ACCESS_KEY", + ): + monkeypatch.setenv(key, "hostile-secret-value") + observed: list[dict[str, object]] = [] + + def fake_runner(argv: tuple[str, ...], **kwargs: object) -> subprocess.CompletedProcess[str]: + observed.append(kwargs) + return subprocess.CompletedProcess(argv, 0, "", "") + + assert _execute(_plan(tmp_path), tmp_path, fake_runner).ok is True + assert observed + for kwargs in observed: + environment = kwargs.get("env") + assert isinstance(environment, dict) + assert not any(key.startswith(("GIT_", "PIP_AUDIT_", "DOCKER_")) for key in environment) + assert "AWS_SECRET_ACCESS_KEY" not in environment + working_directory = Path(str(kwargs["cwd"])).resolve() + assert working_directory != tmp_path.resolve() + assert tmp_path.resolve() not in working_directory.parents + + +def test_native_scanners_are_absolute_and_validator_is_trusted(tmp_path: Path) -> None: + observed: list[tuple[str, ...]] = [] + + def fake_runner(argv: tuple[str, ...], **_kwargs: object) -> subprocess.CompletedProcess[str]: + observed.append(argv) + return subprocess.CompletedProcess(argv, 0, "", "") + + assert _execute(_plan(tmp_path), tmp_path, fake_runner).ok is True + commands = _by_name(_plan(tmp_path)) + validator = commands["validate_sbom"].argv + + scanner_calls = [argv for argv in observed if "--log-opts=--all" in argv or "scout" in argv] + assert len(scanner_calls) == 2 + assert all(Path(argv[0]).is_absolute() for argv in scanner_calls) + assert Path(validator[2]).resolve() == Path(sbom.__file__).resolve() + + +def test_repository_local_native_scanner_is_refused(tmp_path: Path) -> None: + plan = _plan(tmp_path) + local_tool = tmp_path / "gitleaks.exe" + local_tool.write_bytes(b"untrusted") + trusted = _resolver(tmp_path) + + def resolver(name: str) -> str | None: + return str(local_tool) if name == "gitleaks" else trusted(name) + + report = supply.execute_plan( + plan, + runner=lambda *_args, **_kwargs: pytest.fail("runner called"), + resolver=resolver, + ) + + assert report.ok is False + assert report.results[0].status == "failed" + + +def test_gitleaks_policy_materialization_failure_is_fixed_and_fail_closed( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + def unavailable(*_args: object, **_kwargs: object) -> object: + raise OSError("secret temp path") + + monkeypatch.setattr(tempfile, "TemporaryDirectory", unavailable) + + report = _execute( + _plan(tmp_path), + tmp_path, + lambda *_args, **_kwargs: pytest.fail("runner called"), + ) + + assert report.ok is False + assert report.results == (supply.CheckResult("gitleaks_history", "failed", "policy_unavailable"),) + assert "secret temp path" not in json.dumps(report.to_dict()) + + +def test_plan_enforces_strict_cyclonedx_dependency_audit(tmp_path: Path) -> None: + commands = _by_name(_plan(tmp_path)) + audit = commands["pip_audit_project"].argv + validation = commands["validate_sbom"].argv + + assert audit[:4] == (sys.executable, "-I", "-m", "pip_audit") + assert "--strict" in audit + assert str(tmp_path.resolve()) in audit + assert validation[0] == sys.executable + assert validation[validation.index("--expected-name") + 1] == EXPECTED_NAME + assert validation[validation.index("--expected-version") + 1] == EXPECTED_VERSION + + +def test_dependency_audit_is_bound_to_project_and_fixed_service(tmp_path: Path) -> None: + audit = _by_name(_plan(tmp_path))["pip_audit_project"].argv + + assert str(tmp_path.resolve()) in audit + assert audit[audit.index("--vulnerability-service") + 1] == "pypi" + + +def test_release_dependency_audit_includes_docker_extras(tmp_path: Path) -> None: + observed_requirements: list[str] = [] + + def fake_runner(argv: tuple[str, ...], **_kwargs: object) -> subprocess.CompletedProcess[str]: + if "--requirement" in argv: + path = Path(argv[argv.index("--requirement") + 1]) + observed_requirements.extend(path.read_text(encoding="utf-8").splitlines()) + return subprocess.CompletedProcess(argv, 0) + + report = _execute(_plan(tmp_path), tmp_path, fake_runner) + + assert report.ok is True + assert {"requests>=2.31", "tenacity>=8.2"}.issubset(observed_requirements) + assert {"mcp>=1.2", "httpx>=0.27", "cryptography>=42"}.issubset(observed_requirements) + + +def test_project_identity_cannot_be_supplied_by_operator(tmp_path: Path) -> None: + (tmp_path / "pyproject.toml").write_text('[project]\nname = "memorymaster"\nversion = "9.9.9"\n', encoding="utf-8") + + with pytest.raises(ValueError, match="does not match"): + supply.build_command_plan( + repo_root=tmp_path, + sbom_path=tmp_path / "sbom.json", + local_images=[IMAGE_ID], + expected_name="lookalike", + expected_version="0.0.0", + ) + + +def test_python_scanner_and_validator_use_isolated_mode(tmp_path: Path) -> None: + commands = _by_name(_plan(tmp_path)) + + assert commands["pip_audit_project"].argv[:3] == (sys.executable, "-I", "-m") + assert commands["validate_sbom"].argv[:2] == (sys.executable, "-I") + + +def test_plan_enforces_local_only_high_and_critical_image_scan(tmp_path: Path) -> None: + command = next(item for item in _plan(tmp_path) if item.name.startswith("docker_scout_")) + + assert command.argv[1:3] == ("scout", "cves") + assert command.argv[-1] == f"local://{IMAGE_ID}" + severities = command.argv[command.argv.index("--only-severity") + 1] + assert set(severities.split(",")) == {"high", "critical"} + assert "--exit-code" in command.argv + assert "--ignore-base" not in command.argv + + +def test_mutable_image_tag_is_rejected(tmp_path: Path) -> None: + wheel = _prepare_repo(tmp_path) + with pytest.raises(ValueError, match="immutable"): + supply.build_command_plan( + repo_root=tmp_path, + sbom_path=tmp_path / "memorymaster.cdx.json", + local_images=["memorymaster:phase1"], + release_artifact=wheel, + ) + + +def test_local_image_count_is_bounded(tmp_path: Path) -> None: + wheel = _prepare_repo(tmp_path) + with pytest.raises(ValueError, match="image"): + supply.build_command_plan( + repo_root=tmp_path, + sbom_path=tmp_path / "sbom.json", + local_images=[f"sha256:{index:064x}" for index in range(16)], + expected_name=EXPECTED_NAME, + expected_version=EXPECTED_VERSION, + release_artifact=wheel, + ) + + +@pytest.mark.parametrize( + "image", + [ + "https://registry.invalid/memorymaster:latest", + "docker://memorymaster:latest", + "registry.invalid/memorymaster:latest\n--ignore-base", + "", + ], +) +def test_plan_rejects_remote_or_malformed_image_references(tmp_path: Path, image: str) -> None: + wheel = _prepare_repo(tmp_path) + with pytest.raises(ValueError, match="local image"): + supply.build_command_plan( + repo_root=tmp_path, + sbom_path=tmp_path / "sbom.json", + local_images=[image], + expected_name=EXPECTED_NAME, + expected_version=EXPECTED_VERSION, + release_artifact=wheel, + ) + + +def test_plan_fails_closed_when_no_local_image_is_supplied(tmp_path: Path) -> None: + wheel = _prepare_repo(tmp_path) + with pytest.raises(ValueError, match="local image"): + supply.build_command_plan( + repo_root=tmp_path, + sbom_path=tmp_path / "sbom.json", + local_images=[], + expected_name=EXPECTED_NAME, + expected_version=EXPECTED_VERSION, + release_artifact=wheel, + ) + + +def test_every_command_has_bounded_timeout_and_argv(tmp_path: Path) -> None: + for command in _plan(tmp_path): + assert isinstance(command.argv, tuple) + assert command.argv + assert all(isinstance(part, str) and part for part in command.argv) + assert 0 < command.timeout_seconds <= 900 + + +def test_execute_plan_uses_no_shell_and_never_emits_tool_output(tmp_path: Path) -> None: + calls: list[tuple[tuple[str, ...], dict[str, object]]] = [] + + def fake_runner(argv: tuple[str, ...], **kwargs: object) -> subprocess.CompletedProcess[str]: + calls.append((argv, kwargs)) + return subprocess.CompletedProcess(argv, 0, "secret-from-stdout", "secret-from-stderr") + + report = _execute(_plan(tmp_path), tmp_path, fake_runner) + + assert report.ok is True + assert len(calls) == len(_plan(tmp_path)) + assert all(kwargs["shell"] is False for _, kwargs in calls) + assert all(kwargs["stdout"] is subprocess.DEVNULL for _, kwargs in calls) + assert all(kwargs["stderr"] is subprocess.DEVNULL for _, kwargs in calls) + assert all("capture_output" not in kwargs for _, kwargs in calls) + assert all(isinstance(kwargs["timeout"], (int, float)) for _, kwargs in calls) + encoded = json.dumps(report.to_dict()) + assert "secret-from-stdout" not in encoded + assert "secret-from-stderr" not in encoded + + +def test_runner_discards_scanner_streams_instead_of_buffering(tmp_path: Path) -> None: + calls: list[dict[str, object]] = [] + + def fake_runner(argv: tuple[str, ...], **kwargs: object) -> subprocess.CompletedProcess[str]: + calls.append(kwargs) + return subprocess.CompletedProcess(argv, 0, None, None) + + assert _execute(_plan(tmp_path), tmp_path, fake_runner).ok is True + assert calls + assert all(call.get("stdout") is subprocess.DEVNULL for call in calls) + assert all(call.get("stderr") is subprocess.DEVNULL for call in calls) + assert all("capture_output" not in call for call in calls) + + +def test_success_report_contains_bound_evidence(tmp_path: Path) -> None: + report = _execute( + _plan(tmp_path), + tmp_path, + lambda argv, **_: subprocess.CompletedProcess(argv, 0, "", ""), + ).to_dict() + + evidence = report["evidence"] + assert evidence["repository_commit"] + assert evidence["release_artifact_sha256"] + assert evidence["sbom_sha256"] + assert evidence["image_ids"] + assert evidence["tool_sha256"] + + +@pytest.mark.parametrize("returncode", [1, 2, 127]) +def test_nonzero_tool_exit_fails_closed(returncode: int) -> None: + spec = supply.CommandSpec("scanner", ("scanner", "safe"), 10) + + def fake_runner(argv: tuple[str, ...], **_: object) -> subprocess.CompletedProcess[str]: + return subprocess.CompletedProcess(argv, returncode, "sensitive finding", "sensitive detail") + + report = supply.execute_plan((spec,), runner=fake_runner) + + assert report.ok is False + assert report.results[0].status == "failed" + assert report.results[0].failure_kind == "nonzero_exit" + assert "sensitive" not in json.dumps(report.to_dict()) + + +def test_missing_tool_fails_closed_without_leaking_exception() -> None: + spec = supply.CommandSpec("scanner", ("missing",), 10) + + def fake_runner(*_: object, **__: object) -> subprocess.CompletedProcess[str]: + raise FileNotFoundError("missing secret-tool-token") + + report = supply.execute_plan((spec,), runner=fake_runner) + + assert report.ok is False + assert report.results[0].failure_kind == "tool_unavailable" + assert "secret-tool-token" not in json.dumps(report.to_dict()) + + +def test_timeout_fails_closed_without_leaking_captured_output() -> None: + spec = supply.CommandSpec("scanner", ("scanner",), 3) + + def fake_runner(*_: object, **__: object) -> subprocess.CompletedProcess[str]: + raise subprocess.TimeoutExpired( + cmd=["scanner", "secret-argument"], + timeout=3, + output="secret-output", + stderr="secret-error", + ) + + report = supply.execute_plan((spec,), runner=fake_runner) + + assert report.ok is False + assert report.results[0].failure_kind == "timeout" + assert "secret" not in json.dumps(report.to_dict()) + + +def test_unexpected_runner_error_fails_closed_without_leaking_exception() -> None: + spec = supply.CommandSpec("scanner", ("scanner",), 3) + + def fake_runner(*_: object, **__: object) -> subprocess.CompletedProcess[str]: + raise RuntimeError("secret-runtime-detail") + + report = supply.execute_plan((spec,), runner=fake_runner) + + assert report.ok is False + assert report.results[0].failure_kind == "execution_error" + assert "secret-runtime-detail" not in json.dumps(report.to_dict()) + + +def test_execute_plan_stops_after_first_failure() -> None: + calls: list[tuple[str, ...]] = [] + plan = ( + supply.CommandSpec("first", ("first",), 10), + supply.CommandSpec("second", ("second",), 10), + ) + + def fake_runner(argv: tuple[str, ...], **_: object) -> subprocess.CompletedProcess[str]: + calls.append(argv) + return subprocess.CompletedProcess(argv, 1, "", "") + + report = supply.execute_plan(plan, runner=fake_runner) + + assert report.ok is False + assert calls == [("first",)] + assert [item.name for item in report.results] == ["first"] + + +def test_execute_plan_enforces_global_deadline(monkeypatch: pytest.MonkeyPatch) -> None: + plan = ( + supply.CommandSpec("first", ("first",), 10), + supply.CommandSpec("second", ("second",), 10), + ) + clock = iter([0.0, 0.0, supply.MAX_PLAN_SECONDS + 1.0]) + monkeypatch.setattr(supply.time, "monotonic", lambda: next(clock)) + calls: list[tuple[str, ...]] = [] + + def fake_runner(argv: tuple[str, ...], **_: object) -> subprocess.CompletedProcess[str]: + calls.append(argv) + return subprocess.CompletedProcess(argv, 0) + + report = supply.execute_plan(plan, runner=fake_runner) + + assert report.ok is False + assert calls == [("first",)] + assert report.results[-1].failure_kind == "global_deadline" + + +def test_cli_refuses_foreign_repository_root(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + marker = "foreign-secret-marker" + code = supply.main( + [ + "--repo-root", + str(tmp_path), + "--release-artifact", + str(tmp_path / marker), + "--local-image", + IMAGE_ID, + "--command-plan", + ] + ) + + assert code == 2 + assert marker not in capsys.readouterr().out + + +def test_command_plan_mode_does_not_execute_tools(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + def forbidden_runner(*_: object, **__: object) -> subprocess.CompletedProcess[str]: + raise AssertionError("command-plan mode executed an external tool") + + exit_code = supply.main( + [ + "--sbom", + "artifacts/test-sbom.json", + "--release-artifact", + "artifacts/test-release.whl", + "--local-image", + IMAGE_ID, + "--command-plan", + ], + runner=forbidden_runner, + ) + + output = json.loads(capsys.readouterr().out) + assert exit_code == 0 + assert output["mode"] == "command-plan" + assert output["commands"] + + +@pytest.mark.parametrize("flag", ["--command-plan", "--dry-run"]) +def test_both_nonexecuting_modes_are_supported(tmp_path: Path, flag: str, capsys: pytest.CaptureFixture[str]) -> None: + exit_code = supply.main( + [ + "--sbom", + "artifacts/test-sbom.json", + "--release-artifact", + "artifacts/test-release.whl", + "--local-image", + IMAGE_ID, + flag, + ], + runner=lambda *_args, **_kwargs: pytest.fail("runner called"), + ) + + assert exit_code == 0 + assert json.loads(capsys.readouterr().out)["mode"] == "command-plan" + + +def test_valid_cyclonedx_document_passes() -> None: + errors = sbom.validate_sbom_document(_valid_sbom(), expected_name=EXPECTED_NAME, expected_version=EXPECTED_VERSION) + + assert errors == () + + +def test_sbom_rejects_invalid_spec_and_name_version_lookalike() -> None: + document = { + "bomFormat": "CycloneDX", + "specVersion": "anything", + "components": [{"name": EXPECTED_NAME, "version": EXPECTED_VERSION}], + } + + errors = sbom.validate_sbom_document(document, expected_name=EXPECTED_NAME, expected_version=EXPECTED_VERSION) + + assert "invalid_spec_version" in errors + assert "project_component_missing" in errors + + +def test_sbom_reads_at_most_bound_plus_one_byte(monkeypatch: pytest.MonkeyPatch) -> None: + payload = json.dumps(_valid_sbom()).encode() + (b" " * 128) + + class GrowingPath: + def stat(self) -> SimpleNamespace: + return SimpleNamespace(st_size=1) + + def read_bytes(self) -> bytes: + return payload + + def open(self, _mode: str) -> io.BytesIO: + return io.BytesIO(payload) + + monkeypatch.setattr(sbom, "MAX_SBOM_BYTES", 64) + + assert sbom.validate_sbom_file( # type: ignore[arg-type] + GrowingPath(), expected_name=EXPECTED_NAME, expected_version=EXPECTED_VERSION + ) == ("file_too_large",) + + +def test_sbom_large_integer_fails_closed_without_raising(tmp_path: Path) -> None: + path = tmp_path / "large-integer.json" + path.write_text('{"value":' + ("9" * 5000) + "}", encoding="utf-8") + + assert sbom.validate_sbom_file(path, expected_name=EXPECTED_NAME, expected_version=EXPECTED_VERSION) == ( + "invalid_json", + ) + + +@pytest.mark.parametrize( + "payload", + [ + b'{"bomFormat":"CycloneDX","specVersion":"1.6","score":NaN}', + b'{"components":[],"components":[{"name":"memorymaster"}]}', + ], +) +def test_sbom_rejects_nonstandard_constants_and_duplicate_keys(tmp_path: Path, payload: bytes) -> None: + path = tmp_path / "ambiguous.json" + path.write_bytes(payload) + + assert sbom.validate_sbom_file(path, expected_name=EXPECTED_NAME, expected_version=EXPECTED_VERSION) == ( + "invalid_json", + ) + + +@pytest.mark.parametrize( + "mutation", + [ + lambda document: document.update(specVersion=[]), + lambda document: document.update({"$schema": []}), + lambda document: document["components"][0].update(type=[]), + ], +) +def test_sbom_unhashable_field_types_fail_closed_without_traceback(mutation: object) -> None: + document = _valid_sbom() + mutation(document) # type: ignore[operator] + + errors = sbom.validate_sbom_document( + document, + expected_name=EXPECTED_NAME, + expected_version=EXPECTED_VERSION, + ) + + assert errors + + +def test_sbom_must_match_exact_release_wheel_hash(tmp_path: Path) -> None: + wheel = _wheel(tmp_path) + sbom_path = tmp_path / "sbom.json" + sbom_path.write_text(json.dumps(_valid_sbom("0" * 64)), encoding="utf-8") + + errors = sbom.validate_sbom_file( + sbom_path, + expected_name=EXPECTED_NAME, + expected_version=EXPECTED_VERSION, + artifact_path=wheel, + ) + + assert "artifact_hash_mismatch" in errors + assert hashlib.sha256(wheel.read_bytes()).hexdigest() not in json.dumps(errors) + + +def test_valid_sbom_is_bound_to_release_wheel(tmp_path: Path) -> None: + wheel = _wheel(tmp_path) + digest = hashlib.sha256(wheel.read_bytes()).hexdigest() + sbom_path = tmp_path / "sbom.json" + sbom_path.write_text(json.dumps(_valid_sbom(digest)), encoding="utf-8") + + assert ( + sbom.validate_sbom_file( + sbom_path, + expected_name=EXPECTED_NAME, + expected_version=EXPECTED_VERSION, + artifact_path=wheel, + ) + == () + ) + + +def test_non_wheel_archive_is_rejected_as_release_artifact(tmp_path: Path) -> None: + wheel = _wheel(tmp_path) + archive = wheel.with_suffix(".zip") + wheel.replace(archive) + digest = hashlib.sha256(archive.read_bytes()).hexdigest() + sbom_path = tmp_path / "sbom.json" + sbom_path.write_text(json.dumps(_valid_sbom(digest)), encoding="utf-8") + + assert sbom.validate_sbom_file( + sbom_path, + expected_name=EXPECTED_NAME, + expected_version=EXPECTED_VERSION, + artifact_path=archive, + ) == ("artifact_invalid",) + + +def test_secret_shaped_image_is_redacted_from_reports_and_command_plan( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + marker = "ghp_" + "7" * 36 + wheel = _prepare_repo(tmp_path) + with pytest.raises(ValueError, match="immutable"): + supply.build_command_plan( + repo_root=tmp_path, + sbom_path=tmp_path / "memorymaster.cdx.json", + local_images=[marker], + release_artifact=wheel, + ) + exit_code = supply.main( + [ + "--release-artifact", + "artifacts/test-release.whl", + "--local-image", + marker, + "--command-plan", + ] + ) + + assert exit_code == 2 + assert marker not in capsys.readouterr().out + + +@pytest.mark.parametrize("entrypoint", [supply.main, sbom.main]) +def test_cli_parse_errors_redact_raw_arguments(entrypoint: object, capsys: pytest.CaptureFixture[str]) -> None: + marker = "ghp_" + "8" * 36 + try: + code = entrypoint(["--unknown", marker]) # type: ignore[operator] + except SystemExit as exc: + code = int(exc.code) + captured = capsys.readouterr() + + assert code == 2 + assert marker not in captured.out + captured.err + assert json.loads(captured.out)["ok"] is False + + +@pytest.mark.parametrize( + ("mutation", "expected_error"), + [ + (lambda doc: doc.pop("bomFormat"), "invalid_bom_format"), + (lambda doc: doc.update(bomFormat="SPDX"), "invalid_bom_format"), + (lambda doc: doc.pop("specVersion"), "invalid_spec_version"), + (lambda doc: doc.update(specVersion=""), "invalid_spec_version"), + (lambda doc: doc.update(components=[]), "components_missing"), + ( + lambda doc: doc["metadata"]["component"].update(name="lookalike"), + "project_component_missing", + ), + ( + lambda doc: doc["metadata"]["component"].update(version="0.0.0"), + "project_component_missing", + ), + ], +) +def test_invalid_sbom_documents_fail_closed(mutation: object, expected_error: str) -> None: + document = _valid_sbom() + mutation(document) # type: ignore[operator] + + errors = sbom.validate_sbom_document(document, expected_name=EXPECTED_NAME, expected_version=EXPECTED_VERSION) + + assert expected_error in errors + + +@pytest.mark.parametrize("document", [None, [], "CycloneDX", 7]) +def test_non_object_sbom_documents_fail_closed(document: object) -> None: + assert sbom.validate_sbom_document(document, expected_name=EXPECTED_NAME, expected_version=EXPECTED_VERSION) == ( + "document_not_object", + ) + + +def test_sbom_cli_does_not_echo_malformed_or_secret_payload(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + sbom_path = tmp_path / "secret-name.json" + sbom_path.write_text('{"token":"do-not-print"', encoding="utf-8") + + exit_code = sbom.main( + [ + "--sbom", + str(sbom_path), + "--expected-name", + EXPECTED_NAME, + "--expected-version", + EXPECTED_VERSION, + ] + ) + + output = capsys.readouterr().out + assert exit_code != 0 + assert "do-not-print" not in output + assert "secret-name" not in output + + +def test_sbom_cli_fails_closed_on_excessive_json_nesting(tmp_path: Path, capsys: pytest.CaptureFixture[str]) -> None: + sbom_path = tmp_path / "nested.json" + sbom_path.write_text('{"nested":' * 2000 + "null" + "}" * 2000, encoding="utf-8") + + exit_code = sbom.main( + [ + "--sbom", + str(sbom_path), + "--expected-name", + EXPECTED_NAME, + "--expected-version", + EXPECTED_VERSION, + ] + ) + + assert exit_code != 0 + assert json.loads(capsys.readouterr().out)["errors"] + + +def test_dockerignore_excludes_sensitive_and_generated_context() -> None: + lines = { + line.strip() + for line in Path(".dockerignore").read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + } + + assert "**" in lines + assert "!memorymaster/**" in lines + assert not any( + value in lines + for value in { + "!.git/**", + "!.env", + "!.mcp.json", + "!*.db", + "!*.log", + "!.tmp-*", + "!.codex/**", + "!.claude/**", + } + ) + + +def test_docker_context_is_an_exact_allowlist_for_dockerfile_inputs() -> None: + lines = [ + line.strip() + for line in Path(".dockerignore").read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + + assert lines == [ + "**", + "!Dockerfile", + "!pyproject.toml", + "!README.md", + "!memorymaster/", + "!memorymaster/**", + ] From 9b3e16c144b248551d0f0959e5b2c6a811c0b9c4 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sun, 12 Jul 2026 17:25:57 -0300 Subject: [PATCH 24/55] fix(security): enforce secure Qdrant transport --- .env.example | 3 + docker-compose.yml | 25 +- helm/memorymaster/templates/deployment.yaml | 3 + helm/memorymaster/values.yaml | 2 +- memorymaster/recall/qdrant_backend.py | 119 +++- memorymaster/recall/qdrant_recall_fallback.py | 21 +- memorymaster/recall/qdrant_transport.py | 142 ++++ memorymaster/recall/verbatim_store.py | 243 ++++--- memorymaster/surfaces/dashboard.py | 18 +- memorymaster/surfaces/setup_detect.py | 16 +- scripts/index_claims_to_qdrant.py | 22 +- tests/test_deployment_contracts.py | 14 +- tests/test_qdrant_backend.py | 98 +-- tests/test_qdrant_transport.py | 645 ++++++++++++++++++ tests/test_verbatim_qdrant_quarantine.py | 2 +- 15 files changed, 1169 insertions(+), 204 deletions(-) create mode 100644 memorymaster/recall/qdrant_transport.py create mode 100644 tests/test_qdrant_transport.py diff --git a/.env.example b/.env.example index 0fc3b8d2..80b6563c 100644 --- a/.env.example +++ b/.env.example @@ -264,6 +264,9 @@ QDRANT_URL=http://localhost:6333 # Required by the hardened Compose/Helm Qdrant profile. Generate and provision # this out of band; never commit the value. # QDRANT_API_KEY= +# QDRANT_TLS_CERT= +# QDRANT_TLS_KEY= +# QDRANT_CA_CERT= # Optional path to a trusted CA certificate for a TLS-enabled Qdrant endpoint. # QDRANT_CA_CERT= diff --git a/docker-compose.yml b/docker-compose.yml index ce3c037d..61418f34 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,11 +8,16 @@ services: - "8765:8765" volumes: - ./data:/data + - type: bind + source: "${QDRANT_CA_CERT:?Set QDRANT_CA_CERT to the Qdrant CA certificate}" + target: /run/secrets/qdrant-ca.pem + read_only: true environment: MEMORYMASTER_DEFAULT_DB: /data/memorymaster.db MEMORYMASTER_WORKSPACE: /data - QDRANT_URL: http://qdrant:6333 + QDRANT_URL: https://qdrant:6333 QDRANT_API_KEY: "${QDRANT_API_KEY:?Set QDRANT_API_KEY before rendering Compose}" + QDRANT_CA_CERT: /run/secrets/qdrant-ca.pem OLLAMA_URL: http://ollama:11434 depends_on: qdrant: @@ -35,16 +40,32 @@ services: - "127.0.0.1:6334:6334" environment: QDRANT__SERVICE__API_KEY: "${QDRANT_API_KEY:?Set QDRANT_API_KEY before rendering Compose}" + QDRANT__SERVICE__ENABLE_TLS: "true" + QDRANT__TLS__CERT: /qdrant/tls/cert.pem + QDRANT__TLS__KEY: /qdrant/tls/key.pem volumes: - qdrant_data:/qdrant/storage + - type: bind + source: "${QDRANT_TLS_CERT:?Set QDRANT_TLS_CERT to the Qdrant server certificate}" + target: /qdrant/tls/cert.pem + read_only: true + - type: bind + source: "${QDRANT_TLS_KEY:?Set QDRANT_TLS_KEY to the Qdrant private key}" + target: /qdrant/tls/key.pem + read_only: true + - type: bind + source: "${QDRANT_CA_CERT:?Set QDRANT_CA_CERT to the Qdrant CA certificate}" + target: /qdrant/tls/ca.pem + read_only: true restart: unless-stopped healthcheck: test: - CMD-SHELL - >- wget --no-verbose --tries=1 --spider + --ca-certificate=/qdrant/tls/ca.pem --header="api-key: $${QDRANT__SERVICE__API_KEY}" - http://127.0.0.1:6333/collections + https://qdrant:6333/collections interval: 15s timeout: 5s retries: 3 diff --git a/helm/memorymaster/templates/deployment.yaml b/helm/memorymaster/templates/deployment.yaml index 7022e955..17ec3732 100644 --- a/helm/memorymaster/templates/deployment.yaml +++ b/helm/memorymaster/templates/deployment.yaml @@ -11,6 +11,9 @@ {{- if hasKey .Values.env "QDRANT_URL" -}} {{- fail "QDRANT_URL is reserved; configure it through .Values.qdrant" -}} {{- end -}} +{{- if not (hasPrefix "https://" .Values.qdrant.url) -}} +{{- fail "qdrant.url must use HTTPS" -}} +{{- end -}} {{- $qdrantApiKeySecret := required "qdrant.apiKeySecret.name is required" .Values.qdrant.apiKeySecret.name -}} apiVersion: apps/v1 kind: Deployment diff --git a/helm/memorymaster/values.yaml b/helm/memorymaster/values.yaml index 5130217c..cfd8b379 100644 --- a/helm/memorymaster/values.yaml +++ b/helm/memorymaster/values.yaml @@ -21,7 +21,7 @@ env: OLLAMA_URL: http://ollama:11434 qdrant: - url: http://qdrant:6333 + url: https://qdrant:6333 apiKeySecret: # Required: name of an existing Secret. The chart never creates the key. name: "" diff --git a/memorymaster/recall/qdrant_backend.py b/memorymaster/recall/qdrant_backend.py index cbfb07e8..4f55da1c 100644 --- a/memorymaster/recall/qdrant_backend.py +++ b/memorymaster/recall/qdrant_backend.py @@ -24,6 +24,7 @@ from memorymaster.core.models import Claim from memorymaster.core.security import scan_persisted_value +from memorymaster.recall.qdrant_transport import QdrantTransportConfig logger = logging.getLogger(__name__) @@ -37,6 +38,24 @@ RETRY_BASE_DELAY = 0.5 +def _create_http_clients( + transport: QdrantTransportConfig, +) -> tuple[httpx.Client, httpx.Client]: + try: + qdrant_client = httpx.Client(timeout=30.0, **transport.httpx_kwargs()) + except Exception: + raise RuntimeError("Qdrant client initialization failed") from None + try: + ollama_client = httpx.Client(timeout=30.0) + except Exception: + try: + qdrant_client.close() + except Exception: + pass + raise RuntimeError("Ollama client initialization failed") from None + return qdrant_client, ollama_client + + class QdrantBackend: """Thin wrapper around the Qdrant REST API for claim indexing.""" @@ -51,7 +70,9 @@ def __init__( self.ollama_url = (ollama_url or os.environ.get("OLLAMA_URL") or DEFAULT_OLLAMA_URL).rstrip("/") self.collection = collection or os.environ.get("QDRANT_COLLECTION") or DEFAULT_COLLECTION self.embed_model = embed_model or os.environ.get("OLLAMA_EMBED_MODEL") or DEFAULT_EMBED_MODEL - self._client = httpx.Client(timeout=30.0) + transport = QdrantTransportConfig.from_env() + transport.validate_url(self.qdrant_url) + self._qdrant_client, self._ollama_client = _create_http_clients(transport) # ------------------------------------------------------------------ # Collection management @@ -59,19 +80,22 @@ def __init__( def ensure_collection(self) -> None: """Create the Qdrant collection if it does not exist.""" - url = f"{self.qdrant_url}/collections/{self.collection}" - resp = self._client.get(url) - if resp.status_code == 200: - logger.debug("Qdrant collection '%s' already exists", self.collection) - return - body = { - "vectors": { - "size": EMBEDDING_DIMS, - "distance": "Cosine", + try: + url = f"{self.qdrant_url}/collections/{self.collection}" + resp = self._qdrant_client.get(url) + if resp.status_code == 200: + logger.debug("Qdrant collection '%s' already exists", self.collection) + return + body = { + "vectors": { + "size": EMBEDDING_DIMS, + "distance": "Cosine", + } } - } - resp = self._client.put(url, json=body) - resp.raise_for_status() + resp = self._qdrant_client.put(url, json=body) + resp.raise_for_status() + except Exception: + raise RuntimeError("Qdrant collection request failed") from None logger.info("Created Qdrant collection '%s' (%d dims, Cosine)", self.collection, EMBEDDING_DIMS) # ------------------------------------------------------------------ @@ -82,7 +106,7 @@ def _embed(self, text: str) -> list[float] | None: """Get a 4096-dim embedding from Ollama with retry on transient failures.""" for attempt in range(1 + MAX_RETRIES): try: - resp = self._client.post( + resp = self._ollama_client.post( f"{self.ollama_url}/api/embed", json={"model": self.embed_model, "input": [text]}, timeout=OLLAMA_TIMEOUT, @@ -98,13 +122,20 @@ def _embed(self, text: str) -> list[float] | None: EMBEDDING_DIMS, ) return None # dim mismatch is not retryable - except Exception as exc: + except Exception: if attempt < MAX_RETRIES: delay = RETRY_BASE_DELAY * (2 ** attempt) - logger.info("Ollama embed attempt %d failed (%s), retrying in %.1fs", attempt + 1, exc, delay) + logger.info( + "Ollama embed attempt %d failed; retrying in %.1fs", + attempt + 1, + delay, + ) time.sleep(delay) else: - logger.warning("Ollama embed failed after %d attempts: %s", 1 + MAX_RETRIES, exc) + logger.warning( + "Ollama embed failed after %d attempts", + 1 + MAX_RETRIES, + ) return None # ------------------------------------------------------------------ @@ -183,19 +214,28 @@ def upsert_claim(self, claim: Claim, source: str = "memorymaster") -> bool: } for attempt in range(1 + MAX_RETRIES): try: - resp = self._client.put( + resp = self._qdrant_client.put( f"{self.qdrant_url}/collections/{self.collection}/points", json=body, ) resp.raise_for_status() return True - except Exception as exc: + except Exception: if attempt < MAX_RETRIES: delay = RETRY_BASE_DELAY * (2 ** attempt) - logger.info("Qdrant upsert attempt %d for claim %d failed (%s), retrying in %.1fs", attempt + 1, claim.id, exc, delay) + logger.info( + "Qdrant upsert attempt %d for claim %d failed; retrying in %.1fs", + attempt + 1, + claim.id, + delay, + ) time.sleep(delay) else: - logger.warning("Qdrant upsert failed for claim %d after %d attempts: %s", claim.id, 1 + MAX_RETRIES, exc) + logger.warning( + "Qdrant upsert failed for claim %d after %d attempts", + claim.id, + 1 + MAX_RETRIES, + ) return False def delete_claim(self, claim_id: int) -> bool: @@ -203,14 +243,14 @@ def delete_claim(self, claim_id: int) -> bool: point_id = self._point_id(claim_id) body = {"points": [point_id]} try: - resp = self._client.post( + resp = self._qdrant_client.post( f"{self.qdrant_url}/collections/{self.collection}/points/delete", json=body, ) resp.raise_for_status() return True - except Exception as exc: - logger.warning("Qdrant delete failed for claim %d: %s", claim_id, exc) + except Exception: + logger.warning("Qdrant delete failed for claim %d", claim_id) return False def count_points(self) -> int | None: @@ -220,14 +260,14 @@ def count_points(self) -> int | None: metric (P1 spec §2.7). """ try: - resp = self._client.post( + resp = self._qdrant_client.post( f"{self.qdrant_url}/collections/{self.collection}/points/count", json={"exact": True}, ) resp.raise_for_status() return int((resp.json().get("result") or {}).get("count", 0)) - except Exception as exc: - logger.warning("Qdrant count failed: %s", exc) + except Exception: + logger.warning("Qdrant count failed") return None def list_point_claim_ids(self, *, batch_size: int = 1000) -> list[int] | None: @@ -248,7 +288,7 @@ def list_point_claim_ids(self, *, batch_size: int = 1000) -> list[int] | None: } if offset is not None: body["offset"] = offset - resp = self._client.post( + resp = self._qdrant_client.post( f"{self.qdrant_url}/collections/{self.collection}/points/scroll", json=body, ) @@ -261,8 +301,8 @@ def list_point_claim_ids(self, *, batch_size: int = 1000) -> list[int] | None: offset = result.get("next_page_offset") if offset is None: return ids - except Exception as exc: - logger.warning("Qdrant scroll failed: %s", exc) + except Exception: + logger.warning("Qdrant scroll failed") return None def search( @@ -286,19 +326,26 @@ def _batch_upsert(self, points: list[dict[str, Any]]) -> bool: body = {"points": points} for attempt in range(1 + MAX_RETRIES): try: - resp = self._client.put( + resp = self._qdrant_client.put( f"{self.qdrant_url}/collections/{self.collection}/points", json=body, ) resp.raise_for_status() return True - except Exception as exc: + except Exception: if attempt < MAX_RETRIES: delay = RETRY_BASE_DELAY * (2 ** attempt) - logger.info("Qdrant batch upsert attempt %d failed (%s), retrying in %.1fs", attempt + 1, exc, delay) + logger.info( + "Qdrant batch upsert attempt %d failed; retrying in %.1fs", + attempt + 1, + delay, + ) time.sleep(delay) else: - logger.warning("Qdrant batch upsert failed after %d attempts: %s", 1 + MAX_RETRIES, exc) + logger.warning( + "Qdrant batch upsert failed after %d attempts", + 1 + MAX_RETRIES, + ) return False def sync_all(self, store, *, batch_size: int = 50) -> dict[str, int]: @@ -363,4 +410,6 @@ def sync_all(self, store, *, batch_size: int = 50) -> dict[str, int]: return stats def close(self) -> None: - self._client.close() + self._qdrant_client.close() + if self._ollama_client is not self._qdrant_client: + self._ollama_client.close() diff --git a/memorymaster/recall/qdrant_recall_fallback.py b/memorymaster/recall/qdrant_recall_fallback.py index a674873d..3a90537a 100644 --- a/memorymaster/recall/qdrant_recall_fallback.py +++ b/memorymaster/recall/qdrant_recall_fallback.py @@ -37,6 +37,8 @@ from dataclasses import dataclass from typing import Any +from memorymaster.recall.qdrant_transport import QdrantTransportConfig + logger = logging.getLogger(__name__) DEFAULT_COLLECTION = "memorymaster-claims" @@ -189,6 +191,8 @@ def _get_client(): if _client_failed: return None try: + transport = QdrantTransportConfig.from_env() + transport.validate_url(url) from qdrant_client import QdrantClient except ImportError: logger.warning( @@ -197,16 +201,21 @@ def _get_client(): ) _client_failed = True return None - except Exception as exc: - logger.warning("vector fallback disabled: import error: %s", exc) + except Exception: + logger.warning( + "vector fallback disabled: invalid Qdrant transport configuration." + ) _client_failed = True return None try: - _client = QdrantClient(url=url, timeout=5.0) - except Exception as exc: + _client = QdrantClient( + url=url, + timeout=5.0, + **transport.qdrant_client_kwargs(), + ) + except Exception: logger.warning( - "vector fallback disabled: could not create client for %s: %s", - url, exc, + "vector fallback disabled: could not create Qdrant client." ) _client_failed = True return None diff --git a/memorymaster/recall/qdrant_transport.py b/memorymaster/recall/qdrant_transport.py new file mode 100644 index 00000000..661b84ec --- /dev/null +++ b/memorymaster/recall/qdrant_transport.py @@ -0,0 +1,142 @@ +"""Shared authenticated and TLS-verified Qdrant transport configuration.""" + +from __future__ import annotations + +import ipaddress +import os +import ssl +import urllib.parse +import urllib.request +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Mapping + + +_API_KEY_ENV = "QDRANT_API_KEY" +_CA_CERT_ENV = "QDRANT_CA_CERT" + + +class _NoRedirectHandler(urllib.request.HTTPRedirectHandler): + """Turn redirects into HTTP errors so credentials never change origin.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): # noqa: ANN001, ANN201 + return None + + +def _is_literal_loopback(hostname: str) -> bool: + if hostname.lower() == "localhost": + return True + try: + return ipaddress.ip_address(hostname).is_loopback + except ValueError: + return False + + +def _load_ca_path(raw_path: str | None) -> Path | None: + if not raw_path or not raw_path.strip(): + return None + try: + path = Path(raw_path.strip()).expanduser().resolve(strict=True) + if not path.is_file(): + raise ValueError + ssl.create_default_context(cafile=str(path)) + except (OSError, RuntimeError, ValueError, ssl.SSLError): + raise ValueError("QDRANT_CA_CERT must reference a readable CA certificate file") from None + return path + + +@dataclass(frozen=True) +class QdrantTransportConfig: + """Immutable Qdrant credential and trust configuration.""" + + api_key: str | None = field(default=None, repr=False) + ca_cert: Path | None = None + + @classmethod + def from_env( + cls, + environ: Mapping[str, str] | None = None, + ) -> QdrantTransportConfig: + source = os.environ if environ is None else environ + api_key = source.get(_API_KEY_ENV) or None + return cls(api_key=api_key, ca_cert=_load_ca_path(source.get(_CA_CERT_ENV))) + + def ssl_context(self) -> ssl.SSLContext: + """Return a verified context using the custom CA or system trust.""" + try: + cafile = str(self.ca_cert) if self.ca_cert is not None else None + return ssl.create_default_context(cafile=cafile) + except (OSError, ValueError, ssl.SSLError): + raise ValueError("QDRANT_CA_CERT could not be loaded") from None + + def headers(self, base: Mapping[str, str] | None = None) -> dict[str, str]: + """Return fresh request headers with the Qdrant key scoped to them.""" + result = {str(name): str(value) for name, value in (base or {}).items() if str(name).lower() != "api-key"} + if self.api_key: + result["api-key"] = self.api_key + return result + + def validate_url(self, url: str) -> str: + """Reject malformed or remotely reachable plaintext Qdrant URLs.""" + try: + parsed = urllib.parse.urlsplit(url) + hostname = parsed.hostname + _ = parsed.port + except (TypeError, ValueError): + raise ValueError("Qdrant URL must be a valid HTTP(S) URL") from None + if parsed.scheme not in {"http", "https"} or not hostname: + raise ValueError("Qdrant URL must be a valid HTTP(S) URL") + if parsed.username is not None or parsed.password is not None: + raise ValueError("Qdrant URL must not contain credentials") + if parsed.scheme == "http" and (self.ca_cert is not None or not _is_literal_loopback(hostname)): + raise ValueError("Qdrant URL must use HTTPS except for explicit loopback development endpoints") + return url + + def httpx_kwargs(self) -> dict[str, object]: + """Build kwargs for an HTTPX client dedicated to Qdrant.""" + kwargs: dict[str, object] = {} + if self.api_key: + kwargs["headers"] = self.headers() + if self.ca_cert is not None: + kwargs["verify"] = self.ssl_context() + return kwargs + + def qdrant_client_kwargs(self) -> dict[str, object]: + """Build optional kwargs for ``qdrant_client.QdrantClient``.""" + kwargs: dict[str, object] = {} + if self.api_key: + kwargs["api_key"] = self.api_key + if self.ca_cert is not None: + kwargs["verify"] = self.ssl_context() + return kwargs + + def urlopen_kwargs(self) -> dict[str, ssl.SSLContext]: + """Build urllib kwargs without disabling system verification.""" + if self.ca_cert is None: + return {} + return {"context": self.ssl_context()} + + def open(self, request: urllib.request.Request, *, timeout: float) -> Any: + """Open one Qdrant request with verified TLS and redirects disabled.""" + parsed = urllib.parse.urlsplit(self.validate_url(request.full_url)) + handlers: list[urllib.request.BaseHandler] = [_NoRedirectHandler()] + if parsed.scheme == "https": + handlers.append(urllib.request.HTTPSHandler(context=self.ssl_context())) + return urllib.request.build_opener(*handlers).open(request, timeout=timeout) + + def request( + self, + url: str, + *, + data: bytes | None = None, + headers: Mapping[str, str] | None = None, + method: str | None = None, + ) -> urllib.request.Request: + """Build a Qdrant-only urllib request with scoped credentials.""" + self.validate_url(url) + return urllib.request.Request( + url, + data=data, + headers=self.headers(headers), + method=method, + ) diff --git a/memorymaster/recall/verbatim_store.py b/memorymaster/recall/verbatim_store.py index df2a284b..3c5bf90b 100644 --- a/memorymaster/recall/verbatim_store.py +++ b/memorymaster/recall/verbatim_store.py @@ -8,6 +8,7 @@ - Vector/hybrid requests currently downgrade to FTS5 - Qdrant remains available only for index synchronization during quarantine """ + from __future__ import annotations import hashlib @@ -24,12 +25,12 @@ # Credential detection delegated to the canonical filter in memorymaster.core.security. from memorymaster.core import spool from memorymaster.core.security import scan_persisted_value +from memorymaster.recall.qdrant_transport import QdrantTransportConfig from memorymaster.stores._storage_shared import open_conn logger = logging.getLogger(__name__) - def _contains_sensitive(text: str) -> bool: return bool(scan_persisted_value(text)) @@ -90,6 +91,7 @@ def _verbatim_row_has_sensitive_field(row: sqlite3.Row | dict) -> bool: created_at=str(_row_value(row, "created_at") or ""), ) + # Vector search is opt-in: an unset QDRANT_URL means "vector disabled", exactly # like a missing OPENAI_API_KEY. NEVER hardcode a routable private LAN IP here — # a home-lab RFC1918 default previously shipped to PyPI, violating the @@ -268,8 +270,7 @@ def _extract_role_content(entry: dict) -> tuple[str, str]: content = entry.get("content", "") if isinstance(content, list): content = " ".join( - part.get("text", "") for part in content - if isinstance(part, dict) and part.get("type") == "text" + part.get("text", "") for part in content if isinstance(part, dict) and part.get("type") == "text" ) return role, content if isinstance(content, str) else "" @@ -491,117 +492,157 @@ def _search_fts(db_path: str, query: str, scope: str | None, limit: int) -> list def _search_vector(query: str, scope: str | None, limit: int) -> list[dict]: """Reject raw Qdrant payload reads until governed rehydration exists.""" del query, scope, limit - raise PermissionError( - "Verbatim Qdrant retrieval is quarantined pending authoritative rehydration." + raise PermissionError("Verbatim Qdrant retrieval is quarantined pending authoritative rehydration.") + + +def _safe_rows_for_sync( + conn: sqlite3.Connection, + batch_size: int, +) -> tuple[list[sqlite3.Row], int]: + rows = conn.execute( + "SELECT * FROM verbatim_memories WHERE embedding_synced = 0 LIMIT ?", + (batch_size,), + ).fetchall() + safe_rows: list[sqlite3.Row] = [] + unsafe_ids: list[int] = [] + for row in rows: + if _verbatim_row_has_sensitive_field(row): + unsafe_ids.append(int(row["id"])) + else: + safe_rows.append(row) + if unsafe_ids: + placeholders = ",".join("?" for _ in unsafe_ids) + conn.execute( + f"UPDATE verbatim_memories SET embedding_synced = -1 WHERE id IN ({placeholders})", + unsafe_ids, + ) + conn.commit() + return safe_rows, len(unsafe_ids) + + +def _ensure_verbatim_collection(transport: QdrantTransportConfig) -> bool: + collection_url = f"{QDRANT_URL}/collections/{QDRANT_COLLECTION}" + try: + request = transport.request(collection_url, method="GET") + transport.open(request, timeout=5) + return True + except Exception: + payload = {"vectors": {"size": EMBED_DIM, "distance": "Cosine"}} + request = transport.request( + collection_url, + data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json"}, + method="PUT", + ) + try: + transport.open(request, timeout=60) + return True + except Exception: + return False + + +def _openai_embeddings(api_key: str, texts: list[str]) -> list[list[float]] | None: + payload = {"model": "text-embedding-3-small", "input": texts} + request = urllib.request.Request( + "https://api.openai.com/v1/embeddings", + data=json.dumps(payload).encode(), + headers={ + "Content-Type": "application/json", + "Authorization": f"Bearer {api_key}", + }, + method="POST", ) + try: + with urllib.request.urlopen(request, timeout=30) as response: + result = json.loads(response.read().decode()) + return [item["embedding"] for item in result["data"]] + except Exception: + return None -def sync_to_qdrant(db_path: str, batch_size: int = 50) -> dict[str, int]: +def _verbatim_points( + rows: list[sqlite3.Row], + embeddings: list[list[float]], +) -> list[dict]: + return [ + { + "id": row["id"], + "vector": embedding, + "payload": { + "content": row["content"][:2000], + "content_hash": hashlib.sha256(row["content"].encode()).hexdigest(), + "scope": row["scope"], + "session_id": row["session_id"], + "role": row["role"], + }, + } + for row, embedding in zip(rows, embeddings) + ] + + +def _upsert_verbatim_points( + transport: QdrantTransportConfig, + points: list[dict], +) -> bool: + request = transport.request( + f"{QDRANT_URL}/collections/{QDRANT_COLLECTION}/points", + data=json.dumps({"points": points}).encode(), + headers={"Content-Type": "application/json"}, + method="PUT", + ) + try: + transport.open(request, timeout=30) + return True + except Exception: + return False + + +def _mark_verbatim_synced(conn: sqlite3.Connection, rows: list[sqlite3.Row]) -> None: + ids = [row["id"] for row in rows] + placeholders = ",".join("?" for _ in ids) + conn.execute( + f"UPDATE verbatim_memories SET embedding_synced = 1 WHERE id IN ({placeholders})", + ids, + ) + conn.commit() + + +def sync_to_qdrant(db_path: str, batch_size: int = 50) -> dict[str, int | str]: """Sync verbatim rows to the Qdrant index; read retrieval is quarantined.""" api_key = os.environ.get("OPENAI_API_KEY", "") if not api_key: return {"synced": 0, "error": "no OPENAI_API_KEY"} if not QDRANT_URL: return {"synced": 0, "error": "no QDRANT_URL"} + try: + transport = QdrantTransportConfig.from_env() + except (OSError, RuntimeError, ValueError): + return {"synced": 0, "error": "invalid Qdrant transport configuration"} - # closing() guarantees the connection is released on every exit path, - # including the initial SELECT raising or the final UPDATE/commit raising. with closing(_connect(db_path)) as conn: - rows = conn.execute( - "SELECT * FROM verbatim_memories WHERE embedding_synced = 0 LIMIT ?", - (batch_size,), - ).fetchall() - - if not rows: - return {"synced": 0} - - safe_rows: list[sqlite3.Row] = [] - unsafe_ids: list[int] = [] - for row in rows: - if _verbatim_row_has_sensitive_field(row): - unsafe_ids.append(int(row["id"])) - else: - safe_rows.append(row) - excluded_sensitive = len(unsafe_ids) - if unsafe_ids: - placeholders = ",".join("?" for _ in unsafe_ids) - conn.execute( - f"UPDATE verbatim_memories SET embedding_synced = -1 WHERE id IN ({placeholders})", - unsafe_ids, - ) - conn.commit() + safe_rows, excluded = _safe_rows_for_sync(conn, batch_size) if not safe_rows: - return {"synced": 0, "excluded_sensitive": excluded_sensitive} - - # Ensure collection exists - try: - req = urllib.request.Request(f"{QDRANT_URL}/collections/{QDRANT_COLLECTION}") - urllib.request.urlopen(req, timeout=5) - except Exception: - payload = {"vectors": {"size": EMBED_DIM, "distance": "Cosine"}} - req = urllib.request.Request( - f"{QDRANT_URL}/collections/{QDRANT_COLLECTION}", - data=json.dumps(payload).encode(), - headers={"Content-Type": "application/json"}, - method="PUT", - ) - try: - urllib.request.urlopen(req, timeout=60) - except Exception as e: - return {"synced": 0, "error": str(e)} - - # Embed in batches - texts = [r["content"][:2000] for r in safe_rows] - try: - embed_url = "https://api.openai.com/v1/embeddings" - payload = {"model": "text-embedding-3-small", "input": texts} - req = urllib.request.Request( - embed_url, - data=json.dumps(payload).encode(), - headers={"Content-Type": "application/json", "Authorization": f"Bearer {api_key}"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=30) as resp: - result = json.loads(resp.read().decode()) - embeddings = [d["embedding"] for d in result["data"]] - except Exception as e: - return {"synced": 0, "error": str(e)} + result: dict[str, int | str] = {"synced": 0} + if excluded: + result["excluded_sensitive"] = excluded + return result + if not _ensure_verbatim_collection(transport): + return {"synced": 0, "error": "Qdrant collection unavailable"} + embeddings = _openai_embeddings( + api_key, + [row["content"][:2000] for row in safe_rows], + ) + if embeddings is None: + return {"synced": 0, "error": "embedding request failed"} if len(embeddings) != len(safe_rows): return {"synced": 0, "error": "embedding response cardinality mismatch"} - - # Upsert to Qdrant - points = [] - for row, emb in zip(safe_rows, embeddings): - points.append({ - "id": row["id"], - "vector": emb, - "payload": { - "content": row["content"][:2000], - "content_hash": hashlib.sha256(row["content"].encode()).hexdigest(), - "scope": row["scope"], - "session_id": row["session_id"], - "role": row["role"], - }, - }) - - try: - req = urllib.request.Request( - f"{QDRANT_URL}/collections/{QDRANT_COLLECTION}/points", - data=json.dumps({"points": points}).encode(), - headers={"Content-Type": "application/json"}, - method="PUT", - ) - urllib.request.urlopen(req, timeout=30) - except Exception as e: - return {"synced": 0, "error": str(e)} - - # Mark as synced - ids = [r["id"] for r in safe_rows] - placeholders = ",".join("?" for _ in ids) - conn.execute(f"UPDATE verbatim_memories SET embedding_synced = 1 WHERE id IN ({placeholders})", ids) - conn.commit() - + if not _upsert_verbatim_points( + transport, + _verbatim_points(safe_rows, embeddings), + ): + return {"synced": 0, "error": "Qdrant upsert failed"} + _mark_verbatim_synced(conn, safe_rows) result = {"synced": len(safe_rows)} - if excluded_sensitive: - result["excluded_sensitive"] = excluded_sensitive + if excluded: + result["excluded_sensitive"] = excluded return result diff --git a/memorymaster/surfaces/dashboard.py b/memorymaster/surfaces/dashboard.py index 561ebd84..7fbffa71 100644 --- a/memorymaster/surfaces/dashboard.py +++ b/memorymaster/surfaces/dashboard.py @@ -24,6 +24,7 @@ from memorymaster.govern.review import build_review_queue, queue_to_dicts from memorymaster.core.security import is_sensitive_claim from memorymaster.core.service import MemoryService +from memorymaster.recall.qdrant_transport import QdrantTransportConfig import contextlib @@ -595,27 +596,30 @@ def _check_dashboard_db(service: Any) -> dict[str, Any]: def _qdrant_request_headers() -> dict[str, str]: - api_key = os.environ.get("QDRANT_API_KEY") - return {"api-key": api_key} if api_key else {} + return QdrantTransportConfig.from_env().headers() def _check_qdrant(qdrant_url: str | None) -> dict[str, Any]: if not qdrant_url: return {"status": "skipped", "reason": "QDRANT_URL not set"} + try: + transport = QdrantTransportConfig.from_env() + transport.validate_url(qdrant_url) + except (OSError, RuntimeError, ValueError): + return {"status": "fail", "error": "invalid Qdrant transport configuration"} base_url = qdrant_url.rstrip("/") - headers = _qdrant_request_headers() last_error = "" for path in ("/healthz", "/collections"): - request = urllib.request.Request(f"{base_url}{path}", headers=headers, method="GET") + request = transport.request(f"{base_url}{path}", method="GET") try: - with urllib.request.urlopen(request, timeout=0.5) as response: + with transport.open(request, timeout=0.5) as response: if 200 <= int(response.status) < 300: return {"status": "ok", "endpoint": path} last_error = f"HTTP {response.status} from {path}" except urllib.error.HTTPError as exc: last_error = f"HTTP {exc.code} from {path}" - except Exception as exc: - last_error = str(exc) + except Exception: + last_error = f"Qdrant request failed for {path}" return {"status": "fail", "error": last_error or "Qdrant probe failed"} diff --git a/memorymaster/surfaces/setup_detect.py b/memorymaster/surfaces/setup_detect.py index a50344bb..307a2751 100644 --- a/memorymaster/surfaces/setup_detect.py +++ b/memorymaster/surfaces/setup_detect.py @@ -21,6 +21,8 @@ from pathlib import Path from typing import Optional +from memorymaster.recall.qdrant_transport import QdrantTransportConfig + # --------------------------------------------------------------------------- # DTO @@ -53,6 +55,10 @@ class Detected: _PROBE_TIMEOUT = 5 # seconds +class _QdrantProbeUrl(str): + """String-compatible marker that scopes Qdrant transport credentials.""" + + def _run(args: list[str]) -> Optional[str]: """Run a subprocess (shell=False, timeout<=5s). @@ -76,7 +82,13 @@ def _run(args: list[str]) -> Optional[str]: def _http_get(url: str) -> Optional[bytes]: """HTTP GET with timeout. Returns body bytes on 2xx, None otherwise.""" try: - req = urllib.request.urlopen(url, timeout=_PROBE_TIMEOUT) # noqa: S310 + target: str | urllib.request.Request = url + if isinstance(url, _QdrantProbeUrl): + transport = QdrantTransportConfig.from_env() + target = transport.request(str(url), method="GET") + req = transport.open(target, timeout=_PROBE_TIMEOUT) + else: + req = urllib.request.urlopen(target, timeout=_PROBE_TIMEOUT) # noqa: S310 return req.read() except Exception: # noqa: BLE001 return None @@ -127,7 +139,7 @@ def _probe_ollama() -> tuple[bool, tuple[str, ...]]: def _probe_qdrant() -> bool: qdrant_url = os.environ.get("QDRANT_URL", "http://localhost:6333") healthz_url = qdrant_url.rstrip("/") + "/healthz" - return _http_get(healthz_url) is not None + return _http_get(_QdrantProbeUrl(healthz_url)) is not None def _probe_obsidian_vault(cwd: Path) -> Optional[str]: diff --git a/scripts/index_claims_to_qdrant.py b/scripts/index_claims_to_qdrant.py index 7fe530d8..bffec80f 100644 --- a/scripts/index_claims_to_qdrant.py +++ b/scripts/index_claims_to_qdrant.py @@ -35,6 +35,8 @@ if str(REPO) not in sys.path: sys.path.insert(0, str(REPO)) +from memorymaster.recall.qdrant_transport import QdrantTransportConfig # noqa: E402 + # Ensure UTF-8 stdout on Windows so we can print claim text safely. if hasattr(sys.stdout, "buffer"): sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace") @@ -135,6 +137,8 @@ def _load_embedder(model_name: str): def _load_qdrant(url: str): + transport = QdrantTransportConfig.from_env() + transport.validate_url(url) try: from qdrant_client import QdrantClient except ImportError as exc: @@ -145,7 +149,14 @@ def _load_qdrant(url: str): " pip install qdrant-client" ) from exc logger.info("Connecting to Qdrant at %s", url) - return QdrantClient(url=url, timeout=30.0) + try: + return QdrantClient( + url=url, + timeout=30.0, + **transport.qdrant_client_kwargs(), + ) + except Exception: + raise SystemExit("Could not create the Qdrant client") from None def _ensure_collection(client, collection: str, dims: int) -> None: @@ -188,7 +199,10 @@ def index_claims( embedder = _load_embedder(embed_model) dims = int(embedder.get_sentence_embedding_dimension()) client = _load_qdrant(qdrant_url) - _ensure_collection(client, collection, dims) + try: + _ensure_collection(client, collection, dims) + except Exception: + raise SystemExit("Could not prepare the Qdrant collection") from None total = _count_claims(db_path) if limit is not None: @@ -227,8 +241,8 @@ def _flush() -> None: try: _upsert_batch(client, collection, points) stats["indexed"] += len(points) - except Exception as exc: - logger.warning("qdrant upsert of %d points failed: %s", len(points), exc) + except Exception: + logger.warning("qdrant upsert of %d points failed", len(points)) stats["errors"] += len(points) done += len(points) texts = [] diff --git a/tests/test_deployment_contracts.py b/tests/test_deployment_contracts.py index 1bac13f8..136b66df 100644 --- a/tests/test_deployment_contracts.py +++ b/tests/test_deployment_contracts.py @@ -46,10 +46,17 @@ def test_auxiliary_compose_ports_are_not_public(): def test_auxiliary_compose_requires_authenticated_qdrant(): compose = _read("docker-compose.yml") + assert "QDRANT_URL: https://qdrant:6333" in compose assert re.search(r"QDRANT_API_KEY:\s*[\"']?\$\{QDRANT_API_KEY:\?", compose) assert re.search(r"QDRANT__SERVICE__API_KEY:\s*[\"']?\$\{QDRANT_API_KEY:\?", compose) + assert 'QDRANT__SERVICE__ENABLE_TLS: "true"' in compose + assert "QDRANT__TLS__CERT: /qdrant/tls/cert.pem" in compose + assert "QDRANT__TLS__KEY: /qdrant/tls/key.pem" in compose + for variable in ("QDRANT_TLS_CERT", "QDRANT_TLS_KEY", "QDRANT_CA_CERT"): + assert f"${{{variable}:?" in compose assert '--header="api-key: $${QDRANT__SERVICE__API_KEY}"' in compose - assert "http://127.0.0.1:6333/collections" in compose + assert "https://qdrant:6333/collections" in compose + assert "--ca-certificate=/qdrant/tls/ca.pem" in compose @pytest.mark.xfail( @@ -120,6 +127,8 @@ def test_helm_requires_digest_and_existing_qdrant_secrets(): assert ".Values.qdrant.caSecret.name" in deployment assert "QDRANT_CA_CERT" in deployment assert "readOnly: true" in deployment + assert re.search(r"^\s*url:\s*https://", values, re.MULTILINE) + assert 'fail "qdrant.url must use HTTPS"' in deployment def test_postgres_smoke_requires_operator_supplied_dsn(): @@ -136,6 +145,9 @@ def test_environment_example_documents_required_deployment_inputs(): assert "# MEMORYMASTER_POSTGRES_PASSWORD=" in example assert "# QDRANT_API_KEY=" in example + assert "# QDRANT_TLS_CERT=" in example + assert "# QDRANT_TLS_KEY=" in example + assert "# QDRANT_CA_CERT=" in example assert re.search(r"^# QDRANT_IMAGE_DIGEST=sha256:", example, re.MULTILINE) assert re.search(r"^# OLLAMA_IMAGE_DIGEST=sha256:", example, re.MULTILINE) diff --git a/tests/test_qdrant_backend.py b/tests/test_qdrant_backend.py index e224ff29..a87ede19 100644 --- a/tests/test_qdrant_backend.py +++ b/tests/test_qdrant_backend.py @@ -75,24 +75,25 @@ class TestEmbedFailure: def test_upsert_returns_false_on_embed_failure(self): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") - backend._client = MagicMock() + backend._ollama_client = MagicMock() # Simulate embed failure - backend._client.post.side_effect = Exception("connection refused") + backend._ollama_client.post.side_effect = Exception("connection refused") claim = _fake_claim() assert backend.upsert_claim(claim) is False def test_search_is_quarantined_before_embed_failure(self): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") - backend._client = MagicMock() - backend._client.post.side_effect = Exception("connection refused") + backend._qdrant_client = MagicMock() + backend._ollama_client = MagicMock() with pytest.raises(PermissionError, match="quarantined"): backend.search("test query") - backend._client.post.assert_not_called() + backend._qdrant_client.post.assert_not_called() + backend._ollama_client.post.assert_not_called() def test_delete_returns_false_on_failure(self): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") - backend._client = MagicMock() - backend._client.post.side_effect = Exception("connection refused") + backend._qdrant_client = MagicMock() + backend._qdrant_client.post.side_effect = Exception("connection refused") assert backend.delete_claim(1) is False @@ -101,7 +102,7 @@ class TestEmbedSuccess: def _make_backend(self): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") - backend._client = MagicMock() + backend._ollama_client = MagicMock() return backend def _mock_embed_response(self, client_mock, dims=EMBEDDING_DIMS): @@ -114,14 +115,14 @@ def _mock_embed_response(self, client_mock, dims=EMBEDDING_DIMS): def test_embed_returns_vector(self): backend = self._make_backend() - self._mock_embed_response(backend._client) + self._mock_embed_response(backend._ollama_client) vec = backend._embed("test text") assert vec is not None assert len(vec) == EMBEDDING_DIMS def test_embed_wrong_dims_returns_none(self): backend = self._make_backend() - self._mock_embed_response(backend._client, dims=768) # wrong dims + self._mock_embed_response(backend._ollama_client, dims=768) # wrong dims vec = backend._embed("test text") assert vec is None @@ -131,7 +132,7 @@ def test_embed_empty_embeddings_returns_none(self): resp.status_code = 200 resp.raise_for_status = MagicMock() resp.json.return_value = {"embeddings": []} - backend._client.post.return_value = resp + backend._ollama_client.post.return_value = resp vec = backend._embed("test text") assert vec is None @@ -141,8 +142,8 @@ class TestUpsertSuccess: def _make_backend_with_embed(self): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") - backend._client = MagicMock() - # First post = embed, second put = qdrant upsert + backend._qdrant_client = MagicMock() + backend._ollama_client = MagicMock() embed_resp = MagicMock() embed_resp.status_code = 200 embed_resp.raise_for_status = MagicMock() @@ -152,8 +153,8 @@ def _make_backend_with_embed(self): upsert_resp.status_code = 200 upsert_resp.raise_for_status = MagicMock() - backend._client.post.return_value = embed_resp - backend._client.put.return_value = upsert_resp + backend._ollama_client.post.return_value = embed_resp + backend._qdrant_client.put.return_value = upsert_resp return backend def test_upsert_returns_true(self): @@ -165,13 +166,13 @@ def test_upsert_calls_qdrant_put(self): backend = self._make_backend_with_embed() claim = _fake_claim() backend.upsert_claim(claim) - backend._client.put.assert_called_once() - url = backend._client.put.call_args[0][0] + backend._qdrant_client.put.assert_called_once() + url = backend._qdrant_client.put.call_args[0][0] assert "/collections/agent-memories/points" in url def test_upsert_qdrant_failure_returns_false(self): backend = self._make_backend_with_embed() - backend._client.put.side_effect = Exception("qdrant down") + backend._qdrant_client.put.side_effect = Exception("qdrant down") claim = _fake_claim() assert backend.upsert_claim(claim) is False @@ -179,29 +180,30 @@ def test_upsert_qdrant_failure_returns_false(self): class TestDeleteSuccess: def test_delete_returns_true(self): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") - backend._client = MagicMock() + backend._qdrant_client = MagicMock() resp = MagicMock() resp.status_code = 200 resp.raise_for_status = MagicMock() - backend._client.post.return_value = resp + backend._qdrant_client.post.return_value = resp assert backend.delete_claim(1) is True def test_delete_calls_correct_url(self): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") - backend._client = MagicMock() + backend._qdrant_client = MagicMock() resp = MagicMock() resp.status_code = 200 resp.raise_for_status = MagicMock() - backend._client.post.return_value = resp + backend._qdrant_client.post.return_value = resp backend.delete_claim(42) - url = backend._client.post.call_args[0][0] + url = backend._qdrant_client.post.call_args[0][0] assert "/points/delete" in url class TestSearchSuccess: def _make_backend_with_search(self, results=None): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") - backend._client = MagicMock() + backend._qdrant_client = MagicMock() + backend._ollama_client = MagicMock() embed_resp = MagicMock() embed_resp.status_code = 200 @@ -213,8 +215,8 @@ def _make_backend_with_search(self, results=None): search_resp.raise_for_status = MagicMock() search_resp.json.return_value = {"result": results or []} - # post is used for both embed and search - backend._client.post.side_effect = [embed_resp, search_resp] + backend._ollama_client.post.return_value = embed_resp + backend._qdrant_client.post.return_value = search_resp return backend def test_search_does_not_return_raw_results_during_quarantine(self): @@ -222,7 +224,8 @@ def test_search_does_not_return_raw_results_during_quarantine(self): backend = self._make_backend_with_search(hits) with pytest.raises(PermissionError, match="quarantined"): backend.search("test query") - backend._client.post.assert_not_called() + backend._qdrant_client.post.assert_not_called() + backend._ollama_client.post.assert_not_called() def test_search_empty_results_still_reports_quarantine(self): backend = self._make_backend_with_search([]) @@ -233,65 +236,70 @@ def test_search_with_filters_remains_quarantined(self): backend = self._make_backend_with_search([]) with pytest.raises(PermissionError, match="quarantined"): backend.search("test", states=["confirmed"], min_confidence=0.5) - backend._client.post.assert_not_called() + backend._qdrant_client.post.assert_not_called() + backend._ollama_client.post.assert_not_called() def test_search_qdrant_failure_is_never_reached(self): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") - backend._client = MagicMock() + backend._qdrant_client = MagicMock() + backend._ollama_client = MagicMock() embed_resp = MagicMock() embed_resp.status_code = 200 embed_resp.raise_for_status = MagicMock() embed_resp.json.return_value = {"embeddings": [[0.1] * EMBEDDING_DIMS]} - backend._client.post.side_effect = [embed_resp, Exception("qdrant down")] + backend._ollama_client.post.return_value = embed_resp + backend._qdrant_client.post.side_effect = Exception("qdrant down") with pytest.raises(PermissionError, match="quarantined"): backend.search("test") - backend._client.post.assert_not_called() + backend._qdrant_client.post.assert_not_called() + backend._ollama_client.post.assert_not_called() class TestEnsureCollection: def test_existing_collection_skips_create(self): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") - backend._client = MagicMock() + backend._qdrant_client = MagicMock() resp = MagicMock() resp.status_code = 200 - backend._client.get.return_value = resp + backend._qdrant_client.get.return_value = resp backend.ensure_collection() - backend._client.put.assert_not_called() + backend._qdrant_client.put.assert_not_called() def test_missing_collection_creates(self): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") - backend._client = MagicMock() + backend._qdrant_client = MagicMock() get_resp = MagicMock() get_resp.status_code = 404 - backend._client.get.return_value = get_resp + backend._qdrant_client.get.return_value = get_resp put_resp = MagicMock() put_resp.status_code = 200 put_resp.raise_for_status = MagicMock() - backend._client.put.return_value = put_resp + backend._qdrant_client.put.return_value = put_resp backend.ensure_collection() - backend._client.put.assert_called_once() + backend._qdrant_client.put.assert_called_once() class TestSyncAll: def test_sync_all_counts(self): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") - backend._client = MagicMock() + backend._qdrant_client = MagicMock() + backend._ollama_client = MagicMock() # Mock ensure_collection get_resp = MagicMock() get_resp.status_code = 200 - backend._client.get.return_value = get_resp + backend._qdrant_client.get.return_value = get_resp # Mock embed + upsert embed_resp = MagicMock() embed_resp.status_code = 200 embed_resp.raise_for_status = MagicMock() embed_resp.json.return_value = {"embeddings": [[0.1] * EMBEDDING_DIMS]} - backend._client.post.return_value = embed_resp + backend._ollama_client.post.return_value = embed_resp upsert_resp = MagicMock() upsert_resp.status_code = 200 upsert_resp.raise_for_status = MagicMock() - backend._client.put.return_value = upsert_resp + backend._qdrant_client.put.return_value = upsert_resp # Mock store store = MagicMock() @@ -307,9 +315,11 @@ def test_sync_all_counts(self): class TestClose: def test_close_calls_client_close(self): backend = QdrantBackend(qdrant_url="http://localhost:1", ollama_url="http://localhost:2") - backend._client = MagicMock() + backend._qdrant_client = MagicMock() + backend._ollama_client = MagicMock() backend.close() - backend._client.close.assert_called_once() + backend._qdrant_client.close.assert_called_once() + backend._ollama_client.close.assert_called_once() class TestServiceQdrantIntegration: diff --git a/tests/test_qdrant_transport.py b/tests/test_qdrant_transport.py new file mode 100644 index 00000000..fec15b7b --- /dev/null +++ b/tests/test_qdrant_transport.py @@ -0,0 +1,645 @@ +"""RED contracts for authenticated, verified Qdrant transport. + +Every external call is replaced with a deterministic fake. These tests are +intentionally added before the R1.5 transport implementation. +""" + +from __future__ import annotations + +import json +import importlib +import io +import logging +import sqlite3 +import ssl +import sys +import urllib.request +from pathlib import Path +from types import ModuleType +from typing import Any + +import certifi +import pytest + +from memorymaster.recall import qdrant_backend, qdrant_recall_fallback, verbatim_store +from memorymaster.recall.qdrant_transport import QdrantTransportConfig +from memorymaster.surfaces import dashboard, setup_detect + + +QDRANT_KEY = "qdrant-test-secret-never-log-or-forward" +OPENAI_KEY = "openai-test-secret" + + +class _Response: + def __init__(self, payload: dict[str, Any] | None = None, *, status: int = 200) -> None: + self._payload = payload or {} + self.status = status + self.status_code = status + + def __enter__(self) -> _Response: + return self + + def __exit__(self, exc_type, exc, traceback) -> bool: + return False + + def read(self) -> bytes: + return json.dumps(self._payload).encode() + + def json(self) -> dict[str, Any]: + return self._payload + + def raise_for_status(self) -> None: + return None + + +class _ObservedHttpxClient: + def __init__(self, kwargs: dict[str, Any], *, failure_secret: str = "") -> None: + self.constructor_kwargs = kwargs + self.calls: list[tuple[str, str, dict[str, Any]]] = [] + self.failure_secret = failure_secret + + def _call(self, method: str, url: str, kwargs: dict[str, Any]) -> _Response: + self.calls.append((method, url, kwargs)) + if self.failure_secret and "qdrant.invalid" in url: + raise RuntimeError(f"synthetic failure: {self.failure_secret}") + if url.endswith("/api/embed"): + dims = qdrant_backend.EMBEDDING_DIMS + return _Response({"embeddings": [[0.0] * dims]}) + return _Response() + + def get(self, url: str, **kwargs: Any) -> _Response: + return self._call("GET", url, kwargs) + + def put(self, url: str, **kwargs: Any) -> _Response: + return self._call("PUT", url, kwargs) + + def post(self, url: str, **kwargs: Any) -> _Response: + return self._call("POST", url, kwargs) + + def close(self) -> None: + return None + + +def _install_httpx_clients( + monkeypatch: pytest.MonkeyPatch, + *, + failure_secret: str = "", +) -> list[_ObservedHttpxClient]: + clients: list[_ObservedHttpxClient] = [] + + def factory(**kwargs: Any) -> _ObservedHttpxClient: + client = _ObservedHttpxClient(kwargs, failure_secret=failure_secret) + clients.append(client) + return client + + monkeypatch.setattr(qdrant_backend.httpx, "Client", factory) + return clients + + +def _clients_for_url( + clients: list[_ObservedHttpxClient], + needle: str, +) -> list[_ObservedHttpxClient]: + return [client for client in clients if any(needle in call[1] for call in client.calls)] + + +def _effective_headers( + client: _ObservedHttpxClient, + call: tuple[str, str, dict[str, Any]], +) -> dict[str, str]: + headers = dict(client.constructor_kwargs.get("headers") or {}) + headers.update(call[2].get("headers") or {}) + return {str(key).lower(): str(value) for key, value in headers.items()} + + +def _request_headers(request: Any) -> dict[str, str]: + if isinstance(request, str): + return {} + return {key.lower(): value for key, value in request.header_items()} + + +def _request_url(request: Any) -> str: + return request if isinstance(request, str) else request.full_url + + +def _assert_verified_context(context: Any) -> None: + assert isinstance(context, ssl.SSLContext) + assert context.verify_mode == ssl.CERT_REQUIRED + assert context.check_hostname is True + + +def _assert_custom_ca(verify: Any, ca_path: Path) -> None: + if isinstance(verify, ssl.SSLContext): + _assert_verified_context(verify) + return + assert Path(verify).resolve() == ca_path.resolve() + + +def _install_qdrant_open(monkeypatch: pytest.MonkeyPatch, callback: Any) -> None: + def open_request( + transport: QdrantTransportConfig, + request: urllib.request.Request, + *, + timeout: float, + ) -> Any: + context = transport.ssl_context() if request.full_url.startswith("https://") else None + return callback(request, timeout, context=context) + + monkeypatch.setattr(QdrantTransportConfig, "open", open_request) + + +def _install_fake_qdrant_module( + monkeypatch: pytest.MonkeyPatch, + factory: Any, +) -> None: + module = ModuleType("qdrant_client") + module.QdrantClient = factory # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "qdrant_client", module) + + +def _load_indexer_without_rewrapping_pytest_stdout( + monkeypatch: pytest.MonkeyPatch, +) -> Any: + monkeypatch.setattr(sys, "stdout", io.StringIO()) + return importlib.import_module("scripts.index_claims_to_qdrant") + + +@pytest.fixture(autouse=True) +def _clean_qdrant_environment(monkeypatch: pytest.MonkeyPatch): + for name in ("QDRANT_API_KEY", "QDRANT_CA_CERT", "MEMORYMASTER_QDRANT_URL"): + monkeypatch.delenv(name, raising=False) + qdrant_recall_fallback.reset_singletons_for_tests() + yield + qdrant_recall_fallback.reset_singletons_for_tests() + + +def test_transport_rejects_remote_http_and_limits_loopback_exception() -> None: + transport = QdrantTransportConfig(api_key=QDRANT_KEY) + + for url in ("http://qdrant:6333", "http://192.0.2.10:6333"): + with pytest.raises(ValueError, match="HTTPS"): + transport.request(url, method="GET") + + assert transport.request("http://localhost:6333", method="GET").full_url.startswith("http://localhost") + assert transport.request("http://127.0.0.1:6333", method="GET").full_url.startswith("http://127.0.0.1") + + transport_with_ca = QdrantTransportConfig(api_key=QDRANT_KEY, ca_cert=Path(certifi.where())) + with pytest.raises(ValueError, match="HTTPS"): + transport_with_ca.request("http://localhost:6333", method="GET") + + +def test_transport_opener_refuses_redirect_before_key_can_be_forwarded( + monkeypatch: pytest.MonkeyPatch, +) -> None: + handlers: list[object] = [] + forwarded: list[urllib.request.Request] = [] + + class _Opener: + def open(self, request: urllib.request.Request, *, timeout: float) -> _Response: + redirect_handler = next( + handler for handler in handlers if isinstance(handler, urllib.request.HTTPRedirectHandler) + ) + redirected = redirect_handler.redirect_request( + request, + None, + 302, + "Found", + {}, + "https://attacker.invalid/collect", + ) + if redirected is not None: + forwarded.append(redirected) + return _Response() + + def build_opener(*configured: object) -> _Opener: + handlers.extend(configured) + return _Opener() + + monkeypatch.setattr(urllib.request, "build_opener", build_opener) + transport = QdrantTransportConfig(api_key=QDRANT_KEY) + request = transport.request("https://qdrant.invalid/collections", method="GET") + + transport.open(request, timeout=0.1) + + assert forwarded == [] + + +@pytest.mark.parametrize("kind", ["missing", "directory", "invalid"]) +def test_backend_rejects_invalid_ca_before_constructing_clients( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + kind: str, +) -> None: + ca_path = tmp_path / "configured-ca.pem" + if kind == "directory": + ca_path.mkdir() + elif kind == "invalid": + ca_path.write_text("not a CA certificate", encoding="utf-8") + monkeypatch.setenv("QDRANT_CA_CERT", str(ca_path)) + clients = _install_httpx_clients(monkeypatch) + + with pytest.raises((OSError, RuntimeError, ValueError), match="QDRANT_CA_CERT"): + qdrant_backend.QdrantBackend(qdrant_url="https://qdrant.invalid") + + assert clients == [] + + +def test_ca_resolution_error_is_fixed_and_secret_free( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("QDRANT_CA_CERT", "configured-ca.pem") + + def fail_resolve(self: Path, *, strict: bool = False) -> Path: + raise RuntimeError(f"synthetic path failure: {QDRANT_KEY}") + + monkeypatch.setattr(Path, "resolve", fail_resolve) + + with pytest.raises(ValueError) as caught: + QdrantTransportConfig.from_env() + + assert QDRANT_KEY not in str(caught.value) + + +def test_backend_uses_separate_clients_and_scopes_api_key( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("QDRANT_API_KEY", QDRANT_KEY) + clients = _install_httpx_clients(monkeypatch) + backend = qdrant_backend.QdrantBackend( + qdrant_url="https://qdrant.invalid", + ollama_url="http://ollama.invalid", + ) + + backend.ensure_collection() + assert backend._embed("safe text") is not None + + qdrant_clients = _clients_for_url(clients, "qdrant.invalid") + ollama_clients = _clients_for_url(clients, "ollama.invalid") + assert len(qdrant_clients) == len(ollama_clients) == 1 + assert qdrant_clients[0] is not ollama_clients[0] + assert all( + _effective_headers(qdrant_clients[0], call).get("api-key") == QDRANT_KEY for call in qdrant_clients[0].calls + ) + assert QDRANT_KEY not in repr(ollama_clients[0].constructor_kwargs) + assert QDRANT_KEY not in repr(ollama_clients[0].calls) + + +def test_backend_keeps_system_verification_without_custom_ca( + monkeypatch: pytest.MonkeyPatch, +) -> None: + clients = _install_httpx_clients(monkeypatch) + backend = qdrant_backend.QdrantBackend( + qdrant_url="https://qdrant.invalid", + ollama_url="http://ollama.invalid", + ) + backend.ensure_collection() + + qdrant_client = _clients_for_url(clients, "qdrant.invalid")[0] + assert qdrant_client.constructor_kwargs.get("verify", True) is not False + + +def test_backend_custom_ca_reaches_only_qdrant_client( + monkeypatch: pytest.MonkeyPatch, +) -> None: + ca_path = Path(certifi.where()) + monkeypatch.setenv("QDRANT_CA_CERT", str(ca_path)) + clients = _install_httpx_clients(monkeypatch) + backend = qdrant_backend.QdrantBackend( + qdrant_url="https://qdrant.invalid", + ollama_url="https://ollama.invalid", + ) + backend.ensure_collection() + backend._embed("safe text") + + qdrant_client = _clients_for_url(clients, "qdrant.invalid")[0] + ollama_client = _clients_for_url(clients, "ollama.invalid")[0] + _assert_custom_ca(qdrant_client.constructor_kwargs.get("verify"), ca_path) + assert "verify" not in ollama_client.constructor_kwargs + + +def test_backend_failure_logs_redact_qdrant_key( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setenv("QDRANT_API_KEY", QDRANT_KEY) + _install_httpx_clients(monkeypatch, failure_secret=QDRANT_KEY) + caplog.set_level(logging.WARNING, logger=qdrant_backend.__name__) + backend = qdrant_backend.QdrantBackend( + qdrant_url="https://qdrant.invalid", + ollama_url="http://ollama.invalid", + ) + + assert backend.delete_claim(7) is False + assert QDRANT_KEY not in caplog.text + + +def test_backend_constructor_error_redacts_qdrant_key( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("QDRANT_API_KEY", QDRANT_KEY) + + def fail_client(**kwargs: Any) -> object: + raise RuntimeError(f"synthetic failure: {QDRANT_KEY}") + + monkeypatch.setattr(qdrant_backend.httpx, "Client", fail_client) + + with pytest.raises(RuntimeError) as caught: + qdrant_backend.QdrantBackend(qdrant_url="https://qdrant.invalid") + + assert QDRANT_KEY not in str(caught.value) + + +def test_backend_collection_error_redacts_qdrant_key( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("QDRANT_API_KEY", QDRANT_KEY) + _install_httpx_clients(monkeypatch, failure_secret=QDRANT_KEY) + backend = qdrant_backend.QdrantBackend(qdrant_url="https://qdrant.invalid") + + with pytest.raises(RuntimeError) as caught: + backend.ensure_collection() + + assert QDRANT_KEY not in str(caught.value) + + +@pytest.mark.parametrize( + ("loader", "url_env"), + [ + ("fallback", "MEMORYMASTER_QDRANT_URL"), + ("indexer", None), + ], +) +def test_qdrant_client_constructors_receive_api_key_and_ca( + monkeypatch: pytest.MonkeyPatch, + loader: str, + url_env: str | None, +) -> None: + ca_path = Path(certifi.where()) + monkeypatch.setenv("QDRANT_API_KEY", QDRANT_KEY) + monkeypatch.setenv("QDRANT_CA_CERT", str(ca_path)) + url = "https://qdrant.invalid" + if url_env: + monkeypatch.setenv(url_env, url) + observed: list[dict[str, Any]] = [] + + def factory(**kwargs: Any) -> object: + observed.append(kwargs) + return object() + + _install_fake_qdrant_module(monkeypatch, factory) + if loader == "fallback": + assert qdrant_recall_fallback._get_client() is not None + else: + indexer = _load_indexer_without_rewrapping_pytest_stdout(monkeypatch) + indexer._load_qdrant(url) + + assert observed[0]["api_key"] == QDRANT_KEY + _assert_custom_ca(observed[0]["verify"], ca_path) + + +def test_fallback_constructor_error_does_not_log_qdrant_key( + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + monkeypatch.setenv("MEMORYMASTER_QDRANT_URL", "https://qdrant.invalid") + monkeypatch.setenv("QDRANT_API_KEY", QDRANT_KEY) + + def factory(**kwargs: Any) -> object: + raise RuntimeError(f"synthetic failure: {QDRANT_KEY}") + + _install_fake_qdrant_module(monkeypatch, factory) + caplog.set_level(logging.WARNING, logger=qdrant_recall_fallback.__name__) + + assert qdrant_recall_fallback._get_client() is None + assert QDRANT_KEY not in caplog.text + + +def test_indexer_upsert_error_does_not_log_qdrant_key( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + indexer = _load_indexer_without_rewrapping_pytest_stdout(monkeypatch) + models = ModuleType("qdrant_client.models") + models.PointStruct = lambda **kwargs: kwargs # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, "qdrant_client.models", models) + + class _Vector(list): + def tolist(self) -> list[float]: + return list(self) + + class _Embedder: + def get_sentence_embedding_dimension(self) -> int: + return 2 + + def encode(self, texts, **kwargs): + return [_Vector([0.0, 0.0]) for _ in texts] + + monkeypatch.setattr(indexer, "_load_embedder", lambda model: _Embedder()) + monkeypatch.setattr(indexer, "_load_qdrant", lambda url: object()) + monkeypatch.setattr(indexer, "_ensure_collection", lambda *args: None) + monkeypatch.setattr(indexer, "_count_claims", lambda path: 1) + monkeypatch.setattr( + indexer, + "_iter_claims", + lambda path: iter([(1, "project:test", "subject", "safe text", "confirmed", 0.9)]), + ) + + def fail_upsert(*args: Any) -> None: + raise RuntimeError(f"synthetic failure: {QDRANT_KEY}") + + monkeypatch.setattr(indexer, "_upsert_batch", fail_upsert) + caplog.set_level(logging.WARNING, logger="index_claims_to_qdrant") + + result = indexer.index_claims( + tmp_path / "unused.db", + "https://qdrant.invalid", + "collection", + "model", + batch_size=1, + ) + + assert result["errors"] == 1 + assert QDRANT_KEY not in caplog.text + + +def test_indexer_collection_error_redacts_qdrant_key( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + indexer = _load_indexer_without_rewrapping_pytest_stdout(monkeypatch) + + class _Embedder: + def get_sentence_embedding_dimension(self) -> int: + return 2 + + monkeypatch.setattr(indexer, "_load_embedder", lambda model: _Embedder()) + monkeypatch.setattr(indexer, "_load_qdrant", lambda url: object()) + + def fail_collection(*args: Any) -> None: + raise RuntimeError(f"synthetic failure: {QDRANT_KEY}") + + monkeypatch.setattr(indexer, "_ensure_collection", fail_collection) + + with pytest.raises(SystemExit) as caught: + indexer.index_claims( + tmp_path / "unused.db", + "https://qdrant.invalid", + "collection", + "model", + ) + + assert QDRANT_KEY not in str(caught.value) + + +def test_dashboard_qdrant_probe_uses_key_and_ca_context( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("QDRANT_API_KEY", QDRANT_KEY) + monkeypatch.setenv("QDRANT_CA_CERT", certifi.where()) + observed: list[tuple[Any, Any]] = [] + + def fake_urlopen(request, timeout, context=None): + observed.append((request, context)) + return _Response(status=200) + + _install_qdrant_open(monkeypatch, fake_urlopen) + + assert dashboard._check_qdrant("https://qdrant.invalid")["status"] == "ok" + request, context = observed[0] + assert _request_headers(request).get("api-key") == QDRANT_KEY + _assert_verified_context(context) + + +def test_setup_qdrant_probe_does_not_forward_key_to_ollama( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("QDRANT_URL", "https://qdrant.invalid") + monkeypatch.setenv("OLLAMA_URL", "https://ollama.invalid") + monkeypatch.setenv("QDRANT_API_KEY", QDRANT_KEY) + monkeypatch.setenv("QDRANT_CA_CERT", certifi.where()) + observed: list[tuple[Any, Any]] = [] + + def fake_urlopen(request, timeout, context=None): + observed.append((request, context)) + return _Response({"models": []}) + + _install_qdrant_open(monkeypatch, fake_urlopen) + monkeypatch.setattr(setup_detect.urllib.request, "urlopen", fake_urlopen) + monkeypatch.setattr(setup_detect, "_run", lambda args: None) + + assert setup_detect._probe_qdrant() is True + assert setup_detect._probe_ollama()[0] is True + qdrant_request, qdrant_context = next(item for item in observed if "qdrant.invalid" in _request_url(item[0])) + ollama_request, _ = next(item for item in observed if "ollama.invalid" in _request_url(item[0])) + assert _request_headers(qdrant_request).get("api-key") == QDRANT_KEY + _assert_verified_context(qdrant_context) + assert QDRANT_KEY not in repr(_request_headers(ollama_request)) + + +def _create_verbatim_db(db_path: Path) -> None: + conn = sqlite3.connect(db_path) + conn.execute( + """CREATE TABLE verbatim_memories ( + id INTEGER PRIMARY KEY, + session_id TEXT, + role TEXT, + content TEXT, + scope TEXT, + timestamp TEXT, + source_agent TEXT, + embedding_synced INTEGER DEFAULT 0 + )""" + ) + conn.execute( + """INSERT INTO verbatim_memories + (id, session_id, role, content, scope, timestamp, source_agent) + VALUES (1, 'session', 'user', 'safe text', 'project:test', + '2026-07-11T00:00:00Z', 'pytest')""" + ) + conn.commit() + conn.close() + + +def test_verbatim_qdrant_requests_use_key_and_ca_but_openai_does_not( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "verbatim.db" + _create_verbatim_db(db_path) + monkeypatch.setenv("OPENAI_API_KEY", OPENAI_KEY) + monkeypatch.setenv("QDRANT_API_KEY", QDRANT_KEY) + monkeypatch.setenv("QDRANT_CA_CERT", certifi.where()) + monkeypatch.setattr(verbatim_store, "QDRANT_URL", "https://qdrant.invalid") + observed: list[tuple[Any, Any]] = [] + + def fake_urlopen(request, timeout, context=None): + observed.append((request, context)) + if "api.openai.com" in _request_url(request): + return _Response({"data": [{"embedding": [0.0] * verbatim_store.EMBED_DIM}]}) + return _Response() + + _install_qdrant_open(monkeypatch, fake_urlopen) + monkeypatch.setattr(verbatim_store.urllib.request, "urlopen", fake_urlopen) + + assert verbatim_store.sync_to_qdrant(str(db_path)) == {"synced": 1} + qdrant_calls = [item for item in observed if "qdrant.invalid" in _request_url(item[0])] + openai_call = next(item for item in observed if "api.openai.com" in _request_url(item[0])) + assert qdrant_calls + for request, context in qdrant_calls: + assert _request_headers(request).get("api-key") == QDRANT_KEY + _assert_verified_context(context) + assert QDRANT_KEY not in repr(_request_headers(openai_call[0])) + + +def test_dashboard_and_verbatim_errors_redact_qdrant_key( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "verbatim.db" + _create_verbatim_db(db_path) + monkeypatch.setenv("OPENAI_API_KEY", OPENAI_KEY) + monkeypatch.setenv("QDRANT_API_KEY", QDRANT_KEY) + monkeypatch.setattr(verbatim_store, "QDRANT_URL", "https://qdrant.invalid") + + def fail_urlopen(request, timeout, context=None): + raise RuntimeError(f"synthetic failure: {QDRANT_KEY}") + + _install_qdrant_open(monkeypatch, fail_urlopen) + dashboard_result = dashboard._check_qdrant("https://qdrant.invalid") + verbatim_result = verbatim_store.sync_to_qdrant(str(db_path)) + + assert QDRANT_KEY not in repr(dashboard_result) + assert QDRANT_KEY not in repr(verbatim_result) + + +@pytest.mark.parametrize("surface", ["dashboard", "setup", "verbatim"]) +def test_invalid_ca_stops_urllib_qdrant_network( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + surface: str, +) -> None: + missing_ca = tmp_path / "missing-ca.pem" + monkeypatch.setenv("QDRANT_CA_CERT", str(missing_ca)) + monkeypatch.setenv("QDRANT_API_KEY", QDRANT_KEY) + network_calls: list[str] = [] + + def fail_if_called(request, timeout, context=None): + network_calls.append(_request_url(request)) + return _Response() + + _install_qdrant_open(monkeypatch, fail_if_called) + monkeypatch.setattr(dashboard.urllib.request, "urlopen", fail_if_called) + if surface == "dashboard": + assert dashboard._check_qdrant("https://qdrant.invalid")["status"] == "fail" + elif surface == "setup": + monkeypatch.setenv("QDRANT_URL", "https://qdrant.invalid") + assert setup_detect._probe_qdrant() is False + else: + db_path = tmp_path / "verbatim.db" + _create_verbatim_db(db_path) + monkeypatch.setenv("OPENAI_API_KEY", OPENAI_KEY) + monkeypatch.setattr(verbatim_store, "QDRANT_URL", "https://qdrant.invalid") + assert verbatim_store.sync_to_qdrant(str(db_path))["synced"] == 0 + + assert network_calls == [] diff --git a/tests/test_verbatim_qdrant_quarantine.py b/tests/test_verbatim_qdrant_quarantine.py index aa514be2..35edefe5 100644 --- a/tests/test_verbatim_qdrant_quarantine.py +++ b/tests/test_verbatim_qdrant_quarantine.py @@ -89,7 +89,7 @@ def test_claim_qdrant_backend_search_fails_before_embedding( monkeypatch: pytest.MonkeyPatch, ) -> None: backend = QdrantBackend( - qdrant_url="http://qdrant.invalid", + qdrant_url="https://qdrant.invalid", ollama_url="http://ollama.invalid", ) monkeypatch.setattr( From 8d80abb43ecdcbf5cf83163133cff3ed1a7ca37a Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sun, 12 Jul 2026 17:50:07 -0300 Subject: [PATCH 25/55] fix(integrity): close Phase 1 verification regressions --- memorymaster/govern/jobs/sensitivity_inventory.py | 5 ++--- tests/test_auxiliary_persistence_sensitivity.py | 13 +++++++++++-- tests/test_external_persistence_sensitivity.py | 9 +++++---- 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/memorymaster/govern/jobs/sensitivity_inventory.py b/memorymaster/govern/jobs/sensitivity_inventory.py index 42caa15c..a1253ec5 100644 --- a/memorymaster/govern/jobs/sensitivity_inventory.py +++ b/memorymaster/govern/jobs/sensitivity_inventory.py @@ -23,10 +23,10 @@ _reasons, _record, _sql_tokens, - _sqlite_uri, _surface, _table_columns, ) +from memorymaster.stores._storage_shared import connect_ro _MAX_CHUNK_BYTES = 1024 * 1024 _MAX_FILE_BYTES = 64 * 1024 * 1024 @@ -151,8 +151,7 @@ def _sqlite_inventory(path: Path) -> dict[str, object]: return {"reason": "sqlite_not_available", "status": "BLOCKED"} conn: sqlite3.Connection | None = None try: - conn = sqlite3.connect(_sqlite_uri(path), uri=True) - conn.execute("PRAGMA query_only=ON") + conn = connect_ro(path) conn.execute("BEGIN") master = conn.execute( "SELECT type, name, sql FROM sqlite_master WHERE name NOT LIKE 'sqlite_%' ORDER BY type, name" diff --git a/tests/test_auxiliary_persistence_sensitivity.py b/tests/test_auxiliary_persistence_sensitivity.py index c84b51f0..da4c66a7 100644 --- a/tests/test_auxiliary_persistence_sensitivity.py +++ b/tests/test_auxiliary_persistence_sensitivity.py @@ -415,6 +415,13 @@ def read(self) -> bytes: return json.dumps(self.payload).encode() +def _install_qdrant_open(monkeypatch: pytest.MonkeyPatch, callback) -> None: + def open_request(_transport, request, *, timeout): + return callback(request, timeout) + + monkeypatch.setattr(verbatim_store.QdrantTransportConfig, "open", open_request) + + def test_verbatim_qdrant_sync_filters_legacy_secret_before_embedding_and_upsert( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, @@ -433,7 +440,8 @@ def fake_urlopen(req, timeout): return _Response({}) monkeypatch.setenv("OPENAI_API_KEY", "test-key") - monkeypatch.setattr(verbatim_store, "QDRANT_URL", "http://qdrant.invalid") + monkeypatch.setattr(verbatim_store, "QDRANT_URL", "https://qdrant.invalid") + _install_qdrant_open(monkeypatch, fake_urlopen) monkeypatch.setattr(verbatim_store.urllib.request, "urlopen", fake_urlopen) result = verbatim_store.sync_to_qdrant(str(db_path)) @@ -483,7 +491,8 @@ def fake_urlopen(req, timeout): pytest.fail("cardinality mismatch reached Qdrant upsert") monkeypatch.setenv("OPENAI_API_KEY", "test-key") - monkeypatch.setattr(verbatim_store, "QDRANT_URL", "http://qdrant.invalid") + monkeypatch.setattr(verbatim_store, "QDRANT_URL", "https://qdrant.invalid") + _install_qdrant_open(monkeypatch, fake_urlopen) monkeypatch.setattr(verbatim_store.urllib.request, "urlopen", fake_urlopen) result = verbatim_store.sync_to_qdrant(str(db_path)) diff --git a/tests/test_external_persistence_sensitivity.py b/tests/test_external_persistence_sensitivity.py index 403f6695..f3562978 100644 --- a/tests/test_external_persistence_sensitivity.py +++ b/tests/test_external_persistence_sensitivity.py @@ -191,8 +191,9 @@ def put(self, _url: str, *, json: dict) -> SimpleNamespace: def _unsafe_qdrant_backend() -> QdrantBackend: backend = QdrantBackend() - backend._client.close() - backend._client = _FakeClient() + backend._qdrant_client.close() + backend._ollama_client.close() + backend._qdrant_client = _FakeClient() backend._embed = lambda _text: [0.0] * EMBEDDING_DIMS return backend @@ -202,7 +203,7 @@ def test_qdrant_rejects_unsafe_claim_before_embed_or_upsert(field: str) -> None: backend = _unsafe_qdrant_backend() assert backend.upsert_claim(_claim(2, **{field: _ENCODED})) is False - assert backend._client.puts == [] + assert backend._qdrant_client.puts == [] @pytest.mark.parametrize("field", ["source", "locator", "excerpt"]) @@ -210,4 +211,4 @@ def test_qdrant_rejects_unsafe_citation_before_embed_or_upsert(field: str) -> No backend = _unsafe_qdrant_backend() assert backend.upsert_claim(_claim(2, citations=[_citation(field)])) is False - assert backend._client.puts == [] + assert backend._qdrant_client.puts == [] From 132e5d0074f9c627a3ed1d9ef29d71815635881e Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sun, 12 Jul 2026 17:53:11 -0300 Subject: [PATCH 26/55] test(security): align Qdrant ML transport fixture --- tests/test_verbatim_store_qdrant.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/test_verbatim_store_qdrant.py b/tests/test_verbatim_store_qdrant.py index edb44c8d..31760201 100644 --- a/tests/test_verbatim_store_qdrant.py +++ b/tests/test_verbatim_store_qdrant.py @@ -96,7 +96,12 @@ def fake_urlopen(req, timeout): return _Response({}) monkeypatch.setenv("OPENAI_API_KEY", "test-key") - monkeypatch.setattr(verbatim_store, "QDRANT_URL", "http://test-qdrant:6333") + monkeypatch.setattr(verbatim_store, "QDRANT_URL", "https://test-qdrant:6333") + monkeypatch.setattr( + verbatim_store.QdrantTransportConfig, + "open", + lambda _transport, request, *, timeout: fake_urlopen(request, timeout), + ) monkeypatch.setattr(verbatim_store.urllib.request, "urlopen", fake_urlopen) assert verbatim_store.sync_to_qdrant(str(db_path)) == {"synced": 1} From 95c2b3a204d871da2588095dfbec29e70c506f5c Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sun, 12 Jul 2026 18:21:40 -0300 Subject: [PATCH 27/55] docs(audit): record Phase 1 budget convergence --- .planning/REMEDIATION-EXECUTION-V3-BUDGET.md | 214 ++++++++++++++++++ ...EMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md | 5 +- .planning/audit-remediation-ledger.md | 22 +- .../audit-delta.md | 119 ++++++++++ external-actions-required.md | 8 +- 5 files changed, 360 insertions(+), 8 deletions(-) create mode 100644 .planning/REMEDIATION-EXECUTION-V3-BUDGET.md create mode 100644 .planning/audits/2026-07-12-phase1-budget-delta/audit-delta.md diff --git a/.planning/REMEDIATION-EXECUTION-V3-BUDGET.md b/.planning/REMEDIATION-EXECUTION-V3-BUDGET.md new file mode 100644 index 00000000..504c4571 --- /dev/null +++ b/.planning/REMEDIATION-EXECUTION-V3-BUDGET.md @@ -0,0 +1,214 @@ +# MemoryMaster Phase 1 — Budget Execution V3 + +**Created:** 2026-07-12 + +**Supersedes for execution:** `REMEDIATION-EXECUTION-V2.md` + +**Does not replace audit evidence:** the original remediation plan and ledger remain historical sources of truth. + +## Objective + +Finish Phase 1 security convergence without continuing the open-ended +fix/test/review loop. Preserve completed work, close the two remaining Wave B +packages, run the expensive gates once, produce a targeted Phase 1 audit delta, +and stop before Phases 2–4. + +## Authoritative inputs + +- Scheduler: `.planning/REMEDIATION-EXECUTION-V3-BUDGET.md` +- Audit roadmap: `.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md` +- Finding ledger: `.planning/audit-remediation-ledger.md` +- Baseline audit: `.planning/audits/2026-07-10-baseline/` + +When V3 conflicts with V2 on execution frequency, review loops, or completion +criteria, V3 controls. V3 does not erase findings or imply that Phases 2–4 are +complete. + +## Resume checkpoint + +Integration worktree: +`G:\tmp\memorymaster-remediation-20260710` + +Completed Phase 1 packages include R1.1–R1.3 and R1.4 repository work. Recent +commits include: + +- `a3e3824` — bridge persistence transport +- `702b59d` — legacy sensitivity inventory +- `a858419` — private authenticated deployment defaults + +Remaining code packages: + +1. **Supply-chain package**, currently uncommitted in the integration worktree: + `.dockerignore`, `docs/security_supply_chain.md`, + `scripts/run_supply_chain_checks.py`, `scripts/validate_sbom.py`, and + `tests/test_supply_chain_contracts.py`. Its focused suite last passed 65 + tests. A real unsuppressed history scan failed closed with 40 potential + findings across 10 commits; this is external review/rotation evidence, not a + repository test failure to suppress. +2. **Qdrant transport package**, uncommitted in + `G:\tmp\memorymaster-remediation-agents\r15-qdrant`. Worker evidence: 22 + transport adversarial tests, 105 focused non-ML tests, and 28 ML tests + passed. Independent final review and integration remain. + +The authorized read-only legacy inventory completed without changing the live +DB but returned `BLOCKED`: SQLite scanned 4,908,349 records and flagged 223,230, +with one unscannable value; artifacts scanned 1,184 files, flagged 654, and had +270 unscannable plus 2 truncated files; spool completed; Qdrant was +`BLOCKED-EXTERNAL/qdrant_not_configured`. Cleanup/redaction remains forbidden. + +## Budget rules + +1. Freeze scope. Do not add features, broad refactors, release automation, or + general hardening outside the remaining Phase 1 findings. +2. Fix only a reproducible Critical/High security or integrity defect in code + changed by Phase 1. Record Medium/Low observations in the backlog unless a + correction is trivial and required for a passing final gate. +3. Do not require a new adversarial RED for every small correction. Existing + witnessed REDs remain evidence. Add a new test only for a newly discovered + Critical/High bypass that lacks coverage. +4. Do not rerun the full suite after individual edits. Use syntax/import checks + while editing, one focused package run before its commit, and the full + verification ladder once after both packages are integrated. +5. Use at most one child beside root. Use one Sol/high read-only security review + at the combined integration boundary; time-box it to the changed Phase 1 + files. Do not fan out another full audit. +6. Allow one correction batch after the final review. If it reports only + Medium/Low or out-of-scope work, backlog it and continue. +7. Classify unavailable services, credentials, images, scanners, or live-data + actions as `BLOCKED-EXTERNAL` immediately. Do not wait for them and do not + manufacture passing evidence. +8. Preserve the main checkout and all unrelated dirty files. Do not push, + publish, deploy, rotate credentials, rewrite Git history, or mutate product + data. + +## Execution sequence + +### Checkpoint 1 — Close supply-chain package + +- Review the existing five-file diff; do not expand its scope. +- Run its focused tests, Ruff, format check, and diff check once. +- Preserve the real Gitleaks failure as `BLOCKED-EXTERNAL` with aggregate-only + evidence. +- Run GitNexus change detection and make one atomic conventional commit. + +### Checkpoint 2 — Integrate Qdrant transport + +- Review the frozen worker diff and run one Sol/high read-only review covering + only credential scoping, CA/TLS verification, secret-free errors, and R1.3 + quarantine preservation. +- Fix only reproducible Critical/High issues in one batch. +- Run the worker's focused non-ML and required ML gates once, then integrate and + make one atomic conventional commit. + +### Checkpoint 3 — One final verification boundary + +Run these once on the final integrated commit: + +1. `ruff check memorymaster/` plus changed scripts/tests. +2. Combined R1.4/R1.5 targeted adversarial and integration tests. +3. `python -m pytest tests/ -q --tb=short -m "not ml"` once. +4. Test collection once and compare with the prior Phase 1 count. +5. Required Qdrant ML tests once. +6. Compose fail-closed/private-binding configuration checks once. +7. Clean-wheel build/install and SBOM-to-wheel validation once. +8. Dependency/history/image scanners when locally available; otherwise record + exact blockers. Do not build or pull images solely to satisfy this goal. +9. SQLite/Postgres/Qdrant external parity remains blocked unless disposable + services are already available. +10. Run GitNexus change detection before each commit and preserve embeddings + when refreshing the index. + +Do not run browser/a11y, all 13 audit domains, performance benchmarks, recovery +drills, deployment smoke, or Phases 2–4 tests under this goal unless a changed +Phase 1 file directly breaks them. + +### Checkpoint 4 — Targeted audit delta and stop + +Reconcile only the Phase 1 findings and hard stops: + +- `MM-SEC-01` +- `MM-SEC-02` (Phase 1 containment only; R2.1 remains backlog) +- `MM-SEC-03` +- `MM-SEC-04` +- `MM-OPS-01` +- `MM-OPS-04` (Phase 1 defaults only; R3.4 remains backlog) + +Produce one concise audit delta with commit/runtime evidence, external actions, +rollback notes, and explicit Phase 2–4 backlog boundaries. Do not rerun the +full 13-domain audit, blind-spot catalog, or an unbounded convergence loop. + +## Completion contract + +Phase 1 is complete for this budget goal when: + +- The two remaining repository packages are committed and their focused gates + pass. +- The one final verification boundary is green or honestly + `BLOCKED-EXTERNAL`. +- No reproducible unresolved Critical/High regression introduced by the Phase + 1 branch remains. +- Medium/Low and out-of-scope findings are recorded as backlog rather than + expanded into new work. +- The six Phase 1 ledger rows are reconciled to commit/runtime evidence. +- The targeted audit delta is written. +- Phases 2–4 remain explicitly incomplete. + +Stop immediately after the targeted Phase 1 audit delta. Do not continue into +another audit loop. + +## Execution result — 2026-07-12 + +- Checkpoint 1 complete: supply-chain evidence committed as `b71e18f`. +- Checkpoint 2 complete: Qdrant transport/TLS package committed as `9b3e16c`; + final verification corrections are `8d80abb` and `132e5d0`. +- Checkpoint 3 complete under the V3 single-run rule. The one full non-ML run + reported 3,940 passed and 10 failures; those exact failures then passed, and + the focused R1.4 integrity gate passed 46 with 1 environment skip. The full + suite was intentionally not rerun. Collection was 4,126; required Qdrant ML + was 38 passed; Ruff, Compose contracts, clean-wheel install, and SBOM binding + passed. Dependency audits timed out and approved local product images were + unavailable, so those checks are `BLOCKED-EXTERNAL`. +- Checkpoint 4 complete: targeted delta is + `.planning/audits/2026-07-12-phase1-budget-delta/audit-delta.md`. +- Phases 2-4 remain incomplete and were not executed. + +## Replacement goal prompt + +```text +/goal Finish MemoryMaster Phase 1 in budget mode using the isolated worktree +`G:\tmp\memorymaster-remediation-20260710` and +`.planning/REMEDIATION-EXECUTION-V3-BUDGET.md` as the controlling scheduler. +Keep `.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md` and +`.planning/audit-remediation-ledger.md` as audit sources, but let V3 override V2 +for test frequency, review loops, audit scope, and completion criteria. + +Preserve the main checkout and all existing user changes. Resume the current +uncommitted five-file supply-chain package and the frozen Qdrant worker package +at `G:\tmp\memorymaster-remediation-agents\r15-qdrant`. Freeze scope: fix only +reproducible Critical/High security or integrity defects in changed Phase 1 +code; backlog Medium/Low and out-of-scope observations. Do not add features, +broad refactors, or release automation. + +Use at most one child beside root. Perform one time-boxed Sol/high read-only +security review at the Qdrant/combined integration boundary and allow at most +one correction batch. Do not rerun the full suite after individual edits. Run +syntax/import checks while editing, one focused package gate before each atomic +commit, then the full non-ML suite, required Qdrant ML tests, Ruff, collection, +Compose contracts, clean-wheel/SBOM validation, and locally available scanners +once at the final integration boundary. + +Keep all existing safety constraints: temporary/fake services by default; +GitNexus impact before editing existing symbols and change detection before +commits; embedding-preserving reindex; no push, publish, deploy, credential +rotation, history rewrite, live DB mutation, cleanup, redaction, migration, or +backlog operation. Record unavailable infrastructure and the unsuppressed +Gitleaks history findings as BLOCKED-EXTERNAL without suppressing or claiming a +pass. + +Reconcile only MM-SEC-01, MM-SEC-02 Phase 1 containment, MM-SEC-03, MM-SEC-04, +MM-OPS-01, and MM-OPS-04 Phase 1 defaults. Produce one targeted Phase 1 audit +delta. Completion requires no unresolved reproducible Critical/High regression +introduced by the Phase 1 branch; Medium/Low items may remain documented +backlog. Stop after the delta. Do not run a full 13-domain audit or begin Phases +2–4. +``` diff --git a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md index 53e093d8..469c0f33 100644 --- a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md +++ b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md @@ -85,8 +85,9 @@ Phase 0 must replace qualitative capacity goals with recorded numeric gates in t - [x] R1.1 — authenticated request context and MCP authorization - [x] R1.2 — Postgres tenant enforcement and RLS (repository complete; external proof blocked) - [x] R1.3 — immediate Qdrant containment -- [ ] R1.4 — complete sensitivity/write gateway and legacy inventory -- [ ] R1.5 — secure deployment and supply-chain defaults +- [x] R1.4 — complete sensitivity/write gateway and legacy inventory (repository complete; live cleanup/Qdrant inventory blocked externally) +- [x] R1.5 — secure deployment and supply-chain defaults (repository complete; rotation, advisory/image/runtime evidence blocked externally) +- [x] Phase 1 targeted budget audit delta (V3 scope; Phases 2-4 remain open) - [ ] R2.1 — unified RetrievalPlanner and governed Qdrant reintegration - [ ] R2.2 — lifecycle authority and read-only recall - [ ] R2.3 — unified entity model diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 294bdbfa..cc064d03 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -14,16 +14,16 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY |---|---|---|---|---|---|---| | MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | BLOCKED-EXTERNAL | Owner: Codex remediation branch. R1.1 complete. PostgreSQL application connections are team-only and bind tenant, principal, and immutable scopes transaction-locally; schema work requires a distinct verified migrator. v0011 FORCE RLS defines restricted role/table/policy and append-only event contracts, a tenant-derived hash-only event-head function, and deny-only governance/raw tables. v0012 defines six partial unique indexes: public keys are tenant + exact-scope local; non-public keys additionally include exact visibility/principal; ambiguity without exact scope fails closed; every team claim requires a nonblank owner. Startup validates literal-sensitive policy/index/function fingerprints, exact event and claims trigger inventories, required event SELECT/INSERT plus forbidden table/column UPDATE and DELETE, a privileged event-head owner, and the strict validated owner constraint. Supersession rejects self/cross-tenant/scope/visibility/owner references and the canonical path atomically commits reciprocal pointers plus one event; v0012 preflights unsafe legacy edges. Unsupported Postgres source/evidence/action/retry, read-only, merge, delta, and tenant-bound CLI surfaces now fail before driver or filesystem access; whitespace-wrapped DSNs cannot bypass routing. The parity harness requires two distinct roles plus disposable opt-in, uses UUID tenant namespaces, and performs no destructive cleanup. Closure matrix: 349 passed, 47 externally gated skips; direct surface/factory matrix: 57 passed. Full isolated non-ML gate: 3,552 passed, 69 skipped, 95 deselected, 22 intentional xfails, 2 warnings in 858.47s. Commit evidence is the commit containing this row. Rollback: keep the team profile disabled and revert this package; schema rollback requires a verified backup/forward repair. Repository work for R1.1-R1.2 is complete. Real two-role PostgreSQL evidence plus approved brownfield owner/duplicate/supersession-edge inventory/repair remain `BLOCKED-EXTERNAL`; the Team/Postgres profile remains blocked. | | MM-SEC-02 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | Qdrant can return archived, sensitive, cross-scope/tenant, or orphan payloads | R1.3,R2.1 | IN-PROGRESS | Owner: Codex remediation branch. R1.3 containment is repository-complete: local-trusted claim requests and auto-classified Qdrant recommendations fall back to authoritative lexical retrieval with requested/classified/effective metadata; team semantic requests are denied before tool dispatch; prompt-context fallback is disconnected; verbatim vector/hybrid requests use FTS5; CLI denial occurs before service/backend construction; and every direct claim/verbatim/fallback read adapter raises before model, network, or raw payload access. Qdrant upsert/sync/reconcile/count-ID maintenance remains available. Adversarial containment: 18 passed; Qdrant/verbatim/classifier matrix: 153 passed; CLI/setup/MCP regression matrix: 182 passed; explicit ML gate: 14 passed. Full isolated non-ML gate: 3,572 passed, 69 skipped, 95 deselected, 20 intentional xfails, 2 warnings in 882.06s; collection: 3,756; Ruff: clean. Independent blocker-only review found no first-party raw payload-search path. Commit evidence is the commit containing this row. Rollback: keep semantic retrieval disabled and revert this package. R2.1 governed ID-candidate rehydration remains unimplemented, and final authenticated/TLS Qdrant parity is `BLOCKED-EXTERNAL`; do not enable the semantic profile. | -| MM-OPS-01 | DevOps | Critical / EXPLOITABLE-LOW-EFFORT / H4 | Postgres Compose publishes a fixed credential | R1.5 | OPEN | Fail-closed Compose config, private port, external rotation evidence | -| MM-SEC-03 | Security | High / EXPLOITABLE-NOW | Persisted metadata/provenance fields bypass sensitivity scanning | R1.4 | OPEN | Complete field-matrix adversarial suite and legacy dry-run | -| MM-SEC-04 | Security | High / EXPLOITABLE-NOW | Steward, compact-summary, verbatim, and integration writes bypass one gateway | R1.4 | OPEN | Table-driven write-path test; no raw fixture in durable scan | +| MM-OPS-01 | DevOps | Critical / EXPLOITABLE-LOW-EFFORT / H4 | Postgres Compose publishes a fixed credential | R1.5 | BLOCKED-EXTERNAL | Repository defaults are fail-closed and private (`a858419`); Compose rejects missing inputs and renders with synthetic required inputs. Historical credential rotation/recreation and a real network probe require operator action. | +| MM-SEC-03 | Security | High / EXPLOITABLE-NOW | Persisted metadata/provenance fields bypass sensitivity scanning | R1.4 | BLOCKED-EXTERNAL | Repository gateways and the aggregate-only inventory are complete (`a3e3824`, `702b59d`, `8d80abb`); focused final gate: 46 passed, 1 skipped. The authorized live read-only inventory found legacy sensitive/unscannable material, while Qdrant was unavailable; cleanup/redaction and product-data mutation remain forbidden without approval. | +| MM-SEC-04 | Security | High / EXPLOITABLE-NOW | Steward, compact-summary, verbatim, and integration writes bypass one gateway | R1.4 | RESOLVED | Durable auxiliary writers now use the shared sensitivity envelope and adversarial persistence matrices; the final observed compatibility failures were corrected and the exact 10-test failure set passed. Roll back by reverting the R1.4 commits while keeping affected ingestion disabled. | | MM-ARCH-01 | Architecture/Database | High / BAD-PRACTICE | Entity registry and graph own incompatible `entities` schemas | R2.3 | OPEN | Normal init-to-graph MCP integration and FK check | | MM-ARCH-02 | Architecture/Integrity | High / BAD-PRACTICE | MCP/hooks/CLI/Qdrant use contradictory retrieval planners and trust defaults | R2.1 | OPEN | Cross-surface ID-set and conversational-query parity | | MM-LIFE-01 | Database/Reliability | High / BAD-PRACTICE | Scheduled archival uses raw SQL and bypasses lifecycle/vector evidence | R2.2 | OPEN | Version/event/timestamp/outbox assertions through scheduled path | | MM-REL-02 | Reliability/Performance | Medium / BAD-PRACTICE | MCP reads take write locks and context detail modes retrieve twice | R2.2 | OPEN | Zero query write lock; one aggregated telemetry envelope | | MM-OPS-02 | DevOps/Demo | High / BAD-PRACTICE | Docker/Helm publish HTTP on a stdio MCP process and use an invalid healthcheck | R3.4 | OPEN | Built-container readiness and MCP handshake | | MM-OPS-03 | DevOps/Maintainability | High / BAD-PRACTICE | A release tag can publish without a blocking verified test artifact | R4.4 | OPEN | Deliberately failing release candidate cannot publish | -| MM-OPS-04 | Security/DevOps | High / EXPLOITABLE-LOW-EFFORT | Qdrant/Ollama ports are broadly exposed; images are mutable | R1.5,R3.4 | OPEN | Private network defaults, auth/TLS, pinned digests | +| MM-OPS-04 | Security/DevOps | High / EXPLOITABLE-LOW-EFFORT | Qdrant/Ollama ports are broadly exposed; images are mutable | R1.5,R3.4 | IN-PROGRESS | Phase 1 defaults are repository-complete (`a858419`, `9b3e16c`, `b71e18f`): loopback-only ports, required immutable digests, authenticated TLS Qdrant, redirect-safe credentials, and fail-closed supply-chain policy. Real image scans/runtime health remain `BLOCKED-EXTERNAL`; R3.4 service-entrypoint work remains backlog. | | MM-PERF-01 | Performance/Cost | High / BAD-PRACTICE | Hybrid reads recompute and rewrite candidate embeddings | R3.1 | OPEN | Warm query: one query embed, zero candidate embeds/writes | | MM-PERF-02 | Performance | Medium / BAD-PRACTICE | Each process cold-scans corpus token statistics and caches forever | R3.2 | OPEN | Generation-aware/token-specific stats benchmark | | MM-PERF-03 | Performance/Cost | High / EXPLOITABLE-NOW | Qdrant reconciliation repeats/truncates tens of thousands of embeddings | R3.1 | OPEN | Paginated incremental convergence benchmark | @@ -46,6 +46,20 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-INTEGRITY-01 | Code Integrity | Medium / BAD-PRACTICE | `importlib.util` probe is wrong in clean Python and its test masks failure | R4.4 | OPEN | Clean subprocess test and explicit import | | MM-TEST-01 | Maintainability/Reliability | Medium / BAD-PRACTICE | Validator candidate winner changed under load because mutable `updated_at` controlled processing order | Phase 0/R4.4 | RESOLVED | Deterministic timestamp-inversion regression passes; targeted lifecycle suite 35 passed; full isolated non-ML gate 3,094 passed, 56 skipped, 95 deselected, 24 intentional xfails | +## Phase 1 budget reconciliation (2026-07-12) + +This supplement is authoritative for the six rows in the V3 Phase 1 scope. It +does not change or close Phase 2-4 work. + +| Finding | Phase 1 disposition | Final evidence | +|---|---|---| +| MM-SEC-01 | `BLOCKED-EXTERNAL` | R1.1-R1.2 repository work remains complete and fail-closed. Disposable two-role Postgres proof and brownfield inventory/repair approval are still external; the Team/Postgres profile stays disabled. | +| MM-SEC-02 | `IN-PROGRESS` overall; Phase 1 containment complete | R1.3 continues to deny every Qdrant payload-read adapter. R1.5 adds authenticated, TLS-verified, redirect-denying maintenance transport. Required ML gate: 38 passed. R2.1 governed rehydration remains backlog and semantic reads stay disabled. | +| MM-SEC-03 | `BLOCKED-EXTERNAL` | New writes and inventory code are covered; the live read-only inventory accounted for SQLite/artifact/spool surfaces but reported legacy unscannable/truncated material and no configured Qdrant. No cleanup or redaction was authorized. | +| MM-SEC-04 | `RESOLVED` | Shared persisted-envelope gateway covers auxiliary writers; focused final integrity gate passed 46 tests with 1 environment skip. | +| MM-OPS-01 | `BLOCKED-EXTERNAL` | Fixed credentials were removed and Compose is fail-closed/private. Historical rotation/recreation and an external port probe remain operator work. | +| MM-OPS-04 | `IN-PROGRESS` overall; Phase 1 defaults complete | Loopback bindings, digest-only images, Qdrant API-key/TLS propagation, redirect denial, clean-wheel/SBOM binding, and scanner policy are committed. Approved local product images and runtime targets were unavailable; R3.4 remains backlog. | + ## Source-domain reconciliation The canonical rows above deduplicate repeated findings from the 13 domain reports. Original domain IDs remain traceable as follows: diff --git a/.planning/audits/2026-07-12-phase1-budget-delta/audit-delta.md b/.planning/audits/2026-07-12-phase1-budget-delta/audit-delta.md new file mode 100644 index 00000000..bf8f3da3 --- /dev/null +++ b/.planning/audits/2026-07-12-phase1-budget-delta/audit-delta.md @@ -0,0 +1,119 @@ +# MemoryMaster Phase 1 Budget Audit Delta + +**Date:** 2026-07-12 + +**Baseline:** `.planning/audits/2026-07-10-baseline/audit-report.md` + +**Scheduler:** `.planning/REMEDIATION-EXECUTION-V3-BUDGET.md` + +**Scope:** MM-SEC-01, MM-SEC-02 Phase 1 containment, MM-SEC-03, +MM-SEC-04, MM-OPS-01, and MM-OPS-04 Phase 1 defaults only. + +## Verdict + +Phase 1 repository convergence is complete under the V3 budget contract. No +reproducible Critical/High regression introduced by the Phase 1 branch remains +unresolved. External runtime, product-data, credential-rotation, advisory, and +image-scan evidence is explicitly blocked rather than treated as passing. + +This was not a 13-domain reaudit. Phases 2-4 and their findings remain open. + +## Finding delta + +| Finding | Baseline | Phase 1 disposition | Delta | +|---|---|---|---| +| MM-SEC-01 | Critical / open | `BLOCKED-EXTERNAL` | R1.1-R1.2 repository enforcement is complete and fail-closed. Real two-role Postgres proof and brownfield repair authority remain external; the Team/Postgres profile stays disabled. | +| MM-SEC-02 | Critical / open | `IN-PROGRESS` overall; Phase 1 containment complete | All Qdrant payload-read adapters remain quarantined. Maintenance clients now use scoped API keys, verified TLS, and no redirects. R2.1 governed candidate rehydration remains backlog and semantic reads stay disabled. | +| MM-SEC-03 | High / open | `BLOCKED-EXTERNAL` | Persisted-field gateways and aggregate-only inventory are repository-complete. The authorized live read-only inventory found legacy unscannable/truncated material and no configured Qdrant; no cleanup/redaction was authorized. | +| MM-SEC-04 | High / open | `RESOLVED` | Steward, compact-summary, verbatim, bridge, feedback, and integration writes are covered by the shared persisted-envelope policy and adversarial matrices. | +| MM-OPS-01 | Critical / open | `BLOCKED-EXTERNAL` | Fixed deployment credentials were removed; Compose is fail-closed and ports are private. Historical rotation/recreation and an external port probe remain operator actions. | +| MM-OPS-04 | High / open | `IN-PROGRESS` overall; Phase 1 defaults complete | Loopback bindings, immutable digests, Qdrant auth/TLS, redirect denial, clean-wheel/SBOM binding, and scanner policy are committed. Approved runtime images and R3.4 entrypoint/readiness work remain outstanding. | + +## Implemented evidence + +- `a3e3824` — hardened bridge persistence transport. +- `702b59d` — aggregate-only legacy sensitivity inventory and fail-closed + SQLite/artifact/spool/Qdrant accounting. +- `a858419` — fail-closed secret inputs, private bindings, and digest-only + deployment defaults. +- `b71e18f` — bound repository/history/dependency/image/SBOM supply-chain + policy and validator. +- `9b3e16c` — separated Qdrant clients, API-key/CA propagation, remote HTTPS + enforcement, redirect denial, TLS-enabled Compose, and HTTPS Helm default. +- `8d80abb` — routed the new inventory reader through the canonical read-only + SQLite helper and corrected final compatibility fixtures. +- `132e5d0` — aligned the required ML fixture with the secure transport. + +The Qdrant TLS deployment settings follow Qdrant's documented `enable_tls`, +certificate, and key configuration, and the transport rejects plaintext remote +endpoints before client/network construction. + +## Verification boundary + +| Gate | Result | +|---|---| +| Supply-chain focused package | 65 passed; Ruff/format/diff checks passed before `b71e18f`. | +| Qdrant focused non-ML package | 130 passed, 24 deselected, 2 intentional xfails before `9b3e16c`. | +| Independent security review | Two High Qdrant findings (plaintext authenticated defaults and cross-origin redirect credential forwarding), zero Critical; both corrected in `9b3e16c`. | +| Final non-ML run | Single V3 run: 3,940 passed, 70 skipped, 95 deselected, 11 intentional xfails, 10 failures. The failures were nine stale transport test adapters plus one real canonical-connection regression. | +| Failure reconciliation | Exact 10-test failure set passed; focused R1.4 integrity gate passed 46 with 1 environment skip after `8d80abb`. Per V3, the 15-minute full suite was not rerun. This is compositional evidence, not a claim that a second full-suite invocation passed. | +| Collection | 4,126 tests collected. | +| Required Qdrant ML | 38 passed on the final fixture state. | +| Ruff | Final `memorymaster/` plus changed scripts/tests passed. | +| Compose | Missing required inputs failed closed; complete synthetic key/digest/certificate inputs rendered successfully. No containers were started. | +| Clean wheel | Wheel built from `git archive HEAD`, installed into a fresh venv, and imported in isolated mode. SHA-256: `d3a98b7ed6db406a1080b7cb23aad4277845caa6019211d8a2600070a89c39ed`. | +| SBOM | CycloneDX 1.6 root identity and SHA-256 binding validated against that exact wheel. | +| GitNexus | Impact checks were LOW or unavailable for newly added/unindexed private helpers; staged change detection was run before each commit. The index was refreshed with embeddings after commits. | + +## External blockers + +- Known unsuppressed Gitleaks history evidence remains 40 potential findings + across 10 commits and 7 files. No suppression or pass claim was added. A + security owner must classify and rotate before any approved history action. +- Strict project and `mcp,qdrant,security` dependency audits exceeded the + single 15-minute advisory-service cap and were terminated. They are + `BLOCKED-EXTERNAL/advisory-timeout`, not passing. +- Docker and Docker Scout were available, but no approved local immutable + MemoryMaster, Qdrant, or Ollama images existed. V3 forbade pulls/builds solely + for scanning, so image CVE evidence is `BLOCKED-EXTERNAL`. +- Helm was unavailable; no Kubernetes render/runtime claim is made. +- Disposable authenticated/TLS Qdrant and two-role Postgres targets were not + available. Fake/local tests passed, but real-service parity remains external. +- The live inventory was read-only and did not change the database. Cleanup, + redaction, rebuild, migration, backlog, and retention operations remain + unauthorized. +- MemoryMaster MCP recall was attempted before the transport decision but the + MCP transport was closed; no live-DB fallback was used. + +See `external-actions-required.md` for owners and required evidence. + +## Backlog boundary + +- R2.1: governed Qdrant candidate-ID rehydration and unified retrieval planner. +- R2.2-R2.5: lifecycle authority, entity convergence, capture/budget/retention, + and mock-evidence removal. +- R3.1-R3.5: performance, setup truth, service readiness, recovery, + observability, and privacy operations. +- R4.1-R4.4: modularity, UX/accessibility, and generated release truth. +- Medium review note: validate Qdrant transport configuration before loading a + potentially downloadable embedding model in the standalone indexer. + +None of these items was executed or accepted by this goal. + +## Rollback + +- Keep semantic Qdrant reads disabled; revert `9b3e16c`/`132e5d0` only with + Qdrant integrations disabled. +- Revert R1.4 commits only while affected ingestion/bridge paths are disabled; + never restore a sensitivity bypass. +- Revert deployment/supply-chain commits only to another fail-closed private, + immutable, authenticated configuration. +- Postgres schema rollback requires a verified backup or forward repair; do + not edit immutable migrations or mutate live data under this record. + +## Stop condition + +The latest targeted verification has zero unresolved new Critical/High Phase 1 +regressions. All six authorized rows are resolved for repository scope or +validly blocked/in-progress at their explicit Phase 2-4 or external boundary. +Stop here; do not begin another audit loop. diff --git a/external-actions-required.md b/external-actions-required.md index ab1d25cd..40f8f6e0 100644 --- a/external-actions-required.md +++ b/external-actions-required.md @@ -7,11 +7,15 @@ live mutation by itself. | Finding | Owner/system | Required action | Evidence needed to unblock | Review date | Status | |---|---|---|---|---|---| -| MM-OPS-01 | Operator / any Postgres deployment | Rotate the historical `mm_pw` credential and recreate deployments that used it. Keep database ports private. | Rotation record plus network probe showing the port is not externally reachable | 2026-07-31 | PENDING-INVENTORY | +| MM-OPS-01 | Operator / any Postgres deployment | Rotate the historical `mm_pw` credential and recreate deployments that used it. Keep database ports private. | Rotation record plus network probe showing the port is not externally reachable | 2026-07-31 | BLOCKED-EXTERNAL | | MM-SEC-01 / disposable proof | Team Postgres test environment | Provide one fully disposable database, a dedicated schema-owning `SUPERUSER`/`BYPASSRLS` migrator DSN in `MEMORYMASTER_TEST_POSTGRES_DSN`, and a distinct restricted app DSN in `MEMORYMASTER_TEST_POSTGRES_APP_DSN`; set `MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE=1`. Neither DSN may target product/live data. | Run `tests/test_postgres_rls_integration.py` plus every Postgres parameter in `tests/test_postgres_parity.py` and `tests/test_backend_parity.py`. Retain exact FORCE-RLS policy/function/trigger/index catalog evidence; literal-sensitive fingerprints; event-head owner `SUPERUSER`/`BYPASSRLS`; app event SELECT/INSERT with no table/column UPDATE or DELETE; cross-tenant/scope/visibility/principal denials; tenant-chain linearity; scope-local identities and ambiguity rejection; self/cross-boundary supersession denial; successful reciprocal supersession plus one event; rollback after replacement-update/event-insert failure; and parity results from UUID-isolated tenants with no destructive cleanup. No real PostgreSQL proof has been performed here. | 2026-07-31 | BLOCKED-EXTERNAL | | MM-SEC-01 / brownfield data | Live database operator | Authorize a read-only inventory before v0012. Any owner backfill, duplicate/supersession-edge repair, or grant change requires separate explicit approval, a verified backup, and a maintenance window. | Inventory noncanonical visibility; blank/null `source_agent` on every claim; identity duplicates; both supersession columns for self, missing-target, nonreciprocal, cross-tenant/scope/visibility/owner edges; current event/function/table grants; and constraint state without writing. After approval, retain repair/grant evidence and rerun v0012, which validates `ck_claims_identity_visibility_owner`. No product-data verification or mutation has been performed here. | 2026-07-31 | BLOCKED-EXTERNAL | -| MM-OPS-02 | Docker/Helm runtime | Provide Docker and, for final verification, a disposable Kubernetes target if unavailable locally. | Container health/MCP handshake and Helm readiness/network-policy evidence | 2026-08-15 | PENDING-CAPABILITY-CHECK | +| MM-OPS-02 | Docker/Helm runtime | Provide approved immutable MemoryMaster/Qdrant/Ollama images, certificates, and a disposable Kubernetes target. Docker/Compose config was available locally; Helm and approved product images were not. | Container health/MCP handshake, authenticated TLS Qdrant health, and Helm readiness/network-policy evidence | 2026-08-15 | BLOCKED-EXTERNAL | | MM-SEC-02 | Qdrant runtime | Provide a disposable authenticated/TLS Qdrant target for final parity after the fake-backed containment suite. Do not point the tests at a live/product collection. | API-key/TLS connection proof plus real-service sync/reconcile/count-ID evidence and denial of every payload-search adapter; retain exact command output without secrets | 2026-08-15 | BLOCKED-EXTERNAL | +| MM-SEC-03 | Live legacy stores | Review the aggregate inventory, provide a verified backup, and separately authorize any cleanup/redaction/rebuild. Configure a disposable or approved read-only Qdrant inventory target before claiming complete coverage. | SQLite/artifact/spool/Qdrant aggregate rerun with zero unscannable/truncated surfaces, plus approved remediation evidence; never include raw findings | 2026-07-31 | BLOCKED-EXTERNAL | +| MM-OPS-04 / history scan | Security owner | Classify the unsuppressed Gitleaks history result: 40 potential findings across 10 commits and 7 files. Rotate any affected credentials before considering an approved history action. | Aggregate classification and rotation record; no raw secret material in general logs | 2026-07-31 | BLOCKED-EXTERNAL | +| MM-OPS-04 / dependency audit | Release owner / advisory service | Rerun strict project and `mcp,qdrant,security` extra audits in an approved lock/resolver environment; both final attempts exceeded the 15-minute external-service cap and were terminated without a pass claim. | Successful strict PyPI audit bound to the release lock/artifact, or reviewed vulnerability remediation evidence | 2026-07-31 | BLOCKED-EXTERNAL | +| MM-OPS-04 / image scan | Release owner / local image registry | Provide approved immutable local MemoryMaster, Qdrant, and Ollama image IDs. No relevant local images existed and this goal forbade builds or pulls solely for scanning. | Docker Scout high/critical scan results for the exact immutable release images, including base findings | 2026-07-31 | BLOCKED-EXTERNAL | | MM-DATA-01 | Live MemoryMaster operator | Approve a consistent backup/restore drill before any live migration, redaction, backlog, or retention operation. | Restored backup, integrity check, counts/checksums, approval record | 2026-08-15 | BLOCKED-EXTERNAL | | MM-CAP-01 | Host storage operator | Address/monitor the drive at 85.82% used without deleting MemoryMaster data under this goal. | Daily disk telemetry below the critical gate or approved capacity expansion | 2026-07-18 | BLOCKED-EXTERNAL | | MM-PRIV-01 | Product owner / legal reviewer | Decide intended organizational use, jurisdictions, processor disclosures, and retention commitments before compliance claims. | Approved privacy/data-processing statement | 2026-08-31 | BLOCKED-EXTERNAL | From 307be22eb14c790dd31caa8b2296c725f038e8bf Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sun, 12 Jul 2026 21:36:40 -0300 Subject: [PATCH 28/55] docs(plan): split remaining phases into budget goals --- .planning/REMAINING-PHASES-V1-BUDGET.md | 136 ++++++++++++++++++++++++ 1 file changed, 136 insertions(+) create mode 100644 .planning/REMAINING-PHASES-V1-BUDGET.md diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md new file mode 100644 index 00000000..8a307c61 --- /dev/null +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -0,0 +1,136 @@ +# MemoryMaster Remaining Phases — Budget Execution V1 + +**Created:** 2026-07-12 + +**Base:** Phase 1 commit `95c2b3a` + +**Phase 1 review:** draft PR #185. Do not add Phase 2-4 commits to that branch. + +## Objective + +Complete Phases 2-4 without recreating the open-ended remediation loop. Each +package gets an isolated branch/worktree, one focused gate, one atomic commit, +and a stop. A full non-ML suite runs once at each phase boundary, not after +individual edits. + +Production deployment remains a separate release goal after all phase and +external-action gates pass. + +## Shared budget and safety rules + +1. Freeze each package to its named findings and acceptance contract. +2. Add tests only for the behavior being changed or a newly reproduced + Critical/High regression. +3. Use syntax/import checks while editing and one focused package gate before + commit. Do not run the full suite inside package goals. +4. Run GitNexus impact before existing-symbol edits and change detection before + commits; preserve embeddings on reindex. +5. Use temporary databases and fake/local services. Never mutate live product + data or enable a previously quarantined profile without its phase exit gate. +6. Record unavailable models, services, credentials, images, and legal/operator + decisions as `BLOCKED-EXTERNAL`; continue unblocked work once. +7. No push, merge, release, staging, or production deployment inside package + goals. Publish one phase draft PR only after its phase convergence gate. +8. Medium/Low findings outside the package become backlog. No broad refactors. + +## Phase 2 — Governed core convergence + +Phase 2 uses branch `remediation/phase2-governed-core-20260712` and worktree +`G:\tmp\memorymaster-phase2-20260712`. + +### Goal P2-A — RetrievalPlanner contract + +- Establish one immutable request/planner input and one policy-filtered result + contract across service, MCP, hooks, CLI, context packing, task briefing, and + volunteer context. +- Preserve Phase 1 lexical fallback and Qdrant quarantine. +- Do not enable semantic Qdrant reads in this goal. +- Acceptance: same request/policy yields the same authoritative claim IDs and + trust annotations across covered surfaces. + +### Goal P2-B — Governed Qdrant reintegration + +- Overfetch candidate IDs only; rehydrate every candidate from the authoritative + store and reapply the P2-A planner policy. +- Reject orphan, archived, sensitive, wrong-scope, wrong-tenant, stale-hash, and + malformed points before result construction. +- Add exact ID/content-hash reconciliation and a bounded replayable outbox. +- Keep the semantic profile disabled if disposable authenticated/TLS Qdrant is + unavailable; fake-backed repository work may complete as blocked external. + +### Goal P2-C — Lifecycle authority/read-only recall + +- Route scheduled archival through canonical transitions with versions, events, + optimistic locking, cache invalidation, and replayable vector deletion. +- Make top-level recall read-only and spool at most one aggregated telemetry + envelope; remove duplicate detail-level retrieval. + +### Goal P2-D — Entity schema convergence + +- Select one canonical entity registry/graph schema and immutable migrations. +- Remove lazy DDL from read tools and expose explicit graph readiness failures. +- Acceptance covers registry-first, graph-first, backend parity, MCP integration, + and zero FK violations. + +### Goal P2-E — Quiet finite capture and budgets + +- Default to session-start recall, on-demand recall, and distilled session-end + ingest. +- Gate verbatim/per-stop/correction capture behind explicit maximum-capture + flags; persist transcript cursors, provider/global budgets, and usage. +- Add finite age/bytes/session retention and dry-run backlog controls. Never + bulk-confirm or mutate the live backlog. + +### Goal P2-F — Remove fake evidence defaults + +- Require an explicit real media provider in production modes. +- Permit mocks only under conspicuous test/dev configuration and prevent mock + evidence from creating governed claims/actions. + +### Goal P2-Z — Phase 2 convergence + +- Integrate P2-A through P2-F in dependency order. +- Run the Phase 2 targeted matrix, full non-ML once, collection, Ruff, required + ML tests, and disposable backend/runtime gates when available. +- Reconcile only MM-SEC-02, MM-ARCH-01, MM-ARCH-02, MM-LIFE-01, MM-REL-02, + MM-PRIV-01, MM-COST-01, MM-COST-02, MM-COST-03, and MM-DEMO-01. +- Produce one Phase 2 audit delta and draft PR; stop before Phase 3. + +## Phase 3 — Performance and operational readiness + +Run separate goals for R3.1 embeddings/reconciliation, R3.2 query/storage, +R3.3 setup profiles, R3.4 service entrypoints/readiness, and R3.5 +recovery/observability/privacy. Each package uses focused tests only. The Phase +3 convergence goal runs benchmarks, clean-wheel profile setup, Docker/Helm +smoke, backup restore, and privacy dry-runs once, then produces a Phase 3 delta +and draft PR. + +No real backup deletion, privacy erase, deployment, or production-data action +is authorized by this scheduler. + +## Phase 4 — Product focus and maintainability + +Run separate goals for R4.1 extension boundaries, R4.2 gradual decomposition, +R4.3 governance UX/accessibility, and R4.4 generated release truth. Avoid a +flag-day rewrite; keep compatibility facades and measured size budgets. The +Phase 4 convergence goal runs one full verification boundary, browser/a11y +validation, documentation/release drift checks, and produces the final phase +delta and draft PR. + +## Release sequence after Phase 4 + +1. Merge reviewed phase PRs in order. +2. Resolve every item in `external-actions-required.md` or obtain an explicit, + documented risk decision from the appropriate owner. +3. Build immutable release artifacts/images and run the final release-candidate + suite, dependency/history/image scans, SBOM/provenance checks, and disposable + Postgres/Qdrant parity. +4. Deploy to staging/canary and verify health, MCP handshake, migrations, + rollback, telemetry, and data-integrity invariants. +5. Request explicit production cutover approval with the exact image/artifact + digests and rollback command. Production is never inferred from a phase goal. + +## Current next action + +Execute P2-A only. Stop after its atomic commit, focused evidence, and scheduler +update; do not begin governed Qdrant reintegration in the same package. From 7026711c6fd4a4340441afddd12e4c2d7ddd93ac Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Sun, 12 Jul 2026 21:51:03 -0300 Subject: [PATCH 29/55] feat(recall): add governed retrieval planner --- .planning/REMAINING-PHASES-V1-BUDGET.md | 14 ++- memorymaster/core/service.py | 103 +++++++++++++---- memorymaster/recall/planner.py | 136 +++++++++++++++++++++++ memorymaster/surfaces/mcp_server.py | 74 ++++++------ tests/test_mcp_helpers.py | 1 + tests/test_push_volunteer_context.py | 18 ++- tests/test_retrieval_planner_contract.py | 52 +++++++++ tests/test_retrieval_surface_parity.py | 83 +++++++++++--- 8 files changed, 407 insertions(+), 74 deletions(-) create mode 100644 memorymaster/recall/planner.py create mode 100644 tests/test_retrieval_planner_contract.py diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index 8a307c61..7c21f56e 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -132,5 +132,15 @@ delta and draft PR. ## Current next action -Execute P2-A only. Stop after its atomic commit, focused evidence, and scheduler -update; do not begin governed Qdrant reintegration in the same package. +P2-A implementation and focused gate completed on 2026-07-12: + +- Immutable `RetrievalRequest`, `RetrievalPlan`, and `RetrievalResult` govern + service and MCP recall entrypoints. +- Trusted mode is confirmed-only; exploratory status expansion is explicit. +- Conversational lexical recall uses bounded safe token fan-out while Qdrant + remains quarantined behind the planner. +- Focused gate: 141 passed, 1 expected xfail; FTS/planner regression slice: + 32 passed; changed-file Ruff and `git diff --check` passed. + +Next package is P2-B. Start it only in a separate goal/package; do not combine +governed Qdrant reintegration with P2-A. diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index dc1449e5..b4cabf18 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -19,6 +19,11 @@ from memorymaster.recall.context_optimizer import ContextResult, pack_context from memorymaster.core.config import get_config from memorymaster.recall.retrieval import VectorSearchHook, _tier_bonus, rank_claim_rows +from memorymaster.recall.planner import ( + RetrievalRequest, + RetrievalResult, + build_retrieval_plan, +) from memorymaster.core.security import ( is_sensitive_claim, normalize_sensitivity_findings, @@ -977,8 +982,8 @@ def query( query_text: str, *, limit: int = 20, - include_stale: bool = True, - include_conflicted: bool = True, + include_stale: bool = False, + include_conflicted: bool = False, include_candidates: bool = False, retrieval_mode: str = "legacy", vector_hook: VectorSearchHook | None = None, @@ -986,21 +991,52 @@ def query( allow_sensitive: bool = False, scope_allowlist: list[str] | None = None, query_type: str | None = None, + trust_mode: str | None = None, ) -> list[Claim]: - rows = self.query_rows( + resolved_trust = trust_mode or ( + "exploratory" + if include_stale or include_conflicted or include_candidates + else "trusted" + ) + result = self.retrieve(RetrievalRequest( query_text=query_text, limit=limit, + trust_mode=resolved_trust, include_stale=include_stale, include_conflicted=include_conflicted, include_candidates=include_candidates, retrieval_mode=retrieval_mode, - vector_hook=vector_hook, retrieval_profile=retrieval_profile, allow_sensitive=allow_sensitive, - scope_allowlist=scope_allowlist, + scope_allowlist=tuple(scope_allowlist) if scope_allowlist else None, query_type=query_type, + ), vector_hook=vector_hook) + return [row["claim"] for row in result.rows] + + def retrieve( + self, + request: RetrievalRequest, + *, + vector_hook: VectorSearchHook | None = None, + ) -> RetrievalResult: + """Execute one immutable governed retrieval plan.""" + plan = build_retrieval_plan(request) + statuses = set(plan.statuses) + rows = self.query_rows( + query_text=plan.search_text, + limit=plan.limit, + include_stale="stale" in statuses, + include_conflicted="conflicted" in statuses, + include_candidates="candidate" in statuses, + retrieval_mode=plan.effective_mode, + vector_hook=vector_hook, + retrieval_profile=plan.retrieval_profile, + allow_sensitive=plan.allow_sensitive, + scope_allowlist=list(plan.scope_allowlist) if plan.scope_allowlist else None, + requesting_agent=plan.requesting_agent, + query_type=plan.query_type, ) - return [row["claim"] for row in rows] + return RetrievalResult(plan=plan, rows=tuple(rows)) def query_rules( self, @@ -1179,14 +1215,8 @@ def _build_query_statuses(self, include_stale: bool, include_conflicted: bool, i def _query_legacy_mode(self, query_text: str, limit: int, statuses: list[str], normalized_scopes: list[str] | None, include_sensitive: bool, requesting_agent: str | None) -> list[dict[str, Any]]: """Query using legacy retrieval mode.""" - legacy = self.store.list_claims( - limit=limit, - text_query=query_text, - status_in=statuses, - include_archived=False, - include_citations=True, - scope_allowlist=normalized_scopes, - tenant_id=self.tenant_id, + legacy = self._legacy_candidates( + query_text, limit, statuses, normalized_scopes ) if not include_sensitive: legacy = [claim for claim in legacy if not is_sensitive_claim(claim)] @@ -1216,6 +1246,32 @@ def _query_legacy_mode(self, query_text: str, limit: int, statuses: list[str], n self._record_accesses(results, query_text=query_text) return results + def _legacy_candidates( + self, + query_text: str, + limit: int, + statuses: list[str], + normalized_scopes: list[str] | None, + ) -> list[Claim]: + """Resolve planner OR expressions without exposing raw FTS syntax.""" + queries = [query_text] + if " OR " in query_text: + queries = [term for term in query_text.split(" OR ") if term] + merged: dict[int, Claim] = {} + for text_query in queries: + claims = self.store.list_claims( + limit=limit, + text_query=text_query, + status_in=statuses, + include_archived=False, + include_citations=True, + scope_allowlist=normalized_scopes, + tenant_id=self.tenant_id, + ) + for claim in claims: + merged.setdefault(claim.id, claim) + return list(merged.values())[:limit] + def query_rows( self, query_text: str, @@ -1607,14 +1663,15 @@ def query_for_context( token_budget: int = 4000, output_format: str = "text", limit: int = 100, - include_stale: bool = True, - include_conflicted: bool = True, + include_stale: bool = False, + include_conflicted: bool = False, include_candidates: bool = False, retrieval_mode: str = "hybrid", retrieval_profile: str | None = None, allow_sensitive: bool = False, scope_allowlist: list[str] | None = None, provider: str | None = None, + trust_mode: str | None = None, ) -> ContextResult: """Return a formatted text block of the most relevant claims packed into *token_budget* using greedy knapsack. @@ -1638,19 +1695,25 @@ def query_for_context( provider: Optional context-packing strategy for a target LLM provider. """ - rows = self.query_rows( + resolved_trust = trust_mode or ( + "exploratory" + if include_stale or include_conflicted or include_candidates + else "trusted" + ) + retrieval = self.retrieve(RetrievalRequest( query_text=query, limit=limit, + trust_mode=resolved_trust, include_stale=include_stale, include_conflicted=include_conflicted, include_candidates=include_candidates, retrieval_mode=retrieval_mode, retrieval_profile=retrieval_profile, allow_sensitive=allow_sensitive, - scope_allowlist=scope_allowlist, - ) + scope_allowlist=tuple(scope_allowlist) if scope_allowlist else None, + )) return pack_context( - rows, + list(retrieval.rows), token_budget=token_budget, output_format=output_format, provider=provider, diff --git a/memorymaster/recall/planner.py b/memorymaster/recall/planner.py new file mode 100644 index 00000000..59def10b --- /dev/null +++ b/memorymaster/recall/planner.py @@ -0,0 +1,136 @@ +"""Governed retrieval request planning. + +The planner separates caller intent from the existing retrieval machinery. It +is deliberately pure and immutable so every public surface can share the same +trust/status and transport-containment decisions. +""" + +from __future__ import annotations + +from dataclasses import dataclass +import re + + +_TRUST_MODES = frozenset({"trusted", "exploratory"}) +_RETRIEVAL_MODES = frozenset({"legacy", "hybrid", "qdrant"}) +_STOPWORDS = frozenset( + { + "a", "an", "and", "are", "as", "at", "be", "by", "does", "for", + "from", "how", "in", "is", "it", "of", "on", "or", "that", "the", + "this", "to", "what", "when", "where", "which", "who", "why", "with", + } +) + + +@dataclass(frozen=True, slots=True) +class RetrievalRequest: + query_text: str + limit: int = 20 + trust_mode: str = "trusted" + retrieval_mode: str = "legacy" + include_stale: bool | None = None + include_conflicted: bool | None = None + include_candidates: bool | None = None + allow_sensitive: bool = False + scope_allowlist: tuple[str, ...] | None = None + requesting_agent: str | None = None + query_type: str | None = None + retrieval_profile: str | None = None + + +@dataclass(frozen=True, slots=True) +class RetrievalPlan: + query_text: str + search_text: str + limit: int + trust_mode: str + statuses: tuple[str, ...] + requested_mode: str + effective_mode: str + containment_reason: str | None + allow_sensitive: bool + scope_allowlist: tuple[str, ...] | None + requesting_agent: str | None + query_type: str | None + retrieval_profile: str | None + + +@dataclass(frozen=True, slots=True) +class RetrievalResult: + plan: RetrievalPlan + rows: tuple[dict[str, object], ...] + + +def _normalize_search_text(query_text: str) -> str: + tokens: list[str] = [] + seen: set[str] = set() + for token in re.findall(r"[\w-]+", query_text.lower(), flags=re.UNICODE): + if len(token) < 2 or token in _STOPWORDS or token in seen: + continue + seen.add(token) + tokens.append(token) + if len(tokens) == 12: + break + # Short keyword searches already have useful AND semantics. Conversational + # prompts accumulate filler and modifiers, so requiring every term causes + # false negatives; broaden only those longer requests. + if len(tokens) < 5: + return query_text.strip() + return " OR ".join(tokens) or query_text.strip() + + +def _statuses(request: RetrievalRequest) -> tuple[str, ...]: + if request.trust_mode == "trusted": + if any( + value is True + for value in ( + request.include_stale, + request.include_conflicted, + request.include_candidates, + ) + ): + raise ValueError("Expanded statuses require trust_mode='exploratory'.") + return ("confirmed",) + + statuses = ["confirmed"] + choices = ( + ("stale", request.include_stale), + ("conflicted", request.include_conflicted), + ("candidate", request.include_candidates), + ) + statuses.extend(status for status, enabled in choices if enabled is not False) + return tuple(statuses) + + +def build_retrieval_plan(request: RetrievalRequest) -> RetrievalPlan: + """Validate and resolve a retrieval request without performing I/O.""" + if request.limit <= 0: + raise ValueError("Retrieval limit must be positive.") + if request.trust_mode not in _TRUST_MODES: + raise ValueError(f"Unsupported trust mode: {request.trust_mode}") + if request.retrieval_mode not in _RETRIEVAL_MODES: + raise ValueError(f"Unsupported retrieval mode: {request.retrieval_mode}") + + effective_mode = request.retrieval_mode + containment_reason = None + if request.retrieval_mode == "qdrant": + effective_mode = "legacy" + containment_reason = ( + "qdrant retrieval is quarantined pending governed ID rehydration" + ) + + return RetrievalPlan( + query_text=request.query_text, + search_text=_normalize_search_text(request.query_text), + limit=request.limit, + trust_mode=request.trust_mode, + statuses=_statuses(request), + requested_mode=request.retrieval_mode, + effective_mode=effective_mode, + containment_reason=containment_reason, + allow_sensitive=request.allow_sensitive, + scope_allowlist=request.scope_allowlist, + requesting_agent=request.requesting_agent, + query_type=request.query_type, + retrieval_profile=request.retrieval_profile, + ) diff --git a/memorymaster/surfaces/mcp_server.py b/memorymaster/surfaces/mcp_server.py index 4ff79e15..ff38dcb7 100644 --- a/memorymaster/surfaces/mcp_server.py +++ b/memorymaster/surfaces/mcp_server.py @@ -1148,15 +1148,16 @@ def query_memory( workspace: str = ".", limit: int = 20, retrieval_mode: str = "legacy", + trust_mode: str = "trusted", auto_classify: bool = False, - include_stale: bool = True, - include_conflicted: bool = True, - include_candidates: bool = True, + include_stale: bool | None = None, + include_conflicted: bool | None = None, + include_candidates: bool | None = None, allow_sensitive: bool = False, scope_allowlist: str = "", detail_level: str = "standard", ) -> dict[str, Any]: - """Query memory for relevant claims. Includes candidates by default for MCP use. + """Query governed memory; trusted mode returns confirmed claims only. retrieval_mode options: - "legacy" (default, fastest ~0.1s): SQL text search @@ -1178,36 +1179,29 @@ def query_memory( context="mcp.query_memory", ) - requested_retrieval_mode = retrieval_mode classified_retrieval_mode: str | None = None query_type: str | None = None - containment_reason: str | None = None if auto_classify and retrieval_mode == "legacy": query_type = _classify(query) classified_retrieval_mode = recommended_retrieval_mode(query_type) retrieval_mode = classified_retrieval_mode - # Qdrant payloads cannot express the full tenant/scope/visibility - # policy. R1.3 keeps recall useful through the authoritative lexical - # planner until R2.1 can safely reintroduce vector ID candidates. - if retrieval_mode == "qdrant": - retrieval_mode = "legacy" - containment_reason = ( - "qdrant retrieval is quarantined pending the governed retrieval planner" - ) - + from memorymaster.recall.planner import RetrievalRequest svc = _service(db, workspace) - rows_data = svc.query_rows( + retrieval = svc.retrieve(RetrievalRequest( query_text=query, limit=limit, + trust_mode=trust_mode, retrieval_mode=retrieval_mode, include_stale=include_stale, include_conflicted=include_conflicted, include_candidates=include_candidates, allow_sensitive=allow_sensitive, - scope_allowlist=_effective_scope_allowlist(scope_allowlist, workspace), + scope_allowlist=tuple(_effective_scope_allowlist(scope_allowlist, workspace)), requesting_agent=_team_request_principal(), - ) + query_type=query_type, + )) + rows_data = list(retrieval.rows) # For "full" detail level, re-fetch each claim with citations inline. if detail_level == "full": rows_data = _enrich_claims_with_citations(svc, rows_data) @@ -1233,15 +1227,14 @@ def query_memory( "rows": len(claims), "claims": [_apply_detail_level(_claim_to_dict(c), detail_level) for c in claims], "rows_data": serialized_rows, + "trust_mode": retrieval.plan.trust_mode, + "requested_retrieval_mode": retrieval.plan.requested_mode, + "retrieval_mode": retrieval.plan.effective_mode, } if query_type is not None: response["query_type"] = query_type - if containment_reason is not None: - response.update({ - "requested_retrieval_mode": requested_retrieval_mode, - "retrieval_mode": retrieval_mode, - "containment_reason": containment_reason, - }) + if retrieval.plan.containment_reason is not None: + response["containment_reason"] = retrieval.plan.containment_reason if classified_retrieval_mode is not None: response["classified_retrieval_mode"] = classified_retrieval_mode @@ -1362,9 +1355,10 @@ def query_for_context( output_format: str = "text", limit: int = 100, retrieval_mode: str = "legacy", - include_stale: bool = True, - include_conflicted: bool = True, - include_candidates: bool = True, + trust_mode: str = "trusted", + include_stale: bool | None = None, + include_conflicted: bool | None = None, + include_candidates: bool | None = None, allow_sensitive: bool = False, scope_allowlist: str = "", detail_level: str = "standard", @@ -1397,6 +1391,7 @@ def query_for_context( include_conflicted=include_conflicted, include_candidates=include_candidates, retrieval_mode=retrieval_mode, + trust_mode=trust_mode, allow_sensitive=allow_sensitive, scope_allowlist=_effective_scope_allowlist(scope_allowlist, workspace), ) @@ -1412,16 +1407,19 @@ def query_for_context( if detail_level != "standard": # Attach filtered claim list as a convenience for callers who want # structured data alongside the formatted output block. - rows_data = svc.query_rows( + from memorymaster.recall.planner import RetrievalRequest + retrieval = svc.retrieve(RetrievalRequest( query_text=query, limit=limit, + trust_mode=trust_mode, retrieval_mode=retrieval_mode, include_stale=include_stale, include_conflicted=include_conflicted, include_candidates=include_candidates, allow_sensitive=allow_sensitive, - scope_allowlist=_effective_scope_allowlist(scope_allowlist, workspace), - ) + scope_allowlist=tuple(_effective_scope_allowlist(scope_allowlist, workspace)), + )) + rows_data = list(retrieval.rows) if detail_level == "full": filtered_claims = [] for row in rows_data: @@ -1449,9 +1447,10 @@ def volunteer_context( output_format: str = "text", limit: int = 100, retrieval_mode: str = "legacy", - include_stale: bool = True, - include_conflicted: bool = True, - include_candidates: bool = True, + trust_mode: str = "trusted", + include_stale: bool | None = None, + include_conflicted: bool | None = None, + include_candidates: bool | None = None, allow_sensitive: bool = False, scope_allowlist: str = "", detail_level: str = "standard", @@ -1486,16 +1485,19 @@ def volunteer_context( ) svc = _service(db, workspace) effective_allowlist = _effective_scope_allowlist(scope_allowlist, workspace) - rows = svc.query_rows( + from memorymaster.recall.planner import RetrievalRequest + retrieval = svc.retrieve(RetrievalRequest( query_text=query, limit=limit, + trust_mode=trust_mode, retrieval_mode=retrieval_mode, include_stale=include_stale, include_conflicted=include_conflicted, include_candidates=include_candidates, allow_sensitive=allow_sensitive, - scope_allowlist=effective_allowlist, - ) + scope_allowlist=tuple(effective_allowlist), + )) + rows = list(retrieval.rows) # Confidence gate — pure post-filter on the already-ranked, already # sensitivity-filtered rows. >= so min_confidence=0.0 keeps everything # (additive, no recall change vs query_for_context when the gate is open). diff --git a/tests/test_mcp_helpers.py b/tests/test_mcp_helpers.py index 161e4bb4..7ba3913a 100644 --- a/tests/test_mcp_helpers.py +++ b/tests/test_mcp_helpers.py @@ -298,6 +298,7 @@ def test_ingest_and_query_tools(self): query="Python indentation", db=self.db_path, workspace=self.workspace, + trust_mode="exploratory", ) assert query_result["ok"] is True assert query_result["rows"] >= 1 diff --git a/tests/test_push_volunteer_context.py b/tests/test_push_volunteer_context.py index 06097603..4cbfb549 100644 --- a/tests/test_push_volunteer_context.py +++ b/tests/test_push_volunteer_context.py @@ -84,6 +84,7 @@ def test_gate_excludes_low_confidence_claims(env): workspace=env["workspace"], min_confidence=0.8, detail_level="summary", + trust_mode="exploratory", ) assert result["ok"] is True confidences = [c["confidence"] for c in result.get("claims", [])] @@ -100,7 +101,13 @@ def test_open_gate_matches_query_for_context(env): _ingest(env, f"{QUERY_TOKEN} alpha claim", confidence=0.9) _ingest(env, f"{QUERY_TOKEN} beta claim", confidence=0.4) - common = dict(query=QUERY_TOKEN, db=env["db"], workspace=env["workspace"], token_budget=2000) + common = dict( + query=QUERY_TOKEN, + db=env["db"], + workspace=env["workspace"], + token_budget=2000, + trust_mode="exploratory", + ) volunteered = env["volunteer_context"](min_confidence=0.0, **common) baseline = env["query_for_context"](**common) @@ -122,6 +129,7 @@ def test_no_high_confidence_match_returns_empty_ok(env): db=env["db"], workspace=env["workspace"], min_confidence=0.9, + trust_mode="exploratory", ) assert result["ok"] is True # The graceful-empty contract: zero claims volunteered (the formatted block @@ -172,6 +180,7 @@ def cite(t): workspace=workspace, min_confidence=0.0, detail_level="standard", + trust_mode="exploratory", ) assert result["ok"] is True assert "[REDACTED:" not in (result["output"] or ""), "redacted-secret claim leaked into volunteered output" @@ -187,7 +196,12 @@ def test_no_side_effect_on_subsequent_query(env): """ _ingest(env, f"{QUERY_TOKEN} durable claim", confidence=0.5) - common = dict(query=QUERY_TOKEN, db=env["db"], workspace=env["workspace"]) + common = dict( + query=QUERY_TOKEN, + db=env["db"], + workspace=env["workspace"], + trust_mode="exploratory", + ) before = env["query_for_context"](**common) env["volunteer_context"](min_confidence=0.99, **common) # gate out everything after = env["query_for_context"](**common) diff --git a/tests/test_retrieval_planner_contract.py b/tests/test_retrieval_planner_contract.py new file mode 100644 index 00000000..a589ef1a --- /dev/null +++ b/tests/test_retrieval_planner_contract.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from dataclasses import FrozenInstanceError + +import pytest + +from memorymaster.recall.planner import RetrievalRequest, build_retrieval_plan + + +def test_trusted_plan_is_immutable_and_normalizes_conversational_search() -> None: + request = RetrievalRequest( + query_text="How does MemoryMaster explain governed claims and citations?", + retrieval_mode="legacy", + ) + + plan = build_retrieval_plan(request) + + assert plan.trust_mode == "trusted" + assert plan.statuses == ("confirmed",) + assert " OR " in plan.search_text + assert "how" not in plan.search_text.lower().split() + with pytest.raises(FrozenInstanceError): + request.limit = 99 # type: ignore[misc] + + +def test_trusted_mode_rejects_exploratory_status_expansion() -> None: + request = RetrievalRequest( + query_text="candidate", + trust_mode="trusted", + include_candidates=True, + ) + + with pytest.raises(ValueError, match="exploratory"): + build_retrieval_plan(request) + + +def test_exploratory_plan_carries_all_reviewable_statuses() -> None: + plan = build_retrieval_plan( + RetrievalRequest(query_text="candidate", trust_mode="exploratory") + ) + + assert plan.statuses == ("confirmed", "stale", "conflicted", "candidate") + + +def test_qdrant_request_remains_contained_until_governed_rehydration() -> None: + plan = build_retrieval_plan( + RetrievalRequest(query_text="semantic request", retrieval_mode="qdrant") + ) + + assert plan.requested_mode == "qdrant" + assert plan.effective_mode == "legacy" + assert plan.containment_reason diff --git a/tests/test_retrieval_surface_parity.py b/tests/test_retrieval_surface_parity.py index 7d613b6a..a45c7b1d 100644 --- a/tests/test_retrieval_surface_parity.py +++ b/tests/test_retrieval_surface_parity.py @@ -2,12 +2,15 @@ from __future__ import annotations -import pytest - from memorymaster.core.lifecycle import transition_claim from memorymaster.core.models import CitationInput from memorymaster.core.service import MemoryService -from memorymaster.surfaces.mcp_server import _project_scope, query_memory +from memorymaster.surfaces.mcp_server import ( + _project_scope, + query_for_context, + query_memory, + volunteer_context, +) def _service_with_claim(tmp_path, text: str, *, confirmed: bool) -> tuple[MemoryService, int]: @@ -31,10 +34,13 @@ def _service_with_claim(tmp_path, text: str, *, confirmed: bool) -> tuple[Memory return service, claim.id -@pytest.mark.xfail( - strict=True, - reason="R2.1: MCP legacy retrieval sends conversational prompts to FTS5 as raw AND terms", -) +def _claim_ids(result: dict) -> set[int]: + return { + int(claim.get("id", claim.get("claim_id"))) + for claim in result.get("claims", []) + } + + def test_conversational_recall_preserves_keyword_hits(tmp_path, monkeypatch): monkeypatch.delenv("QDRANT_URL", raising=False) service, claim_id = _service_with_claim( @@ -57,17 +63,13 @@ def test_conversational_recall_preserves_keyword_hits(tmp_path, monkeypatch): query="How does MemoryMaster explain governed claims and citations for durable recall?", **common, ) - keyword_ids = {claim["id"] for claim in keyword["claims"]} - conversational_ids = {claim["id"] for claim in conversational["claims"]} + keyword_ids = _claim_ids(keyword) + conversational_ids = _claim_ids(conversational) assert claim_id in keyword_ids assert claim_id in conversational_ids -@pytest.mark.xfail( - strict=True, - reason="R2.1: MCP query_memory defaults to exploratory candidate recall", -) def test_default_mcp_recall_excludes_provisional_claims(tmp_path, monkeypatch): monkeypatch.delenv("QDRANT_URL", raising=False) service, claim_id = _service_with_claim( @@ -81,7 +83,60 @@ def test_default_mcp_recall_excludes_provisional_claims(tmp_path, monkeypatch): db=str(service.store.db_path), workspace=str(tmp_path), ) - returned_ids = {claim["id"] for claim in result["claims"]} + returned_ids = _claim_ids(result) assert claim_id not in returned_ids assert all(claim["status"] == "confirmed" for claim in result["claims"]) + + +def test_exploratory_mode_is_explicit_and_annotated(tmp_path, monkeypatch): + monkeypatch.delenv("QDRANT_URL", raising=False) + service, claim_id = _service_with_claim( + tmp_path, + "exploratorycontracttoken is an unreviewed hypothesis", + confirmed=False, + ) + + result = query_memory( + query="exploratorycontracttoken", + db=str(service.store.db_path), + workspace=str(tmp_path), + trust_mode="exploratory", + ) + + rows = {row["claim"]["id"]: row for row in result["rows_data"]} + assert claim_id in rows + assert rows[claim_id]["annotation"]["active"] is False + assert rows[claim_id]["annotation"]["status"] == "candidate" + + +def test_context_surfaces_share_trusted_default_ids(tmp_path, monkeypatch): + monkeypatch.delenv("QDRANT_URL", raising=False) + service, confirmed_id = _service_with_claim( + tmp_path, + "surfaceparitytoken belongs to a confirmed governed claim", + confirmed=True, + ) + candidate = service.ingest( + text="surfaceparitytoken belongs to an unreviewed candidate", + citations=[CitationInput(source="test")], + scope=_project_scope(str(tmp_path)), + source_agent="retrieval-contract", + ) + common = { + "query": "surfaceparitytoken", + "db": str(service.store.db_path), + "workspace": str(tmp_path), + "detail_level": "summary", + } + + query_result = query_memory(**common) + context_result = query_for_context(**common) + volunteer_result = volunteer_context(**common, min_confidence=0.0) + + id_sets = [ + _claim_ids(result) + for result in (query_result, context_result, volunteer_result) + ] + assert id_sets == [{confirmed_id}, {confirmed_id}, {confirmed_id}] + assert candidate.id not in set().union(*id_sets) From 3e97e5b7430d2866fe0201a7787b6dcc8ec27c9a Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Mon, 13 Jul 2026 00:34:54 -0300 Subject: [PATCH 30/55] feat(recall): govern qdrant reintegration --- .planning/REMAINING-PHASES-V1-BUDGET.md | 20 +- .planning/audit-remediation-ledger.md | 4 +- external-actions-required.md | 2 +- memorymaster/core/service.py | 106 ++++++- memorymaster/govern/jobs/qdrant_reconcile.py | 49 ++- memorymaster/recall/planner.py | 3 +- memorymaster/recall/qdrant_backend.py | 137 +++++++- memorymaster/recall/qdrant_outbox.py | 124 ++++++++ memorymaster/surfaces/mcp_server.py | 8 +- tests/test_qdrant_backend.py | 5 +- tests/test_qdrant_governed_reintegration.py | 311 +++++++++++++++++++ tests/test_qdrant_retrieval_quarantine.py | 8 + 12 files changed, 751 insertions(+), 26 deletions(-) create mode 100644 memorymaster/recall/qdrant_outbox.py create mode 100644 tests/test_qdrant_governed_reintegration.py diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index 7c21f56e..11cb72ba 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -142,5 +142,21 @@ P2-A implementation and focused gate completed on 2026-07-12: - Focused gate: 141 passed, 1 expected xfail; FTS/planner regression slice: 32 passed; changed-file Ruff and `git diff --check` passed. -Next package is P2-B. Start it only in a separate goal/package; do not combine -governed Qdrant reintegration with P2-A. +P2-B implementation and focused gate completed on 2026-07-13: + +- Qdrant exposes bounded ID/hash/score candidates only; stored payloads no + longer duplicate claim text or provenance. +- Primary-store rehydration rejects orphan, archived, sensitive, wrong-scope, + wrong-tenant, private, provisional, stale-hash, and malformed candidates. +- Exact reconciliation detects equal-count ID/hash drift, and failed immediate + writes enter a bounded metadata-only replay outbox. +- Semantic reads remain default-off behind + `MEMORYMASTER_QDRANT_GOVERNED_READS`; team semantic reads remain denied. +- Focused fake-backed gate: 91 passed; explicit Qdrant ML gate: 38 passed; + focused new-code coverage recorded 92% for the outbox and 85% for the planner; + changed-file Ruff and `git diff --check` passed. +- Disposable authenticated/TLS Qdrant runtime parity is `BLOCKED-EXTERNAL` in + `external-actions-required.md`. + +Next package is P2-C. Start it only in a separate goal/package; do not combine +lifecycle/read-only recall changes with P2-B. diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index cc064d03..06a0c5ac 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -13,12 +13,12 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | ID | Primary domain | Severity / exploitability | Summary | Package | Status | Acceptance evidence | |---|---|---|---|---|---|---| | MM-SEC-01 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | MCP roles, project scopes, and Postgres tenant isolation are not enforced centrally | R1.1-R1.2 | BLOCKED-EXTERNAL | Owner: Codex remediation branch. R1.1 complete. PostgreSQL application connections are team-only and bind tenant, principal, and immutable scopes transaction-locally; schema work requires a distinct verified migrator. v0011 FORCE RLS defines restricted role/table/policy and append-only event contracts, a tenant-derived hash-only event-head function, and deny-only governance/raw tables. v0012 defines six partial unique indexes: public keys are tenant + exact-scope local; non-public keys additionally include exact visibility/principal; ambiguity without exact scope fails closed; every team claim requires a nonblank owner. Startup validates literal-sensitive policy/index/function fingerprints, exact event and claims trigger inventories, required event SELECT/INSERT plus forbidden table/column UPDATE and DELETE, a privileged event-head owner, and the strict validated owner constraint. Supersession rejects self/cross-tenant/scope/visibility/owner references and the canonical path atomically commits reciprocal pointers plus one event; v0012 preflights unsafe legacy edges. Unsupported Postgres source/evidence/action/retry, read-only, merge, delta, and tenant-bound CLI surfaces now fail before driver or filesystem access; whitespace-wrapped DSNs cannot bypass routing. The parity harness requires two distinct roles plus disposable opt-in, uses UUID tenant namespaces, and performs no destructive cleanup. Closure matrix: 349 passed, 47 externally gated skips; direct surface/factory matrix: 57 passed. Full isolated non-ML gate: 3,552 passed, 69 skipped, 95 deselected, 22 intentional xfails, 2 warnings in 858.47s. Commit evidence is the commit containing this row. Rollback: keep the team profile disabled and revert this package; schema rollback requires a verified backup/forward repair. Repository work for R1.1-R1.2 is complete. Real two-role PostgreSQL evidence plus approved brownfield owner/duplicate/supersession-edge inventory/repair remain `BLOCKED-EXTERNAL`; the Team/Postgres profile remains blocked. | -| MM-SEC-02 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | Qdrant can return archived, sensitive, cross-scope/tenant, or orphan payloads | R1.3,R2.1 | IN-PROGRESS | Owner: Codex remediation branch. R1.3 containment is repository-complete: local-trusted claim requests and auto-classified Qdrant recommendations fall back to authoritative lexical retrieval with requested/classified/effective metadata; team semantic requests are denied before tool dispatch; prompt-context fallback is disconnected; verbatim vector/hybrid requests use FTS5; CLI denial occurs before service/backend construction; and every direct claim/verbatim/fallback read adapter raises before model, network, or raw payload access. Qdrant upsert/sync/reconcile/count-ID maintenance remains available. Adversarial containment: 18 passed; Qdrant/verbatim/classifier matrix: 153 passed; CLI/setup/MCP regression matrix: 182 passed; explicit ML gate: 14 passed. Full isolated non-ML gate: 3,572 passed, 69 skipped, 95 deselected, 20 intentional xfails, 2 warnings in 882.06s; collection: 3,756; Ruff: clean. Independent blocker-only review found no first-party raw payload-search path. Commit evidence is the commit containing this row. Rollback: keep semantic retrieval disabled and revert this package. R2.1 governed ID-candidate rehydration remains unimplemented, and final authenticated/TLS Qdrant parity is `BLOCKED-EXTERNAL`; do not enable the semantic profile. | +| MM-SEC-02 | Security/Database | Critical / EXPLOITABLE-NOW / H1 | Qdrant can return archived, sensitive, cross-scope/tenant, or orphan payloads | R1.3,R2.1 | BLOCKED-EXTERNAL | R1.3 containment and R2.1 repository work are complete. Qdrant reads are default-off and expose only validated claim ID/content-hash/score candidates; every candidate is rehydrated from the authoritative store and checked for lifecycle status, tenant, scope, sensitivity, visibility, and exact content hash before construction. Stored vector payloads are reduced to ID/hash metadata. Equal-count reconciliation detects stale, missing, sensitive, and orphan points, while failed immediate writes enter a bounded metadata-only replay outbox. Team semantic reads remain denied. Fake-backed gate: 91 passed; explicit Qdrant ML gate: 38 passed; changed-file Ruff and diff checks clean. Commit evidence is the commit containing this row. Disposable authenticated/TLS real-service parity remains `BLOCKED-EXTERNAL`; keep `MEMORYMASTER_QDRANT_GOVERNED_READS` disabled until that evidence exists. | | MM-OPS-01 | DevOps | Critical / EXPLOITABLE-LOW-EFFORT / H4 | Postgres Compose publishes a fixed credential | R1.5 | BLOCKED-EXTERNAL | Repository defaults are fail-closed and private (`a858419`); Compose rejects missing inputs and renders with synthetic required inputs. Historical credential rotation/recreation and a real network probe require operator action. | | MM-SEC-03 | Security | High / EXPLOITABLE-NOW | Persisted metadata/provenance fields bypass sensitivity scanning | R1.4 | BLOCKED-EXTERNAL | Repository gateways and the aggregate-only inventory are complete (`a3e3824`, `702b59d`, `8d80abb`); focused final gate: 46 passed, 1 skipped. The authorized live read-only inventory found legacy sensitive/unscannable material, while Qdrant was unavailable; cleanup/redaction and product-data mutation remain forbidden without approval. | | MM-SEC-04 | Security | High / EXPLOITABLE-NOW | Steward, compact-summary, verbatim, and integration writes bypass one gateway | R1.4 | RESOLVED | Durable auxiliary writers now use the shared sensitivity envelope and adversarial persistence matrices; the final observed compatibility failures were corrected and the exact 10-test failure set passed. Roll back by reverting the R1.4 commits while keeping affected ingestion disabled. | | MM-ARCH-01 | Architecture/Database | High / BAD-PRACTICE | Entity registry and graph own incompatible `entities` schemas | R2.3 | OPEN | Normal init-to-graph MCP integration and FK check | -| MM-ARCH-02 | Architecture/Integrity | High / BAD-PRACTICE | MCP/hooks/CLI/Qdrant use contradictory retrieval planners and trust defaults | R2.1 | OPEN | Cross-surface ID-set and conversational-query parity | +| MM-ARCH-02 | Architecture/Integrity | High / BAD-PRACTICE | MCP/hooks/CLI/Qdrant use contradictory retrieval planners and trust defaults | R2.1 | RESOLVED | P2-A/P2-B route governed lexical and gated semantic recall through one immutable planner, confirmed-only trusted defaults, explicit exploratory expansion, cross-surface ID parity, and authoritative Qdrant rehydration. Focused P2-A gate: 141 passed, 1 expected xfail; P2-B fake-backed gate: 91 passed. | | MM-LIFE-01 | Database/Reliability | High / BAD-PRACTICE | Scheduled archival uses raw SQL and bypasses lifecycle/vector evidence | R2.2 | OPEN | Version/event/timestamp/outbox assertions through scheduled path | | MM-REL-02 | Reliability/Performance | Medium / BAD-PRACTICE | MCP reads take write locks and context detail modes retrieve twice | R2.2 | OPEN | Zero query write lock; one aggregated telemetry envelope | | MM-OPS-02 | DevOps/Demo | High / BAD-PRACTICE | Docker/Helm publish HTTP on a stdio MCP process and use an invalid healthcheck | R3.4 | OPEN | Built-container readiness and MCP handshake | diff --git a/external-actions-required.md b/external-actions-required.md index 40f8f6e0..663c34a9 100644 --- a/external-actions-required.md +++ b/external-actions-required.md @@ -11,7 +11,7 @@ live mutation by itself. | MM-SEC-01 / disposable proof | Team Postgres test environment | Provide one fully disposable database, a dedicated schema-owning `SUPERUSER`/`BYPASSRLS` migrator DSN in `MEMORYMASTER_TEST_POSTGRES_DSN`, and a distinct restricted app DSN in `MEMORYMASTER_TEST_POSTGRES_APP_DSN`; set `MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE=1`. Neither DSN may target product/live data. | Run `tests/test_postgres_rls_integration.py` plus every Postgres parameter in `tests/test_postgres_parity.py` and `tests/test_backend_parity.py`. Retain exact FORCE-RLS policy/function/trigger/index catalog evidence; literal-sensitive fingerprints; event-head owner `SUPERUSER`/`BYPASSRLS`; app event SELECT/INSERT with no table/column UPDATE or DELETE; cross-tenant/scope/visibility/principal denials; tenant-chain linearity; scope-local identities and ambiguity rejection; self/cross-boundary supersession denial; successful reciprocal supersession plus one event; rollback after replacement-update/event-insert failure; and parity results from UUID-isolated tenants with no destructive cleanup. No real PostgreSQL proof has been performed here. | 2026-07-31 | BLOCKED-EXTERNAL | | MM-SEC-01 / brownfield data | Live database operator | Authorize a read-only inventory before v0012. Any owner backfill, duplicate/supersession-edge repair, or grant change requires separate explicit approval, a verified backup, and a maintenance window. | Inventory noncanonical visibility; blank/null `source_agent` on every claim; identity duplicates; both supersession columns for self, missing-target, nonreciprocal, cross-tenant/scope/visibility/owner edges; current event/function/table grants; and constraint state without writing. After approval, retain repair/grant evidence and rerun v0012, which validates `ck_claims_identity_visibility_owner`. No product-data verification or mutation has been performed here. | 2026-07-31 | BLOCKED-EXTERNAL | | MM-OPS-02 | Docker/Helm runtime | Provide approved immutable MemoryMaster/Qdrant/Ollama images, certificates, and a disposable Kubernetes target. Docker/Compose config was available locally; Helm and approved product images were not. | Container health/MCP handshake, authenticated TLS Qdrant health, and Helm readiness/network-policy evidence | 2026-08-15 | BLOCKED-EXTERNAL | -| MM-SEC-02 | Qdrant runtime | Provide a disposable authenticated/TLS Qdrant target for final parity after the fake-backed containment suite. Do not point the tests at a live/product collection. | API-key/TLS connection proof plus real-service sync/reconcile/count-ID evidence and denial of every payload-search adapter; retain exact command output without secrets | 2026-08-15 | BLOCKED-EXTERNAL | +| MM-SEC-02 | Qdrant runtime | Provide a disposable authenticated/TLS Qdrant target for final parity after the fake-backed governed candidate suite. The semantic profile remains off by default; do not point tests at a live/product collection. | API-key/TLS proof plus real-service ID/hash-only search, authoritative rehydration denials, exact equal-count hash reconciliation, bounded outbox replay, and sync/delete evidence; retain exact command output without secrets | 2026-08-15 | BLOCKED-EXTERNAL | | MM-SEC-03 | Live legacy stores | Review the aggregate inventory, provide a verified backup, and separately authorize any cleanup/redaction/rebuild. Configure a disposable or approved read-only Qdrant inventory target before claiming complete coverage. | SQLite/artifact/spool/Qdrant aggregate rerun with zero unscannable/truncated surfaces, plus approved remediation evidence; never include raw findings | 2026-07-31 | BLOCKED-EXTERNAL | | MM-OPS-04 / history scan | Security owner | Classify the unsuppressed Gitleaks history result: 40 potential findings across 10 commits and 7 files. Rotate any affected credentials before considering an approved history action. | Aggregate classification and rotation record; no raw secret material in general logs | 2026-07-31 | BLOCKED-EXTERNAL | | MM-OPS-04 / dependency audit | Release owner / advisory service | Rerun strict project and `mcp,qdrant,security` extra audits in an approved lock/resolver environment; both final attempts exceeded the 15-minute external-service cap and were terminated without a pass claim. | Successful strict PyPI audit bound to the release lock/artifact, or reviewed vulnerability remediation evidence | 2026-07-31 | BLOCKED-EXTERNAL | diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index b4cabf18..59c5bec1 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -20,6 +20,7 @@ from memorymaster.core.config import get_config from memorymaster.recall.retrieval import VectorSearchHook, _tier_bonus, rank_claim_rows from memorymaster.recall.planner import ( + RetrievalPlan, RetrievalRequest, RetrievalResult, build_retrieval_plan, @@ -443,16 +444,32 @@ def _init_qdrant(): return None def _qdrant_sync(self, claim: Claim) -> None: - """Fire-and-forget upsert to Qdrant after a claim state change.""" + """Attempt a vector write and durably enqueue failures for replay.""" if self.qdrant is None: return + operation = "delete" if claim.status == "archived" else "upsert" try: - if claim.status == "archived": + completed = ( self.qdrant.delete_claim(claim.id) - else: - self.qdrant.upsert_claim(claim) + if operation == "delete" + else self.qdrant.upsert_claim(claim) + ) + if completed: + return except Exception as exc: logger.warning("Qdrant sync failed for claim %d: %s", claim.id, exc) + try: + from memorymaster.recall import qdrant_outbox + from memorymaster.recall.qdrant_backend import claim_content_hash + + content_hash = claim_content_hash(claim) if operation == "upsert" else None + db_path = getattr(self.store, "db_path", None) + if db_path is not None and not qdrant_outbox.enqueue( + db_path, operation, claim.id, content_hash + ): + logger.error("Qdrant outbox rejected operation for claim %d", claim.id) + except Exception as exc: + logger.error("Qdrant outbox unavailable for claim %d: %s", claim.id, exc) def _qdrant_post_cycle_sync(self) -> None: """After a lifecycle cycle, push recently-changed claims to Qdrant.""" @@ -1021,6 +1038,8 @@ def retrieve( ) -> RetrievalResult: """Execute one immutable governed retrieval plan.""" plan = build_retrieval_plan(request) + if plan.effective_mode == "qdrant": + return RetrievalResult(plan=plan, rows=tuple(self._query_qdrant_mode(plan))) statuses = set(plan.statuses) rows = self.query_rows( query_text=plan.search_text, @@ -1038,6 +1057,85 @@ def retrieve( ) return RetrievalResult(plan=plan, rows=tuple(rows)) + def _query_qdrant_mode(self, plan: RetrievalPlan) -> list[dict[str, Any]]: + """Rehydrate untrusted vector IDs and reapply authoritative policy.""" + if self.qdrant is None: + return [] + requesting_agent = self._effective_requesting_agent(plan.requesting_agent) + if requesting_agent: + from memorymaster.core.access_control import require_permission + + require_permission(requesting_agent, "query") + self._allow_sensitive( + allow_sensitive=plan.allow_sensitive, + context="service.query_qdrant", + deny_mode="filter", + ) + scopes = self._effective_scope_allowlist( + list(plan.scope_allowlist) if plan.scope_allowlist else None + ) + overfetch = min(max(plan.limit * 4, 20), 200) + try: + candidates = self.qdrant.search_candidates(plan.query_text, limit=overfetch) + except Exception: + logger.warning("Governed Qdrant candidate search failed") + return [] + + rows: list[dict[str, Any]] = [] + seen: set[int] = set() + for candidate in candidates: + claim = self._authoritative_qdrant_claim( + candidate, plan, scopes, requesting_agent + ) + if claim is None or claim.id in seen: + continue + seen.add(claim.id) + rows.append({ + "claim": claim, + "status": claim.status, + "annotation": self._annotation_for_claim(claim), + "score": candidate.score, + "lexical_score": 0.0, + "freshness_score": 0.0, + "confidence_score": claim.confidence, + "vector_score": candidate.score, + "breakdown": {"vector": candidate.score}, + }) + if len(rows) == plan.limit: + break + self._record_accesses(rows, query_text=plan.query_text) + return rows + + def _authoritative_qdrant_claim( + self, + candidate: object, + plan: RetrievalPlan, + scopes: list[str] | None, + requesting_agent: str | None, + ) -> Claim | None: + """Resolve one validated vector reference against primary truth.""" + from memorymaster.recall.qdrant_backend import ( + QdrantCandidate, + claim_content_hash, + ) + + if not isinstance(candidate, QdrantCandidate): + return None + claim = self.store.get_claim(candidate.claim_id, include_citations=True) + if claim is None or claim.status not in plan.statuses: + return None + if self.tenant_id is not None and claim.tenant_id != self.tenant_id: + return None + if scopes is not None and claim.scope not in scopes: + return None + if is_sensitive_claim(claim): + return None + if not _filter_agent_visibility([claim], requesting_agent): + return None + if claim_content_hash(claim) != candidate.content_hash: + return None + return claim + def query_rules( self, query_text: str, diff --git a/memorymaster/govern/jobs/qdrant_reconcile.py b/memorymaster/govern/jobs/qdrant_reconcile.py index 4376bfbe..63a68777 100644 --- a/memorymaster/govern/jobs/qdrant_reconcile.py +++ b/memorymaster/govern/jobs/qdrant_reconcile.py @@ -33,6 +33,9 @@ from memorymaster.stores._storage_shared import connect_ro from memorymaster.govern.jobs.integrity import _due, _record +from memorymaster.core.security import is_sensitive_claim +from memorymaster.recall import qdrant_outbox +from memorymaster.recall.qdrant_backend import claim_content_hash logger = logging.getLogger(__name__) @@ -106,6 +109,30 @@ def _delete_orphan_points(store, qdrant, db_path: str | Path | None) -> int: return deleted +def _exact_reconcile(store, qdrant, db_path: str | Path | None) -> dict[str, int]: + """Converge point IDs and content hashes without trusting vector payloads.""" + if not hasattr(qdrant, "list_point_refs"): + return {"checked": 0, "upserted": 0, "deleted": 0} + refs = qdrant.list_point_refs() + if refs is None: + return {"checked": 0, "upserted": 0, "deleted": 0} + point_ids: set[int] = set() + upserted = 0 + deleted = 0 + for ref in refs: + point_ids.add(ref.claim_id) + claim = store.get_claim(ref.claim_id, include_citations=True) + if claim is None or claim.status == "archived" or is_sensitive_claim(claim): + deleted += int(bool(qdrant.delete_claim(ref.claim_id))) + elif claim_content_hash(claim) != ref.content_hash: + upserted += int(bool(qdrant.upsert_claim(claim))) + for claim_id in _live_claim_ids(store, db_path) - point_ids: + claim = store.get_claim(claim_id, include_citations=True) + if claim is not None and not is_sensitive_claim(claim): + upserted += int(bool(qdrant.upsert_claim(claim))) + return {"checked": len(refs), "upserted": upserted, "deleted": deleted} + + def run( store, qdrant, @@ -150,11 +177,29 @@ def run( "upserted": 0, "deleted": 0, } + try: + exact = _exact_reconcile(store, qdrant, db_path) + except Exception: + logger.warning("qdrant exact reconciliation failed") + exact = {"checked": 0, "upserted": 0, "deleted": 0} + result["exact_reconcile"] = exact + result["upserted"] = exact["upserted"] + result["deleted"] = exact["deleted"] + if db_path is not None: + try: + result["outbox"] = qdrant_outbox.replay( + db_path, store, qdrant, max_operations=500 + ) + except Exception: + logger.warning("qdrant outbox replay failed") + result["outbox"] = {"attempted": 0, "completed": 0, "remaining": -1} if full or drift > limit: - result["deleted"] = _delete_orphan_points(store, qdrant, db_path) + result["deleted"] = int(result["deleted"]) + _delete_orphan_points( + store, qdrant, db_path + ) stats = qdrant.sync_all(store) result["synced"] = True - result["upserted"] = int(stats.get("synced", 0)) + result["upserted"] = int(result["upserted"]) + int(stats.get("synced", 0)) result["sync_stats"] = dict(stats) logger.info( "qdrant reconcile: drift=%d threshold=%d full=%s — sync_all ran: %s", diff --git a/memorymaster/recall/planner.py b/memorymaster/recall/planner.py index 59def10b..67af74f4 100644 --- a/memorymaster/recall/planner.py +++ b/memorymaster/recall/planner.py @@ -36,6 +36,7 @@ class RetrievalRequest: requesting_agent: str | None = None query_type: str | None = None retrieval_profile: str | None = None + qdrant_candidate_reads: bool = False @dataclass(frozen=True, slots=True) @@ -113,7 +114,7 @@ def build_retrieval_plan(request: RetrievalRequest) -> RetrievalPlan: effective_mode = request.retrieval_mode containment_reason = None - if request.retrieval_mode == "qdrant": + if request.retrieval_mode == "qdrant" and not request.qdrant_candidate_reads: effective_mode = "legacy" containment_reason = ( "qdrant retrieval is quarantined pending governed ID rehydration" diff --git a/memorymaster/recall/qdrant_backend.py b/memorymaster/recall/qdrant_backend.py index 4f55da1c..5e1b2f55 100644 --- a/memorymaster/recall/qdrant_backend.py +++ b/memorymaster/recall/qdrant_backend.py @@ -15,9 +15,14 @@ from __future__ import annotations import logging +import hashlib +import json +import math import os +import re import time import uuid +from dataclasses import dataclass from typing import Any import httpx @@ -36,6 +41,40 @@ OLLAMA_TIMEOUT = 120.0 MAX_RETRIES = 2 RETRY_BASE_DELAY = 0.5 +_SHA256_RE = re.compile(r"[0-9a-f]{64}") + + +@dataclass(frozen=True, slots=True) +class QdrantCandidate: + """Untrusted vector match reduced to the only fields recall may consume.""" + + claim_id: int + content_hash: str + score: float + + +def claim_content_hash(claim: Claim) -> str: + """Stable digest of the authoritative fields represented by one vector.""" + representation = { + "text": claim.text, + "subject": claim.subject, + "predicate": claim.predicate, + "object_value": claim.object_value, + "claim_type": claim.claim_type, + "holder": claim.holder, + "scope": claim.scope, + "status": claim.status, + "tenant_id": claim.tenant_id, + "visibility": claim.visibility, + "source_agent": claim.source_agent, + } + encoded = json.dumps( + representation, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ).encode("utf-8") + return hashlib.sha256(encoded).hexdigest() def _create_http_clients( @@ -171,17 +210,10 @@ def _claim_text(claim: Claim) -> str: @staticmethod def _claim_payload(claim: Claim, source: str = "memorymaster") -> dict[str, Any]: + del source return { "claim_id": claim.id, - "subject": claim.subject or "", - "predicate": claim.predicate or "", - "object": claim.object_value or "", - "claim_text": claim.text, - "state": claim.status, - "confidence": claim.confidence, - "source": source, - "created_at": claim.created_at, - "workspace": "main", + "content_hash": claim_content_hash(claim), } @staticmethod @@ -305,6 +337,93 @@ def list_point_claim_ids(self, *, batch_size: int = 1000) -> list[int] | None: logger.warning("Qdrant scroll failed") return None + def list_point_refs(self, *, batch_size: int = 1000) -> list[QdrantCandidate] | None: + """Scroll validated claim ID/hash pairs for exact reconciliation.""" + refs: list[QdrantCandidate] = [] + offset: Any = None + try: + while True: + body: dict[str, Any] = { + "limit": batch_size, + "with_payload": ["claim_id", "content_hash"], + "with_vector": False, + } + if offset is not None: + body["offset"] = offset + resp = self._qdrant_client.post( + f"{self.qdrant_url}/collections/{self.collection}/points/scroll", + json=body, + ) + resp.raise_for_status() + result = resp.json().get("result") or {} + for point in result.get("points", []): + candidate = self._candidate_from_point(point, default_score=0.0) + if candidate is not None: + refs.append(candidate) + offset = result.get("next_page_offset") + if offset is None: + return refs + except Exception: + logger.warning("Qdrant reference scroll failed") + return None + + @staticmethod + def _candidate_from_point( + point: object, + *, + default_score: float | None = None, + ) -> QdrantCandidate | None: + if not isinstance(point, dict): + return None + payload = point.get("payload") + if not isinstance(payload, dict): + return None + claim_id = payload.get("claim_id") + content_hash = payload.get("content_hash") + score = point.get("score", default_score) + if isinstance(claim_id, bool) or not isinstance(claim_id, int) or claim_id <= 0: + return None + if not isinstance(content_hash, str) or _SHA256_RE.fullmatch(content_hash) is None: + return None + if isinstance(score, bool) or not isinstance(score, (int, float)): + return None + numeric_score = float(score) + if not math.isfinite(numeric_score): + return None + return QdrantCandidate(claim_id, content_hash, numeric_score) + + def search_candidates(self, query_text: str, *, limit: int) -> list[QdrantCandidate]: + """Return validated ID/hash candidates; never return raw payload data.""" + if limit <= 0 or limit > 1000: + raise ValueError("Qdrant candidate limit must be between 1 and 1000.") + vector = self._embed(query_text) + if vector is None: + return [] + body = { + "query": vector, + "limit": limit, + "with_payload": ["claim_id", "content_hash"], + "with_vector": False, + } + try: + resp = self._qdrant_client.post( + f"{self.qdrant_url}/collections/{self.collection}/points/query", + json=body, + ) + resp.raise_for_status() + result = resp.json().get("result") or {} + points = result.get("points", []) if isinstance(result, dict) else result + if not isinstance(points, list): + return [] + return [ + candidate + for point in points + if (candidate := self._candidate_from_point(point)) is not None + ] + except Exception: + logger.warning("Qdrant candidate search failed") + return [] + def search( self, query_text: str, diff --git a/memorymaster/recall/qdrant_outbox.py b/memorymaster/recall/qdrant_outbox.py new file mode 100644 index 00000000..e98258c5 --- /dev/null +++ b/memorymaster/recall/qdrant_outbox.py @@ -0,0 +1,124 @@ +"""Bounded metadata-only outbox for replayable Qdrant maintenance writes.""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import re +import threading + +from memorymaster.core.security import is_sensitive_claim + + +ENV_OUTBOX_DIR = "MEMORYMASTER_QDRANT_OUTBOX_DIR" +DEFAULT_MAX_ENTRIES = 10_000 +DEFAULT_MAX_BYTES = 4 * 1024 * 1024 +_HASH_RE = re.compile(r"[0-9a-f]{64}") +_LOCK = threading.Lock() + + +def _path(db_path: str | Path) -> Path: + source = Path(db_path) + resolved = str(source.resolve()) + digest = hashlib.sha256(resolved.encode("utf-8")).hexdigest()[:12] + root = Path(os.environ.get(ENV_OUTBOX_DIR) or Path.home() / ".memorymaster" / "qdrant-outbox") + return root / f"{source.name}-{digest}.jsonl" + + +def _valid(entry: object) -> bool: + if not isinstance(entry, dict) or set(entry) != {"op", "claim_id", "content_hash"}: + return False + claim_id = entry.get("claim_id") + operation = entry.get("op") + content_hash = entry.get("content_hash") + if isinstance(claim_id, bool) or not isinstance(claim_id, int) or claim_id <= 0: + return False + if operation == "delete": + return content_hash is None + return operation == "upsert" and isinstance(content_hash, str) and _HASH_RE.fullmatch(content_hash) is not None + + +def pending(db_path: str | Path) -> list[dict[str, object]]: + path = _path(db_path) + if not path.exists(): + return [] + entries: list[dict[str, object]] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + try: + entry = json.loads(line) + except (TypeError, ValueError): + continue + if _valid(entry): + entries.append(entry) + return entries + + +def enqueue( + db_path: str | Path, + operation: str, + claim_id: int, + content_hash: str | None, + *, + max_entries: int = DEFAULT_MAX_ENTRIES, + max_bytes: int = DEFAULT_MAX_BYTES, +) -> bool: + """Append one operation, returning False when validation/bounds reject it.""" + entry = {"op": operation, "claim_id": claim_id, "content_hash": content_hash} + if not _valid(entry) or max_entries <= 0 or max_bytes <= 0: + return False + line = json.dumps(entry, sort_keys=True, separators=(",", ":")) + "\n" + path = _path(db_path) + with _LOCK: + existing = pending(db_path) + size = path.stat().st_size if path.exists() else 0 + if len(existing) >= max_entries or size + len(line.encode("utf-8")) > max_bytes: + return False + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("a", encoding="utf-8") as handle: + handle.write(line) + return True + + +def _apply(entry: dict[str, object], store, backend) -> bool: + claim_id = int(entry["claim_id"]) + if entry["op"] == "delete": + return bool(backend.delete_claim(claim_id)) + claim = store.get_claim(claim_id, include_citations=True) + if claim is None or claim.status == "archived" or is_sensitive_claim(claim): + return bool(backend.delete_claim(claim_id)) + return bool(backend.upsert_claim(claim)) + + +def replay( + db_path: str | Path, + store, + backend, + *, + max_operations: int = 500, +) -> dict[str, int]: + """Replay a bounded prefix and atomically retain failed/unattempted work.""" + if max_operations <= 0: + raise ValueError("max_operations must be positive") + with _LOCK: + entries = pending(db_path) + attempted = min(len(entries), max_operations) + retained: list[dict[str, object]] = [] + completed = 0 + for index, entry in enumerate(entries): + try: + applied = index < attempted and _apply(entry, store, backend) + except Exception: + applied = False + if not applied: + retained.append(entry) + else: + completed += 1 + path = _path(db_path) + path.parent.mkdir(parents=True, exist_ok=True) + temp = path.with_suffix(".tmp") + payload = "".join(json.dumps(item, sort_keys=True, separators=(",", ":")) + "\n" for item in retained) + temp.write_text(payload, encoding="utf-8") + os.replace(temp, path) + return {"attempted": attempted, "completed": completed, "remaining": len(retained)} diff --git a/memorymaster/surfaces/mcp_server.py b/memorymaster/surfaces/mcp_server.py index ff38dcb7..b4992919 100644 --- a/memorymaster/surfaces/mcp_server.py +++ b/memorymaster/surfaces/mcp_server.py @@ -1179,6 +1179,7 @@ def query_memory( context="mcp.query_memory", ) + requested_retrieval_mode = retrieval_mode classified_retrieval_mode: str | None = None query_type: str | None = None if auto_classify and retrieval_mode == "legacy": @@ -1200,6 +1201,11 @@ def query_memory( scope_allowlist=tuple(_effective_scope_allowlist(scope_allowlist, workspace)), requesting_agent=_team_request_principal(), query_type=query_type, + qdrant_candidate_reads=( + retrieval_mode == "qdrant" + and os.environ.get("MEMORYMASTER_QDRANT_GOVERNED_READS", "").strip().lower() + in {"1", "true", "yes", "on"} + ), )) rows_data = list(retrieval.rows) # For "full" detail level, re-fetch each claim with citations inline. @@ -1228,7 +1234,7 @@ def query_memory( "claims": [_apply_detail_level(_claim_to_dict(c), detail_level) for c in claims], "rows_data": serialized_rows, "trust_mode": retrieval.plan.trust_mode, - "requested_retrieval_mode": retrieval.plan.requested_mode, + "requested_retrieval_mode": requested_retrieval_mode, "retrieval_mode": retrieval.plan.effective_mode, } if query_type is not None: diff --git a/tests/test_qdrant_backend.py b/tests/test_qdrant_backend.py index a87ede19..bdd1174b 100644 --- a/tests/test_qdrant_backend.py +++ b/tests/test_qdrant_backend.py @@ -54,10 +54,7 @@ def test_claim_payload_structure(self): claim = _fake_claim() payload = QdrantBackend._claim_payload(claim, source="test") assert payload["claim_id"] == 1 - assert payload["state"] == "confirmed" - assert payload["confidence"] == 0.85 - assert payload["source"] == "test" - assert payload["workspace"] == "main" + assert set(payload) == {"claim_id", "content_hash"} def test_point_id_is_deterministic(self): id1 = QdrantBackend._point_id(42) diff --git a/tests/test_qdrant_governed_reintegration.py b/tests/test_qdrant_governed_reintegration.py new file mode 100644 index 00000000..e9319ecc --- /dev/null +++ b/tests/test_qdrant_governed_reintegration.py @@ -0,0 +1,311 @@ +"""Adversarial contracts for governed Qdrant candidate reintegration.""" + +from __future__ import annotations + +from dataclasses import replace +from unittest.mock import MagicMock + +import pytest + +import memorymaster.surfaces.mcp_server as mcp_server +from memorymaster.core.lifecycle import transition_claim +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.govern.jobs import qdrant_reconcile +from memorymaster.recall.planner import RetrievalRequest, build_retrieval_plan +from memorymaster.recall.qdrant_backend import ( + EMBEDDING_DIMS, + QdrantBackend, + QdrantCandidate, + claim_content_hash, +) +from memorymaster.recall import qdrant_outbox + + +class FakeCandidates: + def __init__(self, candidates: list[QdrantCandidate]) -> None: + self.candidates = candidates + self.limits: list[int] = [] + + def search_candidates(self, query_text: str, *, limit: int) -> list[QdrantCandidate]: + assert query_text + self.limits.append(limit) + return self.candidates[:limit] + + +def _seed(svc: MemoryService, text: str, *, scope: str = "project:allowed"): + claim = svc.ingest( + text, + [CitationInput(source="test://qdrant-governed")], + scope=scope, + source_agent="seed", + ) + transition_claim( + svc.store, + claim.id, + "confirmed", + reason="governed qdrant fixture", + event_type="validator", + ) + return svc.store.get_claim(claim.id, include_citations=True) + + +def _candidate(claim, *, content_hash: str | None = None, score: float = 0.9): + return QdrantCandidate( + claim_id=claim.id, + content_hash=content_hash or claim_content_hash(claim), + score=score, + ) + + +def test_qdrant_requires_explicit_candidate_read_gate() -> None: + contained = build_retrieval_plan( + RetrievalRequest(query_text="semantic", retrieval_mode="qdrant") + ) + enabled = build_retrieval_plan( + RetrievalRequest( + query_text="semantic", + retrieval_mode="qdrant", + qdrant_candidate_reads=True, + ) + ) + + assert contained.effective_mode == "legacy" + assert contained.containment_reason + assert enabled.effective_mode == "qdrant" + assert enabled.containment_reason is None + + +def test_vector_hash_tracks_authoritative_vector_and_policy_fields_only(tmp_path) -> None: + svc = MemoryService(tmp_path / "hash.db", workspace_root=tmp_path) + svc.init_db() + claim = _seed(svc, "stable vector hash representation") + + assert claim_content_hash(claim) == claim_content_hash(replace(claim, citations=[])) + assert claim_content_hash(claim) != claim_content_hash(replace(claim, status="stale")) + assert claim_content_hash(claim) != claim_content_hash(replace(claim, scope="project:other")) + assert QdrantBackend._claim_payload(claim) == { + "claim_id": claim.id, + "content_hash": claim_content_hash(claim), + } + + +def test_authoritative_rehydration_rejects_every_untrusted_candidate( + tmp_path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("QDRANT_URL", raising=False) + svc = MemoryService(tmp_path / "governed-qdrant.db", workspace_root=tmp_path) + svc.init_db() + allowed = _seed(svc, "allowed semantic memory") + stale_hash = _seed(svc, "changed after vector indexing") + wrong_scope = _seed(svc, "foreign scope memory", scope="project:other") + wrong_tenant = _seed(svc, "foreign tenant memory") + private = replace(_seed(svc, "private agent memory"), visibility="private", source_agent="other") + sensitive = _seed(svc, "credential AKIAIOSFODNN7EXAMPLE must remain hidden") + archived = _seed(svc, "archived semantic memory") + transition_claim(svc.store, archived.id, "archived", reason="fixture", event_type="transition") + candidate = svc.ingest( + "unconfirmed semantic memory", + [CitationInput(source="test://candidate")], + scope="project:allowed", + source_agent="seed", + ) + with svc.store.connect() as conn: + conn.execute("UPDATE claims SET tenant_id = 'tenant-a' WHERE id != ?", (wrong_tenant.id,)) + conn.execute("UPDATE claims SET tenant_id = 'tenant-b' WHERE id = ?", (wrong_tenant.id,)) + conn.execute( + "UPDATE claims SET visibility = 'private', source_agent = 'other' WHERE id = ?", + (private.id,), + ) + conn.commit() + svc.tenant_id = "tenant-a" + refreshed = { + claim.id: svc.store.get_claim(claim.id, include_citations=True) + for claim in (allowed, stale_hash, wrong_scope, wrong_tenant, private, sensitive, archived, candidate) + } + svc.qdrant = FakeCandidates( + [ + _candidate(refreshed[allowed.id], score=0.99), + _candidate(refreshed[stale_hash.id], content_hash="0" * 64), + _candidate(refreshed[wrong_scope.id]), + _candidate(refreshed[wrong_tenant.id]), + _candidate(refreshed[private.id]), + _candidate(refreshed[sensitive.id]), + _candidate(refreshed[archived.id]), + _candidate(refreshed[candidate.id]), + QdrantCandidate(claim_id=999_999, content_hash="a" * 64, score=1.0), + ] + ) + + result = svc.retrieve( + RetrievalRequest( + query_text="semantic memory", + retrieval_mode="qdrant", + qdrant_candidate_reads=True, + scope_allowlist=("project:allowed",), + requesting_agent="reader", + limit=2, + ) + ) + + assert [row["claim"].id for row in result.rows] == [allowed.id] + assert result.rows[0]["vector_score"] == 0.99 + assert svc.qdrant.limits == [20] + + +def test_backend_returns_only_validated_id_hash_candidates() -> None: + backend = object.__new__(QdrantBackend) + backend.qdrant_url = "https://qdrant.test" + backend.collection = "claims" + backend._qdrant_client = MagicMock() + backend._ollama_client = MagicMock() + backend._embed = lambda _text: [0.1] * EMBEDDING_DIMS + response = MagicMock() + response.json.return_value = { + "result": [ + {"score": 0.8, "payload": {"claim_id": 7, "content_hash": "a" * 64, "claim_text": "hostile"}}, + {"score": 1.0, "payload": {"claim_id": True, "content_hash": "b" * 64}}, + {"score": 0.7, "payload": {"claim_id": 8, "content_hash": "short"}}, + {"score": "nan", "payload": {"claim_id": 9, "content_hash": "c" * 64}}, + {"score": 0.6, "payload": None}, + ] + } + backend._qdrant_client.post.return_value = response + + assert backend.search_candidates("query", limit=5) == [ + QdrantCandidate(claim_id=7, content_hash="a" * 64, score=0.8) + ] + body = backend._qdrant_client.post.call_args.kwargs["json"] + assert body["with_payload"] == ["claim_id", "content_hash"] + + +def test_qdrant_outbox_is_bounded_and_replayable(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("MEMORYMASTER_QDRANT_OUTBOX_DIR", str(tmp_path / "outbox")) + svc = MemoryService(tmp_path / "outbox.db", workspace_root=tmp_path) + svc.init_db() + claim = _seed(svc, "replay this authoritative claim") + + assert qdrant_outbox.enqueue(svc.store.db_path, "upsert", claim.id, "0" * 64, max_entries=2) + assert qdrant_outbox.enqueue(svc.store.db_path, "delete", 999, None, max_entries=2) + assert not qdrant_outbox.enqueue(svc.store.db_path, "delete", 1000, None, max_entries=2) + + backend = MagicMock() + backend.upsert_claim.return_value = True + backend.delete_claim.return_value = True + result = qdrant_outbox.replay(svc.store.db_path, svc.store, backend, max_operations=10) + + assert result == {"attempted": 2, "completed": 2, "remaining": 0} + backend.upsert_claim.assert_called_once() + backend.delete_claim.assert_called_once_with(999) + assert qdrant_outbox.pending(svc.store.db_path) == [] + + +def test_qdrant_outbox_retains_operations_when_backend_raises(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("MEMORYMASTER_QDRANT_OUTBOX_DIR", str(tmp_path / "outbox")) + svc = MemoryService(tmp_path / "retry.db", workspace_root=tmp_path) + svc.init_db() + claim = _seed(svc, "retry after transient qdrant exception") + assert qdrant_outbox.enqueue( + svc.store.db_path, + "upsert", + claim.id, + claim_content_hash(claim), + ) + backend = MagicMock() + backend.upsert_claim.side_effect = RuntimeError("transient") + + result = qdrant_outbox.replay(svc.store.db_path, svc.store, backend) + + assert result == {"attempted": 1, "completed": 0, "remaining": 1} + assert len(qdrant_outbox.pending(svc.store.db_path)) == 1 + + +def test_equal_counts_still_reconcile_stale_missing_and_orphan_points( + tmp_path, + monkeypatch, +) -> None: + monkeypatch.setenv("MEMORYMASTER_QDRANT_OUTBOX_DIR", str(tmp_path / "outbox")) + svc = MemoryService(tmp_path / "exact.db", workspace_root=tmp_path) + svc.init_db() + first = _seed(svc, "first exact reconciliation claim") + second = _seed(svc, "second exact reconciliation claim") + + class ExactBackend: + def __init__(self) -> None: + self.upserted: list[int] = [] + self.deleted: list[int] = [] + + def count_points(self): + return 2 + + def list_point_refs(self): + return [ + QdrantCandidate(first.id, "0" * 64, 0.0), + QdrantCandidate(999_999, "f" * 64, 0.0), + ] + + def list_point_claim_ids(self): + return [first.id, 999_999] + + def upsert_claim(self, claim): + self.upserted.append(claim.id) + return True + + def delete_claim(self, claim_id): + self.deleted.append(claim_id) + return True + + def sync_all(self, _store): + pytest.fail("equal cardinality must not trigger full re-embedding") + + backend = ExactBackend() + result = qdrant_reconcile.run(svc.store, backend, force=True, threshold=10) + + assert result["drift"] == 0 + assert set(backend.upserted) == {first.id, second.id} + assert backend.deleted == [999_999] + assert result["exact_reconcile"] == {"checked": 2, "upserted": 2, "deleted": 1} + + +def test_failed_service_sync_enters_metadata_only_outbox(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("MEMORYMASTER_QDRANT_OUTBOX_DIR", str(tmp_path / "outbox")) + svc = MemoryService(tmp_path / "sync.db", workspace_root=tmp_path) + svc.init_db() + claim = _seed(svc, "failed vector sync is replayable") + svc.qdrant = MagicMock() + svc.qdrant.upsert_claim.return_value = False + + svc._qdrant_sync(claim) + + entries = qdrant_outbox.pending(svc.store.db_path) + assert entries == [{ + "op": "upsert", + "claim_id": claim.id, + "content_hash": claim_content_hash(claim), + }] + assert "text" not in entries[0] + + +def test_mcp_semantic_profile_requires_explicit_gate(tmp_path, monkeypatch) -> None: + monkeypatch.setenv("MEMORYMASTER_MCP_AUTH_MODE", "local-trusted") + monkeypatch.setenv("MEMORYMASTER_QDRANT_GOVERNED_READS", "1") + svc = MemoryService(tmp_path / "mcp.db", workspace_root=tmp_path) + svc.init_db() + claim = _seed(svc, "governed semantic MCP result") + fake = FakeCandidates([_candidate(claim)]) + svc.qdrant = fake + monkeypatch.setattr(mcp_server, "_service", lambda *_args, **_kwargs: svc) + + result = mcp_server.query_memory( + query="governed semantic MCP result", + db=str(svc.store.db_path), + workspace=str(tmp_path), + retrieval_mode="qdrant", + scope_allowlist="project:allowed", + ) + + assert {item["id"] for item in result["claims"]} == {claim.id} + assert result["retrieval_mode"] == "qdrant" + assert "containment_reason" not in result + assert fake.limits == [80] diff --git a/tests/test_qdrant_retrieval_quarantine.py b/tests/test_qdrant_retrieval_quarantine.py index e53e508f..b586f24d 100644 --- a/tests/test_qdrant_retrieval_quarantine.py +++ b/tests/test_qdrant_retrieval_quarantine.py @@ -11,6 +11,7 @@ import memorymaster.recall.query_classifier as query_classifier import memorymaster.surfaces.mcp_server as mcp_server from memorymaster.core.models import CitationInput +from memorymaster.core.lifecycle import transition_claim from memorymaster.core.service import MemoryService from memorymaster.surfaces.cli import main @@ -39,6 +40,13 @@ def _seed_lexical_claim(tmp_path, text: str) -> tuple[str, str, int]: scope="project:allowed", source_agent="seed", ) + transition_claim( + service.store, + claim.id, + "confirmed", + reason="trusted retrieval fixture", + event_type="validator", + ) return db, str(workspace), claim.id From 038779f80f2a87cf9084aafef615ec30160c3ac1 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Mon, 13 Jul 2026 10:22:48 -0300 Subject: [PATCH 31/55] chore(agents): pin budget-aware routing --- .codex/agents/mm-docs-ledger.toml | 4 +-- .codex/agents/mm-explorer.toml | 4 +-- .codex/agents/mm-fast-worker.toml | 4 +-- .codex/agents/mm-test-runner.toml | 6 ++--- .codex/agents/mm-worker.toml | 6 ++--- .codex/config.toml | 2 +- .planning/REMAINING-PHASES-V1-BUDGET.md | 35 +++++++++++++++++++++++-- 7 files changed, 46 insertions(+), 15 deletions(-) diff --git a/.codex/agents/mm-docs-ledger.toml b/.codex/agents/mm-docs-ledger.toml index 4c94c5aa..273ccd4c 100644 --- a/.codex/agents/mm-docs-ledger.toml +++ b/.codex/agents/mm-docs-ledger.toml @@ -1,9 +1,9 @@ name = "mm_docs_ledger" description = "Evidence-focused documentation worker for remediation ledgers, audit deltas, runbooks, and external-action records." -model = "gpt-5.6-terra" +model = "gpt-5.4-mini" model_reasoning_effort = "medium" sandbox_mode = "workspace-write" -nickname_candidates = ["Terra Ledger", "Terra Scribe", "Terra Delta"] +nickname_candidates = ["Mini Ledger", "Mini Scribe", "Mini Delta"] developer_instructions = """ Obey the repository AGENTS.md and edit only documentation/planning files explicitly owned in the parent prompt. diff --git a/.codex/agents/mm-explorer.toml b/.codex/agents/mm-explorer.toml index 289fa348..3df388ba 100644 --- a/.codex/agents/mm-explorer.toml +++ b/.codex/agents/mm-explorer.toml @@ -1,9 +1,9 @@ name = "mm_explorer" description = "Fast read-only MemoryMaster code mapper for inventories, dependency tracing, and bounded evidence gathering." -model = "gpt-5.6-terra" +model = "gpt-5.4-mini" model_reasoning_effort = "low" sandbox_mode = "read-only" -nickname_candidates = ["Terra Map", "Terra Trace", "Terra Scout"] +nickname_candidates = ["Mini Map", "Mini Trace", "Mini Scout"] developer_instructions = """ Work read-only and obey the repository AGENTS.md plus the parent task. diff --git a/.codex/agents/mm-fast-worker.toml b/.codex/agents/mm-fast-worker.toml index f2e23383..23bcb24a 100644 --- a/.codex/agents/mm-fast-worker.toml +++ b/.codex/agents/mm-fast-worker.toml @@ -1,9 +1,9 @@ name = "mm_fast_worker" description = "Efficient implementation worker for isolated LOW/MEDIUM-risk fixes with explicit file ownership and acceptance tests." -model = "gpt-5.6-terra" +model = "gpt-5.6-luna" model_reasoning_effort = "medium" sandbox_mode = "workspace-write" -nickname_candidates = ["Terra Patch", "Terra Build", "Terra Fix"] +nickname_candidates = ["Luna Patch", "Luna Build", "Luna Fix"] developer_instructions = """ Obey the repository AGENTS.md and the exact owned-file boundary in the parent prompt. diff --git a/.codex/agents/mm-test-runner.toml b/.codex/agents/mm-test-runner.toml index 9a5fcbbe..32312522 100644 --- a/.codex/agents/mm-test-runner.toml +++ b/.codex/agents/mm-test-runner.toml @@ -1,9 +1,9 @@ name = "mm_test_runner" description = "Fast verification worker for focused pytest, Ruff, scanners, collection checks, and evidence summaries." -model = "gpt-5.6-terra" -model_reasoning_effort = "low" +model = "gpt-5.4-mini" +model_reasoning_effort = "medium" sandbox_mode = "workspace-write" -nickname_candidates = ["Terra Gate", "Terra Probe", "Terra Verify"] +nickname_candidates = ["Mini Gate", "Mini Probe", "Mini Verify"] developer_instructions = """ Obey the repository AGENTS.md and run only the verification scope assigned by the parent. diff --git a/.codex/agents/mm-worker.toml b/.codex/agents/mm-worker.toml index 3609d87a..9f6bb531 100644 --- a/.codex/agents/mm-worker.toml +++ b/.codex/agents/mm-worker.toml @@ -1,9 +1,9 @@ name = "mm_worker" description = "General implementation worker for bounded MemoryMaster packages that need stronger multi-step coding and validation." -model = "gpt-5.6-sol" -model_reasoning_effort = "medium" +model = "gpt-5.6-terra" +model_reasoning_effort = "high" sandbox_mode = "workspace-write" -nickname_candidates = ["Core Worker", "Package Worker", "Integration Worker"] +nickname_candidates = ["Terra Core", "Terra Package", "Terra Worker"] developer_instructions = """ Obey the repository AGENTS.md and the exact owned-file/worktree boundary in the parent prompt. diff --git a/.codex/config.toml b/.codex/config.toml index 809fdedd..42c3f242 100644 --- a/.codex/config.toml +++ b/.codex/config.toml @@ -1,5 +1,5 @@ [agents] -max_threads = 4 +max_threads = 2 max_depth = 1 job_max_runtime_seconds = 1800 interrupt_message = true diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index 11cb72ba..6dbae9f4 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -33,6 +33,36 @@ external-action gates pass. goals. Publish one phase draft PR only after its phase convergence gate. 8. Medium/Low findings outside the package become backlog. No broad refactors. +## Agent routing gate (required before P2-C) + +The root coordinator runs on the composer-selected `gpt-5.6-sol` at high +reasoning. Root owns architecture, HIGH/CRITICAL decisions and warnings, +cross-package integration, staging, commits, and final evidence reconciliation. +Child agents never supervise the root or independently declare a package done. + +MemoryMaster's project agents are pinned in `.codex/agents/`: + +| Agent | Model / effort | Allowed work | +|---|---|---| +| `mm_explorer` | `gpt-5.4-mini` / low | Read-only mapping and inventories | +| `mm_test_runner` | `gpt-5.4-mini` / medium | Focused commands and evidence | +| `mm_docs_ledger` | `gpt-5.4-mini` / medium | Evidence-backed planning/docs only | +| `mm_fast_worker` | `gpt-5.6-luna` / medium | Isolated LOW/MEDIUM implementation | +| `mm_worker` | `gpt-5.6-terra` / high | Bounded multi-step implementation | +| `mm_security_reviewer` | `gpt-5.6-sol` / high | Read-only security/integrity review | + +`.codex/config.toml` sets `max_threads = 2` and `max_depth = 1`: at most one +child may run beside root, and children cannot create grandchildren. Delegate +only a concrete bounded task whose parallelism saves time. Every write worker +must receive an isolated worktree and explicit owned files; root inspects and +integrates its diff. Never run concurrent writers in one checkout. + +Do not silently promote routine work to Sol. If a pinned model is unavailable, +root either performs the task or records the routing limitation; security, +architecture, and HIGH/CRITICAL decisions must not be downgraded to Mini or +Luna. Before starting P2-C, parse every project TOML file, verify each pinned +model and reasoning level exists locally, and commit this routing gate. + ## Phase 2 — Governed core convergence Phase 2 uses branch `remediation/phase2-governed-core-20260712` and worktree @@ -158,5 +188,6 @@ P2-B implementation and focused gate completed on 2026-07-13: - Disposable authenticated/TLS Qdrant runtime parity is `BLOCKED-EXTERNAL` in `external-actions-required.md`. -Next package is P2-C. Start it only in a separate goal/package; do not combine -lifecycle/read-only recall changes with P2-B. +The agent routing gate is the only prerequisite package before P2-C. After it +is validated and atomically committed, start P2-C only in a separate goal; do +not combine lifecycle/read-only recall changes with P2-B or routing changes. From 732cb684efd48083523bcd86200933cd6e928a7a Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Mon, 13 Jul 2026 13:06:29 -0300 Subject: [PATCH 32/55] fix: enforce lifecycle authority and read-only recall --- .planning/REMAINING-PHASES-V1-BUDGET.md | 19 ++++- .../hooks/memorymaster-steward-cycle.py | 16 +--- memorymaster/core/service.py | 42 ++++++---- memorymaster/core/spool.py | 26 +++++- memorymaster/govern/jobs/__init__.py | 12 ++- memorymaster/govern/jobs/scheduled_archive.py | 56 +++++++++++++ memorymaster/govern/jobs/spool_drain.py | 8 ++ memorymaster/recall/context_hook.py | 23 +++-- memorymaster/recall/context_optimizer.py | 2 + memorymaster/surfaces/mcp_server.py | 55 +++++------- .../test_auxiliary_persistence_sensitivity.py | 7 +- tests/test_mcp_read_only_contract.py | 20 +++-- tests/test_ro_recall.py | 26 +++--- tests/test_scheduled_archive_lifecycle.py | 84 +++++++++++++++++-- 14 files changed, 286 insertions(+), 110 deletions(-) create mode 100644 memorymaster/govern/jobs/scheduled_archive.py diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index 6dbae9f4..f57f8845 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -188,6 +188,19 @@ P2-B implementation and focused gate completed on 2026-07-13: - Disposable authenticated/TLS Qdrant runtime parity is `BLOCKED-EXTERNAL` in `external-actions-required.md`. -The agent routing gate is the only prerequisite package before P2-C. After it -is validated and atomically committed, start P2-C only in a separate goal; do -not combine lifecycle/read-only recall changes with P2-B or routing changes. +P2-C implementation and focused gate completed on 2026-07-13 from routing-gate +commit `038779f`: + +- Scheduled stale/unused archival now enters the canonical lifecycle transition + authority, preserving optimistic version checks, transition events, status + timestamps, query-cache invalidation, and replayable Qdrant deletion. +- MCP/context-hook recall opens SQLite query-only, suppresses detail-level and + token-fanout duplicate retrieval, and emits one sanitized `recall` envelope + per successful top-level recall while retaining legacy spool replay support. +- Focused P2-C gate: 40 passed in 6.03s; changed-file Ruff and + `git diff --check` passed. +- Atomic package commit: the conventional `fix: enforce lifecycle authority and + read-only recall` commit containing this scheduler evidence. + +The next separately authorized package is P2-D. Do not combine entity-schema +convergence with P2-C or begin it from this goal. diff --git a/memorymaster/config_templates/hooks/memorymaster-steward-cycle.py b/memorymaster/config_templates/hooks/memorymaster-steward-cycle.py index b46768c8..5390bf45 100644 --- a/memorymaster/config_templates/hooks/memorymaster-steward-cycle.py +++ b/memorymaster/config_templates/hooks/memorymaster-steward-cycle.py @@ -45,20 +45,10 @@ # Auto-archive: stale claims never accessed, older than 14 days try: - from datetime import datetime, timedelta + from memorymaster.govern.jobs import scheduled_archive - # Uniform pragma envelope (WAL + busy_timeout=15000) — a raw connect here - # had busy_timeout=0 and could lose the UPDATE to a write race (spec F8). - from memorymaster.stores._storage_shared import open_conn - cutoff = (datetime.now() - timedelta(days=14)).isoformat() - conn = open_conn(DB_PATH) - conn.execute(""" - UPDATE claims SET status = 'archived', archived_at = datetime('now') - WHERE status = 'stale' AND access_count = 0 AND created_at < ? - """, (cutoff,)) - archived = conn.total_changes - conn.commit() - conn.close() + archive_result = scheduled_archive.run(svc, older_than_days=14) + archived = archive_result["archived"] if archived: print(f"[MemoryMaster] auto-archived {archived} stale unused claims") except Exception as e: diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index 59c5bec1..c1bf8962 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -1311,7 +1311,7 @@ def _build_query_statuses(self, include_stale: bool, include_conflicted: bool, i statuses.append("candidate") return statuses - def _query_legacy_mode(self, query_text: str, limit: int, statuses: list[str], normalized_scopes: list[str] | None, include_sensitive: bool, requesting_agent: str | None) -> list[dict[str, Any]]: + def _query_legacy_mode(self, query_text: str, limit: int, statuses: list[str], normalized_scopes: list[str] | None, include_sensitive: bool, requesting_agent: str | None, record_accesses: bool = True) -> list[dict[str, Any]]: """Query using legacy retrieval mode.""" legacy = self._legacy_candidates( query_text, limit, statuses, normalized_scopes @@ -1341,7 +1341,8 @@ def _query_legacy_mode(self, query_text: str, limit: int, statuses: list[str], n } for row in ranked_rows ] - self._record_accesses(results, query_text=query_text) + if record_accesses: + self._record_accesses(results, query_text=query_text) return results def _legacy_candidates( @@ -1386,6 +1387,7 @@ def query_rows( enrich_with_entities: bool = False, requesting_agent: str | None = None, query_type: str | None = None, + record_accesses: bool = True, ) -> list[dict[str, Any]]: if limit <= 0: return [] @@ -1421,7 +1423,15 @@ def query_rows( retrieval_mode = "hybrid" if retrieval_mode == "legacy": - return self._query_legacy_mode(query_text, limit, statuses, normalized_scopes, include_sensitive, requesting_agent) + return self._query_legacy_mode( + query_text, + limit, + statuses, + normalized_scopes, + include_sensitive, + requesting_agent, + record_accesses, + ) use_llm_rerank = _llm_rerank_enabled() @@ -1430,7 +1440,7 @@ def query_rows( # the corpus generation (bumped by claim-write triggers) is unchanged. cache_path = None cache_key = None - if query_cache.cache_enabled(): + if query_cache.cache_enabled() and not getattr(self.store, "read_only", False): cache_path = query_cache.sqlite_db_path(self.store) if cache_path: cache_key = query_cache.make_cache_key(query_text, { @@ -1454,7 +1464,8 @@ def query_rows( if getattr(r["claim"], "visibility", "public") == "public" or getattr(r["claim"], "source_agent", None) == requesting_agent ] - self._record_accesses(rows, query_text=query_text) + if record_accesses: + self._record_accesses(rows, query_text=query_text) return rows # Capture the corpus generation BEFORE reading candidates so the cache # entry is tagged with the generation it was actually computed against, @@ -1524,7 +1535,8 @@ def _vector_hook(text, claims): from memorymaster.recall.llm_rerank import rerank_with_llm results = rerank_with_llm(query_text, results, top_k=limit) - self._record_accesses(results, query_text=query_text) + if record_accesses: + self._record_accesses(results, query_text=query_text) if enrich_with_entities: results = self._enrich_with_entity_graph(results, query_text, limit) if cache_path and cache_key: @@ -1720,8 +1732,8 @@ def _emit_recall_telemetry(self) -> None: def _spool_accesses(self, claim_ids: list[int], query_text: str) -> None: """RO-store branch of _record_accesses: spool, don't write. - Appends ``access`` (+ ``feedback`` when there is a query) envelopes - to the JSONL spool (spool.py); jobs/spool_drain replays them through + Appends one aggregated ``recall`` envelope to the JSONL spool + (spool.py); jobs/spool_drain replays it through the exact same sinks the RW path uses (record_accesses_batch, FeedbackTracker.record_retrieval). Best-effort like the RW path — a spool I/O failure must never break recall. @@ -1741,14 +1753,14 @@ def _spool_accesses(self, claim_ids: list[int], query_text: str) -> None: ) with contextlib.suppress(Exception): spool.append( - db_path, "access", {"claim_ids": claim_ids, "query_hash": query_hash} + db_path, + "recall", + { + "claim_ids": list(dict.fromkeys(claim_ids)), + "query_hash": query_hash, + "query_text": safe_query, + }, ) - if safe_query: - spool.append( - db_path, - "feedback", - {"claim_ids": claim_ids, "query_text": safe_query}, - ) def recompute_tiers(self) -> dict[str, int]: """Recompute tier assignments for all non-archived claims.""" diff --git a/memorymaster/core/spool.py b/memorymaster/core/spool.py index c235ffcf..8d9ad067 100644 --- a/memorymaster/core/spool.py +++ b/memorymaster/core/spool.py @@ -6,7 +6,7 @@ service paths (``jobs/spool_drain.py``). This file format is the only "wire protocol" in the P1 design: - {"v": 1, "op": "access"|"feedback"|"ingest"|"verbatim"|"dream", + {"v": 1, "op": "recall"|"access"|"feedback"|"ingest"|"verbatim"|"dream", "ts": , "idempotency_key": , "payload": {...}} Layout (spec §2.2): @@ -43,7 +43,7 @@ ENV_WAL_DISCIPLINE = "MEMORYMASTER_WAL_DISCIPLINE" SPOOL_VERSION = 1 -KNOWN_OPS = ("access", "feedback", "ingest", "verbatim", "dream") +KNOWN_OPS = ("recall", "access", "feedback", "ingest", "verbatim", "dream") DRAINING_SUFFIX = ".draining" QUARANTINE_DIRNAME = "quarantine" @@ -61,6 +61,7 @@ ) _ACCESS_PAYLOAD_FIELDS = frozenset({"claim_ids", "query_hash"}) _FEEDBACK_PAYLOAD_FIELDS = frozenset({"claim_ids", "query_text"}) +_RECALL_PAYLOAD_FIELDS = frozenset({"claim_ids", "query_hash", "query_text"}) def _reject_unknown_fields(payload: dict[str, object], allowed: frozenset[str]) -> None: @@ -153,6 +154,25 @@ def _sanitize_feedback_payload(payload: dict[str, object]) -> dict[str, object]: return safe +def _sanitize_recall_payload(payload: dict[str, object]) -> dict[str, object]: + _reject_unknown_fields(payload, _RECALL_PAYLOAD_FIELDS) + query = payload.get("query_text", "") + if not isinstance(query, str): + raise ValueError("Spool recall query_text must be a string.") + validate_persisted_metadata( + {"claim_ids": payload.get("claim_ids"), "query_hash": payload.get("query_hash")} + ) + safe_query, findings = sanitize_persisted_text(query) + safe = { + "claim_ids": list(payload.get("claim_ids") or []), + "query_hash": payload.get("query_hash"), + "query_text": safe_query, + } + if findings: + safe["_sanitization"] = {"findings": findings} + return safe + + def _sanitize_spool_payload( op: str, payload: dict[str, object], @@ -162,6 +182,8 @@ def _sanitize_spool_payload( return _sanitize_ingest_payload(payload, idempotency_key) if op == "feedback": return _sanitize_feedback_payload(payload) + if op == "recall": + return _sanitize_recall_payload(payload) allowed = _VERBATIM_PAYLOAD_FIELDS if op == "verbatim" else _ACCESS_PAYLOAD_FIELDS _reject_unknown_fields(payload, allowed) validate_persisted_metadata({f"{op}_payload": payload}) diff --git a/memorymaster/govern/jobs/__init__.py b/memorymaster/govern/jobs/__init__.py index 54a1a9b6..b5e8ef19 100644 --- a/memorymaster/govern/jobs/__init__.py +++ b/memorymaster/govern/jobs/__init__.py @@ -1,5 +1,13 @@ """Job modules for the memory reliability cycle.""" -from . import compactor, decay, dedup, deterministic, extractor, validator +from . import compactor, decay, dedup, deterministic, extractor, scheduled_archive, validator -__all__ = ["extractor", "deterministic", "validator", "decay", "compactor", "dedup"] +__all__ = [ + "extractor", + "deterministic", + "validator", + "decay", + "compactor", + "dedup", + "scheduled_archive", +] diff --git a/memorymaster/govern/jobs/scheduled_archive.py b/memorymaster/govern/jobs/scheduled_archive.py new file mode 100644 index 00000000..0893eb39 --- /dev/null +++ b/memorymaster/govern/jobs/scheduled_archive.py @@ -0,0 +1,56 @@ +"""Bounded lifecycle-authoritative archival for the scheduled steward hook.""" + +from __future__ import annotations + +from datetime import datetime, timedelta, timezone + +from memorymaster.core import lifecycle +from memorymaster.recall import qdrant_outbox +from memorymaster.stores._storage_shared import ConcurrentModificationError + + +def _eligible(claim, cutoff: datetime) -> bool: + if claim.access_count != 0: + return False + try: + created_at = datetime.fromisoformat(claim.created_at) + except (TypeError, ValueError): + return False + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=timezone.utc) + return created_at < cutoff + + +def _schedule_vector_delete(service, claim) -> None: + if service.qdrant is not None: + service._qdrant_sync(claim) + return + db_path = getattr(service.store, "db_path", None) + if db_path is not None: + qdrant_outbox.enqueue(db_path, "delete", claim.id, None) + + +def run(service, *, older_than_days: int = 14, limit: int = 500) -> dict[str, int]: + """Archive a bounded stale/unused batch through canonical transitions.""" + if older_than_days < 0 or limit <= 0: + raise ValueError("older_than_days must be non-negative and limit must be positive") + cutoff = datetime.now(timezone.utc) - timedelta(days=older_than_days) + stale = service.store.find_by_status("stale", limit=limit, include_citations=False) + matched = [claim for claim in stale if _eligible(claim, cutoff)] + archived = 0 + skipped = 0 + for claim in matched: + try: + updated = lifecycle.transition_claim( + service.store, + claim.id, + "archived", + reason=f"scheduled stale/unused archival after {older_than_days} days", + event_type="staleness", + ) + except (ConcurrentModificationError, ValueError): + skipped += 1 + continue + _schedule_vector_delete(service, updated) + archived += 1 + return {"matched": len(matched), "archived": archived, "skipped": skipped} diff --git a/memorymaster/govern/jobs/spool_drain.py b/memorymaster/govern/jobs/spool_drain.py index 49d5eef6..f13801f7 100644 --- a/memorymaster/govern/jobs/spool_drain.py +++ b/memorymaster/govern/jobs/spool_drain.py @@ -145,6 +145,12 @@ def _replay_feedback(db_path: str | Path, envelope: dict[str, object]) -> None: tracker.record_retrieval(claim_ids, str(payload.get("query_text") or "")) +def _replay_recall(store, db_path: str | Path, envelope: dict[str, object]) -> None: + _replay_access(store, envelope) + if str(envelope["payload"].get("query_text") or ""): + _replay_feedback(db_path, envelope) + + def _validate(envelope: object) -> str | None: """Reason string when an envelope must be quarantined, else None.""" if not isinstance(envelope, dict): @@ -212,6 +218,8 @@ def run( _replay_verbatim(db_path, envelope) elif op == "access": _replay_access(store, envelope) + elif op == "recall": + _replay_recall(store, db_path, envelope) else: # "feedback" — _validate pinned op to KNOWN_OPS _replay_feedback(db_path, envelope) except Exception as exc: diff --git a/memorymaster/recall/context_hook.py b/memorymaster/recall/context_hook.py index b63c10a4..25d31902 100644 --- a/memorymaster/recall/context_hook.py +++ b/memorymaster/recall/context_hook.py @@ -1231,7 +1231,7 @@ def query_for_task( from memorymaster.core.service import MemoryService db = db_path or os.environ.get("MEMORYMASTER_DEFAULT_DB") or "memorymaster.db" - svc = MemoryService(db_target=db, workspace_root=Path.cwd()) + svc = MemoryService(db_target=db, workspace_root=Path.cwd(), read_only=True) scope_filter = [project_scope] if project_scope else None # Historical name retained: this now selects primary-store legacy vs @@ -1295,17 +1295,9 @@ def _recall_impl( """ MemoryService = _memory_service_cls db = db_path or os.environ.get("MEMORYMASTER_DEFAULT_DB") or "memorymaster.db" - if _wal_discipline_enabled(): - # P1 WAL-discipline (spec §2.2): the per-prompt recall hook must - # never take a write lock on the shared multi-GB DB. The RO store - # hands out mode=ro + query_only connections (side lookups follow - # automatically through the store) and _record_accesses spools its - # access/feedback signal for the steward drain instead of UPDATEing - # inline — no tiering/decay/quality signal is lost (the F9 fix). - svc = MemoryService(db_target=db, workspace_root=Path.cwd(), read_only=True) - else: - # Flag off = the untouched legacy RW path, bit-for-bit. - svc = MemoryService(db_target=db, workspace_root=Path.cwd()) + # Recall is always query-only. Access/feedback signals are emitted once, + # after top-level ranking and budget selection, through the RO spool path. + svc = MemoryService(db_target=db, workspace_root=Path.cwd(), read_only=True) # Pre-extract salient tokens before hitting FTS5. Passing the full # prompt verbatim AND-joins every token in FTS5 and rejects nearly all @@ -1356,6 +1348,7 @@ def _recall_impl( retrieval_mode="legacy", include_candidates=True, scope_allowlist=None, + record_accesses=False, ) for row in batch: claim = row.get("claim") @@ -1375,6 +1368,7 @@ def _recall_impl( retrieval_mode="legacy", include_candidates=True, scope_allowlist=None, + record_accesses=False, ) for row in rows: claim = row.get("claim") @@ -2014,6 +2008,7 @@ def _ranking(score_fn) -> list[int]: # Build output — top claims within budget lines = ["# Memory Context", ""] + rendered_rows: list[dict] = [] tokens_used = 0 chars_per_token = 4 for row in ranked: @@ -2034,12 +2029,16 @@ def _ranking(score_fn) -> list[int]: if tokens_used + chunk_tokens > budget: break lines.append(chunk) + rendered_rows.append(row) tokens_used += chunk_tokens if _rendered_ids is not None: cid = getattr(claim, "id", None) if isinstance(cid, int): _rendered_ids.append(cid) + if rendered_rows: + svc._record_accesses(rendered_rows, query_text=query) + # Close the rank_and_build timer right before we emit output so the # measurement covers _relevance/RRF + the budget-trimming loop. phase_ms["rank_and_build"] = (time.perf_counter() - _rank_start) * 1000.0 diff --git a/memorymaster/recall/context_optimizer.py b/memorymaster/recall/context_optimizer.py index 38148eed..0ff09b46 100644 --- a/memorymaster/recall/context_optimizer.py +++ b/memorymaster/recall/context_optimizer.py @@ -34,6 +34,7 @@ class ContextResult: tokens_used: int token_budget: int format: str + rows: tuple[dict[str, Any], ...] = () @dataclass(frozen=True, slots=True) @@ -392,4 +393,5 @@ def pack_context( tokens_used=total_tokens, token_budget=token_budget, format=output_format, + rows=tuple(ranked_rows), ) diff --git a/memorymaster/surfaces/mcp_server.py b/memorymaster/surfaces/mcp_server.py index b4992919..5c9a327b 100644 --- a/memorymaster/surfaces/mcp_server.py +++ b/memorymaster/surfaces/mcp_server.py @@ -335,7 +335,7 @@ def _bind_telemetry_session( pass -def _service(db: str, workspace: str) -> MemoryService: +def _service(db: str, workspace: str, *, read_only: bool = False) -> MemoryService: db_path = _resolve_db(db) workspace_path = _resolve_workspace(workspace) context = current_request_context() @@ -348,12 +348,18 @@ def _service(db: str, workspace: str) -> MemoryService: require_tenant=context is not None and context.mode is AuthMode.TEAM, principal=principal, allowed_scopes=context.allowed_scopes if context is not None else frozenset(), + read_only=read_only, ) - _bind_telemetry_session(svc, db_path, principal, tenant_id) + if not read_only: + _bind_telemetry_session(svc, db_path, principal, tenant_id) svc.source_agent = principal return svc +def _read_service(db: str, workspace: str) -> MemoryService: + return _service(db, workspace, read_only=True) + + def _usage_rollup(db: str) -> dict[str, Any]: """Build the usage-telemetry rollup payload. @@ -1188,7 +1194,7 @@ def query_memory( retrieval_mode = classified_retrieval_mode from memorymaster.recall.planner import RetrievalRequest - svc = _service(db, workspace) + svc = _read_service(db, workspace) retrieval = svc.retrieve(RetrievalRequest( query_text=query, limit=limit, @@ -1244,13 +1250,6 @@ def query_memory( if classified_retrieval_mode is not None: response["classified_retrieval_mode"] = classified_retrieval_mode - # Log to vault chronicle - try: - from memorymaster.knowledge.vault_log import log_query - log_query(query, len(claims)) - except Exception: - pass - return response @mcp.tool() @@ -1288,7 +1287,7 @@ def query_claim_paths( known claim_id can't leak cross-scope or sensitive claim text via graph traversal (audit: claim-paths-scope-gate). """ - svc = _service(db, workspace) + svc = _read_service(db, workspace) normalized_scopes = _effective_scope_allowlist(scope_allowlist, workspace) allow_sensitive = resolve_allow_sensitive_access( allow_sensitive=allow_sensitive, @@ -1338,7 +1337,7 @@ def recall_analysis( allow_sensitive=allow_sensitive, context="mcp.recall_analysis", ) - svc = _service(db, workspace) + svc = _read_service(db, workspace) analysis = svc.recall_analysis( query_text=query, limit=limit, @@ -1387,7 +1386,7 @@ def query_for_context( allow_sensitive=allow_sensitive, context="mcp.query_for_context", ) - svc = _service(db, workspace) + svc = _read_service(db, workspace) result = svc.query_for_context( query=query, token_budget=token_budget, @@ -1413,19 +1412,7 @@ def query_for_context( if detail_level != "standard": # Attach filtered claim list as a convenience for callers who want # structured data alongside the formatted output block. - from memorymaster.recall.planner import RetrievalRequest - retrieval = svc.retrieve(RetrievalRequest( - query_text=query, - limit=limit, - trust_mode=trust_mode, - retrieval_mode=retrieval_mode, - include_stale=include_stale, - include_conflicted=include_conflicted, - include_candidates=include_candidates, - allow_sensitive=allow_sensitive, - scope_allowlist=tuple(_effective_scope_allowlist(scope_allowlist, workspace)), - )) - rows_data = list(retrieval.rows) + rows_data = list(result.rows) if detail_level == "full": filtered_claims = [] for row in rows_data: @@ -1489,7 +1476,7 @@ def volunteer_context( allow_sensitive=allow_sensitive, context="mcp.volunteer_context", ) - svc = _service(db, workspace) + svc = _read_service(db, workspace) effective_allowlist = _effective_scope_allowlist(scope_allowlist, workspace) from memorymaster.recall.planner import RetrievalRequest retrieval = svc.retrieve(RetrievalRequest( @@ -1699,7 +1686,7 @@ def list_claims( allow_sensitive=allow_sensitive, context="mcp.list_claims", ) - svc = _service(db, workspace) + svc = _read_service(db, workspace) claims = svc.list_claims( status=_empty_to_none(status), limit=limit, @@ -1782,7 +1769,7 @@ def query_rules( allow_sensitive=allow_sensitive, context="mcp.query_rules", ) - svc = _service(db, workspace) + svc = _read_service(db, workspace) rules = svc.query_rules(query, limit=limit, allow_sensitive=allow_sensitive) return {"ok": True, "rows": len(rules), "rules": rules} @@ -1808,7 +1795,7 @@ def rules_export( ) from memorymaster.knowledge.rule_export import collect_rules - svc = _service(db, workspace) + svc = _read_service(db, workspace) rows = collect_rules( svc, min_confidence=min_confidence, @@ -1878,7 +1865,7 @@ def list_events( limit: int = 100, ) -> dict[str, Any]: """List events by optional claim_id and event_type.""" - svc = _service(db, workspace) + svc = _read_service(db, workspace) events = svc.list_events( claim_id=claim_id, limit=limit, @@ -1995,7 +1982,7 @@ def list_steward_proposals( """List steward proposals for human override workflow.""" from memorymaster.govern.steward import list_steward_proposals as _list_steward_proposals - svc = _service(db, workspace) + svc = _read_service(db, workspace) rows = _list_steward_proposals( svc, limit=limit, @@ -2116,7 +2103,7 @@ def query_meta_decisions( if request.top_n <= 0: return _structured_error("top_n must be positive.", "VALIDATION_ERROR", "top_n") - svc = _service(request.db, request.workspace) + svc = _read_service(request.db, request.workspace) result = svc.query_meta_decisions( query=request.query, claim_types=request.claim_types, @@ -2139,7 +2126,7 @@ def federated_query( this tool searches every claim regardless of scope, enabling cross-project memory retrieval. Returns claims sorted by relevance. """ - svc = _service(db, workspace) + svc = _read_service(db, workspace) effective_scope = (current_scope or "").strip() or _project_scope(workspace) rows_data = svc.federated_query( query_text=query, diff --git a/tests/test_auxiliary_persistence_sensitivity.py b/tests/test_auxiliary_persistence_sensitivity.py index da4c66a7..e6a9b378 100644 --- a/tests/test_auxiliary_persistence_sensitivity.py +++ b/tests/test_auxiliary_persistence_sensitivity.py @@ -567,10 +567,11 @@ def test_ro_access_spool_hashes_sanitized_query_not_sensitive_input( for path in spool.spool_dir_for(db_path).glob("*.jsonl") for line in path.read_text(encoding="utf-8").splitlines() ] - by_op = {envelope["op"]: envelope for envelope in envelopes} expected_hash = hashlib.sha256(safe_query.encode("utf-8")).hexdigest()[:12] - assert by_op["access"]["payload"]["query_hash"] == expected_hash - assert by_op["feedback"]["payload"]["query_text"] == safe_query + assert len(envelopes) == 1 + assert envelopes[0]["op"] == "recall" + assert envelopes[0]["payload"]["query_hash"] == expected_hash + assert envelopes[0]["payload"]["query_text"] == safe_query def test_spooled_ingest_preserves_holder_and_records_boundary_findings_once( diff --git a/tests/test_mcp_read_only_contract.py b/tests/test_mcp_read_only_contract.py index 911a0789..6e982929 100644 --- a/tests/test_mcp_read_only_contract.py +++ b/tests/test_mcp_read_only_contract.py @@ -1,21 +1,17 @@ """Red contract for one read-only retrieval per MCP context request (MM-REL-02).""" from __future__ import annotations +import json import sqlite3 -import pytest - import memorymaster.surfaces.mcp_server as mcp_server +from memorymaster.core import spool from memorymaster.core.models import CitationInput from memorymaster.core.service import MemoryService -@pytest.mark.xfail( - strict=True, - reason="audit baseline MM-REL-02: MCP context summaries query and write twice", -) def test_mcp_context_summary_queries_once_without_writing_access_count( - tmp_path, monkeypatch: pytest.MonkeyPatch + tmp_path, monkeypatch ) -> None: """Formatting detail must reuse one governed read, not run retrieval again.""" workspace = tmp_path / "readonly" @@ -42,12 +38,14 @@ def counted_query_rows(self, *args, **kwargs): monkeypatch.setattr(mcp_server, "_ENV_DEFAULT_WORKSPACE", "") monkeypatch.setattr(mcp_server, "_ENV_QUERY_INCLUDE_LEGACY_PROJECT", False) monkeypatch.delenv("QDRANT_URL", raising=False) + monkeypatch.setenv(spool.ENV_SPOOL_DIR, str(tmp_path / "spool")) result = mcp_server.query_for_context( query="readonlycontract", db=db, workspace=str(workspace), retrieval_mode="legacy", + trust_mode="exploratory", include_candidates=True, scope_allowlist="project:readonly", detail_level="summary", @@ -59,5 +57,13 @@ def counted_query_rows(self, *args, **kwargs): ).fetchone()[0] ) + envelopes = [ + json.loads(line) + for path in spool.spool_dir_for(db).glob("*.jsonl") + for line in path.read_text(encoding="utf-8").splitlines() + ] assert result["claims"], "fixture must exercise structured context retrieval" assert (calls, access_count) == (1, 0) + assert len(envelopes) == 1 + assert envelopes[0]["op"] == "recall" + assert envelopes[0]["payload"]["claim_ids"] == [claim.id] diff --git a/tests/test_ro_recall.py b/tests/test_ro_recall.py index a1bfb4a1..732b41c1 100644 --- a/tests/test_ro_recall.py +++ b/tests/test_ro_recall.py @@ -144,11 +144,11 @@ def test_ro_recall_spools_access_signal_instead_of_writing(db: Path) -> None: assert _access_counts(db) == before # zero DB writes from RO recall envelopes = _spool_envelopes(db) - by_op = {e["op"]: e for e in envelopes} - assert set(by_op) == {"access", "feedback"} - assert by_op["access"]["payload"]["claim_ids"] - assert by_op["access"]["payload"]["query_hash"] - assert by_op["feedback"]["payload"]["query_text"] == QUERY + assert len(envelopes) == 1 + assert envelopes[0]["op"] == "recall" + assert envelopes[0]["payload"]["claim_ids"] + assert envelopes[0]["payload"]["query_hash"] + assert envelopes[0]["payload"]["query_text"] == QUERY def test_drained_access_signal_feeds_tiering_end_to_end(db: Path) -> None: @@ -173,7 +173,7 @@ def test_drained_access_signal_feeds_tiering_end_to_end(db: Path) -> None: drained = spool_drain.run(rw) assert drained["quarantined"] == 0 - assert drained["by_op"]["access"] == 6 + assert drained["by_op"]["recall"] == 6 counts = _access_counts(db) assert max(counts) == 6 # every spooled access reached the DB rows @@ -185,17 +185,17 @@ def test_drained_access_signal_feeds_tiering_end_to_end(db: Path) -> None: assert tier_rows and all(r[0] == "core" for r in tier_rows) -def test_flag_off_keeps_legacy_direct_write_path(db: Path) -> None: - """REQUIREMENT (governance: flag default OFF, legacy path intact): with - the flag unset, recall records accesses directly in the DB and the spool - stays empty — the v3.27 behavior is the untouched else-branch.""" +def test_recall_entrypoint_is_read_only_by_default(db: Path) -> None: + """Top-level recall always avoids DB writes and emits one spool line.""" from memorymaster.recall.context_hook import recall before = _access_counts(db) rendered = recall(QUERY, db_path=str(db), skip_qdrant=True) assert rendered # fixture claims match — the path actually ran - assert sum(_access_counts(db)) > sum(before) # direct UPDATE happened - assert spool.pending_depth(db) == {"files": 0, "lines": 0} + assert _access_counts(db) == before + envelopes = _spool_envelopes(db) + assert len(envelopes) == 1 + assert envelopes[0]["op"] == "recall" def test_recall_entrypoint_goes_read_only_under_flag( @@ -213,7 +213,7 @@ def test_recall_entrypoint_goes_read_only_under_flag( assert rendered # same fixture claims — RO recall still answers assert _access_counts(db) == before # no per-prompt write lock taken ops = [e["op"] for e in _spool_envelopes(db)] - assert "access" in ops and "feedback" in ops + assert ops == ["recall"] class _FakeEmbedProvider: diff --git a/tests/test_scheduled_archive_lifecycle.py b/tests/test_scheduled_archive_lifecycle.py index 757422bd..277fda42 100644 --- a/tests/test_scheduled_archive_lifecycle.py +++ b/tests/test_scheduled_archive_lifecycle.py @@ -1,9 +1,15 @@ -"""Red static contract for lifecycle-safe scheduled archival (MM-LIFE-01).""" +"""Contracts for lifecycle-safe scheduled archival (MM-LIFE-01).""" from __future__ import annotations +from datetime import datetime, timedelta, timezone from pathlib import Path +from unittest.mock import MagicMock -import pytest +from memorymaster.core import lifecycle +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.govern.jobs import scheduled_archive +from memorymaster.recall import qdrant_outbox, query_cache HOOK_TEMPLATE = ( @@ -15,13 +21,79 @@ ) -@pytest.mark.xfail( - strict=True, - reason="audit baseline MM-LIFE-01: scheduled hook archives with raw status SQL", -) def test_scheduled_archive_contains_no_raw_claim_status_update() -> None: """Scheduled archival must enter through lifecycle authority and its events.""" source = HOOK_TEMPLATE.read_text(encoding="utf-8") normalized = " ".join(source.lower().split()) assert "update claims set status = 'archived'" not in normalized + + +def test_scheduled_archive_preserves_lifecycle_and_vector_evidence( + tmp_path: Path, + monkeypatch, +) -> None: + monkeypatch.setenv( + qdrant_outbox.ENV_OUTBOX_DIR, + str(tmp_path / "qdrant-outbox"), + ) + monkeypatch.delenv("QDRANT_URL", raising=False) + db_path = tmp_path / "scheduled-archive.db" + service = MemoryService(db_path) + service.init_db() + claim = service.ingest( + "scheduled lifecycle archival evidence", + [CitationInput(source="test://scheduled-archive")], + ) + lifecycle.transition_claim( + service.store, + claim.id, + "confirmed", + reason="fixture confirmed", + event_type="validator", + ) + lifecycle.transition_claim( + service.store, + claim.id, + "stale", + reason="fixture stale", + event_type="decay", + ) + old_created_at = ( + datetime.now(timezone.utc) - timedelta(days=30) + ).replace(microsecond=0).isoformat() + with service.store.connect() as conn: + conn.execute( + "UPDATE claims SET created_at = ?, updated_at = ?, access_count = 0 WHERE id = ?", + (old_created_at, old_created_at, claim.id), + ) + conn.commit() + + before = service.store.get_claim(claim.id, include_citations=False) + assert before is not None + generation_before = query_cache.read_generation(db_path) + service.qdrant = MagicMock() + service.qdrant.delete_claim.return_value = False + + result = scheduled_archive.run(service, older_than_days=14) + + archived = service.store.get_claim(claim.id, include_citations=False) + assert result == {"matched": 1, "archived": 1, "skipped": 0} + assert archived is not None + assert archived.status == "archived" + assert archived.version == before.version + 1 + assert archived.archived_at is not None + assert archived.updated_at != before.updated_at + events = service.store.list_events( + claim_id=claim.id, + event_type="staleness", + limit=10, + ) + assert len(events) == 1 + assert events[0].from_status == "stale" + assert events[0].to_status == "archived" + assert query_cache.read_generation(db_path) > generation_before + service.qdrant.delete_claim.assert_called_once_with(claim.id) + assert qdrant_outbox.pending(db_path) == [ + {"op": "delete", "claim_id": claim.id, "content_hash": None} + ] From 4fb1725b234a3144299ec66a674ce1b8833fb728 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Mon, 13 Jul 2026 18:13:59 -0300 Subject: [PATCH 33/55] fix: converge entity schema authority --- .planning/REMAINING-PHASES-V1-BUDGET.md | 21 +- memorymaster/core/service.py | 66 ++-- .../govern/jobs/entity_graph_export.py | 41 ++- memorymaster/knowledge/entity_graph.py | 222 ++++++------- memorymaster/knowledge/entity_registry.py | 50 +-- memorymaster/knowledge/wiki_suggest.py | 46 ++- memorymaster/recall/graph_store.py | 53 +++- .../migrations/0013_canonical_entity_graph.py | 294 ++++++++++++++++++ memorymaster/stores/migrations/__init__.py | 13 + memorymaster/stores/storage.py | 7 - .../surfaces/cli_handlers_curation.py | 4 +- memorymaster/surfaces/mcp_server.py | 34 +- scripts/backfill_graph_store.py | 6 +- tests/test_entity_graph.py | 63 ++-- tests/test_entity_graph_export.py | 60 ++-- tests/test_entity_registry.py | 36 +-- tests/test_entity_registry_extra.py | 52 ++-- tests/test_entity_registry_merge_atomicity.py | 10 +- tests/test_entity_schema_composition.py | 8 +- tests/test_entity_schema_convergence.py | 246 +++++++++++++++ tests/test_graph_distance.py | 2 +- tests/test_graph_store.py | 8 +- tests/test_hebbian_ebbinghaus.py | 59 ++-- tests/test_wiki_suggest.py | 66 ++-- 24 files changed, 1031 insertions(+), 436 deletions(-) create mode 100644 memorymaster/stores/migrations/0013_canonical_entity_graph.py create mode 100644 tests/test_entity_schema_convergence.py diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index f57f8845..4a8b6c0f 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -202,5 +202,22 @@ commit `038779f`: - Atomic package commit: the conventional `fix: enforce lifecycle authority and read-only recall` commit containing this scheduler evidence. -The next separately authorized package is P2-D. Do not combine entity-schema -convergence with P2-C or begin it from this goal. +P2-D implementation and focused gate completed on 2026-07-13 from P2-C commit +`732cb684efd48083523bcd86200933cd6e928a7a`: + +- The integer-ID entity registry is the canonical relational authority; + immutable migration `0013` converges legacy graph-first and registry-first + SQLite stores and declares the matching Postgres schema contract. Optional + Kuzu state remains a derived projection initialized only by its explicit + backfill path. +- Relational graph, MCP, recall enrichment, wiki suggestion, export, and Kuzu + recall opens now validate readiness without lazy DDL. Missing or incompatible + schema produces an actionable initialization failure. +- Focused P2-D gate: 116 passed in 13.02s; changed-file Ruff passed. An + independent disposable initialization probe reached schema version 13 with + all four canonical entity tables and zero foreign-key violations. +- Atomic package commit: the conventional `fix: converge entity schema + authority` commit containing this scheduler evidence. + +The next separately authorized package is P2-E. Do not begin quiet finite +capture or retention work from the P2-D goal. diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index c1bf8962..ef40fb32 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -1625,38 +1625,44 @@ def _enrich_with_entity_graph( self, results: list[dict[str, Any]], query_text: str, limit: int ) -> list[dict[str, Any]]: """Add entity-related claims to query results via knowledge graph traversal.""" - try: - from memorymaster.knowledge.entity_graph import EntityGraph - db_path = str(getattr(self.store, 'db_path', '')) - if not db_path: - return results - eg = EntityGraph(db_path) - eg.ensure_tables() - # Extract entity names from query (simple word-based, no LLM needed) - query_words = [w for w in query_text.split() if len(w) > 3 and w[0].isupper()] - if not query_words: - return results - related_ids = eg.find_related_claims(query_words, hops=2, limit=limit) - existing_ids = {row["claim"].id for row in results if hasattr(row.get("claim"), "id")} - new_ids = [cid for cid in related_ids if cid not in existing_ids] - for cid in new_ids[:limit - len(results)]: - claim = self.store.get_claim(cid, include_citations=True) - if claim and claim.status != "archived": - results.append({ - "claim": claim, - "status": claim.status, - "annotation": self._annotation_for_claim(claim), - "score": 0.3, # entity-graph bonus - "lexical_score": 0.0, - "freshness_score": 0.0, - "confidence_score": claim.confidence, - "vector_score": 0.0, - "source": "entity_graph", - }) - except Exception: - pass # best-effort + from memorymaster.knowledge.entity_graph import EntityGraph + + query_words = [ + word for word in query_text.split() if len(word) > 3 and word[0].isupper() + ] + if not query_words: + return results + db_target = str( + getattr(self.store, "db_path", "") or getattr(self.store, "dsn", "") + ) + if not db_target: + return results + graph = EntityGraph(db_target, read_only=True) + related_ids = graph.find_related_claims(query_words, hops=2, limit=limit) + existing_ids = { + row["claim"].id for row in results if hasattr(row.get("claim"), "id") + } + for claim_id in [cid for cid in related_ids if cid not in existing_ids][ + : limit - len(results) + ]: + claim = self.store.get_claim(claim_id, include_citations=True) + if claim and claim.status != "archived": + results.append(self._entity_graph_row(claim)) return results + def _entity_graph_row(self, claim: Claim) -> dict[str, Any]: + return { + "claim": claim, + "status": claim.status, + "annotation": self._annotation_for_claim(claim), + "score": 0.3, + "lexical_score": 0.0, + "freshness_score": 0.0, + "confidence_score": claim.confidence, + "vector_score": 0.0, + "source": "entity_graph", + } + def _record_accesses(self, rows: list[dict[str, Any]], query_text: str = "") -> None: """Record access + feedback for each claim returned by a query. diff --git a/memorymaster/govern/jobs/entity_graph_export.py b/memorymaster/govern/jobs/entity_graph_export.py index e10f5f39..f73f2851 100644 --- a/memorymaster/govern/jobs/entity_graph_export.py +++ b/memorymaster/govern/jobs/entity_graph_export.py @@ -4,7 +4,6 @@ from dataclasses import dataclass from pathlib import Path -import json import sqlite3 import xml.etree.ElementTree as ET @@ -46,8 +45,8 @@ def export_entity_graph(db_path: str | Path, output: str | Path, fmt: str, scope if normalized_format not in SUPPORTED_FORMATS: raise ValueError(f"unsupported format: {fmt}") - graph = EntityGraph(str(db_path)) - graph.ensure_tables() + graph = EntityGraph(str(db_path), read_only=True) + graph.assert_ready() conn = graph._connect() try: nodes, edges = _load_graph(conn, scope) @@ -77,7 +76,12 @@ def _load_graph(conn: sqlite3.Connection, scope: str | None) -> tuple[list[Entit ).fetchall() node_rows = conn.execute( """ - SELECT DISTINCT en.id, en.name, en.type, en.aliases + SELECT DISTINCT en.id, en.canonical_name AS name, + en.entity_type AS type, + COALESCE(( + SELECT GROUP_CONCAT(a.original_form, ', ') + FROM entity_aliases a WHERE a.entity_id = en.id + ), '') AS aliases FROM entities en WHERE en.id IN ( SELECT cel.entity_id @@ -89,7 +93,7 @@ def _load_graph(conn: sqlite3.Connection, scope: str | None) -> tuple[list[Entit UNION SELECT e.target_id FROM entity_edges e JOIN claims c ON c.id = e.claim_id WHERE c.scope = ? ) - ORDER BY en.name COLLATE NOCASE + ORDER BY en.canonical_name COLLATE NOCASE """, (scope, scope, scope), ).fetchall() @@ -102,7 +106,12 @@ def _load_graph(conn: sqlite3.Connection, scope: str | None) -> tuple[list[Entit """ ).fetchall() node_rows = conn.execute( - "SELECT id, name, type, aliases FROM entities ORDER BY name COLLATE NOCASE" + """SELECT en.id, en.canonical_name AS name, en.entity_type AS type, + COALESCE(( + SELECT GROUP_CONCAT(a.original_form, ', ') + FROM entity_aliases a WHERE a.entity_id = en.id + ), '') AS aliases + FROM entities en ORDER BY en.canonical_name COLLATE NOCASE""" ).fetchall() return [_node_from_row(row) for row in node_rows], [_edge_from_row(row) for row in edge_rows] @@ -117,23 +126,21 @@ def _has_entity_graph_schema(conn: sqlite3.Connection) -> bool: if table_names != {"entities", "entity_edges"}: return False columns = {row["name"] for row in conn.execute("PRAGMA table_info(entities)").fetchall()} - return {"id", "name", "type", "aliases"}.issubset(columns) + return {"id", "canonical_name", "entity_type", "scope"}.issubset(columns) def _node_from_row(row: sqlite3.Row) -> EntityNode: - return EntityNode(row["id"], row["name"], row["type"], _aliases_text(row["aliases"])) + return EntityNode(str(row["id"]), row["name"], row["type"], row["aliases"] or "") def _edge_from_row(row: sqlite3.Row) -> EntityEdge: - return EntityEdge(row["source_id"], row["target_id"], row["relation"], float(row["weight"]), row["claim_id"]) - - -def _aliases_text(raw_aliases: str) -> str: - try: - aliases = json.loads(raw_aliases or "[]") - except json.JSONDecodeError: - return "" - return ", ".join(str(alias) for alias in aliases) + return EntityEdge( + str(row["source_id"]), + str(row["target_id"]), + row["relation"], + float(row["weight"]), + row["claim_id"], + ) def _render_dot(nodes: list[EntityNode], edges: list[EntityEdge]) -> str: diff --git a/memorymaster/knowledge/entity_graph.py b/memorymaster/knowledge/entity_graph.py index d9ac38b8..26cc608b 100644 --- a/memorymaster/knowledge/entity_graph.py +++ b/memorymaster/knowledge/entity_graph.py @@ -18,10 +18,10 @@ import sqlite3 import urllib.error import urllib.request -import uuid from datetime import datetime, timezone -from memorymaster.stores._storage_shared import open_conn +from memorymaster.knowledge.entity_registry import add_alias, resolve_or_create +from memorymaster.stores._storage_shared import connect_ro, open_conn logger = logging.getLogger(__name__) @@ -46,6 +46,10 @@ DEFAULT_MODEL = "deepseek-coder-v2:16b" +class EntityGraphNotReady(RuntimeError): + """Raised when the versioned relational graph schema is unavailable.""" + + def _llm_chat(prompt: str, system: str = "", model: str = "", base_url: str = "") -> str: """Call an Ollama-compatible LLM. Returns raw text response.""" url = (base_url or os.environ.get("OLLAMA_URL") or DEFAULT_OLLAMA_URL).rstrip("/") @@ -96,94 +100,85 @@ def _parse_json(raw: str) -> dict: class EntityGraph: """Entity extraction and graph storage in SQLite.""" - def __init__(self, db_path: str) -> None: + def __init__(self, db_path: str, *, read_only: bool = False) -> None: self.db_path = db_path - # Schema is created once per instance: bulk ingest calls extract_and_link - # once per claim, and re-issuing the full executescript DDL + commit on - # every claim is pure churn. Flag flips True after the first ensure. + self.read_only = bool(read_only) self._schema_ready = False def _connect(self) -> sqlite3.Connection: - return open_conn(self.db_path) - - def ensure_tables(self, conn: sqlite3.Connection | None = None) -> None: - """Create entity tables if they don't exist. Idempotent - safe to call multiple times. - - Guarded by ``self._schema_ready`` so the executescript DDL + commit runs - at most once per instance. Pass an open ``conn`` to reuse a caller's - connection (avoids opening a second one during extract_and_link). - """ + if self.db_path.startswith(("postgres://", "postgresql://")): + raise EntityGraphNotReady( + "Postgres entity extraction is not enabled; the canonical schema " + "is migration-ready but the runtime adapter remains SQLite-only." + ) + return connect_ro(self.db_path) if self.read_only else open_conn(self.db_path) + + def assert_ready(self, conn: sqlite3.Connection | None = None) -> None: + """Validate the migrated schema without creating or altering objects.""" if self._schema_ready: return own_conn = conn is None - conn = conn or self._connect() try: - conn.executescript(""" - CREATE TABLE IF NOT EXISTS entities ( - id TEXT PRIMARY KEY, - name TEXT NOT NULL, - type TEXT NOT NULL DEFAULT 'concept', - aliases TEXT NOT NULL DEFAULT '[]', - created_at TEXT NOT NULL - ); - CREATE UNIQUE INDEX IF NOT EXISTS idx_entity_name - ON entities(name COLLATE NOCASE); - CREATE TABLE IF NOT EXISTS entity_edges ( - source_id TEXT NOT NULL, - target_id TEXT NOT NULL, - relation TEXT NOT NULL DEFAULT 'related_to', - weight REAL NOT NULL DEFAULT 1.0, - claim_id INTEGER, - created_at TEXT NOT NULL, - last_reinforced_at TEXT, - PRIMARY KEY (source_id, target_id, relation) - ); - CREATE TABLE IF NOT EXISTS claim_entity_links ( - claim_id INTEGER NOT NULL, - entity_id TEXT NOT NULL, - PRIMARY KEY (claim_id, entity_id) - ); - CREATE INDEX IF NOT EXISTS idx_cel_entity - ON claim_entity_links(entity_id); - """) - self._migrate_edge_columns(conn) - conn.commit() + conn = conn or self._connect() + tables = { + row["name"] + for row in conn.execute( + "SELECT name FROM sqlite_master WHERE type='table' AND " + "name IN ('entities','entity_aliases','entity_edges','claim_entity_links')" + ) + } + columns = { + row["name"] for row in conn.execute("PRAGMA table_info(entities)") + } + if tables != { + "entities", + "entity_aliases", + "entity_edges", + "claim_entity_links", + } or not {"id", "canonical_name", "entity_type", "scope"} <= columns: + raise EntityGraphNotReady( + "Entity graph schema is not ready; run `memorymaster init-db` " + "with a schema-administration connection." + ) self._schema_ready = True except sqlite3.OperationalError as exc: - logger.warning("ensure_tables already called (idempotent): %s", exc) - conn.rollback() + raise EntityGraphNotReady( + "Entity graph schema is not ready; run `memorymaster init-db` " + "with a schema-administration connection." + ) from exc finally: - if own_conn: + if own_conn and conn is not None: conn.close() - @staticmethod - def _migrate_edge_columns(conn: sqlite3.Connection) -> None: - """Add Hebbian/Ebbinghaus columns to a pre-existing entity_edges table. - - The CREATE TABLE above only fires for fresh DBs; a DB created before the - ``last_reinforced_at`` column existed keeps its old schema. ``ALTER TABLE - ADD COLUMN`` is the additive, default-safe path — guarded so re-running - on an already-migrated DB is a silent no-op (no "duplicate column" raise). - """ - try: - cols = {row[1] for row in conn.execute("PRAGMA table_info(entity_edges)").fetchall()} - except sqlite3.OperationalError: + def ensure_tables(self, conn: sqlite3.Connection | None = None) -> None: + """Compatibility admin entrypoint backed only by immutable migrations.""" + if self._schema_ready: return - if "last_reinforced_at" not in cols: - conn.execute("ALTER TABLE entity_edges ADD COLUMN last_reinforced_at TEXT") + own_conn = conn is None + conn = conn or self._connect() + try: + from memorymaster.knowledge.entity_registry import ensure_entity_schema - def _process_entities(self, data: dict, conn) -> tuple[dict[str, str], list[str]]: + ensure_entity_schema(conn) + self.assert_ready(conn) + finally: + if own_conn: + conn.close() + + def _process_entities(self, data: dict, conn) -> tuple[dict[str, int], list[str]]: """Process extracted entities and return (name->id mapping, original names).""" entity_names = [] - entity_id_map: dict[str, str] = {} + entity_id_map: dict[str, int] = {} for ent in data.get("entities", []): name = (ent.get("name") or "").strip() if not name or len(name) < 2: continue ent_type = ent.get("type", "concept") - aliases = ent.get("aliases", []) + aliases = [str(alias) for alias in ent.get("aliases", []) if alias] ent_id = self._upsert_entity(conn, name, ent_type, aliases) + if ent_id <= 0: + continue entity_id_map[name.lower()] = ent_id entity_names.append(name) for alias in aliases: @@ -204,16 +199,9 @@ def extract_and_link(self, claim_id: int, text: str) -> list[str]: if not text: return [] - # One connection for the whole call: ensure schema (once per instance, - # guarded), read known entities, and write — instead of opening/pragma'ing - # a fresh connection per helper on every claim during bulk ingest. conn = self._connect() try: - try: - self.ensure_tables(conn) - except sqlite3.OperationalError as exc: - logger.error("Failed to ensure entity tables for claim %d: %s", claim_id, exc) - return [] + self.assert_ready(conn) known = self._get_known_entity_names(limit=30, conn=conn) context = f"\nKnown entities: {', '.join(known)}" if known else "" @@ -244,27 +232,22 @@ def extract_and_link(self, claim_id: int, text: str) -> list[str]: return entity_names def find_related_claims(self, entity_names: list[str], hops: int = 2, limit: int = 50) -> list[int]: - """Graph BFS: find claim IDs related to entities. - - Returns empty list for non-existent entities or if tables don't exist. - """ + """Graph BFS: find claim IDs related to canonical names or aliases.""" if not entity_names: return [] conn = self._connect() try: - # Check if tables exist before querying - try: - conn.execute("SELECT 1 FROM entities LIMIT 1") - except sqlite3.OperationalError: - logger.debug("Entity tables don't exist yet, returning empty list") - return [] - + self.assert_ready(conn) placeholders = ",".join("?" * len(entity_names)) names_lower = [n.lower() for n in entity_names] seed_rows = conn.execute( - f"SELECT id FROM entities WHERE LOWER(name) IN ({placeholders})", - names_lower, + f"""SELECT DISTINCT e.id + FROM entities e + LEFT JOIN entity_aliases a ON a.entity_id = e.id + WHERE LOWER(e.canonical_name) IN ({placeholders}) + OR LOWER(a.original_form) IN ({placeholders})""", + names_lower + names_lower, ).fetchall() if not seed_rows: return [] @@ -304,61 +287,40 @@ def find_related_claims(self, entity_names: list[str], hops: int = 2, limit: int seed_ids + [hops, hops, limit], ).fetchall() return [r["claim_id"] for r in rows] - except sqlite3.OperationalError as exc: - logger.warning("find_related_claims failed: %s", exc) - return [] finally: conn.close() def get_stats(self) -> dict: conn = self._connect() try: + self.assert_ready(conn) entities = conn.execute("SELECT COUNT(*) as c FROM entities").fetchone()["c"] edges = conn.execute("SELECT COUNT(*) as c FROM entity_edges").fetchone()["c"] links = conn.execute("SELECT COUNT(*) as c FROM claim_entity_links").fetchone()["c"] types = { - r["type"]: r["cnt"] - for r in conn.execute("SELECT type, COUNT(*) as cnt FROM entities GROUP BY type").fetchall() + r["entity_type"]: r["cnt"] + for r in conn.execute( + "SELECT entity_type, COUNT(*) as cnt FROM entities GROUP BY entity_type" + ).fetchall() } return {"entities": entities, "edges": edges, "claim_links": links, "by_type": types} finally: conn.close() - def _upsert_entity(self, conn, name: str, ent_type: str, aliases: list[str]) -> str: - existing = conn.execute( - "SELECT id, aliases FROM entities WHERE LOWER(name) = LOWER(?)", (name,) - ).fetchone() - if existing: - return self._merge_entity_aliases(conn, existing, aliases) - ent_id = str(uuid.uuid4()) - # INSERT OR IGNORE + re-select: a concurrent writer can insert a - # case-insensitively-equal name between the SELECT above and this INSERT. - # The NOCASE unique index would raise IntegrityError on a plain INSERT and - # abort extract_and_link; OR IGNORE makes the loser a no-op, after which we - # re-select the winning row and merge our aliases into it. - cur = conn.execute( - "INSERT OR IGNORE INTO entities (id, name, type, aliases, created_at) VALUES (?, ?, ?, ?, ?)", - (ent_id, name, ent_type, json.dumps(aliases), datetime.now(timezone.utc).isoformat()), + def _upsert_entity(self, conn, name: str, ent_type: str, aliases: list[str]) -> int: + entity_id = resolve_or_create( + conn, + name, + entity_type=ent_type or "concept", + scope="global", ) - if cur.rowcount: - return ent_id - winner = conn.execute( - "SELECT id, aliases FROM entities WHERE LOWER(name) = LOWER(?)", (name,) - ).fetchone() - if winner is None: - return ent_id # defensive: row vanished; treat our id as authoritative - return self._merge_entity_aliases(conn, winner, aliases) - - @staticmethod - def _merge_entity_aliases(conn, row, aliases: list[str]) -> str: - """Merge new aliases into an existing entity row; return its id.""" - current = json.loads(row["aliases"]) - merged = list(set(current + aliases)) - if merged != current: - conn.execute("UPDATE entities SET aliases = ? WHERE id = ?", (json.dumps(merged), row["id"])) - return row["id"] - - def _upsert_edge(self, conn, source_id: str, target_id: str, relation: str, claim_id: int) -> None: + if entity_id <= 0: + return 0 + for alias in aliases: + add_alias(conn, entity_id, alias) + return entity_id + + def _upsert_edge(self, conn, source_id: int, target_id: int, relation: str, claim_id: int) -> None: # Hebbian potentiation: every co-occurrence strengthens the edge # (weight += 0.1) and stamps last_reinforced_at = NOW. The timestamp is # what the Ebbinghaus decay job reads to compute elapsed-days; without it @@ -379,8 +341,12 @@ def _get_known_entity_names( own_conn = conn is None conn = conn or self._connect() try: - rows = conn.execute("SELECT name FROM entities ORDER BY created_at DESC LIMIT ?", (limit,)).fetchall() - return [r["name"] for r in rows] + self.assert_ready(conn) + rows = conn.execute( + "SELECT canonical_name FROM entities ORDER BY created_at DESC LIMIT ?", + (limit,), + ).fetchall() + return [r["canonical_name"] for r in rows] finally: if own_conn: conn.close() diff --git a/memorymaster/knowledge/entity_registry.py b/memorymaster/knowledge/entity_registry.py index 2325e446..380f7fe2 100644 --- a/memorymaster/knowledge/entity_registry.py +++ b/memorymaster/knowledge/entity_registry.py @@ -148,46 +148,18 @@ def _variant_key(raw: str) -> str: def ensure_entity_schema(conn: sqlite3.Connection) -> None: - """Create entity tables if they don't exist. Idempotent. - - The UNIQUE constraint on ``entity_aliases`` is on - ``(entity_id, variant_key)`` — NOT on ``alias``. The normalized ``alias`` - column is shared across variants of the same entity (by design) so - multiple original_forms (e.g. "Qdrant", "qdrant", "QDRANT", - "qdrant-cloud") can coexist as distinct rows pointing to the same - ``entity_id``. Lookup still uses ``alias`` (the heavily-normalized - form) because that's the key that collapses case/separator variation. - - NOTE: this schema is applied to *new* DBs only. Existing DBs created - with the old UNIQUE(alias) constraint require an explicit one-shot - migration — see :func:`migrate_entity_aliases_schema`. - """ - conn.executescript(""" - CREATE TABLE IF NOT EXISTS entities ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - canonical_name TEXT NOT NULL UNIQUE, - entity_type TEXT NOT NULL DEFAULT 'unknown', - scope TEXT NOT NULL DEFAULT 'global', - created_at TEXT NOT NULL, - updated_at TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS entity_aliases ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - entity_id INTEGER NOT NULL, - alias TEXT NOT NULL, - variant_key TEXT NOT NULL DEFAULT '', - original_form TEXT NOT NULL, - created_at TEXT NOT NULL, - UNIQUE(entity_id, variant_key), - FOREIGN KEY (entity_id) REFERENCES entities(id) ON DELETE CASCADE - ); + """Apply the immutable migration chain that owns the entity schema.""" + baseline = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='claims'" + ).fetchone() + if baseline is None: + raise RuntimeError( + "MemoryMaster baseline schema is missing; run `memorymaster init-db` " + "instead of initializing the entity graph independently." + ) + from memorymaster.stores.migrations import MigrationRunner - CREATE INDEX IF NOT EXISTS idx_entity_aliases_alias - ON entity_aliases(alias); - CREATE INDEX IF NOT EXISTS idx_entity_aliases_entity_id - ON entity_aliases(entity_id); - """) + MigrationRunner(conn, backend="sqlite").apply_pending() def _utc_now() -> str: diff --git a/memorymaster/knowledge/wiki_suggest.py b/memorymaster/knowledge/wiki_suggest.py index c8cfb253..5972afa8 100644 --- a/memorymaster/knowledge/wiki_suggest.py +++ b/memorymaster/knowledge/wiki_suggest.py @@ -7,7 +7,6 @@ from __future__ import annotations -import json import re import sqlite3 from dataclasses import dataclass, field @@ -68,13 +67,14 @@ def suggest_wikilinks( if not article_slugs: return [] + graph = EntityGraph(str(db), read_only=True) + graph.assert_ready() terms = _load_entity_terms(db) matched = _match_entity_terms(clean_text, terms) if not matched: return [] seed_names = [term.name for term in matched] - graph = EntityGraph(str(db)) related_claim_ids = set(graph.find_related_claims(seed_names, hops=hops, limit=max(limit * 50, 100))) if not related_claim_ids: return [] @@ -108,36 +108,28 @@ def load_wiki_article_slugs(wiki_root: str | Path) -> set[str]: def _load_entity_terms(db_path: Path) -> list[EntityTerm]: + conn = connect_ro(db_path) try: - # Read-only; a missing DB keeps the empty-result contract. - conn = connect_ro(db_path) - except sqlite3.OperationalError: - return [] - try: - rows = conn.execute("SELECT id, name, aliases FROM entities").fetchall() - except sqlite3.OperationalError: - return [] + rows = conn.execute( + """SELECT e.id, e.canonical_name, a.original_form + FROM entities e + LEFT JOIN entity_aliases a ON a.entity_id = e.id + ORDER BY e.id, a.id""" + ).fetchall() finally: conn.close() - terms: list[EntityTerm] = [] + grouped: dict[int, tuple[str, list[str]]] = {} for row in rows: - aliases = _parse_aliases(row["aliases"]) - values = tuple(dict.fromkeys([row["name"], *aliases])) - terms.append(EntityTerm(entity_id=str(row["id"]), name=str(row["name"]), terms=values)) - return terms - - -def _parse_aliases(raw: str | None) -> list[str]: - if not raw: - return [] - try: - parsed = json.loads(raw) - except json.JSONDecodeError: - return [] - if not isinstance(parsed, list): - return [] - return [str(item) for item in parsed if str(item).strip()] + entity_id = int(row["id"]) + name = str(row["canonical_name"]) + grouped.setdefault(entity_id, (name, [])) + if row["original_form"]: + grouped[entity_id][1].append(str(row["original_form"])) + return [ + EntityTerm(str(entity_id), name, tuple(dict.fromkeys([name, *aliases]))) + for entity_id, (name, aliases) in grouped.items() + ] def _match_entity_terms(text: str, terms: list[EntityTerm]) -> list[EntityTerm]: diff --git a/memorymaster/recall/graph_store.py b/memorymaster/recall/graph_store.py index 4bd7f4c5..228da04f 100644 --- a/memorymaster/recall/graph_store.py +++ b/memorymaster/recall/graph_store.py @@ -87,13 +87,28 @@ def __init__(self, path: Path | str) -> None: # lifecycle # ------------------------------------------------------------------ def open(self) -> None: - """Open the Kuzu DB, creating the schema if it does not exist. + """Open an initialized Kuzu DB without creating or repairing schema. - Raises :class:`GraphStoreUnavailable` when Kuzu is not importable - or the DB path cannot be created. The exception message is + Raises :class:`GraphStoreUnavailable` when Kuzu is not importable, + the DB is absent, or the required graph tables are unavailable. The + message is intentionally generic — callers log it at DEBUG level because the graph stream is an opt-in enhancement, not a correctness layer. """ + if not self.path.exists(): + raise GraphStoreUnavailable( + f"graph schema is not initialized at {self.path}; " + "run the graph backfill initializer" + ) + self._open_backend(create_parent=False) + self._assert_schema_ready() + + def initialize(self) -> None: + """Explicitly create or evolve the optional derived graph store.""" + self._open_backend(create_parent=True) + self._ensure_schema() + + def _open_backend(self, *, create_parent: bool) -> None: if self._conn is not None: return try: @@ -108,7 +123,8 @@ def open(self) -> None: # We still accept ``.kuzu`` paths for forward compatibility — # just make sure the PARENT directory exists and let Kuzu # create the file itself. - self.path.parent.mkdir(parents=True, exist_ok=True) + if create_parent: + self.path.parent.mkdir(parents=True, exist_ok=True) self._db = kuzu.Database(str(self.path)) self._conn = kuzu.Connection(self._db) except Exception as exc: @@ -120,7 +136,22 @@ def open(self) -> None: f"kuzu open failed at {self.path}: {exc!r}" ) from exc - self._ensure_schema() + + def _assert_schema_ready(self) -> None: + """Raise an actionable error when a read opens an incomplete store.""" + assert self._conn is not None + try: + self._conn.execute("MATCH (c:Claim) RETURN count(c)") + self._conn.execute("MATCH (e:Entity) RETURN count(e)") + self._conn.execute( + f"MATCH (:Claim)-[m:{MENTIONS_REL}]->(:Entity) RETURN count(m)" + ) + except Exception as exc: + self.close() + raise GraphStoreUnavailable( + f"graph schema is not ready at {self.path}; " + "run the graph backfill initializer" + ) from exc def close(self) -> None: """Close the Kuzu connection + database. Safe to call twice.""" @@ -550,16 +581,22 @@ def open_graph_store( path: Path | str, *, allow_networkx: bool = False, + initialize: bool = False, ) -> "GraphStore | _NetworkXGraphStore": - """Factory — return an opened Kuzu store, or the networkx fallback - when ``allow_networkx=True`` and Kuzu is unavailable. + """Return an opened Kuzu store, or an explicitly allowed networkx fallback. + + ``initialize=True`` is reserved for administration/backfill paths. Normal + recall opens and validates existing state without creating schema. Raises :class:`GraphStoreUnavailable` when Kuzu fails AND ``allow_networkx`` is False. """ try: store = GraphStore(path) - store.open() + if initialize: + store.initialize() + else: + store.open() return store except GraphStoreUnavailable: if not allow_networkx: diff --git a/memorymaster/stores/migrations/0013_canonical_entity_graph.py b/memorymaster/stores/migrations/0013_canonical_entity_graph.py new file mode 100644 index 00000000..c79e665b --- /dev/null +++ b/memorymaster/stores/migrations/0013_canonical_entity_graph.py @@ -0,0 +1,294 @@ +"""Converge the entity registry and relational graph on integer entity IDs.""" + +from __future__ import annotations + +import json +import re +from datetime import datetime, timezone + +VERSION = 13 +DESCRIPTION = "Canonical entity registry and relational graph" + +ENTITY_SCHEMA_CONTRACT = { + "entities": ("id", "canonical_name", "entity_type", "scope"), + "entity_aliases": ("id", "entity_id", "alias", "variant_key"), + "entity_edges": ("source_id", "target_id", "relation", "claim_id"), + "claim_entity_links": ("claim_id", "entity_id"), +} + +_NORMALIZE_RE = re.compile(r"[\s_\-\.]+") + +_SQLITE_REGISTRY_DDL = """ +CREATE TABLE IF NOT EXISTS entities ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + canonical_name TEXT NOT NULL UNIQUE, + entity_type TEXT NOT NULL DEFAULT 'unknown', + scope TEXT NOT NULL DEFAULT 'global', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS entity_aliases ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + alias TEXT NOT NULL, + variant_key TEXT NOT NULL DEFAULT '', + original_form TEXT NOT NULL, + created_at TEXT NOT NULL, + UNIQUE(entity_id, variant_key) +); +CREATE INDEX IF NOT EXISTS idx_entity_aliases_alias ON entity_aliases(alias); +CREATE INDEX IF NOT EXISTS idx_entity_aliases_entity_id ON entity_aliases(entity_id); +""" + +_SQLITE_GRAPH_DDL = """ +CREATE TABLE entity_edges ( + source_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + target_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + relation TEXT NOT NULL DEFAULT 'related_to', + weight REAL NOT NULL DEFAULT 1.0, + claim_id INTEGER REFERENCES claims(id) ON DELETE SET NULL, + created_at TEXT NOT NULL, + last_reinforced_at TEXT, + PRIMARY KEY (source_id, target_id, relation) +); +CREATE TABLE claim_entity_links ( + claim_id INTEGER NOT NULL REFERENCES claims(id) ON DELETE CASCADE, + entity_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + PRIMARY KEY (claim_id, entity_id) +); +CREATE INDEX idx_cel_entity ON claim_entity_links(entity_id); +""" + + +def _utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def _columns(conn, table: str) -> set[str]: + return {str(row[1]) for row in conn.execute(f"PRAGMA table_info({table})")} + + +def _table_exists(conn, table: str) -> bool: + return conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,) + ).fetchone() is not None + + +def _normalized(value: str) -> str: + return _NORMALIZE_RE.sub("-", value.strip().lower()).strip("-")[:200] + + +def _variant(value: str) -> str: + return re.sub(r"\s+", " ", value.strip())[:200] + + +def _insert_alias(conn, entity_id: int, value: str, created_at: str) -> None: + alias = _normalized(value) + if not alias: + return + conn.execute( + """INSERT OR IGNORE INTO entity_aliases + (entity_id, alias, variant_key, original_form, created_at) + VALUES (?, ?, ?, ?, ?)""", + (entity_id, alias, _variant(value), value.strip()[:200], created_at), + ) + + +def _rename_legacy_graph(conn) -> tuple[str | None, str | None]: + edge_table = None + link_table = None + if _table_exists(conn, "entity_edges"): + conn.execute("ALTER TABLE entity_edges RENAME TO entity_edges_legacy_0013") + edge_table = "entity_edges_legacy_0013" + if _table_exists(conn, "claim_entity_links"): + conn.execute( + "ALTER TABLE claim_entity_links RENAME TO claim_entity_links_legacy_0013" + ) + link_table = "claim_entity_links_legacy_0013" + conn.execute("DROP INDEX IF EXISTS idx_cel_entity") + return edge_table, link_table + + +def _migrate_legacy_entities(conn) -> dict[str, int]: + if not _table_exists(conn, "entities"): + conn.executescript(_SQLITE_REGISTRY_DDL) + return {} + columns = _columns(conn, "entities") + if "canonical_name" in columns: + required = { + "id", + "canonical_name", + "entity_type", + "scope", + "created_at", + "updated_at", + } + if not required <= columns: + raise RuntimeError( + "Entity migration found an incomplete canonical registry." + ) + conn.executescript(_SQLITE_REGISTRY_DDL) + return {str(row[0]): int(row[0]) for row in conn.execute("SELECT id FROM entities")} + if not {"id", "name", "type", "aliases", "created_at"} <= columns: + raise RuntimeError("Entity migration found an unsupported entities table shape.") + + conn.execute("ALTER TABLE entities RENAME TO entities_legacy_0013") + conn.executescript(_SQLITE_REGISTRY_DDL) + mapping: dict[str, int] = {} + rows = conn.execute( + "SELECT id, name, type, aliases, created_at FROM entities_legacy_0013 ORDER BY name" + ).fetchall() + for old_id, name, entity_type, aliases_json, created_at in rows: + stamp = str(created_at or _utc_now()) + cur = conn.execute( + """INSERT INTO entities + (canonical_name, entity_type, scope, created_at, updated_at) + VALUES (?, ?, 'global', ?, ?)""", + (str(name), str(entity_type or "unknown"), stamp, stamp), + ) + entity_id = int(cur.lastrowid) + mapping[str(old_id)] = entity_id + _insert_alias(conn, entity_id, str(name), stamp) + try: + aliases = json.loads(str(aliases_json or "[]")) + except json.JSONDecodeError: + aliases = [] + for alias in aliases if isinstance(aliases, list) else []: + _insert_alias(conn, entity_id, str(alias), stamp) + return mapping + + +def _migrate_legacy_aliases(conn) -> None: + if not _table_exists(conn, "entity_aliases"): + conn.executescript(_SQLITE_REGISTRY_DDL) + return + columns = _columns(conn, "entity_aliases") + required = {"id", "entity_id", "alias", "original_form", "created_at"} + if not required <= columns: + raise RuntimeError("Entity migration found an unsupported alias table shape.") + fk_targets = { + str(row[2]) for row in conn.execute("PRAGMA foreign_key_list(entity_aliases)") + } + if "variant_key" in columns and fk_targets == {"entities"}: + return + + conn.executescript( + """ + DROP INDEX IF EXISTS idx_entity_aliases_alias; + DROP INDEX IF EXISTS idx_entity_aliases_entity_id; + ALTER TABLE entity_aliases RENAME TO entity_aliases_legacy_0013; + """ + ) + conn.executescript(_SQLITE_REGISTRY_DDL) + variant_expr = "variant_key" if "variant_key" in columns else "NULL" + rows = conn.execute( + "SELECT entity_id, alias, original_form, created_at, " + f"{variant_expr} FROM entity_aliases_legacy_0013 ORDER BY id" + ).fetchall() + for entity_id, alias, original_form, created_at, variant_key in rows: + display = str(original_form or alias or "") + conn.execute( + """INSERT OR IGNORE INTO entity_aliases + (entity_id, alias, variant_key, original_form, created_at) + VALUES (?, ?, ?, ?, ?)""", + (entity_id, alias, variant_key or _variant(display), display, created_at), + ) + conn.execute("DROP TABLE entity_aliases_legacy_0013") + + +def _claim_exists(conn, claim_id: object) -> bool: + if claim_id is None or not _table_exists(conn, "claims"): + return claim_id is None + return conn.execute("SELECT 1 FROM claims WHERE id = ?", (claim_id,)).fetchone() is not None + + +def _migrate_graph_rows(conn, mapping: dict[str, int], edge_table: str | None, link_table: str | None) -> None: + conn.executescript(_SQLITE_GRAPH_DDL) + if edge_table: + edge_columns = _columns(conn, edge_table) + reinforced = ( + "last_reinforced_at" + if "last_reinforced_at" in edge_columns + else "NULL AS last_reinforced_at" + ) + rows = conn.execute( + f"SELECT source_id, target_id, relation, weight, claim_id, created_at, " + f"{reinforced} FROM {edge_table}" + ).fetchall() + for source, target, relation, weight, claim_id, created_at, reinforced in rows: + source_id = mapping.get(str(source)) + target_id = mapping.get(str(target)) + if source_id is None or target_id is None or not _claim_exists(conn, claim_id): + raise RuntimeError("Entity migration refused orphaned legacy graph rows.") + conn.execute( + """INSERT INTO entity_edges VALUES (?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(source_id, target_id, relation) DO NOTHING""", + (source_id, target_id, relation, weight, claim_id, created_at, reinforced), + ) + if link_table: + for claim_id, entity_id in conn.execute( + f"SELECT claim_id, entity_id FROM {link_table}" + ).fetchall(): + mapped = mapping.get(str(entity_id)) + if mapped is None or not _claim_exists(conn, claim_id): + raise RuntimeError("Entity migration refused orphaned legacy claim links.") + conn.execute( + "INSERT OR IGNORE INTO claim_entity_links VALUES (?, ?)", + (claim_id, mapped), + ) + + +def _add_claim_entity_reference(conn) -> None: + if not _table_exists(conn, "claims") or "entity_id" in _columns(conn, "claims"): + return + conn.execute( + "ALTER TABLE claims ADD COLUMN entity_id INTEGER REFERENCES entities(id) ON DELETE SET NULL" + ) + + +def apply_sqlite(conn) -> None: + edge_table, link_table = _rename_legacy_graph(conn) + mapping = _migrate_legacy_entities(conn) + _migrate_legacy_aliases(conn) + _add_claim_entity_reference(conn) + _migrate_graph_rows(conn, mapping, edge_table, link_table) + for table in (edge_table, link_table, "entities_legacy_0013"): + if table: + conn.execute(f"DROP TABLE IF EXISTS {table}") + violations = conn.execute("PRAGMA foreign_key_check").fetchall() + if violations: + raise RuntimeError(f"Entity migration produced {len(violations)} FK violation(s).") + conn.commit() + + +_POSTGRES_STATEMENTS = ( + """CREATE TABLE IF NOT EXISTS entities ( + id BIGSERIAL PRIMARY KEY, canonical_name TEXT NOT NULL UNIQUE, + entity_type TEXT NOT NULL DEFAULT 'unknown', scope TEXT NOT NULL DEFAULT 'global', + created_at TEXT NOT NULL, updated_at TEXT NOT NULL)""", + """CREATE TABLE IF NOT EXISTS entity_aliases ( + id BIGSERIAL PRIMARY KEY, entity_id BIGINT NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + alias TEXT NOT NULL, variant_key TEXT NOT NULL DEFAULT '', original_form TEXT NOT NULL, + created_at TEXT NOT NULL, UNIQUE(entity_id, variant_key))""", + "CREATE INDEX IF NOT EXISTS idx_entity_aliases_alias ON entity_aliases(alias)", + "CREATE INDEX IF NOT EXISTS idx_entity_aliases_entity_id ON entity_aliases(entity_id)", + "ALTER TABLE claims ADD COLUMN IF NOT EXISTS entity_id BIGINT REFERENCES entities(id) ON DELETE SET NULL", + """CREATE TABLE IF NOT EXISTS entity_edges ( + source_id BIGINT NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + target_id BIGINT NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + relation TEXT NOT NULL DEFAULT 'related_to', weight DOUBLE PRECISION NOT NULL DEFAULT 1.0, + claim_id BIGINT REFERENCES claims(id) ON DELETE SET NULL, created_at TEXT NOT NULL, + last_reinforced_at TEXT, PRIMARY KEY(source_id, target_id, relation))""", + """CREATE TABLE IF NOT EXISTS claim_entity_links ( + claim_id BIGINT NOT NULL REFERENCES claims(id) ON DELETE CASCADE, + entity_id BIGINT NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + PRIMARY KEY(claim_id, entity_id))""", + "CREATE INDEX IF NOT EXISTS idx_cel_entity ON claim_entity_links(entity_id)", +) + + +def apply_postgres(conn) -> None: + with conn.cursor() as cur: + for statement in _POSTGRES_STATEMENTS: + cur.execute(statement) + conn.commit() diff --git a/memorymaster/stores/migrations/__init__.py b/memorymaster/stores/migrations/__init__.py index 09571f52..c85ff34b 100644 --- a/memorymaster/stores/migrations/__init__.py +++ b/memorymaster/stores/migrations/__init__.py @@ -41,8 +41,21 @@ def apply_postgres(conn) -> None: ... # Postgres-backend DDL discover_migrations, ) + +def entity_schema_contract(backend: str) -> dict[str, tuple[str, ...]]: + """Return the backend-independent P2-D entity schema contract.""" + if backend not in {"sqlite", "postgres"}: + raise ValueError("backend must be 'sqlite' or 'postgres'") + from importlib import import_module + + migration = import_module( + "memorymaster.stores.migrations.0013_canonical_entity_graph" + ) + return dict(migration.ENTITY_SCHEMA_CONTRACT) + __all__ = [ "MigrationDriftError", "MigrationRunner", "discover_migrations", + "entity_schema_contract", ] diff --git a/memorymaster/stores/storage.py b/memorymaster/stores/storage.py index 7e77da86..aec6f83d 100644 --- a/memorymaster/stores/storage.py +++ b/memorymaster/stores/storage.py @@ -156,13 +156,6 @@ def init_db(self) -> None: self._ensure_version_column(conn) self._ensure_embeddings_schema(conn) self._ensure_atlas_source_schema(conn) - # Entity registry (GBrain-inspired canonical entities + alias resolution) - from memorymaster.knowledge.entity_registry import ensure_entity_schema - ensure_entity_schema(conn) - try: - conn.execute("ALTER TABLE claims ADD COLUMN entity_id INTEGER") - except sqlite3.OperationalError: - pass # already exists conn.commit() # v3.20.0-S1: apply versioned migrations after the legacy init flow. diff --git a/memorymaster/surfaces/cli_handlers_curation.py b/memorymaster/surfaces/cli_handlers_curation.py index f0f1cf07..822e9881 100644 --- a/memorymaster/surfaces/cli_handlers_curation.py +++ b/memorymaster/surfaces/cli_handlers_curation.py @@ -492,7 +492,6 @@ def _handle_extract_entities(args: argparse.Namespace, service, parser: argparse from memorymaster.knowledge.entity_graph import EntityGraph t0 = time.perf_counter() eg = EntityGraph(str(effective_db)) - eg.ensure_tables() claims = service.store.find_by_status(args.status, limit=args.limit, include_citations=False) total_entities = 0 for claim in claims: @@ -512,8 +511,7 @@ def _handle_extract_entities(args: argparse.Namespace, service, parser: argparse def _handle_entity_stats(args: argparse.Namespace, service, parser: argparse.ArgumentParser, effective_db: str) -> int: from memorymaster.knowledge.entity_graph import EntityGraph - eg = EntityGraph(str(effective_db)) - eg.ensure_tables() + eg = EntityGraph(str(effective_db), read_only=True) stats = eg.get_stats() if args.json_output: print(_json_envelope(stats)) diff --git a/memorymaster/surfaces/mcp_server.py b/memorymaster/surfaces/mcp_server.py index 5c9a327b..a10cb1dd 100644 --- a/memorymaster/surfaces/mcp_server.py +++ b/memorymaster/surfaces/mcp_server.py @@ -2020,7 +2020,7 @@ def extract_entities( workspace: str = ".", ) -> dict[str, Any]: """Extract entities from a claim's text and link them to the knowledge graph.""" - from memorymaster.knowledge.entity_graph import EntityGraph + from memorymaster.knowledge.entity_graph import EntityGraph, EntityGraphNotReady svc = _service(db, workspace) if not text: claim = svc.store.get_claim(claim_id, include_citations=False) @@ -2028,24 +2028,32 @@ def extract_entities( return {"ok": False, "error": f"Claim {claim_id} not found"} text = claim.text eg = EntityGraph(_resolve_db(db)) - eg.ensure_tables() - names = eg.extract_and_link(claim_id, text) + try: + names = eg.extract_and_link(claim_id, text) + except EntityGraphNotReady as exc: + return _structured_error(str(exc), "ENTITY_GRAPH_NOT_READY") return {"ok": True, "entities": names, "count": len(names)} @mcp.tool() def entity_stats( db: str = "memorymaster.db", + workspace: str = ".", ) -> dict[str, Any]: """Get entity knowledge graph statistics.""" - from memorymaster.knowledge.entity_graph import EntityGraph - eg = EntityGraph(_resolve_db(db)) - eg.ensure_tables() - return {"ok": True, **eg.get_stats()} + from memorymaster.knowledge.entity_graph import EntityGraph, EntityGraphNotReady + + _read_service(db, workspace) + graph = EntityGraph(_resolve_db(db), read_only=True) + try: + return {"ok": True, **graph.get_stats()} + except EntityGraphNotReady as exc: + return _structured_error(str(exc), "ENTITY_GRAPH_NOT_READY") @mcp.tool() def find_related_claims( entity_names: str, db: str = "memorymaster.db", + workspace: str = ".", hops: int = 2, limit: int = 50, ) -> dict[str, Any]: @@ -2053,11 +2061,15 @@ def find_related_claims( entity_names: comma-separated entity names to search from. """ - from memorymaster.knowledge.entity_graph import EntityGraph - eg = EntityGraph(_resolve_db(db)) - eg.ensure_tables() + from memorymaster.knowledge.entity_graph import EntityGraph, EntityGraphNotReady + + _read_service(db, workspace) + graph = EntityGraph(_resolve_db(db), read_only=True) names = [n.strip() for n in entity_names.split(",") if n.strip()] - claim_ids = eg.find_related_claims(names, hops=hops, limit=limit) + try: + claim_ids = graph.find_related_claims(names, hops=hops, limit=limit) + except EntityGraphNotReady as exc: + return _structured_error(str(exc), "ENTITY_GRAPH_NOT_READY") return {"ok": True, "claim_ids": claim_ids, "count": len(claim_ids)} @mcp.tool() diff --git a/scripts/backfill_graph_store.py b/scripts/backfill_graph_store.py index 1807c463..09f39837 100644 --- a/scripts/backfill_graph_store.py +++ b/scripts/backfill_graph_store.py @@ -149,7 +149,11 @@ def run_backfill( store = None if not dry_run: try: - store = open_graph_store(graph_path, allow_networkx=allow_networkx) + store = open_graph_store( + graph_path, + allow_networkx=allow_networkx, + initialize=True, + ) except GraphStoreUnavailable as exc: logger.error( "graph_store unavailable: %s — pass --allow-networkx for " diff --git a/tests/test_entity_graph.py b/tests/test_entity_graph.py index 80f2b47c..2e24a2ee 100644 --- a/tests/test_entity_graph.py +++ b/tests/test_entity_graph.py @@ -9,7 +9,27 @@ import pytest -from memorymaster.knowledge.entity_graph import EntityGraph, _llm_chat, _parse_json +from memorymaster.core.service import MemoryService +from memorymaster.knowledge.entity_graph import ( + EntityGraph, + EntityGraphNotReady, + _llm_chat, + _parse_json, +) + + +def _seed_claims(graph: EntityGraph, *claim_ids: int) -> None: + now = "2026-01-01T00:00:00+00:00" + conn = graph._connect() + try: + conn.executemany( + "INSERT INTO claims (id, text, scope, status, created_at, updated_at) " + "VALUES (?, 'entity graph test', 'project:test', 'confirmed', ?, ?)", + [(claim_id, now, now) for claim_id in claim_ids], + ) + conn.commit() + finally: + conn.close() class TestParseJson: @@ -68,6 +88,7 @@ def db_path(self): @pytest.fixture def graph(self, db_path): """Create EntityGraph instance.""" + MemoryService(db_path, workspace_root=db_path.parent).init_db() return EntityGraph(str(db_path)) def test_ensure_tables_creates_schema(self, graph): @@ -108,6 +129,7 @@ def test_extract_and_link_empty_response(self, mock_llm, graph): def test_extract_and_link_with_entities(self, mock_llm, graph): """extract_and_link extracts and links entities.""" graph.ensure_tables() + _seed_claims(graph, 42) mock_llm.return_value = json.dumps({ "entities": [ {"name": "Alice", "type": "person", "aliases": ["Ally"]}, @@ -140,6 +162,7 @@ def test_extract_and_link_with_entities(self, mock_llm, graph): def test_extract_and_link_skips_short_names(self, mock_llm, graph): """extract_and_link skips entity names with length < 2.""" graph.ensure_tables() + _seed_claims(graph, 1) mock_llm.return_value = json.dumps({ "entities": [ {"name": "A", "type": "person", "aliases": []}, # Should skip @@ -162,6 +185,7 @@ def test_find_related_claims_empty(self, graph): def test_find_related_claims_with_data(self, mock_llm, graph): """find_related_claims finds claims via entity relationships.""" graph.ensure_tables() + _seed_claims(graph, 1, 2) # Extract entities for claim 1 mock_llm.return_value = json.dumps({ @@ -193,6 +217,7 @@ def test_get_stats_empty_graph(self, graph): def test_get_stats_with_data(self, mock_llm, graph): """get_stats returns accurate counts.""" graph.ensure_tables() + _seed_claims(graph, 1) mock_llm.return_value = json.dumps({ "entities": [ {"name": "Alice", "type": "person", "aliases": []}, @@ -222,34 +247,33 @@ def db_path(self): yield Path(tmpdir) / "race.db" def test_concurrent_caseonly_insert_does_not_raise(self, db_path): + MemoryService(db_path, workspace_root=db_path.parent).init_db() graph = EntityGraph(str(db_path)) graph.ensure_tables() conn = graph._connect() try: - # Simulate the race: SELECT sees nothing, then a concurrent writer - # inserts "Acme" before our INSERT for "ACME" (NOCASE-equal) lands. - assert conn.execute( - "SELECT id FROM entities WHERE LOWER(name) = LOWER(?)", ("ACME",) - ).fetchone() is None + # Simulate the winning writer through the canonical registry so its + # normalized alias is present before the case variant arrives. other = graph._connect() try: - other.execute( - "INSERT INTO entities (id, name, type, aliases, created_at) " - "VALUES ('winner', 'Acme', 'org', '[]', '2026-01-01T00:00:00')" - ) + winner_id = graph._upsert_entity(other, "Acme", "org", []) other.commit() finally: other.close() # Must resolve to the existing row, not raise IntegrityError. ent_id = graph._upsert_entity(conn, "ACME", "org", ["alias-from-loser"]) conn.commit() - assert ent_id == "winner" + assert ent_id == winner_id # Our aliases were merged into the winning row. - row = conn.execute("SELECT aliases FROM entities WHERE id = 'winner'").fetchone() - assert "alias-from-loser" in json.loads(row["aliases"]) + aliases = conn.execute( + "SELECT original_form FROM entity_aliases WHERE entity_id = ?", + (ent_id,), + ).fetchall() + assert "alias-from-loser" in {row["original_form"] for row in aliases} # No duplicate row was created. count = conn.execute( - "SELECT COUNT(*) AS c FROM entities WHERE LOWER(name) = 'acme'" + "SELECT COUNT(*) AS c FROM entities " + "WHERE LOWER(canonical_name) = 'acme'" ).fetchone()["c"] assert count == 1 finally: @@ -285,6 +309,7 @@ def _wrapper(conn=None): return _wrapper, calls def test_ensure_tables_runs_executescript_once(self, db_path): + MemoryService(db_path, workspace_root=db_path.parent).init_db() graph = EntityGraph(str(db_path)) assert graph._schema_ready is False wrapper, calls = self._count_real_ddl(graph) @@ -296,14 +321,12 @@ def test_ensure_tables_runs_executescript_once(self, db_path): assert calls["n"] == 1, f"DDL ran {calls['n']} times, expected 1" @patch("memorymaster.knowledge.entity_graph._llm_chat") - def test_extract_and_link_does_not_reissue_ddl_per_claim(self, mock_llm, db_path): + def test_extract_and_link_does_not_initialize_schema(self, mock_llm, db_path): mock_llm.return_value = json.dumps({"entities": [], "relations": []}) graph = EntityGraph(str(db_path)) wrapper, calls = self._count_real_ddl(graph) graph.ensure_tables = wrapper - graph.extract_and_link(claim_id=1, text="one") - graph.extract_and_link(claim_id=2, text="two") - graph.extract_and_link(claim_id=3, text="three") - # DDL issued exactly once across three claims, not three times. - assert calls["n"] == 1, f"DDL issued {calls['n']} times across 3 claims" + with pytest.raises(EntityGraphNotReady, match="init-db"): + graph.extract_and_link(claim_id=1, text="one") + assert calls["n"] == 0 diff --git a/tests/test_entity_graph_export.py b/tests/test_entity_graph_export.py index 012896ed..ce0640ee 100644 --- a/tests/test_entity_graph_export.py +++ b/tests/test_entity_graph_export.py @@ -5,54 +5,52 @@ import xml.etree.ElementTree as ET from memorymaster.surfaces.cli import main -from memorymaster.knowledge.entity_graph import EntityGraph +from memorymaster.core.service import MemoryService from memorymaster.govern.jobs.entity_graph_export import export_entity_graph def _init_db(path) -> None: - EntityGraph(str(path)).ensure_tables() - with sqlite3.connect(path) as conn: - conn.execute( - """ - CREATE TABLE claims ( - id INTEGER PRIMARY KEY, - text TEXT NOT NULL, - scope TEXT NOT NULL - ) - """ - ) + MemoryService(path, workspace_root=path.parent).init_db() def _seed_five_entity_graph(path) -> None: now = datetime.now(timezone.utc).isoformat() entities = [ - ("alice", "Alice", "person", '["A. Example"]'), - ("bob", "Bob", "person", "[]"), - ("acme", "Acme", "org", "[]"), - ("qdrant", "Qdrant", "product", "[]"), - ("memorymaster", "MemoryMaster", "project", "[]"), + (1, "Alice", "person"), + (2, "Bob", "person"), + (3, "Acme", "org"), + (4, "Qdrant", "product"), + (5, "MemoryMaster", "project"), ] edges = [ - ("alice", "acme", "works_at", 1.0, 1), - ("bob", "acme", "works_at", 1.0, 1), - ("memorymaster", "qdrant", "uses", 2.5, 2), - ("memorymaster", "acme", "depends_on", 1.2, 2), + (1, 3, "works_at", 1.0, 1), + (2, 3, "works_at", 1.0, 1), + (5, 4, "uses", 2.5, 2), + (5, 3, "depends_on", 1.2, 2), ] with sqlite3.connect(path) as conn: conn.executemany( """ - INSERT INTO claims (id, text, scope) - VALUES (?, ?, ?) + INSERT INTO claims (id, text, scope, status, created_at, updated_at) + VALUES (?, ?, ?, 'confirmed', ?, ?) """, [ - (1, "people at acme", "project:foo"), - (2, "memorymaster uses qdrant", "project:foo"), - (3, "other scope", "project:bar"), + (1, "people at acme", "project:foo", now, now), + (2, "memorymaster uses qdrant", "project:foo", now, now), + (3, "other scope", "project:bar", now, now), ], ) conn.executemany( - "INSERT INTO entities (id, name, type, aliases, created_at) VALUES (?, ?, ?, ?, ?)", - [(entity_id, name, entity_type, aliases, now) for entity_id, name, entity_type, aliases in entities], + "INSERT INTO entities " + "(id, canonical_name, entity_type, scope, created_at, updated_at) " + "VALUES (?, ?, ?, 'global', ?, ?)", + [(entity_id, name, entity_type, now, now) for entity_id, name, entity_type in entities], + ) + conn.execute( + "INSERT INTO entity_aliases " + "(entity_id, alias, variant_key, original_form, created_at) " + "VALUES (1, 'a-example', 'A. Example', 'A. Example', ?)", + (now,), ) conn.executemany( """ @@ -63,7 +61,7 @@ def _seed_five_entity_graph(path) -> None: ) conn.executemany( "INSERT INTO claim_entity_links (claim_id, entity_id) VALUES (?, ?)", - [(1, "alice"), (1, "bob"), (1, "acme"), (2, "memorymaster"), (2, "qdrant")], + [(1, 1), (1, 2), (1, 3), (2, 5), (2, 4)], ) @@ -107,7 +105,7 @@ def test_export_five_entity_graph_dot_with_scope(tmp_path): assert result.nodes == 5 assert result.edges == 4 - assert '"memorymaster" -> "qdrant"' in text + assert '"5" -> "4"' in text assert 'label="uses"' in text assert 'aliases="A. Example"' in text @@ -126,7 +124,7 @@ def test_export_five_entity_graph_graphml_with_scope(tmp_path): assert result.edges == 4 assert len(root.findall(".//g:node", ns)) == 5 assert len(root.findall(".//g:edge", ns)) == 4 - assert root.find(".//g:edge[@source='memorymaster'][@target='qdrant']", ns) is not None + assert root.find(".//g:edge[@source='5'][@target='4']", ns) is not None def test_entity_graph_export_cli_writes_dot(tmp_path): diff --git a/tests/test_entity_registry.py b/tests/test_entity_registry.py index d43d7526..cfac365e 100644 --- a/tests/test_entity_registry.py +++ b/tests/test_entity_registry.py @@ -12,6 +12,7 @@ import pytest +from memorymaster.stores._storage_schema import load_schema_sql from memorymaster.knowledge.entity_registry import ( _has_legacy_alias_unique, _variant_key, @@ -86,17 +87,19 @@ def test_different_surface_forms_stay_distinct(self): def _fresh_db() -> sqlite3.Connection: """New in-memory DB with the claims + entity tables wired up.""" conn = sqlite3.connect(":memory:") - conn.execute(""" - CREATE TABLE claims ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - subject TEXT, - entity_id INTEGER - ) - """) + conn.executescript(load_schema_sql()) ensure_entity_schema(conn) return conn +def _insert_claim(conn: sqlite3.Connection, subject: str) -> None: + conn.execute( + "INSERT INTO claims (text, subject, scope, status, created_at, updated_at) " + "VALUES ('entity registry test', ?, 'project:test', 'candidate', 't', 't')", + (subject,), + ) + + class TestResolveOrCreate: def test_same_normalized_form_same_entity(self): """'Qdrant', 'qdrant', 'QDRANT' must resolve to ONE entity.""" @@ -201,7 +204,7 @@ def test_four_subjects(self): "qdrant-cloud", ] for s in subjects: - conn.execute("INSERT INTO claims (subject) VALUES (?)", (s,)) + _insert_claim(conn, s) ids = [resolve_or_create(conn, s) for s in subjects] # First three (case variants of "qdrant") → same entity. @@ -248,13 +251,7 @@ def test_all_four_resolve_to_same_entity_if_we_alias_manually(self): def _legacy_schema_db() -> sqlite3.Connection: """Simulate a pre-fix DB: UNIQUE on alias, no variant_key column.""" conn = sqlite3.connect(":memory:") - conn.execute(""" - CREATE TABLE claims ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - subject TEXT, - entity_id INTEGER - ) - """) + conn.executescript(load_schema_sql()) conn.executescript(""" CREATE TABLE entities ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -338,7 +335,7 @@ def test_backfill_reaches_avg_above_three(self): "Qdrant", "qdrant", "QDRANT", " qdrant", ] for s in test_subjects: - conn.execute("INSERT INTO claims (subject) VALUES (?)", (s,)) + _insert_claim(conn, s) out = backfill_entities_normalized(conn, migrate_schema=False) assert out["total_entities"] == 5, out @@ -348,7 +345,7 @@ def test_backfill_migrates_legacy_then_backfills(self): legacy = _legacy_schema_db() # Old-schema DB with 3 claims but only 1 alias (the bug). for s in ["Paperclip", "paperclip", "PAPERCLIP"]: - legacy.execute("INSERT INTO claims (subject) VALUES (?)", (s,)) + _insert_claim(legacy, s) legacy.execute( "INSERT INTO entities (canonical_name, entity_type, scope, " "created_at, updated_at) VALUES ('Paperclip', 'unknown', 'global', 't', 't')" @@ -359,7 +356,10 @@ def test_backfill_migrates_legacy_then_backfills(self): ) out = backfill_entities_normalized(legacy, migrate_schema=True) - assert out["schema_migration"]["migrated"] is True + assert out["schema_migration"] == { + "migrated": False, + "reason": "schema_already_current", + } # After backfill we should have 1 entity and 3 aliases. assert out["total_entities"] == 1 assert out["total_aliases"] == 3 diff --git a/tests/test_entity_registry_extra.py b/tests/test_entity_registry_extra.py index 120f143b..38fb0b84 100644 --- a/tests/test_entity_registry_extra.py +++ b/tests/test_entity_registry_extra.py @@ -21,6 +21,7 @@ import sqlite3 +from memorymaster.stores._storage_schema import load_schema_sql from memorymaster.knowledge.entity_registry import ( add_alias, backfill_entities, @@ -35,19 +36,32 @@ def _fresh_db() -> sqlite3.Connection: """In-memory DB with claims + entity tables, mirroring the live schema.""" conn = sqlite3.connect(":memory:") - conn.execute( - """ - CREATE TABLE claims ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - subject TEXT, - entity_id INTEGER - ) - """ - ) + conn.executescript(load_schema_sql()) ensure_entity_schema(conn) return conn +def _insert_claim( + conn: sqlite3.Connection, + subject: str, + entity_id: int | None = None, +) -> None: + columns = {row[1] for row in conn.execute("PRAGMA table_info(claims)")} + if "entity_id" in columns: + conn.execute( + "INSERT INTO claims " + "(text, subject, entity_id, scope, status, created_at, updated_at) " + "VALUES ('entity registry test', ?, ?, 'project:test', 'candidate', 't', 't')", + (subject, entity_id), + ) + else: + conn.execute( + "INSERT INTO claims (text, subject, scope, status, created_at, updated_at) " + "VALUES ('entity registry test', ?, 'project:test', 'candidate', 't', 't')", + (subject,), + ) + + # --------------------------------------------------------------------------- # resolve_or_create — entity_type/scope persistence + collision recovery # --------------------------------------------------------------------------- @@ -111,8 +125,8 @@ def test_merge_moves_aliases_and_claims_with_counts(): assert keep != merge # Two claims reference the merged entity. - conn.execute("INSERT INTO claims (subject, entity_id) VALUES ('a', ?)", (merge,)) - conn.execute("INSERT INTO claims (subject, entity_id) VALUES ('b', ?)", (merge,)) + _insert_claim(conn, "a", merge) + _insert_claim(conn, "b", merge) out = merge_entities(conn, keep, merge) @@ -178,8 +192,8 @@ def test_list_entities_orders_by_claim_count_and_counts_aliases(): add_alias(conn, busy, "Busy-alt") quiet = resolve_or_create(conn, "Quiet") # Give 'busy' two claims, 'quiet' none. - conn.execute("INSERT INTO claims (subject, entity_id) VALUES ('x', ?)", (busy,)) - conn.execute("INSERT INTO claims (subject, entity_id) VALUES ('y', ?)", (busy,)) + _insert_claim(conn, "x", busy) + _insert_claim(conn, "y", busy) rows = list_entities(conn) assert [r["name"] for r in rows] == ["Busy", "Quiet"] # busy first @@ -231,11 +245,9 @@ def test_backfill_entities_resolves_all_subjects(): canonical entity so the claims graph isn't fragmented.""" conn = sqlite3.connect(":memory:") # Claims table WITHOUT entity_id — backfill must ALTER it in. - conn.execute( - "CREATE TABLE claims (id INTEGER PRIMARY KEY AUTOINCREMENT, subject TEXT)" - ) + conn.executescript(load_schema_sql()) for s in ["Qdrant", "qdrant", "Postgres"]: - conn.execute("INSERT INTO claims (subject) VALUES (?)", (s,)) + _insert_claim(conn, s) out = backfill_entities(conn) @@ -261,10 +273,8 @@ def test_backfill_entities_is_safe_to_rerun(): """Running backfill twice must not double-create entities or crash on the already-present entity_id column (the ALTER is guarded).""" conn = sqlite3.connect(":memory:") - conn.execute( - "CREATE TABLE claims (id INTEGER PRIMARY KEY AUTOINCREMENT, subject TEXT)" - ) - conn.execute("INSERT INTO claims (subject) VALUES ('Qdrant')") + conn.executescript(load_schema_sql()) + _insert_claim(conn, "Qdrant") backfill_entities(conn) out2 = backfill_entities(conn) # second pass: nothing left to resolve assert out2["subjects_processed"] == 0 diff --git a/tests/test_entity_registry_merge_atomicity.py b/tests/test_entity_registry_merge_atomicity.py index e2377410..61e39c2e 100644 --- a/tests/test_entity_registry_merge_atomicity.py +++ b/tests/test_entity_registry_merge_atomicity.py @@ -16,6 +16,7 @@ import pytest +from memorymaster.stores._storage_schema import load_schema_sql from memorymaster.knowledge.entity_registry import ( add_alias, ensure_entity_schema, @@ -28,10 +29,8 @@ def conn(): c = sqlite3.connect(":memory:") c.execute("PRAGMA foreign_keys=ON") + c.executescript(load_schema_sql()) ensure_entity_schema(c) - c.execute( - "CREATE TABLE claims (id INTEGER PRIMARY KEY, subject TEXT, entity_id INTEGER)" - ) yield c c.close() @@ -44,7 +43,10 @@ def test_merge_with_colliding_variant_key_does_not_corrupt_graph(conn): add_alias(conn, keep, "shared-alias") add_alias(conn, merge, "shared-alias") conn.execute( - "INSERT INTO claims (subject, entity_id) VALUES ('QdrantDB', ?)", (merge,) + "INSERT INTO claims " + "(text, subject, entity_id, scope, status, created_at, updated_at) " + "VALUES ('merge test', 'QdrantDB', ?, 'project:test', 'candidate', 't', 't')", + (merge,), ) conn.commit() diff --git a/tests/test_entity_schema_composition.py b/tests/test_entity_schema_composition.py index 418ccdf8..0a9fca96 100644 --- a/tests/test_entity_schema_composition.py +++ b/tests/test_entity_schema_composition.py @@ -4,22 +4,16 @@ import sqlite3 -import pytest - from memorymaster.core.service import MemoryService from memorymaster.knowledge.entity_graph import EntityGraph -@pytest.mark.xfail( - strict=True, - reason="R2.3: EntityGraph DDL conflicts with the canonical registry schema", -) def test_normal_init_produces_a_graph_ready_entity_schema(tmp_path): db_path = tmp_path / "entity-composition.db" MemoryService(db_path, workspace_root=tmp_path).init_db() graph = EntityGraph(str(db_path)) - graph.ensure_tables() + graph.assert_ready() with sqlite3.connect(db_path) as conn: tables = { diff --git a/tests/test_entity_schema_convergence.py b/tests/test_entity_schema_convergence.py new file mode 100644 index 00000000..875d6cbc --- /dev/null +++ b/tests/test_entity_schema_convergence.py @@ -0,0 +1,246 @@ +"""P2-D contracts for one versioned entity registry and graph schema.""" + +from __future__ import annotations + +import sqlite3 +from importlib import import_module +from pathlib import Path +from unittest.mock import patch + +import pytest + +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.knowledge.entity_graph import EntityGraph, EntityGraphNotReady +from memorymaster.surfaces import mcp_server + + +def _table_columns(db_path: Path, table: str) -> set[str]: + with sqlite3.connect(db_path) as conn: + return {row[1] for row in conn.execute(f"PRAGMA table_info({table})")} + + +def _create_legacy_graph_first(db_path: Path) -> None: + with sqlite3.connect(db_path) as conn: + conn.executescript( + """ + CREATE TABLE entities ( + id TEXT PRIMARY KEY, name TEXT NOT NULL, type TEXT NOT NULL, + aliases TEXT NOT NULL DEFAULT '[]', created_at TEXT NOT NULL + ); + CREATE TABLE entity_edges ( + source_id TEXT NOT NULL, target_id TEXT NOT NULL, + relation TEXT NOT NULL, weight REAL NOT NULL DEFAULT 1.0, + claim_id INTEGER, created_at TEXT NOT NULL, + PRIMARY KEY (source_id, target_id, relation) + ); + CREATE TABLE claim_entity_links ( + claim_id INTEGER NOT NULL, entity_id TEXT NOT NULL, + PRIMARY KEY (claim_id, entity_id) + ); + INSERT INTO entities VALUES + ('legacy-alice', 'Alice', 'person', '["A. Example"]', '2026-01-01T00:00:00Z'); + """ + ) + + +def test_graph_first_database_migrates_to_canonical_registry(tmp_path: Path) -> None: + db_path = tmp_path / "graph-first.db" + _create_legacy_graph_first(db_path) + + MemoryService(db_path, workspace_root=tmp_path).init_db() + EntityGraph(str(db_path)).assert_ready() + + assert {"id", "canonical_name", "entity_type", "scope"} <= _table_columns( + db_path, "entities" + ) + with sqlite3.connect(db_path) as conn: + assert conn.execute( + "SELECT canonical_name FROM entities WHERE canonical_name = 'Alice'" + ).fetchone() == ("Alice",) + assert conn.execute("PRAGMA foreign_key_check").fetchall() == [] + + +def test_registry_first_database_adds_graph_without_replacing_entities( + tmp_path: Path, +) -> None: + db_path = tmp_path / "registry-first.db" + migration = import_module( + "memorymaster.stores.migrations.0013_canonical_entity_graph" + ) + with sqlite3.connect(db_path) as conn: + conn.executescript( + """ + PRAGMA foreign_keys = ON; + CREATE TABLE claims (id INTEGER PRIMARY KEY); + CREATE TABLE entities ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + canonical_name TEXT NOT NULL UNIQUE, + entity_type TEXT NOT NULL DEFAULT 'unknown', + scope TEXT NOT NULL DEFAULT 'global', + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE TABLE entity_aliases ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_id INTEGER NOT NULL REFERENCES entities(id) ON DELETE CASCADE, + alias TEXT NOT NULL, + variant_key TEXT NOT NULL DEFAULT '', + original_form TEXT NOT NULL, + created_at TEXT NOT NULL, + UNIQUE(entity_id, variant_key) + ); + INSERT INTO entities VALUES + (7, 'MemoryMaster', 'project', 'project:memorymaster', + '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z'); + """ + ) + migration.apply_sqlite(conn) + assert conn.execute( + "SELECT id, canonical_name FROM entities" + ).fetchall() == [(7, "MemoryMaster")] + assert conn.execute("PRAGMA foreign_key_check").fetchall() == [] + + EntityGraph(str(db_path), read_only=True).assert_ready() + + +def test_missing_graph_schema_fails_explicitly_without_ddl(tmp_path: Path) -> None: + db_path = tmp_path / "not-ready.db" + sqlite3.connect(db_path).close() + + with pytest.raises(EntityGraphNotReady, match="init-db"): + EntityGraph(str(db_path), read_only=True).get_stats() + + with sqlite3.connect(db_path) as conn: + assert conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table'" + ).fetchall() == [] + + +def test_entity_migration_declares_sqlite_postgres_parity() -> None: + from memorymaster.stores.migrations import entity_schema_contract + + sqlite_contract = entity_schema_contract("sqlite") + postgres_contract = entity_schema_contract("postgres") + + assert sqlite_contract == postgres_contract + assert sqlite_contract == { + "entities": ("id", "canonical_name", "entity_type", "scope"), + "entity_aliases": ("id", "entity_id", "alias", "variant_key"), + "entity_edges": ("source_id", "target_id", "relation", "claim_id"), + "claim_entity_links": ("claim_id", "entity_id"), + } + + +def test_postgres_migration_materializes_the_canonical_contract() -> None: + statements: list[str] = [] + + class _Cursor: + def __enter__(self): + return self + + def __exit__(self, *_args): + return False + + def execute(self, statement: str) -> None: + statements.append(" ".join(statement.split()).lower()) + + class _Connection: + committed = False + + def cursor(self): + return _Cursor() + + def commit(self) -> None: + self.committed = True + + connection = _Connection() + migration = import_module( + "memorymaster.stores.migrations.0013_canonical_entity_graph" + ) + migration.apply_postgres(connection) + + ddl = "\n".join(statements) + assert connection.committed is True + for table in ( + "entities", + "entity_aliases", + "entity_edges", + "claim_entity_links", + ): + assert f"table if not exists {table}" in ddl + assert "references entities(id)" in ddl + assert "references claims(id)" in ddl + + +def test_mcp_entity_reads_never_call_schema_creation(tmp_path: Path, monkeypatch) -> None: + db_path = tmp_path / "mcp-entity.db" + MemoryService(db_path, workspace_root=tmp_path).init_db() + + def _forbidden(*_args, **_kwargs): + raise AssertionError("read tool attempted schema creation") + + monkeypatch.setattr(EntityGraph, "ensure_tables", _forbidden) + stats = mcp_server.entity_stats(db=str(db_path)) + related = mcp_server.find_related_claims("Unknown", db=str(db_path)) + + assert stats == { + "ok": True, + "entities": 0, + "edges": 0, + "claim_links": 0, + "by_type": {}, + } + assert related == {"ok": True, "claim_ids": [], "count": 0} + + +def test_mcp_entity_readiness_failure_is_actionable(tmp_path: Path) -> None: + db_path = tmp_path / "mcp-not-ready.db" + sqlite3.connect(db_path).close() + + result = mcp_server.entity_stats(db=str(db_path)) + + assert result["ok"] is False + assert result["code"] == "ENTITY_GRAPH_NOT_READY" + assert "init-db" in result["error"] + + +def test_init_extract_stats_related_and_enriched_recall(tmp_path: Path) -> None: + db_path = tmp_path / "entity-e2e.db" + service = MemoryService(db_path, workspace_root=tmp_path) + service.init_db() + alice = service.ingest( + "Alice uses the canonical entity graph", + [CitationInput(source="test://alice")], + ) + bob = service.ingest( + "Bob collaborates with Alice", + [CitationInput(source="test://bob")], + ) + payload = ( + '{"entities":[{"name":"Alice","type":"person","aliases":[]},' + '{"name":"Bob","type":"person","aliases":[]}],' + '"relations":[{"source":"Bob","target":"Alice","relation":"knows"}]}' + ) + + graph = EntityGraph(str(db_path)) + with patch("memorymaster.knowledge.entity_graph._llm_chat", return_value=payload): + assert graph.extract_and_link(bob.id, bob.text) == ["Alice", "Bob"] + + assert graph.get_stats() == { + "entities": 2, + "edges": 1, + "claim_links": 2, + "by_type": {"person": 2}, + } + assert bob.id in graph.find_related_claims(["Alice"], hops=1) + rows = service.query_rows( + "Alice", + limit=10, + include_candidates=True, + enrich_with_entities=True, + ) + assert {row["claim"].id for row in rows} >= {alice.id, bob.id} + with sqlite3.connect(db_path) as conn: + conn.execute("PRAGMA foreign_keys = ON") + assert conn.execute("PRAGMA foreign_key_check").fetchall() == [] diff --git a/tests/test_graph_distance.py b/tests/test_graph_distance.py index 76016bcd..e903ff07 100644 --- a/tests/test_graph_distance.py +++ b/tests/test_graph_distance.py @@ -34,7 +34,7 @@ def kuzu_store(tmp_path: Path): """Fresh Kuzu DB. Skips when Kuzu is missing on the platform.""" try: store = GraphStore(tmp_path / "g.kuzu") - store.open() + store.initialize() except GraphStoreUnavailable: pytest.skip("Kuzu unavailable on this platform") yield store diff --git a/tests/test_graph_store.py b/tests/test_graph_store.py index d4fa1e71..447783a3 100644 --- a/tests/test_graph_store.py +++ b/tests/test_graph_store.py @@ -33,7 +33,7 @@ def kuzu_store(tmp_path: Path): """Fresh Kuzu DB in a tempdir, auto-closed after the test.""" try: store = GraphStore(tmp_path / "g.kuzu") - store.open() + store.initialize() except GraphStoreUnavailable: pytest.skip("Kuzu unavailable on this platform") yield store @@ -67,6 +67,12 @@ def cognee_edges() -> list[GraphEdge]: # public API # ---------------------------------------------------------------------- class TestGraphStoreKuzu: + def test_open_requires_explicit_initialization(self, tmp_path): + store = GraphStore(tmp_path / "missing.kuzu") + with pytest.raises(GraphStoreUnavailable, match="not initialized"): + store.open() + assert not store.path.exists() + def test_ingest_edges_idempotent(self, kuzu_store, cognee_edges): first = kuzu_store.ingest_edges(cognee_edges) assert first == len(cognee_edges) diff --git a/tests/test_hebbian_ebbinghaus.py b/tests/test_hebbian_ebbinghaus.py index 714ffde0..efc3ed9c 100644 --- a/tests/test_hebbian_ebbinghaus.py +++ b/tests/test_hebbian_ebbinghaus.py @@ -16,14 +16,15 @@ from __future__ import annotations import tempfile -import uuid from datetime import datetime, timedelta, timezone from pathlib import Path import pytest +from memorymaster.core.service import MemoryService from memorymaster.govern.jobs import decay from memorymaster.knowledge.entity_graph import EntityGraph +from memorymaster.knowledge.entity_registry import resolve_or_create HEBBIAN_FLAG = "MEMORYMASTER_HEBBIAN_DECAY" @@ -36,8 +37,20 @@ def db_path(): @pytest.fixture def graph(db_path): + MemoryService(db_path, workspace_root=db_path.parent).init_db() g = EntityGraph(str(db_path)) g.ensure_tables() + now = datetime.now(timezone.utc).isoformat() + conn = g._connect() + try: + conn.executemany( + "INSERT INTO claims (id, text, scope, status, created_at, updated_at) " + "VALUES (?, 'hebbian test', 'project:test', 'confirmed', ?, ?)", + [(claim_id, now, now) for claim_id in (1, 2, 10, 11, 99, 100, 200)], + ) + conn.commit() + finally: + conn.close() return g @@ -47,21 +60,17 @@ def enable_decay(monkeypatch): monkeypatch.setenv(HEBBIAN_FLAG, "1") -def _seed_entity(graph: EntityGraph, name: str) -> str: +def _seed_entity(graph: EntityGraph, name: str) -> int: conn = graph._connect() try: - ent_id = str(uuid.uuid4()) - conn.execute( - "INSERT INTO entities (id, name, type, aliases, created_at) VALUES (?, ?, 'concept', '[]', ?)", - (ent_id, name, datetime.now(timezone.utc).isoformat()), - ) + ent_id = resolve_or_create(conn, name, entity_type="concept") conn.commit() return ent_id finally: conn.close() -def _read_edge(graph: EntityGraph, src: str, tgt: str, relation: str): +def _read_edge(graph: EntityGraph, src: int, tgt: int, relation: str): conn = graph._connect() try: return conn.execute( @@ -255,8 +264,6 @@ def test_run_cycle_includes_edge_decay(monkeypatch, tmp_path): steward and dashboard can observe it — and must remain failure-isolated (no exception escapes). This anchors the integration contract, not just the unit behavior.""" - from memorymaster.core.service import MemoryService - monkeypatch.setenv(HEBBIAN_FLAG, "1") import sqlite3 @@ -264,32 +271,24 @@ def test_run_cycle_includes_edge_decay(monkeypatch, tmp_path): svc = MemoryService(db_file, workspace_root=str(tmp_path)) svc.init_db() - # Seed a real entity_edges row (the only table this feature mutates) directly - # on the service DB. We create the table with raw DDL rather than - # EntityGraph.ensure_tables() because the graph's executescript also - # (re)declares an `entities` table that collides with the registry `entities` - # init_db() already created — a pre-existing schema-name overlap unrelated to - # this feature. The decay job reads ONLY entity_edges, so this is sufficient. + # Seed a canonical edge and its referenced claim/entity rows directly. conn = sqlite3.connect(db_file) old = (datetime.now(timezone.utc) - timedelta(days=20)).isoformat() - conn.executescript( - """ - CREATE TABLE IF NOT EXISTS entity_edges ( - source_id TEXT NOT NULL, - target_id TEXT NOT NULL, - relation TEXT NOT NULL DEFAULT 'related_to', - weight REAL NOT NULL DEFAULT 1.0, - claim_id INTEGER, - created_at TEXT NOT NULL, - last_reinforced_at TEXT, - PRIMARY KEY (source_id, target_id, relation) - ); - """ + conn.execute( + "INSERT INTO claims (id, text, scope, status, created_at, updated_at) " + "VALUES (1, 'cycle edge', 'project:test', 'confirmed', ?, ?)", + (old, old), + ) + conn.executemany( + "INSERT INTO entities " + "(id, canonical_name, entity_type, scope, created_at, updated_at) " + "VALUES (?, ?, 'concept', 'global', ?, ?)", + [(1, "cycle-a", old, old), (2, "cycle-b", old, old)], ) conn.execute( "INSERT INTO entity_edges " "(source_id, target_id, relation, weight, claim_id, created_at, last_reinforced_at) " - "VALUES ('cycle-a', 'cycle-b', 'related_to', 2.0, 1, ?, ?)", + "VALUES (1, 2, 'related_to', 2.0, 1, ?, ?)", (datetime.now(timezone.utc).isoformat(), old), ) conn.commit() diff --git a/tests/test_wiki_suggest.py b/tests/test_wiki_suggest.py index 0f73655f..d97935ec 100644 --- a/tests/test_wiki_suggest.py +++ b/tests/test_wiki_suggest.py @@ -5,56 +5,61 @@ from pathlib import Path from memorymaster.surfaces.cli import main -from memorymaster.knowledge.entity_graph import EntityGraph +import pytest + +from memorymaster.core.service import MemoryService +from memorymaster.knowledge.entity_graph import EntityGraphNotReady from memorymaster.knowledge.wiki_suggest import suggest_wikilinks def _seed_graph(db_path: Path) -> None: - graph = EntityGraph(str(db_path)) - graph.ensure_tables() + MemoryService(db_path, workspace_root=db_path.parent).init_db() conn = sqlite3.connect(db_path) try: - conn.execute( - """ - CREATE TABLE claims ( - id INTEGER PRIMARY KEY, - status TEXT NOT NULL, - wiki_article TEXT - ) - """ - ) + now = "2026-01-01T00:00:00Z" conn.executemany( - "INSERT INTO entities (id, name, type, aliases, created_at) VALUES (?, ?, ?, ?, ?)", + "INSERT INTO entities " + "(id, canonical_name, entity_type, scope, created_at, updated_at) " + "VALUES (?, ?, ?, 'global', ?, ?)", [ - ("e-claim", "claim", "concept", '["claims"]', "2026-01-01T00:00:00Z"), - ("e-status", "status", "concept", "[]", "2026-01-01T00:00:00Z"), - ("e-qdrant", "qdrant", "product", "[]", "2026-01-01T00:00:00Z"), + (1, "claim", "concept", now, now), + (2, "status", "concept", now, now), + (3, "qdrant", "product", now, now), ], ) + conn.executemany( + "INSERT INTO entity_aliases " + "(entity_id, alias, variant_key, original_form, created_at) " + "VALUES (?, ?, ?, ?, ?)", + [(1, "claim", "claim", "claim", now), (1, "claims", "claims", "claims", now), + (2, "status", "status", "status", now), (3, "qdrant", "qdrant", "qdrant", now)], + ) conn.executemany( "INSERT INTO entity_edges (source_id, target_id, relation, claim_id, created_at) VALUES (?, ?, ?, ?, ?)", [ - ("e-claim", "e-status", "related_to", 1, "2026-01-01T00:00:00Z"), - ("e-status", "e-qdrant", "related_to", 2, "2026-01-01T00:00:00Z"), + (1, 2, "related_to", 1, now), + (2, 3, "related_to", 2, now), ], ) conn.executemany( - "INSERT INTO claims (id, status, wiki_article) VALUES (?, ?, ?)", + "INSERT INTO claims " + "(id, text, scope, status, wiki_article, created_at, updated_at) " + "VALUES (?, 'wiki graph claim', 'project:test', ?, ?, ?, ?)", [ - (1, "confirmed", "claims-lifecycle"), - (2, "confirmed", "status-governance"), - (3, "confirmed", "qdrant-sync"), - (4, "confirmed", "missing-article"), + (1, "confirmed", "claims-lifecycle", now, now), + (2, "confirmed", "status-governance", now, now), + (3, "confirmed", "qdrant-sync", now, now), + (4, "confirmed", "missing-article", now, now), ], ) conn.executemany( "INSERT INTO claim_entity_links (claim_id, entity_id) VALUES (?, ?)", [ - (1, "e-claim"), - (1, "e-status"), - (2, "e-status"), - (3, "e-qdrant"), - (4, "e-claim"), + (1, 1), + (1, 2), + (2, 2), + (3, 3), + (4, 1), ], ) conn.commit() @@ -117,8 +122,9 @@ def test_wiki_suggest_links_cli_prints_json_list(tmp_path: Path, capsys) -> None assert payload[0]["slug"] == "claims-lifecycle" -def test_suggest_wikilinks_returns_empty_for_missing_graph(tmp_path: Path) -> None: +def test_suggest_wikilinks_reports_missing_graph(tmp_path: Path) -> None: wiki_root = tmp_path / "wiki" _seed_wiki(wiki_root) - assert suggest_wikilinks(tmp_path / "empty.db", "claim status", wiki_root=wiki_root) == [] + with pytest.raises(EntityGraphNotReady, match="init-db"): + suggest_wikilinks(tmp_path / "empty.db", "claim status", wiki_root=wiki_root) From a1d54773fa5e540ad90b6617e02c983ddbfe3d32 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Mon, 13 Jul 2026 18:41:08 -0300 Subject: [PATCH 34/55] fix: bound quiet capture and retention --- .planning/REMAINING-PHASES-V1-BUDGET.md | 26 +- .../hooks/memorymaster-auto-ingest.py | 138 +++++++++-- .../hooks/memorymaster-precompact.py | 5 +- .../hooks/memorymaster-session-end.py | 61 +++++ memorymaster/core/capture_control.py | 229 ++++++++++++++++++ memorymaster/govern/review.py | 31 +++ memorymaster/govern/verbatim_cleanup.py | 79 ++++++ memorymaster/recall/verbatim_store.py | 51 ++++ memorymaster/surfaces/setup_hooks.py | 20 +- tests/test_ambient_spool.py | 12 +- tests/test_capture_control.py | 55 +++++ tests/test_capture_retention_and_backlog.py | 87 +++++++ tests/test_precompact_capture_policy.py | 43 ++++ tests/test_session_end_capture_hook.py | 51 ++++ tests/test_setup_hooks.py | 1 + tests/test_stop_hook_capture_policy.py | 91 ++++++- 16 files changed, 937 insertions(+), 43 deletions(-) create mode 100644 memorymaster/config_templates/hooks/memorymaster-session-end.py create mode 100644 memorymaster/core/capture_control.py create mode 100644 tests/test_capture_control.py create mode 100644 tests/test_capture_retention_and_backlog.py create mode 100644 tests/test_precompact_capture_policy.py create mode 100644 tests/test_session_end_capture_hook.py diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index 4a8b6c0f..91b99fb0 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -219,5 +219,27 @@ P2-D implementation and focused gate completed on 2026-07-13 from P2-C commit - Atomic package commit: the conventional `fix: converge entity schema authority` commit containing this scheduler evidence. -The next separately authorized package is P2-E. Do not begin quiet finite -capture or retention work from the P2-D goal. +P2-E implementation and focused gate completed on 2026-07-13 from P2-D commit +`4fb1725b234a3144299ec66a674ce1b8833fb728`: + +- Stop and PreCompact are quiet by default. Verbatim, per-stop distillation, + correction mining, and checkpoint blocking require explicit + maximum-capture flags; SessionEnd performs the default distilled ingest. +- A separate WAL-backed capture ledger persists complete-line transcript + cursors, atomic global/provider/session reservations, effective limits, and + input/output usage. Default worst-case candidate inflow is bounded to 600/day + against the 688/day safe intake ceiling. +- Verbatim capture uses appended-only APIs and fails closed when the dry-run + 30-day/512-MiB/75,000-session retention plan is over bounds or its bounded + scan truncates. Candidate backlog planning is reviewable and read-only, with + zero automatic transitions. +- The focused gate collected 84 tests: 80 passed in 13.26s and exposed four + bounded integration issues; the single correction batch passed all 12 + affected/new contracts in 1.88s. Changed-file Ruff and `git diff --check` + passed. The independent invariant probe confirmed a 600/day maximum + candidate inflow and the exact retention defaults. +- Atomic package commit: the conventional `fix: bound quiet capture and + retention` commit containing this scheduler evidence. + +The next separately authorized package is P2-F. Do not begin fake-evidence +default removal from the P2-E goal. diff --git a/memorymaster/config_templates/hooks/memorymaster-auto-ingest.py b/memorymaster/config_templates/hooks/memorymaster-auto-ingest.py index 296e5661..9597bb19 100644 --- a/memorymaster/config_templates/hooks/memorymaster-auto-ingest.py +++ b/memorymaster/config_templates/hooks/memorymaster-auto-ingest.py @@ -1,16 +1,15 @@ -"""Stop hook: Block-based memory capture (MemPalace-inspired). +"""Optional maximum-capture Stop hook; quiet and non-blocking by default. -Every SAVE_INTERVAL human messages, BLOCK Claude from stopping and force a save. -Uses decision:block + reason as system message. Claude saves to MemoryMaster, -then next Stop fires with stop_hook_active=true → passes through. - -Also runs Gemini extraction as fallback for non-block turns. +When explicitly enabled, checkpoint blocking can pause every SAVE_INTERVAL. +Verbatim, per-stop distillation, and correction mining are separately opt-in; +each enabled data path consumes only appended transcript lines. """ import json import os import sys import re import hashlib +import tempfile from datetime import datetime from pathlib import Path @@ -21,7 +20,21 @@ sys.path.insert(0, PROJECT_ROOT) -os.makedirs(STATE_DIR, exist_ok=True) +def _enabled(name): + return os.environ.get(name, "").strip().lower() not in ("", "0", "false", "no", "off") + + +def _approve(): + sys.stdout.write(json.dumps({"decision": "approve"})) + + +def _temporary_chunk(text): + handle = tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".jsonl", delete=False) + try: + handle.write(text) + return handle.name + finally: + handle.close() def _spool_enabled() -> bool: @@ -261,22 +274,88 @@ def _run_rule_extraction(transcript_path, cwd): pass +def _run_incremental_llm(ledger, transcript_path, session_id, cwd, provider, operation, runner): + chunk = ledger.read_increment(transcript_path, f"{session_id}:{operation}") + if not chunk.text: + return + reservation = ledger.reserve_llm(provider, session_id, operation) + if reservation is None: + return + temp_path = "" + try: + temp_path = _temporary_chunk(chunk.text) + runner(temp_path, cwd) + ledger.finish_llm( + reservation, + input_bytes=len(chunk.text.encode("utf-8")), + output_bytes=0, + outcome="ok", + ) + ledger.commit_cursor(chunk) + except Exception: + ledger.finish_llm(reservation, input_bytes=0, output_bytes=0, outcome="error") + finally: + if temp_path: + Path(temp_path).unlink(missing_ok=True) + + +def _run_incremental_verbatim(ledger, transcript_path, session_id, scope): + chunk = ledger.read_increment(transcript_path, f"{session_id}:stop-verbatim") + if not chunk.text: + return + if os.path.exists(DB_PATH): + from memorymaster.govern.verbatim_cleanup import plan_retention + + if not plan_retention(DB_PATH).get("within_bounds", True): + return + if _spool_enabled(): + from memorymaster.recall.verbatim_store import spool_transcript_text + + spool_transcript_text( + DB_PATH, chunk.text, session_id=session_id, scope=scope, source_agent="stop-hook" + ) + else: + from memorymaster.recall.verbatim_store import store_transcript_text + + store_transcript_text( + DB_PATH, chunk.text, session_id=session_id, scope=scope, source_agent="stop-hook" + ) + ledger.commit_cursor(chunk) + + def main(): try: data = json.loads(sys.stdin.read() or "{}") except Exception: data = {} - session_id = data.get("session_id", "unknown") + session_id = str(data.get("session_id", "unknown")) # Sanitize session_id session_id = re.sub(r"[^a-zA-Z0-9_-]", "", session_id) or "unknown" transcript_path = data.get("transcript_path", "") cwd = data.get("cwd", os.getcwd()) stop_hook_active = data.get("stop_hook_active", False) + blocking = _enabled("MEMORYMASTER_STOP_BLOCKING") + verbatim = _enabled("MEMORYMASTER_STOP_CAPTURE_VERBATIM") + extract = _enabled("MEMORYMASTER_STOP_EXTRACT") + rule_mining = _enabled("MEMORYMASTER_STOP_RULE_MINING") # If already in a save cycle, let through (prevents infinite loop) if stop_hook_active in (True, "True", "true"): - sys.stdout.write(json.dumps({"decision": "approve"})) + _approve() + return + + if not any((blocking, verbatim, extract, rule_mining)): + _approve() + return + + os.makedirs(STATE_DIR, exist_ok=True) + try: + from memorymaster.core.capture_control import CaptureLedger, capture_state_path + + ledger = CaptureLedger(capture_state_path()) + except Exception: + _approve() return # Count human messages @@ -289,7 +368,7 @@ def main(): f.write(f"[{datetime.now().strftime('%H:%M:%S')}] {session_id}: {exchange_count} exchanges, {since_last} since last save\n") # Time to block and force save? - if since_last >= SAVE_INTERVAL and exchange_count > 0: + if blocking and since_last >= SAVE_INTERVAL and exchange_count > 0: _set_last_save(session_id, exchange_count) with open(os.path.join(STATE_DIR, "hook.log"), "a") as f: @@ -303,29 +382,40 @@ def main(): # Store verbatim on every stop (raw conversation storage) try: - if transcript_path and os.path.exists(transcript_path): + if verbatim and transcript_path and os.path.exists(transcript_path): scope = "project:" + os.path.basename(cwd).lower().replace(" ", "-") if cwd else "global" - if _spool_enabled(): - # P1 spec §2.3: append op:"verbatim" envelopes (~10 ms) instead - # of opening the multi-GB DB on every stop; the steward drain - # lands them in verbatim_memories via store_verbatim. - from memorymaster.recall.verbatim_store import spool_transcript - spool_transcript(DB_PATH, transcript_path, scope=scope, source_agent="stop-hook") - else: - from memorymaster.recall.verbatim_store import store_transcript - store_transcript(DB_PATH, transcript_path, scope=scope, source_agent="stop-hook") + _run_incremental_verbatim(ledger, transcript_path, session_id, scope) except Exception: pass # Not time to block — run passive Gemini extraction - _run_gemini_extraction(transcript_path, cwd) + provider = os.environ.get("MEMORYMASTER_LLM_PROVIDER", "google") + if extract: + _run_incremental_llm( + ledger, + transcript_path, + session_id, + cwd, + provider, + "stop-distill", + _run_gemini_extraction, + ) # R1b: mine the latest correction in this session into a rule claim. # Accepted P1 residual (spec step 9): mine_transcript_rules needs a live # MemoryService, so this stays a direct DB path even under the flag. - _run_rule_extraction(transcript_path, cwd) - - sys.stdout.write(json.dumps({"decision": "approve"})) + if rule_mining: + _run_incremental_llm( + ledger, + transcript_path, + session_id, + cwd, + provider, + "stop-correction", + _run_rule_extraction, + ) + + _approve() if __name__ == "__main__": diff --git a/memorymaster/config_templates/hooks/memorymaster-precompact.py b/memorymaster/config_templates/hooks/memorymaster-precompact.py index fdfdfd36..7aca69d4 100644 --- a/memorymaster/config_templates/hooks/memorymaster-precompact.py +++ b/memorymaster/config_templates/hooks/memorymaster-precompact.py @@ -21,11 +21,14 @@ from datetime import datetime STATE_DIR = os.path.join(os.path.expanduser("~"), ".memorymaster", "hook_state") -os.makedirs(STATE_DIR, exist_ok=True) MARKER_TTL_SECONDS = 24 * 3600 # re-warn if marker older than 24h def main(): + maximum_capture = os.environ.get("MEMORYMASTER_PRECOMPACT_BLOCKING", "").strip().lower() + if maximum_capture in ("", "0", "false", "no", "off"): + return + os.makedirs(STATE_DIR, exist_ok=True) try: data = json.loads(sys.stdin.read() or "{}") except Exception: diff --git a/memorymaster/config_templates/hooks/memorymaster-session-end.py b/memorymaster/config_templates/hooks/memorymaster-session-end.py new file mode 100644 index 00000000..e469a647 --- /dev/null +++ b/memorymaster/config_templates/hooks/memorymaster-session-end.py @@ -0,0 +1,61 @@ +"""Quiet default session-end distillation with durable cursors and budgets.""" + +import json +import os +import sys +import tempfile +from pathlib import Path + +PROJECT_ROOT = "__MEMORYMASTER_PROJECT_ROOT__" +DB_PATH = os.path.join(PROJECT_ROOT, "memorymaster.db") +sys.path.insert(0, PROJECT_ROOT) + + +def _enabled(name, default=True): + raw = os.environ.get(name) + if raw is None: + return default + return raw.strip().lower() not in ("0", "false", "no", "off", "") + + +def main(): + if not _enabled("MEMORYMASTER_SESSION_END_CAPTURE", True): + return + try: + data = json.loads(sys.stdin.read() or "{}") + except Exception: + data = {} + transcript_path = data.get("transcript_path", "") + session_id = str(data.get("session_id") or "unknown") + cwd = str(data.get("cwd") or PROJECT_ROOT) + try: + from memorymaster.core.capture_control import CaptureLedger, capture_state_path + from scripts.agent_session_end_ingest import run + + ledger = CaptureLedger(capture_state_path()) + chunk = ledger.read_increment(transcript_path, f"{session_id}:session-end") + if not chunk.text: + return + provider = os.environ.get("MEMORYMASTER_LLM_PROVIDER", "google") + reservation = ledger.reserve_llm(provider, session_id, "session-end-distill") + if reservation is None: + return + temp_path = "" + try: + with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".jsonl", delete=False) as handle: + handle.write(chunk.text) + temp_path = handle.name + run(DB_PATH, temp_path, source_agent="session-end-hook", cwd=cwd) + ledger.finish_llm(reservation, input_bytes=len(chunk.text.encode("utf-8")), output_bytes=0, outcome="ok") + ledger.commit_cursor(chunk) + except Exception: + ledger.finish_llm(reservation, input_bytes=len(chunk.text.encode("utf-8")), output_bytes=0, outcome="error") + finally: + if temp_path: + Path(temp_path).unlink(missing_ok=True) + except Exception: + return + + +if __name__ == "__main__": + main() diff --git a/memorymaster/core/capture_control.py b/memorymaster/core/capture_control.py new file mode 100644 index 00000000..e41621ec --- /dev/null +++ b/memorymaster/core/capture_control.py @@ -0,0 +1,229 @@ +"""Durable, finite controls for optional transcript capture paths.""" + +from __future__ import annotations + +import hashlib +import os +import sqlite3 +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + + +DEFAULT_GLOBAL_DAILY_CALLS = 200 +DEFAULT_PROVIDER_DAILY_CALLS = 100 +DEFAULT_SESSION_DAILY_CALLS = 3 + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS capture_cursors ( + stream_key TEXT PRIMARY KEY, + transcript_path_hash TEXT NOT NULL, + prefix_hash TEXT NOT NULL, + committed_offset INTEGER NOT NULL, + updated_at TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS capture_usage ( + reservation_id TEXT PRIMARY KEY, + usage_day TEXT NOT NULL, + provider TEXT NOT NULL, + session_hash TEXT NOT NULL, + operation TEXT NOT NULL, + outcome TEXT NOT NULL DEFAULT 'reserved', + input_bytes INTEGER NOT NULL DEFAULT 0, + output_bytes INTEGER NOT NULL DEFAULT 0, + created_at TEXT NOT NULL, + finished_at TEXT +); +CREATE INDEX IF NOT EXISTS idx_capture_usage_day ON capture_usage(usage_day); +CREATE TABLE IF NOT EXISTS capture_policy ( + key TEXT PRIMARY KEY, + value INTEGER NOT NULL, + updated_at TEXT NOT NULL +); +""" + + +def _env_int(name: str, default: int) -> int: + try: + return max(0, int(os.environ.get(name, str(default)))) + except ValueError: + return default + + +@dataclass(frozen=True, slots=True) +class CaptureLimits: + global_daily_calls: int = DEFAULT_GLOBAL_DAILY_CALLS + provider_daily_calls: int = DEFAULT_PROVIDER_DAILY_CALLS + session_daily_calls: int = DEFAULT_SESSION_DAILY_CALLS + + @classmethod + def from_env(cls) -> "CaptureLimits": + return cls( + global_daily_calls=_env_int("MEMORYMASTER_CAPTURE_GLOBAL_DAILY_CALLS", DEFAULT_GLOBAL_DAILY_CALLS), + provider_daily_calls=_env_int("MEMORYMASTER_CAPTURE_PROVIDER_DAILY_CALLS", DEFAULT_PROVIDER_DAILY_CALLS), + session_daily_calls=_env_int("MEMORYMASTER_CAPTURE_SESSION_DAILY_CALLS", DEFAULT_SESSION_DAILY_CALLS), + ) + + +@dataclass(frozen=True, slots=True) +class TranscriptChunk: + stream_key: str + transcript_path_hash: str + prefix_hash: str + start_offset: int + end_offset: int + text: str + generation_reset: bool = False + + +@dataclass(frozen=True, slots=True) +class LLMReservation: + reservation_id: str + usage_day: str + + +def capture_state_path() -> Path: + configured = os.environ.get("MEMORYMASTER_CAPTURE_STATE_DB", "").strip() + return Path(configured) if configured else Path.home() / ".memorymaster" / "capture-control.db" + + +class CaptureLedger: + def __init__(self, db_path: str | Path, *, limits: CaptureLimits | None = None) -> None: + self.db_path = Path(db_path) + self.limits = limits or CaptureLimits.from_env() + self.db_path.parent.mkdir(parents=True, exist_ok=True) + with self._connect() as conn: + conn.executescript(_SCHEMA) + now = datetime.now(timezone.utc).isoformat() + conn.executemany( + """INSERT INTO capture_policy VALUES (?, ?, ?) + ON CONFLICT(key) DO UPDATE SET value=excluded.value, updated_at=excluded.updated_at""", + [ + ("global_daily_calls", self.limits.global_daily_calls, now), + ("provider_daily_calls", self.limits.provider_daily_calls, now), + ("session_daily_calls", self.limits.session_daily_calls, now), + ], + ) + + def _connect(self) -> sqlite3.Connection: + conn = sqlite3.connect(self.db_path, timeout=15) + conn.row_factory = sqlite3.Row + conn.execute("PRAGMA journal_mode=WAL") + conn.execute("PRAGMA busy_timeout=15000") + return conn + + @staticmethod + def _hash(value: str) -> str: + return hashlib.sha256(value.encode("utf-8", errors="replace")).hexdigest() + + @staticmethod + def _prefix(path: Path, limit: int = 4096) -> str: + with path.open("rb") as handle: + return hashlib.sha256(handle.read(max(0, limit))).hexdigest() + + def read_increment(self, transcript_path: str | Path, stream_key: str) -> TranscriptChunk: + path = Path(transcript_path) + path_hash = self._hash(str(path.resolve())) + if not path.is_file(): + return TranscriptChunk(stream_key, path_hash, "", 0, 0, "") + size = path.stat().st_size + with self._connect() as conn: + row = conn.execute("SELECT * FROM capture_cursors WHERE stream_key = ?", (stream_key,)).fetchone() + start = int(row["committed_offset"]) if row else 0 + comparison_hash = self._prefix(path, min(4096, start or size)) + generation_reset = bool( + row + and ( + row["transcript_path_hash"] != path_hash + or row["prefix_hash"] != comparison_hash + or size < start + ) + ) + if generation_reset: + start = 0 + with path.open("rb") as handle: + handle.seek(start) + pending = handle.read() + last_newline = pending.rfind(b"\n") + complete = pending[: last_newline + 1] if last_newline >= 0 else b"" + end = start + len(complete) + committed_prefix = self._prefix(path, min(4096, end or size)) + return TranscriptChunk( + stream_key, + path_hash, + committed_prefix, + start, + end, + complete.decode("utf-8", errors="replace"), + generation_reset, + ) + + def commit_cursor(self, chunk: TranscriptChunk) -> None: + if chunk.end_offset <= chunk.start_offset: + return + now = datetime.now(timezone.utc).isoformat() + with self._connect() as conn: + current = conn.execute( + "SELECT committed_offset FROM capture_cursors WHERE stream_key=?", + (chunk.stream_key,), + ).fetchone() + if current and not chunk.generation_reset and int(current[0]) > chunk.end_offset: + return + conn.execute( + """INSERT INTO capture_cursors VALUES (?, ?, ?, ?, ?) + ON CONFLICT(stream_key) DO UPDATE SET + transcript_path_hash=excluded.transcript_path_hash, + prefix_hash=excluded.prefix_hash, + committed_offset=excluded.committed_offset, + updated_at=excluded.updated_at""", + (chunk.stream_key, chunk.transcript_path_hash, chunk.prefix_hash, chunk.end_offset, now), + ) + + def reserve_llm(self, provider: str, session_id: str, operation: str) -> LLMReservation | None: + usage_day = datetime.now(timezone.utc).date().isoformat() + provider = (provider or "unknown").strip().lower() + session_hash = self._hash(session_id or "unknown") + conn = self._connect() + try: + conn.execute("BEGIN IMMEDIATE") + global_count = conn.execute("SELECT COUNT(*) FROM capture_usage WHERE usage_day=?", (usage_day,)).fetchone()[0] + provider_count = conn.execute("SELECT COUNT(*) FROM capture_usage WHERE usage_day=? AND provider=?", (usage_day, provider)).fetchone()[0] + session_count = conn.execute("SELECT COUNT(*) FROM capture_usage WHERE usage_day=? AND session_hash=?", (usage_day, session_hash)).fetchone()[0] + if global_count >= self.limits.global_daily_calls or provider_count >= self.limits.provider_daily_calls or session_count >= self.limits.session_daily_calls: + conn.rollback() + return None + reservation_id = uuid.uuid4().hex + conn.execute( + "INSERT INTO capture_usage (reservation_id, usage_day, provider, session_hash, operation, created_at) VALUES (?, ?, ?, ?, ?, ?)", + (reservation_id, usage_day, provider, session_hash, operation, datetime.now(timezone.utc).isoformat()), + ) + conn.commit() + return LLMReservation(reservation_id, usage_day) + finally: + conn.close() + + def finish_llm(self, reservation: LLMReservation, *, input_bytes: int, output_bytes: int, outcome: str) -> None: + with self._connect() as conn: + conn.execute( + "UPDATE capture_usage SET outcome=?, input_bytes=?, output_bytes=?, finished_at=? WHERE reservation_id=?", + (outcome, max(0, input_bytes), max(0, output_bytes), datetime.now(timezone.utc).isoformat(), reservation.reservation_id), + ) + + def usage(self, usage_day: str | None = None) -> dict[str, object]: + day = usage_day or datetime.now(timezone.utc).date().isoformat() + with self._connect() as conn: + totals = conn.execute("SELECT COUNT(*), COALESCE(SUM(input_bytes),0), COALESCE(SUM(output_bytes),0) FROM capture_usage WHERE usage_day=?", (day,)).fetchone() + providers = conn.execute("SELECT provider, COUNT(*) FROM capture_usage WHERE usage_day=? GROUP BY provider ORDER BY provider", (day,)).fetchall() + return { + "usage_day": day, + "global_calls": int(totals[0]), + "input_bytes": int(totals[1]), + "output_bytes": int(totals[2]), + "providers": {str(row[0]): int(row[1]) for row in providers}, + "limits": { + "global_daily_calls": self.limits.global_daily_calls, + "provider_daily_calls": self.limits.provider_daily_calls, + "session_daily_calls": self.limits.session_daily_calls, + }, + } diff --git a/memorymaster/govern/review.py b/memorymaster/govern/review.py index 66a93f24..ba580b95 100644 --- a/memorymaster/govern/review.py +++ b/memorymaster/govern/review.py @@ -119,3 +119,34 @@ def build_review_queue( def queue_to_dicts(items: list[ReviewItem]) -> list[dict[str, object]]: return [asdict(item) for item in items] + + +def build_candidate_backlog_plan( + service, + *, + daily_capacity: int = 688, + batch_size: int = 100, + scan_limit: int = 25_000, +) -> dict[str, object]: + """Return a bounded, read-only candidate review plan.""" + if daily_capacity <= 0 or batch_size <= 0 or scan_limit <= 0: + raise ValueError("daily_capacity, batch_size, and scan_limit must be positive") + claims = service.list_claims( + status="candidate", + include_archived=False, + limit=scan_limit, + allow_sensitive=False, + ) + count = len(claims) + return { + "dry_run": True, + "candidate_count": count, + "scan_limit": scan_limit, + "truncated": count >= scan_limit, + "daily_capacity": daily_capacity, + "batch_size": batch_size, + "review_batches": math.ceil(count / batch_size), + "minimum_days": math.ceil(count / daily_capacity), + "automatic_transitions": 0, + "candidate_ids": [int(claim.id) for claim in claims[:batch_size]], + } diff --git a/memorymaster/govern/verbatim_cleanup.py b/memorymaster/govern/verbatim_cleanup.py index 0ce206a2..59ccbcfc 100644 --- a/memorymaster/govern/verbatim_cleanup.py +++ b/memorymaster/govern/verbatim_cleanup.py @@ -21,12 +21,18 @@ import logging import sqlite3 +from datetime import datetime, timedelta, timezone +from pathlib import Path from typing import Any from memorymaster.stores._storage_shared import open_conn logger = logging.getLogger(__name__) +DEFAULT_MAX_AGE_DAYS = 30 +DEFAULT_MAX_BYTES = 512 * 1024 * 1024 +DEFAULT_MAX_SESSIONS = 75_000 + # Known pre-#128 junk content prefixes (internal LLM prompts that the broken # capture path mistakenly stored as if they were conversation turns). _JUNK_PREFIXES: tuple[str, ...] = ( @@ -204,3 +210,76 @@ def cleanup( return out finally: conn.close() + + +def _read_retention_rows(db_path: str, scan_limit: int) -> tuple[bool, list[dict[str, Any]], bool]: + uri = f"file:{Path(db_path).resolve().as_posix()}?mode=ro" + conn = sqlite3.connect(uri, uri=True) + conn.row_factory = sqlite3.Row + try: + if not _verbatim_present(conn): + return False, [], False + rows = conn.execute( + """SELECT id, session_id, length(CAST(content AS BLOB)) AS byte_count, + COALESCE(NULLIF(timestamp, ''), created_at) AS retained_at + FROM verbatim_memories ORDER BY retained_at ASC, id ASC LIMIT ?""", + (scan_limit + 1,), + ).fetchall() + return True, [dict(row) for row in rows[:scan_limit]], len(rows) > scan_limit + finally: + conn.close() + + +def _retained_at(value: object) -> datetime: + try: + parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00")) + return parsed if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) + except (TypeError, ValueError): + return datetime.min.replace(tzinfo=timezone.utc) + + +def plan_retention( + db_path: str, + *, + max_age_days: int = DEFAULT_MAX_AGE_DAYS, + max_bytes: int = DEFAULT_MAX_BYTES, + max_sessions: int = DEFAULT_MAX_SESSIONS, + scan_limit: int = 100_000, + now: datetime | None = None, +) -> dict[str, Any]: + """Plan finite verbatim retention without changing persistent data.""" + if min(max_age_days, max_bytes, max_sessions) < 0 or scan_limit <= 0: + raise ValueError("retention limits must be non-negative") + verbatim_present, rows, truncated = _read_retention_rows(db_path, scan_limit) + current = now or datetime.now(timezone.utc) + cutoff = current - timedelta(days=max_age_days) + session_last: dict[str, datetime] = {} + for row in rows: + key = str(row["session_id"]) + session_last[key] = max(session_last.get(key, datetime.min.replace(tzinfo=timezone.utc)), _retained_at(row["retained_at"])) + kept_sessions = {key for key, _ in sorted(session_last.items(), key=lambda item: item[1], reverse=True)[:max_sessions]} + candidates = { + int(row["id"]) + for row in rows + if _retained_at(row["retained_at"]) < cutoff or str(row["session_id"]) not in kept_sessions + } + retained = [row for row in rows if int(row["id"]) not in candidates] + retained_bytes = sum(int(row["byte_count"] or 0) for row in retained) + for row in retained: + if retained_bytes <= max_bytes: + break + candidates.add(int(row["id"])) + retained_bytes -= int(row["byte_count"] or 0) + return { + "dry_run": True, + "verbatim_present": verbatim_present, + "truncated": truncated, + "scan_limit": scan_limit, + "limits": {"max_age_days": max_age_days, "max_bytes": max_bytes, "max_sessions": max_sessions}, + "total_rows": None if truncated else len(rows), + "candidate_rows": len(candidates), + "candidate_ids_sample": sorted(candidates)[:100], + "retained_rows": len(rows) - len(candidates), + "retained_bytes": retained_bytes, + "within_bounds": not candidates and not truncated, + } diff --git a/memorymaster/recall/verbatim_store.py b/memorymaster/recall/verbatim_store.py index 3c5bf90b..c08e0c2f 100644 --- a/memorymaster/recall/verbatim_store.py +++ b/memorymaster/recall/verbatim_store.py @@ -384,6 +384,57 @@ def spool_transcript( return stats +def _iter_transcript_text_turns(transcript_text: str): + for line in transcript_text.splitlines(): + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if isinstance(entry, dict): + yield _extract_role_content(entry) + + +def store_transcript_text( + db_path: str, + transcript_text: str, + *, + session_id: str, + scope: str = "project", + source_agent: str = "transcript", +) -> dict[str, int]: + """Store only the supplied JSONL increment under its authoritative session.""" + stats = {"stored": 0, "skipped": 0} + with closing(_connect(db_path)) as conn: + for role, content in _iter_transcript_text_turns(transcript_text): + if role not in ("user", "assistant") or len(content) < 20: + stats["skipped"] += 1 + continue + row_id = _store_verbatim_conn(conn, session_id, role, content, scope, source_agent) + stats["stored" if row_id else "skipped"] += 1 + conn.commit() + return stats + + +def spool_transcript_text( + db_path: str, + transcript_text: str, + *, + session_id: str, + scope: str = "project", + source_agent: str = "transcript", +) -> dict[str, int]: + """Spool only the supplied JSONL increment under its authoritative session.""" + stats = {"spooled": 0, "skipped": 0} + now = datetime.now(timezone.utc).isoformat() + for role, content in _iter_transcript_text_turns(transcript_text): + if role not in ("user", "assistant") or len(content) < 20 or _row_has_sensitive_field(role, source_agent, content): + stats["skipped"] += 1 + continue + spool.append(db_path, "verbatim", {"session_id": session_id, "role": role, "content": content, "scope": scope, "source_agent": source_agent, "timestamp": now}) + stats["spooled"] += 1 + return stats + + def _row_dedup_key(r: dict) -> tuple: """Stable unique key for hybrid-mode search-result dedup. diff --git a/memorymaster/surfaces/setup_hooks.py b/memorymaster/surfaces/setup_hooks.py index ffd39f3e..e9aa99e9 100644 --- a/memorymaster/surfaces/setup_hooks.py +++ b/memorymaster/surfaces/setup_hooks.py @@ -125,8 +125,8 @@ def replace_placeholder(content, project_root): # 1. LLM Provider config # --------------------------------------------------------------------------- def setup_llm_provider(): - banner("LLM Provider for Auto-Ingest Stop Hook") - print(" The Stop hook uses a cheap LLM to extract learnings from each session.") + banner("LLM Provider for Session-End Distillation") + print(" The quiet session-end hook uses a cheap LLM to distill key learnings.") print(" Supported: google (Gemini Flash Lite, ~free), openai, anthropic, ollama\n") provider = ask("Provider", "google") @@ -249,6 +249,18 @@ def install_hooks(llm_config, include_pretooluse: bool = False): ss_hooks[:] = [h for h in ss_hooks if "memorymaster" not in json.dumps(h)] ss_hooks.append(session_start_hook) + # SessionEnd — one distilled ingest pass, quiet and cursor/budget bounded. + session_end_hook = { + "hooks": [{ + "type": "command", + "command": f'python "{hooks_dir / "memorymaster-session-end.py"}"', + "timeout": 30 + }] + } + se_hooks = hooks.setdefault("SessionEnd", []) + se_hooks[:] = [h for h in se_hooks if "memorymaster" not in json.dumps(h)] + se_hooks.append(session_end_hook) + # Stop — auto-ingest stop_hook = { "hooks": [{ @@ -692,7 +704,7 @@ def build_arg_parser() -> argparse.ArgumentParser: p.add_argument( "--provider", choices=["google", "openai", "anthropic", "ollama"], - help="LLM provider for the auto-ingest Stop hook", + help="LLM provider for quiet session-end distillation", ) p.add_argument("--api-key", help="API key for the chosen provider") p.add_argument("--model", help="LLM model id") @@ -905,7 +917,7 @@ def _run_main(args: argparse.Namespace) -> tuple[int, Optional[dict[str, Any]]]: banner("Setup Complete!") print(" What actually happened (skips reflect what's installed on this box):") if detected.claude_code: - print(" - Claude hooks (recall + auto-ingest + session-start) — installed") + print(" - Claude hooks (session-start + on-demand recall + session-end distill) — installed") print(" - MemoryMaster MCP — registered (memorymaster.surfaces.mcp_server)") else: print(" - Claude Code not detected — hooks + MCP SKIPPED. Install Claude") diff --git a/tests/test_ambient_spool.py b/tests/test_ambient_spool.py index b61e134c..519d4eaa 100644 --- a/tests/test_ambient_spool.py +++ b/tests/test_ambient_spool.py @@ -358,7 +358,9 @@ def test_auto_ingest_hook_template_spools_under_flag( "stop_hook_active": False, }) - proc = _run_hook(hook, payload, _hook_env(tmp_path / "home", spool_root, "1")) + env = _hook_env(tmp_path / "home", spool_root, "1") + env["MEMORYMASTER_STOP_CAPTURE_VERBATIM"] = "1" + proc = _run_hook(hook, payload, env) assert proc.returncode == 0, proc.stderr assert json.loads(proc.stdout)["decision"] == "approve" @@ -373,9 +375,7 @@ def test_auto_ingest_hook_template_spools_under_flag( def test_auto_ingest_hook_template_direct_path_with_flag_off( tmp_path: Path, spool_root: Path ) -> None: - """REQUIREMENT (flag default OFF): without the flag the template keeps the - legacy direct write — rows land in verbatim_memories, spool stays empty. - The else-branch must remain intact for the §5 rollback.""" + """Explicit verbatim capture with WAL discipline off uses the direct path.""" project_root = tmp_path / "proj" project_root.mkdir() db = project_root / "memorymaster.db" @@ -389,7 +389,9 @@ def test_auto_ingest_hook_template_direct_path_with_flag_off( "stop_hook_active": False, }) - proc = _run_hook(hook, payload, _hook_env(tmp_path / "home", spool_root, "0")) + env = _hook_env(tmp_path / "home", spool_root, "0") + env["MEMORYMASTER_STOP_CAPTURE_VERBATIM"] = "1" + proc = _run_hook(hook, payload, env) assert proc.returncode == 0, proc.stderr assert json.loads(proc.stdout)["decision"] == "approve" diff --git a/tests/test_capture_control.py b/tests/test_capture_control.py new file mode 100644 index 00000000..d7044450 --- /dev/null +++ b/tests/test_capture_control.py @@ -0,0 +1,55 @@ +from __future__ import annotations + +from pathlib import Path + +from memorymaster.core.capture_control import CaptureLedger, CaptureLimits + + +def test_cursor_is_restart_safe_and_only_returns_complete_appends(tmp_path: Path) -> None: + state = tmp_path / "capture.db" + transcript = tmp_path / "session.jsonl" + transcript.write_bytes(b'{"turn": 1}\n') + + first = CaptureLedger(state).read_increment(transcript, "session:end") + assert first.text == '{"turn": 1}\n' + CaptureLedger(state).commit_cursor(first) + assert CaptureLedger(state).read_increment(transcript, "session:end").text == "" + + with transcript.open("ab") as handle: + handle.write(b'{"turn": 2}') + assert CaptureLedger(state).read_increment(transcript, "session:end").text == "" + with transcript.open("ab") as handle: + handle.write(b"\n") + second = CaptureLedger(state).read_increment(transcript, "session:end") + assert second.text == '{"turn": 2}\n' + CaptureLedger(state).commit_cursor(second) + + transcript.write_bytes(b'{"turn": "replacement"}\n') + replacement = CaptureLedger(state).read_increment(transcript, "session:end") + assert replacement.start_offset == 0 + assert "replacement" in replacement.text + + +def test_provider_global_and_session_budgets_survive_restart(tmp_path: Path) -> None: + state = tmp_path / "capture.db" + limits = CaptureLimits(global_daily_calls=3, provider_daily_calls=2, session_daily_calls=1) + ledger = CaptureLedger(state, limits=limits) + + first = ledger.reserve_llm("google", "session-a", "session-end") + assert first is not None + ledger.finish_llm(first, input_bytes=40, output_bytes=20, outcome="ok") + assert CaptureLedger(state, limits=limits).reserve_llm("google", "session-a", "retry") is None + + second = CaptureLedger(state, limits=limits).reserve_llm("google", "session-b", "session-end") + assert second is not None + assert CaptureLedger(state, limits=limits).reserve_llm("google", "session-c", "session-end") is None + + third = CaptureLedger(state, limits=limits).reserve_llm("openai", "session-c", "session-end") + assert third is not None + assert CaptureLedger(state, limits=limits).reserve_llm("openai", "session-d", "session-end") is None + + usage = CaptureLedger(state, limits=limits).usage() + assert usage["global_calls"] == 3 + assert usage["providers"] == {"google": 2, "openai": 1} + assert usage["input_bytes"] == 40 + assert usage["output_bytes"] == 20 diff --git a/tests/test_capture_retention_and_backlog.py b/tests/test_capture_retention_and_backlog.py new file mode 100644 index 00000000..22ca276d --- /dev/null +++ b/tests/test_capture_retention_and_backlog.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +import sqlite3 +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +from memorymaster.govern.review import build_candidate_backlog_plan +from memorymaster.govern.verbatim_cleanup import plan_retention + + +def _verbatim_db(path: Path) -> None: + conn = sqlite3.connect(path) + conn.execute( + """CREATE TABLE verbatim_memories ( + id INTEGER PRIMARY KEY, + session_id TEXT NOT NULL, + content TEXT NOT NULL, + timestamp TEXT NOT NULL, + created_at TEXT NOT NULL + )""" + ) + conn.executemany( + "INSERT INTO verbatim_memories VALUES (?, ?, ?, ?, ?)", + [ + (1, "old", "x" * 40, "2026-01-01T00:00:00+00:00", "2026-01-01T00:00:00+00:00"), + (2, "recent-a", "y" * 40, "2026-07-12T00:00:00+00:00", "2026-07-12T00:00:00+00:00"), + (3, "recent-b", "z" * 40, "2026-07-13T00:00:00+00:00", "2026-07-13T00:00:00+00:00"), + ], + ) + conn.commit() + conn.close() + + +def test_retention_plan_enforces_age_bytes_sessions_without_mutation(tmp_path: Path) -> None: + db = tmp_path / "verbatim.db" + _verbatim_db(db) + before = db.read_bytes() + + plan = plan_retention( + str(db), + max_age_days=30, + max_bytes=50, + max_sessions=1, + now=datetime(2026, 7, 13, tzinfo=timezone.utc), + ) + + assert plan["dry_run"] is True + assert plan["limits"] == {"max_age_days": 30, "max_bytes": 50, "max_sessions": 1} + assert plan["candidate_rows"] == 2 + assert plan["retained_rows"] == 1 + assert plan["retained_bytes"] <= 50 + assert db.read_bytes() == before + + +@dataclass +class _Claim: + id: int + status: str = "candidate" + + +class _ReadOnlyService: + def __init__(self) -> None: + self.calls: list[dict[str, object]] = [] + + def list_claims(self, **kwargs): + self.calls.append(kwargs) + return [_Claim(index) for index in range(1, 1201)] + + +def test_backlog_plan_is_bounded_reviewable_and_read_only() -> None: + service = _ReadOnlyService() + plan = build_candidate_backlog_plan(service, daily_capacity=688, batch_size=100, scan_limit=5000) + + assert plan["dry_run"] is True + assert plan["candidate_count"] == 1200 + assert plan["review_batches"] == 12 + assert plan["minimum_days"] == 2 + assert plan["automatic_transitions"] == 0 + assert service.calls == [ + { + "status": "candidate", + "include_archived": False, + "limit": 5000, + "allow_sensitive": False, + } + ] diff --git a/tests/test_precompact_capture_policy.py b/tests/test_precompact_capture_policy.py new file mode 100644 index 00000000..b7a671f6 --- /dev/null +++ b/tests/test_precompact_capture_policy.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +TEMPLATE = ROOT / "memorymaster" / "config_templates" / "hooks" / "memorymaster-precompact.py" + + +def _run_hook(tmp_path: Path, *, maximum: bool) -> subprocess.CompletedProcess[str]: + hook = tmp_path / "precompact.py" + hook.write_text(TEMPLATE.read_text(encoding="utf-8"), encoding="utf-8") + env = os.environ.copy() + env.update({"HOME": str(tmp_path / "home"), "USERPROFILE": str(tmp_path / "home")}) + if maximum: + env["MEMORYMASTER_PRECOMPACT_BLOCKING"] = "1" + else: + env.pop("MEMORYMASTER_PRECOMPACT_BLOCKING", None) + return subprocess.run( + [sys.executable, str(hook)], + input=json.dumps({"session_id": "precompact-policy"}), + capture_output=True, + text=True, + env=env, + timeout=30, + ) + + +def test_precompact_is_quiet_by_default(tmp_path: Path) -> None: + result = _run_hook(tmp_path, maximum=False) + assert result.returncode == 0 + assert result.stdout == "" + assert not (tmp_path / "home" / ".memorymaster").exists() + + +def test_precompact_blocking_is_explicit_maximum_capture(tmp_path: Path) -> None: + result = _run_hook(tmp_path, maximum=True) + assert result.returncode == 0 + assert json.loads(result.stdout)["decision"] == "block" diff --git a/tests/test_session_end_capture_hook.py b/tests/test_session_end_capture_hook.py new file mode 100644 index 00000000..97533443 --- /dev/null +++ b/tests/test_session_end_capture_hook.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +TEMPLATE = ROOT / "memorymaster" / "config_templates" / "hooks" / "memorymaster-session-end.py" + + +def test_session_end_distills_once_and_cursor_survives_restart(tmp_path: Path) -> None: + project = tmp_path / "project" + script_dir = project / "scripts" + script_dir.mkdir(parents=True) + (script_dir / "__init__.py").write_text("", encoding="utf-8") + (script_dir / "agent_session_end_ingest.py").write_text( + """from pathlib import Path +def run(db_path, transcript_path, *, source_agent, cwd): + Path(cwd, 'distilled.txt').write_text(Path(transcript_path).read_text(encoding='utf-8'), encoding='utf-8') + return 1 +""", + encoding="utf-8", + ) + hook = tmp_path / "session-end.py" + hook.write_text( + TEMPLATE.read_text(encoding="utf-8").replace("__MEMORYMASTER_PROJECT_ROOT__", str(project).replace("\\", "/")), + encoding="utf-8", + ) + transcript = tmp_path / "session.jsonl" + transcript.write_text(json.dumps({"message": {"role": "assistant", "content": "A durable session learning with sufficient content."}}) + "\n") + env = os.environ.copy() + env.update( + { + "MEMORYMASTER_CAPTURE_STATE_DB": str(tmp_path / "capture.db"), + "MEMORYMASTER_LLM_PROVIDER": "fake", + "PYTHONPATH": os.pathsep.join((str(ROOT), str(project))), + } + ) + payload = {"session_id": "session-end", "transcript_path": str(transcript), "cwd": str(project)} + + first = subprocess.run([sys.executable, str(hook)], input=json.dumps(payload), text=True, capture_output=True, env=env) + assert first.returncode == 0, first.stderr + assert "durable session learning" in (project / "distilled.txt").read_text().lower() + marker_mtime = (project / "distilled.txt").stat().st_mtime_ns + + second = subprocess.run([sys.executable, str(hook)], input=json.dumps(payload), text=True, capture_output=True, env=env) + assert second.returncode == 0, second.stderr + assert (project / "distilled.txt").stat().st_mtime_ns == marker_mtime diff --git a/tests/test_setup_hooks.py b/tests/test_setup_hooks.py index d7c09ff3..62507361 100644 --- a/tests/test_setup_hooks.py +++ b/tests/test_setup_hooks.py @@ -233,6 +233,7 @@ def test_no_duplicate_hooks_on_rerun(self, hermetic_home, monkeypatch): mm_entries = [h for h in ups if "memorymaster" in json.dumps(h)] # recall + classify = exactly 2, not 4. assert len(mm_entries) == 2 + assert len(settings["hooks"]["SessionEnd"]) == 1 def test_settings_json_stays_valid(self, hermetic_home): llm = {"provider": "ollama", "api_key": "", "model": "llama3.2:3b"} diff --git a/tests/test_stop_hook_capture_policy.py b/tests/test_stop_hook_capture_policy.py index faf269d4..23d0e7eb 100644 --- a/tests/test_stop_hook_capture_policy.py +++ b/tests/test_stop_hook_capture_policy.py @@ -8,9 +8,6 @@ import sys from pathlib import Path -import pytest - - ROOT = Path(__file__).resolve().parents[1] TEMPLATE = ROOT / "memorymaster" / "config_templates" / "hooks" / "memorymaster-auto-ingest.py" @@ -40,10 +37,6 @@ def _write_transcript(path: Path) -> Path: return path -@pytest.mark.xfail( - strict=True, - reason="R2.4: the default Stop hook blocks and captures instead of remaining quiet", -) def test_default_stop_hook_is_quiet_and_nonblocking(tmp_path): project_root = tmp_path / "project" project_root.mkdir() @@ -95,3 +88,87 @@ def test_default_stop_hook_is_quiet_and_nonblocking(tmp_path): assert result.returncode == 0, result.stderr assert json.loads(result.stdout) == {"decision": "approve"} assert envelopes == [] + + +def test_blocking_requires_explicit_maximum_capture_flag(tmp_path): + project_root = tmp_path / "project" + project_root.mkdir() + hook = _render_hook(project_root, tmp_path / "stop-hook.py") + transcript = _write_transcript(tmp_path / "session.jsonl") + env = os.environ.copy() + env.update( + { + "HOME": str(tmp_path / "home"), + "USERPROFILE": str(tmp_path / "home"), + "MEMORYMASTER_STOP_BLOCKING": "1", + "PYTHONPATH": str(ROOT), + } + ) + payload = { + "session_id": "maximum-policy", + "transcript_path": str(transcript), + "cwd": str(project_root), + "stop_hook_active": False, + } + + result = subprocess.run( + [sys.executable, str(hook)], + input=json.dumps(payload), + capture_output=True, + text=True, + env=env, + timeout=30, + ) + + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout)["decision"] == "block" + + +def test_verbatim_capture_only_spools_appended_turns(tmp_path): + project_root = tmp_path / "project" + project_root.mkdir() + hook = _render_hook(project_root, tmp_path / "stop-hook.py") + transcript = _write_transcript(tmp_path / "session.jsonl") + spool_root = tmp_path / "spool" + env = os.environ.copy() + env.update( + { + "HOME": str(tmp_path / "home"), + "USERPROFILE": str(tmp_path / "home"), + "MEMORYMASTER_CAPTURE_STATE_DB": str(tmp_path / "capture.db"), + "MEMORYMASTER_SPOOL_DIR": str(spool_root), + "MEMORYMASTER_WAL_DISCIPLINE": "1", + "MEMORYMASTER_STOP_CAPTURE_VERBATIM": "1", + "PYTHONPATH": str(ROOT), + } + ) + payload = { + "session_id": "cursor-policy", + "transcript_path": str(transcript), + "cwd": str(project_root), + "stop_hook_active": False, + } + + def run_hook() -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, str(hook)], + input=json.dumps(payload), + capture_output=True, + text=True, + env=env, + timeout=30, + ) + + assert run_hook().returncode == 0 + first = [line for path in spool_root.rglob("*.jsonl") for line in path.read_text().splitlines()] + assert len(first) == 15 + + assert run_hook().returncode == 0 + second = [line for path in spool_root.rglob("*.jsonl") for line in path.read_text().splitlines()] + assert second == first + + with transcript.open("a", encoding="utf-8") as handle: + handle.write(json.dumps({"message": {"role": "assistant", "content": "A newly appended assistant turn with enough detail."}}) + "\n") + assert run_hook().returncode == 0 + third = [line for path in spool_root.rglob("*.jsonl") for line in path.read_text().splitlines()] + assert len(third) == 16 From c66e240830e0526a1e86fc67bff10c0135219593 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Mon, 13 Jul 2026 20:01:05 -0300 Subject: [PATCH 35/55] fix: remove fake evidence defaults --- .planning/REMAINING-PHASES-V1-BUDGET.md | 23 +- docs/atlas-api-contract-v1.md | 12 +- docs/cli-cookbook.md | 6 +- memorymaster/bridges/action_exporters.py | 26 ++ memorymaster/bridges/action_extractor.py | 4 + memorymaster/bridges/atlas_claim_extractor.py | 3 + memorymaster/bridges/atlas_contract.py | 18 +- memorymaster/bridges/atlas_llm_extractor.py | 3 + memorymaster/bridges/evidence_policy.py | 50 ++++ memorymaster/bridges/media_processing.py | 66 ++++- memorymaster/bridges/media_providers.py | 30 ++- memorymaster/surfaces/cli.py | 16 +- memorymaster/surfaces/cli_handlers_basic.py | 4 +- tests/test_atlas_contract.py | 22 +- tests/test_atlas_mock_evidence_guard.py | 4 - tests/test_evidence_authenticity_policy.py | 226 ++++++++++++++++++ tests/test_media_processing.py | 14 +- 17 files changed, 481 insertions(+), 46 deletions(-) create mode 100644 memorymaster/bridges/evidence_policy.py create mode 100644 tests/test_evidence_authenticity_policy.py diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index 91b99fb0..fb991cc8 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -241,5 +241,24 @@ P2-E implementation and focused gate completed on 2026-07-13 from P2-D commit - Atomic package commit: the conventional `fix: bound quiet capture and retention` commit containing this scheduler evidence. -The next separately authorized package is P2-F. Do not begin fake-evidence -default removal from the P2-E goal. +P2-F implementation and focused gate completed on 2026-07-13 from P2-E commit +`a1d54773fa5e540ad90b6617e02c983ddbfe3d32`: + +- Production media commands now require an explicit, configuration-ready real + provider. The Atlas contract records the breaking fail-closed change as + version `2.0.0`; processing/configuration failures return non-zero status. +- Mock, synthetic, placeholder, fake, and fixture providers require both an + explicit test/development mode and `MEMORYMASTER_ALLOW_SYNTHETIC_MEDIA=1`. + Synthetic evidence is excluded before deterministic claims, LLM prompts, + action proposals, citations, and approved-action export. +- Media idempotency is provider-specific, so historical mock evidence cannot + block later real enrichment. Imported `wacli` evidence remains eligible. +- Focused P2-F gate: 111 passed in 23.30s. Changed-file Ruff and + `git diff --check` passed; the independent policy/contract invariant probe + confirmed production fail-closed behavior, explicit test opt-in, required + providers, and preservation of legitimate imported evidence. +- No paid or production provider was called and no live/external data or + configuration was mutated. + +The next separately authorized package is P2-Z Phase 2 convergence. Do not +begin it from the P2-F goal. diff --git a/docs/atlas-api-contract-v1.md b/docs/atlas-api-contract-v1.md index e2c2b9eb..0efb6c2b 100644 --- a/docs/atlas-api-contract-v1.md +++ b/docs/atlas-api-contract-v1.md @@ -79,8 +79,8 @@ Non-Atlas subcommands (the rest of MemoryMaster) emit the same envelope **withou | `process-media-retry-queue` | `--limit ` (def 25) | `{attempted, expired, recovered, failed, pending_remaining, rows:[MediaRetryItem]}` | `attempted` | | `record-media-retry-outcome` | `--retry-id ` (req), `--status {pending,retrying,expired,done,failed}` (req), `--media-path` (required for `done`), `--last-http-status`, `--last-error`, `--next-attempt-time` | `MediaRetryItem` | `1` | | `list-media-retries` | `--status`, `--source-item-id`, `--limit` (def 100) | `list[MediaRetryItem]` | `len(data)` | -| `transcribe-source-item` | `--source-item-id ` (req), `--provider {mock,openai}` (def `mock`) | `{source_item_id, created, evidence, error, provider}` | `1 if evidence else 0` | -| `ocr-source-item` | `--source-item-id ` (req), `--provider {mock,tesseract}` (def `mock`) | same shape | same | +| `transcribe-source-item` | `--source-item-id ` (req), `--provider {mock,openai}` (req) | `{source_item_id, created, evidence, error, provider}` | `1 if evidence else 0` | +| `ocr-source-item` | `--source-item-id ` (req), `--provider {mock,tesseract}` (req) | same shape | same | | `export-actions` | `--output ` (req), `--destination` (def `super-productivity`), `--limit` (def 100), `--dry-run` | `{destination, output_path, exported, proposal_ids:[int]}` | `exported` | | `atlas-version` | (none) | `{atlas_contract_version, atlas_contract_name, subcommands, endpoints, breaking_changes_since}` | `1` | @@ -103,14 +103,14 @@ LifeAgent should treat these as authoritative backend labels and use them to fil ### Real provider adapters (v1.5.0) -`memorymaster/media_providers.py` ships two real adapters behind the existing `TranscriptionProvider` / `OcrProvider` `Protocol`s. Mock providers remain the default. +`memorymaster/media_providers.py` ships two real adapters behind the existing `TranscriptionProvider` / `OcrProvider` `Protocol`s. Production has no synthetic fallback: callers select a ready real provider explicitly. | Adapter | Class | Optional dependency | Env vars | |---|---|---|---| | OpenAI Whisper transcription | `OpenAIWhisperTranscriptionProvider` | None (stdlib only — urllib + manual multipart) | `OPENAI_API_KEY` (required at call time), `OPENAI_BASE_URL` (default `https://api.openai.com/v1`) | | Tesseract OCR | `TesseractOcrProvider` | `pytesseract` Python package + system `tesseract` binary | None | -**Lazy validation:** importing the module never fails. Optional dependencies are checked only when `.transcribe(path)` / `.extract(path)` is called. Missing deps raise `RuntimeError` with a helpful install message — `_process_media` catches it and records a `media_process_failed` event so the source_item is preserved. +**Readiness:** importing the module remains safe. The CLI factory checks credentials and dependencies before processing; direct provider calls retain actionable runtime checks. Missing configuration never produces placeholder evidence. **Factory:** @@ -120,6 +120,8 @@ provider = get_transcription_provider("openai") # or "mock" provider = get_ocr_provider("tesseract") # or "mock" ``` +Mocks are test/development-only and require both `MEMORYMASTER_MEDIA_MODE=test` (or `development`) and `MEMORYMASTER_ALLOW_SYNTHETIC_MEDIA=1`. Synthetic evidence is excluded from claim extraction, LLM prompts, action proposals, citations, and action export. + **CLI:** ```bash @@ -132,7 +134,7 @@ python -m memorymaster --db /data/atlas-inbox.db --json ocr-source-item \ --source-item-id 99 --provider tesseract ``` -Both CLIs run the existing `process_transcription` / `process_ocr` pipeline — failures are recorded as `media_process` events with `details="media_process_failed"`, never raised. Re-running on a source_item that already has the corresponding evidence type returns `created=false` and the existing evidence row. +Both CLIs run the existing `process_transcription` / `process_ocr` pipeline. Failures are recorded as `media_process` events with `details="media_process_failed"` and return a non-zero exit code. Idempotency is provider-specific, so an old mock row cannot block later real enrichment. ### Media retry queue (v1.4.0) diff --git a/docs/cli-cookbook.md b/docs/cli-cookbook.md index d526fdae..9e876355 100644 --- a/docs/cli-cookbook.md +++ b/docs/cli-cookbook.md @@ -482,16 +482,18 @@ python -m memorymaster --db memorymaster.db record-media-retry-outcome --retry-i **Purpose**: Transcribe a source item through the selected provider. **Example**: ``` -python -m memorymaster --db memorymaster.db transcribe-source-item --source-item-id 1 --provider mock +python -m memorymaster --db memorymaster.db transcribe-source-item --source-item-id 1 --provider openai ``` ### ocr-source-item **Purpose**: Run OCR on a source item through the selected provider. **Example**: ``` -python -m memorymaster --db memorymaster.db ocr-source-item --source-item-id 1 --provider mock +python -m memorymaster --db memorymaster.db ocr-source-item --source-item-id 1 --provider tesseract ``` +Mock providers require both `MEMORYMASTER_MEDIA_MODE=test` (or `development`) and `MEMORYMASTER_ALLOW_SYNTHETIC_MEDIA=1`. They never feed governed claims, actions, citations, or exports. + ### export-actions **Purpose**: Export approved Atlas action proposals. **Example**: diff --git a/memorymaster/bridges/action_exporters.py b/memorymaster/bridges/action_exporters.py index 6faf926f..fdf35229 100644 --- a/memorymaster/bridges/action_exporters.py +++ b/memorymaster/bridges/action_exporters.py @@ -6,6 +6,8 @@ from pathlib import Path from typing import Any +from memorymaster.bridges.evidence_policy import is_governed_evidence_eligible + @dataclass(frozen=True) class ActionExportResult: @@ -36,6 +38,23 @@ def export_approved_actions( destination=destination, limit=limit, ) + needed_evidence_ids = { + proposal.evidence_item_id + for proposal in proposals + if proposal.evidence_item_id is not None + } + evidence_by_id = {} + if needed_evidence_ids: + evidence_by_id = { + evidence.id: evidence + for evidence in service.list_evidence_items(limit=max(needed_evidence_ids)) + if evidence.id in needed_evidence_ids + } + proposals = [ + proposal + for proposal in proposals + if _proposal_evidence_is_eligible(proposal, evidence_by_id) + ] path = Path(output_path) path.parent.mkdir(parents=True, exist_ok=True) tasks = [_proposal_to_super_productivity_bridge_task(proposal) for proposal in proposals] @@ -66,6 +85,13 @@ def export_approved_actions( ) +def _proposal_evidence_is_eligible(proposal, evidence_by_id: dict[int, Any]) -> bool: + if proposal.evidence_item_id is None: + return True + evidence = evidence_by_id.get(proposal.evidence_item_id) + return evidence is not None and is_governed_evidence_eligible(evidence) + + def _proposal_to_super_productivity_bridge_task(proposal) -> dict[str, Any]: notes = proposal.description or "" if proposal.source_item_id is not None: diff --git a/memorymaster/bridges/action_extractor.py b/memorymaster/bridges/action_extractor.py index 5b35f10a..1e49c53b 100644 --- a/memorymaster/bridges/action_extractor.py +++ b/memorymaster/bridges/action_extractor.py @@ -11,6 +11,7 @@ from datetime import datetime, time, timedelta from typing import Any +from memorymaster.bridges.evidence_policy import is_governed_evidence_eligible from memorymaster.core.models import ActionProposal, EvidenceItem, SourceItem @@ -84,6 +85,8 @@ def propose_actions_from_evidence( for evidence in evidence_items: scanned += 1 + if not is_governed_evidence_eligible(evidence): + continue source_item = service.get_source_item_by_id(evidence.source_item_id) draft = _draft_from_evidence(evidence, source_item) if draft is None: @@ -103,6 +106,7 @@ def propose_actions_from_evidence( payload_json={ "extractor": "atlas-rule-v1", "evidence_type": evidence.evidence_type, + "evidence_provider": evidence.provider, "source_item_external_id": source_item.source_item_id if source_item else None, }, idempotency_key=idempotency_key, diff --git a/memorymaster/bridges/atlas_claim_extractor.py b/memorymaster/bridges/atlas_claim_extractor.py index 224b7b85..be857fcb 100644 --- a/memorymaster/bridges/atlas_claim_extractor.py +++ b/memorymaster/bridges/atlas_claim_extractor.py @@ -6,6 +6,7 @@ from dataclasses import asdict, dataclass from typing import Any +from memorymaster.bridges.evidence_policy import is_governed_evidence_eligible from memorymaster.core.models import CitationInput, Claim, EvidenceItem, SourceItem @@ -55,6 +56,8 @@ def extract_atlas_claims_from_evidence( for evidence in evidence_items: scanned += 1 + if not is_governed_evidence_eligible(evidence): + continue source_item = service.get_source_item_by_id(evidence.source_item_id) draft = _draft_from_evidence(evidence) if draft is None: diff --git a/memorymaster/bridges/atlas_contract.py b/memorymaster/bridges/atlas_contract.py index 043e6e29..2e251507 100644 --- a/memorymaster/bridges/atlas_contract.py +++ b/memorymaster/bridges/atlas_contract.py @@ -42,7 +42,7 @@ from typing import Any -ATLAS_CONTRACT_VERSION = "1.5.1" +ATLAS_CONTRACT_VERSION = "2.0.0" """Semver string for the Atlas API/CLI contract. LifeAgent and any other consumer MUST refuse to start if the major component @@ -214,10 +214,10 @@ }, { "name": "transcribe-source-item", - "description": "Run transcription on a source_item via the chosen provider; stores transcript as an evidence_item. Mock is default. Real provider 'openai' uses Whisper API via OPENAI_API_KEY + OPENAI_BASE_URL.", + "description": "Run transcription using an explicitly selected provider. Production requires ready OpenAI configuration; mock requires conspicuous test/development opt-in.", "inputs": { "--source-item-id": {"type": "int", "required": True}, - "--provider": {"type": "str", "required": False, "default": "mock", "allowed": ["mock", "openai"]}, + "--provider": {"type": "str", "required": True, "allowed": ["mock", "openai"]}, }, "data_keys": ["source_item_id", "created", "evidence", "error", "provider"], "meta_total": "1 if evidence else 0", @@ -225,10 +225,10 @@ }, { "name": "ocr-source-item", - "description": "Run OCR on a source_item via the chosen provider; stores OCR text as an evidence_item. Mock is default. Real provider 'tesseract' requires the pytesseract Python package AND a system 'tesseract' binary on PATH.", + "description": "Run OCR using an explicitly selected provider. Production requires ready Tesseract configuration; mock requires conspicuous test/development opt-in.", "inputs": { "--source-item-id": {"type": "int", "required": True}, - "--provider": {"type": "str", "required": False, "default": "mock", "allowed": ["mock", "tesseract"]}, + "--provider": {"type": "str", "required": True, "allowed": ["mock", "tesseract"]}, }, "data_keys": ["source_item_id", "created", "evidence", "error", "provider"], "meta_total": "1 if evidence else 0", @@ -331,7 +331,13 @@ ] -BREAKING_CHANGES_SINCE: list[dict[str, str]] = [] +BREAKING_CHANGES_SINCE: list[dict[str, str]] = [ + { + "version": "2.0.0", + "summary": "Media commands require an explicit, ready provider; mock media is restricted to opted-in test/development modes.", + "date": "2026-07-13", + }, +] """History of breaking changes. Each entry: {"version": "X.Y.Z", "summary": "...", "date": "YYYY-MM-DD"}. Empty in 1.0.0 — the contract was born here. Future major bumps must append. diff --git a/memorymaster/bridges/atlas_llm_extractor.py b/memorymaster/bridges/atlas_llm_extractor.py index cd640b42..b2a760ae 100644 --- a/memorymaster/bridges/atlas_llm_extractor.py +++ b/memorymaster/bridges/atlas_llm_extractor.py @@ -25,6 +25,7 @@ from typing import Any from memorymaster.bridges.atlas_claim_extractor import AtlasClaimExtractionResult +from memorymaster.bridges.evidence_policy import is_governed_evidence_eligible from memorymaster.core.llm_provider import call_llm, parse_json_response from memorymaster.core.models import CitationInput, Claim, EvidenceItem, SourceItem @@ -172,6 +173,8 @@ def extract_atlas_claims_llm( for evidence in evidence_items: scanned += 1 + if not is_governed_evidence_eligible(evidence): + continue source_item = service.get_source_item_by_id(evidence.source_item_id) typed_claims = _extract_for_evidence(evidence, source_item, model=model) if typed_claims is None: diff --git a/memorymaster/bridges/evidence_policy.py b/memorymaster/bridges/evidence_policy.py new file mode 100644 index 00000000..a93684c6 --- /dev/null +++ b/memorymaster/bridges/evidence_policy.py @@ -0,0 +1,50 @@ +"""Authenticity policy for media-derived evidence. + +Synthetic evidence may be useful in tests and demonstrations, but it must be +both conspicuously enabled and permanently excluded from governed knowledge +and action paths. +""" +from __future__ import annotations + +import os +from collections.abc import Mapping +from typing import Any + +_NONPRODUCTION_MODES = frozenset({"development", "test"}) +_TRUE_VALUES = frozenset({"1", "true", "yes", "on"}) +_SYNTHETIC_PROVIDER_TOKENS = frozenset({"mock", "synthetic", "placeholder", "fake", "fixture"}) + + +def is_synthetic_provider(provider: str | None) -> bool: + """Return whether a provider identity explicitly denotes fabricated data.""" + normalized = (provider or "").strip().lower().replace("_", "-") + if not normalized: + return False + tokens = frozenset(part for part in normalized.split("-") if part) + return bool(tokens & _SYNTHETIC_PROVIDER_TOKENS) + + +def synthetic_media_configuration_error( + provider: str | None, + *, + environ: Mapping[str, str] | None = None, +) -> str | None: + """Return an actionable error when a synthetic provider is not allowed.""" + if not is_synthetic_provider(provider): + return None + env = os.environ if environ is None else environ + mode = env.get("MEMORYMASTER_MEDIA_MODE", "production").strip().lower() + allowed = env.get("MEMORYMASTER_ALLOW_SYNTHETIC_MEDIA", "").strip().lower() + if mode in _NONPRODUCTION_MODES and allowed in _TRUE_VALUES: + return None + return ( + f"Synthetic media provider '{provider}' is disabled. Production requires an explicitly " + "configured real provider. For conspicuous test/development use only, set " + "MEMORYMASTER_MEDIA_MODE=test (or development) and " + "MEMORYMASTER_ALLOW_SYNTHETIC_MEDIA=1." + ) + + +def is_governed_evidence_eligible(evidence: Any) -> bool: + """Exclude fabricated evidence from claims, actions, citations, and export.""" + return not is_synthetic_provider(getattr(evidence, "provider", None)) diff --git a/memorymaster/bridges/media_processing.py b/memorymaster/bridges/media_processing.py index 1fe49ec4..cf22f260 100644 --- a/memorymaster/bridges/media_processing.py +++ b/memorymaster/bridges/media_processing.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any, Protocol +from memorymaster.bridges.evidence_policy import synthetic_media_configuration_error from memorymaster.core.models import EvidenceItem @@ -115,19 +116,31 @@ def _process_media( if source_item is None: raise ValueError(f"Source item {source_item_row_id} does not exist.") + configuration_error = synthetic_media_configuration_error(provider_name) + if configuration_error: + return _media_failure_outcome( + service, + source_item_id=source_item.id, + evidence_type=evidence_type, + provider=provider_name, + media_path="", + error=configuration_error, + ) + existing = service.list_evidence_items( source_item_id=source_item.id, evidence_type=evidence_type, - limit=1, + limit=100, ) - if existing: - return MediaProcessOutcome(source_item_id=source_item.id, evidence=existing[0], created=False) + matching = [item for item in existing if _same_provider(item.provider, provider_name)] + if matching: + return MediaProcessOutcome(source_item_id=source_item.id, evidence=matching[0], created=False) media_path = _extract_media_path(source_item.payload_json) try: result = processor(media_path) except Exception as exc: - _record_media_failure( + return _media_failure_outcome( service, source_item_id=source_item.id, evidence_type=evidence_type, @@ -135,11 +148,19 @@ def _process_media( media_path=media_path, error=str(exc), ) - return MediaProcessOutcome( + + result_error = synthetic_media_configuration_error(result.provider) + if result_error or not _same_provider(result.provider, provider_name): + error = result_error or ( + f"Media provider identity mismatch: configured '{provider_name}', returned '{result.provider}'." + ) + return _media_failure_outcome( + service, source_item_id=source_item.id, - evidence=None, - created=False, - error=str(exc), + evidence_type=evidence_type, + provider=provider_name, + media_path=media_path, + error=error, ) evidence = service.add_evidence_item( @@ -154,6 +175,35 @@ def _process_media( return MediaProcessOutcome(source_item_id=source_item.id, evidence=evidence, created=True) +def _same_provider(left: str | None, right: str | None) -> bool: + return (left or "").strip().lower() == (right or "").strip().lower() + + +def _media_failure_outcome( + service, + *, + source_item_id: int, + evidence_type: str, + provider: str, + media_path: str, + error: str, +) -> MediaProcessOutcome: + _record_media_failure( + service, + source_item_id=source_item_id, + evidence_type=evidence_type, + provider=provider, + media_path=media_path, + error=error, + ) + return MediaProcessOutcome( + source_item_id=source_item_id, + evidence=None, + created=False, + error=error, + ) + + def _extract_media_path(payload_json: str | None) -> str: if not payload_json: return "" diff --git a/memorymaster/bridges/media_providers.py b/memorymaster/bridges/media_providers.py index 0d37baf7..0efa4a31 100644 --- a/memorymaster/bridges/media_providers.py +++ b/memorymaster/bridges/media_providers.py @@ -1,10 +1,9 @@ """Real provider adapters for Atlas Inbox media processing. These plug into the existing ``TranscriptionProvider`` / ``OcrProvider`` -``Protocol``s in ``memorymaster.bridges.media_processing``. They are **opt-in** — -``MockTranscriptionProvider`` / ``MockOcrProvider`` remain the defaults. -LifeAgent (or any consumer) imports these classes and passes the instance -into ``process_transcription`` / ``process_ocr``. +``Protocol``s in ``memorymaster.bridges.media_processing``. Production callers +must explicitly select a ready real provider. Mock providers require +conspicuous test/development configuration and are never governed evidence. Failure handling: each adapter raises ``RuntimeError`` on failure. The existing ``_process_media`` helper in ``media_processing`` catches the @@ -22,16 +21,19 @@ """ from __future__ import annotations +import importlib.util import json import logging import mimetypes import os +import shutil import urllib.error import urllib.request import uuid from pathlib import Path from typing import Any +from memorymaster.bridges.evidence_policy import synthetic_media_configuration_error from memorymaster.bridges.media_processing import EvidenceResult logger = logging.getLogger(__name__) @@ -223,12 +225,20 @@ def extract(self, path: str) -> EvidenceResult: def get_transcription_provider(name: str) -> Any: - """Return a transcription provider instance by name. Mock + openai supported.""" + """Return an explicitly selected, configuration-ready provider.""" name = name.strip().lower() if name in ("mock", "mock-transcription"): + error = synthetic_media_configuration_error("mock-transcription") + if error: + raise ValueError(error) from memorymaster.bridges.media_processing import MockTranscriptionProvider return MockTranscriptionProvider() if name in ("openai", "openai-whisper", "whisper"): + if not os.environ.get("OPENAI_API_KEY", "").strip(): + raise ValueError( + "OpenAI transcription is not ready: configure OPENAI_API_KEY " + "before selecting provider 'openai'." + ) return OpenAIWhisperTranscriptionProvider() raise ValueError( f"Unknown transcription provider '{name}'. Supported: mock, openai." @@ -236,12 +246,20 @@ def get_transcription_provider(name: str) -> Any: def get_ocr_provider(name: str) -> Any: - """Return an OCR provider instance by name. Mock + tesseract supported.""" + """Return an explicitly selected, configuration-ready provider.""" name = name.strip().lower() if name in ("mock", "mock-ocr"): + error = synthetic_media_configuration_error("mock-ocr") + if error: + raise ValueError(error) from memorymaster.bridges.media_processing import MockOcrProvider return MockOcrProvider() if name in ("tesseract", "tesseract-ocr", "local"): + if importlib.util.find_spec("pytesseract") is None or shutil.which("tesseract") is None: + raise ValueError( + "Tesseract OCR is not ready: install the 'pytesseract' package " + "and a 'tesseract' binary on PATH." + ) return TesseractOcrProvider() raise ValueError( f"Unknown OCR provider '{name}'. Supported: mock, tesseract." diff --git a/memorymaster/surfaces/cli.py b/memorymaster/surfaces/cli.py index a78bac3d..534af42c 100644 --- a/memorymaster/surfaces/cli.py +++ b/memorymaster/surfaces/cli.py @@ -174,13 +174,21 @@ def build_parser() -> argparse.ArgumentParser: transcribe_evidence = sub.add_parser("transcribe-source-item", help="Run transcription on a source_item via the chosen provider; stores transcript as evidence_item") transcribe_evidence.add_argument("--source-item-id", type=int, required=True, help="source_items.id") - transcribe_evidence.add_argument("--provider", choices=["mock", "openai"], default="mock", - help="'openai' uses Whisper API via OPENAI_API_KEY + OPENAI_BASE_URL.") + transcribe_evidence.add_argument( + "--provider", + choices=["mock", "openai"], + required=True, + help="Required. 'openai' uses Whisper via OPENAI_API_KEY; 'mock' needs explicit test/dev opt-in.", + ) ocr_evidence = sub.add_parser("ocr-source-item", help="Run OCR on a source_item via the chosen provider; stores OCR text as evidence_item") ocr_evidence.add_argument("--source-item-id", type=int, required=True, help="source_items.id") - ocr_evidence.add_argument("--provider", choices=["mock", "tesseract"], default="mock", - help="'tesseract' requires pytesseract package + system tesseract binary.") + ocr_evidence.add_argument( + "--provider", + choices=["mock", "tesseract"], + required=True, + help="Required. 'tesseract' needs pytesseract + binary; 'mock' needs explicit test/dev opt-in.", + ) export_actions = sub.add_parser("export-actions", help="Export approved Atlas action proposals") export_actions.add_argument("--output", required=True, help="Output JSON path") diff --git a/memorymaster/surfaces/cli_handlers_basic.py b/memorymaster/surfaces/cli_handlers_basic.py index bd51b04b..6a4d1d5f 100644 --- a/memorymaster/surfaces/cli_handlers_basic.py +++ b/memorymaster/surfaces/cli_handlers_basic.py @@ -585,7 +585,7 @@ def _handle_transcribe_source_item( f"len={len(outcome.evidence.text or '')}") else: print(f"transcription failed via {provider.provider_name}: {outcome.error}") - return 0 + return 0 if outcome.evidence else 2 def _handle_ocr_source_item( @@ -623,7 +623,7 @@ def _handle_ocr_source_item( f"len={len(outcome.evidence.text or '')}") else: print(f"ocr failed via {provider.provider_name}: {outcome.error}") - return 0 + return 0 if outcome.evidence else 2 def _handle_enqueue_media_retry( diff --git a/tests/test_atlas_contract.py b/tests/test_atlas_contract.py index 8f316074..a769aa99 100644 --- a/tests/test_atlas_contract.py +++ b/tests/test_atlas_contract.py @@ -538,7 +538,6 @@ def _seed_with_proposal(tmp_path: Path) -> tuple[Path, int, int]: main(["--db", str(db), "import-whatsapp", "--input", str(fixture)]) from memorymaster.core.service import MemoryService svc = MemoryService(db, workspace_root=tmp_path) - items = svc.store.connect() with svc.store.connect() as conn: rows = conn.execute("SELECT id FROM source_items LIMIT 1").fetchall() source_item_id = int(rows[0]["id"]) @@ -878,7 +877,9 @@ def test_init_db_migrates_stale_atlas_db_without_sensitivity_column(tmp_path: Pa # --------------------------------------------------------------------------- -def test_provider_factory_returns_mock() -> None: +def test_provider_factory_returns_mock(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MEMORYMASTER_MEDIA_MODE", "test") + monkeypatch.setenv("MEMORYMASTER_ALLOW_SYNTHETIC_MEDIA", "1") from memorymaster.bridges.media_providers import get_ocr_provider, get_transcription_provider assert get_transcription_provider("mock").provider_name == "mock-transcription" assert get_ocr_provider("mock").provider_name == "mock-ocr" @@ -914,14 +915,19 @@ def test_tesseract_class_lazy_imports() -> None: assert provider.provider_name == "tesseract" -def test_transcribe_source_item_mock_envelope(tmp_path: Path, capsys) -> None: +def test_transcribe_source_item_mock_envelope( + tmp_path: Path, + capsys, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MEMORYMASTER_MEDIA_MODE", "test") + monkeypatch.setenv("MEMORYMASTER_ALLOW_SYNTHETIC_MEDIA", "1") fixture = _FIXTURE_DIR / "whatsapp_wacli_basic.json" db = tmp_path / "atlas.db" main(["--db", str(db), "init-db"]) main(["--db", str(db), "import-whatsapp", "--input", str(fixture)]) from memorymaster.core.service import MemoryService svc = MemoryService(db, workspace_root=tmp_path) - audio_item = next((i for i in svc.list_evidence_items() if i.evidence_type == "message_text"), None) # Find the audio source_item with svc.store.connect() as conn: rows = conn.execute("SELECT id FROM source_items WHERE item_type='audio' LIMIT 1").fetchall() @@ -940,7 +946,13 @@ def test_transcribe_source_item_mock_envelope(tmp_path: Path, capsys) -> None: assert env["data"]["evidence"]["evidence_type"] == "transcript" -def test_ocr_source_item_mock_envelope(tmp_path: Path, capsys) -> None: +def test_ocr_source_item_mock_envelope( + tmp_path: Path, + capsys, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MEMORYMASTER_MEDIA_MODE", "test") + monkeypatch.setenv("MEMORYMASTER_ALLOW_SYNTHETIC_MEDIA", "1") fixture = _FIXTURE_DIR / "whatsapp_wacli_basic.json" db = tmp_path / "atlas.db" main(["--db", str(db), "init-db"]) diff --git a/tests/test_atlas_mock_evidence_guard.py b/tests/test_atlas_mock_evidence_guard.py index f93f96aa..2ea4c90c 100644 --- a/tests/test_atlas_mock_evidence_guard.py +++ b/tests/test_atlas_mock_evidence_guard.py @@ -17,10 +17,6 @@ def _run_cli_allowing_required_provider(argv: list[str]) -> int: return int(exc.code or 0) -@pytest.mark.xfail( - strict=True, - reason="MM-DEMO-01: media CLI defaults persist fabricated 0.99-confidence evidence", -) @pytest.mark.parametrize( ("command", "item_type", "evidence_type"), [ diff --git a/tests/test_evidence_authenticity_policy.py b/tests/test_evidence_authenticity_policy.py new file mode 100644 index 00000000..9bb76807 --- /dev/null +++ b/tests/test_evidence_authenticity_policy.py @@ -0,0 +1,226 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from memorymaster.bridges import atlas_llm_extractor +from memorymaster.bridges.action_exporters import export_approved_actions +from memorymaster.bridges.action_extractor import propose_actions_from_evidence +from memorymaster.bridges.atlas_claim_extractor import extract_atlas_claims_from_evidence +from memorymaster.bridges.media_processing import ( + EvidenceResult, + MockTranscriptionProvider, + process_transcription, +) +from memorymaster.bridges.media_providers import get_transcription_provider +from memorymaster.core.service import MemoryService +from memorymaster.surfaces.cli import main + + +@pytest.fixture() +def service(tmp_path: Path) -> MemoryService: + svc = MemoryService(tmp_path / "atlas.db", workspace_root=tmp_path) + svc.init_db() + return svc + + +def _audio_item(service: MemoryService, tmp_path: Path, *, item_id: str = "voice-1"): + media_path = tmp_path / f"{item_id}.ogg" + media_path.write_bytes(b"test audio") + source = service.upsert_external_source(source_type="test", display_name="test") + item = service.upsert_source_item( + source_id=source.id, + source_item_id=item_id, + item_type="audio", + payload_json={"media_path": str(media_path)}, + ) + return item, media_path + + +def _enable_test_media(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("MEMORYMASTER_MEDIA_MODE", "test") + monkeypatch.setenv("MEMORYMASTER_ALLOW_SYNTHETIC_MEDIA", "1") + + +def test_mock_media_fails_closed_without_explicit_nonproduction_opt_in( + service: MemoryService, + tmp_path: Path, +) -> None: + item, media_path = _audio_item(service, tmp_path) + provider = MockTranscriptionProvider({str(media_path): "Please call the client tomorrow"}) + + outcome = process_transcription(service, item.id, provider) + + assert outcome.created is False + assert outcome.evidence is None + assert outcome.error is not None + assert "MEMORYMASTER_MEDIA_MODE" in outcome.error + assert service.list_evidence_items(source_item_id=item.id) == [] + + +def test_mock_media_is_conspicuously_enabled_for_tests( + service: MemoryService, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _enable_test_media(monkeypatch) + item, media_path = _audio_item(service, tmp_path) + provider = MockTranscriptionProvider({str(media_path): "Please call the client tomorrow"}) + + outcome = process_transcription(service, item.id, provider) + + assert outcome.created is True + assert outcome.evidence is not None + assert outcome.evidence.provider == "mock-transcription" + + +def test_synthetic_evidence_cannot_create_claims_or_actions( + service: MemoryService, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _enable_test_media(monkeypatch) + item, media_path = _audio_item(service, tmp_path) + outcome = process_transcription( + service, + item.id, + MockTranscriptionProvider({str(media_path): "Please send the installation quote tomorrow"}), + ) + assert outcome.evidence is not None + + claims = extract_atlas_claims_from_evidence(service, scope="project:test") + actions = propose_actions_from_evidence(service) + + assert claims.ingested == 0 + assert actions.created == 0 + + +def test_synthetic_evidence_never_reaches_llm( + service: MemoryService, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _enable_test_media(monkeypatch) + item, media_path = _audio_item(service, tmp_path) + outcome = process_transcription( + service, + item.id, + MockTranscriptionProvider({str(media_path): "Remember that Acme chose SQLite"}), + ) + assert outcome.evidence is not None + + def _unexpected_llm_call(*args, **kwargs): + raise AssertionError("synthetic evidence reached the LLM") + + monkeypatch.setattr(atlas_llm_extractor, "_call_llm_safe", _unexpected_llm_call) + result = atlas_llm_extractor.extract_atlas_claims_llm( + service, + scope="project:test", + dry_run=True, + ) + + assert result.ingested == 0 + assert result.emitted == 0 + + +def test_approved_action_backed_by_synthetic_evidence_is_not_exported( + service: MemoryService, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _enable_test_media(monkeypatch) + item, media_path = _audio_item(service, tmp_path) + outcome = process_transcription( + service, + item.id, + MockTranscriptionProvider({str(media_path): "Please pay the bill"}), + ) + assert outcome.evidence is not None + proposal = service.create_action_proposal( + proposal_type="task", + title="Pay the bill", + source_item_id=item.id, + evidence_item_id=outcome.evidence.id, + destination="super-productivity", + confidence=0.9, + idempotency_key="synthetic-export-guard", + ) + service.update_action_proposal_status(proposal.id, status="approved") + output_path = tmp_path / "actions.json" + + result = export_approved_actions(service, output_path) + + assert result.exported == 0 + assert json.loads(output_path.read_text(encoding="utf-8"))["tasks"] == [] + assert [row.id for row in service.list_action_proposals(status="approved")] == [proposal.id] + + +def test_real_provider_reprocesses_when_only_mock_evidence_exists( + service: MemoryService, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + _enable_test_media(monkeypatch) + item, media_path = _audio_item(service, tmp_path) + mock = process_transcription( + service, + item.id, + MockTranscriptionProvider({str(media_path): "Mock transcript"}), + ) + assert mock.created is True + monkeypatch.delenv("MEMORYMASTER_ALLOW_SYNTHETIC_MEDIA") + monkeypatch.setenv("MEMORYMASTER_MEDIA_MODE", "production") + + class LocalRealProvider: + provider_name = "local-real-test" + + def transcribe(self, path: str) -> EvidenceResult: + return EvidenceResult( + evidence_type="transcript", + text="Authentic transcript", + provider=self.provider_name, + media_path=path, + confidence=0.8, + ) + + real = process_transcription(service, item.id, LocalRealProvider()) + + assert real.created is True + assert real.evidence is not None + assert real.evidence.provider == "local-real-test" + assert len(service.list_evidence_items(source_item_id=item.id, evidence_type="transcript")) == 2 + + +def test_provider_factory_rejects_mock_in_production() -> None: + with pytest.raises(ValueError, match="MEMORYMASTER_MEDIA_MODE"): + get_transcription_provider("mock") + + +def test_provider_factory_requires_real_provider_readiness(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("OPENAI_API_KEY", raising=False) + + with pytest.raises(ValueError, match="OPENAI_API_KEY"): + get_transcription_provider("openai") + + +def test_cli_requires_explicit_provider(tmp_path: Path) -> None: + db_path = tmp_path / "atlas.db" + service = MemoryService(db_path, workspace_root=tmp_path) + service.init_db() + item, _ = _audio_item(service, tmp_path) + + with pytest.raises(SystemExit) as exc_info: + main([ + "--db", + str(db_path), + "--workspace", + str(tmp_path), + "transcribe-source-item", + "--source-item-id", + str(item.id), + ]) + + assert exc_info.value.code != 0 + assert service.list_evidence_items(source_item_id=item.id) == [] diff --git a/tests/test_media_processing.py b/tests/test_media_processing.py index 5f8b86f1..c88c887e 100644 --- a/tests/test_media_processing.py +++ b/tests/test_media_processing.py @@ -27,7 +27,12 @@ def service(tmp_path: Path) -> MemoryService: return svc -def test_mock_transcription_creates_transcript_evidence(service: MemoryService) -> None: +def test_mock_transcription_creates_transcript_evidence( + service: MemoryService, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MEMORYMASTER_MEDIA_MODE", "test") + monkeypatch.setenv("MEMORYMASTER_ALLOW_SYNTHETIC_MEDIA", "1") source = service.upsert_external_source(source_type="whatsapp", display_name="primary") item = service.upsert_source_item( source_id=source.id, @@ -49,7 +54,12 @@ def test_mock_transcription_creates_transcript_evidence(service: MemoryService) assert second.evidence.id == first.evidence.id -def test_mock_ocr_creates_ocr_evidence(service: MemoryService) -> None: +def test_mock_ocr_creates_ocr_evidence( + service: MemoryService, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("MEMORYMASTER_MEDIA_MODE", "test") + monkeypatch.setenv("MEMORYMASTER_ALLOW_SYNTHETIC_MEDIA", "1") source = service.upsert_external_source(source_type="whatsapp", display_name="primary") item = service.upsert_source_item( source_id=source.id, From fb78bea95dbc8857cf34b9410b2a49b70778cc1e Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Mon, 13 Jul 2026 22:46:09 -0300 Subject: [PATCH 36/55] fix: converge governed phase 2 core --- .planning/REMAINING-PHASES-V1-BUDGET.md | 30 ++++- ...EMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md | 11 +- .planning/audit-remediation-ledger.md | 18 +-- .../audit-delta.md | 92 ++++++++++++++ .../pr-draft.md | 49 ++++++++ memorymaster/core/capture_control.py | 8 +- memorymaster/govern/verbatim_cleanup.py | 7 +- memorymaster/recall/context_hook.py | 78 +++++++++--- .../surfaces/cli_handlers_curation.py | 2 +- tests/test_cli_handlers_extra.py | 7 +- tests/test_context_hook.py | 18 +++ tests/test_context_hook_extra.py | 11 +- tests/test_eval_harness.py | 7 +- tests/test_fresh_eyes_fixes.py | 7 +- tests/test_graph_harvest.py | 8 +- tests/test_guarded_fuzzy_resolver.py | 81 +++++-------- tests/test_mcp_tenant_scope_boundary.py | 7 ++ tests/test_prompt_recall_governance.py | 113 ++++++++++++++++++ tests/test_query_expansion.py | 10 +- tests/test_recall_entity_fanout.py | 5 +- tests/test_recall_precision_at_5.py | 7 +- tests/test_ro_recall.py | 7 +- tests/test_scope_boost.py | 15 ++- tests/test_two_pass_recall.py | 23 ++-- tests/test_wiki_binding.py | 7 +- 25 files changed, 497 insertions(+), 131 deletions(-) create mode 100644 .planning/audits/2026-07-13-phase2-budget-delta/audit-delta.md create mode 100644 .planning/audits/2026-07-13-phase2-budget-delta/pr-draft.md create mode 100644 tests/test_prompt_recall_governance.py diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index fb991cc8..36f58c8b 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -260,5 +260,31 @@ P2-F implementation and focused gate completed on 2026-07-13 from P2-E commit - No paid or production provider was called and no live/external data or configuration was mutated. -The next separately authorized package is P2-Z Phase 2 convergence. Do not -begin it from the P2-F goal. +P2-Z Phase 2 convergence completed on 2026-07-13: + +- The convergence review found and witnessed a High prompt-recall bypass: + downstream graph/entity/two-pass streams returned allowed, candidate, + foreign-scope, and private claims together. One immutable trusted planner + policy now filters both initial lexical results and every optional stream. +- Capture-ledger and verbatim retention readers now use the canonical SQLite + connection helpers. P2-C/P2-D compatibility fixtures were converged without + restoring lazy DDL, candidate trust, invalid entity foreign keys, or + read-path writes. +- Targeted matrix: 226 passed, 2 failed; exact correction: 3 passed. The single + full non-ML completion produced 3,964 passed, 28 failed, 10 errors, 70 + skipped, 95 deselected, and 3 xfailed in 915.59s. The bounded correction + matrix produced 287 passed/12 failed, its remaining cluster 71 passed/1 + failed, and the final exact scope correction passed 2 tests. This is + compositional evidence; no second full-suite pass is claimed. +- Final boundary: 4,171 tests collected; required Qdrant ML slice 34 passed; + project/changed-file Ruff and `git diff --check` passed. +- Disposable two-role Postgres and authenticated/TLS Qdrant were unavailable; + no DSNs, safe opt-in, Qdrant URL, API key, or CA were configured. Their + existing `BLOCKED-EXTERNAL` entries remain authoritative. +- Audit delta: `.planning/audits/2026-07-13-phase2-budget-delta/audit-delta.md`. + Draft PR: `.planning/audits/2026-07-13-phase2-budget-delta/pr-draft.md`. +- Atomic package commit: the conventional Phase 2 convergence commit + containing this scheduler evidence. + +The next package is R3.1 embedding and reconciliation efficiency. The active +roadmap goal authorizes continuing without a new goal. diff --git a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md index 469c0f33..0eabe4fc 100644 --- a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md +++ b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md @@ -88,11 +88,12 @@ Phase 0 must replace qualitative capacity goals with recorded numeric gates in t - [x] R1.4 — complete sensitivity/write gateway and legacy inventory (repository complete; live cleanup/Qdrant inventory blocked externally) - [x] R1.5 — secure deployment and supply-chain defaults (repository complete; rotation, advisory/image/runtime evidence blocked externally) - [x] Phase 1 targeted budget audit delta (V3 scope; Phases 2-4 remain open) -- [ ] R2.1 — unified RetrievalPlanner and governed Qdrant reintegration -- [ ] R2.2 — lifecycle authority and read-only recall -- [ ] R2.3 — unified entity model -- [ ] R2.4 — quiet capture, budgets, retention, and backlog control -- [ ] R2.5 — remove fake evidence defaults +- [x] R2.1 — unified RetrievalPlanner and governed Qdrant reintegration +- [x] R2.2 — lifecycle authority and read-only recall +- [x] R2.3 — unified entity model +- [x] R2.4 — quiet capture, budgets, retention, and backlog control (repository scope; legal review and account-wide cost convergence continue in R3) +- [x] R2.5 — remove fake evidence defaults +- [x] Phase 2 targeted convergence and audit delta - [ ] R3.1 — embedding and reconciliation efficiency - [ ] R3.2 — query and storage efficiency - [ ] R3.3 — truthful setup profiles diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 06a0c5ac..2e16b478 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -17,10 +17,10 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-OPS-01 | DevOps | Critical / EXPLOITABLE-LOW-EFFORT / H4 | Postgres Compose publishes a fixed credential | R1.5 | BLOCKED-EXTERNAL | Repository defaults are fail-closed and private (`a858419`); Compose rejects missing inputs and renders with synthetic required inputs. Historical credential rotation/recreation and a real network probe require operator action. | | MM-SEC-03 | Security | High / EXPLOITABLE-NOW | Persisted metadata/provenance fields bypass sensitivity scanning | R1.4 | BLOCKED-EXTERNAL | Repository gateways and the aggregate-only inventory are complete (`a3e3824`, `702b59d`, `8d80abb`); focused final gate: 46 passed, 1 skipped. The authorized live read-only inventory found legacy sensitive/unscannable material, while Qdrant was unavailable; cleanup/redaction and product-data mutation remain forbidden without approval. | | MM-SEC-04 | Security | High / EXPLOITABLE-NOW | Steward, compact-summary, verbatim, and integration writes bypass one gateway | R1.4 | RESOLVED | Durable auxiliary writers now use the shared sensitivity envelope and adversarial persistence matrices; the final observed compatibility failures were corrected and the exact 10-test failure set passed. Roll back by reverting the R1.4 commits while keeping affected ingestion disabled. | -| MM-ARCH-01 | Architecture/Database | High / BAD-PRACTICE | Entity registry and graph own incompatible `entities` schemas | R2.3 | OPEN | Normal init-to-graph MCP integration and FK check | -| MM-ARCH-02 | Architecture/Integrity | High / BAD-PRACTICE | MCP/hooks/CLI/Qdrant use contradictory retrieval planners and trust defaults | R2.1 | RESOLVED | P2-A/P2-B route governed lexical and gated semantic recall through one immutable planner, confirmed-only trusted defaults, explicit exploratory expansion, cross-surface ID parity, and authoritative Qdrant rehydration. Focused P2-A gate: 141 passed, 1 expected xfail; P2-B fake-backed gate: 91 passed. | -| MM-LIFE-01 | Database/Reliability | High / BAD-PRACTICE | Scheduled archival uses raw SQL and bypasses lifecycle/vector evidence | R2.2 | OPEN | Version/event/timestamp/outbox assertions through scheduled path | -| MM-REL-02 | Reliability/Performance | Medium / BAD-PRACTICE | MCP reads take write locks and context detail modes retrieve twice | R2.2 | OPEN | Zero query write lock; one aggregated telemetry envelope | +| MM-ARCH-01 | Architecture/Database | High / BAD-PRACTICE | Entity registry and graph own incompatible `entities` schemas | R2.3 | RESOLVED | P2-D (`4fb1725`) selects the integer-ID registry as authority, applies immutable migration 0013, removes read-path DDL, and exposes explicit readiness errors. Focused gate: 116 passed; schema-version-13 probe found all four canonical tables and zero FK violations. | +| MM-ARCH-02 | Architecture/Integrity | High / BAD-PRACTICE | MCP/hooks/CLI/Qdrant use contradictory retrieval planners and trust defaults | R2.1 | RESOLVED | P2-A/P2-B established the planner. P2-Z then witnessed a downstream prompt-stream bypass returning allowed, candidate, foreign, and private IDs together; the convergence correction applies the same trusted status/scope/visibility/sensitivity policy before and after all lexical, entity, graph, two-pass, closets, and verbatim streams. Adversarial RED and compositional correction evidence are in the Phase 2 delta. | +| MM-LIFE-01 | Database/Reliability | High / BAD-PRACTICE | Scheduled archival uses raw SQL and bypasses lifecycle/vector evidence | R2.2 | RESOLVED | P2-C (`732cb68`) routes scheduled archival through canonical optimistic transitions with events, timestamps, cache invalidation, and replayable vector deletion; focused gate: 40 passed. | +| MM-REL-02 | Reliability/Performance | Medium / BAD-PRACTICE | MCP reads take write locks and context detail modes retrieve twice | R2.2 | RESOLVED | P2-C makes top-level recall query-only, removes duplicate detail retrieval, and emits at most one sanitized aggregate telemetry envelope; read-only and telemetry contracts passed the focused/convergence matrices. | | MM-OPS-02 | DevOps/Demo | High / BAD-PRACTICE | Docker/Helm publish HTTP on a stdio MCP process and use an invalid healthcheck | R3.4 | OPEN | Built-container readiness and MCP handshake | | MM-OPS-03 | DevOps/Maintainability | High / BAD-PRACTICE | A release tag can publish without a blocking verified test artifact | R4.4 | OPEN | Deliberately failing release candidate cannot publish | | MM-OPS-04 | Security/DevOps | High / EXPLOITABLE-LOW-EFFORT | Qdrant/Ollama ports are broadly exposed; images are mutable | R1.5,R3.4 | IN-PROGRESS | Phase 1 defaults are repository-complete (`a858419`, `9b3e16c`, `b71e18f`): loopback-only ports, required immutable digests, authenticated TLS Qdrant, redirect-safe credentials, and fail-closed supply-chain policy. Real image scans/runtime health remain `BLOCKED-EXTERNAL`; R3.4 service-entrypoint work remains backlog. | @@ -31,12 +31,12 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-UX-01 | UX/Maintainability | High / BAD-PRACTICE | Setup reports success without verifying the requested memory loop | R3.3 | OPEN | Component-level profile verification and nonzero failures | | MM-UX-02 | UX/Reliability | High / BAD-PRACTICE | Dashboard removes review rows before mutation success | R4.3 | OPEN | Browser failure/retry evidence; row remains on rejected POST | | MM-UX-03 | UX | Medium / BAD-PRACTICE | Dashboard hides failures and has labeling, contrast, responsive, and evidence-discovery gaps | R4.3 | OPEN | Browser/a11y/mobile acceptance suite | -| MM-PRIV-01 | Compliance | High / BAD-PRACTICE | Automatic transcript capture/remote processing lacks an explicit consent boundary | R2.4,R3.5 | OPEN | Quiet default, explicit processor/capture choices | +| MM-PRIV-01 | Compliance | High / BAD-PRACTICE | Automatic transcript capture/remote processing lacks an explicit consent boundary | R2.4,R3.5 | BLOCKED-EXTERNAL | P2-E repository controls are quiet by default and require explicit maximum-capture/provider choices. Product-owner/legal approval for intended use, jurisdictions, processor disclosures, and retention commitments remains external. WARNING LEGAL REVIEW REQUIRED; no compliance-pass or risk-acceptance claim is made. | | MM-PRIV-02 | Compliance | High / BAD-PRACTICE | No complete export, erasure, or retention workflow across copies | R3.5 | OPEN | Privacy export/erase dry-run manifest across all stores | -| MM-COST-01 | Cost | High / EXPLOITABLE-NOW | Default Stop-hook LLM calls have no finite persisted budget | R2.4 | OPEN | Restart-safe global/provider/session caps | -| MM-COST-02 | Cost | High / EXPLOITABLE-NOW | Verbatim capture reprocesses sessions and retains data indefinitely | R2.4 | OPEN | Incremental cursor plus frozen retention envelope | -| MM-COST-03 | Cost/Reliability | Medium / BAD-PRACTICE | Cost and intake controls are fragmented/process-local | R2.4,R3.2 | OPEN | Atomic durable ledger and multi-process tests | -| MM-DEMO-01 | Demo/Integrity | High / EXPLOITABLE-NOW | Atlas defaults persist fabricated mock evidence at high confidence | R2.5 | OPEN | Default command rejects; mock rows cannot feed claims/actions | +| MM-COST-01 | Cost | High / EXPLOITABLE-NOW | Default Stop-hook LLM calls have no finite persisted budget | R2.4 | RESOLVED | P2-E (`a1d5477`) adds restart-safe atomic global/provider/session reservations and persisted usage; maximum default candidate inflow is 600/day against the 688/day ceiling. | +| MM-COST-02 | Cost | High / EXPLOITABLE-NOW | Verbatim capture reprocesses sessions and retains data indefinitely | R2.4 | RESOLVED | P2-E persists complete-line cursors, processes appended content only, and enforces dry-run 30-day/512-MiB/75,000-session retention bounds. | +| MM-COST-03 | Cost/Reliability | Medium / BAD-PRACTICE | Cost and intake controls are fragmented/process-local | R2.4,R3.2 | IN-PROGRESS | Capture-hook budgets/cursors are durable after P2-E, but core LLM, embedding, provider, and MCP quota accounting is not yet one account-wide durable ledger. Continue in R3.2. | +| MM-DEMO-01 | Demo/Integrity | High / EXPLOITABLE-NOW | Atlas defaults persist fabricated mock evidence at high confidence | R2.5 | RESOLVED | P2-F (`c66e240`) requires a real configured production provider, confines synthetic providers to explicit dev/test opt-in, excludes synthetic evidence from claims/prompts/actions/citations/exports, and fails configuration nonzero. Focused gate: 111 passed. | | MM-MAINT-01 | Maintainability/Integrity | Medium / BAD-PRACTICE | Versions, tool counts, roadmaps, and install claims drift | R4.4 | OPEN | Generated single-source values checked by CI | | MM-MAINT-02 | Architecture/Maintainability | Medium / BAD-PRACTICE | Oversized facades, hardwired extensions, dead plugin seam, and lingering shims | R4.1-R4.2 | OPEN | Measured boundary/size budgets and supported shim policy | | MM-REL-03 | Reliability | Medium / BAD-PRACTICE | Media retry rows have no expired-lease recovery | R3.5 | OPEN | Worker-death lease reclaim test | diff --git a/.planning/audits/2026-07-13-phase2-budget-delta/audit-delta.md b/.planning/audits/2026-07-13-phase2-budget-delta/audit-delta.md new file mode 100644 index 00000000..ea4f9ca4 --- /dev/null +++ b/.planning/audits/2026-07-13-phase2-budget-delta/audit-delta.md @@ -0,0 +1,92 @@ +# MemoryMaster Phase 2 Budget Audit Delta + +**Date:** 2026-07-13 + +**Baseline:** `.planning/audits/2026-07-10-baseline/audit-report.md` + +**Scheduler:** `.planning/REMAINING-PHASES-V1-BUDGET.md` + +**Scope:** MM-SEC-02, MM-ARCH-01, MM-ARCH-02, MM-LIFE-01, MM-REL-02, +MM-PRIV-01, MM-COST-01, MM-COST-02, MM-COST-03, and MM-DEMO-01 only. + +## Verdict + +Phase 2 repository convergence is complete for its scheduled scope. No +unresolved reproducible Critical/High regression introduced by the Phase 2 +branch remains. MM-COST-03 remains an explicit Medium R3.2 backlog item. +MM-SEC-02 and MM-PRIV-01 remain blocked on external runtime and legal/product +evidence. This was a targeted Phase 2 reconciliation, not a 13-domain reaudit. + +## Finding delta + +| Finding | Disposition | Phase 2 evidence | +|---|---|---| +| MM-SEC-02 | `BLOCKED-EXTERNAL` | P2-B exposes only ID/hash/score candidates, rehydrates from the authoritative store, rejects policy/hash/lifecycle failures, and provides exact reconciliation plus a bounded metadata-only outbox. Semantic reads stay default-off; authenticated/TLS Qdrant parity is unavailable. | +| MM-ARCH-01 | `RESOLVED` | P2-D establishes the integer-ID registry, immutable migration 0013, read-only readiness failures, backend schema parity, MCP integration, and zero-FK validation. | +| MM-ARCH-02 | `RESOLVED` | P2-A/P2-B unify retrieval. P2-Z witnessed a downstream prompt-stream bypass returning allowed, candidate, foreign, and private IDs together, then applied the same trusted planner policy before and after every optional stream. | +| MM-LIFE-01 | `RESOLVED` | P2-C routes scheduled archival through canonical optimistic transitions, events, timestamps, cache invalidation, and replayable vector deletion. | +| MM-REL-02 | `RESOLVED` | P2-C makes top-level recall query-only, removes duplicate detail retrieval, and emits one aggregate telemetry envelope. | +| MM-PRIV-01 | `BLOCKED-EXTERNAL` | Capture is quiet by default and maximum capture/providers require explicit configuration. Product-owner/legal decisions on use, processors, jurisdictions, and retention remain external. **WARNING LEGAL REVIEW REQUIRED; no compliance-pass or risk acceptance is claimed.** | +| MM-COST-01 | `RESOLVED` | P2-E persists atomic restart-safe global/provider/session reservations and usage; default candidate inflow is capped at 600/day. | +| MM-COST-02 | `RESOLVED` | P2-E persists complete-line cursors and finite 30-day/512-MiB/75,000-session dry-run retention bounds. | +| MM-COST-03 | `IN-PROGRESS` | Capture-hook accounting is durable, but core LLM, embedding, provider, and MCP quotas are not yet one account-wide ledger. Continue in R3.2. | +| MM-DEMO-01 | `RESOLVED` | P2-F requires a real production provider and prevents explicitly opted-in synthetic development evidence from reaching governed truth, prompts, actions, citations, or exports. | + +## Implemented evidence + +- `7026711` — immutable governed retrieval planner. +- `3e97e5b` — governed Qdrant reintegration and replayable outbox. +- `732cb68` — lifecycle authority and read-only recall. +- `4fb1725` — entity schema convergence through migration 0013. +- `a1d5477` — quiet finite capture, budgets, cursors, and retention. +- `c66e240` — fail-closed authentic media evidence defaults. +- The conventional P2-Z commit containing this delta — prompt-stream + governance correction, compatibility convergence, and exact boundary proof. + +## Verification boundary + +| Gate | Exact result | +|---|---| +| Targeted Phase 2 matrix | 226 passed, 2 failed; exact two-failure correction: 3 passed. | +| Full non-ML, single completion | 3,964 passed, 28 failed, 10 errors, 70 skipped, 95 deselected, 3 xfailed, 1 warning in 915.59s. | +| Bounded correction matrix | 287 passed, 12 failed in 31.99s. | +| Remaining-cluster verification | 71 passed, 1 failed in 2.82s. | +| Final exact scope correction | 2 passed. | +| Interpretation | Compositional convergence evidence. The 15-minute full suite was not rerun and no second clean full-suite invocation is claimed. | +| Collection | 4,171 tests collected. | +| Required Qdrant ML | 34 passed. | +| Ruff | Project and changed-file Ruff clean after one exact unused-import correction. | +| Diff integrity | `git diff --check` clean. | +| Disposable services | No Postgres DSNs/safe opt-in and no Qdrant URL/API key/CA were configured. | + +## External blockers + +- Disposable authenticated/TLS Qdrant parity remains required before enabling + governed semantic production reads. +- Disposable two-role Postgres parity remains required; no product database was + contacted or mutated. +- The product owner and legal reviewer must approve the privacy, processor, and + retention statement before compliance claims. + +See `external-actions-required.md` for owners and evidence requirements. + +## Backlog boundary + +- MM-COST-03 continues in R3.2 for account-wide durable quota convergence. +- R3.1-R3.5 cover performance, setup truth, service readiness, recovery, + observability, audit, and privacy operations. +- R4.1-R4.4 cover extensions, decomposition, governance UX, accessibility, and + generated release truth. + +## Rollback + +- Disable semantic Qdrant reads first and fall back to authoritative lexical + retrieval. +- Return capture to quiet/no-capture; never restore unlimited per-stop capture. +- Disable media enrichment instead of restoring synthetic production evidence. +- Revert package commits in reverse order only in a disposable/staged + environment. +- Never edit or reverse migration 0013 in place; use a verified backup or a + documented forward repair. +- Preserve metadata-only Qdrant outbox records until reconciliation is safely + disabled or replayed. diff --git a/.planning/audits/2026-07-13-phase2-budget-delta/pr-draft.md b/.planning/audits/2026-07-13-phase2-budget-delta/pr-draft.md new file mode 100644 index 00000000..ba2add23 --- /dev/null +++ b/.planning/audits/2026-07-13-phase2-budget-delta/pr-draft.md @@ -0,0 +1,49 @@ +# Draft PR — Governed MemoryMaster Phase 2 Core + +## Suggested title + +`feat: converge governed MemoryMaster Phase 2 core` + +## Summary + +- Unifies trusted retrieval across service, MCP, prompt hooks, and gated + Qdrant candidate discovery. +- Rehydrates semantic candidates from the authoritative store and reconciles + exact IDs/content hashes through a bounded replayable outbox. +- Routes archival through canonical lifecycle authority and makes top-level + recall query-only with aggregate telemetry. +- Converges entity/graph schemas through immutable migration 0013 and explicit + readiness checks. +- Makes capture quiet, finite, replay-safe, budgeted, and retention-bounded. +- Prevents synthetic media from becoming production evidence. +- Corrects a convergence-discovered prompt/entity/graph policy bypass. + +## Verification + +- Targeted Phase 2 matrix: 226 passed; its two failures passed in the exact + 3-test correction slice. +- Single full non-ML invocation: 3,964 passed, 28 failed, 10 errors, 70 skipped, + 95 deselected, and 3 xfailed in 915.59s. +- Bounded reconciliation: 287 passed/12 failed, then 71 passed/1 failed, then + the final two affected tests passed. +- Collection: 4,171 tests. +- Required Qdrant ML: 34 passed. +- Project/changed Ruff and `git diff --check`: clean. + +The full-suite failures were reconciled compositionally under the budget +scheduler. This PR does not claim a second clean full-suite invocation. + +## External blockers + +- Real authenticated/TLS Qdrant parity. +- Disposable two-role Postgres evidence. +- Approved privacy/data-processing and retention statement. +- Semantic/team production profiles remain disabled until their required + evidence exists. + +## Rollback + +Disable semantic reads, capture, and media enrichment first. Revert package +commits in reverse order only in a disposable/staged environment. Do not edit +immutable migration 0013; restore a verified backup or use a forward repair. +No production rollback command is required because this branch is not deployed. diff --git a/memorymaster/core/capture_control.py b/memorymaster/core/capture_control.py index e41621ec..07833593 100644 --- a/memorymaster/core/capture_control.py +++ b/memorymaster/core/capture_control.py @@ -10,6 +10,8 @@ from datetime import datetime, timezone from pathlib import Path +from memorymaster.stores._storage_shared import open_conn + DEFAULT_GLOBAL_DAILY_CALLS = 200 DEFAULT_PROVIDER_DAILY_CALLS = 100 @@ -107,11 +109,7 @@ def __init__(self, db_path: str | Path, *, limits: CaptureLimits | None = None) ) def _connect(self) -> sqlite3.Connection: - conn = sqlite3.connect(self.db_path, timeout=15) - conn.row_factory = sqlite3.Row - conn.execute("PRAGMA journal_mode=WAL") - conn.execute("PRAGMA busy_timeout=15000") - return conn + return open_conn(self.db_path, busy_ms=15_000) @staticmethod def _hash(value: str) -> str: diff --git a/memorymaster/govern/verbatim_cleanup.py b/memorymaster/govern/verbatim_cleanup.py index 59ccbcfc..7f23b645 100644 --- a/memorymaster/govern/verbatim_cleanup.py +++ b/memorymaster/govern/verbatim_cleanup.py @@ -22,10 +22,9 @@ import logging import sqlite3 from datetime import datetime, timedelta, timezone -from pathlib import Path from typing import Any -from memorymaster.stores._storage_shared import open_conn +from memorymaster.stores._storage_shared import connect_ro, open_conn logger = logging.getLogger(__name__) @@ -213,9 +212,7 @@ def cleanup( def _read_retention_rows(db_path: str, scan_limit: int) -> tuple[bool, list[dict[str, Any]], bool]: - uri = f"file:{Path(db_path).resolve().as_posix()}?mode=ro" - conn = sqlite3.connect(uri, uri=True) - conn.row_factory = sqlite3.Row + conn = connect_ro(db_path) try: if not _verbatim_present(conn): return False, [], False diff --git a/memorymaster/recall/context_hook.py b/memorymaster/recall/context_hook.py index 25d31902..85ebd987 100644 --- a/memorymaster/recall/context_hook.py +++ b/memorymaster/recall/context_hook.py @@ -1150,6 +1150,61 @@ def recall( _emit_recall_latency(phase_ms, total_start) +def _prompt_recall_plan(query_text: str, *, limit: int): + """Build the immutable trusted policy shared by every prompt stream.""" + from memorymaster.recall.planner import RetrievalRequest, build_retrieval_plan + + scopes = [_current_scope(), "global"] + include_legacy = os.environ.get( + "MEMORYMASTER_QUERY_INCLUDE_LEGACY_PROJECT", "1" + ).strip().lower() not in {"0", "false", "no"} + if include_legacy: + scopes.append("project") + return build_retrieval_plan( + RetrievalRequest( + query_text=query_text, + limit=limit, + trust_mode="trusted", + retrieval_mode="legacy", + scope_allowlist=tuple(dict.fromkeys(scopes)), + ) + ) + + +def _prompt_claim_allowed(claim, plan) -> bool: + """Fail closed for claims introduced by any prompt-recall stream.""" + from memorymaster.core.security import is_sensitive_claim + + if claim is None or getattr(claim, "status", "") not in set(plan.statuses): + return False + scopes = plan.scope_allowlist + if scopes is not None and (getattr(claim, "scope", "") or "").strip() not in scopes: + return False + visibility = (getattr(claim, "visibility", "public") or "public").strip().lower() + return visibility == "public" and not is_sensitive_claim(claim) + + +def _filter_prompt_rows(rows: list[dict], plan) -> list[dict]: + return [row for row in rows if _prompt_claim_allowed(row.get("claim"), plan)] + + +def _governed_prompt_rows(svc, query_text: str, *, limit: int) -> list[dict]: + """Run prompt-hook lexical recall through the immutable trusted planner.""" + plan = _prompt_recall_plan(query_text, limit=limit) + statuses = set(plan.statuses) + rows = svc.query_rows( + query_text=plan.search_text, + limit=plan.limit, + retrieval_mode=plan.effective_mode, + include_stale="stale" in statuses, + include_conflicted="conflicted" in statuses, + include_candidates="candidate" in statuses, + scope_allowlist=list(plan.scope_allowlist or ()), + record_accesses=False, + ) + return _filter_prompt_rows(rows, plan) + + def query_for_task( task_description: str, project_scope: str, @@ -1298,6 +1353,7 @@ def _recall_impl( # Recall is always query-only. Access/feedback signals are emitted once, # after top-level ranking and budget selection, through the RO spool path. svc = MemoryService(db_target=db, workspace_root=Path.cwd(), read_only=True) + prompt_plan = _prompt_recall_plan(query, limit=100) # Pre-extract salient tokens before hitting FTS5. Passing the full # prompt verbatim AND-joins every token in FTS5 and rejects nearly all @@ -1342,14 +1398,7 @@ def _recall_impl( # Fan out: top token first (highest IDF), then widen by OR. per_token_limit = max(3, 8 // max(1, len(token_list))) * _lr_overfetch for tok in token_list: - batch = svc.query_rows( - query_text=tok, - limit=per_token_limit, - retrieval_mode="legacy", - include_candidates=True, - scope_allowlist=None, - record_accesses=False, - ) + batch = _governed_prompt_rows(svc, tok, limit=per_token_limit) for row in batch: claim = row.get("claim") cid = getattr(claim, "id", None) @@ -1362,14 +1411,7 @@ def _recall_impl( if not rows: # Fallback to raw prompt — preserves the old behaviour. - rows = svc.query_rows( - query_text=query, - limit=_fts_cap, - retrieval_mode="legacy", - include_candidates=True, - scope_allowlist=None, - record_accesses=False, - ) + rows = _governed_prompt_rows(svc, query, limit=_fts_cap) for row in rows: claim = row.get("claim") cid = getattr(claim, "id", None) @@ -1657,6 +1699,10 @@ def _recall_impl( except Exception as exc: # noqa: BLE001 — defensive (claim 11907) logger.debug("graph stream skipped: %s", exc) + # Every optional stream is untrusted candidate discovery. Apply the same + # trusted status/scope/visibility/sensitivity policy after all streams so + # graph, entity, two-pass, closets, and verbatim cannot bypass P2-A. + rows = _filter_prompt_rows(rows, prompt_plan) if not rows: return "" diff --git a/memorymaster/surfaces/cli_handlers_curation.py b/memorymaster/surfaces/cli_handlers_curation.py index 822e9881..50696160 100644 --- a/memorymaster/surfaces/cli_handlers_curation.py +++ b/memorymaster/surfaces/cli_handlers_curation.py @@ -514,7 +514,7 @@ def _handle_entity_stats(args: argparse.Namespace, service, parser: argparse.Arg eg = EntityGraph(str(effective_db), read_only=True) stats = eg.get_stats() if args.json_output: - print(_json_envelope(stats)) + print(_json_envelope(stats, query_ms=0.0)) else: print(f"Entities: {stats['entities']}, Edges: {stats['edges']}, Claim links: {stats['claim_links']}") for t, c in stats.get('by_type', {}).items(): diff --git a/tests/test_cli_handlers_extra.py b/tests/test_cli_handlers_extra.py index f534942b..229bd284 100644 --- a/tests/test_cli_handlers_extra.py +++ b/tests/test_cli_handlers_extra.py @@ -186,7 +186,7 @@ def test_wiki_breakdown_routes_and_emits_envelope(monkeypatch, capsys): def test_entity_stats_human_path_ok_json_path_raises(monkeypatch, capsys): class FakeEG: - def __init__(self, db): + def __init__(self, db, *, read_only=False): pass def ensure_tables(self): @@ -201,8 +201,9 @@ def get_stats(self): assert rc == 0 assert "Entities: 2" in capsys.readouterr().out - with pytest.raises(TypeError): - C._handle_entity_stats(_ns(json_output=True), _FakeService(), None, "db.sqlite") + assert C._handle_entity_stats( + _ns(json_output=True), _FakeService(), None, "db.sqlite" + ) == 0 def test_feedback_stats_human_path_ok_json_path_raises(monkeypatch, capsys): diff --git a/tests/test_context_hook.py b/tests/test_context_hook.py index 250ed3dd..66a04c54 100644 --- a/tests/test_context_hook.py +++ b/tests/test_context_hook.py @@ -76,6 +76,12 @@ def test_recall_with_results(self, mock_service_class: MagicMock) -> None: mock_claim = MagicMock() mock_claim.text = "working on memorymaster project" mock_claim.id = 1 + mock_claim.status = "confirmed" + mock_claim.scope = "project" + mock_claim.visibility = "public" + mock_claim.object_value = None + mock_claim.subject = None + mock_claim.predicate = None mock_service.query_rows.return_value = [ {"claim": mock_claim, "lexical_score": 0.8, "confidence_score": 0.9}, ] @@ -105,6 +111,12 @@ def test_recall_with_custom_budget(self, mock_service_class: MagicMock) -> None: mock_claim = MagicMock() mock_claim.text = "a" * 200 + mock_claim.status = "confirmed" + mock_claim.scope = "project" + mock_claim.visibility = "public" + mock_claim.object_value = None + mock_claim.subject = None + mock_claim.predicate = None mock_service.query_rows.return_value = [ {"claim": mock_claim, "lexical_score": 0.5, "confidence_score": 0.5}, ] @@ -121,6 +133,12 @@ def test_recall_sanitizes_non_ascii(self, mock_service_class: MagicMock) -> None mock_claim = MagicMock() mock_claim.text = "claim with \u00f1 and \u00e9 characters" + mock_claim.status = "confirmed" + mock_claim.scope = "project" + mock_claim.visibility = "public" + mock_claim.object_value = None + mock_claim.subject = None + mock_claim.predicate = None mock_service.query_rows.return_value = [ {"claim": mock_claim, "lexical_score": 0.5, "confidence_score": 0.5}, ] diff --git a/tests/test_context_hook_extra.py b/tests/test_context_hook_extra.py index 22547743..1503d458 100644 --- a/tests/test_context_hook_extra.py +++ b/tests/test_context_hook_extra.py @@ -423,26 +423,29 @@ class _C: def _seed(tmp_path): + from memorymaster.core.lifecycle import transition_claim from memorymaster.core.models import CitationInput from memorymaster.core.service import MemoryService db = tmp_path / "seeded.db" svc = MemoryService(db_target=str(db), workspace_root=tmp_path) svc.init_db() - svc.ingest( + first = svc.ingest( text="The user prefers PostgreSQL for production databases.", citations=[CitationInput(source="test")], claim_type="preference", - scope="project:test", + scope="project", confidence=0.9, ) - svc.ingest( + second = svc.ingest( text="The team decided to use Qdrant for vector search backends.", citations=[CitationInput(source="test")], claim_type="decision", - scope="project:test", + scope="project", confidence=0.8, ) + transition_claim(svc.store, first.id, "confirmed", "trusted recall fixture") + transition_claim(svc.store, second.id, "confirmed", "trusted recall fixture") return str(db) diff --git a/tests/test_eval_harness.py b/tests/test_eval_harness.py index 99c7c1ff..5580604f 100644 --- a/tests/test_eval_harness.py +++ b/tests/test_eval_harness.py @@ -102,18 +102,21 @@ def seeded_db(tmp_path_factory: pytest.TempPathFactory) -> Path: test wall time and are fully deterministic given identical inputs.""" tmp = tmp_path_factory.mktemp("eval_harness_db") db = tmp / "test.db" + from memorymaster.core.lifecycle import transition_claim + svc = MemoryService(db_target=str(db), workspace_root=tmp) svc.init_db() for text, subject in _SEED_CLAIMS: - svc.ingest( + claim = svc.ingest( text=text, citations=[CitationInput(source="test-harness")], subject=subject, claim_type="fact", - scope="project:memorymaster", + scope="project", confidence=0.7, source_agent="test", ) + transition_claim(svc.store, claim.id, "confirmed", "trusted eval fixture") return db diff --git a/tests/test_fresh_eyes_fixes.py b/tests/test_fresh_eyes_fixes.py index f18df45f..d1703fa6 100644 --- a/tests/test_fresh_eyes_fixes.py +++ b/tests/test_fresh_eyes_fixes.py @@ -166,7 +166,12 @@ def test_volunteer_default_gate_actually_gates(tmp_path): confidence=0.3, source_agent="t") ingest_claim(text="volunteerfixtoken strong fact", db=db, workspace=ws, confidence=0.9, source_agent="t") - res = volunteer_context(query="volunteerfixtoken", db=db, workspace=ws) + res = volunteer_context( + query="volunteerfixtoken", + db=db, + workspace=ws, + trust_mode="exploratory", + ) assert res["ok"] is True assert res["min_confidence"] >= 0.5, "default gate is open again — tool adds nothing" included_texts = json.dumps(res.get("claims", [])) + str(res.get("output", "")) diff --git a/tests/test_graph_harvest.py b/tests/test_graph_harvest.py index 7d6a0db3..8150d978 100644 --- a/tests/test_graph_harvest.py +++ b/tests/test_graph_harvest.py @@ -39,6 +39,7 @@ def _clear_recall_env(monkeypatch): def _seed(tmp_path): """Two FTS5-reachable claims. Returns (db_path, [seeded_claim_ids]).""" from memorymaster.core.models import CitationInput + from memorymaster.core.lifecycle import transition_claim from memorymaster.core.service import MemoryService db = tmp_path / "harvest.db" @@ -53,9 +54,10 @@ def _seed(tmp_path): text=text, citations=[CitationInput(source="test")], claim_type=ctype, - scope="project:test", + scope="project", confidence=conf, ) + transition_claim(svc.store, claim.id, "confirmed", "trusted recall fixture") ids.append(int(claim.id)) return str(db), ids @@ -65,6 +67,7 @@ def _ingest_offgraph_claim(db, text="A graph-only fact about Kubernetes scaling. can only enter the candidate pool via the graph harvest path. Returns id. """ from memorymaster.core.models import CitationInput + from memorymaster.core.lifecycle import transition_claim from memorymaster.core.service import MemoryService svc = MemoryService(db_target=db, workspace_root=os.path.dirname(db) or ".") @@ -72,9 +75,10 @@ def _ingest_offgraph_claim(db, text="A graph-only fact about Kubernetes scaling. text=text, citations=[CitationInput(source="test")], claim_type="fact", - scope="project:test", + scope="project", confidence=0.7, ) + transition_claim(svc.store, claim.id, "confirmed", "trusted recall fixture") return int(claim.id) diff --git a/tests/test_guarded_fuzzy_resolver.py b/tests/test_guarded_fuzzy_resolver.py index 9c792b34..ab434866 100644 --- a/tests/test_guarded_fuzzy_resolver.py +++ b/tests/test_guarded_fuzzy_resolver.py @@ -17,25 +17,21 @@ import sqlite3 import threading +from pathlib import Path from memorymaster.knowledge.entity_registry import ( - ensure_entity_schema, resolve_or_create, ) +from memorymaster.stores.storage import SQLiteStore _FLAG = "MEMORYMASTER_ENTITY_FUZZY_RESOLVE" -def _fresh_db() -> sqlite3.Connection: - """In-memory DB with the claims + entity tables wired up (mirrors the - fixture in test_entity_registry.py).""" - conn = sqlite3.connect(":memory:") - conn.execute( - "CREATE TABLE claims (id INTEGER PRIMARY KEY AUTOINCREMENT, " - "subject TEXT, entity_id INTEGER)" - ) - ensure_entity_schema(conn) - return conn +def _fresh_db(tmp_path: Path) -> sqlite3.Connection: + """Canonical initialized DB with the migrated entity registry.""" + store = SQLiteStore(tmp_path / "guarded-fuzzy.db") + store.init_db() + return store.connect() def _entity_count(conn: sqlite3.Connection) -> int: @@ -47,14 +43,14 @@ def _entity_count(conn: sqlite3.Connection) -> int: # --------------------------------------------------------------------------- # -def test_off_by_default_near_match_creates_two_entities(monkeypatch): +def test_off_by_default_near_match_creates_two_entities(tmp_path, monkeypatch): """WHY: the feature is recall-altering, so absent the env flag a near-miss subject MUST still become its own entity. If this regresses, every shipped DB silently starts merging entities on upgrade — unacceptable for a released package. """ monkeypatch.delenv(_FLAG, raising=False) - conn = _fresh_db() + conn = _fresh_db(tmp_path) id1 = resolve_or_create(conn, "MemoryMaster") id2 = resolve_or_create(conn, "memory-master") # near, but distinct norm form assert id1 > 0 and id2 > 0 @@ -67,13 +63,13 @@ def test_off_by_default_near_match_creates_two_entities(monkeypatch): # --------------------------------------------------------------------------- # -def test_unambiguous_fuzzy_match_returns_existing_entity(monkeypatch): +def test_unambiguous_fuzzy_match_returns_existing_entity(tmp_path, monkeypatch): """WHY: a single high-confidence candidate is exactly when fuzzy matching adds value — "memory master" should collapse onto the existing "MemoryMaster" entity instead of spawning a duplicate. """ monkeypatch.setenv(_FLAG, "1") - conn = _fresh_db() + conn = _fresh_db(tmp_path) canonical = resolve_or_create(conn, "MemoryMaster") assert canonical > 0 before = _entity_count(conn) @@ -83,13 +79,13 @@ def test_unambiguous_fuzzy_match_returns_existing_entity(monkeypatch): assert _entity_count(conn) == before, "no new entity should be created" -def test_unambiguous_match_records_new_alias_for_fast_path(monkeypatch): +def test_unambiguous_match_records_new_alias_for_fast_path(tmp_path, monkeypatch): """WHY: once a fuzzy match is accepted, the surface form should be recorded as an alias so the NEXT lookup is an exact hit (cheap) — and so the registry's alias graph reflects reality. """ monkeypatch.setenv(_FLAG, "1") - conn = _fresh_db() + conn = _fresh_db(tmp_path) canonical = resolve_or_create(conn, "MemoryMaster") resolve_or_create(conn, "memory master") @@ -106,7 +102,7 @@ def test_unambiguous_match_records_new_alias_for_fast_path(monkeypatch): # --------------------------------------------------------------------------- # -def test_ambiguous_match_refuses_and_creates_nothing(monkeypatch): +def test_ambiguous_match_refuses_and_creates_nothing(tmp_path, monkeypatch): """WHY: this is the whole reason the resolver is 'guarded'. When a subject is roughly equidistant from two existing entities, picking either one is a hallucinated merge. The guard must REFUSE: return 0 AND leave the DB @@ -116,7 +112,7 @@ def test_ambiguous_match_refuses_and_creates_nothing(monkeypatch): both above the accept threshold and within the ambiguity margin — and matches NEITHER exactly, so it genuinely exercises the fuzzy tie path. """ - conn = _fresh_db() + conn = _fresh_db(tmp_path) # Seed the two confusable entities with fuzzy OFF so they stay distinct # (turning it on first would itself collapse them — which is the bug we're # guarding against, just on the setup side). @@ -144,13 +140,13 @@ def test_ambiguous_match_refuses_and_creates_nothing(monkeypatch): # --------------------------------------------------------------------------- # -def test_below_threshold_creates_new_entity(monkeypatch): +def test_below_threshold_creates_new_entity(tmp_path, monkeypatch): """WHY: a guard that refuses too eagerly is as broken as one that merges too eagerly. A subject with no real similarity to anything must still get its own entity — the fuzzy path only intercepts CONFIDENT matches. """ monkeypatch.setenv(_FLAG, "1") - conn = _fresh_db() + conn = _fresh_db(tmp_path) resolve_or_create(conn, "Qdrant") resolve_or_create(conn, "SQLite") before = _entity_count(conn) @@ -165,35 +161,28 @@ def test_below_threshold_creates_new_entity(monkeypatch): # --------------------------------------------------------------------------- # -def test_fanout_unaffected_by_flag(monkeypatch): +def test_fanout_unaffected_by_flag(tmp_path, monkeypatch): """WHY: the fuzzy flag lives in the registry, but the recall fanout (``_entity_fanout_claim_ids``) resolves entities by EXACT alias only. The flag must not change fanout output — recall stays bit-identical whether or not an operator has opted into fuzzy *ingest* resolution. """ from memorymaster.recall.context_hook import _entity_fanout_claim_ids + from memorymaster.core.lifecycle import transition_claim + from memorymaster.core.models import CitationInput + from memorymaster.core.service import MemoryService # Build a DB with an entity + linked claim that the fanout can resolve. - conn = _fresh_db() - # claims table here needs the columns the fanout SQL reads. - conn.execute("DROP TABLE claims") - conn.execute( - "CREATE TABLE claims (id INTEGER PRIMARY KEY AUTOINCREMENT, " - "text TEXT, status TEXT DEFAULT 'confirmed', visibility TEXT DEFAULT 'public', " - "updated_at TEXT DEFAULT '2026-01-01', entity_id INTEGER)" - ) - eid = resolve_or_create(conn, "context_hook.py") - conn.execute( - "INSERT INTO claims (text, entity_id) VALUES ('about context_hook.py', ?)", - (eid,), + service = MemoryService(tmp_path / "fanout.db") + service.init_db() + claim = service.ingest( + "about context_hook.py", + [CitationInput(source="test://fanout")], + subject="context_hook.py", + scope="project", ) - conn.commit() - - class _Store: - def connect(self): - return conn - - store = _Store() + transition_claim(service.store, claim.id, "confirmed", "test fixture") + store = service.store prompt = "fix context_hook.py now" monkeypatch.delenv(_FLAG, raising=False) @@ -219,14 +208,8 @@ def test_concurrent_resolve_creates_single_entity(tmp_path, monkeypatch): """ monkeypatch.setenv(_FLAG, "1") db = tmp_path / "concurrent.sqlite" - # Seed schema once. - with sqlite3.connect(db) as seed: - seed.execute( - "CREATE TABLE claims (id INTEGER PRIMARY KEY AUTOINCREMENT, " - "subject TEXT, entity_id INTEGER)" - ) - ensure_entity_schema(seed) - seed.commit() + # Seed the full immutable migration chain once. + SQLiteStore(db).init_db() barrier = threading.Barrier(10) errors: list[Exception] = [] diff --git a/tests/test_mcp_tenant_scope_boundary.py b/tests/test_mcp_tenant_scope_boundary.py index 3b69f7a5..1bd5635d 100644 --- a/tests/test_mcp_tenant_scope_boundary.py +++ b/tests/test_mcp_tenant_scope_boundary.py @@ -6,6 +6,7 @@ import memorymaster.core.access_control as access_control import memorymaster.surfaces.mcp_server as mcp_server +from memorymaster.core.lifecycle import transition_claim from memorymaster.core.models import CitationInput from memorymaster.core.service import MemoryService @@ -36,6 +37,12 @@ def team_claims(tmp_path, monkeypatch): scope="project:alpha", source_agent="seed", ) + for service, claim in ( + (alpha, alpha_claim), + (alpha, beta_scope_claim), + (beta_tenant, beta_tenant_claim), + ): + transition_claim(service.store, claim.id, "confirmed", "trusted MCP fixture") access_control._agent_roles.clear() monkeypatch.setattr(access_control, "_loaded", True) diff --git a/tests/test_prompt_recall_governance.py b/tests/test_prompt_recall_governance.py new file mode 100644 index 00000000..a98efe56 --- /dev/null +++ b/tests/test_prompt_recall_governance.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +import memorymaster.recall.context_hook as context_hook +from memorymaster.core.lifecycle import transition_claim +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.recall.context_hook import recall + + +def test_prompt_recall_uses_trusted_scope_governance( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "prompt-recall.db" + monkeypatch.setenv("MEMORYMASTER_SCOPE_DEFAULT", "project:allowed") + monkeypatch.setenv("MEMORYMASTER_SPOOL_DIR", str(tmp_path / "spool")) + monkeypatch.delenv("QDRANT_URL", raising=False) + service = MemoryService(db_path, workspace_root=tmp_path) + service.init_db() + + confirmed = service.ingest( + "plannerboundary confirmed allowed fact", + [CitationInput(source="test://prompt", locator="confirmed")], + scope="project:allowed", + ) + transition_claim(service.store, confirmed.id, "confirmed", "test fixture") + candidate = service.ingest( + "plannerboundary candidate allowed draft", + [CitationInput(source="test://prompt", locator="candidate")], + scope="project:allowed", + ) + foreign = service.ingest( + "plannerboundary confirmed foreign fact", + [CitationInput(source="test://prompt", locator="foreign")], + scope="project:foreign", + ) + transition_claim(service.store, foreign.id, "confirmed", "test fixture") + + rendered, claim_ids = recall( + "plannerboundary", + db_path=str(db_path), + skip_qdrant=True, + return_ids=True, + ) + + assert confirmed.id in claim_ids + assert "confirmed allowed fact" in rendered + assert candidate.id not in claim_ids + assert "candidate allowed draft" not in rendered + assert foreign.id not in claim_ids + assert "confirmed foreign fact" not in rendered + + +def test_downstream_graph_candidates_cannot_bypass_prompt_policy( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + db_path = tmp_path / "prompt-graph.db" + monkeypatch.setenv("MEMORYMASTER_SCOPE_DEFAULT", "project:allowed") + monkeypatch.setenv("MEMORYMASTER_SPOOL_DIR", str(tmp_path / "spool")) + monkeypatch.setenv("MEMORYMASTER_RECALL_GRAPH", "1") + monkeypatch.setenv("MEMORYMASTER_RECALL_GRAPH_CANDIDATES", "1") + monkeypatch.setenv("MEMORYMASTER_RECALL_W_GRAPH", "1") + monkeypatch.delenv("QDRANT_URL", raising=False) + service = MemoryService(db_path, workspace_root=tmp_path) + service.init_db() + + anchor = service.ingest( + "graphpolicyanchor confirmed allowed fact", + [CitationInput(source="test://graph", locator="anchor")], + scope="project:allowed", + ) + transition_claim(service.store, anchor.id, "confirmed", "test fixture") + candidate = service.ingest( + "downstream candidate draft", + [CitationInput(source="test://graph", locator="candidate")], + scope="project:allowed", + ) + foreign = service.ingest( + "downstream foreign fact", + [CitationInput(source="test://graph", locator="foreign")], + scope="project:foreign", + ) + transition_claim(service.store, foreign.id, "confirmed", "test fixture") + private = service.ingest( + "downstream private fact", + [CitationInput(source="test://graph", locator="private")], + scope="project:allowed", + visibility="private", + ) + transition_claim(service.store, private.id, "confirmed", "test fixture") + + monkeypatch.setattr( + context_hook, + "_graph_reached_claim_distance", + lambda _query, _store: {candidate.id: 1, foreign.id: 1, private.id: 1}, + ) + rendered, claim_ids = context_hook.recall( + "graphpolicyanchor", + db_path=str(db_path), + skip_qdrant=True, + return_ids=True, + ) + + assert claim_ids == [anchor.id] + assert "confirmed allowed fact" in rendered + assert "candidate draft" not in rendered + assert "foreign fact" not in rendered + assert "private fact" not in rendered diff --git a/tests/test_query_expansion.py b/tests/test_query_expansion.py index f041f9a1..ae50e8cf 100644 --- a/tests/test_query_expansion.py +++ b/tests/test_query_expansion.py @@ -16,7 +16,6 @@ import pytest from memorymaster.knowledge.entity_registry import ( - ensure_entity_schema, resolve_or_create, ) from memorymaster.recall.query_expansion import ( @@ -24,6 +23,7 @@ MAX_TOTAL_ALIASES, expand_query, ) +from memorymaster.stores.storage import SQLiteStore # --------------------------------------------------------------------------- # @@ -53,10 +53,10 @@ def _seed_entity( @pytest.fixture() -def entity_db() -> sqlite3.Connection: - conn = sqlite3.connect(":memory:") - conn.row_factory = sqlite3.Row - ensure_entity_schema(conn) +def entity_db(tmp_path) -> sqlite3.Connection: + store = SQLiteStore(tmp_path / "query-expansion.db") + store.init_db() + conn = store.connect() yield conn conn.close() diff --git a/tests/test_recall_entity_fanout.py b/tests/test_recall_entity_fanout.py index e6f1bdd1..7adf0e8d 100644 --- a/tests/test_recall_entity_fanout.py +++ b/tests/test_recall_entity_fanout.py @@ -346,10 +346,13 @@ def __init__(self, store: _FakeStore, fts_rows: list[dict]) -> None: def query_rows(self, **_: object) -> list[dict]: return list(self._rows) + def _record_accesses(self, *_args, **_kwargs) -> None: + return None + def _patch_svc(monkeypatch: pytest.MonkeyPatch, svc: _FakeSvcWithStore, fts_tokens: str) -> None: - def _fake_ctor(db_target, workspace_root): # noqa: ARG001 + def _fake_ctor(db_target, workspace_root, **_kwargs): # noqa: ARG001 return svc monkeypatch.setattr("memorymaster.core.service.MemoryService", _fake_ctor) monkeypatch.setattr( diff --git a/tests/test_recall_precision_at_5.py b/tests/test_recall_precision_at_5.py index 4a377d29..0069c318 100644 --- a/tests/test_recall_precision_at_5.py +++ b/tests/test_recall_precision_at_5.py @@ -37,7 +37,7 @@ def _claim(cid: int, text: str, *, subject: str | None = None, subject=subject, predicate=None, object_value=None, - scope="project:test", + scope="project", volatility="medium", status="confirmed", confidence=confidence, @@ -93,9 +93,12 @@ def __init__(self, rows: list[dict]) -> None: def query_rows(self, **_: object) -> list[dict]: return list(self._rows) + def _record_accesses(self, *_args, **_kwargs) -> None: + return None + def _patch_service(monkeypatch: pytest.MonkeyPatch, rows: list[dict]) -> None: - def _fake_ctor(db_target: str, workspace_root: Path): # noqa: ARG001 + def _fake_ctor(db_target: str, workspace_root: Path, **_kwargs): # noqa: ARG001 return _FakeService(rows) monkeypatch.setattr("memorymaster.core.service.MemoryService", _fake_ctor) diff --git a/tests/test_ro_recall.py b/tests/test_ro_recall.py index 732b41c1..0c14ea4d 100644 --- a/tests/test_ro_recall.py +++ b/tests/test_ro_recall.py @@ -26,6 +26,7 @@ import pytest from memorymaster.core import spool +from memorymaster.core.lifecycle import transition_claim from memorymaster.stores._storage_shared import open_conn from memorymaster.govern.jobs import spool_drain from memorymaster.core.models import CitationInput @@ -56,7 +57,11 @@ def db(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: "The WAL checkpoint discipline truncates the log every cycle", "Recall hooks read claims through a read-only connection", ): - svc.ingest(text, [CitationInput(source="session://chat", locator="turn-1")]) + claim = svc.ingest( + text, + [CitationInput(source="session://chat", locator="turn-1")], + ) + transition_claim(svc.store, claim.id, "confirmed", "trusted recall fixture") return path diff --git a/tests/test_scope_boost.py b/tests/test_scope_boost.py index 2c9875c4..e2cccf47 100644 --- a/tests/test_scope_boost.py +++ b/tests/test_scope_boost.py @@ -72,9 +72,12 @@ def __init__(self, rows: list[dict]) -> None: def query_rows(self, **_: object) -> list[dict]: return list(self._rows) + def _record_accesses(self, *_args, **_kwargs) -> None: + return None + def _patch_service(monkeypatch: pytest.MonkeyPatch, rows: list[dict]) -> None: - def _fake_ctor(db_target: str, workspace_root: Path): # noqa: ARG001 + def _fake_ctor(db_target: str, workspace_root: Path, **_kwargs): # noqa: ARG001 return _FakeService(rows) monkeypatch.setattr("memorymaster.core.service.MemoryService", _fake_ctor) @@ -154,7 +157,7 @@ def _three_row_fixture() -> list[dict]: cross_scope_strong = _claim( 1, "recall pipeline tuning notes from another project", - scope="project:other", + scope="global", ) current_scope_weaker = _claim( 2, @@ -164,7 +167,7 @@ def _three_row_fixture() -> list[dict]: cross_scope_distractor = _claim( 3, "recall subsystem background reading", - scope="project:yetanother", + scope="project", ) return [ {"claim": cross_scope_strong, "lexical_score": 0.95, @@ -219,7 +222,7 @@ def test_boost_half_promotes_current_scope_claim( # Pick weaker/stronger rows whose ratio fits under 1.5x so boost # genuinely flips the ordering (0.45 * 1.5 = 0.675 > 0.60). rows = [ - {"claim": _claim(1, "recall pipeline elsewhere", scope="project:other"), + {"claim": _claim(1, "recall pipeline elsewhere", scope="global"), "lexical_score": 0.60, "freshness_score": 0.0, "confidence_score": 0.5, "vector_score": 0.0}, {"claim": _claim(2, "recall path runs", scope="project:memorymaster"), @@ -269,7 +272,7 @@ def test_boost_zero_point_one_yields_required_score_margin( monkeypatch.setenv("MEMORYMASTER_LEXICAL_BM25", "0") current_scope_claim = _claim(1, "alpha", scope="project:memorymaster") - cross_scope_claim = _claim(2, "alpha", scope="project:other") + cross_scope_claim = _claim(2, "alpha", scope="global") rows = [ {"claim": current_scope_claim, "lexical_score": 1.0, "freshness_score": 0.0, "confidence_score": 0.5, "vector_score": 0.0}, @@ -288,7 +291,7 @@ def test_boost_zero_point_one_yields_required_score_margin( # current-scope claim is listed first in the rendered output. out = recall("alpha", db_path=str(tmp_path / "nope.db"), skip_qdrant=True) lines = [ln for ln in out.splitlines() if ln.startswith("- ")] - assert lines, f"no claims returned: {out!r}" + assert len(lines) == 2, f"expected both governed scopes: {out!r}" # Both claims have the same text ("alpha") so we can't distinguish by # line content alone — assert the count is 2 and the current-scope # claim comes first by checking that the _relevance-driven rank diff --git a/tests/test_two_pass_recall.py b/tests/test_two_pass_recall.py index f4e6ecb9..02ecbb27 100644 --- a/tests/test_two_pass_recall.py +++ b/tests/test_two_pass_recall.py @@ -94,31 +94,34 @@ def test_neighbor_ids_walks_claim_entity_links_on_real_store(tmp_path): table no schema ever created, via a ``store._conn`` attribute real stores don't have, so the stream silently returned [] even when the operator enabled it.""" + from memorymaster.core.models import CitationInput + from memorymaster.knowledge.entity_registry import resolve_or_create from memorymaster.stores.storage import SQLiteStore db = str(tmp_path / "two_pass.db") store = SQLiteStore(db) store.init_db() + first = store.create_claim(text="Redis seed", citations=[CitationInput(source="test")]) + second = store.create_claim(text="Redis neighbor", citations=[CitationInput(source="test")]) + third = store.create_claim(text="Postgres other", citations=[CitationInput(source="test")]) with store.connect() as conn: # Same DDL as EntityGraph.ensure_tables (entity_graph.py). Created # directly because ensure_tables' full script aborts when the claims # DB already holds the registry-style ``entities`` table — the # junction is what the walker contract depends on. - conn.execute( - "CREATE TABLE IF NOT EXISTS claim_entity_links (" - " claim_id INTEGER NOT NULL," - " entity_id TEXT NOT NULL," - " PRIMARY KEY (claim_id, entity_id))" - ) + redis_id = resolve_or_create(conn, "Redis") + postgres_id = resolve_or_create(conn, "Postgres") conn.executemany( "INSERT INTO claim_entity_links (claim_id, entity_id) VALUES (?, ?)", - [(1, "ent-redis"), (2, "ent-redis"), (3, "ent-postgres")], + [(first.id, redis_id), (second.id, redis_id), (third.id, postgres_id)], ) conn.commit() # Claim 2 shares ent-redis with seed claim 1; claim 3 shares nothing. - out = context_hook._two_pass_neighbor_ids(store, [1], {1}) - assert out == [2] + out = context_hook._two_pass_neighbor_ids(store, [first.id], {first.id}) + assert out == [second.id] # Excluded/seen IDs are never reintroduced. - assert context_hook._two_pass_neighbor_ids(store, [1], {1, 2}) == [] + assert context_hook._two_pass_neighbor_ids( + store, [first.id], {first.id, second.id} + ) == [] diff --git a/tests/test_wiki_binding.py b/tests/test_wiki_binding.py index 3de52b3c..ac00e496 100644 --- a/tests/test_wiki_binding.py +++ b/tests/test_wiki_binding.py @@ -123,7 +123,7 @@ def test_recall_appends_wiki_pointer(tmp_path: Path, monkeypatch: pytest.MonkeyP subject="qdrant", predicate="runs_on", object_value="vm", - scope="project:test", + scope="project", volatility="medium", status="confirmed", confidence=0.9, @@ -141,7 +141,10 @@ class _FakeService: def query_rows(self, **_: object) -> list[dict]: return [{"claim": sample, "lexical_score": 1.0, "confidence_score": 0.9}] - def _fake_ctor(db_target: str, workspace_root: Path): # noqa: ARG001 + def _record_accesses(self, *_args, **_kwargs) -> None: + return None + + def _fake_ctor(db_target: str, workspace_root: Path, **_kwargs): # noqa: ARG001 return _FakeService() monkeypatch.setattr("memorymaster.core.service.MemoryService", _fake_ctor) From 6528f64b5aa9acf51100202be43deecb9ef86acd Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Mon, 13 Jul 2026 23:10:11 -0300 Subject: [PATCH 37/55] perf: make embedding sync incremental --- .planning/REMAINING-PHASES-V1-BUDGET.md | 28 ++- .planning/audit-remediation-ledger.md | 4 +- memorymaster/core/service.py | 16 +- memorymaster/recall/embeddings.py | 43 ++++ memorymaster/recall/qdrant_backend.py | 67 +++--- memorymaster/schema.sql | 8 + memorymaster/schema_postgres.sql | 9 + memorymaster/stores/_storage_lifecycle.py | 178 +++++++++----- memorymaster/stores/_storage_schema.py | 1 + .../migrations/0014_embedding_efficiency.py | 71 ++++++ memorymaster/stores/postgres_store.py | 225 ++++++++++++++---- tests/test_embedding_efficiency.py | 126 ++++++++++ tests/test_qdrant_backend.py | 9 +- tests/test_qdrant_sync_pagination.py | 105 ++++++++ 14 files changed, 743 insertions(+), 147 deletions(-) create mode 100644 memorymaster/stores/migrations/0014_embedding_efficiency.py create mode 100644 tests/test_embedding_efficiency.py create mode 100644 tests/test_qdrant_sync_pagination.py diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index 36f58c8b..4af23bea 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -286,5 +286,29 @@ P2-Z Phase 2 convergence completed on 2026-07-13: - Atomic package commit: the conventional Phase 2 convergence commit containing this scheduler evidence. -The next package is R3.1 embedding and reconciliation efficiency. The active -roadmap goal authorizes continuing without a new goal. +R3.1 embedding and reconciliation efficiency completed on 2026-07-13: + +- Immutable migration 0014 and both baseline schemas persist the exact + embedding-content hash plus an authority-scoped durable Qdrant sync cursor. +- SQLite/Postgres embedding caches refresh only missing, model-changed, or + content-changed claims. Candidate refresh uses the provider batch surface; + a warm retrieval performs one query embed and zero candidate embeds/writes. +- The semantic downgrade probe is reused as the query vector, preserving the + post-probe safety check without embedding the query twice. +- Qdrant full synchronization now keyset-pages every authoritative eligible + claim, checkpoints only completed pages, replays the last incomplete page, + and resets its cursor only after full convergence. The former per-status + 10,000-row truncation is removed. +- RED evidence: the initial efficiency/cursor contract failed 6 tests; the + service probe separately demonstrated 2 query embeddings instead of 1. +- Focused correction evidence: embedding/read-only slice 42 passed; Qdrant + sync/reconciliation slice 43 passed with one existing Pydantic warning; + migration/semantic-downgrade slice 15 passed. Changed-file Ruff and + `git diff --check` passed. The first combined invocation hit its 244-second + execution ceiling without a result and is recorded as a timeout, not a pass. +- Authenticated/TLS Qdrant and disposable Postgres runtime parity remain + `BLOCKED-EXTERNAL`; no production semantic profile was enabled. +- Atomic package commit: the conventional R3.1 commit containing this evidence. + +The next package is R3.2 query and storage efficiency. The active roadmap goal +authorizes continuing without a new goal. diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 2e16b478..db1ea12d 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -24,9 +24,9 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-OPS-02 | DevOps/Demo | High / BAD-PRACTICE | Docker/Helm publish HTTP on a stdio MCP process and use an invalid healthcheck | R3.4 | OPEN | Built-container readiness and MCP handshake | | MM-OPS-03 | DevOps/Maintainability | High / BAD-PRACTICE | A release tag can publish without a blocking verified test artifact | R4.4 | OPEN | Deliberately failing release candidate cannot publish | | MM-OPS-04 | Security/DevOps | High / EXPLOITABLE-LOW-EFFORT | Qdrant/Ollama ports are broadly exposed; images are mutable | R1.5,R3.4 | IN-PROGRESS | Phase 1 defaults are repository-complete (`a858419`, `9b3e16c`, `b71e18f`): loopback-only ports, required immutable digests, authenticated TLS Qdrant, redirect-safe credentials, and fail-closed supply-chain policy. Real image scans/runtime health remain `BLOCKED-EXTERNAL`; R3.4 service-entrypoint work remains backlog. | -| MM-PERF-01 | Performance/Cost | High / BAD-PRACTICE | Hybrid reads recompute and rewrite candidate embeddings | R3.1 | OPEN | Warm query: one query embed, zero candidate embeds/writes | +| MM-PERF-01 | Performance/Cost | High / BAD-PRACTICE | Hybrid reads recompute and rewrite candidate embeddings | R3.1 | RESOLVED | R3.1 migration 0014 persists model/content fingerprints; focused warm-query evidence proves one query embed and zero candidate embeds/writes. | | MM-PERF-02 | Performance | Medium / BAD-PRACTICE | Each process cold-scans corpus token statistics and caches forever | R3.2 | OPEN | Generation-aware/token-specific stats benchmark | -| MM-PERF-03 | Performance/Cost | High / EXPLOITABLE-NOW | Qdrant reconciliation repeats/truncates tens of thousands of embeddings | R3.1 | OPEN | Paginated incremental convergence benchmark | +| MM-PERF-03 | Performance/Cost | High / EXPLOITABLE-NOW | Qdrant reconciliation repeats/truncates tens of thousands of embeddings | R3.1 | RESOLVED | R3.1 replaces four truncated 10,000-row scans with authority-scoped keyset pages and a replayable durable cursor; fake-backend convergence/failure-resume evidence passed. Real authenticated/TLS Qdrant remains externally blocked. | | MM-PERF-04 | Database/Performance | Medium / BAD-PRACTICE | Million-row event queries lack event-type composite indexes | R3.2 | OPEN | Versioned parity migration plus EXPLAIN/timing evidence | | MM-UX-01 | UX/Maintainability | High / BAD-PRACTICE | Setup reports success without verifying the requested memory loop | R3.3 | OPEN | Component-level profile verification and nonzero failures | | MM-UX-02 | UX/Reliability | High / BAD-PRACTICE | Dashboard removes review rows before mutation success | R4.3 | OPEN | Browser failure/retry evidence; row remains on rejected POST | diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index ef40fb32..a87bf97e 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -1488,9 +1488,7 @@ def query_rows( candidates = _filter_agent_visibility(candidates, requesting_agent) semantic = False if vector_hook is None and hasattr(self.store, "vector_scores"): - def _vector_hook(text, claims): - return self.store.vector_scores(text, claims, self.embedding_provider) - vector_hook = _vector_hook + query_vector = None semantic = self.embedding_provider.is_semantic if semantic: # The Gemini->hash downgrade is lazy: is_semantic only reflects a @@ -1499,8 +1497,18 @@ def _vector_hook(text, claims): # degraded) so a degraded provider is not reported as semantic — which # would apply retrieval's lenient vector-only filter to non-semantic # hash vectors. (audit: embeddings TOCTOU) - self.embedding_provider.embed(query_text) + query_vector = self.embedding_provider.embed(query_text) semantic = self.embedding_provider.is_semantic + def _vector_hook(text, claims): + if query_vector is None: + return self.store.vector_scores(text, claims, self.embedding_provider) + return self.store.vector_scores( + text, + claims, + self.embedding_provider, + query_vector=query_vector, + ) + vector_hook = _vector_hook rank_limit = len(candidates) if profile_weights is not None else (max(limit, 50) if use_llm_rerank else limit) final_rank_limit = max(limit, 50) if use_llm_rerank else limit ranked_rows = rank_claim_rows( diff --git a/memorymaster/recall/embeddings.py b/memorymaster/recall/embeddings.py index 9a1cfad6..055afba9 100644 --- a/memorymaster/recall/embeddings.py +++ b/memorymaster/recall/embeddings.py @@ -46,6 +46,20 @@ def embed(self, text: str) -> list[float]: return self._gemini_embed(text) return self._semantic_embed(text) + def embed_batch(self, texts: list[str]) -> list[list[float]]: + """Embed a bounded candidate batch with one provider operation.""" + if not texts: + return [] + if self.model.startswith("hash"): + return [hash_embed(text, dims=self.dims) for text in texts] + if self.model.startswith("gemini:"): + return self._gemini_embed_batch(texts) + if self._transformer is None: + self._transformer = _load_transformer(self.model) + embeddings = self._transformer.encode(texts, normalize_embeddings=True) + self.dims = len(embeddings[0]) + return [embedding.tolist() for embedding in embeddings] + def _semantic_embed(self, text: str) -> list[float]: if self._transformer is None: self._transformer = _load_transformer(self.model) @@ -80,6 +94,30 @@ def _gemini_embed(self, text: str) -> list[float]: self.dims = len(vec) return normalize(list(vec)) + def _gemini_embed_batch(self, texts: list[str]) -> list[list[float]]: + if self._transformer is None: + self._transformer = _load_gemini_client() + gemini_model = self.model.split(":", 1)[1] + try: + result = self._transformer.models.embed_content( + model=gemini_model, + contents=texts, + ) + except Exception as exc: + logger.warning( + "Gemini batch embedding failed (%s); falling back to hash-v1", + exc, + ) + self.model = "hash-v1" + self.dims = 1536 + self.degraded = True + return [hash_embed(text, dims=self.dims) for text in texts] + vectors = [normalize(list(item.values)) for item in result.embeddings] + if len(vectors) != len(texts): + raise ValueError("embedding provider returned an incomplete batch") + self.dims = len(vectors[0]) + return vectors + def create_semantic_provider(model: str = "all-MiniLM-L6-v2") -> EmbeddingProvider: """Create a provider using sentence-transformers for real semantic embeddings. @@ -215,3 +253,8 @@ def cosine_similarity(a: list[float], b: list[float]) -> float: dot = sum(x * y for x, y in zip(a, b, strict=True)) # vectors are normalized, but clamp for numeric safety. return max(-1.0, min(1.0, dot)) + + +def embedding_content_hash(text: str) -> str: + """Stable fingerprint of the exact authoritative text embedded.""" + return hashlib.sha256(text.encode("utf-8")).hexdigest() diff --git a/memorymaster/recall/qdrant_backend.py b/memorymaster/recall/qdrant_backend.py index 5e1b2f55..67419296 100644 --- a/memorymaster/recall/qdrant_backend.py +++ b/memorymaster/recall/qdrant_backend.py @@ -468,32 +468,28 @@ def _batch_upsert(self, points: list[dict[str, Any]]) -> bool: return False def sync_all(self, store, *, batch_size: int = 50) -> dict[str, int]: - """Bulk-push all active claims from the store to Qdrant. - - Uses batch upsert for much better throughput than one-by-one. - - Parameters - ---------- - store : SQLiteStore | PostgresStore - The primary data store. - batch_size : int - Points per Qdrant upsert request (default 50). - - Returns - ------- - dict with keys: total, synced, skipped, errors - """ + """Replayably keyset-page every eligible authoritative claim.""" + if batch_size <= 0: + raise ValueError("batch_size must be positive") self.ensure_collection() stats = {"total": 0, "synced": 0, "skipped": 0, "errors": 0} - - for status in ("confirmed", "stale", "candidate", "conflicted"): - claims = store.find_by_status(status, limit=10_000, include_citations=False) + authority = str(getattr(store, "tenant_id", None) or "local") + stream_key = f"{self.collection}:{self.embed_model}:{authority}" + cursor = store.get_qdrant_sync_cursor(stream_key) + if isinstance(cursor, bool) or not isinstance(cursor, int) or cursor < 0: + raise TypeError("Qdrant sync cursor store returned an invalid cursor") + + while True: + claims = store.list_qdrant_sync_page(after_id=cursor, limit=batch_size) + if not isinstance(claims, list): + raise TypeError("Qdrant sync store returned an invalid page") + if not claims: + store.set_qdrant_sync_cursor(stream_key, 0) + break stats["total"] += len(claims) - if claims: - logger.info("Syncing %d %s claims to Qdrant...", len(claims), status) - - batch: list[dict[str, Any]] = [] - for _idx, claim in enumerate(claims): + points: list[dict[str, Any]] = [] + page_errors = 0 + for claim in claims: findings = scan_persisted_value(self._claim_representation(claim)) if findings: stats["skipped"] += 1 @@ -502,25 +498,22 @@ def sync_all(self, store, *, batch_size: int = 50) -> dict[str, int]: vec = self._embed(self._claim_text(claim)) if vec is None: stats["errors"] += 1 + page_errors += 1 continue - batch.append({ + points.append({ "id": self._point_id(claim.id), "vector": vec, "payload": self._claim_payload(claim), }) - if len(batch) >= batch_size: - if self._batch_upsert(batch): - stats["synced"] += len(batch) - else: - stats["errors"] += len(batch) - batch = [] - - # Flush remaining - if batch: - if self._batch_upsert(batch): - stats["synced"] += len(batch) - else: - stats["errors"] += len(batch) + if points and self._batch_upsert(points): + stats["synced"] += len(points) + elif points: + stats["errors"] += len(points) + page_errors += len(points) + if page_errors: + break + cursor = claims[-1].id + store.set_qdrant_sync_cursor(stream_key, cursor) logger.info( "Qdrant sync_all complete: %d total, %d synced, %d errors", diff --git a/memorymaster/schema.sql b/memorymaster/schema.sql index 57ff68b0..9f54617c 100644 --- a/memorymaster/schema.sql +++ b/memorymaster/schema.sql @@ -98,11 +98,19 @@ END; CREATE TABLE IF NOT EXISTS claim_embeddings ( claim_id INTEGER PRIMARY KEY, model TEXT NOT NULL, + content_hash TEXT NOT NULL DEFAULT '', embedding_json TEXT NOT NULL, updated_at TEXT NOT NULL, FOREIGN KEY (claim_id) REFERENCES claims(id) ON DELETE CASCADE ); +CREATE TABLE IF NOT EXISTS qdrant_sync_state ( + stream_key TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL DEFAULT '', + last_claim_id INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL +); + CREATE TABLE IF NOT EXISTS external_sources ( id INTEGER PRIMARY KEY AUTOINCREMENT, source_type TEXT NOT NULL, diff --git a/memorymaster/schema_postgres.sql b/memorymaster/schema_postgres.sql index 61b253ba..f2129ca9 100644 --- a/memorymaster/schema_postgres.sql +++ b/memorymaster/schema_postgres.sql @@ -375,6 +375,7 @@ BEGIN CREATE TABLE IF NOT EXISTS claim_embeddings ( claim_id BIGINT PRIMARY KEY REFERENCES claims(id) ON DELETE CASCADE, model TEXT NOT NULL, + content_hash TEXT NOT NULL DEFAULT '''', embedding VECTOR(1536) NOT NULL, updated_at TIMESTAMPTZ NOT NULL ) @@ -388,6 +389,7 @@ BEGIN CREATE TABLE IF NOT EXISTS claim_embeddings ( claim_id BIGINT PRIMARY KEY REFERENCES claims(id) ON DELETE CASCADE, model TEXT NOT NULL, + content_hash TEXT NOT NULL DEFAULT '''', embedding_json TEXT NOT NULL, updated_at TIMESTAMPTZ NOT NULL ) @@ -397,3 +399,10 @@ END $$; CREATE INDEX IF NOT EXISTS idx_embeddings_updated_at ON claim_embeddings(updated_at); + +CREATE TABLE IF NOT EXISTS qdrant_sync_state ( + stream_key TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + last_claim_id BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL +); diff --git a/memorymaster/stores/_storage_lifecycle.py b/memorymaster/stores/_storage_lifecycle.py index cc3029a2..61eb79c0 100644 --- a/memorymaster/stores/_storage_lifecycle.py +++ b/memorymaster/stores/_storage_lifecycle.py @@ -13,7 +13,11 @@ from typing import TYPE_CHECKING, Any from memorymaster.core.lifecycle import can_transition -from memorymaster.recall.embeddings import EmbeddingProvider, cosine_similarity +from memorymaster.recall.embeddings import ( + EmbeddingProvider, + cosine_similarity, + embedding_content_hash, +) from memorymaster.core.models import ( CLAIM_LINK_TYPES, CLAIM_STATUSES, @@ -563,53 +567,65 @@ def _embedding_text(claim: Claim) -> str: def upsert_embeddings(self, claims: list[Claim], provider: EmbeddingProvider) -> int: if not claims: return 0 + texts = {claim.id: self._embedding_text(claim) for claim in claims} + hashes = { + claim_id: embedding_content_hash(text) + for claim_id, text in texts.items() + } + claim_ids = list(texts) + placeholders = ",".join("?" for _ in claim_ids) + with self.connect() as conn: + cached = { + int(row["claim_id"]): (str(row["model"]), str(row["content_hash"])) + for row in conn.execute( + f"SELECT claim_id, model, content_hash FROM claim_embeddings " + f"WHERE claim_id IN ({placeholders})", + claim_ids, + ).fetchall() + } + stale = [ + claim + for claim in claims + if cached.get(claim.id) != (provider.model, hashes[claim.id]) + ] + if not stale: + return 0 + requested_model = provider.model + batch = getattr(provider, "embed_batch", None) + embeddings = ( + batch([texts[claim.id] for claim in stale]) + if callable(batch) + else [provider.embed(texts[claim.id]) for claim in stale] + ) + if len(embeddings) != len(stale): + raise ValueError("embedding provider returned an incomplete batch") + if provider.model != requested_model: + stale = list(claims) + embeddings = ( + batch([texts[claim.id] for claim in stale]) + if callable(batch) + else [provider.embed(texts[claim.id]) for claim in stale] + ) now = utc_now() - rows = [] - for claim in claims: - embedding = provider.embed(self._embedding_text(claim)) - rows.append((claim.id, provider.model, json.dumps(embedding), now)) - - try: - with self.connect() as conn: - conn.executemany( - """ - INSERT INTO claim_embeddings (claim_id, model, embedding_json, updated_at) - VALUES (?, ?, ?, ?) - ON CONFLICT(claim_id) DO UPDATE SET - model = excluded.model, - embedding_json = excluded.embedding_json, - updated_at = excluded.updated_at - """, - rows, - ) - conn.commit() - except sqlite3.OperationalError as exc: - if "no such table" in str(exc).lower(): - logger.warning("claim_embeddings table missing, recreating: %s", exc) - try: - # Create the table and ensure it's committed before retrying - with self.connect() as create_conn: - self._ensure_embeddings_schema(create_conn) - create_conn.commit() - # Retry the insert - with self.connect() as conn: - conn.executemany( - """ - INSERT INTO claim_embeddings (claim_id, model, embedding_json, updated_at) - VALUES (?, ?, ?, ?) - ON CONFLICT(claim_id) DO UPDATE SET - model = excluded.model, - embedding_json = excluded.embedding_json, - updated_at = excluded.updated_at - """, - rows, - ) - conn.commit() - except Exception as retry_exc: - logger.error("Failed to recreate claim_embeddings: %s", retry_exc) - return 0 - else: - raise + rows = [ + (claim.id, provider.model, hashes[claim.id], json.dumps(embedding), now) + for claim, embedding in zip(stale, embeddings, strict=True) + ] + with self.connect() as conn: + conn.executemany( + """ + INSERT INTO claim_embeddings + (claim_id, model, content_hash, embedding_json, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT(claim_id) DO UPDATE SET + model = excluded.model, + content_hash = excluded.content_hash, + embedding_json = excluded.embedding_json, + updated_at = excluded.updated_at + """, + rows, + ) + conn.commit() return len(rows) @@ -618,6 +634,8 @@ def vector_scores( query_text: str, claims: list[Claim], provider: EmbeddingProvider, + *, + query_vector: list[float] | None = None, ) -> dict[int, float]: if not claims: return {} @@ -632,27 +650,79 @@ def vector_scores( read_only = bool(getattr(self, "read_only", False)) if not read_only: self.upsert_embeddings(claims, provider) - query_vec = provider.embed(query_text) + query_vec = query_vector if query_vector is not None else provider.embed(query_text) claim_ids = [c.id for c in claims] placeholders = ",".join("?" for _ in claim_ids) with self.connect() as conn: rows = conn.execute( - f"SELECT claim_id, embedding_json FROM claim_embeddings WHERE claim_id IN ({placeholders})", + f"SELECT claim_id, model, content_hash, embedding_json " + f"FROM claim_embeddings WHERE claim_id IN ({placeholders})", claim_ids, ).fetchall() scores: dict[int, float] = {} + claims_by_id = {claim.id: claim for claim in claims} for row in rows: + claim_id = int(row["claim_id"]) + claim = claims_by_id[claim_id] + expected_hash = embedding_content_hash(self._embedding_text(claim)) + if str(row["model"]) != provider.model or str(row["content_hash"]) != expected_hash: + continue emb = json.loads(str(row["embedding_json"])) sim = cosine_similarity(query_vec, emb) - scores[int(row["claim_id"])] = max(0.0, min(1.0, (sim + 1.0) / 2.0)) + scores[claim_id] = max(0.0, min(1.0, (sim + 1.0) / 2.0)) if read_only: - for claim in claims: - if claim.id in scores: - continue - sim = cosine_similarity(query_vec, provider.embed(self._embedding_text(claim))) + missing = [claim for claim in claims if claim.id not in scores] + batch = getattr(provider, "embed_batch", None) + embeddings = ( + batch([self._embedding_text(claim) for claim in missing]) + if callable(batch) + else [provider.embed(self._embedding_text(claim)) for claim in missing] + ) + for claim, embedding in zip(missing, embeddings, strict=True): + sim = cosine_similarity(query_vec, embedding) scores[claim.id] = max(0.0, min(1.0, (sim + 1.0) / 2.0)) return scores + def list_qdrant_sync_page(self, *, after_id: int, limit: int) -> list[Claim]: + """Return one authoritative keyset page for vector maintenance.""" + if after_id < 0 or limit <= 0: + raise ValueError("after_id must be nonnegative and limit must be positive") + statuses = ("confirmed", "stale", "candidate", "conflicted") + placeholders = ",".join("?" for _ in statuses) + with self.connect() as conn: + rows = conn.execute( + f"SELECT * FROM claims WHERE id > ? AND status IN ({placeholders}) " + "ORDER BY id ASC LIMIT ?", + (after_id, *statuses, limit), + ).fetchall() + return [self._row_to_claim(row) for row in rows] + + def get_qdrant_sync_cursor(self, stream_key: str) -> int: + if not stream_key or len(stream_key) > 512: + raise ValueError("invalid Qdrant sync stream key") + with self.connect() as conn: + row = conn.execute( + "SELECT last_claim_id FROM qdrant_sync_state WHERE stream_key = ?", + (stream_key,), + ).fetchone() + return int(row[0]) if row is not None else 0 + + def set_qdrant_sync_cursor(self, stream_key: str, last_claim_id: int) -> None: + if not stream_key or len(stream_key) > 512 or last_claim_id < 0: + raise ValueError("invalid Qdrant sync cursor") + with self.connect() as conn: + conn.execute( + """ + INSERT INTO qdrant_sync_state (stream_key, last_claim_id, updated_at) + VALUES (?, ?, ?) + ON CONFLICT(stream_key) DO UPDATE SET + last_claim_id = excluded.last_claim_id, + updated_at = excluded.updated_at + """, + (stream_key, last_claim_id, utc_now()), + ) + conn.commit() + def record_access(self, claim_id: int) -> None: """Increment access_count and set last_accessed for a claim.""" diff --git a/memorymaster/stores/_storage_schema.py b/memorymaster/stores/_storage_schema.py index 644ac0e9..f2af316b 100644 --- a/memorymaster/stores/_storage_schema.py +++ b/memorymaster/stores/_storage_schema.py @@ -98,6 +98,7 @@ def _ensure_embeddings_schema(conn: sqlite3.Connection) -> None: CREATE TABLE IF NOT EXISTS claim_embeddings ( claim_id INTEGER PRIMARY KEY, model TEXT NOT NULL, + content_hash TEXT NOT NULL DEFAULT '', embedding_json TEXT NOT NULL, updated_at TEXT NOT NULL, FOREIGN KEY (claim_id) REFERENCES claims(id) ON DELETE CASCADE diff --git a/memorymaster/stores/migrations/0014_embedding_efficiency.py b/memorymaster/stores/migrations/0014_embedding_efficiency.py new file mode 100644 index 00000000..1535bd76 --- /dev/null +++ b/memorymaster/stores/migrations/0014_embedding_efficiency.py @@ -0,0 +1,71 @@ +"""Persist embedding fingerprints and replayable Qdrant sync cursors.""" + +from __future__ import annotations + +VERSION = 14 +DESCRIPTION = "Embedding fingerprints and Qdrant sync cursor state" + + +def apply_sqlite(conn) -> None: + columns = { + str(row[1]) + for row in conn.execute("PRAGMA table_info(claim_embeddings)").fetchall() + } + if columns and "content_hash" not in columns: + conn.execute( + "ALTER TABLE claim_embeddings " + "ADD COLUMN content_hash TEXT NOT NULL DEFAULT ''" + ) + conn.execute( + """ + CREATE TABLE IF NOT EXISTS qdrant_sync_state ( + stream_key TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL DEFAULT '', + last_claim_id INTEGER NOT NULL DEFAULT 0, + updated_at TEXT NOT NULL + ) + """ + ) + conn.commit() + + +def apply_postgres(conn) -> None: + with conn.cursor() as cur: + cur.execute( + "ALTER TABLE IF EXISTS claim_embeddings " + "ADD COLUMN IF NOT EXISTS content_hash TEXT NOT NULL DEFAULT ''" + ) + cur.execute( + """ + CREATE TABLE IF NOT EXISTS qdrant_sync_state ( + stream_key TEXT PRIMARY KEY, + tenant_id TEXT NOT NULL, + last_claim_id BIGINT NOT NULL DEFAULT 0, + updated_at TIMESTAMPTZ NOT NULL + ) + """ + ) + cur.execute("ALTER TABLE qdrant_sync_state ENABLE ROW LEVEL SECURITY") + cur.execute("ALTER TABLE qdrant_sync_state FORCE ROW LEVEL SECURITY") + cur.execute( + """ + DO $$ BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_policies + WHERE schemaname = 'public' + AND tablename = 'qdrant_sync_state' + AND policyname = 'memorymaster_qdrant_sync_tenant' + ) THEN + CREATE POLICY memorymaster_qdrant_sync_tenant + ON qdrant_sync_state + USING ( + tenant_id = current_setting('memorymaster.tenant_id', true) + ) + WITH CHECK ( + tenant_id = current_setting('memorymaster.tenant_id', true) + ); + END IF; + END $$ + """ + ) + conn.commit() diff --git a/memorymaster/stores/postgres_store.py b/memorymaster/stores/postgres_store.py index 6d952f92..a16728a4 100644 --- a/memorymaster/stores/postgres_store.py +++ b/memorymaster/stores/postgres_store.py @@ -6,7 +6,11 @@ from typing import Any, Iterable from memorymaster.core.lifecycle import can_transition -from memorymaster.recall.embeddings import EmbeddingProvider, cosine_similarity +from memorymaster.recall.embeddings import ( + EmbeddingProvider, + cosine_similarity, + embedding_content_hash, +) from memorymaster.core.models import ( ActionProposal, CLAIM_LINK_TYPES, @@ -171,6 +175,7 @@ def __init__( ) self._psycopg: Any = None self._vector_table_available: bool | None = None + self._vector_storage_kind: str | None = None def _require_team_authority(self) -> tuple[str, str, tuple[str, ...]]: if self.tenant_id is None: @@ -3013,21 +3018,28 @@ def record_event( ) def _has_vector_table(self) -> bool: - if self._vector_table_available is not None: - return self._vector_table_available + return self._embedding_storage_kind() == "vector" + + def _embedding_storage_kind(self) -> str: + if self._vector_storage_kind is not None: + return self._vector_storage_kind with self.connect() as conn, conn.cursor() as cur: cur.execute( """ - SELECT EXISTS ( - SELECT 1 - FROM information_schema.tables - WHERE table_schema = 'public' AND table_name = 'claim_embeddings' - ) AS ok + SELECT column_name + FROM information_schema.columns + WHERE table_schema = 'public' + AND table_name = 'claim_embeddings' + AND column_name IN ('embedding', 'embedding_json') """ ) - row = cur.fetchone() - self._vector_table_available = bool(row["ok"]) if row is not None else False - return self._vector_table_available + rows = cur.fetchall() + columns = {str(row["column_name"]) for row in rows} + self._vector_storage_kind = ( + "vector" if "embedding" in columns else "json" if "embedding_json" in columns else "missing" + ) + self._vector_table_available = self._vector_storage_kind == "vector" + return self._vector_storage_kind @staticmethod def _vector_literal(vec: list[float]) -> str: @@ -3036,31 +3048,77 @@ def _vector_literal(vec: list[float]) -> str: def upsert_embeddings(self, claims: list[Claim], provider: EmbeddingProvider) -> int: if not claims: return 0 - if not self._has_vector_table(): + storage_kind = self._embedding_storage_kind() + if storage_kind == "missing": return 0 - now = utc_now() - rows: list[tuple[object, ...]] = [] - for claim in claims: - text = " ".join( - part - for part in [claim.text, claim.normalized_text or "", claim.subject or "", claim.object_value or ""] - if part - ) - emb = provider.embed(text) - rows.append((claim.id, provider.model, self._vector_literal(emb), now)) - + texts = {claim.id: self._embedding_text(claim) for claim in claims} + hashes = { + claim_id: embedding_content_hash(text) + for claim_id, text in texts.items() + } + ids = list(texts) with self.connect() as conn, conn.cursor() as cur: - cur.executemany( - """ - INSERT INTO claim_embeddings (claim_id, model, embedding, updated_at) - VALUES (%s, %s, %s::vector, %s) - ON CONFLICT (claim_id) DO UPDATE SET - model = EXCLUDED.model, - embedding = EXCLUDED.embedding, - updated_at = EXCLUDED.updated_at - """, - rows, + cur.execute( + "SELECT claim_id, model, content_hash FROM claim_embeddings " + "WHERE claim_id = ANY(%s)", + (ids,), ) + cached = { + int(row["claim_id"]): (str(row["model"]), str(row["content_hash"])) + for row in cur.fetchall() + } + stale = [ + claim + for claim in claims + if cached.get(claim.id) != (provider.model, hashes[claim.id]) + ] + if not stale: + return 0 + requested_model = provider.model + batch = getattr(provider, "embed_batch", None) + embeddings = ( + batch([texts[claim.id] for claim in stale]) + if callable(batch) + else [provider.embed(texts[claim.id]) for claim in stale] + ) + if provider.model != requested_model: + stale = list(claims) + embeddings = ( + batch([texts[claim.id] for claim in stale]) + if callable(batch) + else [provider.embed(texts[claim.id]) for claim in stale] + ) + if len(embeddings) != len(stale): + raise ValueError("embedding provider returned an incomplete batch") + now = utc_now() + if storage_kind == "vector": + rows = [ + (claim.id, provider.model, hashes[claim.id], self._vector_literal(embedding), now) + for claim, embedding in zip(stale, embeddings, strict=True) + ] + sql = """ + INSERT INTO claim_embeddings + (claim_id, model, content_hash, embedding, updated_at) + VALUES (%s, %s, %s, %s::vector, %s) + ON CONFLICT (claim_id) DO UPDATE SET + model = EXCLUDED.model, content_hash = EXCLUDED.content_hash, + embedding = EXCLUDED.embedding, updated_at = EXCLUDED.updated_at + """ + else: + rows = [ + (claim.id, provider.model, hashes[claim.id], json.dumps(embedding), now) + for claim, embedding in zip(stale, embeddings, strict=True) + ] + sql = """ + INSERT INTO claim_embeddings + (claim_id, model, content_hash, embedding_json, updated_at) + VALUES (%s, %s, %s, %s, %s) + ON CONFLICT (claim_id) DO UPDATE SET + model = EXCLUDED.model, content_hash = EXCLUDED.content_hash, + embedding_json = EXCLUDED.embedding_json, updated_at = EXCLUDED.updated_at + """ + with self.connect() as conn, conn.cursor() as cur: + cur.executemany(sql, rows) return len(rows) def vector_scores( @@ -3068,15 +3126,45 @@ def vector_scores( query_text: str, claims: list[Claim], provider: EmbeddingProvider, + *, + query_vector: list[float] | None = None, ) -> dict[int, float]: if not claims: return {} - if not self._has_vector_table(): - return self._vector_scores_fallback(query_text, claims, provider) - + storage_kind = self._embedding_storage_kind() + if storage_kind == "missing": + return self._vector_scores_fallback( + query_text, claims, provider, query_vector=query_vector + ) self.upsert_embeddings(claims, provider) - query_vec = self._vector_literal(provider.embed(query_text)) + raw_query_vec = query_vector if query_vector is not None else provider.embed(query_text) ids = [c.id for c in claims] + if storage_kind == "json": + with self.connect() as conn, conn.cursor() as cur: + cur.execute( + "SELECT claim_id, embedding_json FROM claim_embeddings " + "WHERE claim_id = ANY(%s) AND model = %s", + (ids, provider.model), + ) + rows = cur.fetchall() + return { + int(row["claim_id"]): max( + 0.0, + min( + 1.0, + ( + cosine_similarity( + raw_query_vec, + json.loads(str(row["embedding_json"])), + ) + + 1.0 + ) + / 2.0, + ), + ) + for row in rows + } + query_vec = self._vector_literal(raw_query_vec) with self.connect() as conn, conn.cursor() as cur: cur.execute( """ @@ -3098,20 +3186,67 @@ def _vector_scores_fallback( query_text: str, claims: list[Claim], provider: EmbeddingProvider, + *, + query_vector: list[float] | None = None, ) -> dict[int, float]: - query_vec = provider.embed(query_text) + query_vec = query_vector if query_vector is not None else provider.embed(query_text) out: dict[int, float] = {} - for claim in claims: - text = " ".join( - part - for part in [claim.text, claim.normalized_text or "", claim.subject or "", claim.object_value or ""] - if part - ) - emb = provider.embed(text) + batch = getattr(provider, "embed_batch", None) + texts = [self._embedding_text(claim) for claim in claims] + embeddings = batch(texts) if callable(batch) else [provider.embed(text) for text in texts] + for claim, emb in zip(claims, embeddings, strict=True): sim = cosine_similarity(query_vec, emb) out[claim.id] = max(0.0, min(1.0, (sim + 1.0) / 2.0)) return out + def list_qdrant_sync_page(self, *, after_id: int, limit: int) -> list[Claim]: + if after_id < 0 or limit <= 0: + raise ValueError("after_id must be nonnegative and limit must be positive") + tenant_id = self._tenant_for_operation() + statuses = ("confirmed", "stale", "candidate", "conflicted") + with self.connect() as conn, conn.cursor() as cur: + cur.execute( + """ + SELECT * FROM claims + WHERE id > %s AND tenant_id = %s AND status = ANY(%s) + ORDER BY id ASC LIMIT %s + """, + (after_id, tenant_id, list(statuses), limit), + ) + rows = cur.fetchall() + return [self._row_to_claim(row) for row in rows] + + def get_qdrant_sync_cursor(self, stream_key: str) -> int: + if not stream_key or len(stream_key) > 512: + raise ValueError("invalid Qdrant sync stream key") + tenant_id = self._tenant_for_operation() + with self.connect() as conn, conn.cursor() as cur: + cur.execute( + "SELECT last_claim_id FROM qdrant_sync_state " + "WHERE stream_key = %s AND tenant_id = %s", + (stream_key, tenant_id), + ) + row = cur.fetchone() + return int(row["last_claim_id"]) if row is not None else 0 + + def set_qdrant_sync_cursor(self, stream_key: str, last_claim_id: int) -> None: + if not stream_key or len(stream_key) > 512 or last_claim_id < 0: + raise ValueError("invalid Qdrant sync cursor") + tenant_id = self._tenant_for_operation() + with self.connect() as conn, conn.cursor() as cur: + cur.execute( + """ + INSERT INTO qdrant_sync_state + (stream_key, tenant_id, last_claim_id, updated_at) + VALUES (%s, %s, %s, %s) + ON CONFLICT (stream_key) DO UPDATE SET + tenant_id = EXCLUDED.tenant_id, + last_claim_id = EXCLUDED.last_claim_id, + updated_at = EXCLUDED.updated_at + """, + (stream_key, tenant_id, last_claim_id, utc_now()), + ) + @staticmethod def _as_iso(value: object) -> str | None: if value is None: diff --git a/tests/test_embedding_efficiency.py b/tests/test_embedding_efficiency.py new file mode 100644 index 00000000..c5d67917 --- /dev/null +++ b/tests/test_embedding_efficiency.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import json +from dataclasses import replace + +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.recall.embeddings import hash_embed +from memorymaster.stores.storage import SQLiteStore + + +class _CountingProvider: + model = "counting-v1" + is_semantic = False + + def __init__(self) -> None: + self.calls: list[tuple[str, ...]] = [] + + def embed(self, text: str) -> list[float]: + self.calls.append((text,)) + return hash_embed(text, dims=16) + + def embed_batch(self, texts: list[str]) -> list[list[float]]: + self.calls.append(tuple(texts)) + return [hash_embed(text, dims=16) for text in texts] + + +def _claim(store: SQLiteStore, text: str): + return store.create_claim( + text=text, + citations=[CitationInput(source="test")], + ) + + +def test_warm_vector_scores_embed_query_once_and_do_not_rewrite_candidates(tmp_path) -> None: + store = SQLiteStore(tmp_path / "warm.db") + store.init_db() + claims = [_claim(store, "alpha memory"), _claim(store, "beta memory")] + provider = _CountingProvider() + + store.vector_scores("memory", claims, provider) + with store.connect() as conn: + before = [ + tuple(row) + for row in conn.execute( + "SELECT claim_id, model, content_hash, embedding_json, updated_at " + "FROM claim_embeddings ORDER BY claim_id" + ).fetchall() + ] + + provider.calls.clear() + store.vector_scores("memory", claims, provider) + with store.connect() as conn: + after = [ + tuple(row) + for row in conn.execute( + "SELECT claim_id, model, content_hash, embedding_json, updated_at " + "FROM claim_embeddings ORDER BY claim_id" + ).fetchall() + ] + + assert provider.calls == [("memory",)] + assert after == before + + +def test_only_changed_embedding_input_is_recomputed(tmp_path) -> None: + store = SQLiteStore(tmp_path / "stale.db") + store.init_db() + first = _claim(store, "first memory") + second = _claim(store, "second memory") + provider = _CountingProvider() + store.upsert_embeddings([first, second], provider) + provider.calls.clear() + + changed = replace(first, text="first memory changed") + assert store.upsert_embeddings([changed, second], provider) == 1 + assert provider.calls == [(store._embedding_text(changed),)] + + with store.connect() as conn: + rows = conn.execute( + "SELECT claim_id, content_hash, embedding_json FROM claim_embeddings ORDER BY claim_id" + ).fetchall() + assert len(rows) == 2 + assert all(row["content_hash"] for row in rows) + assert all(len(json.loads(row["embedding_json"])) == 16 for row in rows) + + +def test_embedding_efficiency_migration_adds_hash_and_cursor_state(tmp_path) -> None: + store = SQLiteStore(tmp_path / "migration.db") + store.init_db() + with store.connect() as conn: + columns = { + str(row[1]) for row in conn.execute("PRAGMA table_info(claim_embeddings)") + } + cursor_columns = { + str(row[1]) for row in conn.execute("PRAGMA table_info(qdrant_sync_state)") + } + version = int(conn.execute("SELECT MAX(version) FROM schema_versions").fetchone()[0]) + + assert "content_hash" in columns + assert cursor_columns == {"stream_key", "tenant_id", "last_claim_id", "updated_at"} + assert version >= 14 + + +def test_service_reuses_semantic_probe_as_the_query_embedding(tmp_path) -> None: + service = MemoryService(tmp_path / "service.db", workspace_root=tmp_path) + service.init_db() + service.ingest( + text="authoritative candidate text", + citations=[CitationInput(source="test")], + scope="project:test", + ) + provider = _CountingProvider() + provider.model = "semantic-counting-v1" + provider.is_semantic = True + service.embedding_provider = provider + + service.query_rows( + query_text="unique lookup query", + retrieval_mode="hybrid", + include_candidates=True, + scope_allowlist=["project:test"], + ) + + query_calls = [call for call in provider.calls if call == ("unique lookup query",)] + assert len(query_calls) == 1 diff --git a/tests/test_qdrant_backend.py b/tests/test_qdrant_backend.py index bdd1174b..1a5b84a7 100644 --- a/tests/test_qdrant_backend.py +++ b/tests/test_qdrant_backend.py @@ -301,12 +301,15 @@ def test_sync_all_counts(self): # Mock store store = MagicMock() claims = [_fake_claim(id=i) for i in range(3)] - store.find_by_status.return_value = claims + store.tenant_id = None + store.get_qdrant_sync_cursor.return_value = 0 + store.list_qdrant_sync_page.side_effect = [claims, []] result = backend.sync_all(store) - assert result["total"] == 12 # 3 claims * 4 statuses - assert result["synced"] == 12 + assert result["total"] == 3 + assert result["synced"] == 3 assert result["errors"] == 0 + assert store.set_qdrant_sync_cursor.call_count == 2 class TestClose: diff --git a/tests/test_qdrant_sync_pagination.py b/tests/test_qdrant_sync_pagination.py new file mode 100644 index 00000000..fd1ec3f8 --- /dev/null +++ b/tests/test_qdrant_sync_pagination.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +from memorymaster.core.models import Claim +from memorymaster.recall.qdrant_backend import QdrantBackend + + +def _claim(claim_id: int) -> Claim: + return Claim( + id=claim_id, + text=f"claim {claim_id}", + idempotency_key=None, + normalized_text=None, + claim_type="fact", + subject=None, + predicate=None, + object_value=None, + scope="project:test", + volatility="medium", + status="confirmed", + confidence=0.9, + pinned=False, + supersedes_claim_id=None, + replaced_by_claim_id=None, + created_at="2026-01-01T00:00:00+00:00", + updated_at="2026-01-01T00:00:00+00:00", + last_validated_at=None, + archived_at=None, + ) + + +class _PagedStore: + tenant_id = "tenant-a" + + def __init__(self, claims: list[Claim], cursor: int = 0) -> None: + self.claims = claims + self.cursor = cursor + self.pages: list[int] = [] + self.saved: list[int] = [] + + def get_qdrant_sync_cursor(self, stream_key: str) -> int: + assert "tenant-a" in stream_key + return self.cursor + + def list_qdrant_sync_page(self, *, after_id: int, limit: int) -> list[Claim]: + self.pages.append(after_id) + return [claim for claim in self.claims if claim.id > after_id][:limit] + + def set_qdrant_sync_cursor(self, stream_key: str, last_claim_id: int) -> None: + assert "tenant-a" in stream_key + self.cursor = last_claim_id + self.saved.append(last_claim_id) + + +def _backend() -> QdrantBackend: + backend = object.__new__(QdrantBackend) + backend.collection = "test-collection" + backend.embed_model = "test-model" + backend.ensure_collection = lambda: None + backend._embed = lambda text: [float(len(text))] + return backend + + +def test_sync_all_pages_every_authoritative_claim_and_resets_cursor() -> None: + store = _PagedStore([_claim(i) for i in range(1, 8)]) + backend = _backend() + batches: list[list[int]] = [] + backend._batch_upsert = lambda points: batches.append( + [int(point["payload"]["claim_id"]) for point in points] + ) or True + + result = backend.sync_all(store, batch_size=3) + + assert result == {"total": 7, "synced": 7, "skipped": 0, "errors": 0} + assert batches == [[1, 2, 3], [4, 5, 6], [7]] + assert store.pages == [0, 3, 6, 7] + assert store.saved == [3, 6, 7, 0] + + +def test_sync_all_failure_preserves_last_completed_page_for_replay() -> None: + store = _PagedStore([_claim(i) for i in range(1, 6)]) + backend = _backend() + outcomes = iter([True, False]) + backend._batch_upsert = lambda _points: next(outcomes) + + result = backend.sync_all(store, batch_size=2) + + assert result["synced"] == 2 + assert result["errors"] == 2 + assert store.cursor == 2 + assert store.saved == [2] + + +def test_sync_cursor_is_durable_and_authority_scoped(tmp_path) -> None: + from memorymaster.stores.storage import SQLiteStore + + store = SQLiteStore(tmp_path / "cursor.db") + store.init_db() + key_a = "collection:model:tenant-a" + key_b = "collection:model:tenant-b" + store.set_qdrant_sync_cursor(key_a, 41) + store.set_qdrant_sync_cursor(key_b, 9) + + reopened = SQLiteStore(store.db_path) + assert reopened.get_qdrant_sync_cursor(key_a) == 41 + assert reopened.get_qdrant_sync_cursor(key_b) == 9 From dc5455b8a39aa32ee184ca5488559c6383b0c4dd Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Mon, 13 Jul 2026 23:32:08 -0300 Subject: [PATCH 38/55] perf: bound query and provider costs --- .planning/REMAINING-PHASES-V1-BUDGET.md | 34 ++- .planning/audit-remediation-ledger.md | 8 +- helm/memorymaster/templates/pvc.yaml | 6 +- helm/memorymaster/values.yaml | 11 +- memorymaster/core/intake_policy.py | 27 +++ memorymaster/core/llm_provider.py | 25 ++- memorymaster/core/service.py | 49 +++++ memorymaster/core/usage_ledger.py | 197 ++++++++++++++++++ memorymaster/recall/embeddings.py | 21 ++ memorymaster/recall/recall_tokenizer.py | 17 +- memorymaster/schema.sql | 4 + memorymaster/schema_postgres.sql | 4 + memorymaster/stores/_storage_pagination.py | 121 +++++++++++ .../0015_query_storage_efficiency.py | 41 ++++ memorymaster/stores/postgres_store.py | 112 ++++++++++ memorymaster/stores/storage.py | 19 +- memorymaster/surfaces/mcp_server.py | 93 ++++++++- tests/test_deployment_contracts.py | 18 ++ tests/test_initdb_fastpath.py | 12 ++ tests/test_r32_query_storage_efficiency.py | 114 ++++++++++ tests/test_retrieval_surface_parity.py | 5 +- tests/test_usage_ledger.py | 94 +++++++++ 22 files changed, 1004 insertions(+), 28 deletions(-) create mode 100644 memorymaster/core/usage_ledger.py create mode 100644 memorymaster/stores/_storage_pagination.py create mode 100644 memorymaster/stores/migrations/0015_query_storage_efficiency.py create mode 100644 tests/test_r32_query_storage_efficiency.py create mode 100644 tests/test_usage_ledger.py diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index 4af23bea..5ac1c0db 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -310,5 +310,35 @@ R3.1 embedding and reconciliation efficiency completed on 2026-07-13: `BLOCKED-EXTERNAL`; no production semantic profile was enabled. - Atomic package commit: the conventional R3.1 commit containing this evidence. -The next package is R3.2 query and storage efficiency. The active roadmap goal -authorizes continuing without a new goal. +R3.2 query and storage efficiency completed on 2026-07-13: + +- Immutable migration 0015 and both baseline schemas add event-type/time and + event-type/details/time indexes. SQLite EXPLAIN selects the ordered index; + disposable Postgres runtime evidence remains part of the existing external + backend gate. +- Recall corpus/alias statistics are generation-keyed instead of cached forever. + MCP result limits are finite, list claims/events expose opaque keyset cursors, + and retrieval v2 rows reference the separately serialized claim array rather + than duplicating full claim payloads. +- A shared WAL-backed usage ledger provides atomic global/provider/actor + reservations across LLM primary/fallback calls, Gemini embeddings, MCP + ingest/checkpoint, and explicitly configured core intake quotas. Unlimited + defaults remain backward compatible. +- The init fast-path stamp now covers legacy ensure-helper source. Helm PVC + capacity is validated against the finite verbatim/database/artifact/WAL, + backup, and operator-headroom envelope instead of a fixed 1Gi request. +- RED evidence: the initial focused contract failed collection on the missing + limit helper and the durable-ledger contract failed on the missing module. + The first package boundary then found five migration-runner failures because + migration 0015 assumed a pre-existing events table. The bounded correction + makes the migration a no-op on schema-less runner fixtures. +- Focused boundary: 265 passed, 45 skipped, 2 expected xfails, 5 failed; exact + correction: 21 passed. Supporting focused slices: 132 durable quota/provider/ + intake/embedding tests passed; 82 tokenizer/MCP/fast-path tests passed. + Changed-file Ruff and `git diff --check` passed. Helm CLI was unavailable, so + chart validation is static at this package boundary and will be retried at + Phase 3 convergence. +- Atomic package commit: the conventional R3.2 commit containing this evidence. + +The next package is R3.3 setup-profile truth. The active roadmap goal authorizes +continuing without a new goal. diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index db1ea12d..575e2ed5 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -25,9 +25,9 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-OPS-03 | DevOps/Maintainability | High / BAD-PRACTICE | A release tag can publish without a blocking verified test artifact | R4.4 | OPEN | Deliberately failing release candidate cannot publish | | MM-OPS-04 | Security/DevOps | High / EXPLOITABLE-LOW-EFFORT | Qdrant/Ollama ports are broadly exposed; images are mutable | R1.5,R3.4 | IN-PROGRESS | Phase 1 defaults are repository-complete (`a858419`, `9b3e16c`, `b71e18f`): loopback-only ports, required immutable digests, authenticated TLS Qdrant, redirect-safe credentials, and fail-closed supply-chain policy. Real image scans/runtime health remain `BLOCKED-EXTERNAL`; R3.4 service-entrypoint work remains backlog. | | MM-PERF-01 | Performance/Cost | High / BAD-PRACTICE | Hybrid reads recompute and rewrite candidate embeddings | R3.1 | RESOLVED | R3.1 migration 0014 persists model/content fingerprints; focused warm-query evidence proves one query embed and zero candidate embeds/writes. | -| MM-PERF-02 | Performance | Medium / BAD-PRACTICE | Each process cold-scans corpus token statistics and caches forever | R3.2 | OPEN | Generation-aware/token-specific stats benchmark | +| MM-PERF-02 | Performance | Medium / BAD-PRACTICE | Each process cold-scans corpus token statistics and caches forever | R3.2 | RESOLVED | R3.2 keys corpus and alias statistics by the persisted corpus generation, so warm reads reuse one scan and every claim mutation invalidates it. Focused evidence proves a new cache miss after generation change while tokenizer ranking/retired-status contracts remain green. | | MM-PERF-03 | Performance/Cost | High / EXPLOITABLE-NOW | Qdrant reconciliation repeats/truncates tens of thousands of embeddings | R3.1 | RESOLVED | R3.1 replaces four truncated 10,000-row scans with authority-scoped keyset pages and a replayable durable cursor; fake-backend convergence/failure-resume evidence passed. Real authenticated/TLS Qdrant remains externally blocked. | -| MM-PERF-04 | Database/Performance | Medium / BAD-PRACTICE | Million-row event queries lack event-type composite indexes | R3.2 | OPEN | Versioned parity migration plus EXPLAIN/timing evidence | +| MM-PERF-04 | Database/Performance | Medium / BAD-PRACTICE | Million-row event queries lack event-type composite indexes | R3.2 | RESOLVED | Immutable migration 0015 and both baseline schemas add `(event_type, created_at DESC, id DESC)` and `(event_type, details, created_at DESC)` indexes. SQLite EXPLAIN selects the ordered type index; Postgres DDL parity is covered statically pending the already-recorded disposable-Postgres external gate. | | MM-UX-01 | UX/Maintainability | High / BAD-PRACTICE | Setup reports success without verifying the requested memory loop | R3.3 | OPEN | Component-level profile verification and nonzero failures | | MM-UX-02 | UX/Reliability | High / BAD-PRACTICE | Dashboard removes review rows before mutation success | R4.3 | OPEN | Browser failure/retry evidence; row remains on rejected POST | | MM-UX-03 | UX | Medium / BAD-PRACTICE | Dashboard hides failures and has labeling, contrast, responsive, and evidence-discovery gaps | R4.3 | OPEN | Browser/a11y/mobile acceptance suite | @@ -35,14 +35,14 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-PRIV-02 | Compliance | High / BAD-PRACTICE | No complete export, erasure, or retention workflow across copies | R3.5 | OPEN | Privacy export/erase dry-run manifest across all stores | | MM-COST-01 | Cost | High / EXPLOITABLE-NOW | Default Stop-hook LLM calls have no finite persisted budget | R2.4 | RESOLVED | P2-E (`a1d5477`) adds restart-safe atomic global/provider/session reservations and persisted usage; maximum default candidate inflow is 600/day against the 688/day ceiling. | | MM-COST-02 | Cost | High / EXPLOITABLE-NOW | Verbatim capture reprocesses sessions and retains data indefinitely | R2.4 | RESOLVED | P2-E persists complete-line cursors, processes appended content only, and enforces dry-run 30-day/512-MiB/75,000-session retention bounds. | -| MM-COST-03 | Cost/Reliability | Medium / BAD-PRACTICE | Cost and intake controls are fragmented/process-local | R2.4,R3.2 | IN-PROGRESS | Capture-hook budgets/cursors are durable after P2-E, but core LLM, embedding, provider, and MCP quota accounting is not yet one account-wide durable ledger. Continue in R3.2. | +| MM-COST-03 | Cost/Reliability | Medium / BAD-PRACTICE | Cost and intake controls are fragmented/process-local | R2.4,R3.2 | RESOLVED | R3.2 adds one WAL-backed atomic usage ledger shared by LLM primary/fallback calls, Gemini embedding batches, MCP ingest/checkpoint, and configured core intake quotas. Caps remain backward-compatible/unlimited unless configured; restart/cross-instance and local-bucket-reset tests prove durable global/provider/actor enforcement without consuming capacity on rejected reservations. | | MM-DEMO-01 | Demo/Integrity | High / EXPLOITABLE-NOW | Atlas defaults persist fabricated mock evidence at high confidence | R2.5 | RESOLVED | P2-F (`c66e240`) requires a real configured production provider, confines synthetic providers to explicit dev/test opt-in, excludes synthetic evidence from claims/prompts/actions/citations/exports, and fails configuration nonzero. Focused gate: 111 passed. | | MM-MAINT-01 | Maintainability/Integrity | Medium / BAD-PRACTICE | Versions, tool counts, roadmaps, and install claims drift | R4.4 | OPEN | Generated single-source values checked by CI | | MM-MAINT-02 | Architecture/Maintainability | Medium / BAD-PRACTICE | Oversized facades, hardwired extensions, dead plugin seam, and lingering shims | R4.1-R4.2 | OPEN | Measured boundary/size budgets and supported shim policy | | MM-REL-03 | Reliability | Medium / BAD-PRACTICE | Media retry rows have no expired-lease recovery | R3.5 | OPEN | Worker-death lease reclaim test | | MM-OPS-05 | Missing Operations | High / BAD-PRACTICE | Recovery defaults stop at same-machine SQLite snapshots | R3.5 | BLOCKED-EXTERNAL | Off-device SQLite/Postgres restore drill and RPO/RTO | | MM-OBS-01 | Missing Operations | High / BAD-PRACTICE | Failures/metrics are process-local without central alert ownership | R3.5 | OPEN | Persistent metrics, trace/error capture, alert tests/runbook | -| MM-DB-01 | Database | Medium / BAD-PRACTICE | Fast schema fingerprint can omit legacy ensure-helper changes | R3.2 | OPEN | All DDL versioned or fingerprint covers every schema source | +| MM-DB-01 | Database | Medium / BAD-PRACTICE | Fast schema fingerprint can omit legacy ensure-helper changes | R3.2 | RESOLVED | `schema_stamp()` now hashes baseline schema, immutable migrations, and the complete legacy ensure-helper source. A helper-body mutation regression changes the stamp and forces the full initialization path. New R3.2 DDL exists only in migration 0015/baselines. | | MM-INTEGRITY-01 | Code Integrity | Medium / BAD-PRACTICE | `importlib.util` probe is wrong in clean Python and its test masks failure | R4.4 | OPEN | Clean subprocess test and explicit import | | MM-TEST-01 | Maintainability/Reliability | Medium / BAD-PRACTICE | Validator candidate winner changed under load because mutable `updated_at` controlled processing order | Phase 0/R4.4 | RESOLVED | Deterministic timestamp-inversion regression passes; targeted lifecycle suite 35 passed; full isolated non-ML gate 3,094 passed, 56 skipped, 95 deselected, 24 intentional xfails | diff --git a/helm/memorymaster/templates/pvc.yaml b/helm/memorymaster/templates/pvc.yaml index 9f4c4f61..8e4707ec 100644 --- a/helm/memorymaster/templates/pvc.yaml +++ b/helm/memorymaster/templates/pvc.yaml @@ -1,4 +1,8 @@ {{- if .Values.persistence.enabled }} +{{- $requiredMi := add .Values.retentionCapacity.verbatimMaxMi .Values.retentionCapacity.databaseMaxMi .Values.retentionCapacity.artifactsMaxMi .Values.retentionCapacity.walAndBackupHeadroomMi .Values.retentionCapacity.operatorHeadroomMi }} +{{- if lt (int .Values.persistence.sizeMi) (int $requiredMi) }} +{{- fail (printf "persistence.sizeMi=%v is below the retention envelope of %vMi" .Values.persistence.sizeMi $requiredMi) }} +{{- end }} apiVersion: v1 kind: PersistentVolumeClaim metadata: @@ -8,7 +12,7 @@ spec: - ReadWriteOnce resources: requests: - storage: {{ .Values.persistence.size }} + storage: {{ .Values.persistence.sizeMi }}Mi {{- if .Values.persistence.storageClass }} storageClassName: {{ .Values.persistence.storageClass }} {{- end }} diff --git a/helm/memorymaster/values.yaml b/helm/memorymaster/values.yaml index cfd8b379..88a4cf79 100644 --- a/helm/memorymaster/values.yaml +++ b/helm/memorymaster/values.yaml @@ -12,9 +12,18 @@ service: persistence: enabled: true - size: 1Gi + # PVC capacity is validated against the finite retention envelope below. + # Raise sizeMi when increasing any retained-data budget. + sizeMi: 2048 storageClass: "" +retentionCapacity: + verbatimMaxMi: 512 + databaseMaxMi: 512 + artifactsMaxMi: 256 + walAndBackupHeadroomMi: 512 + operatorHeadroomMi: 256 + env: MEMORYMASTER_DEFAULT_DB: /data/memorymaster.db MEMORYMASTER_WORKSPACE: /data diff --git a/memorymaster/core/intake_policy.py b/memorymaster/core/intake_policy.py index cede58f8..30d60077 100644 --- a/memorymaster/core/intake_policy.py +++ b/memorymaster/core/intake_policy.py @@ -329,6 +329,33 @@ def evaluate_intake( quota = cfg.quota_per_agent_per_window if quota and quota > 0 and effective_agent not in cfg.quota_exempt_agents: wkey = _window_key(cfg.quota_window, moment) + from memorymaster.core.usage_ledger import ( + UsageQuotaExceeded, + reserve_intake_configured, + ) + + try: + durable = reserve_intake_configured( + actor=effective_agent, + units=1, + actor_limit=quota, + window_key=wkey, + ) + except UsageQuotaExceeded: + if intake_batch_id: + with _STATE_LOCK: + if intake_batch_id in _BATCH_COUNTS: + _BATCH_COUNTS[intake_batch_id] = max( + 0, _BATCH_COUNTS[intake_batch_id] - 1 + ) + raise IntakeRejected( + f"source_agent '{effective_agent}' exceeded its durable quota " + f"of {quota} claims per {cfg.quota_window}.", + rule="quota", + reason="quota_exceeded", + ) from None + if durable: + return IntakeDecision(accept=True, mutated_fields=mutated) with _STATE_LOCK: current_window, count = _QUOTA_COUNTS.get(effective_agent, (wkey, 0)) if current_window != wkey: diff --git a/memorymaster/core/llm_provider.py b/memorymaster/core/llm_provider.py index 9f827fac..0753e5a3 100644 --- a/memorymaster/core/llm_provider.py +++ b/memorymaster/core/llm_provider.py @@ -697,8 +697,21 @@ def call_llm(prompt: str, text: str) -> str: # of silently overspending. When no scope is active, this is a no-op # — preserves backwards-compat for callers outside run_cycle/wiki/daydream. llm_budget.check_before_call(primary_name) + from memorymaster.core.usage_ledger import UsageQuotaExceeded, reserve_configured - primary_response = primary_fn(prompt, text) + try: + primary_usage = reserve_configured( + operation="llm", provider=primary_name, actor="account" + ) + except UsageQuotaExceeded as exc: + raise llm_budget.LLMBudgetExceeded( + "durable_daily_exhausted", primary_name + ) from exc + try: + primary_response = primary_fn(prompt, text) + finally: + if primary_usage is not None: + primary_usage[0].finish(primary_usage[1], outcome="attempted") llm_budget.record_call( primary_name, tokens=llm_budget.estimate_tokens(prompt, text, primary_response), @@ -731,6 +744,14 @@ def call_llm(prompt: str, text: str) -> str: # Budget gate for the fallback provider too. llm_budget.check_before_call(fallback_name) + try: + fallback_usage = reserve_configured( + operation="llm", provider=fallback_name, actor="account" + ) + except UsageQuotaExceeded as exc: + raise llm_budget.LLMBudgetExceeded( + "durable_daily_exhausted", fallback_name + ) from exc # Override MEMORYMASTER_LLM_MODEL to the fallback model for the duration of # the call WITHOUT mutating os.environ — a ContextVar override keeps the swap @@ -751,6 +772,8 @@ def call_llm(prompt: str, text: str) -> str: ) finally: _ENV_OVERRIDES.reset(token) + if fallback_usage is not None: + fallback_usage[0].finish(fallback_usage[1], outcome="attempted") if fallback_response and not _looks_like_quota_error(fallback_response): return fallback_response diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index a87bf97e..796a0ce4 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -2123,6 +2123,40 @@ def list_claims( claims = [claim for claim in claims if not is_sensitive_claim(claim)] return claims + def list_claims_page( + self, + *, + status: str | None = None, + limit: int = 50, + cursor: str = "", + include_archived: bool = False, + allow_sensitive: bool = False, + holder: str | None = None, + scope_allowlist: list[str] | None = None, + requesting_agent: str | None = None, + ) -> tuple[list[Claim], str]: + requesting_agent = self._effective_requesting_agent(requesting_agent) + scope_allowlist = self._effective_scope_allowlist(scope_allowlist) + include_sensitive = self._allow_sensitive( + allow_sensitive=allow_sensitive, + context="service.list_claims_page", + deny_mode="filter", + ) + claims, next_cursor = self.store.list_claims_page( + status=status, + limit=limit, + cursor=cursor, + include_archived=include_archived, + include_citations=True, + scope_allowlist=scope_allowlist, + tenant_id=self.tenant_id, + holder=holder, + ) + claims = _filter_agent_visibility(claims, requesting_agent) + if not include_sensitive: + claims = [claim for claim in claims if not is_sensitive_claim(claim)] + return claims, next_cursor + def redact_claim_payload( self, claim_id: int, @@ -2165,6 +2199,21 @@ def list_events( event_type=event_type, ) + def list_events_page( + self, + *, + claim_id: int | None = None, + limit: int = 100, + event_type: str | None = None, + cursor: str = "", + ) -> tuple[list[Event], str]: + return self.store.list_events_page( + claim_id=claim_id, + limit=limit, + event_type=event_type, + cursor=cursor, + ) + def upsert_external_source( self, *, diff --git a/memorymaster/core/usage_ledger.py b/memorymaster/core/usage_ledger.py new file mode 100644 index 00000000..46508178 --- /dev/null +++ b/memorymaster/core/usage_ledger.py @@ -0,0 +1,197 @@ +"""Shared durable quota reservations for paid calls and external intake.""" + +from __future__ import annotations + +import hashlib +import os +import uuid +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path + +from memorymaster.stores._storage_shared import open_conn + + +class UsageQuotaExceeded(RuntimeError): + def __init__(self, partition: str, limit: int) -> None: + self.partition = partition + self.limit = limit + super().__init__(f"durable quota exhausted: partition={partition} limit={limit}") + + +@dataclass(frozen=True, slots=True) +class UsageReservation: + reservation_id: str + units: int + + +_SCHEMA = """ +CREATE TABLE IF NOT EXISTS usage_reservations ( + reservation_id TEXT PRIMARY KEY, + window_key TEXT NOT NULL, + operation TEXT NOT NULL, + provider TEXT NOT NULL, + actor_hash TEXT NOT NULL, + units INTEGER NOT NULL CHECK (units > 0), + outcome TEXT NOT NULL DEFAULT 'reserved', + created_at TEXT NOT NULL, + finished_at TEXT +); +CREATE INDEX IF NOT EXISTS idx_usage_window_operation + ON usage_reservations(window_key, operation); +CREATE INDEX IF NOT EXISTS idx_usage_window_provider + ON usage_reservations(window_key, operation, provider); +CREATE INDEX IF NOT EXISTS idx_usage_window_actor + ON usage_reservations(window_key, operation, actor_hash); +""" + + +def _actor_hash(actor: str) -> str: + return hashlib.sha256((actor or "unknown").encode("utf-8")).hexdigest() + + +def _env_int(name: str) -> int: + try: + return max(0, int(os.environ.get(name, "0"))) + except ValueError: + return 0 + + +class UsageLedger: + def __init__(self, db_path: str | Path) -> None: + self.db_path = Path(db_path) + self.db_path.parent.mkdir(parents=True, exist_ok=True) + with open_conn(self.db_path) as conn: + conn.executescript(_SCHEMA) + + def reserve( + self, + *, + operation: str, + provider: str, + actor: str, + units: int = 1, + global_limit: int = 0, + provider_limit: int = 0, + actor_limit: int = 0, + window_key: str | None = None, + ) -> UsageReservation: + units = max(1, int(units)) + window = window_key or datetime.now(timezone.utc).date().isoformat() + provider = (provider or "unknown").strip().lower() + actor_digest = _actor_hash(actor) + conn = open_conn(self.db_path) + try: + conn.execute("BEGIN IMMEDIATE") + partitions = ( + ("global", global_limit, "", (window, operation)), + ("provider", provider_limit, " AND provider=?", (window, operation, provider)), + ("actor", actor_limit, " AND actor_hash=?", (window, operation, actor_digest)), + ) + for name, limit, suffix, params in partitions: + if limit <= 0: + continue + current = int( + conn.execute( + "SELECT COALESCE(SUM(units), 0) FROM usage_reservations " + f"WHERE window_key=? AND operation=?{suffix}", + params, + ).fetchone()[0] + ) + if current + units > limit: + conn.rollback() + raise UsageQuotaExceeded(name, limit) + reservation = UsageReservation(uuid.uuid4().hex, units) + conn.execute( + "INSERT INTO usage_reservations " + "(reservation_id, window_key, operation, provider, actor_hash, units, created_at) " + "VALUES (?, ?, ?, ?, ?, ?, ?)", + ( + reservation.reservation_id, + window, + operation, + provider, + actor_digest, + units, + datetime.now(timezone.utc).isoformat(), + ), + ) + conn.commit() + return reservation + finally: + conn.close() + + def finish(self, reservation: UsageReservation, *, outcome: str) -> None: + with open_conn(self.db_path) as conn: + conn.execute( + "UPDATE usage_reservations SET outcome=?, finished_at=? " + "WHERE reservation_id=?", + ( + outcome, + datetime.now(timezone.utc).isoformat(), + reservation.reservation_id, + ), + ) + + +def configured_ledger() -> UsageLedger | None: + configured = os.environ.get("MEMORYMASTER_USAGE_LEDGER_DB", "").strip() + cap_names = ( + "MEMORYMASTER_MAX_LLM_CALLS_PER_DAY", + "MEMORYMASTER_MAX_PROVIDER_CALLS_PER_DAY", + "MEMORYMASTER_MAX_EMBEDDING_ITEMS_PER_DAY", + "MEMORYMASTER_MAX_MCP_INGESTS_PER_DAY", + "MEMORYMASTER_MAX_MCP_INGESTS_PER_AGENT_PER_DAY", + ) + if not configured and not any(_env_int(name) for name in cap_names): + return None + path = Path(configured) if configured else Path.home() / ".memorymaster" / "usage-ledger.db" + return UsageLedger(path) + + +def reserve_configured( + *, operation: str, provider: str, actor: str, units: int = 1 +) -> tuple[UsageLedger, UsageReservation] | None: + ledger = configured_ledger() + if ledger is None: + return None + if operation == "llm": + global_limit = _env_int("MEMORYMASTER_MAX_LLM_CALLS_PER_DAY") + provider_limit = _env_int("MEMORYMASTER_MAX_PROVIDER_CALLS_PER_DAY") + actor_limit = 0 + elif operation == "embedding": + global_limit = _env_int("MEMORYMASTER_MAX_EMBEDDING_ITEMS_PER_DAY") + provider_limit = global_limit + actor_limit = 0 + else: + global_limit = _env_int("MEMORYMASTER_MAX_MCP_INGESTS_PER_DAY") + provider_limit = 0 + actor_limit = _env_int("MEMORYMASTER_MAX_MCP_INGESTS_PER_AGENT_PER_DAY") + reservation = ledger.reserve( + operation=operation, + provider=provider, + actor=actor, + units=units, + global_limit=global_limit, + provider_limit=provider_limit, + actor_limit=actor_limit, + ) + return ledger, reservation + + +def reserve_intake_configured( + *, actor: str, units: int, actor_limit: int, window_key: str +) -> bool: + """Use the shared ledger for intake when an explicit ledger is configured.""" + configured = os.environ.get("MEMORYMASTER_USAGE_LEDGER_DB", "").strip() + if not configured: + return False + UsageLedger(configured).reserve( + operation="ingest", + provider="core", + actor=actor, + units=units, + actor_limit=actor_limit, + window_key=window_key, + ) + return True diff --git a/memorymaster/recall/embeddings.py b/memorymaster/recall/embeddings.py index 055afba9..4f062479 100644 --- a/memorymaster/recall/embeddings.py +++ b/memorymaster/recall/embeddings.py @@ -71,7 +71,13 @@ def _gemini_embed(self, text: str) -> list[float]: if self._transformer is None: self._transformer = _load_gemini_client() gemini_model = self.model.split(":", 1)[1] + usage = None try: + from memorymaster.core.usage_ledger import reserve_configured + + usage = reserve_configured( + operation="embedding", provider="gemini", actor="account" + ) result = self._transformer.models.embed_content( model=gemini_model, contents=text, @@ -90,6 +96,9 @@ def _gemini_embed(self, text: str) -> list[float]: self.dims = 1536 self.degraded = True return hash_embed(text, dims=self.dims) + finally: + if usage is not None: + usage[0].finish(usage[1], outcome="attempted") vec = result.embeddings[0].values self.dims = len(vec) return normalize(list(vec)) @@ -98,7 +107,16 @@ def _gemini_embed_batch(self, texts: list[str]) -> list[list[float]]: if self._transformer is None: self._transformer = _load_gemini_client() gemini_model = self.model.split(":", 1)[1] + usage = None try: + from memorymaster.core.usage_ledger import reserve_configured + + usage = reserve_configured( + operation="embedding", + provider="gemini", + actor="account", + units=len(texts), + ) result = self._transformer.models.embed_content( model=gemini_model, contents=texts, @@ -112,6 +130,9 @@ def _gemini_embed_batch(self, texts: list[str]) -> list[list[float]]: self.dims = 1536 self.degraded = True return [hash_embed(text, dims=self.dims) for text in texts] + finally: + if usage is not None: + usage[0].finish(usage[1], outcome="attempted") vectors = [normalize(list(item.values)) for item in result.embeddings] if len(vectors) != len(texts): raise ValueError("embedding provider returned an incomplete batch") diff --git a/memorymaster/recall/recall_tokenizer.py b/memorymaster/recall/recall_tokenizer.py index fe4f662c..ab9b071c 100644 --- a/memorymaster/recall/recall_tokenizer.py +++ b/memorymaster/recall/recall_tokenizer.py @@ -179,8 +179,10 @@ def _stem(tok: str) -> str | None: _NON_CANONICAL_STATUSES = ("archived", "superseded") -@lru_cache(maxsize=8) -def _corpus_stats(db_path: str) -> tuple[int, dict[str, int]]: +@lru_cache(maxsize=16) +def _corpus_stats( + db_path: str, generation: int | None = None +) -> tuple[int, dict[str, int]]: """Return (total_docs, {token: doc_frequency}) for IDF. Read-only. Only canonical claims (everything except ``archived``/``superseded``) @@ -222,8 +224,8 @@ def _corpus_stats(db_path: str) -> tuple[int, dict[str, int]]: conn.close() -@lru_cache(maxsize=8) -def _alias_set(db_path: str) -> frozenset[str]: +@lru_cache(maxsize=16) +def _alias_set(db_path: str, generation: int | None = None) -> frozenset[str]: """Return lowercased entity aliases. Empty if table missing.""" try: conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True) @@ -279,8 +281,11 @@ def extract_query_tokens(raw_prompt: str, db_path: str, max_tokens: int = 6) -> if not tokens: return "" - total_docs, df = _corpus_stats(db_path) - aliases = _alias_set(db_path) + from memorymaster.recall.query_cache import read_generation + + generation = read_generation(db_path) + total_docs, df = _corpus_stats(db_path, generation) + aliases = _alias_set(db_path, generation) # Resolve each unique token to the best-matching corpus form (stem # or synonym fallback). Preserve first-seen order. diff --git a/memorymaster/schema.sql b/memorymaster/schema.sql index 9f54617c..b32a3470 100644 --- a/memorymaster/schema.sql +++ b/memorymaster/schema.sql @@ -209,6 +209,10 @@ CREATE INDEX IF NOT EXISTS idx_events_tenant_head CREATE INDEX IF NOT EXISTS idx_events_tenant_algo_head ON events(tenant_id, hash_algo, id DESC); CREATE INDEX IF NOT EXISTS idx_events_created_at ON events(created_at); +CREATE INDEX IF NOT EXISTS idx_events_type_created_id + ON events(event_type, created_at DESC, id DESC); +CREATE INDEX IF NOT EXISTS idx_events_type_details_created + ON events(event_type, details, created_at DESC); CREATE INDEX IF NOT EXISTS idx_embeddings_updated_at ON claim_embeddings(updated_at); CREATE INDEX IF NOT EXISTS idx_claims_human_id ON claims(human_id); CREATE UNIQUE INDEX IF NOT EXISTS idx_claims_public_human_id_unique diff --git a/memorymaster/schema_postgres.sql b/memorymaster/schema_postgres.sql index f2129ca9..4abc6b6f 100644 --- a/memorymaster/schema_postgres.sql +++ b/memorymaster/schema_postgres.sql @@ -236,6 +236,10 @@ CREATE INDEX IF NOT EXISTS idx_events_tenant_head CREATE INDEX IF NOT EXISTS idx_events_tenant_algo_head ON events(tenant_id, hash_algo, id DESC); CREATE INDEX IF NOT EXISTS idx_events_created_at ON events(created_at); +CREATE INDEX IF NOT EXISTS idx_events_type_created_id + ON events(event_type, created_at DESC, id DESC); +CREATE INDEX IF NOT EXISTS idx_events_type_details_created + ON events(event_type, details, created_at DESC); CREATE TABLE IF NOT EXISTS external_sources ( id BIGSERIAL PRIMARY KEY, diff --git a/memorymaster/stores/_storage_pagination.py b/memorymaster/stores/_storage_pagination.py new file mode 100644 index 00000000..a46c2ebb --- /dev/null +++ b/memorymaster/stores/_storage_pagination.py @@ -0,0 +1,121 @@ +from __future__ import annotations + +import base64 +import json +from typing import Any + + +def _encode_cursor(values: list[object]) -> str: + raw = json.dumps(values, separators=(",", ":"), ensure_ascii=True).encode() + return base64.urlsafe_b64encode(raw).decode().rstrip("=") + + +def _decode_cursor(cursor: str, size: int) -> list[Any]: + if not cursor: + return [] + try: + raw = base64.urlsafe_b64decode(cursor + "=" * (-len(cursor) % 4)) + values = json.loads(raw) + except (ValueError, TypeError, json.JSONDecodeError) as exc: + raise ValueError("invalid pagination cursor") from exc + if not isinstance(values, list) or len(values) != size: + raise ValueError("invalid pagination cursor") + return values + + +class _PaginationMixin: + def list_claims_page( + self, + *, + limit: int, + cursor: str = "", + status: str | None = None, + include_archived: bool = False, + include_citations: bool = False, + scope_allowlist: list[str] | None = None, + tenant_id: str | None = None, + holder: str | None = None, + ): + clauses, params = self._build_list_clauses( + status, None, include_archived, scope_allowlist, tenant_id, holder + ) + after = _decode_cursor(cursor, 4) + if after: + pinned, confidence, updated_at, claim_id = after + clauses.append( + "(pinned < ? OR (pinned = ? AND confidence < ?) OR " + "(pinned = ? AND confidence = ? AND updated_at < ?) OR " + "(pinned = ? AND confidence = ? AND updated_at = ? AND id < ?))" + ) + params.extend( + [ + pinned, + pinned, + confidence, + pinned, + confidence, + updated_at, + pinned, + confidence, + updated_at, + claim_id, + ] + ) + where_sql = f"WHERE {' AND '.join(clauses)}" if clauses else "" + params.append(max(1, int(limit)) + 1) + sql = ( + f"SELECT * FROM claims {where_sql} " + "ORDER BY pinned DESC, confidence DESC, updated_at DESC, id DESC LIMIT ?" + ) + with self.connect() as conn: + rows = conn.execute(sql, params).fetchall() + has_more = len(rows) > limit + rows = rows[:limit] + claims = [self._row_to_claim(row) for row in rows] + if include_citations and claims: + citation_map = self.list_citations_batch([claim.id for claim in claims]) + for claim in claims: + claim.citations = citation_map.get(claim.id, []) + next_cursor = "" + if has_more and rows: + row = rows[-1] + next_cursor = _encode_cursor( + [int(row["pinned"]), float(row["confidence"]), row["updated_at"], int(row["id"])] + ) + return claims, next_cursor + + def list_events_page( + self, + *, + limit: int, + cursor: str = "", + claim_id: int | None = None, + event_type: str | None = None, + ): + clauses: list[str] = [] + params: list[object] = [] + if claim_id is not None: + clauses.append("claim_id = ?") + params.append(claim_id) + if event_type is not None: + clauses.append("event_type = ?") + params.append(event_type) + after = _decode_cursor(cursor, 2) + if after: + clauses.append("(created_at < ? OR (created_at = ? AND id < ?))") + params.extend([after[0], after[0], after[1]]) + where_sql = f"WHERE {' AND '.join(clauses)}" if clauses else "" + params.append(max(1, int(limit)) + 1) + with self.connect() as conn: + rows = conn.execute( + f"SELECT * FROM events {where_sql} " + "ORDER BY created_at DESC, id DESC LIMIT ?", + params, + ).fetchall() + has_more = len(rows) > limit + rows = rows[:limit] + events = [self._row_to_event(row) for row in rows] + next_cursor = "" + if has_more and rows: + next_cursor = _encode_cursor([rows[-1]["created_at"], int(rows[-1]["id"])]) + return events, next_cursor diff --git a/memorymaster/stores/migrations/0015_query_storage_efficiency.py b/memorymaster/stores/migrations/0015_query_storage_efficiency.py new file mode 100644 index 00000000..12135601 --- /dev/null +++ b/memorymaster/stores/migrations/0015_query_storage_efficiency.py @@ -0,0 +1,41 @@ +"""Indexes for the event query shapes used by governance and dashboards.""" + +from __future__ import annotations + +VERSION = 15 +DESCRIPTION = "Event query indexes for type, details, and time ordering" + + +def apply_sqlite(conn) -> None: + exists = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='events'" + ).fetchone() + if not exists: + conn.commit() + return + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_events_type_created_id " + "ON events(event_type, created_at DESC, id DESC)" + ) + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_events_type_details_created " + "ON events(event_type, details, created_at DESC)" + ) + conn.commit() + + +def apply_postgres(conn) -> None: + with conn.cursor() as cur: + cur.execute( + """ + DO $$ BEGIN + IF to_regclass('public.events') IS NOT NULL THEN + CREATE INDEX IF NOT EXISTS idx_events_type_created_id + ON events(event_type, created_at DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_events_type_details_created + ON events(event_type, details, created_at DESC); + END IF; + END $$ + """ + ) + conn.commit() diff --git a/memorymaster/stores/postgres_store.py b/memorymaster/stores/postgres_store.py index a16728a4..95ad853a 100644 --- a/memorymaster/stores/postgres_store.py +++ b/memorymaster/stores/postgres_store.py @@ -47,6 +47,7 @@ compute_tenant_event_hash, generate_top_level_human_id, ) +from memorymaster.stores._storage_pagination import _decode_cursor, _encode_cursor from memorymaster.stores.claim_identity import ( normalize_claim_identity, require_unambiguous_identity_row, @@ -2091,6 +2092,117 @@ def list_events( rows = cur.fetchall() return [self._row_to_event(row) for row in rows] + def list_claims_page( + self, + *, + limit: int, + cursor: str = "", + status: str | None = None, + include_archived: bool = False, + include_citations: bool = False, + scope_allowlist: list[str] | None = None, + tenant_id: str | None = None, + holder: str | None = None, + ): + clauses: list[str] = [] + params: list[object] = [] + effective_tenant = tenant_id if tenant_id is not None else self.tenant_id + if effective_tenant is not None: + clauses.append("tenant_id IS NOT DISTINCT FROM %s") + params.append(effective_tenant) + if holder and holder.strip(): + clauses.append("holder = %s") + params.append(holder.strip()) + if status is not None: + clauses.append("status = %s") + params.append(status) + elif not include_archived: + clauses.append("status <> 'archived'") + if scope_allowlist: + scopes = [scope.strip() for scope in scope_allowlist if scope.strip()] + if scopes: + clauses.append(f"scope IN ({','.join('%s' for _ in scopes)})") + params.extend(scopes) + after = _decode_cursor(cursor, 4) + if after: + pinned, confidence, updated_at, claim_id = after + clauses.append( + "(pinned < %s OR (pinned = %s AND confidence < %s) OR " + "(pinned = %s AND confidence = %s AND updated_at < %s) OR " + "(pinned = %s AND confidence = %s AND updated_at = %s AND id < %s))" + ) + params.extend( + [pinned, pinned, confidence, pinned, confidence, updated_at, + pinned, confidence, updated_at, claim_id] + ) + where_sql = f"WHERE {' AND '.join(clauses)}" if clauses else "" + params.append(max(1, int(limit)) + 1) + with self.connect() as conn, conn.cursor() as cur: + cur.execute( + f"SELECT * FROM claims {where_sql} " + "ORDER BY pinned DESC, confidence DESC, updated_at DESC, id DESC LIMIT %s", + params, + ) + rows = cur.fetchall() + has_more = len(rows) > limit + rows = rows[:limit] + claims = [self._row_to_claim(row) for row in rows] + if include_citations: + for claim in claims: + claim.citations = self.list_citations(claim.id) + next_cursor = "" + if has_more and rows: + row = rows[-1] + updated = row["updated_at"] + next_cursor = _encode_cursor( + [bool(row["pinned"]), float(row["confidence"]), + updated.isoformat() if hasattr(updated, "isoformat") else str(updated), int(row["id"])] + ) + return claims, next_cursor + + def list_events_page( + self, + *, + limit: int, + cursor: str = "", + claim_id: int | None = None, + event_type: str | None = None, + ): + clauses: list[str] = [] + params: list[object] = [] + if self.tenant_id is not None: + clauses.append("tenant_id IS NOT DISTINCT FROM %s") + params.append(self._tenant_for_operation()) + if claim_id is not None: + clauses.append("claim_id = %s") + params.append(claim_id) + if event_type is not None: + clauses.append("event_type = %s") + params.append(event_type) + after = _decode_cursor(cursor, 2) + if after: + clauses.append("(created_at < %s OR (created_at = %s AND id < %s))") + params.extend([after[0], after[0], after[1]]) + where_sql = f"WHERE {' AND '.join(clauses)}" if clauses else "" + params.append(max(1, int(limit)) + 1) + with self.connect() as conn, conn.cursor() as cur: + cur.execute( + f"SELECT * FROM events {where_sql} " + "ORDER BY created_at DESC, id DESC LIMIT %s", + params, + ) + rows = cur.fetchall() + has_more = len(rows) > limit + rows = rows[:limit] + events = [self._row_to_event(row) for row in rows] + next_cursor = "" + if has_more and rows: + created = rows[-1]["created_at"] + next_cursor = _encode_cursor( + [created.isoformat() if hasattr(created, "isoformat") else str(created), int(rows[-1]["id"])] + ) + return events, next_cursor + def count_citations(self, claim_id: int) -> int: with self.connect() as conn, conn.cursor() as cur: cur.execute("SELECT COUNT(*) AS c FROM citations WHERE claim_id = %s", (claim_id,)) diff --git a/memorymaster/stores/storage.py b/memorymaster/stores/storage.py index aec6f83d..5c083ab9 100644 --- a/memorymaster/stores/storage.py +++ b/memorymaster/stores/storage.py @@ -8,6 +8,7 @@ from pathlib import Path from memorymaster.stores._storage_lifecycle import _LifecycleMixin +from memorymaster.stores._storage_pagination import _PaginationMixin from memorymaster.stores._storage_read import _ReadMixin from memorymaster.stores._storage_schema import _SchemaMixin # Re-export shared helpers for backward compat with external imports. @@ -52,6 +53,13 @@ def initdb_fastpath_enabled() -> bool: return raw not in ("0", "false", "False", "no", "off", "") +def _legacy_schema_source() -> bytes: + """Return legacy ensure-helper source included in the fast-path stamp.""" + from memorymaster.stores import _storage_schema + + return Path(_storage_schema.__file__).read_bytes() + + def schema_stamp() -> int: """31-bit fingerprint of the full schema lineage, for ``PRAGMA user_version``. @@ -66,13 +74,22 @@ def schema_stamp() -> int: from memorymaster.stores.migrations import discover_migrations h = hashlib.sha256(load_schema_sql().encode("utf-8")) + h.update(b"\x00legacy-schema-helpers\x00") + h.update(_legacy_schema_source()) for m in discover_migrations(): h.update(f"\x00{m.version}:{m.checksum()}".encode("utf-8")) stamp = int.from_bytes(h.digest()[:4], "big") & 0x7FFFFFFF return stamp or 1 -class SQLiteStore(_SchemaMixin, _ReadMixin, _WriteClaimsMixin, _LifecycleMixin, _SourceItemsMixin): +class SQLiteStore( + _SchemaMixin, + _ReadMixin, + _WriteClaimsMixin, + _LifecycleMixin, + _SourceItemsMixin, + _PaginationMixin, +): def __init__(self, db_path: str | Path, *, read_only: bool = False) -> None: self.db_path = str(db_path) # P1 WAL-discipline (spec §2.2): an RO store hands out mode=ro + diff --git a/memorymaster/surfaces/mcp_server.py b/memorymaster/surfaces/mcp_server.py index a10cb1dd..aa7ad125 100644 --- a/memorymaster/surfaces/mcp_server.py +++ b/memorymaster/surfaces/mcp_server.py @@ -241,7 +241,7 @@ def _check_ingest_rate_limit( """ limit = _ingest_rate_limit_per_min() if limit == 0: - return None + return _check_durable_ingest_quota(source_agent, cost) timestamp = _monotonic() if now is None else now key = _empty_to_none(source_agent) or _ANONYMOUS_SOURCE_AGENT @@ -275,6 +275,31 @@ def _check_ingest_rate_limit( _INGEST_RATE_BUCKETS[key] = (agent_tokens - cost, timestamp) _INGEST_RATE_BUCKETS[_GLOBAL_RATE_AGENT] = (global_tokens - cost, timestamp) _evict_rate_buckets() + return _check_durable_ingest_quota(source_agent, cost) + + +def _check_durable_ingest_quota( + source_agent: str, cost: float +) -> dict[str, Any] | None: + from memorymaster.core.usage_ledger import UsageQuotaExceeded, reserve_configured + + actor = _empty_to_none(source_agent) or _ANONYMOUS_SOURCE_AGENT + try: + reserve_configured( + operation="ingest", + provider="mcp", + actor=actor, + units=max(1, int(cost)), + ) + except UsageQuotaExceeded as exc: + return _structured_error( + "durable MCP ingest quota exhausted", + "RATE_LIMITED", + "source_agent", + source_agent=actor, + partition=exc.partition, + limit_per_window=exc.limit, + ) return None @@ -389,6 +414,15 @@ def _empty_to_none(value: str) -> str | None: return v if v else None +def _bounded_limit(value: int, *, maximum: int) -> int: + """Clamp caller-controlled result sizes to a finite positive range.""" + try: + parsed = int(value) + except (TypeError, ValueError): + parsed = 1 + return min(maximum, max(1, parsed)) + + def _parse_sources_json(sources_json: str) -> list[CitationInput]: if not sources_json.strip(): return [] @@ -1101,6 +1135,7 @@ def run_steward( def classify_query(query: str) -> dict[str, Any]: """Classify a query and report its recommended and effective modes.""" from memorymaster.recall.query_classifier import classify_query as _classify, recommended_retrieval_mode + qtype = _classify(query) recommended_mode = recommended_retrieval_mode(qtype) effective_mode = "legacy" if recommended_mode == "qdrant" else recommended_mode @@ -1180,6 +1215,7 @@ def query_memory( """ from memorymaster.recall.query_classifier import classify_query as _classify, recommended_retrieval_mode + limit = _bounded_limit(limit, maximum=100) resolve_allow_sensitive_access( allow_sensitive=allow_sensitive, context="mcp.query_memory", @@ -1219,11 +1255,15 @@ def query_memory( rows_data = _enrich_claims_with_citations(svc, rows_data) claims = [row["claim"] for row in rows_data] + serialized_claims = [ + _apply_detail_level(_claim_to_dict(claim), detail_level) + for claim in claims + ] serialized_rows: list[dict[str, Any]] = [] - for row in rows_data: + for index, row in enumerate(rows_data): serialized_rows.append( { - "claim": _apply_detail_level(_claim_to_dict(row["claim"]), detail_level), + "claim_index": index, "status": row.get("status"), "annotation": row.get("annotation", {}), "score": row.get("score", 0.0), @@ -1237,8 +1277,9 @@ def query_memory( response: dict[str, Any] = { "ok": True, "rows": len(claims), - "claims": [_apply_detail_level(_claim_to_dict(c), detail_level) for c in claims], + "claims": serialized_claims, "rows_data": serialized_rows, + "response_contract": "memorymaster.retrieval.v2", "trust_mode": retrieval.plan.trust_mode, "requested_retrieval_mode": requested_retrieval_mode, "retrieval_mode": retrieval.plan.effective_mode, @@ -1382,6 +1423,8 @@ def query_for_context( - "standard" (default): full claim dict - "full": full claim dict with citations inlined """ + limit = _bounded_limit(limit, maximum=250) + token_budget = _bounded_limit(token_budget, maximum=32_000) resolve_allow_sensitive_access( allow_sensitive=allow_sensitive, context="mcp.query_for_context", @@ -1472,6 +1515,9 @@ def volunteer_context( """ from memorymaster.recall.context_optimizer import pack_context + limit = _bounded_limit(limit, maximum=250) + token_budget = _bounded_limit(token_budget, maximum=32_000) + resolve_allow_sensitive_access( allow_sensitive=allow_sensitive, context="mcp.volunteer_context", @@ -1680,6 +1726,7 @@ def list_claims( include_archived: bool = False, allow_sensitive: bool = False, holder: str = "", + cursor: str = "", ) -> dict[str, Any]: """List claims by optional status and/or belief holder (takes-vs-facts).""" resolve_allow_sensitive_access( @@ -1687,16 +1734,23 @@ def list_claims( context="mcp.list_claims", ) svc = _read_service(db, workspace) - claims = svc.list_claims( + limit = _bounded_limit(limit, maximum=250) + claims, next_cursor = svc.list_claims_page( status=_empty_to_none(status), limit=limit, + cursor=cursor, include_archived=include_archived, allow_sensitive=allow_sensitive, holder=_empty_to_none(holder), scope_allowlist=_effective_scope_allowlist("", workspace), requesting_agent=_team_request_principal(), ) - return {"ok": True, "rows": len(claims), "claims": [_claim_to_dict(c) for c in claims]} + return { + "ok": True, + "rows": len(claims), + "claims": [_claim_to_dict(c) for c in claims], + "next_cursor": next_cursor or None, + } @mcp.tool() def ingest_rule( @@ -1765,6 +1819,7 @@ def query_rules( hybrid retriever. Use this when you want only behavioural rules, not descriptive fact claims. """ + limit = _bounded_limit(limit, maximum=100) resolve_allow_sensitive_access( allow_sensitive=allow_sensitive, context="mcp.query_rules", @@ -1789,6 +1844,7 @@ def rules_export( out unless ``allow_sensitive`` is granted by policy, so this never leaks another agent's secret rules. """ + limit = _bounded_limit(limit, maximum=1000) resolve_allow_sensitive_access( allow_sensitive=allow_sensitive, context="mcp.rules_export", @@ -1863,15 +1919,23 @@ def list_events( claim_id: int | None = None, event_type: str = "", limit: int = 100, + cursor: str = "", ) -> dict[str, Any]: """List events by optional claim_id and event_type.""" svc = _read_service(db, workspace) - events = svc.list_events( + limit = _bounded_limit(limit, maximum=500) + events, next_cursor = svc.list_events_page( claim_id=claim_id, limit=limit, event_type=_empty_to_none(event_type), + cursor=cursor, ) - return {"ok": True, "rows": len(events), "events": [asdict(e) for e in events]} + return { + "ok": True, + "rows": len(events), + "events": [asdict(e) for e in events], + "next_cursor": next_cursor or None, + } @mcp.tool() def search_verbatim( @@ -1889,6 +1953,7 @@ def search_verbatim( verbatim search finds exact conversation fragments. """ from memorymaster.recall.verbatim_store import search_verbatim as _search + limit = _bounded_limit(limit, maximum=100) requested_mode = str(mode).strip().lower() effective_mode = "fts" if requested_mode in {"vector", "hybrid"} else requested_mode results = _search( @@ -1982,6 +2047,7 @@ def list_steward_proposals( """List steward proposals for human override workflow.""" from memorymaster.govern.steward import list_steward_proposals as _list_steward_proposals + limit = _bounded_limit(limit, maximum=500) svc = _read_service(db, workspace) rows = _list_steward_proposals( svc, @@ -2063,6 +2129,8 @@ def find_related_claims( """ from memorymaster.knowledge.entity_graph import EntityGraph, EntityGraphNotReady + limit = _bounded_limit(limit, maximum=250) + hops = _bounded_limit(hops, maximum=5) _read_service(db, workspace) graph = EntityGraph(_resolve_db(db), read_only=True) names = [n.strip() for n in entity_names.split(",") if n.strip()] @@ -2138,6 +2206,7 @@ def federated_query( this tool searches every claim regardless of scope, enabling cross-project memory retrieval. Returns claims sorted by relevance. """ + limit = _bounded_limit(limit, maximum=100) svc = _read_service(db, workspace) effective_scope = (current_scope or "").strip() or _project_scope(workspace) rows_data = svc.federated_query( @@ -2147,9 +2216,10 @@ def federated_query( scope_allowlist=_parse_scope_allowlist(scope_allowlist), ) claims = [row["claim"] for row in rows_data] + serialized_claims = [_claim_to_dict(claim) for claim in claims] serialized_rows: list[dict[str, Any]] = [ { - "claim": _claim_to_dict(row["claim"]), + "claim_index": index, "status": row.get("status"), "annotation": row.get("annotation", {}), "score": row.get("score", 0.0), @@ -2158,13 +2228,14 @@ def federated_query( "confidence_score": row.get("confidence_score", 0.0), "vector_score": row.get("vector_score", 0.0), } - for row in rows_data + for index, row in enumerate(rows_data) ] return { "ok": True, "rows": len(claims), - "claims": [_claim_to_dict(c) for c in claims], + "claims": serialized_claims, "rows_data": serialized_rows, + "response_contract": "memorymaster.retrieval.v2", } diff --git a/tests/test_deployment_contracts.py b/tests/test_deployment_contracts.py index 136b66df..7e7c6a77 100644 --- a/tests/test_deployment_contracts.py +++ b/tests/test_deployment_contracts.py @@ -131,6 +131,24 @@ def test_helm_requires_digest_and_existing_qdrant_secrets(): assert 'fail "qdrant.url must use HTTPS"' in deployment +def test_helm_storage_is_coupled_to_finite_retention_envelope(): + values = _read("helm/memorymaster/values.yaml") + pvc = _read("helm/memorymaster/templates/pvc.yaml") + + assert "size: 1Gi" not in values + for key in ( + "verbatimMaxMi", + "databaseMaxMi", + "artifactsMaxMi", + "walAndBackupHeadroomMi", + "operatorHeadroomMi", + ): + assert key in values + assert key in pvc + assert "retention envelope" in pvc + assert "persistence.sizeMi" in pvc + + def test_postgres_smoke_requires_operator_supplied_dsn(): script = _read("scripts/smoke_postgres.ps1") diff --git a/tests/test_initdb_fastpath.py b/tests/test_initdb_fastpath.py index ca133ec0..c8289cf3 100644 --- a/tests/test_initdb_fastpath.py +++ b/tests/test_initdb_fastpath.py @@ -167,6 +167,18 @@ def test_schema_change_without_manual_bump_is_caught( assert _user_version(db) == schema_stamp() != old_stamp +def test_legacy_schema_helper_change_changes_stamp(monkeypatch: pytest.MonkeyPatch) -> None: + """Fast-path fingerprints include legacy ensure-helper source bytes.""" + old_stamp = schema_stamp() + real_source = storage_mod._legacy_schema_source + monkeypatch.setattr( + storage_mod, + "_legacy_schema_source", + lambda: real_source() + b"\n# simulated ensure helper edit\n", + ) + assert schema_stamp() != old_stamp + + def test_new_migration_changes_stamp_and_forces_full_path( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, counters: dict[str, int] ) -> None: diff --git a/tests/test_r32_query_storage_efficiency.py b/tests/test_r32_query_storage_efficiency.py new file mode 100644 index 00000000..1aba803f --- /dev/null +++ b/tests/test_r32_query_storage_efficiency.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import sqlite3 +from pathlib import Path + +import pytest + +from memorymaster.recall.recall_tokenizer import _corpus_stats, extract_query_tokens +from memorymaster.core.models import CitationInput +from memorymaster.stores.storage import SQLiteStore +from memorymaster.surfaces.mcp_server import _bounded_limit + + +def test_event_queries_use_versioned_composite_indexes(tmp_path: Path) -> None: + db = tmp_path / "events.db" + store = SQLiteStore(db) + store.init_db() + with sqlite3.connect(db) as conn: + indexes = { + str(row[1]) + for row in conn.execute("PRAGMA index_list(events)").fetchall() + } + assert "idx_events_type_created_id" in indexes + assert "idx_events_type_details_created" in indexes + plan = conn.execute( + "EXPLAIN QUERY PLAN SELECT * FROM events " + "WHERE event_type=? ORDER BY created_at DESC, id DESC LIMIT 10", + ("system",), + ).fetchall() + assert "idx_events_type_created_id" in " ".join(str(row) for row in plan) + + +def test_token_stats_invalidate_on_corpus_generation(tmp_path: Path) -> None: + db = tmp_path / "tokens.db" + store = SQLiteStore(db) + store.init_db() + store.create_claim( + "alpha steward", [CitationInput(source="test")], scope="project:test" + ) + _corpus_stats.cache_clear() + + assert "alpha" in extract_query_tokens("alpha steward", str(db), max_tokens=2) + before = _corpus_stats.cache_info() + store.create_claim( + "qdrant qdrant", [CitationInput(source="test")], scope="project:test" + ) + extract_query_tokens("qdrant steward", str(db), max_tokens=2) + after = _corpus_stats.cache_info() + + assert after.misses == before.misses + 1 + + +@pytest.mark.parametrize( + ("value", "maximum", "expected"), + [(-5, 100, 1), (0, 100, 1), (20, 100, 20), (10_000, 100, 100)], +) +def test_mcp_limits_are_clamped(value: int, maximum: int, expected: int) -> None: + assert _bounded_limit(value, maximum=maximum) == expected + + +def test_store_keyset_pages_do_not_repeat_rows(tmp_path: Path) -> None: + db = tmp_path / "pages.db" + store = SQLiteStore(db) + store.init_db() + for index in range(5): + store.create_claim( + f"claim {index}", + [CitationInput(source="test")], + scope="project:test", + confidence=0.5, + ) + + first, cursor = store.list_claims_page(limit=2) + second, next_cursor = store.list_claims_page(limit=2, cursor=cursor) + + assert len(first) == len(second) == 2 + assert {claim.id for claim in first}.isdisjoint(claim.id for claim in second) + assert cursor and next_cursor and cursor != next_cursor + + +def test_retrieval_rows_reference_claims_without_duplicate_payload( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + from memorymaster.core.lifecycle import transition_claim + from memorymaster.core.service import MemoryService + from memorymaster.surfaces.mcp_server import _project_scope, query_memory + + monkeypatch.delenv("QDRANT_URL", raising=False) + service = MemoryService(tmp_path / "mcp.db", workspace_root=tmp_path) + service.init_db() + claim = service.ingest( + text="bounded retrieval payload", + citations=[CitationInput(source="test")], + scope=_project_scope(str(tmp_path)), + source_agent="test", + ) + transition_claim( + service.store, + claim.id, + "confirmed", + reason="test fixture", + event_type="validator", + ) + result = query_memory( + query="bounded retrieval payload", + db=str(service.store.db_path), + workspace=str(tmp_path), + trust_mode="exploratory", + ) + + assert result["response_contract"] == "memorymaster.retrieval.v2" + assert result["rows_data"] + assert all("claim" not in row for row in result["rows_data"]) + assert all(0 <= row["claim_index"] < len(result["claims"]) for row in result["rows_data"]) diff --git a/tests/test_retrieval_surface_parity.py b/tests/test_retrieval_surface_parity.py index a45c7b1d..cdabc364 100644 --- a/tests/test_retrieval_surface_parity.py +++ b/tests/test_retrieval_surface_parity.py @@ -104,7 +104,10 @@ def test_exploratory_mode_is_explicit_and_annotated(tmp_path, monkeypatch): trust_mode="exploratory", ) - rows = {row["claim"]["id"]: row for row in result["rows_data"]} + rows = { + result["claims"][row["claim_index"]]["id"]: row + for row in result["rows_data"] + } assert claim_id in rows assert rows[claim_id]["annotation"]["active"] is False assert rows[claim_id]["annotation"]["status"] == "candidate" diff --git a/tests/test_usage_ledger.py b/tests/test_usage_ledger.py new file mode 100644 index 00000000..44383c69 --- /dev/null +++ b/tests/test_usage_ledger.py @@ -0,0 +1,94 @@ +from __future__ import annotations + +import pytest + +from memorymaster.core import llm_provider +from memorymaster.core.usage_ledger import UsageLedger, UsageQuotaExceeded + + +def test_durable_quota_survives_new_ledger_instance(tmp_path) -> None: + db = tmp_path / "usage.db" + first = UsageLedger(db) + first.reserve( + operation="llm", + provider="google", + actor="account", + units=1, + global_limit=1, + ) + + second = UsageLedger(db) + with pytest.raises(UsageQuotaExceeded): + second.reserve( + operation="llm", + provider="google", + actor="account", + units=1, + global_limit=1, + ) + + +def test_provider_and_actor_partitions_are_atomic(tmp_path) -> None: + ledger = UsageLedger(tmp_path / "usage.db") + ledger.reserve( + operation="embedding", + provider="gemini", + actor="agent-a", + units=2, + provider_limit=3, + actor_limit=2, + ) + with pytest.raises(UsageQuotaExceeded): + ledger.reserve( + operation="embedding", + provider="gemini", + actor="agent-a", + units=1, + provider_limit=3, + actor_limit=2, + ) + # A rejected reservation does not consume provider capacity. + ledger.reserve( + operation="embedding", + provider="gemini", + actor="agent-b", + units=1, + provider_limit=3, + actor_limit=2, + ) + + +def test_call_llm_blocks_before_provider_when_durable_cap_is_exhausted( + tmp_path, monkeypatch +) -> None: + db = tmp_path / "usage.db" + monkeypatch.setenv("MEMORYMASTER_USAGE_LEDGER_DB", str(db)) + monkeypatch.setenv("MEMORYMASTER_MAX_LLM_CALLS_PER_DAY", "1") + monkeypatch.setenv("MEMORYMASTER_LLM_PROVIDER", "google") + calls: list[int] = [] + monkeypatch.setitem( + llm_provider._PROVIDERS, + "google", + lambda _prompt, _text: calls.append(1) or "ok", + ) + + assert llm_provider.call_llm("p", "t") == "ok" + with pytest.raises(Exception, match="durable_daily_exhausted"): + llm_provider.call_llm("p", "t") + assert len(calls) == 1 + + +def test_mcp_quota_survives_process_local_bucket_reset(tmp_path, monkeypatch) -> None: + import memorymaster.surfaces.mcp_server as mcp_server + + monkeypatch.setenv("MEMORYMASTER_USAGE_LEDGER_DB", str(tmp_path / "usage.db")) + monkeypatch.setenv("MEMORYMASTER_MAX_MCP_INGESTS_PER_DAY", "1") + monkeypatch.setenv("MM_INGEST_RATE_LIMIT_PER_MIN", "0") + + assert mcp_server._check_ingest_rate_limit("agent-a") is None + mcp_server._INGEST_RATE_BUCKETS.clear() + rejected = mcp_server._check_ingest_rate_limit("agent-b") + + assert rejected is not None + assert rejected["code"] == "RATE_LIMITED" + assert rejected["partition"] == "global" From 0125a6341b73c20be9bdde6a86d60dc15f7c0c26 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Mon, 13 Jul 2026 23:44:05 -0300 Subject: [PATCH 39/55] fix: make setup profiles truthful --- .planning/REMAINING-PHASES-V1-BUDGET.md | 30 +++ .planning/audit-remediation-ledger.md | 2 +- .../codex-agents-md-append.md | 2 +- .../hooks/memorymaster-session-end.py | 2 +- memorymaster/surfaces/session_end_ingest.py | 150 ++++++++++++ memorymaster/surfaces/setup_hooks.py | 220 ++++++++++++++++-- pyproject.toml | 1 + tests/test_session_end_capture_hook.py | 8 +- tests/test_setup_hooks.py | 27 ++- tests/test_setup_profiles.py | 47 ++++ 10 files changed, 462 insertions(+), 27 deletions(-) create mode 100644 memorymaster/surfaces/session_end_ingest.py create mode 100644 tests/test_setup_profiles.py diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index 5ac1c0db..0ee3ef53 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -342,3 +342,33 @@ R3.2 query and storage efficiency completed on 2026-07-13: The next package is R3.3 setup-profile truth. The active roadmap goal authorizes continuing without a new goal. + +R3.3 truthful setup profiles completed on 2026-07-13: + +- `memorymaster-setup --profile` now exposes explicit `minimal`, `semantic`, + `team`, and `full-lab` component contracts. JSON and human results report DB, + MCP, recall hook, capture hook, provider, steward, vector backend, and + dashboard evidence as PASS, PARTIAL, or BLOCKED. +- Requested PARTIAL/BLOCKED profiles return nonzero (3/2 respectively); a + successful local sentinel no longer masks a missing requested component. + Team verification requires a PostgreSQL DSN and retains the disposable + two-role external gate. +- MCP status requires registration plus an importable FastMCP runtime; hooks + require installed packaged files; remote providers are never called merely + for setup and therefore remain PARTIAL when only credentials are configured. + Started-but-unreprobed vector/provider services are PARTIAL, not PASS. +- Session-end distillation is shipped as + `memorymaster.surfaces.session_end_ingest` and the + `memorymaster-session-end` entrypoint. Hooks and Codex instructions no longer + depend on the repository-only `scripts/` tree. Installed-wheel setup now + fails closed when a project-local Compose bundle is absent instead of + invoking an unrelated default Compose project. +- RED evidence: four profile tests initially failed because no profile DTO, + aggregation, parser flag, or nonzero semantic result existed. Focused setup, + detection, hook, packaged-session, and Qdrant-transport boundary: 161 passed + with one existing Pydantic deprecation warning. Changed-file Ruff and + `git diff --check` passed. +- Atomic package commit: the conventional R3.3 commit containing this evidence. + +The next package is R3.4 service entrypoints and readiness contracts. The +active roadmap goal authorizes continuing without a new goal. diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 575e2ed5..4b95cb2a 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -28,7 +28,7 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-PERF-02 | Performance | Medium / BAD-PRACTICE | Each process cold-scans corpus token statistics and caches forever | R3.2 | RESOLVED | R3.2 keys corpus and alias statistics by the persisted corpus generation, so warm reads reuse one scan and every claim mutation invalidates it. Focused evidence proves a new cache miss after generation change while tokenizer ranking/retired-status contracts remain green. | | MM-PERF-03 | Performance/Cost | High / EXPLOITABLE-NOW | Qdrant reconciliation repeats/truncates tens of thousands of embeddings | R3.1 | RESOLVED | R3.1 replaces four truncated 10,000-row scans with authority-scoped keyset pages and a replayable durable cursor; fake-backend convergence/failure-resume evidence passed. Real authenticated/TLS Qdrant remains externally blocked. | | MM-PERF-04 | Database/Performance | Medium / BAD-PRACTICE | Million-row event queries lack event-type composite indexes | R3.2 | RESOLVED | Immutable migration 0015 and both baseline schemas add `(event_type, created_at DESC, id DESC)` and `(event_type, details, created_at DESC)` indexes. SQLite EXPLAIN selects the ordered type index; Postgres DDL parity is covered statically pending the already-recorded disposable-Postgres external gate. | -| MM-UX-01 | UX/Maintainability | High / BAD-PRACTICE | Setup reports success without verifying the requested memory loop | R3.3 | OPEN | Component-level profile verification and nonzero failures | +| MM-UX-01 | UX/Maintainability | High / BAD-PRACTICE | Setup reports success without verifying the requested memory loop | R3.3 | RESOLVED | R3.3 adds explicit minimal/semantic/team/full-lab contracts and stable DB, MCP, recall-hook, capture-hook, provider, steward, vector, and dashboard component evidence. Requested PARTIAL/BLOCKED profiles return nonzero; a DB-only sentinel can no longer mask missing requested components. Session-end ingest is now a packaged module/entrypoint, and installed-wheel setup refuses to imply an absent Compose bundle was shipped. Focused setup/profile gate: 161 passed. | | MM-UX-02 | UX/Reliability | High / BAD-PRACTICE | Dashboard removes review rows before mutation success | R4.3 | OPEN | Browser failure/retry evidence; row remains on rejected POST | | MM-UX-03 | UX | Medium / BAD-PRACTICE | Dashboard hides failures and has labeling, contrast, responsive, and evidence-discovery gaps | R4.3 | OPEN | Browser/a11y/mobile acceptance suite | | MM-PRIV-01 | Compliance | High / BAD-PRACTICE | Automatic transcript capture/remote processing lacks an explicit consent boundary | R2.4,R3.5 | BLOCKED-EXTERNAL | P2-E repository controls are quiet by default and require explicit maximum-capture/provider choices. Product-owner/legal approval for intended use, jurisdictions, processor disclosures, and retention commitments remains external. WARNING LEGAL REVIEW REQUIRED; no compliance-pass or risk-acceptance claim is made. | diff --git a/memorymaster/config_templates/codex-agents-md-append.md b/memorymaster/config_templates/codex-agents-md-append.md index 7a3d6682..c9f49cd1 100644 --- a/memorymaster/config_templates/codex-agents-md-append.md +++ b/memorymaster/config_templates/codex-agents-md-append.md @@ -43,7 +43,7 @@ operator can install a session-end distiller that reads your last transcript, distills <=3 learnings, and ingests them with `source_agent="codex-session"`: ``` -python scripts/agent_session_end_ingest.py \ +python -m memorymaster.surfaces.session_end_ingest \ --db /memorymaster.db \ --transcript ~/.codex/sessions/rollout-.jsonl \ --source-agent codex-session \ diff --git a/memorymaster/config_templates/hooks/memorymaster-session-end.py b/memorymaster/config_templates/hooks/memorymaster-session-end.py index e469a647..7b0ae9dc 100644 --- a/memorymaster/config_templates/hooks/memorymaster-session-end.py +++ b/memorymaster/config_templates/hooks/memorymaster-session-end.py @@ -30,7 +30,7 @@ def main(): cwd = str(data.get("cwd") or PROJECT_ROOT) try: from memorymaster.core.capture_control import CaptureLedger, capture_state_path - from scripts.agent_session_end_ingest import run + from memorymaster.surfaces.session_end_ingest import run ledger = CaptureLedger(capture_state_path()) chunk = ledger.read_increment(transcript_path, f"{session_id}:session-end") diff --git a/memorymaster/surfaces/session_end_ingest.py b/memorymaster/surfaces/session_end_ingest.py new file mode 100644 index 00000000..9bfb4c2f --- /dev/null +++ b/memorymaster/surfaces/session_end_ingest.py @@ -0,0 +1,150 @@ +"""Packaged session-end distillation for Codex and generic agents.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import sys +import uuid +from pathlib import Path +from typing import Any + +MAX_LEARNINGS = 3 +DISTILL_PROMPT = ( + "Extract at most 3 non-obvious bug causes, decisions, gotchas, or constraints. " + "Return JSON objects with text, claim_type, subject, and predicate. " + "Never include credentials, IPs, paths, or code; return [] when empty." +) + + +def _sensitive(claim: dict[str, Any]) -> bool: + from memorymaster.core.security import redact_text + + joined = " | ".join( + str(claim.get(key, "") or "") + for key in ("text", "subject", "predicate", "object_value") + ) + return bool(redact_text(joined)[1]) + + +def _assistant_text(transcript_path: str) -> str: + path = Path(transcript_path) + if not path.is_file(): + return "" + messages: list[str] = [] + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + for line in reversed(lines[-200:]): + try: + entry = json.loads(line) + except json.JSONDecodeError: + continue + if not isinstance(entry, dict): + continue + record = entry + for key in ("message", "payload"): + if isinstance(entry.get(key), dict): + record = entry[key] + break + if str(record.get("role") or "") not in {"assistant", "model"}: + continue + content = record.get("content", "") + if isinstance(content, list): + content = " ".join( + str(part.get("text", "")) + for part in content + if isinstance(part, dict) and part.get("type") in (None, "text") + ) + if isinstance(content, str) and len(content.strip()) > 30: + messages.append(content[:500]) + if sum(map(len, messages)) > 3000: + break + return "\n---\n".join(reversed(messages)) if sum(map(len, messages)) >= 50 else "" + + +def _distill(text: str) -> list[dict[str, Any]]: + from memorymaster.core.llm_provider import call_llm, parse_json_response + + claims = parse_json_response(call_llm(DISTILL_PROMPT, text)) + clean = [claim for claim in claims if isinstance(claim, dict) and not _sensitive(claim)] + return clean[:MAX_LEARNINGS] + + +def ingest_learnings( + db_path: str, + claims: list[dict[str, Any]], + *, + source_agent: str, + cwd: str | None, +) -> int: + if not source_agent.strip(): + raise ValueError("source_agent is required") + if not Path(db_path).is_file(): + raise FileNotFoundError(f"DB not found: {db_path}") + from memorymaster.core.models import CitationInput + from memorymaster.core.service import MemoryService + + service = MemoryService(db_path, workspace_root=Path(cwd or os.getcwd())) + scope = "global" if not cwd else f"project:{Path(cwd).name.lower().replace(' ', '-')}" + batch_id = f"session-end-{uuid.uuid4().hex[:16]}" + ingested = 0 + for claim in claims[:MAX_LEARNINGS]: + text = str(claim.get("text", "") or "").strip() + if len(text) < 10 or _sensitive(claim): + continue + digest = hashlib.sha256(text.lower().encode()).hexdigest()[:16] + service.ingest( + text=text, + citations=[CitationInput(source=source_agent, locator=scope, excerpt=text[:200])], + idempotency_key=f"{source_agent}-{digest}", + claim_type=str(claim.get("claim_type", "fact") or "fact"), + subject=str(claim.get("subject", "codebase") or "codebase"), + predicate=str(claim.get("predicate", "observation") or "observation"), + object_value=claim.get("object_value"), + scope=scope, + confidence=0.6, + source_agent=source_agent, + intake_batch_id=batch_id, + intake_batch_max=MAX_LEARNINGS, + ) + ingested += 1 + return ingested + + +def run( + db_path: str, + transcript_path: str, + *, + source_agent: str, + cwd: str | None, +) -> int: + text = _assistant_text(transcript_path) + return ingest_learnings( + db_path, _distill(text), source_agent=source_agent, cwd=cwd + ) if text else 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Distill and ingest session-end learnings") + parser.add_argument("--db", required=True) + parser.add_argument("--transcript", required=True) + parser.add_argument("--source-agent", default="codex-session") + parser.add_argument("--cwd", default=None) + args = parser.parse_args(argv) + try: + count = run( + args.db, + args.transcript, + source_agent=args.source_agent, + cwd=args.cwd, + ) + except (FileNotFoundError, ValueError) as exc: + print(f"[MemoryMaster] session-end ingest error: {exc}", file=sys.stderr) + return 1 + print(f"[MemoryMaster] session-end: ingested {count} learning(s)", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/memorymaster/surfaces/setup_hooks.py b/memorymaster/surfaces/setup_hooks.py index e9aa99e9..e866d8dd 100644 --- a/memorymaster/surfaces/setup_hooks.py +++ b/memorymaster/surfaces/setup_hooks.py @@ -24,6 +24,7 @@ import subprocess import sys import time +from dataclasses import asdict, dataclass from pathlib import Path from typing import Any, Optional @@ -49,6 +50,130 @@ CODEX_DIR = HOME / ".codex" PYTHON_EXE = sys.executable +SETUP_PROFILES: dict[str, tuple[str, ...]] = { + "minimal": ("db", "mcp"), + "semantic": ("db", "mcp", "provider", "vector_backend"), + "team": ("db", "mcp"), + "full-lab": ( + "db", + "mcp", + "recall_hook", + "capture_hook", + "provider", + "steward", + "vector_backend", + "dashboard", + ), +} + + +@dataclass(frozen=True, slots=True) +class ComponentResult: + name: str + status: str + detail: str + + +def evaluate_setup_profile( + profile: str, components: dict[str, ComponentResult] +) -> tuple[str, int]: + required = SETUP_PROFILES[profile] + statuses = [components[name].status for name in required] + if "BLOCKED" in statuses: + return "BLOCKED", 2 + if "PARTIAL" in statuses: + return "PARTIAL", 3 + return "PASS", 0 + + +def _provider_component( + detected: Detected, applied: dict[str, Any], config: dict[str, str] +) -> ComponentResult: + provider = config.get("provider", "").lower() + stack = applied.get("full_stack", {}) + if provider == "ollama": + if detected.ollama or stack.get("ollama") == "reused": + return ComponentResult("provider", "PASS", "Ollama readiness probe passed") + if stack.get("ollama") == "started": + return ComponentResult("provider", "PARTIAL", "Ollama started; readiness not yet reprobed") + return ComponentResult("provider", "BLOCKED", "Ollama is not reachable") + key_env = { + "google": "GEMINI_API_KEY", + "openai": "OPENAI_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + }.get(provider, "") + if config.get("api_key") or (key_env and os.environ.get(key_env)): + return ComponentResult( + "provider", "PARTIAL", "credentials configured; paid provider was not called" + ) + return ComponentResult("provider", "BLOCKED", f"{provider or 'remote'} provider is not configured") + + +def _setup_components( + *, + profile: str, + detected: Detected, + applied: dict[str, Any], + verify: dict[str, Any], + llm_config: dict[str, str], + db_target: str | Path, +) -> dict[str, ComponentResult]: + verify_status = str(verify.get("status", "FAIL")) + db_status = "PASS" if verify_status == "PASS" else "PARTIAL" if verify_status == "PARTIAL" else "BLOCKED" + db_detail = str(verify.get("detail", "database verification unavailable")) + if profile == "team" and not str(db_target).lower().startswith(("postgres://", "postgresql://")): + db_status = "BLOCKED" + db_detail = "team profile requires a PostgreSQL DSN and disposable two-role verification" + + mcp_actions = (str(applied.get("mcp_claude", "")), str(applied.get("mcp_codex", ""))) + try: + from memorymaster.surfaces import mcp_server as installed_mcp + + mcp_importable = installed_mcp.FastMCP is not None + except Exception: + mcp_importable = False + mcp_ok = mcp_importable and any( + value in {"present", "registered"} for value in mcp_actions + ) + recall_ok = (CLAUDE_DIR / "hooks" / "memorymaster-recall.py").is_file() + capture_ok = (CLAUDE_DIR / "hooks" / "memorymaster-session-end.py").is_file() + cron_state = str(applied.get("cron", "blocked")) + stack = applied.get("full_stack", {}) if isinstance(applied.get("full_stack"), dict) else {} + qdrant = stack.get("qdrant") + if detected.qdrant or qdrant == "reused": + vector = ComponentResult("vector_backend", "PASS", "Qdrant readiness probe passed") + elif qdrant == "started": + vector = ComponentResult("vector_backend", "PARTIAL", "Qdrant started; authenticated/TLS readiness not reprobed") + else: + vector = ComponentResult("vector_backend", "BLOCKED", "governed Qdrant backend is unavailable") + + return { + "db": ComponentResult("db", db_status, db_detail), + "mcp": ComponentResult( + "mcp", + "PASS" if mcp_ok else "BLOCKED", + "MCP registration and import contract verified" if mcp_ok else "no supported MCP client was registered", + ), + "recall_hook": ComponentResult( + "recall_hook", "PASS" if recall_ok else "BLOCKED", + "recall hook resource and installed file verified" if recall_ok else "recall hook unavailable", + ), + "capture_hook": ComponentResult( + "capture_hook", "PASS" if capture_ok else "BLOCKED", + "quiet capture hook resource and installed file verified" if capture_ok else "capture hook unavailable", + ), + "provider": _provider_component(detected, applied, llm_config), + "steward": ComponentResult( + "steward", + "PASS" if cron_state == "configured" else "PARTIAL" if cron_state == "manual" else "BLOCKED", + "steward schedule configured" if cron_state == "configured" else "manual cron action required" if cron_state == "manual" else "steward schedule unavailable", + ), + "vector_backend": vector, + "dashboard": ComponentResult( + "dashboard", "PARTIAL", "dashboard entrypoint is installed; HTTP readiness belongs to R3.4" + ), + } + # --------------------------------------------------------------------------- # Non-interactive mode (set by main() when --yes is passed). When True, # ask()/ask_yn() never call input() — they return the supplied default. This @@ -455,12 +580,11 @@ def append_instructions(): # rather than auto-scheduling a daemon they didn't ask for. The script sets # source_agent + caps the batch at 3 and routes through service.ingest. if CODEX_DIR.exists(): - session_end_script = PROJECT_ROOT / "scripts" / "agent_session_end_ingest.py" print(" Codex BEAT-3 (session-end distilled ingest) — no native Stop hook.") - print(f" Turnkey reference: {session_end_script}") + print(" Packaged command: memorymaster-session-end") print(" Wire it as a Codex notify/exit hook or run at session end:") print( - f' "{PYTHON_EXE}" "{session_end_script}" ' + f' "{PYTHON_EXE}" -m memorymaster.surfaces.session_end_ingest ' "--db /memorymaster.db --transcript " "--source-agent codex-session --cwd " ) @@ -470,11 +594,11 @@ def append_instructions(): # --------------------------------------------------------------------------- # 5. Steward cron # --------------------------------------------------------------------------- -def setup_steward_cron(): +def setup_steward_cron() -> str: banner("Steward Cycle (every 6 hours)") if not ask_yn("Set up automatic steward cycle every 6 hours?"): - return + return "blocked" steward_script = str(CLAUDE_DIR / "hooks" / "memorymaster-steward-cycle.py") @@ -489,14 +613,18 @@ def setup_steward_cron(): ) if result.returncode == 0: print(" Created Windows scheduled task: MemoryMasterSteward (every 6h)") + return "configured" else: print(f" Failed to create task: {result.stderr}") + return "blocked" except Exception as e: print(f" Error: {e}") + return "blocked" else: cron_line = f"0 */6 * * * {PYTHON_EXE} {steward_script} >> /var/log/memorymaster-steward.log 2>&1" print(" Add this to your crontab (crontab -e):") print(f" {cron_line}") + return "manual" # --------------------------------------------------------------------------- @@ -594,9 +722,17 @@ def setup_full_stack(detected: Detected, *, interactive: bool, yes: bool, model: return result compose_file = PROJECT_ROOT / "docker-compose.yml" + if not compose_file.is_file(): + result["degraded"] = True + result["message"] = ( + "No docker-compose.yml exists in the selected project. Installed-wheel " + "setup does not ship deployment assets; configure authenticated services " + "explicitly or run from a release bundle." + ) + print(f" {result['message']}") + return result up_args = ["docker", "compose"] - if compose_file.is_file(): - up_args += ["-f", str(compose_file)] + up_args += ["-f", str(compose_file)] services = [s for s, needed in (("qdrant", qdrant_needed), ("ollama", ollama_needed)) if needed] up_args += ["up", "-d", *services] @@ -700,6 +836,12 @@ def build_arg_parser() -> argparse.ArgumentParser: description="Detect-first, idempotent MemoryMaster onboarding.", ) p.add_argument("-y", "--yes", action="store_true", help="non-interactive; accept all defaults") + p.add_argument( + "--profile", + choices=tuple(SETUP_PROFILES), + default=None, + help="verify a named setup profile (default: minimal for normal setup)", + ) p.add_argument("--db", help="path to memorymaster.db (overrides project-root default)") p.add_argument( "--provider", @@ -799,15 +941,41 @@ def _run_main(args: argparse.Namespace) -> tuple[int, Optional[dict[str, Any]]]: # non-interactive (--yes), or for a quick --verify-only check — stdin may # not be a tty (CI, `docker exec`, an agent), where input() raises EOFError. set_non_interactive(bool(args.yes) or bool(args.json) or bool(args.verify_only)) - want_full_stack = True if args.full_stack is None else bool(args.full_stack) + profile = args.profile or "minimal" + want_full_stack = ( + profile in {"semantic", "full-lab"} + if args.full_stack is None + else bool(args.full_stack) + ) # --verify-only short-circuits BEFORE any project-root prompt: it only needs # --db (defaulting to cwd/memorymaster.db) and must never block on input. if args.verify_only: - vdb = Path(args.db).expanduser().resolve() if args.db else Path.cwd() / "memorymaster.db" + vdb: str | Path + if args.db and "://" in args.db: + vdb = args.db + else: + vdb = Path(args.db).expanduser().resolve() if args.db else Path.cwd() / "memorymaster.db" verify = verify_install(vdb) - rc = 0 if verify.get("status") in ("PASS", "PARTIAL") else 1 - return rc, ({"verify": verify} if args.json else None) + if args.profile is None: + rc = 0 if verify.get("status") == "PASS" else 3 if verify.get("status") == "PARTIAL" else 1 + return rc, ({"verify": verify} if args.json else None) + detected = detect_environment(cwd=Path.cwd()) + components = _setup_components( + profile=profile, + detected=detected, + applied={}, + verify=verify, + llm_config=_resolve_provider_config(args), + db_target=vdb, + ) + status, rc = evaluate_setup_profile(profile, components) + payload = { + "verify": verify, + "profile": {"name": profile, "status": status}, + "components": {name: asdict(result) for name, result in components.items()}, + } + return rc, (payload if args.json else None) # --- Resolve project root (flag > prompt > cwd) --- if args.project_root: @@ -818,7 +986,10 @@ def _run_main(args: argparse.Namespace) -> tuple[int, Optional[dict[str, Any]]]: PROJECT_ROOT = Path(root_input).expanduser().resolve() PROJECT_ROOT.mkdir(parents=True, exist_ok=True) - db_path = Path(args.db).expanduser().resolve() if args.db else PROJECT_ROOT / "memorymaster.db" + if args.db and "://" in args.db: + db_path: str | Path = args.db + else: + db_path = Path(args.db).expanduser().resolve() if args.db else PROJECT_ROOT / "memorymaster.db" # --- Detect-first --- detected = detect_environment(cwd=PROJECT_ROOT) @@ -837,7 +1008,7 @@ def _run_main(args: argparse.Namespace) -> tuple[int, Optional[dict[str, Any]]]: applied: dict[str, Any] = {} # --- Init DB if needed --- - if not db_path.exists(): + if isinstance(db_path, Path) and not db_path.exists(): if not args.json: print("\n Initializing database...") try: @@ -888,8 +1059,7 @@ def _run_main(args: argparse.Namespace) -> tuple[int, Optional[dict[str, Any]]]: # --- Steward cron (references the Claude hook script; needs ~/.claude) --- if not args.no_cron and detected.claude_code: - setup_steward_cron() - applied["cron"] = "configured" + applied["cron"] = setup_steward_cron() else: applied["cron"] = "skipped" if args.no_cron else "skipped (no ~/.claude/)" @@ -914,7 +1084,17 @@ def _run_main(args: argparse.Namespace) -> tuple[int, Optional[dict[str, Any]]]: # --- Verify --- verify = verify_install(db_path) - banner("Setup Complete!") + components = _setup_components( + profile=profile, + detected=detected, + applied=applied, + verify=verify, + llm_config=llm_config, + db_target=db_path, + ) + profile_status, profile_rc = evaluate_setup_profile(profile, components) + + banner(f"Setup {profile_status}") print(" What actually happened (skips reflect what's installed on this box):") if detected.claude_code: print(" - Claude hooks (session-start + on-demand recall + session-end distill) — installed") @@ -937,17 +1117,17 @@ def _run_main(args: argparse.Namespace) -> tuple[int, Optional[dict[str, Any]]]: print() if args.json: - from dataclasses import asdict - payload = { "detected": asdict(detected), "planned": plan, "applied": applied, "verify": verify, "degraded": degraded, + "profile": {"name": profile, "status": profile_status}, + "components": {name: asdict(result) for name, result in components.items()}, } - return 0, payload - return 0, None + return profile_rc, payload + return profile_rc, None if __name__ == "__main__": diff --git a/pyproject.toml b/pyproject.toml index 4807e79c..9dd23817 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,7 @@ memorymaster-mcp = "memorymaster.surfaces.mcp_server:main" memorymaster-dashboard = "memorymaster.surfaces.dashboard:main" memorymaster-steward = "memorymaster.govern.llm_steward:main" memorymaster-setup = "memorymaster.surfaces.setup_hooks:main" +memorymaster-session-end = "memorymaster.surfaces.session_end_ingest:main" [tool.ruff] target-version = "py310" diff --git a/tests/test_session_end_capture_hook.py b/tests/test_session_end_capture_hook.py index 97533443..26989215 100644 --- a/tests/test_session_end_capture_hook.py +++ b/tests/test_session_end_capture_hook.py @@ -24,6 +24,12 @@ def run(db_path, transcript_path, *, source_agent, cwd): """, encoding="utf-8", ) + (project / "sitecustomize.py").write_text( + "import importlib, sys\n" + "sys.modules['memorymaster.surfaces.session_end_ingest'] = " + "importlib.import_module('scripts.agent_session_end_ingest')\n", + encoding="utf-8", + ) hook = tmp_path / "session-end.py" hook.write_text( TEMPLATE.read_text(encoding="utf-8").replace("__MEMORYMASTER_PROJECT_ROOT__", str(project).replace("\\", "/")), @@ -36,7 +42,7 @@ def run(db_path, transcript_path, *, source_agent, cwd): { "MEMORYMASTER_CAPTURE_STATE_DB": str(tmp_path / "capture.db"), "MEMORYMASTER_LLM_PROVIDER": "fake", - "PYTHONPATH": os.pathsep.join((str(ROOT), str(project))), + "PYTHONPATH": os.pathsep.join((str(project), str(ROOT))), } ) payload = {"session_id": "session-end", "transcript_path": str(transcript), "cwd": str(project)} diff --git a/tests/test_setup_hooks.py b/tests/test_setup_hooks.py index 62507361..63a959b2 100644 --- a/tests/test_setup_hooks.py +++ b/tests/test_setup_hooks.py @@ -264,6 +264,9 @@ def test_reuses_already_healthy(self, hermetic_home, no_real_subprocess): assert result["ollama"] == "reused" def test_compose_up_when_available(self, hermetic_home, no_real_subprocess): + (hermetic_home["project"] / "docker-compose.yml").write_text( + "services: {}\n", encoding="utf-8" + ) det = _detected(docker_compose=True, qdrant=False, ollama=False) result = sh.setup_full_stack(det, interactive=False, yes=True, model="llama3.2:3b") assert result["compose_run"] is True @@ -395,6 +398,23 @@ def test_no_docker_fallback_exits_zero(self, monkeypatch, hermetic_home, capsys) payload = json.loads(out) assert payload["degraded"] is True + def test_semantic_profile_blocks_when_provider_and_vector_are_unavailable( + self, monkeypatch, hermetic_home, capsys + ): + det = _detected(docker_compose=False, qdrant=False, ollama=False) + argv = self._argv( + hermetic_home, + extra=["--profile", "semantic", "--json"], + ) + rc = _run_main(monkeypatch, hermetic_home, det, argv) + payload = json.loads(capsys.readouterr().out) + + assert rc != 0 + assert payload["profile"]["name"] == "semantic" + assert payload["profile"]["status"] == "BLOCKED" + assert payload["components"]["provider"]["status"] == "BLOCKED" + assert payload["components"]["vector_backend"]["status"] == "BLOCKED" + def test_json_output_parses(self, monkeypatch, hermetic_home, capsys): det = _detected(claude_code=True) rc = _run_main( @@ -402,6 +422,7 @@ def test_json_output_parses(self, monkeypatch, hermetic_home, capsys): ) assert rc == 0 payload = json.loads(capsys.readouterr().out) + assert payload["profile"]["status"] == "PASS" assert set(["detected", "planned", "applied", "verify", "degraded"]).issubset(payload) assert payload["verify"]["status"] in ("PASS", "PARTIAL") @@ -472,8 +493,7 @@ class TestFromZeroRegressions: """ def test_no_claude_code_completes_without_crash(self, hermetic_home, no_real_subprocess, monkeypatch, capsys): - """~/.claude absent + claude_code=False: setup must finish (exit 0), not - crash in append_instructions writing into a non-existent ~/.claude.""" + """Missing clients are reported BLOCKED without crashing or writing HOME.""" import shutil shutil.rmtree(hermetic_home["claude_dir"], ignore_errors=True) monkeypatch.setattr(sh, "detect_environment", lambda *a, **kw: _detected(claude_code=False)) @@ -482,8 +502,9 @@ def test_no_claude_code_completes_without_crash(self, hermetic_home, no_real_sub ["--yes", "--no-full-stack", "--no-cron", "--no-obsidian-skills", "--project-root", str(hermetic_home["project"]), "--db", str(db), "--json"] ) - assert rc == 0 + assert rc == 2 payload = json.loads(capsys.readouterr().out) + assert payload["profile"]["status"] == "BLOCKED" assert payload["applied"]["hooks"].startswith("skipped") assert payload["applied"]["mcp_claude"].startswith("skipped") assert not (hermetic_home["claude_dir"] / "CLAUDE.md").exists() # never created a fake ~/.claude diff --git a/tests/test_setup_profiles.py b/tests/test_setup_profiles.py new file mode 100644 index 00000000..10c07a6b --- /dev/null +++ b/tests/test_setup_profiles.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from importlib import resources + +import memorymaster.surfaces.setup_hooks as setup + + +def test_setup_profiles_have_explicit_component_contracts() -> None: + assert set(setup.SETUP_PROFILES) == {"minimal", "semantic", "team", "full-lab"} + assert setup.SETUP_PROFILES["minimal"] == ("db", "mcp") + assert set(setup.SETUP_PROFILES["full-lab"]) == { + "db", + "mcp", + "recall_hook", + "capture_hook", + "provider", + "steward", + "vector_backend", + "dashboard", + } + + +def test_profile_aggregation_is_nonzero_for_partial_or_blocked() -> None: + passed = { + "db": setup.ComponentResult("db", "PASS", "ok"), + "mcp": setup.ComponentResult("mcp", "PASS", "ok"), + } + assert setup.evaluate_setup_profile("minimal", passed)[0] == "PASS" + passed["mcp"] = setup.ComponentResult("mcp", "PARTIAL", "restart required") + assert setup.evaluate_setup_profile("minimal", passed) == ("PARTIAL", 3) + passed["mcp"] = setup.ComponentResult("mcp", "BLOCKED", "not registered") + assert setup.evaluate_setup_profile("minimal", passed) == ("BLOCKED", 2) + + +def test_explicit_profile_parser_and_packaged_hook_assets() -> None: + args = setup.build_arg_parser().parse_args(["--profile", "semantic"]) + assert args.profile == "semantic" + hook = resources.files("memorymaster").joinpath( + "config_templates", "hooks", "memorymaster-session-end.py" + ) + assert hook.is_file() + hook_text = hook.read_text(encoding="utf-8") + assert "memorymaster.surfaces.session_end_ingest" in hook_text + assert "scripts.agent_session_end_ingest" not in hook_text + assert resources.files("memorymaster").joinpath( + "surfaces", "session_end_ingest.py" + ).is_file() From 2bfbf0ca4a4c0ab1f16a7cc937d380d442029f68 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Mon, 13 Jul 2026 23:57:39 -0300 Subject: [PATCH 40/55] fix: separate authenticated service entrypoints --- .github/workflows/ci.yml | 68 +++++++++ .planning/REMAINING-PHASES-V1-BUDGET.md | 37 +++++ .planning/audit-remediation-ledger.md | 4 +- Dockerfile | 6 +- docker-compose.yml | 23 +++- external-actions-required.md | 2 +- helm/memorymaster/templates/deployment.yaml | 50 +++++++ helm/memorymaster/values.yaml | 9 ++ memorymaster/surfaces/mcp_http.py | 144 ++++++++++++++++++++ pyproject.toml | 5 +- scripts/smoke_mcp_transport.py | 63 +++++++++ tests/test_deployment_contracts.py | 48 +++++-- tests/test_mcp_http_entrypoint.py | 138 +++++++++++++++++++ tests/test_supply_chain_contracts.py | 4 +- 14 files changed, 580 insertions(+), 21 deletions(-) create mode 100644 memorymaster/surfaces/mcp_http.py create mode 100644 scripts/smoke_mcp_transport.py create mode 100644 tests/test_mcp_http_entrypoint.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f89620e8..d250271e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -78,3 +78,71 @@ jobs: artifacts/eval/eval_results.json artifacts/eval/eval_report.md artifacts/eval/eval_checks.csv + + deployment-smoke: + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: azure/setup-helm@v4 + - name: Install MCP smoke dependencies + run: pip install -e ".[mcp]" + - name: Smoke stdio MCP handshake + run: python scripts/smoke_mcp_transport.py --transport stdio + - name: Build deployment image + run: docker build -t memorymaster-ci . + - name: Initialize disposable deployment database + run: | + mkdir -p .ci-data + docker run --rm -v "$PWD/.ci-data:/data" memorymaster-ci \ + memorymaster --db /data/memorymaster.db --workspace /data init-db + - name: Smoke dashboard health and readiness + run: | + docker run -d --name memorymaster-dashboard -p 127.0.0.1:8765:8765 \ + -e MEMORYMASTER_DASHBOARD_TOKEN_VIEWER=ci-dashboard-token \ + -v "$PWD/.ci-data:/data" memorymaster-ci + trap 'docker rm -f memorymaster-dashboard >/dev/null 2>&1 || true' EXIT + for attempt in $(seq 1 30); do + curl --fail --silent http://127.0.0.1:8765/healthz >/dev/null && \ + curl --fail --silent http://127.0.0.1:8765/readyz >/dev/null && exit 0 + sleep 1 + done + docker logs memorymaster-dashboard + exit 1 + - name: Smoke authenticated streamable-HTTP MCP handshake + env: + MEMORYMASTER_MCP_HTTP_TOKEN: ci-mcp-http-token + run: | + docker rm -f memorymaster-dashboard >/dev/null 2>&1 || true + docker run -d --name memorymaster-mcp-http -p 127.0.0.1:8765:8765 \ + -e MEMORYMASTER_MCP_HTTP_TOKEN \ + -e MEMORYMASTER_MCP_AUTH_MODE=local-trusted \ + -v "$PWD/.ci-data:/data" memorymaster-ci \ + memorymaster-mcp-http --host 0.0.0.0 --port 8765 \ + --db /data/memorymaster.db --workspace /data + trap 'docker rm -f memorymaster-mcp-http >/dev/null 2>&1 || true' EXIT + for attempt in $(seq 1 30); do + curl --fail --silent http://127.0.0.1:8765/readyz >/dev/null && break + sleep 1 + done + python scripts/smoke_mcp_transport.py --transport http + - name: Render both Helm HTTP profiles + env: + IMAGE_DIGEST: sha256:0000000000000000000000000000000000000000000000000000000000000000 + run: | + helm lint helm/memorymaster \ + --set image.digest="$IMAGE_DIGEST" \ + --set qdrant.apiKeySecret.name=qdrant-ci \ + --set service.dashboardTokenSecret.name=dashboard-ci + helm template mm-dashboard helm/memorymaster \ + --set image.digest="$IMAGE_DIGEST" \ + --set qdrant.apiKeySecret.name=qdrant-ci \ + --set service.dashboardTokenSecret.name=dashboard-ci > /tmp/dashboard.yaml + helm template mm-mcp helm/memorymaster \ + --set image.digest="$IMAGE_DIGEST" \ + --set qdrant.apiKeySecret.name=qdrant-ci \ + --set service.profile=mcp-http \ + --set service.mcpHttpTokenSecret.name=mcp-http-ci > /tmp/mcp-http.yaml diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index 0ee3ef53..4b593d51 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -372,3 +372,40 @@ R3.3 truthful setup profiles completed on 2026-07-13: The next package is R3.4 service entrypoints and readiness contracts. The active roadmap goal authorizes continuing without a new goal. + +R3.4 service entrypoints and readiness completed on 2026-07-13: + +- Stdio MCP remains the private agent-process entrypoint. A distinct + `memorymaster-mcp-http` entrypoint uses the official stateless streamable-HTTP + transport, requires a startup bearer token, validates allowed Host patterns, + leaves only `/healthz` and `/readyz` unauthenticated, and reports DB + connectivity failures as readiness 503 responses. The MCP dependency floor + now matches the streamable-HTTP API used by the installed wheel. +- Docker defaults to the authenticated dashboard HTTP profile instead of + publishing a stdio process. Compose exposes only loopback port 8765, selects + dashboard or MCP HTTP explicitly, uses real readiness, and bounds CPU/memory + for MemoryMaster, Qdrant, and Ollama. Helm rejects stdio/unknown service + profiles, sources dashboard/MCP tokens from existing Secrets, and defines + liveness, readiness, immutable image, storage, and resource contracts. +- CI now performs stdio initialize/tools-list, builds the image, runs dashboard + readiness, runs authenticated MCP HTTP initialize/tools-list, and renders + both Helm profiles. The protocol smoke uses the official MCP client rather + than treating an open port as proof. +- RED evidence: eight deployment/auth/readiness tests initially failed. Focused + deployment/dashboard/setup/supply-chain boundary: 97 passed with one existing + Pydantic deprecation warning. Changed-file Ruff and `git diff --check` pass + after the one unused-import correction. +- Runtime evidence: local stdio handshake passed; a disposable local HTTP MCP + returned readiness 200 and completed initialize/tools-list; the built + `memorymaster:r34-local` image returned dashboard readiness 200 and MCP HTTP + readiness 200 with a successful bearer-authenticated handshake. Compose + rendered successfully with synthetic required inputs. Temporary containers + and the named test volume were removed. +- Helm CLI and a disposable Kubernetes target were unavailable. The existing + MM-OPS-02 external runtime row remains `BLOCKED-EXTERNAL`; no Helm runtime + pass is claimed. Approved Qdrant/Ollama images, TLS runtime, and image scans + also remain external. +- Atomic package commit: the conventional R3.4 commit containing this evidence. + +The next package is R3.5 recovery, observability, and privacy. The active +roadmap goal authorizes continuing without a new goal. diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 4b95cb2a..93adf25e 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -21,9 +21,9 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-ARCH-02 | Architecture/Integrity | High / BAD-PRACTICE | MCP/hooks/CLI/Qdrant use contradictory retrieval planners and trust defaults | R2.1 | RESOLVED | P2-A/P2-B established the planner. P2-Z then witnessed a downstream prompt-stream bypass returning allowed, candidate, foreign, and private IDs together; the convergence correction applies the same trusted status/scope/visibility/sensitivity policy before and after all lexical, entity, graph, two-pass, closets, and verbatim streams. Adversarial RED and compositional correction evidence are in the Phase 2 delta. | | MM-LIFE-01 | Database/Reliability | High / BAD-PRACTICE | Scheduled archival uses raw SQL and bypasses lifecycle/vector evidence | R2.2 | RESOLVED | P2-C (`732cb68`) routes scheduled archival through canonical optimistic transitions with events, timestamps, cache invalidation, and replayable vector deletion; focused gate: 40 passed. | | MM-REL-02 | Reliability/Performance | Medium / BAD-PRACTICE | MCP reads take write locks and context detail modes retrieve twice | R2.2 | RESOLVED | P2-C makes top-level recall query-only, removes duplicate detail retrieval, and emits at most one sanitized aggregate telemetry envelope; read-only and telemetry contracts passed the focused/convergence matrices. | -| MM-OPS-02 | DevOps/Demo | High / BAD-PRACTICE | Docker/Helm publish HTTP on a stdio MCP process and use an invalid healthcheck | R3.4 | OPEN | Built-container readiness and MCP handshake | +| MM-OPS-02 | DevOps/Demo | High / BAD-PRACTICE | Docker/Helm publish HTTP on a stdio MCP process and use an invalid healthcheck | R3.4 | RESOLVED | R3.4 keeps stdio MCP private to the agent process and adds distinct authenticated streamable-HTTP MCP and dashboard entrypoints. Docker defaults to the dashboard; Compose is loopback-only, resource-bounded, and probes `/readyz`; Helm selects an explicit authenticated HTTP profile with liveness/readiness probes and resource limits. A locally built image returned 200 from dashboard and MCP HTTP readiness, and the official MCP client completed initialize/tools-list handshakes over stdio and authenticated HTTP. Focused deployment/supply-chain boundary: 97 passed. Helm CLI/disposable cluster evidence remains tracked externally without reopening the repository protocol mismatch. | | MM-OPS-03 | DevOps/Maintainability | High / BAD-PRACTICE | A release tag can publish without a blocking verified test artifact | R4.4 | OPEN | Deliberately failing release candidate cannot publish | -| MM-OPS-04 | Security/DevOps | High / EXPLOITABLE-LOW-EFFORT | Qdrant/Ollama ports are broadly exposed; images are mutable | R1.5,R3.4 | IN-PROGRESS | Phase 1 defaults are repository-complete (`a858419`, `9b3e16c`, `b71e18f`): loopback-only ports, required immutable digests, authenticated TLS Qdrant, redirect-safe credentials, and fail-closed supply-chain policy. Real image scans/runtime health remain `BLOCKED-EXTERNAL`; R3.4 service-entrypoint work remains backlog. | +| MM-OPS-04 | Security/DevOps | High / EXPLOITABLE-LOW-EFFORT | Qdrant/Ollama ports are broadly exposed; images are mutable | R1.5,R3.4 | BLOCKED-EXTERNAL | Repository defaults are complete: loopback-only service and auxiliary ports, required immutable digests, authenticated TLS Qdrant, redirect-safe credentials, explicit authenticated HTTP service profiles, real readiness, resource limits, and fail-closed supply-chain policy. The local MemoryMaster image passed dashboard/MCP HTTP runtime smoke. Approved release-image scans plus authenticated Qdrant/Ollama and disposable Kubernetes runtime evidence remain external. | | MM-PERF-01 | Performance/Cost | High / BAD-PRACTICE | Hybrid reads recompute and rewrite candidate embeddings | R3.1 | RESOLVED | R3.1 migration 0014 persists model/content fingerprints; focused warm-query evidence proves one query embed and zero candidate embeds/writes. | | MM-PERF-02 | Performance | Medium / BAD-PRACTICE | Each process cold-scans corpus token statistics and caches forever | R3.2 | RESOLVED | R3.2 keys corpus and alias statistics by the persisted corpus generation, so warm reads reuse one scan and every claim mutation invalidates it. Focused evidence proves a new cache miss after generation change while tokenizer ranking/retired-status contracts remain green. | | MM-PERF-03 | Performance/Cost | High / EXPLOITABLE-NOW | Qdrant reconciliation repeats/truncates tens of thousands of embeddings | R3.1 | RESOLVED | R3.1 replaces four truncated 10,000-row scans with authority-scoped keyset pages and a replayable durable cursor; fake-backend convergence/failure-resume evidence passed. Real authenticated/TLS Qdrant remains externally blocked. | diff --git a/Dockerfile b/Dockerfile index 229d4a8c..fc8c2c4d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,5 +34,7 @@ VOLUME /data # Expose dashboard port EXPOSE 8765 -# Default command: run MCP server -CMD ["memorymaster-mcp"] +# The image defaults to the HTTP dashboard. The stdio and authenticated HTTP +# MCP services remain explicit alternatives: memorymaster-mcp and +# memorymaster-mcp-http. +CMD ["memorymaster-dashboard", "--host", "0.0.0.0", "--port", "8765", "--db", "/data/memorymaster.db", "--workspace", "/data"] diff --git a/docker-compose.yml b/docker-compose.yml index 61418f34..56a3bf9e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,8 +4,18 @@ services: context: . dockerfile: Dockerfile container_name: memorymaster + command: + - "${MEMORYMASTER_SERVICE_ENTRYPOINT:-memorymaster-dashboard}" + - --host + - 0.0.0.0 + - --port + - "8765" + - --db + - /data/memorymaster.db + - --workspace + - /data ports: - - "8765:8765" + - "127.0.0.1:8765:8765" volumes: - ./data:/data - type: bind @@ -15,6 +25,9 @@ services: environment: MEMORYMASTER_DEFAULT_DB: /data/memorymaster.db MEMORYMASTER_WORKSPACE: /data + MEMORYMASTER_DASHBOARD_TOKEN_VIEWER: "${MEMORYMASTER_DASHBOARD_TOKEN_VIEWER:-}" + MEMORYMASTER_MCP_HTTP_TOKEN: "${MEMORYMASTER_MCP_HTTP_TOKEN:-}" + MEMORYMASTER_MCP_HTTP_ALLOWED_HOSTS: "127.0.0.1:*,localhost:*" QDRANT_URL: https://qdrant:6333 QDRANT_API_KEY: "${QDRANT_API_KEY:?Set QDRANT_API_KEY before rendering Compose}" QDRANT_CA_CERT: /run/secrets/qdrant-ca.pem @@ -25,8 +38,10 @@ services: ollama: condition: service_started restart: unless-stopped + cpus: "0.50" + mem_limit: 512m healthcheck: - test: ["CMD", "memorymaster", "--version"] + test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8765/readyz', timeout=5)"] interval: 30s timeout: 10s retries: 3 @@ -58,6 +73,8 @@ services: target: /qdrant/tls/ca.pem read_only: true restart: unless-stopped + cpus: "1.00" + mem_limit: 1g healthcheck: test: - CMD-SHELL @@ -78,6 +95,8 @@ services: volumes: - ollama_data:/root/.ollama restart: unless-stopped + cpus: "2.00" + mem_limit: 4g volumes: qdrant_data: diff --git a/external-actions-required.md b/external-actions-required.md index 663c34a9..c548e635 100644 --- a/external-actions-required.md +++ b/external-actions-required.md @@ -10,7 +10,7 @@ live mutation by itself. | MM-OPS-01 | Operator / any Postgres deployment | Rotate the historical `mm_pw` credential and recreate deployments that used it. Keep database ports private. | Rotation record plus network probe showing the port is not externally reachable | 2026-07-31 | BLOCKED-EXTERNAL | | MM-SEC-01 / disposable proof | Team Postgres test environment | Provide one fully disposable database, a dedicated schema-owning `SUPERUSER`/`BYPASSRLS` migrator DSN in `MEMORYMASTER_TEST_POSTGRES_DSN`, and a distinct restricted app DSN in `MEMORYMASTER_TEST_POSTGRES_APP_DSN`; set `MEMORYMASTER_TEST_POSTGRES_RLS_DISPOSABLE=1`. Neither DSN may target product/live data. | Run `tests/test_postgres_rls_integration.py` plus every Postgres parameter in `tests/test_postgres_parity.py` and `tests/test_backend_parity.py`. Retain exact FORCE-RLS policy/function/trigger/index catalog evidence; literal-sensitive fingerprints; event-head owner `SUPERUSER`/`BYPASSRLS`; app event SELECT/INSERT with no table/column UPDATE or DELETE; cross-tenant/scope/visibility/principal denials; tenant-chain linearity; scope-local identities and ambiguity rejection; self/cross-boundary supersession denial; successful reciprocal supersession plus one event; rollback after replacement-update/event-insert failure; and parity results from UUID-isolated tenants with no destructive cleanup. No real PostgreSQL proof has been performed here. | 2026-07-31 | BLOCKED-EXTERNAL | | MM-SEC-01 / brownfield data | Live database operator | Authorize a read-only inventory before v0012. Any owner backfill, duplicate/supersession-edge repair, or grant change requires separate explicit approval, a verified backup, and a maintenance window. | Inventory noncanonical visibility; blank/null `source_agent` on every claim; identity duplicates; both supersession columns for self, missing-target, nonreciprocal, cross-tenant/scope/visibility/owner edges; current event/function/table grants; and constraint state without writing. After approval, retain repair/grant evidence and rerun v0012, which validates `ck_claims_identity_visibility_owner`. No product-data verification or mutation has been performed here. | 2026-07-31 | BLOCKED-EXTERNAL | -| MM-OPS-02 | Docker/Helm runtime | Provide approved immutable MemoryMaster/Qdrant/Ollama images, certificates, and a disposable Kubernetes target. Docker/Compose config was available locally; Helm and approved product images were not. | Container health/MCP handshake, authenticated TLS Qdrant health, and Helm readiness/network-policy evidence | 2026-08-15 | BLOCKED-EXTERNAL | +| MM-OPS-02 | Docker/Helm runtime | Provide approved immutable MemoryMaster/Qdrant/Ollama release images, certificates, Helm CLI, and a disposable Kubernetes target. R3.4 built and smoked a local MemoryMaster development image plus Compose rendering; it did not use approved release images or a cluster. | Authenticated TLS Qdrant/Ollama health, image scans, and Helm dashboard/MCP-HTTP readiness, handshake, Secret, resource, and network-policy evidence | 2026-08-15 | BLOCKED-EXTERNAL | | MM-SEC-02 | Qdrant runtime | Provide a disposable authenticated/TLS Qdrant target for final parity after the fake-backed governed candidate suite. The semantic profile remains off by default; do not point tests at a live/product collection. | API-key/TLS proof plus real-service ID/hash-only search, authoritative rehydration denials, exact equal-count hash reconciliation, bounded outbox replay, and sync/delete evidence; retain exact command output without secrets | 2026-08-15 | BLOCKED-EXTERNAL | | MM-SEC-03 | Live legacy stores | Review the aggregate inventory, provide a verified backup, and separately authorize any cleanup/redaction/rebuild. Configure a disposable or approved read-only Qdrant inventory target before claiming complete coverage. | SQLite/artifact/spool/Qdrant aggregate rerun with zero unscannable/truncated surfaces, plus approved remediation evidence; never include raw findings | 2026-07-31 | BLOCKED-EXTERNAL | | MM-OPS-04 / history scan | Security owner | Classify the unsuppressed Gitleaks history result: 40 potential findings across 10 commits and 7 files. Rotate any affected credentials before considering an approved history action. | Aggregate classification and rotation record; no raw secret material in general logs | 2026-07-31 | BLOCKED-EXTERNAL | diff --git a/helm/memorymaster/templates/deployment.yaml b/helm/memorymaster/templates/deployment.yaml index 17ec3732..77589e34 100644 --- a/helm/memorymaster/templates/deployment.yaml +++ b/helm/memorymaster/templates/deployment.yaml @@ -15,6 +15,14 @@ {{- fail "qdrant.url must use HTTPS" -}} {{- end -}} {{- $qdrantApiKeySecret := required "qdrant.apiKeySecret.name is required" .Values.qdrant.apiKeySecret.name -}} +{{- if not (has .Values.service.profile (list "dashboard" "mcp-http")) -}} +{{- fail "service.profile must be dashboard or mcp-http; stdio MCP cannot be published as HTTP" -}} +{{- end -}} +{{- if eq .Values.service.profile "dashboard" -}} +{{- $_ := required "service.dashboardTokenSecret.name is required for the dashboard profile" .Values.service.dashboardTokenSecret.name -}} +{{- else -}} +{{- $_ := required "service.mcpHttpTokenSecret.name is required for the mcp-http profile" .Values.service.mcpHttpTokenSecret.name -}} +{{- end -}} apiVersion: apps/v1 kind: Deployment metadata: @@ -33,6 +41,21 @@ spec: - name: memorymaster image: "{{ .Values.image.repository }}@{{ $imageDigest }}" imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + {{- if eq .Values.service.profile "dashboard" }} + - memorymaster-dashboard + {{- else }} + - memorymaster-mcp-http + {{- end }} + args: + - --host + - 0.0.0.0 + - --port + - "8765" + - --db + - /data/memorymaster.db + - --workspace + - /data ports: - containerPort: 8765 env: @@ -47,6 +70,21 @@ spec: secretKeyRef: name: {{ $qdrantApiKeySecret | quote }} key: {{ .Values.qdrant.apiKeySecret.key | quote }} + {{- if eq .Values.service.profile "dashboard" }} + - name: MEMORYMASTER_DASHBOARD_TOKEN_VIEWER + valueFrom: + secretKeyRef: + name: {{ .Values.service.dashboardTokenSecret.name | quote }} + key: {{ .Values.service.dashboardTokenSecret.key | quote }} + {{- else }} + - name: MEMORYMASTER_MCP_HTTP_TOKEN + valueFrom: + secretKeyRef: + name: {{ .Values.service.mcpHttpTokenSecret.name | quote }} + key: {{ .Values.service.mcpHttpTokenSecret.key | quote }} + - name: MEMORYMASTER_MCP_HTTP_ALLOWED_HOSTS + value: {{ printf "%s-memorymaster:*,127.0.0.1:*,localhost:*" .Release.Name | quote }} + {{- end }} {{- if .Values.qdrant.caSecret.name }} - name: QDRANT_CA_CERT value: {{ printf "%s/ca.crt" .Values.qdrant.caSecret.mountPath | quote }} @@ -61,6 +99,18 @@ spec: {{- end }} resources: {{- toYaml .Values.resources | nindent 12 }} + livenessProbe: + httpGet: + path: /healthz + port: 8765 + initialDelaySeconds: 5 + periodSeconds: 15 + readinessProbe: + httpGet: + path: /readyz + port: 8765 + initialDelaySeconds: 5 + periodSeconds: 10 volumes: - name: data {{- if .Values.persistence.enabled }} diff --git a/helm/memorymaster/values.yaml b/helm/memorymaster/values.yaml index 88a4cf79..899a3625 100644 --- a/helm/memorymaster/values.yaml +++ b/helm/memorymaster/values.yaml @@ -7,8 +7,17 @@ image: pullPolicy: IfNotPresent service: + # Supported HTTP deployment profiles. Stdio MCP is intentionally not exposed + # through a Kubernetes Service; run memorymaster-mcp from an agent process. + profile: dashboard type: ClusterIP port: 8765 + dashboardTokenSecret: + name: "" + key: viewer-token + mcpHttpTokenSecret: + name: "" + key: bearer-token persistence: enabled: true diff --git a/memorymaster/surfaces/mcp_http.py b/memorymaster/surfaces/mcp_http.py new file mode 100644 index 00000000..73f75b7f --- /dev/null +++ b/memorymaster/surfaces/mcp_http.py @@ -0,0 +1,144 @@ +"""Authenticated streamable-HTTP entrypoint for MemoryMaster MCP.""" + +from __future__ import annotations + +import argparse +import hmac +import os +from pathlib import Path +from typing import Any + +from starlette.responses import JSONResponse +from starlette.routing import Route + +from memorymaster.surfaces.mcp_server import FastMCP, _read_service, mcp + + +TOKEN_ENV = "MEMORYMASTER_MCP_HTTP_TOKEN" +ALLOWED_HOSTS_ENV = "MEMORYMASTER_MCP_HTTP_ALLOWED_HOSTS" +DEFAULT_ALLOWED_HOSTS = ("127.0.0.1:*", "localhost:*", "[::1]:*") + + +class BearerAuthMiddleware: + """Protect MCP traffic without blocking unauthenticated health probes.""" + + def __init__(self, app: Any, *, token: str) -> None: + self.app = app + self.token = token + + async def __call__(self, scope: dict[str, Any], receive: Any, send: Any) -> None: + if scope.get("type") == "http" and scope.get("path") not in {"/healthz", "/readyz"}: + supplied = self._bearer_token(scope.get("headers", [])) + if supplied is None or not hmac.compare_digest(supplied, self.token): + response = JSONResponse( + {"status": "fail", "error": "unauthorized"}, + status_code=401, + headers={"WWW-Authenticate": "Bearer"}, + ) + await response(scope, receive, send) + return + await self.app(scope, receive, send) + + @staticmethod + def _bearer_token(headers: list[tuple[bytes, bytes]]) -> str | None: + for name, value in headers: + if name.lower() != b"authorization": + continue + scheme, separator, token = value.decode("latin-1").partition(" ") + if separator and scheme.lower() == "bearer" and token.strip(): + return token.strip() + return None + + +def _required_token(explicit: str | None = None) -> str: + token = (explicit if explicit is not None else os.environ.get(TOKEN_ENV, "")).strip() + if not token: + raise RuntimeError(f"{TOKEN_ENV} is required for the streamable-HTTP MCP service") + return token + + +def _allowed_hosts(explicit: list[str] | tuple[str, ...] | None = None) -> list[str]: + if explicit is not None: + values = [str(value).strip() for value in explicit] + else: + configured = os.environ.get(ALLOWED_HOSTS_ENV, "") + values = configured.split(",") if configured else list(DEFAULT_ALLOWED_HOSTS) + resolved = [value for value in values if value] + if not resolved: + raise RuntimeError(f"{ALLOWED_HOSTS_ENV} must contain at least one host pattern") + return resolved + + +def _db_readiness(db_target: str, workspace: str) -> dict[str, str]: + try: + service = _read_service(db_target, workspace) + with service.store.connect() as connection: + connection.execute("SELECT 1") + except Exception as exc: + return {"status": "fail", "error": str(exc)} + return {"status": "ok"} + + +def create_http_app( + *, + token: str | None = None, + db_target: str | Path | None = None, + workspace: str | Path | None = None, + allowed_hosts: list[str] | tuple[str, ...] | None = None, +) -> Any: + """Build the authenticated MCP ASGI app with liveness/readiness routes.""" + if FastMCP is None: # pragma: no cover - optional dependency guard + raise RuntimeError("MCP support is not installed. Install with: pip install 'memorymaster[mcp]'") + if not hasattr(mcp, "streamable_http_app"): # pragma: no cover - old SDK guard + raise RuntimeError("The installed MCP SDK lacks streamable HTTP support; install 'mcp>=1.8.1'") + resolved_token = _required_token(token) + resolved_db = str(db_target or os.environ.get("MEMORYMASTER_DEFAULT_DB", "memorymaster.db")) + resolved_workspace = str(workspace or os.environ.get("MEMORYMASTER_WORKSPACE", ".")) + + async def healthz(_request: Any) -> JSONResponse: + return JSONResponse({"status": "ok", "service": "memorymaster-mcp-http"}) + + async def readyz(_request: Any) -> JSONResponse: + db_check = _db_readiness(resolved_db, resolved_workspace) + ready = db_check["status"] == "ok" + return JSONResponse( + {"status": "ok" if ready else "fail", "checks": {"db": db_check}}, + status_code=200 if ready else 503, + ) + + mcp.settings.streamable_http_path = "/mcp" + mcp.settings.json_response = True + mcp.settings.stateless_http = True + mcp.settings.transport_security.allowed_hosts = _allowed_hosts(allowed_hosts) + app = mcp.streamable_http_app() + app.routes.insert(0, Route("/readyz", readyz, methods=["GET"])) + app.routes.insert(0, Route("/healthz", healthz, methods=["GET"])) + app.add_middleware(BearerAuthMiddleware, token=resolved_token) + return app + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Run authenticated MemoryMaster streamable-HTTP MCP") + parser.add_argument("--host", default="127.0.0.1", help="Bind host") + parser.add_argument("--port", type=int, default=8765, help="Bind port") + parser.add_argument("--db", default=os.environ.get("MEMORYMASTER_DEFAULT_DB", "memorymaster.db")) + parser.add_argument("--workspace", default=os.environ.get("MEMORYMASTER_WORKSPACE", ".")) + parser.add_argument("--allowed-host", action="append", dest="allowed_hosts") + return parser + + +def main(argv: list[str] | None = None) -> int: + args = _build_parser().parse_args(argv) + app = create_http_app( + db_target=args.db, + workspace=args.workspace, + allowed_hosts=args.allowed_hosts, + ) + import uvicorn + + uvicorn.run(app, host=args.host, port=args.port) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 9dd23817..04484bbf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,12 +45,15 @@ vector = ["sentence-transformers>=3.0", "qdrant-client>=1.9"] # MEMORYMASTER_RECALL_GRAPH=1; off by default. graph = ["kuzu>=0.4"] dev = ["pytest>=8.2", "pytest-cov>=6.0", "pytest-timeout>=2.3"] -mcp = ["mcp>=1.2"] +# Streamable HTTP and its ASGI app landed in the 1.8 line. Stdio remains the +# default transport; the higher floor makes the explicit HTTP entrypoint real. +mcp = ["mcp>=1.8.1"] ml = ["scikit-learn>=1.3", "joblib>=1.3"] [project.scripts] memorymaster = "memorymaster.surfaces.cli:main" memorymaster-mcp = "memorymaster.surfaces.mcp_server:main" +memorymaster-mcp-http = "memorymaster.surfaces.mcp_http:main" memorymaster-dashboard = "memorymaster.surfaces.dashboard:main" memorymaster-steward = "memorymaster.govern.llm_steward:main" memorymaster-setup = "memorymaster.surfaces.setup_hooks:main" diff --git a/scripts/smoke_mcp_transport.py b/scripts/smoke_mcp_transport.py new file mode 100644 index 00000000..55c7d1c4 --- /dev/null +++ b/scripts/smoke_mcp_transport.py @@ -0,0 +1,63 @@ +"""Protocol-level MCP initialize/tools-list smoke for stdio or HTTP.""" + +from __future__ import annotations + +import argparse +import asyncio +import os + +import httpx +from mcp import ClientSession, StdioServerParameters +from mcp.client.stdio import stdio_client +from mcp.client.streamable_http import streamable_http_client + + +async def _assert_session(read_stream, write_stream) -> None: + async with ClientSession(read_stream, write_stream) as session: + initialized = await session.initialize() + server_info = getattr(initialized, "serverInfo", None) + server_name = getattr(server_info, "name", "") + if server_name != "memorymaster": + raise RuntimeError(f"unexpected MCP server: {server_name}") + tools = await session.list_tools() + names = {tool.name for tool in tools.tools} + if "query_memory" not in names: + raise RuntimeError("MemoryMaster MCP handshake omitted query_memory") + + +async def _smoke_stdio() -> None: + parameters = StdioServerParameters( + command="memorymaster-mcp", + env={**os.environ, "MEMORYMASTER_MCP_AUTH_MODE": "local-trusted"}, + ) + async with stdio_client(parameters) as (read_stream, write_stream): + await _assert_session(read_stream, write_stream) + + +async def _smoke_http(url: str, token: str) -> None: + if not token: + raise RuntimeError("--token or MEMORYMASTER_MCP_HTTP_TOKEN is required for HTTP smoke") + async with httpx.AsyncClient(headers={"Authorization": f"Bearer {token}"}) as client: + async with streamable_http_client(url, http_client=client) as (read_stream, write_stream, _): + await _assert_session(read_stream, write_stream) + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser() + parser.add_argument("--transport", required=True, choices=("stdio", "http")) + parser.add_argument("--url", default="http://127.0.0.1:8765/mcp") + parser.add_argument("--token", default=os.environ.get("MEMORYMASTER_MCP_HTTP_TOKEN", "")) + return parser + + +def main() -> int: + args = _build_parser().parse_args() + if args.transport == "stdio": + asyncio.run(_smoke_stdio()) + else: + asyncio.run(_smoke_http(args.url, args.token)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_deployment_contracts.py b/tests/test_deployment_contracts.py index 7e7c6a77..cd8d12e2 100644 --- a/tests/test_deployment_contracts.py +++ b/tests/test_deployment_contracts.py @@ -5,9 +5,6 @@ import re from pathlib import Path -import pytest - - ROOT = Path(__file__).resolve().parents[1] @@ -59,10 +56,6 @@ def test_auxiliary_compose_requires_authenticated_qdrant(): assert "--ca-certificate=/qdrant/tls/ca.pem" in compose -@pytest.mark.xfail( - strict=True, - reason="R3.4: container publishes HTTP but launches stdio MCP with a CLI-only healthcheck", -) def test_container_entrypoint_and_healthcheck_share_an_http_contract(): dockerfile = _read("Dockerfile") compose = _read("docker-compose.yml") @@ -72,10 +65,6 @@ def test_container_entrypoint_and_healthcheck_share_an_http_contract(): assert '"--version"' not in compose -@pytest.mark.xfail( - strict=True, - reason="R3.4: Helm deployment has no liveness/readiness probes", -) def test_helm_deployment_defines_health_and_readiness_probes(): deployment = _read("helm/memorymaster/templates/deployment.yaml") @@ -85,6 +74,43 @@ def test_helm_deployment_defines_health_and_readiness_probes(): assert "/readyz" in deployment +def test_compose_http_service_is_private_and_resource_bounded(): + compose = _read("docker-compose.yml") + + assert '"127.0.0.1:8765:8765"' in compose + assert "MEMORYMASTER_SERVICE_ENTRYPOINT" in compose + assert "MEMORYMASTER_MCP_HTTP_TOKEN" in compose + assert "MEMORYMASTER_DASHBOARD_TOKEN_VIEWER" in compose + assert "cpus:" in compose + assert "mem_limit:" in compose + + +def test_helm_selects_an_explicit_authenticated_http_profile(): + values = _read("helm/memorymaster/values.yaml") + deployment = _read("helm/memorymaster/templates/deployment.yaml") + + assert "profile: dashboard" in values + assert "memorymaster-dashboard" in deployment + assert "memorymaster-mcp-http" in deployment + assert "dashboardTokenSecret" in values + assert "mcpHttpTokenSecret" in values + assert "MEMORYMASTER_MCP_HTTP_TOKEN" in deployment + assert "MEMORYMASTER_DASHBOARD_TOKEN_VIEWER" in deployment + + +def test_ci_builds_and_smokes_http_profiles_and_mcp_handshakes(): + workflow = _read(".github/workflows/ci.yml") + + assert "deployment-smoke:" in workflow + assert "docker build" in workflow + assert "memorymaster-dashboard" in workflow + assert "memorymaster-mcp-http" in workflow + assert "smoke_mcp_transport.py --transport stdio" in workflow + assert "smoke_mcp_transport.py --transport http" in workflow + assert "helm lint" in workflow + assert workflow.count("helm template") >= 2 + + def test_deployment_images_reject_latest_tags(): deployment_text = "\n".join( [ diff --git a/tests/test_mcp_http_entrypoint.py b/tests/test_mcp_http_entrypoint.py new file mode 100644 index 00000000..d57fd6bd --- /dev/null +++ b/tests/test_mcp_http_entrypoint.py @@ -0,0 +1,138 @@ +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import Any + +import pytest + + +TOKEN = "test-only-http-token" + + +async def _asgi_request( + app: Any, + path: str, + *, + method: str = "GET", + headers: dict[str, str] | None = None, + body: bytes = b"", +) -> tuple[int, dict[str, str], bytes]: + sent: list[dict[str, Any]] = [] + request_sent = False + + async def receive() -> dict[str, Any]: + nonlocal request_sent + if not request_sent: + request_sent = True + return {"type": "http.request", "body": body, "more_body": False} + return {"type": "http.disconnect"} + + async def send(message: dict[str, Any]) -> None: + sent.append(message) + + raw_headers = [(key.lower().encode(), value.encode()) for key, value in (headers or {}).items()] + scope = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": method, + "scheme": "http", + "path": path, + "raw_path": path.encode(), + "query_string": b"", + "headers": raw_headers, + "client": ("127.0.0.1", 12345), + "server": ("127.0.0.1", 8765), + "state": {}, + } + await app(scope, receive, send) + start = next(message for message in sent if message["type"] == "http.response.start") + response_headers = {key.decode(): value.decode() for key, value in start.get("headers", [])} + response_body = b"".join( + message.get("body", b"") for message in sent if message["type"] == "http.response.body" + ) + return int(start["status"]), response_headers, response_body + + +def test_http_entrypoint_requires_a_startup_token(monkeypatch: pytest.MonkeyPatch) -> None: + from memorymaster.surfaces import mcp_http + + monkeypatch.delenv("MEMORYMASTER_MCP_HTTP_TOKEN", raising=False) + with pytest.raises(RuntimeError, match="MEMORYMASTER_MCP_HTTP_TOKEN"): + mcp_http.create_http_app(db_target="unused.db") + + +def test_http_entrypoint_exposes_unauthenticated_liveness_but_protects_mcp( + tmp_path: Path, +) -> None: + from memorymaster.surfaces import mcp_http + + app = mcp_http.create_http_app(token=TOKEN, db_target=str(tmp_path / "missing.db")) + status, _, payload = asyncio.run(_asgi_request(app, "/healthz")) + assert status == 200 + assert json.loads(payload)["status"] == "ok" + + status, _, payload = asyncio.run(_asgi_request(app, "/mcp", method="POST")) + assert status == 401 + assert json.loads(payload)["error"] == "unauthorized" + + status, _, _ = asyncio.run( + _asgi_request( + app, + "/mcp", + method="POST", + headers={"authorization": "Bearer wrong-token"}, + ) + ) + assert status == 401 + + +def test_http_readiness_reports_missing_database(tmp_path: Path) -> None: + from memorymaster.surfaces import mcp_http + + app = mcp_http.create_http_app(token=TOKEN, db_target=str(tmp_path / "missing.db")) + status, _, payload = asyncio.run(_asgi_request(app, "/readyz")) + assert status == 503 + assert json.loads(payload)["checks"]["db"]["status"] == "fail" + + +def test_http_mcp_initialize_handshake_accepts_valid_bearer(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from memorymaster.core.service import MemoryService + from memorymaster.surfaces import mcp_http + + db = tmp_path / "ready.db" + MemoryService(db, workspace_root=tmp_path).init_db() + monkeypatch.setenv("MEMORYMASTER_MCP_AUTH_MODE", "local-trusted") + app = mcp_http.create_http_app(token=TOKEN, db_target=str(db), workspace=str(tmp_path)) + request = json.dumps( + { + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "r34-smoke", "version": "1"}, + }, + } + ).encode() + async def initialize() -> tuple[int, dict[str, str], bytes]: + async with app.router.lifespan_context(app): + return await _asgi_request( + app, + "/mcp", + method="POST", + headers={ + "authorization": f"Bearer {TOKEN}", + "content-type": "application/json", + "accept": "application/json, text/event-stream", + "host": "127.0.0.1:8765", + }, + body=request, + ) + + status, _, payload = asyncio.run(initialize()) + assert status == 200 + assert b'"serverInfo"' in payload diff --git a/tests/test_supply_chain_contracts.py b/tests/test_supply_chain_contracts.py index 02046db6..c16c2f69 100644 --- a/tests/test_supply_chain_contracts.py +++ b/tests/test_supply_chain_contracts.py @@ -63,7 +63,7 @@ def _prepare_repo(tmp_path: Path) -> Path: f'[project]\nname = "{EXPECTED_NAME}"\nversion = "{EXPECTED_VERSION}"\n' 'dependencies = ["requests>=2.31", "tenacity>=8.2"]\n' "[project.optional-dependencies]\n" - 'mcp = ["mcp>=1.2"]\n' + 'mcp = ["mcp>=1.8.1"]\n' 'qdrant = ["httpx>=0.27"]\n' 'security = ["cryptography>=42"]\n' ), @@ -278,7 +278,7 @@ def fake_runner(argv: tuple[str, ...], **_kwargs: object) -> subprocess.Complete assert report.ok is True assert {"requests>=2.31", "tenacity>=8.2"}.issubset(observed_requirements) - assert {"mcp>=1.2", "httpx>=0.27", "cryptography>=42"}.issubset(observed_requirements) + assert {"mcp>=1.8.1", "httpx>=0.27", "cryptography>=42"}.issubset(observed_requirements) def test_project_identity_cannot_be_supplied_by_operator(tmp_path: Path) -> None: From b656869c7ad5921bcb3372f370a8faddbac7b47b Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Tue, 14 Jul 2026 00:16:40 -0300 Subject: [PATCH 41/55] feat: add governed recovery operations --- .planning/REMAINING-PHASES-V1-BUDGET.md | 34 ++++ .planning/audit-remediation-ledger.md | 6 +- docs/operations.md | 41 +++++ external-actions-required.md | 2 + memorymaster/core/audit_envelope.py | 35 ++++ memorymaster/core/models.py | 2 + memorymaster/core/service.py | 14 +- memorymaster/govern/operational_health.py | 127 +++++++++++++ memorymaster/govern/privacy_ops.py | 153 ++++++++++++++++ memorymaster/govern/recovery.py | 170 ++++++++++++++++++ memorymaster/schema.sql | 3 + memorymaster/schema_postgres.sql | 3 + memorymaster/stores/_storage_schema.py | 9 + memorymaster/stores/_storage_sources.py | 72 +++++++- .../migrations/0016_media_retry_leases.py | 45 +++++ memorymaster/stores/postgres_store.py | 65 ++++++- memorymaster/surfaces/operations.py | 100 +++++++++++ pyproject.toml | 1 + tests/test_atlas_persisted_envelope.py | 19 +- tests/test_media_retry_lease_recovery.py | 95 ++++++++++ tests/test_operational_health.py | 57 ++++++ tests/test_operations_cli.py | 43 +++++ tests/test_privacy_operations.py | 69 +++++++ tests/test_recovery_operations.py | 77 ++++++++ 24 files changed, 1225 insertions(+), 17 deletions(-) create mode 100644 docs/operations.md create mode 100644 memorymaster/core/audit_envelope.py create mode 100644 memorymaster/govern/operational_health.py create mode 100644 memorymaster/govern/privacy_ops.py create mode 100644 memorymaster/govern/recovery.py create mode 100644 memorymaster/stores/migrations/0016_media_retry_leases.py create mode 100644 memorymaster/surfaces/operations.py create mode 100644 tests/test_media_retry_lease_recovery.py create mode 100644 tests/test_operational_health.py create mode 100644 tests/test_operations_cli.py create mode 100644 tests/test_privacy_operations.py create mode 100644 tests/test_recovery_operations.py diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index 4b593d51..c467f51d 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -409,3 +409,37 @@ R3.4 service entrypoints and readiness completed on 2026-07-13: The next package is R3.5 recovery, observability, and privacy. The active roadmap goal authorizes continuing without a new goal. + +R3.5 recovery, observability, privacy planning, and lease recovery completed +on 2026-07-13: + +- Media retries now use bounded owner/expiry leases on SQLite and Postgres. + Expired work is atomically returned to pending with an audit event; concurrent + claimers cannot receive the same row. Immutable migration 0016 and both + baseline schemas carry the lease contract. +- `memorymaster-ops` creates authenticated encrypted SQLite online backups, + records plaintext/ciphertext checksums and RPO/RTO metadata, and restores only + into a disposable drill target with integrity and foreign-key validation. + PostgreSQL and real off-device recovery remain explicit external evidence. +- Operational health evaluates backup age, retry backlog, SQLite integrity, + WAL/disk pressure, provider failures, and optional telemetry readiness. It can + persist one aggregate audit envelope with an owner and runbook instead of + leaving the evidence process-local. +- Privacy planning is read-only and principal/scope/tenant selective. It + inventories authoritative rows, citations/events, verbatim, artifacts, + caches, backups, and vector/backend copies; ambiguous tenants and unavailable + Qdrant/Postgres attribution fail closed. No erase/export/retention mutation is + implemented or implied. +- RED lease evidence initially failed three tests on the absent lease API; the + recovery/privacy/operations modules initially failed collection while absent. + The focused package gate produced 257 passed, 40 skipped, and two failures in + legacy Postgres test doubles. The bounded correction passed the exact two + regressions plus recovery/CLI coverage (7 passed). Changed-file Ruff and + `git diff --check` passed. +- MM-REL-03 and the repository portion of MM-OBS-01 are resolved. MM-OPS-05, + MM-PRIV-01, and MM-PRIV-02 retain explicit external evidence requirements; + no live data, backup, provider, or external system was mutated. +- Atomic package commit: the conventional R3.5 commit containing this evidence. + +The next boundary is Phase 3 convergence. The active roadmap goal authorizes +continuing without a new goal. diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 93adf25e..3e8ae86b 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -32,16 +32,16 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-UX-02 | UX/Reliability | High / BAD-PRACTICE | Dashboard removes review rows before mutation success | R4.3 | OPEN | Browser failure/retry evidence; row remains on rejected POST | | MM-UX-03 | UX | Medium / BAD-PRACTICE | Dashboard hides failures and has labeling, contrast, responsive, and evidence-discovery gaps | R4.3 | OPEN | Browser/a11y/mobile acceptance suite | | MM-PRIV-01 | Compliance | High / BAD-PRACTICE | Automatic transcript capture/remote processing lacks an explicit consent boundary | R2.4,R3.5 | BLOCKED-EXTERNAL | P2-E repository controls are quiet by default and require explicit maximum-capture/provider choices. Product-owner/legal approval for intended use, jurisdictions, processor disclosures, and retention commitments remains external. WARNING LEGAL REVIEW REQUIRED; no compliance-pass or risk-acceptance claim is made. | -| MM-PRIV-02 | Compliance | High / BAD-PRACTICE | No complete export, erasure, or retention workflow across copies | R3.5 | OPEN | Privacy export/erase dry-run manifest across all stores | +| MM-PRIV-02 | Compliance | High / BAD-PRACTICE | No complete export, erasure, or retention workflow across copies | R3.5 | BLOCKED-EXTERNAL | R3.5 adds a read-only, tenant-aware dry-run inventory across authoritative claims/citations/events, verbatim, artifacts/spool/wiki/cache, backups, Qdrant, and Postgres. Ambiguous tenants and unattributable copies fail closed. Legal disposition, authenticated backend evidence, and any live export/erase/retention application require external approval and infrastructure; no mutation pass is claimed. | | MM-COST-01 | Cost | High / EXPLOITABLE-NOW | Default Stop-hook LLM calls have no finite persisted budget | R2.4 | RESOLVED | P2-E (`a1d5477`) adds restart-safe atomic global/provider/session reservations and persisted usage; maximum default candidate inflow is 600/day against the 688/day ceiling. | | MM-COST-02 | Cost | High / EXPLOITABLE-NOW | Verbatim capture reprocesses sessions and retains data indefinitely | R2.4 | RESOLVED | P2-E persists complete-line cursors, processes appended content only, and enforces dry-run 30-day/512-MiB/75,000-session retention bounds. | | MM-COST-03 | Cost/Reliability | Medium / BAD-PRACTICE | Cost and intake controls are fragmented/process-local | R2.4,R3.2 | RESOLVED | R3.2 adds one WAL-backed atomic usage ledger shared by LLM primary/fallback calls, Gemini embedding batches, MCP ingest/checkpoint, and configured core intake quotas. Caps remain backward-compatible/unlimited unless configured; restart/cross-instance and local-bucket-reset tests prove durable global/provider/actor enforcement without consuming capacity on rejected reservations. | | MM-DEMO-01 | Demo/Integrity | High / EXPLOITABLE-NOW | Atlas defaults persist fabricated mock evidence at high confidence | R2.5 | RESOLVED | P2-F (`c66e240`) requires a real configured production provider, confines synthetic providers to explicit dev/test opt-in, excludes synthetic evidence from claims/prompts/actions/citations/exports, and fails configuration nonzero. Focused gate: 111 passed. | | MM-MAINT-01 | Maintainability/Integrity | Medium / BAD-PRACTICE | Versions, tool counts, roadmaps, and install claims drift | R4.4 | OPEN | Generated single-source values checked by CI | | MM-MAINT-02 | Architecture/Maintainability | Medium / BAD-PRACTICE | Oversized facades, hardwired extensions, dead plugin seam, and lingering shims | R4.1-R4.2 | OPEN | Measured boundary/size budgets and supported shim policy | -| MM-REL-03 | Reliability | Medium / BAD-PRACTICE | Media retry rows have no expired-lease recovery | R3.5 | OPEN | Worker-death lease reclaim test | +| MM-REL-03 | Reliability | Medium / BAD-PRACTICE | Media retry rows have no expired-lease recovery | R3.5 | RESOLVED | Immutable migration 0016 and SQLite/Postgres parity add bounded owner/expiry leases, atomic expired-work recovery with an audit event, lease clearing on terminal outcomes, and concurrent single-winner coverage. The focused R3.5 gate includes worker-death reclaim, unexpired non-steal, and concurrent claimer tests. | | MM-OPS-05 | Missing Operations | High / BAD-PRACTICE | Recovery defaults stop at same-machine SQLite snapshots | R3.5 | BLOCKED-EXTERNAL | Off-device SQLite/Postgres restore drill and RPO/RTO | -| MM-OBS-01 | Missing Operations | High / BAD-PRACTICE | Failures/metrics are process-local without central alert ownership | R3.5 | OPEN | Persistent metrics, trace/error capture, alert tests/runbook | +| MM-OBS-01 | Missing Operations | High / BAD-PRACTICE | Failures/metrics are process-local without central alert ownership | R3.5 | RESOLVED | R3.5 adds deterministic backup/retry/integrity/WAL/disk/provider/telemetry checks and can persist one aggregate audit envelope with explicit owner and runbook. Focused tests cover alert severity, persistence, and audit fields. Delivery to an organization-selected OTel/error backend remains environment-specific readiness evidence, not a repository correctness claim. | | MM-DB-01 | Database | Medium / BAD-PRACTICE | Fast schema fingerprint can omit legacy ensure-helper changes | R3.2 | RESOLVED | `schema_stamp()` now hashes baseline schema, immutable migrations, and the complete legacy ensure-helper source. A helper-body mutation regression changes the stamp and forces the full initialization path. New R3.2 DDL exists only in migration 0015/baselines. | | MM-INTEGRITY-01 | Code Integrity | Medium / BAD-PRACTICE | `importlib.util` probe is wrong in clean Python and its test masks failure | R4.4 | OPEN | Clean subprocess test and explicit import | | MM-TEST-01 | Maintainability/Reliability | Medium / BAD-PRACTICE | Validator candidate winner changed under load because mutable `updated_at` controlled processing order | Phase 0/R4.4 | RESOLVED | Deterministic timestamp-inversion regression passes; targeted lifecycle suite 35 passed; full isolated non-ML gate 3,094 passed, 56 skipped, 95 deselected, 24 intentional xfails | diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 00000000..191c92ac --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,41 @@ +# MemoryMaster Operations + +MemoryMaster's operational tools are fail-closed and use disposable targets by +default. They never upload a backup, erase product data, or run PostgreSQL +restore commands automatically. + +## Recovery contract + +- SQLite backups use the online backup API, validate `integrity_check` and + foreign keys, encrypt before leaving temporary local storage, and write an + authenticated checksum manifest. +- Configure `MEMORYMASTER_BACKUP_KEY` with a Fernet key through the deployment + secret manager. The key is intentionally not accepted on the command line. +- Default objectives are RPO 24 hours and RTO 30 minutes. Override them only + with an operator-approved service objective. +- `memorymaster-ops backup-create --db DB --destination BACKUP --off-device` + asserts that the operator-selected destination is off-device; the tool does + not infer storage topology. +- `memorymaster-ops restore-drill --backup BACKUP` restores only into a + disposable temporary database and reruns full integrity and foreign-key + checks. +- PostgreSQL remains plan-only until separate dump and restore endpoints, + `pg_dump`/`pg_restore`, and a disposable restricted-role database are + supplied. + +## Health and alerts + +`memorymaster-ops health` evaluates backup age, retry backlog, provider +failures, DB integrity, WAL size, disk use, OpenTelemetry readiness, and error +tracking readiness. With `--persist`, one aggregate append-only system event is +written with the owner and runbook. Alert delivery is configured externally; +the persisted snapshot remains the source of evidence across restarts. + +## Privacy operations + +`memorymaster-ops privacy-plan` is always dry-run. It inventories attributable +primary DB and verbatim rows and explicitly reports Qdrant, artifacts, spool, +wiki, caches, and backups that cannot be proven complete. Backups expire by +policy; they are never deleted immediately by an erasure request. Any apply +workflow requires a verified backup, legal/product approval, an exact tenant +and principal selector, and separate live-data authorization. diff --git a/external-actions-required.md b/external-actions-required.md index c548e635..a81463e0 100644 --- a/external-actions-required.md +++ b/external-actions-required.md @@ -19,3 +19,5 @@ live mutation by itself. | MM-DATA-01 | Live MemoryMaster operator | Approve a consistent backup/restore drill before any live migration, redaction, backlog, or retention operation. | Restored backup, integrity check, counts/checksums, approval record | 2026-08-15 | BLOCKED-EXTERNAL | | MM-CAP-01 | Host storage operator | Address/monitor the drive at 85.82% used without deleting MemoryMaster data under this goal. | Daily disk telemetry below the critical gate or approved capacity expansion | 2026-07-18 | BLOCKED-EXTERNAL | | MM-PRIV-01 | Product owner / legal reviewer | Decide intended organizational use, jurisdictions, processor disclosures, and retention commitments before compliance claims. | Approved privacy/data-processing statement | 2026-08-31 | BLOCKED-EXTERNAL | +| MM-OPS-05 | Backup / database operator | Provide an approved off-device encrypted destination and disposable PostgreSQL target; execute restore drills without production cutover. | Immutable backup IDs, checksums, SQLite/Postgres restore logs, integrity/FK/row-count evidence, measured RPO/RTO | 2026-08-15 | BLOCKED-EXTERNAL | +| MM-PRIV-02 | Product owner / privacy reviewer / backend operator | Approve subject-copy attribution and legal disposition, then validate the dry-run plan against authenticated disposable Postgres/Qdrant and managed artifact stores before any live apply workflow is authorized. | Tenant-scoped export/erase/retention dry-run manifest, reviewer approval, backend reconciliation, backup-expiry disposition | 2026-08-31 | BLOCKED-EXTERNAL | diff --git a/memorymaster/core/audit_envelope.py b/memorymaster/core/audit_envelope.py new file mode 100644 index 00000000..21ece424 --- /dev/null +++ b/memorymaster/core/audit_envelope.py @@ -0,0 +1,35 @@ +"""Canonical attributable audit metadata for operational actions.""" + +from __future__ import annotations + +from datetime import datetime, timezone + + +def build_audit_envelope( + *, + principal: str, + tenant_id: str | None, + role: str, + request_id: str, + session_id: str, + action: str, + target: str, + result: str, +) -> dict[str, str | None]: + required = { + "principal": principal, + "role": role, + "request_id": request_id, + "session_id": session_id, + "action": action, + "target": target, + "result": result, + } + blanks = [name for name, value in required.items() if not str(value or "").strip()] + if blanks: + raise ValueError(f"audit envelope requires: {', '.join(blanks)}") + return { + **{name: str(value).strip() for name, value in required.items()}, + "tenant_id": str(tenant_id).strip() if tenant_id else None, + "occurred_at": datetime.now(timezone.utc).replace(microsecond=0).isoformat(), + } diff --git a/memorymaster/core/models.py b/memorymaster/core/models.py index d3e268ed..39e4eb26 100644 --- a/memorymaster/core/models.py +++ b/memorymaster/core/models.py @@ -394,6 +394,8 @@ class MediaRetryItem: last_http_status: int | None last_error: str | None next_attempt_time: str | None + lease_owner: str | None + lease_expires_at: str | None created_at: str updated_at: str diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index 796a0ce4..3bbf3fa3 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -2314,8 +2314,18 @@ def enqueue_media_retry( next_attempt_time=next_attempt_time, ) - def claim_pending_media_retries(self, limit: int = 25) -> list[MediaRetryItem]: - return self.store.claim_pending_media_retries(limit=limit) + def claim_pending_media_retries( + self, + limit: int = 25, + *, + lease_owner: str = "media-worker", + lease_seconds: int = 300, + ) -> list[MediaRetryItem]: + return self.store.claim_pending_media_retries( + limit=limit, + lease_owner=lease_owner, + lease_seconds=lease_seconds, + ) def record_media_retry_outcome( self, diff --git a/memorymaster/govern/operational_health.py b/memorymaster/govern/operational_health.py new file mode 100644 index 00000000..35226114 --- /dev/null +++ b/memorymaster/govern/operational_health.py @@ -0,0 +1,127 @@ +"""Persistent aggregate health snapshots and deterministic alerts.""" + +from __future__ import annotations + +import importlib.util +import json +import os +from pathlib import Path +import shutil +from typing import Any + +from memorymaster.core.audit_envelope import build_audit_envelope +from memorymaster.govern.recovery import backup_status + + +def otel_status() -> dict[str, str]: + endpoint = os.environ.get("MEMORYMASTER_OTEL_ENDPOINT", "").strip() + if not endpoint: + return {"status": "disabled"} + if importlib.util.find_spec("opentelemetry.sdk") is None: + return {"status": "unavailable", "reason": "install the OpenTelemetry SDK"} + return {"status": "configured"} + + +def error_tracking_status() -> dict[str, str]: + endpoint = os.environ.get("MEMORYMASTER_ERROR_TRACKING_DSN", "").strip() + if not endpoint: + return {"status": "disabled"} + if importlib.util.find_spec("sentry_sdk") is None: + return {"status": "unavailable", "reason": "install the configured error-tracking SDK"} + return {"status": "configured"} + + +def _retry_alerts(service: Any, threshold: int) -> tuple[dict[str, int], list[dict[str, Any]]]: + counts = service.media_retry_status_counts() + backlog = int(counts.get("pending", 0)) + int(counts.get("retrying", 0)) + alerts = [] + if backlog > threshold: + alerts.append({"code": "media_backlog_high", "value": backlog, "threshold": threshold}) + return counts, alerts + + +def _database_signals(service: Any) -> dict[str, Any]: + store = service.store + with store.connect() as connection: + connection.execute("SELECT 1") + if hasattr(store, "db_path"): + integrity = str(connection.execute("PRAGMA quick_check").fetchone()[0]) + else: + integrity = "ok" + db_path = Path(store.db_path) if hasattr(store, "db_path") else None + wal_bytes = Path(f"{db_path}-wal").stat().st_size if db_path and Path(f"{db_path}-wal").exists() else 0 + disk = shutil.disk_usage(db_path.parent if db_path else Path.cwd()) + return { + "integrity": integrity, + "wal_bytes": wal_bytes, + "disk_percent": round((disk.used / max(1, disk.total)) * 100, 2), + } + + +def _provider_failure_count(service: Any) -> int: + count = 0 + for event in service.list_events(limit=1000): + detail = str(event.details or "").lower() + if "provider" in detail and any(marker in detail for marker in ("fail", "error", "unavailable")): + count += 1 + return count + + +def evaluate_operational_health( + service: Any, + *, + backup_manifest_dir: str | Path, + persist: bool, + owner: str, + runbook: str, + backup_max_age_hours: int = 24, + retry_backlog_threshold: int = 100, + wal_max_bytes: int = 512 * 1024 * 1024, + disk_max_percent: float = 90.0, + provider_failure_threshold: int = 0, +) -> dict[str, Any]: + backup = backup_status(backup_manifest_dir, max_age_hours=backup_max_age_hours) + retry_counts, alerts = _retry_alerts(service, retry_backlog_threshold) + if backup["status"] != "ok": + alerts.append({"code": backup["code"], "age_hours": backup.get("age_hours")}) + database = _database_signals(service) + if database["integrity"] != "ok": + alerts.append({"code": "database_integrity_failed"}) + if database["wal_bytes"] > max(0, int(wal_max_bytes)): + alerts.append({"code": "wal_size_high", "value": database["wal_bytes"], "threshold": wal_max_bytes}) + if database["disk_percent"] > max(1.0, float(disk_max_percent)): + alerts.append({"code": "disk_usage_high", "value": database["disk_percent"], "threshold": disk_max_percent}) + provider_failures = _provider_failure_count(service) + if provider_failures > max(0, int(provider_failure_threshold)): + alerts.append({"code": "provider_failures", "value": provider_failures}) + result = { + "schema_version": "memorymaster.operational-health.v1", + "status": "alert" if alerts else "ok", + "owner": owner, + "runbook": runbook, + "alerts": alerts, + "backup": backup, + "media_retry": retry_counts, + "database": database, + "provider_failures": provider_failures, + "otel": otel_status(), + "error_tracking": error_tracking_status(), + } + if persist: + envelope = build_audit_envelope( + principal=owner, + tenant_id=getattr(service, "tenant_id", None), + role="operator", + request_id="scheduled-health", + session_id="operational-health", + action="operations.evaluate", + target="memorymaster", + result=result["status"], + ) + service.store.record_event( + claim_id=None, + event_type="system", + details="operational_health_snapshot", + payload={"audit": envelope, "health": json.loads(json.dumps(result))}, + ) + return result diff --git a/memorymaster/govern/privacy_ops.py b/memorymaster/govern/privacy_ops.py new file mode 100644 index 00000000..92072658 --- /dev/null +++ b/memorymaster/govern/privacy_ops.py @@ -0,0 +1,153 @@ +"""Read-only, inventory-driven privacy export/erase planning.""" + +from __future__ import annotations + +from dataclasses import dataclass +import os +from pathlib import Path +import sqlite3 +from typing import Any + + +@dataclass(frozen=True, slots=True) +class PrivacySelector: + principal: str + tenant_id: str | None = None + scope: str | None = None + + def __post_init__(self) -> None: + principal = self.principal.strip() + if not principal: + raise ValueError("privacy selector principal must not be blank") + object.__setattr__(self, "principal", principal) + object.__setattr__(self, "tenant_id", (self.tenant_id or "").strip() or None) + object.__setattr__(self, "scope", (self.scope or "").strip() or None) + + +def _connect_read_only(db_path: Path) -> sqlite3.Connection: + connection = sqlite3.connect(f"file:{db_path.as_posix()}?mode=ro", uri=True) + connection.row_factory = sqlite3.Row + connection.execute("PRAGMA query_only = ON") + return connection + + +def _has_table(connection: sqlite3.Connection, name: str) -> bool: + return connection.execute( + "SELECT 1 FROM sqlite_master WHERE type IN ('table','view') AND name = ?", + (name,), + ).fetchone() is not None + + +def _claim_inventory(connection: sqlite3.Connection, selector: PrivacySelector) -> dict[str, Any]: + clauses = ["(source_agent = ? OR holder = ?)"] + params: list[Any] = [selector.principal, selector.principal] + if selector.tenant_id: + clauses.append("tenant_id = ?") + params.append(selector.tenant_id) + if selector.scope: + clauses.append("scope = ?") + params.append(selector.scope) + where = " AND ".join(clauses) + if selector.tenant_id is None: + tenants = { + str(row[0]) + for row in connection.execute( + "SELECT DISTINCT COALESCE(tenant_id, '') FROM claims WHERE (source_agent = ? OR holder = ?)", + (selector.principal, selector.principal), + ) + } + if len(tenants) > 1: + return { + "surface": "primary_db", + "status": "BLOCKED-EXTERNAL", + "reason": "principal spans multiple tenant boundaries; tenant_id is required", + } + ids = [int(row[0]) for row in connection.execute(f"SELECT id FROM claims WHERE {where}", params)] + if not ids: + return {"surface": "primary_db", "status": "NOT_FOUND", "claims": 0, "citations": 0, "events": 0} + placeholders = ",".join("?" for _ in ids) + citations = connection.execute(f"SELECT COUNT(*) FROM citations WHERE claim_id IN ({placeholders})", ids).fetchone()[0] + events = connection.execute(f"SELECT COUNT(*) FROM events WHERE claim_id IN ({placeholders})", ids).fetchone()[0] + return {"surface": "primary_db", "status": "FOUND", "claims": len(ids), "citations": citations, "events": events} + + +def _verbatim_inventory(connection: sqlite3.Connection, selector: PrivacySelector) -> dict[str, Any]: + if not _has_table(connection, "verbatim_memories"): + return {"surface": "verbatim", "status": "NOT_FOUND", "rows": 0} + clauses = ["source_agent = ?"] + params: list[Any] = [selector.principal] + if selector.scope: + clauses.append("scope = ?") + params.append(selector.scope) + count = connection.execute( + f"SELECT COUNT(*) FROM verbatim_memories WHERE {' AND '.join(clauses)}", + params, + ).fetchone()[0] + return {"surface": "verbatim", "status": "FOUND" if count else "NOT_FOUND", "rows": count} + + +def _path_surface(workspace: Path, name: str, relative: str) -> dict[str, Any]: + root = workspace / relative + count = sum(1 for path in root.rglob("*") if path.is_file()) if root.exists() else 0 + return { + "surface": name, + "status": "BLOCKED-EXTERNAL" if count else "NOT_FOUND", + "objects": count, + "reason": "objects are not reliably principal-attributable" if count else "surface absent", + } + + +def build_privacy_plan( + *, + db_target: str | Path, + workspace: str | Path, + selector: PrivacySelector, +) -> dict[str, Any]: + if str(db_target).strip().lower().startswith(("postgres://", "postgresql://")): + return { + "schema_version": "memorymaster.privacy-plan.v1", + "dry_run": True, + "selector": {"principal": selector.principal, "tenant_id": selector.tenant_id, "scope": selector.scope}, + "surfaces": [{ + "surface": "primary_db", + "status": "BLOCKED-EXTERNAL", + "reason": "Postgres privacy inventory requires a disposable restricted-role runtime", + }], + "complete": False, + "mutation_count": 0, + } + db_path = Path(db_target).resolve() + workspace_path = Path(workspace).resolve() + with _connect_read_only(db_path) as connection: + surfaces = [ + _claim_inventory(connection, selector), + _verbatim_inventory(connection, selector), + _path_surface(workspace_path, "artifacts", "artifacts"), + _path_surface(workspace_path, "spool", ".memorymaster/spool"), + _path_surface(workspace_path, "wiki", "obsidian-vault"), + ] + cache_rows = connection.execute("SELECT COUNT(*) FROM query_cache").fetchone()[0] if _has_table(connection, "query_cache") else 0 + surfaces.append({ + "surface": "query_cache", + "status": "BLOCKED-EXTERNAL" if cache_rows else "NOT_FOUND", + "rows": cache_rows, + "reason": "cache rows lack principal attribution" if cache_rows else "cache empty", + }) + surfaces.append({ + "surface": "qdrant", + "status": "BLOCKED-EXTERNAL", + "configured": bool(os.environ.get("QDRANT_URL", "").strip()), + "reason": "requires disposable authenticated/TLS inventory and deletion evidence", + }) + backups = _path_surface(workspace_path, "backups", ".memorymaster/snapshots") + backups["disposition"] = "expire_by_policy" + surfaces.append(backups) + complete = all(row["status"] in {"FOUND", "NOT_FOUND"} for row in surfaces) + return { + "schema_version": "memorymaster.privacy-plan.v1", + "dry_run": True, + "selector": {"principal": selector.principal, "tenant_id": selector.tenant_id, "scope": selector.scope}, + "surfaces": surfaces, + "complete": complete, + "mutation_count": 0, + } diff --git a/memorymaster/govern/recovery.py b/memorymaster/govern/recovery.py new file mode 100644 index 00000000..04419472 --- /dev/null +++ b/memorymaster/govern/recovery.py @@ -0,0 +1,170 @@ +"""Encrypted backup manifests and disposable recovery drills.""" + +from __future__ import annotations + +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path +import sqlite3 +import tempfile +import time +from typing import Any + +from memorymaster.stores import snapshot + + +def _utc_now() -> str: + return datetime.now(timezone.utc).replace(microsecond=0).isoformat() + + +def _sha256_bytes(payload: bytes) -> str: + return hashlib.sha256(payload).hexdigest() + + +def _manifest_path(backup_path: Path) -> Path: + return backup_path.with_suffix(f"{backup_path.suffix}.manifest.json") + + +def _fernet(encryption_key: str): + try: + from cryptography.fernet import Fernet + except ImportError as exc: + raise RuntimeError( + "encrypted recovery requires the 'security' extra: " + "pip install 'memorymaster[security]'" + ) from exc + return Fernet(encryption_key.encode("ascii")) + + +def _decrypt(encryption_key: str, payload: bytes) -> bytes: + try: + from cryptography.fernet import InvalidToken + except ImportError as exc: + raise RuntimeError( + "encrypted recovery requires the 'security' extra: " + "pip install 'memorymaster[security]'" + ) from exc + try: + return _fernet(encryption_key).decrypt(payload) + except InvalidToken as exc: + raise RuntimeError("encrypted backup authentication failed") from exc + + +def _sqlite_checks(db_path: Path) -> tuple[str, int]: + connection = sqlite3.connect(str(db_path)) + try: + integrity = str(connection.execute("PRAGMA integrity_check").fetchone()[0]) + fk_rows = connection.execute("PRAGMA foreign_key_check").fetchall() + finally: + connection.close() + return integrity, len(fk_rows) + + +def _write_atomic(path: Path, payload: bytes) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + partial = path.with_suffix(f"{path.suffix}.part") + partial.write_bytes(payload) + partial.replace(path) + + +def create_encrypted_sqlite_backup( + db_path: str | Path, + destination: str | Path, + *, + encryption_key: str, + off_device: bool, + rpo_hours: int = 24, + rto_minutes: int = 30, +) -> dict[str, Any]: + source = Path(db_path).resolve() + target = Path(destination).resolve() + cipher = _fernet(encryption_key) + with tempfile.TemporaryDirectory(prefix="memorymaster-backup-") as temporary: + plaintext_path = Path(temporary) / "online-backup.db" + snapshot.backup(source, plaintext_path) + integrity, fk_violations = _sqlite_checks(plaintext_path) + if integrity != "ok" or fk_violations: + raise RuntimeError("online SQLite backup failed integrity validation") + plaintext = plaintext_path.read_bytes() + encrypted = cipher.encrypt(plaintext) + _write_atomic(target, encrypted) + manifest = { + "schema_version": "memorymaster.recovery.v1", + "backend": "sqlite", + "created_at": _utc_now(), + "encrypted": True, + "off_device": bool(off_device), + "rpo_hours": max(1, int(rpo_hours)), + "rto_minutes": max(1, int(rto_minutes)), + "plaintext_sha256": _sha256_bytes(plaintext), + "ciphertext_sha256": _sha256_bytes(encrypted), + "size_bytes": len(encrypted), + "integrity_check": integrity, + "foreign_key_violations": fk_violations, + } + _write_atomic(_manifest_path(target), (json.dumps(manifest, sort_keys=True, indent=2) + "\n").encode()) + return manifest + + +def run_sqlite_restore_drill( + backup_path: str | Path, + *, + encryption_key: str, +) -> dict[str, Any]: + source = Path(backup_path).resolve() + manifest = json.loads(_manifest_path(source).read_text(encoding="utf-8")) + encrypted = source.read_bytes() + if _sha256_bytes(encrypted) != manifest["ciphertext_sha256"]: + raise RuntimeError("encrypted backup checksum mismatch") + started = time.monotonic() + plaintext = _decrypt(encryption_key, encrypted) + if _sha256_bytes(plaintext) != manifest["plaintext_sha256"]: + raise RuntimeError("decrypted backup checksum mismatch") + with tempfile.TemporaryDirectory(prefix="memorymaster-restore-") as temporary: + backup = Path(temporary) / "backup.db" + restored = Path(temporary) / "restored.db" + backup.write_bytes(plaintext) + snapshot.restore(backup, restored) + integrity, fk_violations = _sqlite_checks(restored) + elapsed = time.monotonic() - started + rto_seconds = int(manifest["rto_minutes"]) * 60 + return { + "backend": "sqlite", + "integrity_check": integrity, + "foreign_key_violations": fk_violations, + "elapsed_seconds": round(elapsed, 3), + "rto_seconds": rto_seconds, + "rto_met": integrity == "ok" and fk_violations == 0 and elapsed <= rto_seconds, + } + + +def backup_status(manifest_dir: str | Path, *, max_age_hours: int) -> dict[str, Any]: + manifests = sorted(Path(manifest_dir).glob("*.manifest.json"), key=lambda path: path.stat().st_mtime) + if not manifests: + return {"status": "alert", "code": "backup_missing", "age_hours": None} + payload = json.loads(manifests[-1].read_text(encoding="utf-8")) + created = datetime.fromisoformat(str(payload["created_at"]).replace("Z", "+00:00")) + if created.tzinfo is None: + created = created.replace(tzinfo=timezone.utc) + age_hours = (datetime.now(timezone.utc) - created).total_seconds() / 3600 + return { + "status": "ok" if age_hours <= max(1, int(max_age_hours)) else "alert", + "code": "backup_fresh" if age_hours <= max(1, int(max_age_hours)) else "backup_stale", + "age_hours": round(age_hours, 2), + } + + +def postgres_recovery_plan() -> dict[str, Any]: + return { + "status": "BLOCKED-EXTERNAL", + "backend": "postgres", + "required_tools": ["pg_dump", "pg_restore"], + "required_evidence": [ + "consistent custom-format dump", + "restore into an empty disposable database", + "schema and row-count verification", + "restricted application-role smoke", + ], + "executes": False, + } diff --git a/memorymaster/schema.sql b/memorymaster/schema.sql index b32a3470..ceb8f641 100644 --- a/memorymaster/schema.sql +++ b/memorymaster/schema.sql @@ -264,6 +264,8 @@ CREATE TABLE IF NOT EXISTS media_retry_queue ( last_http_status INTEGER, last_error TEXT, next_attempt_time TEXT, + lease_owner TEXT, + lease_expires_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, FOREIGN KEY (source_item_id) REFERENCES source_items(id) ON DELETE CASCADE, @@ -272,4 +274,5 @@ CREATE TABLE IF NOT EXISTS media_retry_queue ( CREATE INDEX IF NOT EXISTS idx_media_retry_status ON media_retry_queue(status); CREATE INDEX IF NOT EXISTS idx_media_retry_next_attempt ON media_retry_queue(next_attempt_time); +CREATE INDEX IF NOT EXISTS idx_media_retry_lease_expiry ON media_retry_queue(status, lease_expires_at); CREATE INDEX IF NOT EXISTS idx_media_retry_source_item ON media_retry_queue(source_item_id); diff --git a/memorymaster/schema_postgres.sql b/memorymaster/schema_postgres.sql index 4abc6b6f..f94b3776 100644 --- a/memorymaster/schema_postgres.sql +++ b/memorymaster/schema_postgres.sql @@ -349,6 +349,8 @@ CREATE TABLE IF NOT EXISTS media_retry_queue ( last_http_status INTEGER, last_error TEXT, next_attempt_time TIMESTAMPTZ, + lease_owner TEXT, + lease_expires_at TIMESTAMPTZ, created_at TIMESTAMPTZ NOT NULL, updated_at TIMESTAMPTZ NOT NULL, UNIQUE (source_item_id, media_key) @@ -356,6 +358,7 @@ CREATE TABLE IF NOT EXISTS media_retry_queue ( CREATE INDEX IF NOT EXISTS idx_media_retry_status ON media_retry_queue(status); CREATE INDEX IF NOT EXISTS idx_media_retry_next_attempt ON media_retry_queue(next_attempt_time); +CREATE INDEX IF NOT EXISTS idx_media_retry_lease_expiry ON media_retry_queue(status, lease_expires_at); CREATE INDEX IF NOT EXISTS idx_media_retry_source_item ON media_retry_queue(source_item_id); CREATE TABLE IF NOT EXISTS claim_links ( diff --git a/memorymaster/stores/_storage_schema.py b/memorymaster/stores/_storage_schema.py index f2af316b..3220e782 100644 --- a/memorymaster/stores/_storage_schema.py +++ b/memorymaster/stores/_storage_schema.py @@ -212,6 +212,8 @@ def _ensure_atlas_source_schema(conn: sqlite3.Connection) -> None: last_http_status INTEGER, last_error TEXT, next_attempt_time TEXT, + lease_owner TEXT, + lease_expires_at TEXT, created_at TEXT NOT NULL, updated_at TEXT NOT NULL, FOREIGN KEY (source_item_id) REFERENCES source_items(id) ON DELETE CASCADE, @@ -230,6 +232,12 @@ def _ensure_atlas_source_schema(conn: sqlite3.Connection) -> None: except sqlite3.OperationalError as exc: if "duplicate column name" not in str(exc).lower(): raise + for column in ("lease_owner TEXT", "lease_expires_at TEXT"): + try: + conn.execute(f"ALTER TABLE media_retry_queue ADD COLUMN {column}") + except sqlite3.OperationalError as exc: + if "duplicate column name" not in str(exc).lower(): + raise # Phase 3: indexes (now safe — all columns guaranteed to exist). conn.executescript( @@ -250,6 +258,7 @@ def _ensure_atlas_source_schema(conn: sqlite3.Connection) -> None: CREATE INDEX IF NOT EXISTS idx_evidence_items_sensitivity ON evidence_items(sensitivity); CREATE INDEX IF NOT EXISTS idx_media_retry_status ON media_retry_queue(status); CREATE INDEX IF NOT EXISTS idx_media_retry_next_attempt ON media_retry_queue(next_attempt_time); + CREATE INDEX IF NOT EXISTS idx_media_retry_lease_expiry ON media_retry_queue(status, lease_expires_at); CREATE INDEX IF NOT EXISTS idx_media_retry_source_item ON media_retry_queue(source_item_id); """ ) diff --git a/memorymaster/stores/_storage_sources.py b/memorymaster/stores/_storage_sources.py index 0a0e841a..df66e1f5 100644 --- a/memorymaster/stores/_storage_sources.py +++ b/memorymaster/stores/_storage_sources.py @@ -9,6 +9,7 @@ import json import re import sqlite3 +from datetime import datetime, timedelta, timezone from typing import TYPE_CHECKING, Any from memorymaster.stores._storage_shared import utc_now @@ -93,7 +94,7 @@ def _atlas_row_is_safe(row: Any, fields: tuple[str, ...]) -> bool: ) _RETRY_FIELDS = ( "media_key", "chat_id", "media_type", "media_path", "media_url", "status", "last_error", - "next_attempt_time", "created_at", "updated_at", + "next_attempt_time", "lease_owner", "lease_expires_at", "created_at", "updated_at", ) @@ -726,7 +727,13 @@ def enqueue_media_retry( conn.commit() return self._row_to_media_retry(row) - def claim_pending_media_retries(self, limit: int = 25) -> list[MediaRetryItem]: + def claim_pending_media_retries( + self, + limit: int = 25, + *, + lease_owner: str = "media-worker", + lease_seconds: int = 300, + ) -> list[MediaRetryItem]: """Atomically claim up to N pending rows whose next_attempt_time is past. Transitions claimed rows from 'pending' to 'retrying' and increments @@ -734,7 +741,15 @@ def claim_pending_media_retries(self, limit: int = 25) -> list[MediaRetryItem]: """ if limit <= 0: return [] + owner = (lease_owner or "").strip() + if not owner: + raise ValueError("lease_owner must not be blank.") + validate_persisted_metadata({"lease_owner": owner}) + duration = max(1, min(int(lease_seconds), 3600)) now = utc_now() + lease_expires_at = ( + datetime.now(timezone.utc) + timedelta(seconds=duration) + ).replace(microsecond=0).isoformat() with self.connect() as conn: # BEGIN IMMEDIATE takes the write lock up front so the SELECT and # the claiming UPDATE are one atomic read-modify-write. Without it, @@ -746,6 +761,38 @@ def claim_pending_media_retries(self, limit: int = 25) -> list[MediaRetryItem]: # second guard: a row already moved out of 'pending' is never # re-claimed even if a stale id slips through. conn.execute("BEGIN IMMEDIATE") + expired_rows = conn.execute( + """SELECT * FROM media_retry_queue + WHERE status = 'retrying' + AND lease_expires_at IS NOT NULL + AND lease_expires_at <= ? + ORDER BY id ASC""", + (now,), + ).fetchall() + expired_ids = [ + int(row["id"]) + for row in expired_rows + if _atlas_row_is_safe(row, _RETRY_FIELDS) + ] + for retry_id in expired_ids: + conn.execute( + """UPDATE media_retry_queue + SET status = 'pending', lease_owner = NULL, + lease_expires_at = NULL, updated_at = ? + WHERE id = ? AND status = 'retrying' + AND lease_expires_at <= ?""", + (now, retry_id, now), + ) + self._insert_event_row( + conn, + claim_id=None, + event_type="media_process", + from_status="retrying", + to_status="pending", + details="media_retry_lease_expired", + payload_json=json.dumps({"retry_id": retry_id}, sort_keys=True), + created_at=now, + ) ids: list[int] = [] cursor_id = 0 page_size = min(max(limit, 25), 250) @@ -775,11 +822,13 @@ def claim_pending_media_retries(self, limit: int = 25) -> list[MediaRetryItem]: UPDATE media_retry_queue SET status = 'retrying', attempt_count = attempt_count + 1, + lease_owner = ?, + lease_expires_at = ?, updated_at = ? WHERE status = 'pending' AND id IN ({placeholders}) """, - [now, *ids], + [owner, lease_expires_at, now, *ids], ) for retry_id in ids: self._insert_event_row( @@ -845,6 +894,7 @@ def record_media_retry_outcome( new_http = last_http_status if last_http_status is not None else current["last_http_status"] new_err = last_error if last_error is not None else current["last_error"] new_next = next_attempt_time if next_attempt_time is not None else current["next_attempt_time"] + clear_lease = status != "retrying" conn.execute( """ UPDATE media_retry_queue @@ -853,10 +903,22 @@ def record_media_retry_outcome( last_http_status = ?, last_error = ?, next_attempt_time = ?, + lease_owner = ?, + lease_expires_at = ?, updated_at = ? WHERE id = ? """, - (status, new_path, new_http, new_err, new_next, now, retry_id), + ( + status, + new_path, + new_http, + new_err, + new_next, + None if clear_lease else current["lease_owner"], + None if clear_lease else current["lease_expires_at"], + now, + retry_id, + ), ) self._insert_event_row( conn, @@ -953,6 +1015,8 @@ def _row_to_media_retry(row: sqlite3.Row) -> MediaRetryItem: last_http_status=int(row["last_http_status"]) if row["last_http_status"] is not None else None, last_error=row["last_error"], next_attempt_time=row["next_attempt_time"], + lease_owner=row["lease_owner"], + lease_expires_at=row["lease_expires_at"], created_at=str(row["created_at"]), updated_at=str(row["updated_at"]), ) diff --git a/memorymaster/stores/migrations/0016_media_retry_leases.py b/memorymaster/stores/migrations/0016_media_retry_leases.py new file mode 100644 index 00000000..088bb416 --- /dev/null +++ b/memorymaster/stores/migrations/0016_media_retry_leases.py @@ -0,0 +1,45 @@ +"""Replay-safe worker leases for media retry rows.""" + +from __future__ import annotations + +VERSION = 16 +DESCRIPTION = "Add explicit owner and expiry to media retry worker leases" + + +def apply_sqlite(conn) -> None: + exists = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name='media_retry_queue'" + ).fetchone() + if not exists: + conn.commit() + return + columns = { + str(row[1]) + for row in conn.execute("PRAGMA table_info(media_retry_queue)").fetchall() + } + if "lease_owner" not in columns: + conn.execute("ALTER TABLE media_retry_queue ADD COLUMN lease_owner TEXT") + if "lease_expires_at" not in columns: + conn.execute("ALTER TABLE media_retry_queue ADD COLUMN lease_expires_at TEXT") + conn.execute( + "CREATE INDEX IF NOT EXISTS idx_media_retry_lease_expiry " + "ON media_retry_queue(status, lease_expires_at)" + ) + conn.commit() + + +def apply_postgres(conn) -> None: + with conn.cursor() as cursor: + cursor.execute( + """ + DO $$ BEGIN + IF to_regclass('public.media_retry_queue') IS NOT NULL THEN + ALTER TABLE media_retry_queue ADD COLUMN IF NOT EXISTS lease_owner TEXT; + ALTER TABLE media_retry_queue ADD COLUMN IF NOT EXISTS lease_expires_at TIMESTAMPTZ; + CREATE INDEX IF NOT EXISTS idx_media_retry_lease_expiry + ON media_retry_queue(status, lease_expires_at); + END IF; + END $$ + """ + ) + conn.commit() diff --git a/memorymaster/stores/postgres_store.py b/memorymaster/stores/postgres_store.py index 95ad853a..27ce748d 100644 --- a/memorymaster/stores/postgres_store.py +++ b/memorymaster/stores/postgres_store.py @@ -4374,6 +4374,8 @@ def _row_to_media_retry(cls, row: Any) -> MediaRetryItem: last_http_status=int(row["last_http_status"]) if row.get("last_http_status") is not None else None, last_error=cls._as_text(row.get("last_error")), next_attempt_time=cls._as_iso(row.get("next_attempt_time")), + lease_owner=cls._as_text(row.get("lease_owner")), + lease_expires_at=cls._as_iso(row.get("lease_expires_at")), created_at=cls._as_iso(row["created_at"]) or "", updated_at=cls._as_iso(row["updated_at"]) or "", ) @@ -4454,12 +4456,54 @@ def enqueue_media_retry( ) return self._row_to_media_retry(row) - def claim_pending_media_retries(self, limit: int = 25) -> list[MediaRetryItem]: + def claim_pending_media_retries( + self, + limit: int = 25, + *, + lease_owner: str = "media-worker", + lease_seconds: int = 300, + ) -> list[MediaRetryItem]: self._deny_unsupported_team_surface("claim_pending_media_retries") if limit <= 0: return [] + owner = (lease_owner or "").strip() + if not owner: + raise ValueError("lease_owner must not be blank.") + validate_persisted_metadata({"lease_owner": owner}) + duration = max(1, min(int(lease_seconds), 3600)) now = utc_now() + lease_expires_at = now + timedelta(seconds=duration) with self.connect() as conn, conn.cursor() as cur: + cur.execute( + """SELECT * FROM media_retry_queue + WHERE status = 'retrying' + AND lease_expires_at IS NOT NULL + AND lease_expires_at <= %s + ORDER BY id ASC FOR UPDATE SKIP LOCKED""", + (now,), + ) + for expired in cur.fetchall(): + if not _atlas_row_is_safe(expired, _RETRY_FIELDS): + continue + retry_id = int(expired["id"]) + cur.execute( + """UPDATE media_retry_queue + SET status = 'pending', lease_owner = NULL, + lease_expires_at = NULL, updated_at = %s + WHERE id = %s AND status = 'retrying' + AND lease_expires_at <= %s""", + (now, retry_id, now), + ) + self._insert_event_row( + conn, + claim_id=None, + event_type="media_process", + from_status="retrying", + to_status="pending", + details="media_retry_lease_expired", + payload={"retry_id": retry_id}, + created_at=now, + ) safe_ids: list[int] = [] cursor_id = 0 page_size = min(max(limit, 25), 250) @@ -4489,11 +4533,13 @@ def claim_pending_media_retries(self, limit: int = 25) -> list[MediaRetryItem]: UPDATE media_retry_queue SET status = 'retrying', attempt_count = attempt_count + 1, + lease_owner = %s, + lease_expires_at = %s, updated_at = %s WHERE status = 'pending' AND id IN ({placeholders}) RETURNING * """, - (now, *safe_ids), + (owner, lease_expires_at, now, *safe_ids), ) rows = cur.fetchall() by_id = {int(row["id"]): row for row in rows} @@ -4542,6 +4588,7 @@ def record_media_retry_outcome( new_http = last_http_status if last_http_status is not None else current["last_http_status"] new_err = last_error if last_error is not None else current["last_error"] new_next = next_attempt_time if next_attempt_time is not None else current["next_attempt_time"] + clear_lease = status != "retrying" cur.execute( """ UPDATE media_retry_queue @@ -4550,11 +4597,23 @@ def record_media_retry_outcome( last_http_status = %s, last_error = %s, next_attempt_time = %s, + lease_owner = %s, + lease_expires_at = %s, updated_at = %s WHERE id = %s RETURNING * """, - (status, new_path, new_http, new_err, new_next, now, retry_id), + ( + status, + new_path, + new_http, + new_err, + new_next, + None if clear_lease else current["lease_owner"], + None if clear_lease else current["lease_expires_at"], + now, + retry_id, + ), ) row = cur.fetchone() self._insert_event_row( diff --git a/memorymaster/surfaces/operations.py b/memorymaster/surfaces/operations.py new file mode 100644 index 00000000..7a7abb08 --- /dev/null +++ b/memorymaster/surfaces/operations.py @@ -0,0 +1,100 @@ +"""Fail-closed operational CLI for recovery, health, and privacy planning.""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path +from typing import Any + +from memorymaster.core.service import MemoryService +from memorymaster.govern.operational_health import evaluate_operational_health +from memorymaster.govern.privacy_ops import PrivacySelector, build_privacy_plan +from memorymaster.govern.recovery import ( + create_encrypted_sqlite_backup, + postgres_recovery_plan, + run_sqlite_restore_drill, +) + + +def _print(payload: dict[str, Any]) -> None: + print(json.dumps(payload, ensure_ascii=True, indent=2, sort_keys=True)) + + +def _required_backup_key() -> str: + key = os.environ.get("MEMORYMASTER_BACKUP_KEY", "").strip() + if not key: + raise RuntimeError("MEMORYMASTER_BACKUP_KEY is required; keys are never accepted as CLI arguments") + return key + + +def _build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="MemoryMaster recovery, health, and privacy operations") + commands = parser.add_subparsers(dest="command", required=True) + backup = commands.add_parser("backup-create") + backup.add_argument("--db", required=True) + backup.add_argument("--destination", required=True) + backup.add_argument("--off-device", action="store_true") + backup.add_argument("--rpo-hours", type=int, default=24) + backup.add_argument("--rto-minutes", type=int, default=30) + drill = commands.add_parser("restore-drill") + drill.add_argument("--backup", required=True) + health = commands.add_parser("health") + health.add_argument("--db", required=True) + health.add_argument("--workspace", default=".") + health.add_argument("--manifest-dir", required=True) + health.add_argument("--owner", required=True) + health.add_argument("--runbook", default="docs/operations.md") + health.add_argument("--persist", action="store_true") + privacy = commands.add_parser("privacy-plan") + privacy.add_argument("--db", required=True) + privacy.add_argument("--workspace", default=".") + privacy.add_argument("--principal", required=True) + privacy.add_argument("--tenant-id") + privacy.add_argument("--scope") + commands.add_parser("postgres-recovery-plan") + return parser + + +def _run(args: argparse.Namespace) -> dict[str, Any]: + if args.command == "backup-create": + return create_encrypted_sqlite_backup( + args.db, + args.destination, + encryption_key=_required_backup_key(), + off_device=args.off_device, + rpo_hours=args.rpo_hours, + rto_minutes=args.rto_minutes, + ) + if args.command == "restore-drill": + return run_sqlite_restore_drill(args.backup, encryption_key=_required_backup_key()) + if args.command == "postgres-recovery-plan": + return postgres_recovery_plan() + if args.command == "privacy-plan": + return build_privacy_plan( + db_target=args.db, + workspace=args.workspace, + selector=PrivacySelector(args.principal, args.tenant_id, args.scope), + ) + service = MemoryService(args.db, workspace_root=Path(args.workspace)) + return evaluate_operational_health( + service, + backup_manifest_dir=args.manifest_dir, + persist=args.persist, + owner=args.owner, + runbook=args.runbook, + ) + + +def main(argv: list[str] | None = None) -> int: + try: + _print(_run(_build_parser().parse_args(argv))) + except (OSError, RuntimeError, ValueError) as exc: + _print({"ok": False, "error": str(exc)}) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/pyproject.toml b/pyproject.toml index 04484bbf..35cd742b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ memorymaster-dashboard = "memorymaster.surfaces.dashboard:main" memorymaster-steward = "memorymaster.govern.llm_steward:main" memorymaster-setup = "memorymaster.surfaces.setup_hooks:main" memorymaster-session-end = "memorymaster.surfaces.session_end_ingest:main" +memorymaster-ops = "memorymaster.surfaces.operations:main" [tool.ruff] target-version = "py310" diff --git a/tests/test_atlas_persisted_envelope.py b/tests/test_atlas_persisted_envelope.py index 9171e588..a1a56bba 100644 --- a/tests/test_atlas_persisted_envelope.py +++ b/tests/test_atlas_persisted_envelope.py @@ -648,7 +648,7 @@ def test_postgres_lists_count_safe_rows_after_unsafe_prefix( ] retry_base = {**common, "source_item_id": 1, "chat_id": None, "media_type": None, "media_path": None, "media_url": None, "status": "failed", "attempt_count": 0, "last_http_status": None, - "next_attempt_time": None} + "next_attempt_time": None, "lease_owner": None, "lease_expires_at": None} retries = [{**retry_base, "id": index + 1, "media_key": f"unsafe-{index}", "last_error": ENCODED, "updated_at": "2026-01-03T00:00:00Z"} for index in range(251)] + [ {**retry_base, "id": index + 252, "media_key": f"safe-{index}", "last_error": None, @@ -722,16 +722,24 @@ def __exit__(self, *args: object) -> None: def execute(self, sql: str, params: tuple[Any, ...]) -> None: self.executed.append((sql, params)) if sql.lstrip().startswith("SELECT"): + if "status = 'retrying'" in sql: + self.result = [] + return cursor_id = int(params[0]) self.result = [row for row in self.rows if int(row["id"]) > cursor_id][ : int(params[-1]) ] return - safe_ids = set(params[1:]) + safe_ids = set(params[3:]) self.result = [] for row in self.rows: if row["id"] in safe_ids: - row.update(status="retrying", attempt_count=row["attempt_count"] + 1) + row.update( + status="retrying", + attempt_count=row["attempt_count"] + 1, + lease_owner=params[0], + lease_expires_at=params[1], + ) self.result.append(dict(row)) def fetchall(self) -> list[dict[str, Any]]: @@ -757,7 +765,8 @@ def test_postgres_claim_filters_before_update_and_event(monkeypatch: pytest.Monk "id": 252, "source_item_id": 1, "media_key": "safe", "chat_id": None, "media_type": None, "media_path": None, "media_url": None, "status": "pending", "attempt_count": 0, "last_http_status": None, "last_error": None, - "next_attempt_time": None, "created_at": "2026-01-01T00:00:00Z", + "next_attempt_time": None, "lease_owner": None, "lease_expires_at": None, + "created_at": "2026-01-01T00:00:00Z", "updated_at": "2026-01-01T00:00:00Z", } unsafe_rows = [ @@ -776,7 +785,7 @@ def test_postgres_claim_filters_before_update_and_event(monkeypatch: pytest.Monk assert [row.id for row in claimed] == [252] assert all(row["status"] == "pending" and row["attempt_count"] == 0 for row in unsafe_rows) update_params = cursor.executed[-1][1] - assert update_params[1:] == (252,) + assert update_params[3:] == (252,) assert [event["payload"]["retry_id"] for event in events] == [252] assert "FOR UPDATE SKIP LOCKED" in cursor.executed[0][0] diff --git a/tests/test_media_retry_lease_recovery.py b/tests/test_media_retry_lease_recovery.py new file mode 100644 index 00000000..032cf404 --- /dev/null +++ b/tests/test_media_retry_lease_recovery.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +import importlib +from pathlib import Path +import sqlite3 + +from memorymaster.core.service import MemoryService + + +def _queued_service(tmp_path: Path) -> tuple[MemoryService, int]: + service = MemoryService(tmp_path / "lease.db", workspace_root=tmp_path) + service.init_db() + source = service.upsert_external_source(source_type="whatsapp", display_name="lease-test") + item = service.upsert_source_item( + source_id=source.id, + source_item_id="media-1", + item_type="audio", + payload_json={"media_path": "media/one.ogg"}, + ) + queued = service.enqueue_media_retry(source_item_id=item.id, media_key="media/one.ogg") + return service, queued.id + + +def test_expired_worker_lease_is_reclaimed_once(tmp_path: Path) -> None: + service, retry_id = _queued_service(tmp_path) + claimed = service.claim_pending_media_retries(limit=1, lease_owner="worker-a", lease_seconds=30) + assert claimed[0].lease_owner == "worker-a" + assert claimed[0].lease_expires_at + + with service.store.connect() as connection: + connection.execute( + "UPDATE media_retry_queue SET lease_expires_at = ? WHERE id = ?", + ("2000-01-01T00:00:00+00:00", retry_id), + ) + connection.commit() + + reclaimed = service.claim_pending_media_retries(limit=1, lease_owner="worker-b", lease_seconds=30) + assert [row.id for row in reclaimed] == [retry_id] + assert reclaimed[0].lease_owner == "worker-b" + assert reclaimed[0].attempt_count == 2 + details = [event.details for event in service.list_events(limit=50)] + assert details.count("media_retry_lease_expired") == 1 + + +def test_unexpired_worker_lease_is_not_stolen(tmp_path: Path) -> None: + service, _ = _queued_service(tmp_path) + service.claim_pending_media_retries(limit=1, lease_owner="worker-a", lease_seconds=300) + + assert service.claim_pending_media_retries(limit=1, lease_owner="worker-b", lease_seconds=300) == [] + + +def test_concurrent_reclaim_has_one_winner(tmp_path: Path) -> None: + service, retry_id = _queued_service(tmp_path) + service.claim_pending_media_retries(limit=1, lease_owner="dead-worker", lease_seconds=30) + with service.store.connect() as connection: + connection.execute( + "UPDATE media_retry_queue SET lease_expires_at = ? WHERE id = ?", + ("2000-01-01T00:00:00+00:00", retry_id), + ) + connection.commit() + + def claim(owner: str): + contender = MemoryService(service.store.db_path, workspace_root=tmp_path) + return contender.claim_pending_media_retries(limit=1, lease_owner=owner, lease_seconds=300) + + with ThreadPoolExecutor(max_workers=2) as pool: + results = list(pool.map(claim, ("worker-b", "worker-c"))) + + winners = [rows[0] for rows in results if rows] + assert len(winners) == 1 + assert winners[0].attempt_count == 2 + + +def test_lease_migration_upgrades_a_stale_queue() -> None: + connection = sqlite3.connect(":memory:") + connection.execute("CREATE TABLE media_retry_queue (id INTEGER PRIMARY KEY, status TEXT)") + migration = importlib.import_module("memorymaster.stores.migrations.0016_media_retry_leases") + + migration.apply_sqlite(connection) + + columns = {row[1] for row in connection.execute("PRAGMA table_info(media_retry_queue)")} + assert {"lease_owner", "lease_expires_at"} <= columns + indexes = {row[1] for row in connection.execute("PRAGMA index_list(media_retry_queue)")} + assert "idx_media_retry_lease_expiry" in indexes + + +def test_baseline_schemas_keep_sqlite_postgres_lease_parity() -> None: + root = Path(__file__).resolve().parents[1] + sqlite_schema = (root / "memorymaster/schema.sql").read_text(encoding="utf-8") + postgres_schema = (root / "memorymaster/schema_postgres.sql").read_text(encoding="utf-8") + + for field in ("lease_owner", "lease_expires_at", "idx_media_retry_lease_expiry"): + assert field in sqlite_schema + assert field in postgres_schema diff --git a/tests/test_operational_health.py b/tests/test_operational_health.py new file mode 100644 index 00000000..3edc416e --- /dev/null +++ b/tests/test_operational_health.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from pathlib import Path + +from memorymaster.core.audit_envelope import build_audit_envelope +from memorymaster.core.service import MemoryService +from memorymaster.govern.operational_health import evaluate_operational_health, otel_status + + +def test_audit_envelope_is_attributable() -> None: + envelope = build_audit_envelope( + principal="operator-a", + tenant_id="tenant-a", + role="admin", + request_id="request-a", + session_id="session-a", + action="recovery.drill", + target="sqlite-backup", + result="pass", + ) + + assert set(envelope) >= { + "principal", + "tenant_id", + "role", + "request_id", + "session_id", + "action", + "target", + "result", + "occurred_at", + } + + +def test_operational_health_persists_alert_snapshot(tmp_path: Path) -> None: + db = tmp_path / "ops.db" + service = MemoryService(db, workspace_root=tmp_path) + service.init_db() + + result = evaluate_operational_health( + service, + backup_manifest_dir=tmp_path / "missing-backups", + persist=True, + owner="platform-owner", + runbook="docs/operations.md", + ) + + assert result["status"] == "alert" + assert any(alert["code"] == "backup_missing" for alert in result["alerts"]) + events = service.list_events(limit=20, event_type="system") + assert any(event.details == "operational_health_snapshot" for event in events) + assert result["owner"] == "platform-owner" + + +def test_otel_is_explicitly_disabled_by_default(monkeypatch) -> None: + monkeypatch.delenv("MEMORYMASTER_OTEL_ENDPOINT", raising=False) + assert otel_status()["status"] == "disabled" diff --git a/tests/test_operations_cli.py b/tests/test_operations_cli.py new file mode 100644 index 00000000..caf67c8a --- /dev/null +++ b/tests/test_operations_cli.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from memorymaster.core.service import MemoryService +from memorymaster.surfaces.operations import main + + +def test_privacy_cli_is_dry_run_only(tmp_path: Path, capsys) -> None: + db = tmp_path / "ops.db" + service = MemoryService(db, workspace_root=tmp_path) + service.init_db() + + code = main([ + "privacy-plan", + "--db", + str(db), + "--workspace", + str(tmp_path), + "--principal", + "operator-a", + ]) + + payload = json.loads(capsys.readouterr().out) + assert code == 0 + assert payload["dry_run"] is True + assert payload["mutation_count"] == 0 + + +def test_backup_cli_refuses_command_line_key_and_missing_secret(tmp_path: Path, monkeypatch, capsys) -> None: + monkeypatch.delenv("MEMORYMASTER_BACKUP_KEY", raising=False) + code = main([ + "backup-create", + "--db", + str(tmp_path / "unused.db"), + "--destination", + str(tmp_path / "unused.enc"), + ]) + + payload = json.loads(capsys.readouterr().out) + assert code == 2 + assert "MEMORYMASTER_BACKUP_KEY" in payload["error"] diff --git a/tests/test_privacy_operations.py b/tests/test_privacy_operations.py new file mode 100644 index 00000000..e7116a88 --- /dev/null +++ b/tests/test_privacy_operations.py @@ -0,0 +1,69 @@ +from __future__ import annotations + +import hashlib +from pathlib import Path + +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.govern.privacy_ops import PrivacySelector, build_privacy_plan +from memorymaster.recall.verbatim_store import ensure_verbatim_schema, store_verbatim + + +def _sha(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def test_privacy_plan_is_inventory_driven_and_strictly_read_only(tmp_path: Path) -> None: + db = tmp_path / "privacy.db" + service = MemoryService(db, workspace_root=tmp_path) + service.init_db() + service.ingest( + "A principal-bound privacy export claim.", + citations=[CitationInput(source="test", locator="privacy")], + source_agent="agent-a", + holder="agent-a", + scope="project:privacy", + visibility="private", + ) + ensure_verbatim_schema(str(db)) + store_verbatim( + str(db), + "session-a", + "user", + "This is a sufficiently long principal-bound transcript row.", + scope="project:privacy", + source_agent="agent-a", + ) + artifact = tmp_path / "artifacts" / "privacy" / "opaque.json" + artifact.parent.mkdir(parents=True) + artifact.write_text('{"principal":"agent-a"}', encoding="utf-8") + backup = tmp_path / ".memorymaster" / "snapshots" / "old.db" + backup.parent.mkdir(parents=True) + backup.write_bytes(b"not-a-live-database") + before = {path: _sha(path) for path in (db, artifact, backup)} + + plan = build_privacy_plan( + db_target=db, + workspace=tmp_path, + selector=PrivacySelector(principal="agent-a", scope="project:privacy"), + ) + + after = {path: _sha(path) for path in (db, artifact, backup)} + assert before == after + assert plan["dry_run"] is True + assert plan["complete"] is False + by_surface = {row["surface"]: row for row in plan["surfaces"]} + assert by_surface["primary_db"]["status"] == "FOUND" + assert by_surface["verbatim"]["status"] == "FOUND" + assert by_surface["artifacts"]["status"] == "BLOCKED-EXTERNAL" + assert by_surface["qdrant"]["status"] == "BLOCKED-EXTERNAL" + assert by_surface["backups"]["disposition"] == "expire_by_policy" + + +def test_privacy_selector_requires_a_principal() -> None: + try: + PrivacySelector(principal=" ") + except ValueError as exc: + assert "principal" in str(exc) + else: + raise AssertionError("blank privacy principal was accepted") diff --git a/tests/test_recovery_operations.py b/tests/test_recovery_operations.py new file mode 100644 index 00000000..be8517b0 --- /dev/null +++ b/tests/test_recovery_operations.py @@ -0,0 +1,77 @@ +from __future__ import annotations + +from datetime import datetime, timedelta, timezone +import json +from pathlib import Path + +from cryptography.fernet import Fernet + +from memorymaster.core.models import CitationInput +from memorymaster.core.service import MemoryService +from memorymaster.govern.recovery import ( + backup_status, + create_encrypted_sqlite_backup, + postgres_recovery_plan, + run_sqlite_restore_drill, +) + + +def _database(tmp_path: Path) -> Path: + db = tmp_path / "source" / "memory.db" + db.parent.mkdir(parents=True) + service = MemoryService(db, workspace_root=tmp_path) + service.init_db() + service.ingest( + "Recovery drill preserves this claim.", + citations=[CitationInput(source="test", locator="recovery")], + source_agent="recovery-test", + ) + return db + + +def test_encrypted_off_device_backup_roundtrip_and_integrity(tmp_path: Path) -> None: + db = _database(tmp_path) + key = Fernet.generate_key().decode() + destination = tmp_path / "off-device" / "memory.db.enc" + + manifest = create_encrypted_sqlite_backup( + db, + destination, + encryption_key=key, + off_device=True, + rpo_hours=24, + rto_minutes=15, + ) + + assert destination.exists() + assert not destination.with_suffix(".plain").exists() + assert manifest["encrypted"] is True + assert manifest["off_device"] is True + assert len(manifest["plaintext_sha256"]) == 64 + assert json.loads(destination.with_suffix(".enc.manifest.json").read_text())["backend"] == "sqlite" + + drill = run_sqlite_restore_drill(destination, encryption_key=key) + assert drill["integrity_check"] == "ok" + assert drill["foreign_key_violations"] == 0 + assert drill["rto_met"] is True + + +def test_backup_age_alerts_on_stale_manifest(tmp_path: Path) -> None: + manifest = tmp_path / "stale.manifest.json" + manifest.write_text( + json.dumps({"created_at": (datetime.now(timezone.utc) - timedelta(hours=49)).isoformat()}), + encoding="utf-8", + ) + + status = backup_status(tmp_path, max_age_hours=24) + assert status["status"] == "alert" + assert status["age_hours"] >= 48 + + +def test_postgres_recovery_is_a_redacted_external_plan() -> None: + plan = postgres_recovery_plan() + + assert plan["status"] == "BLOCKED-EXTERNAL" + assert plan["backend"] == "postgres" + assert "pg_dump" in plan["required_tools"] + assert "dsn" not in json.dumps(plan).lower() From c4a4f4fa942f0f1e400603649d1c2f0ddffd7da6 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Tue, 14 Jul 2026 00:48:29 -0300 Subject: [PATCH 42/55] test: converge phase 3 readiness gates --- .planning/REMAINING-PHASES-V1-BUDGET.md | 46 +++++++++++++++ .planning/audit-remediation-ledger.md | 2 +- .../audit-delta.md | 59 +++++++++++++++++++ .../pr-draft.md | 33 +++++++++++ benchmarks/perf_smoke.py | 5 ++ memorymaster/govern/recovery.py | 4 +- tests/test_bm25_per_field.py | 5 ++ tests/test_local_rerank.py | 4 ++ tests/test_recall_latency.py | 3 + tests/test_recall_vector_fallback.py | 3 + tests/test_retrieval_profile.py | 10 +++- 11 files changed, 170 insertions(+), 4 deletions(-) create mode 100644 .planning/audits/2026-07-14-phase3-budget-delta/audit-delta.md create mode 100644 .planning/audits/2026-07-14-phase3-budget-delta/pr-draft.md diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index c467f51d..ab885bf4 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -443,3 +443,49 @@ on 2026-07-13: The next boundary is Phase 3 convergence. The active roadmap goal authorizes continuing without a new goal. + +Phase 3 convergence completed on 2026-07-14: + +- Collection discovered 4,218 tests. The single full non-ML invocation + completed with 4,042 passed, 70 skipped, 95 deselected, one expected xfail, + and ten failures in 942.30s. Nine failures were stale trusted-recall/vector + test doubles and one was R3.5's direct SQLite open; the one bounded correction + batch passed the exact combined scope (19 tests). No second full-suite pass is + claimed. +- The Phase 3 targeted matrix passed 193 tests with 40 marker deselections. + Project Ruff and `git diff --check` passed. The isolated ML gate produced 93 + passed and two stale candidate/scope fixture failures; canonical confirmation + and explicit in-scope setup passed the exact two tests. This is compositional + ML evidence, not a rerun claim. +- The initial three-attempt quick SLO gate exposed an environment-dependent + benchmark: optional sentence-transformers reduced query throughput to + 1.36-1.46 ops/s even though CI installs no ML extra. The core smoke now pins + deterministic hash embeddings while the dedicated ML gate owns semantic + evidence. The corrected gate passed in 6.734s: ingest 52.77 ops/s, query + 20.65 ops/s, query p95 0.0547s, zero misses. +- A clean wheel built and installed into an isolated venv. Minimal setup passed + after an isolated Codex client root was present; semantic, team, and full-lab + returned the expected nonzero BLOCKED status without external services. +- Disposable SQLite encrypted backup/restore passed authentication, checksums, + integrity, zero foreign-key violations, and RTO (0.046s versus 1,800s). The + operational health result was OK. The tenant-aware privacy plan was dry-run, + incomplete by design for Qdrant, and reported zero mutations. +- The freshly built `memorymaster:phase3-local` image + (`sha256:42fa9252110d8b5458c1bf099bb6f9b158b49013622efce06cc88af661b8d146`) + returned dashboard and MCP HTTP readiness 200; stdio and authenticated HTTP + initialize/tools-list handshakes passed. Compose rendered with synthetic + secrets/digests. Test containers and volumes were removed. +- Helm/disposable Kubernetes, approved immutable release images, authenticated + TLS Qdrant, disposable two-role Postgres, off-device recovery, and an + organization telemetry backend remain `BLOCKED-EXTERNAL`. No pass is claimed + for them. +- Same-scope audit found no new unresolved reproducible Critical/High branch + regression. Scheduled repository findings are resolved; MM-OPS-04, + MM-OPS-05, MM-PRIV-01, and MM-PRIV-02 remain explicit external blockers. +- Audit delta: `.planning/audits/2026-07-14-phase3-budget-delta/audit-delta.md`. + Draft PR: `.planning/audits/2026-07-14-phase3-budget-delta/pr-draft.md`. +- Atomic convergence commit: the conventional Phase 3 convergence commit + containing this evidence. + +The next package is R4.1 extension boundaries. The active roadmap goal +authorizes continuing without a new goal. diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 3e8ae86b..040bba0f 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -40,7 +40,7 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-MAINT-01 | Maintainability/Integrity | Medium / BAD-PRACTICE | Versions, tool counts, roadmaps, and install claims drift | R4.4 | OPEN | Generated single-source values checked by CI | | MM-MAINT-02 | Architecture/Maintainability | Medium / BAD-PRACTICE | Oversized facades, hardwired extensions, dead plugin seam, and lingering shims | R4.1-R4.2 | OPEN | Measured boundary/size budgets and supported shim policy | | MM-REL-03 | Reliability | Medium / BAD-PRACTICE | Media retry rows have no expired-lease recovery | R3.5 | RESOLVED | Immutable migration 0016 and SQLite/Postgres parity add bounded owner/expiry leases, atomic expired-work recovery with an audit event, lease clearing on terminal outcomes, and concurrent single-winner coverage. The focused R3.5 gate includes worker-death reclaim, unexpired non-steal, and concurrent claimer tests. | -| MM-OPS-05 | Missing Operations | High / BAD-PRACTICE | Recovery defaults stop at same-machine SQLite snapshots | R3.5 | BLOCKED-EXTERNAL | Off-device SQLite/Postgres restore drill and RPO/RTO | +| MM-OPS-05 | Missing Operations | High / BAD-PRACTICE | Recovery defaults stop at same-machine SQLite snapshots | R3.5 | BLOCKED-EXTERNAL | R3.5/Phase 3 convergence adds authenticated encrypted SQLite backups, plaintext/ciphertext checksums, online backup, disposable restore, integrity/FK checks, backup-age health, and measured RPO/RTO (0.046s restore versus 1,800s objective). Approved off-device storage and disposable PostgreSQL dump/restore evidence remain external; same-machine success is not represented as full recovery readiness. | | MM-OBS-01 | Missing Operations | High / BAD-PRACTICE | Failures/metrics are process-local without central alert ownership | R3.5 | RESOLVED | R3.5 adds deterministic backup/retry/integrity/WAL/disk/provider/telemetry checks and can persist one aggregate audit envelope with explicit owner and runbook. Focused tests cover alert severity, persistence, and audit fields. Delivery to an organization-selected OTel/error backend remains environment-specific readiness evidence, not a repository correctness claim. | | MM-DB-01 | Database | Medium / BAD-PRACTICE | Fast schema fingerprint can omit legacy ensure-helper changes | R3.2 | RESOLVED | `schema_stamp()` now hashes baseline schema, immutable migrations, and the complete legacy ensure-helper source. A helper-body mutation regression changes the stamp and forces the full initialization path. New R3.2 DDL exists only in migration 0015/baselines. | | MM-INTEGRITY-01 | Code Integrity | Medium / BAD-PRACTICE | `importlib.util` probe is wrong in clean Python and its test masks failure | R4.4 | OPEN | Clean subprocess test and explicit import | diff --git a/.planning/audits/2026-07-14-phase3-budget-delta/audit-delta.md b/.planning/audits/2026-07-14-phase3-budget-delta/audit-delta.md new file mode 100644 index 00000000..69c79201 --- /dev/null +++ b/.planning/audits/2026-07-14-phase3-budget-delta/audit-delta.md @@ -0,0 +1,59 @@ +# Phase 3 Budget Audit Delta — 2026-07-14 + +## Scope and outcome + +This targeted delta reconciles only MM-OPS-02, MM-OPS-04, MM-OPS-05, +MM-PERF-01..04, MM-UX-01, MM-PRIV-01..02, MM-COST-03, MM-REL-03, +MM-OBS-01, and MM-DB-01. It is not a new 13-domain audit. + +No unresolved reproducible Critical/High regression introduced by the Phase 3 +branch remains in the local scope. The gate discovered benchmark-environment +coupling and stale governed-recall fixtures; both were corrected within the one +bounded correction batch. No new finding remains open. + +| Finding set | Delta | +|---|---| +| MM-PERF-01..04, MM-COST-03, MM-DB-01 | RESOLVED — durable embedding/query/storage/cursor/index/budget evidence and corrected deterministic core SLO | +| MM-UX-01 | RESOLVED — clean-wheel minimal setup PASS; requested external profiles fail truthfully | +| MM-REL-03 | RESOLVED — replay-safe bounded retry leases with concurrent/worker-death evidence | +| MM-OBS-01 | RESOLVED — persistent aggregate health envelope with owner/runbook; external collector selection remains environment evidence | +| MM-OPS-02 | RESOLVED — distinct authenticated HTTP services and protocol/readiness runtime evidence | +| MM-OPS-04 | BLOCKED-EXTERNAL — approved immutable image/TLS/Kubernetes/scanner evidence unavailable | +| MM-OPS-05 | BLOCKED-EXTERNAL — local encrypted SQLite drill passes; off-device/Postgres drill unavailable | +| MM-PRIV-01 | BLOCKED-EXTERNAL — repository capture controls complete; legal/product consent decision unavailable | +| MM-PRIV-02 | BLOCKED-EXTERNAL — complete read-only inventory plan exists; legal disposition/backend attribution/live apply approval unavailable | + +## Runtime evidence + +| Gate | Evidence | +|---|---| +| Collection | 4,218 tests collected | +| Full non-ML, once | 4,042 passed, 10 failed, 70 skipped, 95 deselected, 1 xfailed in 942.30s | +| Bounded correction | Exact combined regression/recovery scope: 19 passed; no second full run claimed | +| Targeted Phase 3 matrix | 193 passed, 40 deselected | +| ML | 93 passed, 2 governed-fixture failures; exact correction 2 passed | +| Quick performance SLO | 6.734s total; ingest 52.77 ops/s; query 20.65 ops/s; p95 0.0547s; zero misses | +| Clean wheel | Build/install PASS; minimal setup PASS; semantic/team/full-lab truthfully BLOCKED | +| Recovery/privacy | Encrypted restore integrity OK, 0 FK violations, RTO met at 0.046s; privacy mutation_count=0 | +| Deployment | Fresh image dashboard/MCP readiness 200; stdio and authenticated HTTP MCP handshakes PASS; Compose config PASS | +| Static quality | Project Ruff and `git diff --check` PASS | + +## External evidence retained + +- Helm CLI/disposable Kubernetes and approved immutable release images. +- Authenticated/TLS disposable Qdrant and two-role disposable Postgres. +- Approved off-device backup destination and Postgres dump/restore drill. +- Organization OTel/error backend and alert delivery ownership. +- Legal/product privacy, consent, processor, retention, and data-disposition decisions. +- Existing unsuppressed Gitleaks history classification and approved release-image scans. + +No live database, paid provider, production service, credential, backup, or +external/product data was mutated. + +## Rollback + +Revert the Phase 3 convergence commit first, then revert R3.5 through R3.1 in +reverse order. Do not downgrade a database in place: restore a verified backup +or keep forward-compatible migrations 0014–0016 applied. Return service +profiles to the last reviewed authenticated dashboard/MCP configuration; never +restore stdio behind an HTTP port or unbounded capture/retry behavior. diff --git a/.planning/audits/2026-07-14-phase3-budget-delta/pr-draft.md b/.planning/audits/2026-07-14-phase3-budget-delta/pr-draft.md new file mode 100644 index 00000000..728f2a9e --- /dev/null +++ b/.planning/audits/2026-07-14-phase3-budget-delta/pr-draft.md @@ -0,0 +1,33 @@ +# Draft PR — Phase 3 performance and operational readiness + +## Summary + +- Persist embedding fingerprints and replayable Qdrant reconciliation cursors. +- Add generation-aware query statistics, durable shared budgets, event indexes, + finite result contracts, and deterministic performance smoke. +- Make setup profiles truthful from a clean wheel. +- Separate stdio MCP, authenticated HTTP MCP, and dashboard deployment profiles + with real readiness and bounded resources. +- Add retry leases, encrypted recovery drills, persistent operational health, + attributable audit envelopes, and read-only privacy planning. + +## Verification + +- Phase 3 matrix: 193 passed. +- Full non-ML: 4,042 passed with 10 corrected failures; exact correction 19 + passed. A second full run was intentionally not performed. +- ML: 93 passed plus exact two-test correction. +- Quick SLO: PASS at 20.65 query ops/s and 0.0547s p95. +- Clean-wheel minimal setup, encrypted SQLite restore, Compose, dashboard/MCP + readiness, and stdio/authenticated HTTP MCP handshakes: PASS. +- Project Ruff and diff check: PASS. + +## Required external review before release + +- Resolve every applicable BLOCKED-EXTERNAL row in + `external-actions-required.md`, especially Postgres/Qdrant/Kubernetes, + off-device recovery, history/image scans, telemetry, and privacy/legal review. +- Build and scan approved immutable release artifacts; the local image digest in + this delta is development evidence only. +- No push, merge, publish, deploy, credential rotation, or live-data operation + is part of this PR draft. diff --git a/benchmarks/perf_smoke.py b/benchmarks/perf_smoke.py index 60ed371e..1a80e3ed 100644 --- a/benchmarks/perf_smoke.py +++ b/benchmarks/perf_smoke.py @@ -10,6 +10,7 @@ from typing import Any from memorymaster.models import CitationInput +from memorymaster.recall.embeddings import EmbeddingProvider from memorymaster.service import MemoryService DEFAULT_INGEST_P95_MAX = 0.060 @@ -156,6 +157,10 @@ def run_perf_smoke(*, claims: int, queries: int, cycles: int, workspace_root: Pa db_path_str, workspace_root=Path(normalize_fs_path(workspace_root)), ) + # Keep the core CI SLO independent of whichever optional ML/provider extras + # happen to be installed on the runner. Semantic quality/performance is + # covered by its dedicated ML gate; this smoke measures storage/retrieval. + service.embedding_provider = EmbeddingProvider(model="hash-v1", dims=1536) try: service.init_db() diff --git a/memorymaster/govern/recovery.py b/memorymaster/govern/recovery.py index 04419472..ed51ef00 100644 --- a/memorymaster/govern/recovery.py +++ b/memorymaster/govern/recovery.py @@ -6,12 +6,12 @@ import hashlib import json from pathlib import Path -import sqlite3 import tempfile import time from typing import Any from memorymaster.stores import snapshot +from memorymaster.stores._storage_shared import connect_ro def _utc_now() -> str: @@ -52,7 +52,7 @@ def _decrypt(encryption_key: str, payload: bytes) -> bytes: def _sqlite_checks(db_path: Path) -> tuple[str, int]: - connection = sqlite3.connect(str(db_path)) + connection = connect_ro(db_path) try: integrity = str(connection.execute("PRAGMA integrity_check").fetchone()[0]) fk_rows = connection.execute("PRAGMA foreign_key_check").fetchall() diff --git a/tests/test_bm25_per_field.py b/tests/test_bm25_per_field.py index fbe91003..991d5a34 100644 --- a/tests/test_bm25_per_field.py +++ b/tests/test_bm25_per_field.py @@ -55,6 +55,11 @@ def _make_claim(cid: int, subject: str, text: str) -> MagicMock: claim.subject = subject claim.text = text # ``recall`` attribute-probes several fields; default them to sane values. + claim.status = "confirmed" + claim.scope = "project" + claim.visibility = "public" + claim.object_value = None + claim.predicate = None claim.confidence_score = 0.0 return claim diff --git a/tests/test_local_rerank.py b/tests/test_local_rerank.py index 8d44df01..aa5f5282 100644 --- a/tests/test_local_rerank.py +++ b/tests/test_local_rerank.py @@ -83,6 +83,10 @@ def _row(cid: int, text: str) -> dict: claim.subject = None claim.wiki_article = None claim.status = "confirmed" + claim.scope = "project" + claim.visibility = "public" + claim.object_value = None + claim.predicate = None return {"claim": claim, "lexical_score": 0.5, "confidence_score": 0.5} diff --git a/tests/test_recall_latency.py b/tests/test_recall_latency.py index 547944a2..0682ae30 100644 --- a/tests/test_recall_latency.py +++ b/tests/test_recall_latency.py @@ -26,6 +26,9 @@ def _mock_claim(cid: int, text: str = "working on memorymaster project", subject c.subject = subject c.scope = "project:memorymaster" c.status = "confirmed" + c.visibility = "public" + c.object_value = None + c.predicate = None c.confidence = 0.9 c.wiki_article = None return c diff --git a/tests/test_recall_vector_fallback.py b/tests/test_recall_vector_fallback.py index 28d16e26..32405fb8 100644 --- a/tests/test_recall_vector_fallback.py +++ b/tests/test_recall_vector_fallback.py @@ -12,6 +12,7 @@ import pytest from memorymaster.recall import context_hook, qdrant_recall_fallback +from memorymaster.core.lifecycle import transition_claim from memorymaster.core.models import CitationInput from memorymaster.core.service import MemoryService @@ -228,6 +229,8 @@ def test_prompt_recall_keeps_authoritative_lexical_results_during_quarantine( ): """Containment removes vector candidates without degrading lexical recall.""" svc, seeded = service + monkeypatch.setenv("MEMORYMASTER_SCOPE_DEFAULT", "test") + transition_claim(svc.store, seeded[0], "confirmed", "trusted recall fixture") monkeypatch.setenv("MEMORYMASTER_RECALL_VECTOR_FALLBACK", "1") monkeypatch.setenv("MEMORYMASTER_QDRANT_URL", "http://mocked.local:6333") monkeypatch.setenv("MEMORYMASTER_RECALL_W_VECTOR", "1.0") diff --git a/tests/test_retrieval_profile.py b/tests/test_retrieval_profile.py index d92f7a7b..65d73f4d 100644 --- a/tests/test_retrieval_profile.py +++ b/tests/test_retrieval_profile.py @@ -50,7 +50,15 @@ def __init__(self, claims: list[Claim], vector_scores: dict[int, float] | None = def list_claims(self, **kwargs): return self.claims[: kwargs.get("limit", len(self.claims))] - def vector_scores(self, query: str, claims: list[Claim], provider: EmbeddingProvider): + def vector_scores( + self, + query: str, + claims: list[Claim], + provider: EmbeddingProvider, + *, + query_vector: list[float] | None = None, + ): + del query, provider, query_vector if not self._vector_scores: return {} return {claim.id: self._vector_scores.get(claim.id, 0.0) for claim in claims} From b361a13c03fc634de841691bf196f0933fae8f00 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Tue, 14 Jul 2026 00:59:23 -0300 Subject: [PATCH 43/55] refactor: enforce companion extension boundaries --- .planning/REMAINING-PHASES-V1-BUDGET.md | 25 ++ ...EMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md | 2 +- .planning/audit-remediation-ledger.md | 2 +- .planning/codebase/STRUCTURE.md | 13 +- docs/architecture.md | 31 +- memorymaster/core/plugins.py | 127 -------- memorymaster/core/service.py | 20 +- memorymaster/govern/steward_features.py | 14 +- memorymaster/knowledge/wiki_similarity.py | 4 + memorymaster/plugins.py | 11 - tests/test_extension_boundaries.py | 99 ++++++ tests/test_p2_cycle_cuts.py | 4 +- tests/test_plugins.py | 284 ------------------ 13 files changed, 182 insertions(+), 454 deletions(-) delete mode 100644 memorymaster/core/plugins.py delete mode 100644 memorymaster/plugins.py create mode 100644 tests/test_extension_boundaries.py delete mode 100644 tests/test_plugins.py diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index ab885bf4..1463e5b8 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -489,3 +489,28 @@ Phase 3 convergence completed on 2026-07-14: The next package is R4.1 extension boundaries. The active roadmap goal authorizes continuing without a new goal. + +R4.1 core and companion extension boundaries completed on 2026-07-14: + +- Claims, lifecycle, citations, policy, recall, stewardship, stores, and + telemetry remain in the authoritative core. Wiki/Obsidian, Dream/OpenClaw, + Atlas/media/actions, local search, and specialized integrations are explicit + companion namespaces composed by surface handlers. +- Import-boundary acceptance pins prevent core, governance, recall, and store + modules from importing optional Wiki/vault or bridge modules. Importing + `MemoryService` no longer installs a Wiki side effect; importing the Wiki + companion explicitly retains lifecycle autopromotion. +- Stewardship consumes Wiki similarity through the narrow read-only + `WikiSimilarityCorpus` protocol. Existing local-search, transcription, and + OCR provider protocols remain the supported integration seams. +- The deprecated generic plugin registry and root compatibility shim were + removed after their documented one-minor window. GitNexus found zero + production consumers; the only direct consumer was its deleted test suite. + No arbitrary validator/retrieval/ingestion entry-point boundary was added. +- RED boundary evidence initially failed three of four tests. The focused + companion/provider/autopromotion gate then passed 87 tests. Changed-file + Ruff, syntax compilation, and `git diff --check` passed. +- Atomic package commit: the conventional R4.1 commit containing this evidence. + +The next package is R4.2 gradual orchestration decomposition. The active +roadmap goal authorizes continuing without a new goal. diff --git a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md index 0eabe4fc..0872da27 100644 --- a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md +++ b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md @@ -99,7 +99,7 @@ Phase 0 must replace qualitative capacity goals with recorded numeric gates in t - [ ] R3.3 — truthful setup profiles - [ ] R3.4 — service entrypoints and deployment health - [ ] R3.5 — recovery, observability, audit, and privacy operations -- [ ] R4.1 — core/extension boundaries +- [x] R4.1 — core/extension boundaries - [ ] R4.2 — decompose oversized orchestration points - [ ] R4.3 — governance UX and accessibility - [ ] R4.4 — generated release/documentation truth diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 040bba0f..01140f06 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -38,7 +38,7 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-COST-03 | Cost/Reliability | Medium / BAD-PRACTICE | Cost and intake controls are fragmented/process-local | R2.4,R3.2 | RESOLVED | R3.2 adds one WAL-backed atomic usage ledger shared by LLM primary/fallback calls, Gemini embedding batches, MCP ingest/checkpoint, and configured core intake quotas. Caps remain backward-compatible/unlimited unless configured; restart/cross-instance and local-bucket-reset tests prove durable global/provider/actor enforcement without consuming capacity on rejected reservations. | | MM-DEMO-01 | Demo/Integrity | High / EXPLOITABLE-NOW | Atlas defaults persist fabricated mock evidence at high confidence | R2.5 | RESOLVED | P2-F (`c66e240`) requires a real configured production provider, confines synthetic providers to explicit dev/test opt-in, excludes synthetic evidence from claims/prompts/actions/citations/exports, and fails configuration nonzero. Focused gate: 111 passed. | | MM-MAINT-01 | Maintainability/Integrity | Medium / BAD-PRACTICE | Versions, tool counts, roadmaps, and install claims drift | R4.4 | OPEN | Generated single-source values checked by CI | -| MM-MAINT-02 | Architecture/Maintainability | Medium / BAD-PRACTICE | Oversized facades, hardwired extensions, dead plugin seam, and lingering shims | R4.1-R4.2 | OPEN | Measured boundary/size budgets and supported shim policy | +| MM-MAINT-02 | Architecture/Maintainability | Medium / BAD-PRACTICE | Oversized facades, hardwired extensions, dead plugin seam, and lingering shims | R4.1-R4.2 | PARTIAL | R4.1 removes the zero-consumer generic plugin registry, stops core service imports from installing Wiki behavior, routes Wiki similarity through a narrow protocol, and enforces core-to-companion import boundaries. R4.2 owns facade decomposition, size budgets, and the dated shim policy. | | MM-REL-03 | Reliability | Medium / BAD-PRACTICE | Media retry rows have no expired-lease recovery | R3.5 | RESOLVED | Immutable migration 0016 and SQLite/Postgres parity add bounded owner/expiry leases, atomic expired-work recovery with an audit event, lease clearing on terminal outcomes, and concurrent single-winner coverage. The focused R3.5 gate includes worker-death reclaim, unexpired non-steal, and concurrent claimer tests. | | MM-OPS-05 | Missing Operations | High / BAD-PRACTICE | Recovery defaults stop at same-machine SQLite snapshots | R3.5 | BLOCKED-EXTERNAL | R3.5/Phase 3 convergence adds authenticated encrypted SQLite backups, plaintext/ciphertext checksums, online backup, disposable restore, integrity/FK checks, backup-age health, and measured RPO/RTO (0.046s restore versus 1,800s objective). Approved off-device storage and disposable PostgreSQL dump/restore evidence remain external; same-machine success is not represented as full recovery readiness. | | MM-OBS-01 | Missing Operations | High / BAD-PRACTICE | Failures/metrics are process-local without central alert ownership | R3.5 | RESOLVED | R3.5 adds deterministic backup/retry/integrity/WAL/disk/provider/telemetry checks and can persist one aggregate audit envelope with explicit owner and runbook. Focused tests cover alert severity, persistence, and audit fields. Delivery to an organization-selected OTel/error backend remains environment-specific readiness evidence, not a repository correctness claim. | diff --git a/.planning/codebase/STRUCTURE.md b/.planning/codebase/STRUCTURE.md index b495e2ee..931e86b2 100644 --- a/.planning/codebase/STRUCTURE.md +++ b/.planning/codebase/STRUCTURE.md @@ -1,4 +1,10 @@ -# STRUCTURE.md — Directory Structure (v3.28.0) +# STRUCTURE.md — Directory Structure (R4.1 boundary refresh) + +The authoritative core is split across `memorymaster/core`, `stores`, `recall`, +and `govern`. Optional integrations live in the built-in companion namespaces +`memorymaster/bridges` and the `memorymaster/knowledge/wiki_*` / `vault_*` +modules. `memorymaster/surfaces` is the composition root. Core-to-companion +imports are forbidden and pinned by `tests/test_extension_boundaries.py`. ``` G:/_OneDrive/OneDrive/Desktop/Py Apps/memorymaster/ @@ -39,7 +45,7 @@ G:/_OneDrive/OneDrive/Desktop/Py Apps/memorymaster/ │ ├── atlas_contract.py / atlas_claim_extractor.py / action_extractor.py / action_exporters.py │ ├── media_processing.py / media_providers.py / connectors/whatsapp.py # Atlas Inbox evidence │ ├── transcript_miner.py / turn_schema.py / session_tracker.py # Transcript -> claims pipeline -│ ├── operator.py / operator_queue.py / scheduler.py / webhook.py / plugins.py / retry.py +│ ├── operator.py / operator_queue.py / scheduler.py / webhook.py / retry.py │ ├── scope_utils.py / config.py / policy.py / federated_graphify.py / qmd_bridge.py │ ├── setup_hooks.py / config_templates/ # Installer + hook/MD templates for Claude/Codex │ └── skill_evolver.py / auto_extractor.py / candidate_dedupe.py / review.py @@ -65,3 +71,6 @@ Notes - Storage and CLI were deliberately split into `_storage_*` / `cli_handlers_*` modules to keep every file under 800 LOC. - Migration 0005 does not exist; runner.py tolerates gaps and verifies checksums (drift detection). +- The retired generic plugin registry is not a supported extension surface. + Optional integrations use the typed provider protocols and companion package + boundaries documented in `docs/architecture.md`. diff --git a/docs/architecture.md b/docs/architecture.md index 652730a9..71fc793b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -37,6 +37,36 @@ External surfaces Obsidian vault, dashboard, Qdrant, LLM providers, Auto Dream, Atlas adapters ``` +## Core and companion extension boundary + +The authoritative memory system lives in `memorymaster.core`, `memorymaster.stores`, +`memorymaster.recall`, and `memorymaster.govern`: claims, lifecycle, citations, +policy, retrieval, conflict/stewardship, and telemetry remain usable without +loading optional integrations. + +Optional product integrations are built-in companion namespaces composed only +at explicit surfaces: + +| Companion | Owned namespace | Composition roots | +|---|---|---| +| Wiki and Obsidian | `memorymaster.knowledge.wiki_*`, `memorymaster.knowledge.vault_*` | CLI/MCP handlers, wiki jobs and opt-in hooks | +| Dream and OpenClaw | `memorymaster.bridges.dream_bridge`, `db_merge`, `delta_sync`, `qmd_bridge` | CLI/integration handlers | +| Atlas, media and actions | `memorymaster.bridges.atlas_*`, `media_*`, `action_*`, `connectors` | CLI/MCP integration handlers | +| Local search | `memorymaster.bridges.local_search` | CLI/MCP tools | +| Specialized bridges | remaining `memorymaster.bridges.*` modules | explicit integration handlers | + +Dependency direction is one-way: companions may consume core contracts, while +core modules must not import companion modules. Optional behavior is installed +by importing its companion; for example, importing `wiki_engine` registers the +wiki lifecycle adapter, while importing `MemoryService` alone does not. + +The supported extension seams are narrow typed provider protocols +(`LocalSearchProvider`, `TranscriptionProvider`, `OcrProvider`) plus the +read-only `WikiSimilarityCorpus` stewardship protocol. The former generic +`memorymaster.plugins` callback registry had no production consumers and was +removed in R4.1 after its deprecation window. Arbitrary validator, retrieval, +or ingestion callbacks are not a supported security boundary. + ## Data Flow **Ingest path** @@ -211,7 +241,6 @@ this track's requested inventory. | `memorymaster/models.py` | Domain model dataclasses and validation helpers. | `fe4dbc2 feat(atlas): Atlas Inbox V1 contract (v1.0.0 → v1.5.1, 7 commits) (#27)` | | `memorymaster/operator.py` | Operator loop support for reviewing and applying queued memory work. | `886bec4 fix: green CI + v3.2.0 release prep + repo cleanup` | | `memorymaster/operator_queue.py` | WAL-backed durable queue for pending operator turns. | `761eabc refactor: extract migration helpers to reduce migrate_from_json complexity (11→8)` | -| `memorymaster/plugins.py` | Entry-point plugin registry for validators, probes, retrieval hooks, and exporters. | `8c4f302 feat: plugin system + cross-agent scope isolation with RBAC` | | `memorymaster/policy.py` | Policy-mode configuration and cadence override helpers. | `0dff74a feat(policy): MEMORYMASTER_POLICY_MODE env-var opt-in for cadence` | | `memorymaster/postgres_store.py` | Postgres storage backend with parity methods for the service layer. | `e337c07 chore(storage): audit SQLite/Postgres parity, add 3 missing pg methods (#35)` | | `memorymaster/qdrant_backend.py` | Qdrant maintenance-index backend; payload search fails closed while upsert/sync/reconcile count/ID operations remain available. | `7b049c5 chore: prepare for open-source release — scrub private data, add docs` | diff --git a/memorymaster/core/plugins.py b/memorymaster/core/plugins.py deleted file mode 100644 index 935ffa32..00000000 --- a/memorymaster/core/plugins.py +++ /dev/null @@ -1,127 +0,0 @@ -"""Plugin system for custom validators, probes, and retrieval hooks. - -Deprecated: P2 census found zero live consumers of the plugin registry or the -entry-point group (test-only surface). Kept per operator verdict (KEEP-DEPRECATED, -2026-06-10); wire-or-remove decision deferred to P5 review. - -Plugins register via entry points in pyproject.toml: - - [project.entry-points."memorymaster.plugins"] - my_validator = "my_package:MyValidator" - -Or programmatically: - - from memorymaster.core.plugins import register_plugin, PluginType - register_plugin("my_validator", PluginType.VALIDATOR, my_validate_fn) - -Plugin types: - - VALIDATOR: Called during run-cycle to validate claims - - PROBE: Called during steward to probe claim accuracy - - RETRIEVAL_HOOK: Called during query to add/rerank results - - INGESTION_HOOK: Called before/after claim ingestion -""" - -from __future__ import annotations - -import logging -from enum import Enum -from typing import Any, Callable - -logger = logging.getLogger(__name__) - - -class PluginType(str, Enum): - VALIDATOR = "validator" - PROBE = "probe" - RETRIEVAL_HOOK = "retrieval_hook" - INGESTION_HOOK = "ingestion_hook" - - -_registry: dict[str, dict[str, Any]] = {} - - -def register_plugin( - name: str, - plugin_type: PluginType, - handler: Callable, - *, - priority: int = 100, - description: str = "", -) -> None: - """Register a plugin handler.""" - _registry[name] = { - "type": plugin_type, - "handler": handler, - "priority": priority, - "description": description, - } - logger.info("Registered plugin: %s (type=%s, priority=%d)", name, plugin_type.value, priority) - - -def unregister_plugin(name: str) -> bool: - """Remove a registered plugin.""" - if name in _registry: - del _registry[name] - return True - return False - - -def get_plugins(plugin_type: PluginType | None = None) -> list[dict[str, Any]]: - """Get all registered plugins, optionally filtered by type.""" - plugins = [] - for name, info in sorted(_registry.items(), key=lambda x: x[1]["priority"]): - if plugin_type is None or info["type"] == plugin_type: - plugins.append({"name": name, **info}) - return plugins - - -def run_plugins( - plugin_type: PluginType, - *args: Any, - **kwargs: Any, -) -> list[dict[str, Any]]: - """Run all plugins of a given type and collect results.""" - results = [] - for plugin in get_plugins(plugin_type): - try: - result = plugin["handler"](*args, **kwargs) - results.append({"name": plugin["name"], "result": result, "ok": True}) - except Exception as exc: - logger.warning("Plugin '%s' failed: %s", plugin["name"], exc) - results.append({"name": plugin["name"], "error": str(exc), "ok": False}) - return results - - -def load_entry_point_plugins() -> int: - """Load plugins from setuptools entry points.""" - loaded = 0 - try: - from importlib.metadata import entry_points - eps = entry_points() - mm_plugins = eps.get("memorymaster.plugins", []) if isinstance(eps, dict) else eps.select(group="memorymaster.plugins") - for ep in mm_plugins: - try: - handler = ep.load() - plugin_type = getattr(handler, "plugin_type", PluginType.VALIDATOR) - priority = getattr(handler, "priority", 100) - description = getattr(handler, "description", ep.name) - register_plugin(ep.name, plugin_type, handler, priority=priority, description=description) - loaded += 1 - except Exception as exc: - logger.warning("Failed to load plugin '%s': %s", ep.name, exc) - except Exception: - pass # No entry points available - return loaded - - -def get_stats() -> dict[str, Any]: - """Return plugin system statistics.""" - by_type: dict[str, int] = {} - for info in _registry.values(): - t = info["type"].value - by_type[t] = by_type.get(t, 0) + 1 - return { - "total": len(_registry), - "by_type": by_type, - "plugins": [{"name": n, "type": i["type"].value, "priority": i["priority"]} for n, i in _registry.items()], - } diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index 3bbf3fa3..476bdaf8 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -8,7 +8,7 @@ import logging import os -from memorymaster.core import lifecycle, observability +from memorymaster.core import observability from memorymaster.core import llm_budget from memorymaster.govern import candidate_dedupe from memorymaster.recall import query_cache @@ -49,24 +49,6 @@ logger = logging.getLogger(__name__) - -def _wiki_autopromote_adapter(claim_id: int, db_path: str | None = None) -> None: - """Lazy adapter for lifecycle's wiki autopromote hook. - - P2 phase0 cycle cut: lifecycle (core) must never import wiki_engine - (knowledge), so service wiring registers this callback instead. The import - stays inside the function so wiki_engine is only loaded when the - autopromote threshold actually fires (same laziness as the old - lifecycle-internal import). - """ - from memorymaster.knowledge.wiki_engine import absorb_single_claim - - absorb_single_claim(claim_id, db_path=db_path) - - -if lifecycle.on_claim_confirmed is None: - lifecycle.on_claim_confirmed = _wiki_autopromote_adapter - RetrievalWeights = tuple[float, float, float, float] # Rule-mining steward phase (P3). DEFAULT OFF: run_cycle only mines verbatim diff --git a/memorymaster/govern/steward_features.py b/memorymaster/govern/steward_features.py index 1038c2f4..a073e91b 100644 --- a/memorymaster/govern/steward_features.py +++ b/memorymaster/govern/steward_features.py @@ -24,10 +24,13 @@ import re import sqlite3 from datetime import datetime, timezone -from typing import TYPE_CHECKING, Any +from typing import Any, Protocol -if TYPE_CHECKING: - from memorymaster.knowledge.wiki_similarity import WikiCorpus + +class WikiSimilarityCorpus(Protocol): + """Narrow optional-extension seam consumed by core stewardship.""" + + def similarity_for_claim(self, claim: dict[str, Any]) -> float: ... FEATURE_VERSION = "v3" @@ -271,7 +274,7 @@ def extract_features( claim: Any, conn: sqlite3.Connection, *, - wiki_corpus: WikiCorpus | None = None, + wiki_corpus: WikiSimilarityCorpus | None = None, ) -> dict[str, float]: """Return the v3 feature dict for a single claim. ``claim`` may be a dict, ``sqlite3.Row``, or dataclass with standard Claim fields (``id``, @@ -315,8 +318,7 @@ def extract_features( has_entity = 1.0 if c.get("entity_id") else 0.0 if wiki_corpus is not None: - from memorymaster.knowledge.wiki_similarity import compute_wiki_similarity - wiki_sim = compute_wiki_similarity(c, wiki_corpus) + wiki_sim = wiki_corpus.similarity_for_claim(c) else: wiki_sim = 0.0 diff --git a/memorymaster/knowledge/wiki_similarity.py b/memorymaster/knowledge/wiki_similarity.py index f872f4ec..4f6a3cdf 100644 --- a/memorymaster/knowledge/wiki_similarity.py +++ b/memorymaster/knowledge/wiki_similarity.py @@ -121,6 +121,10 @@ def iter_articles_for_scope(self, scope: str | None) -> list[WikiArticle]: return list(self.articles.values()) return [a for a in self.articles.values() if a.article_scope == scope] + def similarity_for_claim(self, claim: dict[str, Any]) -> float: + """Implement the narrow stewardship extension protocol.""" + return compute_wiki_similarity(claim, self) + # --------------------------------------------------------------------------- # Article loading diff --git a/memorymaster/plugins.py b/memorymaster/plugins.py deleted file mode 100644 index df7a316d..00000000 --- a/memorymaster/plugins.py +++ /dev/null @@ -1,11 +0,0 @@ -"""Deprecated compatibility shim — moved to ``memorymaster.core.plugins``. - -P2 restructure: this alias keeps the old import path (including submodule -attribute access) working for one minor version. Update imports to -``memorymaster.core.plugins``. -""" -import sys as _sys - -from memorymaster.core import plugins as _new - -_sys.modules[__name__] = _new diff --git a/tests/test_extension_boundaries.py b/tests/test_extension_boundaries.py new file mode 100644 index 00000000..d7c94645 --- /dev/null +++ b/tests/test_extension_boundaries.py @@ -0,0 +1,99 @@ +"""R4.1 acceptance pins for core and optional companion boundaries.""" + +from __future__ import annotations + +import ast +import subprocess +import sys +import textwrap +from pathlib import Path + + +REPO = Path(__file__).resolve().parents[1] +PACKAGE = REPO / "memorymaster" +CORE_TREES = ("core", "govern", "recall", "stores") +OPTIONAL_PREFIXES = ( + "memorymaster.bridges", + "memorymaster.knowledge.wiki", + "memorymaster.knowledge.vault", +) + + +def _imports(path: Path) -> set[str]: + tree = ast.parse(path.read_text(encoding="utf-8")) + names: set[str] = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + names.update(alias.name for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.module: + names.add(node.module) + return names + + +def test_core_layers_do_not_import_optional_companions() -> None: + offenders: list[str] = [] + for tree_name in CORE_TREES: + for path in (PACKAGE / tree_name).rglob("*.py"): + for imported in _imports(path): + if imported.startswith(OPTIONAL_PREFIXES): + offenders.append(f"{path.relative_to(REPO)} -> {imported}") + assert offenders == [] + + +def test_core_service_import_does_not_load_or_register_companions() -> None: + code = r''' + import importlib.abc + import sys + + blocked = ( + "memorymaster.bridges", + "memorymaster.knowledge.wiki", + "memorymaster.knowledge.vault", + ) + + class Blocker(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname.startswith(blocked): + raise ImportError(f"optional companion loaded by core: {fullname}") + return None + + sys.meta_path.insert(0, Blocker()) + import memorymaster.core.service # noqa: F401 + from memorymaster.core import lifecycle + + assert lifecycle.on_claim_confirmed is None + print("OK") + ''' + proc = subprocess.run( + [sys.executable, "-c", textwrap.dedent(code)], + cwd=REPO, + capture_output=True, + text=True, + timeout=120, + ) + assert proc.returncode == 0, proc.stderr + + +def test_explicit_wiki_import_registers_its_lifecycle_adapter() -> None: + code = r''' + from memorymaster.core import lifecycle + assert lifecycle.on_claim_confirmed is None + + import memorymaster.knowledge.wiki_engine # noqa: F401 + assert lifecycle.on_claim_confirmed is not None + print("OK") + ''' + proc = subprocess.run( + [sys.executable, "-c", textwrap.dedent(code)], + cwd=REPO, + capture_output=True, + text=True, + timeout=120, + ) + assert proc.returncode == 0, proc.stderr + + +def test_dead_generic_plugin_surface_is_removed() -> None: + assert not (PACKAGE / "core" / "plugins.py").exists() + assert not (PACKAGE / "plugins.py").exists() + assert "memorymaster.plugins" not in (REPO / "pyproject.toml").read_text(encoding="utf-8") diff --git a/tests/test_p2_cycle_cuts.py b/tests/test_p2_cycle_cuts.py index 6d59ce9f..2172069d 100644 --- a/tests/test_p2_cycle_cuts.py +++ b/tests/test_p2_cycle_cuts.py @@ -4,8 +4,8 @@ memorymaster.core.key_rotator (RoundRobinKeyRotator); llm_steward only re-exports it for backward compatibility. Cut 2: lifecycle must never import wiki_engine. The wiki autopromote trigger - is inverted into lifecycle.on_claim_confirmed, registered by wiring - modules (service.py, wiki_engine.py). + is inverted into lifecycle.on_claim_confirmed and registered only when + the optional wiki companion is explicitly imported. Cut 3: llm_steward must never import store_factory. Auto-validation accepts an injected store; the default resolves via jobs.deterministic.open_store. diff --git a/tests/test_plugins.py b/tests/test_plugins.py deleted file mode 100644 index ba2e1fd3..00000000 --- a/tests/test_plugins.py +++ /dev/null @@ -1,284 +0,0 @@ -"""Tests for plugin system.""" - -from __future__ import annotations - - -from memorymaster.core.plugins import ( - PluginType, - get_plugins, - register_plugin, - run_plugins, - unregister_plugin, -) - - -class TestPluginType: - """Test PluginType enum.""" - - def test_plugin_type_values(self): - """PluginType has correct values.""" - assert PluginType.VALIDATOR.value == "validator" - assert PluginType.PROBE.value == "probe" - assert PluginType.RETRIEVAL_HOOK.value == "retrieval_hook" - assert PluginType.INGESTION_HOOK.value == "ingestion_hook" - - -class TestRegisterPlugin: - """Test plugin registration.""" - - def teardown_method(self): - """Clean up registry after each test.""" - import memorymaster.core.plugins as p - p._registry.clear() - - def test_register_plugin_simple(self): - """Register a simple plugin.""" - def my_handler(): - return "result" - - register_plugin("my_plugin", PluginType.VALIDATOR, my_handler) - - plugins = get_plugins() - assert len(plugins) == 1 - assert plugins[0]["name"] == "my_plugin" - - def test_register_plugin_with_priority(self): - """Register plugin with custom priority.""" - def handler1(): - pass - def handler2(): - pass - - register_plugin("plugin1", PluginType.VALIDATOR, handler1, priority=100) - register_plugin("plugin2", PluginType.VALIDATOR, handler2, priority=50) - - plugins = get_plugins(PluginType.VALIDATOR) - # Should be sorted by priority (lower priority first, but stored as-is) - assert plugins[0]["priority"] == 50 - assert plugins[1]["priority"] == 100 - - def test_register_plugin_with_description(self): - """Register plugin with description.""" - def handler(): - pass - - register_plugin( - "documented", - PluginType.VALIDATOR, - handler, - description="A test plugin", - ) - - plugins = get_plugins() - assert plugins[0]["description"] == "A test plugin" - - def test_register_plugin_overwrites_existing(self): - """Registering same name overwrites existing.""" - def handler1(): - return "first" - def handler2(): - return "second" - - register_plugin("plugin", PluginType.VALIDATOR, handler1) - register_plugin("plugin", PluginType.VALIDATOR, handler2) - - plugins = get_plugins() - assert len(plugins) == 1 - assert plugins[0]["handler"]() == "second" - - -class TestUnregisterPlugin: - """Test plugin unregistration.""" - - def teardown_method(self): - """Clean up registry after each test.""" - import memorymaster.core.plugins as p - p._registry.clear() - - def test_unregister_existing_plugin(self): - """Unregister an existing plugin.""" - register_plugin("plugin", PluginType.VALIDATOR, lambda: None) - - result = unregister_plugin("plugin") - assert result is True - assert len(get_plugins()) == 0 - - def test_unregister_nonexistent_plugin(self): - """Unregister non-existent plugin returns False.""" - result = unregister_plugin("nonexistent") - assert result is False - - def test_unregister_leaves_others(self): - """Unregistering one plugin leaves others.""" - register_plugin("plugin1", PluginType.VALIDATOR, lambda: None) - register_plugin("plugin2", PluginType.VALIDATOR, lambda: None) - - unregister_plugin("plugin1") - - plugins = get_plugins() - assert len(plugins) == 1 - assert plugins[0]["name"] == "plugin2" - - -class TestGetPlugins: - """Test plugin retrieval.""" - - def teardown_method(self): - """Clean up registry after each test.""" - import memorymaster.core.plugins as p - p._registry.clear() - - def test_get_plugins_empty(self): - """Get plugins from empty registry.""" - result = get_plugins() - assert result == [] - - def test_get_plugins_all(self): - """Get all plugins without filter.""" - register_plugin("validator_plugin", PluginType.VALIDATOR, lambda: None) - register_plugin("probe_plugin", PluginType.PROBE, lambda: None) - - plugins = get_plugins() - assert len(plugins) == 2 - - def test_get_plugins_by_type(self): - """Get plugins filtered by type.""" - register_plugin("validator1", PluginType.VALIDATOR, lambda: None) - register_plugin("validator2", PluginType.VALIDATOR, lambda: None) - register_plugin("probe1", PluginType.PROBE, lambda: None) - - validators = get_plugins(PluginType.VALIDATOR) - assert len(validators) == 2 - assert all(p["type"] == PluginType.VALIDATOR for p in validators) - - def test_get_plugins_sorted_by_priority(self): - """Get plugins sorted by priority.""" - register_plugin("p1", PluginType.VALIDATOR, lambda: None, priority=300) - register_plugin("p2", PluginType.VALIDATOR, lambda: None, priority=100) - register_plugin("p3", PluginType.VALIDATOR, lambda: None, priority=200) - - plugins = get_plugins(PluginType.VALIDATOR) - priorities = [p["priority"] for p in plugins] - assert priorities == [100, 200, 300] - - -class TestRunPlugins: - """Test plugin execution.""" - - def teardown_method(self): - """Clean up registry after each test.""" - import memorymaster.core.plugins as p - p._registry.clear() - - def test_run_plugins_no_plugins(self): - """Run plugins when none registered.""" - result = run_plugins(PluginType.VALIDATOR) - assert result == [] - - def test_run_plugins_single_plugin(self): - """Run a single plugin.""" - def my_validator(claim): - return {"valid": True} - - register_plugin("validator", PluginType.VALIDATOR, my_validator) - - results = run_plugins(PluginType.VALIDATOR, claim="test") - assert len(results) == 1 - assert results[0]["name"] == "validator" - assert results[0]["ok"] is True - assert results[0]["result"]["valid"] is True - - def test_run_plugins_multiple(self): - """Run multiple plugins.""" - register_plugin("v1", PluginType.VALIDATOR, lambda c: {"id": 1}) - register_plugin("v2", PluginType.VALIDATOR, lambda c: {"id": 2}) - - results = run_plugins(PluginType.VALIDATOR, claim="test") - assert len(results) == 2 - - def test_run_plugins_passes_arguments(self): - """Plugin receives passed arguments.""" - def handler(claim, score): - return {"claim": claim, "score": score} - - register_plugin("handler", PluginType.VALIDATOR, handler) - - results = run_plugins(PluginType.VALIDATOR, claim=42, score=0.8) - assert results[0]["result"]["claim"] == 42 - assert results[0]["result"]["score"] == 0.8 - - def test_run_plugins_exception_handling(self): - """Plugin exception is caught and reported.""" - def failing_handler(): - raise ValueError("Plugin error") - - register_plugin("failing", PluginType.VALIDATOR, failing_handler) - - results = run_plugins(PluginType.VALIDATOR) - assert len(results) == 1 - assert results[0]["ok"] is False - assert "error" in results[0] - assert "Plugin error" in results[0]["error"] - - def test_run_plugins_mixed_success_failure(self): - """Some plugins succeed, some fail.""" - register_plugin("good", PluginType.VALIDATOR, lambda: {"ok": True}) - register_plugin("bad", PluginType.VALIDATOR, lambda: 1 / 0) - - results = run_plugins(PluginType.VALIDATOR) - assert len(results) == 2 - - good = [r for r in results if r["ok"]] - bad = [r for r in results if not r["ok"]] - assert len(good) == 1 - assert len(bad) == 1 - - def test_run_plugins_by_type_only(self): - """Only plugins of specified type are run.""" - register_plugin("v1", PluginType.VALIDATOR, lambda: "validator") - register_plugin("p1", PluginType.PROBE, lambda: "probe") - - results = run_plugins(PluginType.VALIDATOR) - assert len(results) == 1 - assert results[0]["result"] == "validator" - - -class TestPluginIntegration: - """Integration tests.""" - - def teardown_method(self): - """Clean up registry after each test.""" - import memorymaster.core.plugins as p - p._registry.clear() - - def test_register_and_run_workflow(self): - """Full workflow: register, get, run.""" - def validator1(claim): - return {"score": 0.8} - - def validator2(claim): - return {"score": 0.9} - - register_plugin("v1", PluginType.VALIDATOR, validator1, priority=100) - register_plugin("v2", PluginType.VALIDATOR, validator2, priority=50) - - # Get validators sorted by priority - plugins = get_plugins(PluginType.VALIDATOR) - assert len(plugins) == 2 - assert plugins[0]["name"] == "v2" # priority 50 comes first - - # Run validators - results = run_plugins(PluginType.VALIDATOR, claim="test") - assert len(results) == 2 - assert all(r["ok"] for r in results) - - def test_multiple_plugin_types(self): - """Register and run different plugin types.""" - register_plugin("val", PluginType.VALIDATOR, lambda: "validator") - register_plugin("prb", PluginType.PROBE, lambda: "probe") - register_plugin("ret", PluginType.RETRIEVAL_HOOK, lambda: "retrieval") - - assert len(get_plugins(PluginType.VALIDATOR)) == 1 - assert len(get_plugins(PluginType.PROBE)) == 1 - assert len(get_plugins(PluginType.RETRIEVAL_HOOK)) == 1 - assert len(get_plugins()) == 3 From d78de9c9fc6c23855f9eac2f86047e3d288af8a2 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Tue, 14 Jul 2026 01:15:25 -0300 Subject: [PATCH 44/55] refactor: ratchet orchestration boundaries --- .planning/REMAINING-PHASES-V1-BUDGET.md | 31 ++ ...EMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md | 2 +- .planning/audit-remediation-ledger.md | 2 +- docs/architecture.md | 17 ++ docs/compatibility.md | 34 +++ memorymaster/core/service.py | 244 +--------------- memorymaster/core/services/__init__.py | 5 + memorymaster/core/services/integration.py | 267 ++++++++++++++++++ memorymaster/surfaces/dashboard.py | 248 ++++------------ memorymaster/surfaces/dashboard_commands.py | 67 +++++ .../surfaces/dashboard_read_models.py | 219 ++++++++++++++ tests/test_architecture_budgets.py | 68 +++++ tests/test_dashboard_coverage.py | 10 +- 13 files changed, 774 insertions(+), 440 deletions(-) create mode 100644 docs/compatibility.md create mode 100644 memorymaster/core/services/__init__.py create mode 100644 memorymaster/core/services/integration.py create mode 100644 memorymaster/surfaces/dashboard_commands.py create mode 100644 memorymaster/surfaces/dashboard_read_models.py create mode 100644 tests/test_architecture_budgets.py diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index 1463e5b8..d50d0e01 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -514,3 +514,34 @@ R4.1 core and companion extension boundaries completed on 2026-07-14: The next package is R4.2 gradual orchestration decomposition. The active roadmap goal authorizes continuing without a new goal. + +R4.2 gradual orchestration decomposition completed on 2026-07-14: + +- The public `MemoryService` facade now inherits the real store-backed + `IntegrationService` implementation for external sources, Atlas/media + evidence, retry leases, and action proposals. Exact public signatures are + preserved; ingest, retrieval, cycle, and initialization bodies were not + changed. +- The manually HIGH facade boundary was limited to the new base class. The + `get_action_proposal_by_idempotency_key` symbol was left in the facade after + GitNexus reported a self-referential CRITICAL graph result; its behavior was + not changed. +- Dashboard claim/event/conflict/review/mobile/action/audit/namespace read + models now live outside the HTTP handler. Action-status, triage, and operator + mutations now use explicit command functions; handlers parse, invoke, and + serialize. +- Ratcheting tests cap `service.py` at 2,450 lines, `dashboard.py` at 1,550, + `DashboardRequestHandler` at 720, extracted modules at 800, and extracted + functions at 50. Actual results are 2,205, 1,381, and 691 respectively. +- `docs/compatibility.md` freezes the root-shim inventory, retains it through + 4.5.x, and requires the dated 2026-09-30/v5 major removal gate plus consumer + evidence. Telemetry/lifecycle, stewardship/ingestion, and retrieval remain + prioritized gradual ratchets; no placeholder services were added. +- RED architecture evidence initially failed three of four checks. The focused + package boundary produced 137 passes and one stale monkeypatch-path failure; + the exact correction plus architecture checks passed five tests. Changed-file + Ruff, syntax compilation, and `git diff --check` passed. +- Atomic package commit: the conventional R4.2 commit containing this evidence. + +The next package is R4.3 governance UX and accessibility. The active roadmap +goal authorizes continuing without a new goal. diff --git a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md index 0872da27..7abfcab1 100644 --- a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md +++ b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md @@ -100,7 +100,7 @@ Phase 0 must replace qualitative capacity goals with recorded numeric gates in t - [ ] R3.4 — service entrypoints and deployment health - [ ] R3.5 — recovery, observability, audit, and privacy operations - [x] R4.1 — core/extension boundaries -- [ ] R4.2 — decompose oversized orchestration points +- [x] R4.2 — decompose oversized orchestration points - [ ] R4.3 — governance UX and accessibility - [ ] R4.4 — generated release/documentation truth - [ ] Final audit delta and convergence evidence diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 01140f06..299f2b25 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -38,7 +38,7 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-COST-03 | Cost/Reliability | Medium / BAD-PRACTICE | Cost and intake controls are fragmented/process-local | R2.4,R3.2 | RESOLVED | R3.2 adds one WAL-backed atomic usage ledger shared by LLM primary/fallback calls, Gemini embedding batches, MCP ingest/checkpoint, and configured core intake quotas. Caps remain backward-compatible/unlimited unless configured; restart/cross-instance and local-bucket-reset tests prove durable global/provider/actor enforcement without consuming capacity on rejected reservations. | | MM-DEMO-01 | Demo/Integrity | High / EXPLOITABLE-NOW | Atlas defaults persist fabricated mock evidence at high confidence | R2.5 | RESOLVED | P2-F (`c66e240`) requires a real configured production provider, confines synthetic providers to explicit dev/test opt-in, excludes synthetic evidence from claims/prompts/actions/citations/exports, and fails configuration nonzero. Focused gate: 111 passed. | | MM-MAINT-01 | Maintainability/Integrity | Medium / BAD-PRACTICE | Versions, tool counts, roadmaps, and install claims drift | R4.4 | OPEN | Generated single-source values checked by CI | -| MM-MAINT-02 | Architecture/Maintainability | Medium / BAD-PRACTICE | Oversized facades, hardwired extensions, dead plugin seam, and lingering shims | R4.1-R4.2 | PARTIAL | R4.1 removes the zero-consumer generic plugin registry, stops core service imports from installing Wiki behavior, routes Wiki similarity through a narrow protocol, and enforces core-to-companion import boundaries. R4.2 owns facade decomposition, size budgets, and the dated shim policy. | +| MM-MAINT-02 | Architecture/Maintainability | Medium / BAD-PRACTICE | Oversized facades, hardwired extensions, dead plugin seam, and lingering shims | R4.1-R4.2 | RESOLVED | R4.1 removes the zero-consumer generic plugin registry and enforces core-to-companion imports. R4.2 extracts the real integration service plus dashboard read/command layers, reduces `service.py` from 2,617 to 2,205 lines and the handler from 848 to 691, enforces ratcheting budgets, and publishes a dated v5 shim-removal gate. Remaining collaborator extraction is an explicit gradual ratchet, not an unbounded facade regression. | | MM-REL-03 | Reliability | Medium / BAD-PRACTICE | Media retry rows have no expired-lease recovery | R3.5 | RESOLVED | Immutable migration 0016 and SQLite/Postgres parity add bounded owner/expiry leases, atomic expired-work recovery with an audit event, lease clearing on terminal outcomes, and concurrent single-winner coverage. The focused R3.5 gate includes worker-death reclaim, unexpired non-steal, and concurrent claimer tests. | | MM-OPS-05 | Missing Operations | High / BAD-PRACTICE | Recovery defaults stop at same-machine SQLite snapshots | R3.5 | BLOCKED-EXTERNAL | R3.5/Phase 3 convergence adds authenticated encrypted SQLite backups, plaintext/ciphertext checksums, online backup, disposable restore, integrity/FK checks, backup-age health, and measured RPO/RTO (0.046s restore versus 1,800s objective). Approved off-device storage and disposable PostgreSQL dump/restore evidence remain external; same-machine success is not represented as full recovery readiness. | | MM-OBS-01 | Missing Operations | High / BAD-PRACTICE | Failures/metrics are process-local without central alert ownership | R3.5 | RESOLVED | R3.5 adds deterministic backup/retry/integrity/WAL/disk/provider/telemetry checks and can persist one aggregate audit envelope with explicit owner and runbook. Focused tests cover alert severity, persistence, and audit fields. Delivery to an organization-selected OTel/error backend remains environment-specific readiness evidence, not a repository correctness claim. | diff --git a/docs/architecture.md b/docs/architecture.md index 71fc793b..63fc453b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -67,6 +67,23 @@ read-only `WikiSimilarityCorpus` stewardship protocol. The former generic removed in R4.1 after its deprecation window. Arbitrary validator, retrieval, or ingestion callbacks are not a supported security boundary. +### Gradual orchestration decomposition + +`MemoryService` remains the supported facade. Its Atlas/media/action persistence +API is implemented by `core.services.IntegrationService`, preserving instance +method signatures through inheritance while removing integration ownership from +the 2,617-line orchestration class. The first enforced ratchet caps +`core/service.py` at 2,450 lines; the R4.2 result is 2,205 lines. Telemetry and +lifecycle are the next extraction pair, followed by stewardship and ingestion; +policy-dense retrieval is last. + +Dashboard HTTP handlers own transport only. Read-model construction lives in +`surfaces/dashboard_read_models.py`, while mutation application lives in +`surfaces/dashboard_commands.py`. Non-growth tests cap `dashboard.py` at 1,550 +lines and `DashboardRequestHandler` at 720 lines; R4.2 results are 1,381 and 691. +Compatibility-shim ownership and the dated v5 removal gate are documented in +`docs/compatibility.md`. + ## Data Flow **Ingest path** diff --git a/docs/compatibility.md b/docs/compatibility.md new file mode 100644 index 00000000..b91e01cb --- /dev/null +++ b/docs/compatibility.md @@ -0,0 +1,34 @@ +# Compatibility and shim retirement policy + +Inventory frozen: 2026-07-14. + +`MemoryService` remains the public compatibility facade while bounded services +are extracted behind it. R4.2 starts with the Atlas/media/action integration +service and dashboard application models. The next ratchets are telemetry and +lifecycle, then stewardship and ingestion, with policy-dense retrieval last. +No placeholder collaborator counts as an extraction. + +## Root import shims + +The repository currently retains the root-level import family introduced by +the package-layer move, including `memorymaster.service`. Those shims forward +to `memorymaster.core`, `stores`, `recall`, `govern`, `knowledge`, `bridges`, or +`surfaces`; new code must import the canonical path. + +Existing shims remain available through the 4.5.x line. Their earliest removal +date is 2026-09-30 and removal is permitted only in v5.0 or later. + +## Removal gate + +A shim may be deleted only when all of the following are true: + +1. Repository imports and clean-wheel tests use the canonical path. +2. Release notes name the old path, replacement path, and removal release. +3. Available import/deprecation telemetry shows no supported consumer, or the + owner records an explicit compatibility decision. +4. A major-version test proves the old import fails intentionally and the new + import preserves the supported behavior. + +No new root shim may be added without an owner, canonical replacement, +introduction release, planned removal release, and test. Version-source drift +is intentionally owned by R4.4; this policy does not choose a version value. diff --git a/memorymaster/core/service.py b/memorymaster/core/service.py index 476bdaf8..d48cb691 100644 --- a/memorymaster/core/service.py +++ b/memorymaster/core/service.py @@ -14,7 +14,7 @@ from memorymaster.recall import query_cache from memorymaster.recall.embeddings import EmbeddingProvider, create_best_provider from memorymaster.govern.jobs import compact_summaries, compactor, decay, dedup, deterministic, extractor, integrity, qdrant_reconcile, spool_drain, validator -from memorymaster.core.models import ActionProposal, CitationInput, Claim, ClaimLink, Event, EvidenceItem, ExternalSource, MediaRetryItem, SourceItem, validate_temporal_fields +from memorymaster.core.models import ActionProposal, CitationInput, Claim, ClaimLink, Event, validate_temporal_fields from memorymaster.core.policy import select_revalidation_candidates from memorymaster.recall.context_optimizer import ContextResult, pack_context from memorymaster.core.config import get_config @@ -38,6 +38,7 @@ IntakeRejected, evaluate_intake, ) +from memorymaster.core.services.integration import IntegrationService from memorymaster.stores.claim_identity import ( normalize_claim_identity, normalize_claim_visibility, @@ -353,7 +354,7 @@ def _recall_result_entry(row: dict[str, Any]) -> dict[str, Any]: } -class MemoryService: +class MemoryService(IntegrationService): def __init__( self, db_target: str | Path, @@ -2196,248 +2197,9 @@ def list_events_page( cursor=cursor, ) - def upsert_external_source( - self, - *, - source_type: str, - display_name: str, - config_json: dict[str, object] | str | None = None, - ) -> ExternalSource: - return self.store.upsert_external_source( - source_type=source_type, - display_name=display_name, - config_json=config_json, - ) - - def upsert_source_item( - self, - *, - source_id: int, - source_item_id: str, - item_type: str, - chat_id: str | None = None, - sender_id: str | None = None, - sender_name: str | None = None, - occurred_at: str | None = None, - text: str | None = None, - payload_json: dict[str, object] | str | None = None, - content_hash: str | None = None, - sensitivity: str | None = None, - ) -> SourceItem: - return self.store.upsert_source_item( - source_id=source_id, - source_item_id=source_item_id, - item_type=item_type, - chat_id=chat_id, - sender_id=sender_id, - sender_name=sender_name, - occurred_at=occurred_at, - text=text, - payload_json=payload_json, - content_hash=content_hash, - sensitivity=sensitivity, - ) - - def get_source_item(self, *, source_id: int, source_item_id: str) -> SourceItem | None: - return self.store.get_source_item(source_id=source_id, source_item_id=source_item_id) - - def get_source_item_by_id(self, source_item_row_id: int) -> SourceItem | None: - return self.store.get_source_item_by_id(source_item_row_id) - - def add_evidence_item( - self, - *, - source_item_id: int, - evidence_type: str, - text: str | None = None, - media_path: str | None = None, - provider: str | None = None, - confidence: float | None = None, - payload_json: dict[str, object] | str | None = None, - sensitivity: str | None = None, - ) -> EvidenceItem: - return self.store.add_evidence_item( - source_item_id=source_item_id, - evidence_type=evidence_type, - text=text, - media_path=media_path, - provider=provider, - confidence=confidence, - payload_json=payload_json, - sensitivity=sensitivity, - ) - - def set_source_item_sensitivity(self, source_item_row_id: int, sensitivity: str | None) -> SourceItem: - return self.store.set_source_item_sensitivity(source_item_row_id, sensitivity) - - def set_evidence_item_sensitivity(self, evidence_item_row_id: int, sensitivity: str | None) -> EvidenceItem: - return self.store.set_evidence_item_sensitivity(evidence_item_row_id, sensitivity) - - def enqueue_media_retry( - self, - *, - source_item_id: int, - media_key: str, - chat_id: str | None = None, - media_type: str | None = None, - media_path: str | None = None, - media_url: str | None = None, - status: str = "pending", - next_attempt_time: str | None = None, - ) -> MediaRetryItem: - return self.store.enqueue_media_retry( - source_item_id=source_item_id, - media_key=media_key, - chat_id=chat_id, - media_type=media_type, - media_path=media_path, - media_url=media_url, - status=status, - next_attempt_time=next_attempt_time, - ) - - def claim_pending_media_retries( - self, - limit: int = 25, - *, - lease_owner: str = "media-worker", - lease_seconds: int = 300, - ) -> list[MediaRetryItem]: - return self.store.claim_pending_media_retries( - limit=limit, - lease_owner=lease_owner, - lease_seconds=lease_seconds, - ) - - def record_media_retry_outcome( - self, - retry_id: int, - *, - status: str, - media_path: str | None = None, - last_http_status: int | None = None, - last_error: str | None = None, - next_attempt_time: str | None = None, - ) -> MediaRetryItem: - return self.store.record_media_retry_outcome( - retry_id, - status=status, - media_path=media_path, - last_http_status=last_http_status, - last_error=last_error, - next_attempt_time=next_attempt_time, - ) - - def list_media_retries( - self, - *, - status: str | None = None, - source_item_id: int | None = None, - limit: int = 100, - ) -> list[MediaRetryItem]: - return self.store.list_media_retries( - status=status, - source_item_id=source_item_id, - limit=limit, - ) - - def media_retry_status_counts(self) -> dict[str, int]: - return self.store.media_retry_status_counts() - - def list_evidence_items( - self, - *, - source_item_id: int | None = None, - evidence_type: str | None = None, - limit: int = 100, - ) -> list[EvidenceItem]: - return self.store.list_evidence_items( - source_item_id=source_item_id, - evidence_type=evidence_type, - limit=limit, - ) - - def create_action_proposal( - self, - *, - proposal_type: str, - title: str, - description: str | None = None, - source_item_id: int | None = None, - evidence_item_id: int | None = None, - claim_id: int | None = None, - suggested_due_at: str | None = None, - destination: str = "manual", - confidence: float = 0.5, - payload_json: dict[str, object] | str | None = None, - idempotency_key: str | None = None, - ) -> ActionProposal: - return self.store.create_action_proposal( - proposal_type=proposal_type, - title=title, - description=description, - source_item_id=source_item_id, - evidence_item_id=evidence_item_id, - claim_id=claim_id, - suggested_due_at=suggested_due_at, - destination=destination, - confidence=confidence, - payload_json=payload_json, - idempotency_key=idempotency_key, - ) - def get_action_proposal_by_idempotency_key(self, idempotency_key: str) -> ActionProposal | None: return self.store.get_action_proposal_by_idempotency_key(idempotency_key) - def update_action_proposal_status( - self, - proposal_id: int, - *, - status: str, - external_ref: str | None = None, - exported_at: str | None = None, - payload_json: dict[str, object] | str | None = None, - ) -> ActionProposal: - return self.store.update_action_proposal_status( - proposal_id, - status=status, - external_ref=external_ref, - exported_at=exported_at, - payload_json=payload_json, - ) - - def list_action_proposals( - self, - *, - status: str | None = None, - destination: str | None = None, - limit: int = 100, - ) -> list[ActionProposal]: - return self.store.list_action_proposals( - status=status, - destination=destination, - limit=limit, - ) - - def update_action_proposal_fields( - self, - proposal_id: int, - *, - title: str | None = None, - description: str | None = None, - suggested_due_at: str | None = None, - confidence: float | None = None, - payload_json: dict[str, object] | str | None = None, - ) -> ActionProposal: - return self.store.update_action_proposal_fields( - proposal_id, - title=title, - description=description, - suggested_due_at=suggested_due_at, - confidence=confidence, - payload_json=payload_json, - ) - def add_claim_link(self, source_id: int, target_id: int, link_type: str) -> ClaimLink: for cid in (source_id, target_id): claim = self.store.get_claim(cid, include_citations=False) diff --git a/memorymaster/core/services/__init__.py b/memorymaster/core/services/__init__.py new file mode 100644 index 00000000..a3a6ebde --- /dev/null +++ b/memorymaster/core/services/__init__.py @@ -0,0 +1,5 @@ +"""Bounded collaborators inherited by the public MemoryService facade.""" + +from memorymaster.core.services.integration import IntegrationService + +__all__ = ["IntegrationService"] diff --git a/memorymaster/core/services/integration.py b/memorymaster/core/services/integration.py new file mode 100644 index 00000000..873495db --- /dev/null +++ b/memorymaster/core/services/integration.py @@ -0,0 +1,267 @@ +"""Atlas/media/action persistence collaborator for ``MemoryService``.""" + +from __future__ import annotations + +from typing import Any + +from memorymaster.core.models import ( + ActionProposal, + EvidenceItem, + ExternalSource, + MediaRetryItem, + SourceItem, +) + + +class IntegrationService: + """Store-backed integration API inherited by the compatibility facade. + + Subclasses provide ``store``. Keeping the implementations here preserves + every public method signature while removing integration ownership from the + authoritative orchestration module. + """ + + store: Any + + def upsert_external_source( + self, + *, + source_type: str, + display_name: str, + config_json: dict[str, object] | str | None = None, + ) -> ExternalSource: + return self.store.upsert_external_source( + source_type=source_type, + display_name=display_name, + config_json=config_json, + ) + + def upsert_source_item( + self, + *, + source_id: int, + source_item_id: str, + item_type: str, + chat_id: str | None = None, + sender_id: str | None = None, + sender_name: str | None = None, + occurred_at: str | None = None, + text: str | None = None, + payload_json: dict[str, object] | str | None = None, + content_hash: str | None = None, + sensitivity: str | None = None, + ) -> SourceItem: + return self.store.upsert_source_item( + source_id=source_id, + source_item_id=source_item_id, + item_type=item_type, + chat_id=chat_id, + sender_id=sender_id, + sender_name=sender_name, + occurred_at=occurred_at, + text=text, + payload_json=payload_json, + content_hash=content_hash, + sensitivity=sensitivity, + ) + + def get_source_item(self, *, source_id: int, source_item_id: str) -> SourceItem | None: + return self.store.get_source_item(source_id=source_id, source_item_id=source_item_id) + + def get_source_item_by_id(self, source_item_row_id: int) -> SourceItem | None: + return self.store.get_source_item_by_id(source_item_row_id) + + def add_evidence_item( + self, + *, + source_item_id: int, + evidence_type: str, + text: str | None = None, + media_path: str | None = None, + provider: str | None = None, + confidence: float | None = None, + payload_json: dict[str, object] | str | None = None, + sensitivity: str | None = None, + ) -> EvidenceItem: + return self.store.add_evidence_item( + source_item_id=source_item_id, + evidence_type=evidence_type, + text=text, + media_path=media_path, + provider=provider, + confidence=confidence, + payload_json=payload_json, + sensitivity=sensitivity, + ) + + def set_source_item_sensitivity( + self, source_item_row_id: int, sensitivity: str | None + ) -> SourceItem: + return self.store.set_source_item_sensitivity(source_item_row_id, sensitivity) + + def set_evidence_item_sensitivity( + self, evidence_item_row_id: int, sensitivity: str | None + ) -> EvidenceItem: + return self.store.set_evidence_item_sensitivity(evidence_item_row_id, sensitivity) + + def enqueue_media_retry( + self, + *, + source_item_id: int, + media_key: str, + chat_id: str | None = None, + media_type: str | None = None, + media_path: str | None = None, + media_url: str | None = None, + status: str = "pending", + next_attempt_time: str | None = None, + ) -> MediaRetryItem: + return self.store.enqueue_media_retry( + source_item_id=source_item_id, + media_key=media_key, + chat_id=chat_id, + media_type=media_type, + media_path=media_path, + media_url=media_url, + status=status, + next_attempt_time=next_attempt_time, + ) + + def claim_pending_media_retries( + self, + limit: int = 25, + *, + lease_owner: str = "media-worker", + lease_seconds: int = 300, + ) -> list[MediaRetryItem]: + return self.store.claim_pending_media_retries( + limit=limit, + lease_owner=lease_owner, + lease_seconds=lease_seconds, + ) + + def record_media_retry_outcome( + self, + retry_id: int, + *, + status: str, + media_path: str | None = None, + last_http_status: int | None = None, + last_error: str | None = None, + next_attempt_time: str | None = None, + ) -> MediaRetryItem: + return self.store.record_media_retry_outcome( + retry_id, + status=status, + media_path=media_path, + last_http_status=last_http_status, + last_error=last_error, + next_attempt_time=next_attempt_time, + ) + + def list_media_retries( + self, + *, + status: str | None = None, + source_item_id: int | None = None, + limit: int = 100, + ) -> list[MediaRetryItem]: + return self.store.list_media_retries( + status=status, + source_item_id=source_item_id, + limit=limit, + ) + + def media_retry_status_counts(self) -> dict[str, int]: + return self.store.media_retry_status_counts() + + def list_evidence_items( + self, + *, + source_item_id: int | None = None, + evidence_type: str | None = None, + limit: int = 100, + ) -> list[EvidenceItem]: + return self.store.list_evidence_items( + source_item_id=source_item_id, + evidence_type=evidence_type, + limit=limit, + ) + + def create_action_proposal( + self, + *, + proposal_type: str, + title: str, + description: str | None = None, + source_item_id: int | None = None, + evidence_item_id: int | None = None, + claim_id: int | None = None, + suggested_due_at: str | None = None, + destination: str = "manual", + confidence: float = 0.5, + payload_json: dict[str, object] | str | None = None, + idempotency_key: str | None = None, + ) -> ActionProposal: + return self.store.create_action_proposal( + proposal_type=proposal_type, + title=title, + description=description, + source_item_id=source_item_id, + evidence_item_id=evidence_item_id, + claim_id=claim_id, + suggested_due_at=suggested_due_at, + destination=destination, + confidence=confidence, + payload_json=payload_json, + idempotency_key=idempotency_key, + ) + + def update_action_proposal_status( + self, + proposal_id: int, + *, + status: str, + external_ref: str | None = None, + exported_at: str | None = None, + payload_json: dict[str, object] | str | None = None, + ) -> ActionProposal: + return self.store.update_action_proposal_status( + proposal_id, + status=status, + external_ref=external_ref, + exported_at=exported_at, + payload_json=payload_json, + ) + + def list_action_proposals( + self, + *, + status: str | None = None, + destination: str | None = None, + limit: int = 100, + ) -> list[ActionProposal]: + return self.store.list_action_proposals( + status=status, + destination=destination, + limit=limit, + ) + + def update_action_proposal_fields( + self, + proposal_id: int, + *, + title: str | None = None, + description: str | None = None, + suggested_due_at: str | None = None, + confidence: float | None = None, + payload_json: dict[str, object] | str | None = None, + ) -> ActionProposal: + return self.store.update_action_proposal_fields( + proposal_id, + title=title, + description=description, + suggested_due_at=suggested_due_at, + confidence=confidence, + payload_json=payload_json, + ) diff --git a/memorymaster/surfaces/dashboard.py b/memorymaster/surfaces/dashboard.py index 7fbffa71..768f2010 100644 --- a/memorymaster/surfaces/dashboard.py +++ b/memorymaster/surfaces/dashboard.py @@ -21,10 +21,25 @@ from memorymaster.surfaces import dashboard_auth from memorymaster.core.config import get_config -from memorymaster.govern.review import build_review_queue, queue_to_dicts -from memorymaster.core.security import is_sensitive_claim +from memorymaster.govern.review import build_review_queue from memorymaster.core.service import MemoryService from memorymaster.recall.qdrant_transport import QdrantTransportConfig +from memorymaster.surfaces.dashboard_commands import ( + apply_triage_action, + control_operator, + update_action_proposal_status, +) +from memorymaster.surfaces.dashboard_read_models import ( + action_proposals_payload, + audit_payload, + claims_payload, + conflicts_payload, + events_payload, + mobile_review_queue_payload, + namespaces_payload, + review_queue_payload, + triage_flags, +) import contextlib @@ -1091,19 +1106,7 @@ def _write_dashboard(self) -> None: self.wfile.write(body) def _triage_flags(self, limit: int) -> dict[int, dict[str, bool]]: - flags: dict[int, dict[str, bool]] = {} - for event in reversed(self._server.service.list_events(limit=limit, event_type="audit")): - if event.claim_id is None: - continue - ref = flags.setdefault(int(event.claim_id), {"reviewed": False, "suppressed": False}) - details = str(event.details or "") - if details == "triage_mark_reviewed": - ref["reviewed"] = True - if details == "triage_suppress": - ref["suppressed"] = True - if details == "triage_unsuppress": - ref["suppressed"] = False - return flags + return triage_flags(self._server.service, limit) def _handle_claims(self, query_string: str) -> None: query = parse_qs(query_string) @@ -1111,10 +1114,11 @@ def _handle_claims(self, query_string: str) -> None: include_archived = _parse_bool(_first_query_value(query, "include_archived"), default=False) allow_sensitive = _parse_bool(_first_query_value(query, "allow_sensitive"), default=False) status = _first_query_value(query, "status") - claims = self._server.service.list_claims(status=status, limit=limit, include_archived=include_archived) - if not allow_sensitive: - claims = [claim for claim in claims if not is_sensitive_claim(claim)] - self._write_json({"ok": True, "rows": len(claims), "claims": [_claim_to_dict(c) for c in claims]}) + self._write_json(claims_payload( + self._server.service, status=status, limit=limit, + include_archived=include_archived, allow_sensitive=allow_sensitive, + serialize=_claim_to_dict, + )) def _handle_events(self, query_string: str) -> None: query = parse_qs(query_string) @@ -1124,39 +1128,28 @@ def _handle_events(self, query_string: str) -> None: claim_id = int(claim_id_raw) if claim_id_raw is not None else None if claim_id is not None and claim_id <= 0: raise ValueError("claim_id must be positive") - events = self._server.service.list_events(claim_id=claim_id, limit=limit, event_type=event_type) - self._write_json({"ok": True, "rows": len(events), "events": [_event_to_dict(e) for e in events]}) + self._write_json(events_payload( + self._server.service, limit=limit, claim_id=claim_id, + event_type=event_type, serialize=_event_to_dict, + )) def _handle_timeline(self, query_string: str) -> None: query = parse_qs(query_string) limit = _parse_int(_first_query_value(query, "limit"), default=100, minimum=1, maximum=2000) event_type = _first_query_value(query, "event_type") - events = self._server.service.list_events(limit=limit, event_type=event_type) - self._write_json({"ok": True, "rows": len(events), "timeline": [_event_to_dict(e) for e in events]}) + self._write_json(events_payload( + self._server.service, limit=limit, claim_id=None, + event_type=event_type, serialize=_event_to_dict, output_key="timeline", + )) def _handle_conflicts(self, query_string: str) -> None: query = parse_qs(query_string) limit = _parse_int(_first_query_value(query, "limit"), default=50, minimum=1, maximum=500) include_stale = _parse_bool(_first_query_value(query, "include_stale"), default=False) - conflicted = self._server.service.list_claims(status="conflicted", limit=limit, include_archived=False) - if not conflicted: - self._write_json({"ok": True, "rows": 0, "groups": []}) - return - statuses = ["confirmed", "conflicted"] + (["stale"] if include_stale else []) - active = self._server.service.store.list_claims(limit=max(limit * 12, 200), status_in=statuses, include_archived=False, include_citations=True) - grouped: dict[tuple[str, str, str], list[Any]] = {} - for claim in conflicted: - grouped.setdefault((str(claim.subject or ""), str(claim.predicate or ""), str(claim.scope or "project")), []) - for claim in active: - key = (str(claim.subject or ""), str(claim.predicate or ""), str(claim.scope or "project")) - if key in grouped: - grouped[key].append(claim) - groups_payload: list[dict[str, Any]] = [] - for key, claims in grouped.items(): - claims_sorted = sorted(claims, key=lambda c: (str(c.updated_at), int(c.id)), reverse=True) - groups_payload.append({"subject": key[0], "predicate": key[1], "scope": key[2], "claims": [_claim_to_dict(c) for c in claims_sorted]}) - groups_payload.sort(key=lambda g: (g["subject"], g["predicate"], g["scope"])) - self._write_json({"ok": True, "rows": len(groups_payload), "groups": groups_payload}) + self._write_json(conflicts_payload( + self._server.service, limit=limit, include_stale=include_stale, + serialize=_claim_to_dict, + )) def _handle_review_queue(self, query_string: str) -> None: query = parse_qs(query_string) @@ -1166,88 +1159,29 @@ def _handle_review_queue(self, query_string: str) -> None: allow_sensitive = _parse_bool(_first_query_value(query, "allow_sensitive"), default=False) exclude_reviewed = _parse_bool(_first_query_value(query, "exclude_reviewed"), default=False) exclude_suppressed = _parse_bool(_first_query_value(query, "exclude_suppressed"), default=False) - items = build_review_queue( - self._server.service, - limit=limit, - include_stale=include_stale, - include_conflicted=include_conflicted, - include_sensitive=allow_sensitive, - ) - flags = self._triage_flags(max(limit * 20, 200)) - queue = queue_to_dicts(items) - out: list[dict[str, Any]] = [] - for item in queue: - claim_id = int(item["claim_id"]) - triage = flags.get(claim_id, {"reviewed": False, "suppressed": False}) - item["reviewed"] = bool(triage["reviewed"]) - item["suppressed"] = bool(triage["suppressed"]) - if exclude_reviewed and bool(item["reviewed"]): - continue - if exclude_suppressed and bool(item["suppressed"]): - continue - out.append(item) - self._write_json({"ok": True, "rows": len(out), "items": out}) + self._write_json(review_queue_payload( + self._server.service, limit=limit, include_stale=include_stale, + include_conflicted=include_conflicted, allow_sensitive=allow_sensitive, + exclude_reviewed=exclude_reviewed, exclude_suppressed=exclude_suppressed, + )) def _handle_mobile_review_queue(self, query_string: str) -> None: query = parse_qs(query_string) limit = _parse_review_queue_limit(_first_query_value(query, "n")) scope = _first_query_value(query, "scope") cursor = _parse_review_queue_cursor(_first_query_value(query, "cursor")) - - clauses = ["archived_at IS NULL"] - params: list[Any] = [] - if scope is not None: - clauses.append("scope = ?") - params.append(scope) - if cursor is not None: - clauses.append("id > ?") - params.append(cursor) - where_sql = "WHERE " + " AND ".join(clauses) - sql = f""" - SELECT id, text, created_at, scope, claim_type, confidence - FROM claims - {where_sql} - ORDER BY created_at ASC, id ASC - LIMIT ? - """ - params.append(limit + 1) - - with self._server.service.store.connect() as conn: - rows = conn.execute(sql, params).fetchall() - - page_rows = rows[:limit] - now = datetime.now(timezone.utc) - queue = [ - { - "id": int(row["id"]), - "text_preview": str(row["text"] or "")[:200], - "age_days": _claim_age_days(str(row["created_at"]), now), - "scope": row["scope"], - "type": row["claim_type"], - "score": float(row["confidence"]), - } - for row in page_rows - ] - next_cursor = int(page_rows[-1]["id"]) if len(rows) > limit and page_rows else None - self._write_json({"queue": queue, "cursor": next_cursor}) + self._write_json(mobile_review_queue_payload( + self._server.service, limit=limit, scope=scope, cursor=cursor, + )) def _handle_action_proposals(self, query_string: str) -> None: query = parse_qs(query_string) limit = _parse_int(_first_query_value(query, "limit"), default=100, minimum=1, maximum=500) status = _first_query_value(query, "status") destination = _first_query_value(query, "destination") - proposals = self._server.service.list_action_proposals( - status=status, - destination=destination, - limit=limit, - ) - self._write_json( - { - "ok": True, - "rows": len(proposals), - "proposals": [asdict(proposal) for proposal in proposals], - } - ) + self._write_json(action_proposals_payload( + self._server.service, status=status, destination=destination, limit=limit, + )) def _handle_atlas_version(self, query_string: str) -> None: from memorymaster.bridges.atlas_contract import atlas_contract_payload @@ -1256,17 +1190,7 @@ def _handle_atlas_version(self, query_string: str) -> None: self._write_json({"ok": True, **payload}) def _handle_action_proposal_status(self, payload: dict[str, Any]) -> None: - proposal_id = int(payload.get("proposal_id") or 0) - status = str(payload.get("status") or "").strip().lower() - external_ref = payload.get("external_ref") - if proposal_id <= 0: - raise ValueError("proposal_id must be positive") - proposal = self._server.service.update_action_proposal_status( - proposal_id, - status=status, - external_ref=str(external_ref) if external_ref not in (None, "") else None, - ) - self._write_json({"ok": True, "proposal": asdict(proposal)}) + self._write_json(update_action_proposal_status(self._server.service, payload)) def _handle_retrieval(self, query_string: str) -> None: params = _parse_retrieval_query_params(query_string) @@ -1334,47 +1258,12 @@ def _handle_recall_analysis(self, query_string: str) -> None: def _handle_audit(self, query_string: str) -> None: query = parse_qs(query_string) limit = _parse_int(_first_query_value(query, "limit"), default=50, minimum=1, maximum=1000) - rows: list[dict[str, Any]] = [] - for event_type in ("audit", "policy_decision"): - events = self._server.service.list_events(limit=limit, event_type=event_type) - rows.extend(_event_to_dict(event) for event in events) - rows.sort(key=lambda e: (str(e.get("created_at", "")), int(e.get("id", 0))), reverse=True) - self._write_json({"ok": True, "rows": min(limit, len(rows)), "events": rows[:limit]}) + self._write_json(audit_payload(self._server.service, limit=limit, serialize=_event_to_dict)) def _handle_namespaces(self, query_string: str) -> None: query = parse_qs(query_string) limit = _parse_int(_first_query_value(query, "limit"), default=200, minimum=1, maximum=5000) - claims = self._server.service.list_claims(limit=limit, include_archived=False) - buckets: dict[str, list[Any]] = {"facts": [], "decisions": [], "workflows": [], "project_overview": []} - for claim in claims: - text = str(claim.text or "").lower() - claim_type = str(claim.claim_type or "").lower() - predicate = str(claim.predicate or "").lower() - is_decision = ("decision" in claim_type) or (predicate in {"decision", "policy"}) or ("decid" in text) - is_workflow = any(k in claim_type for k in {"workflow", "runbook", "process"}) or predicate in {"workflow", "runbook", "command", "step"} - if is_decision: - buckets["decisions"].append(claim) - elif is_workflow: - buckets["workflows"].append(claim) - else: - buckets["facts"].append(claim) - buckets["project_overview"].append(claim) - - def _samples(rows: list[Any]) -> list[dict[str, Any]]: - return [{"id": int(c.id), "subject": c.subject, "predicate": c.predicate, "object_value": c.object_value, "status": c.status} for c in rows[:5]] - - self._write_json( - { - "ok": True, - "rows": len(claims), - "namespaces": { - "facts": {"count": len(buckets["facts"]), "samples": _samples(buckets["facts"])}, - "decisions": {"count": len(buckets["decisions"]), "samples": _samples(buckets["decisions"])}, - "workflows": {"count": len(buckets["workflows"]), "samples": _samples(buckets["workflows"])}, - "project_overview": {"count": len(buckets["project_overview"]), "samples": _samples(buckets["project_overview"])}, - }, - } - ) + self._write_json(namespaces_payload(self._server.service, limit=limit)) def _handle_provenance(self, query_string: str) -> None: """Per-agent provenance: ingest counts + status mix by source_agent. @@ -1520,43 +1409,12 @@ def _handle_validation_latency(self, query_string: str) -> None: self._write_json(_validation_latency_metric(self._server.service)) def _handle_triage_action(self, payload: dict[str, Any]) -> None: - action = str(payload.get("action") or "").strip().lower() - claim_id = int(payload.get("claim_id", 0)) - if claim_id <= 0: - raise ValueError("claim_id must be positive") - if action not in {"pin", "unpin", "mark_reviewed", "suppress", "unsuppress", "approve_proposal", "reject_proposal"}: - raise ValueError("unsupported action") - if action == "pin": - self._write_json({"ok": True, "action": action, "claim": _claim_to_dict(self._server.service.pin(claim_id, pin=True))}) - return - if action == "unpin": - self._write_json({"ok": True, "action": action, "claim": _claim_to_dict(self._server.service.pin(claim_id, pin=False))}) - return - if action in {"approve_proposal", "reject_proposal"}: - from memorymaster.govern.steward import resolve_steward_proposal - - resolved = resolve_steward_proposal( - self._server.service, - action=("approve" if action == "approve_proposal" else "reject"), - claim_id=claim_id, - apply_on_approve=True, - ) - self._write_json({"ok": True, "action": action, "result": resolved}) - return - detail_map = {"mark_reviewed": "triage_mark_reviewed", "suppress": "triage_suppress", "unsuppress": "triage_unsuppress"} - self._server.service.store.record_event(claim_id=claim_id, event_type="audit", details=detail_map[action], payload={"source": "dashboard"}) - self._write_json({"ok": True, "action": action, "claim_id": claim_id}) + self._write_json(apply_triage_action( + self._server.service, payload, serialize_claim=_claim_to_dict, + )) def _handle_operator_control(self, payload: dict[str, Any]) -> None: - action = str(payload.get("action") or "").strip().lower() - if action == "start": - inbox_jsonl = str(payload.get("inbox_jsonl") or "artifacts/operator/operator_inbox.jsonl") - self._write_json({"ok": True, **self._server.start_operator(inbox_jsonl)}) - return - if action == "stop": - self._write_json({"ok": True, **self._server.stop_operator()}) - return - raise ValueError("action must be start or stop") + self._write_json(control_operator(self._server, payload)) def _send_sse_event(self, record: dict[str, Any]) -> bool: """Send a server-sent event record. Returns False if connection is closed.""" diff --git a/memorymaster/surfaces/dashboard_commands.py b/memorymaster/surfaces/dashboard_commands.py new file mode 100644 index 00000000..65b32443 --- /dev/null +++ b/memorymaster/surfaces/dashboard_commands.py @@ -0,0 +1,67 @@ +"""Mutation application for the dashboard composition root.""" + +from __future__ import annotations + +from dataclasses import asdict +from typing import Any, Callable + + +def update_action_proposal_status(service: Any, payload: dict[str, Any]) -> dict[str, Any]: + proposal_id = int(payload.get("proposal_id") or 0) + status = str(payload.get("status") or "").strip().lower() + external_ref = payload.get("external_ref") + if proposal_id <= 0: + raise ValueError("proposal_id must be positive") + proposal = service.update_action_proposal_status( + proposal_id, + status=status, + external_ref=str(external_ref) if external_ref not in (None, "") else None, + ) + return {"ok": True, "proposal": asdict(proposal)} + + +def apply_triage_action( + service: Any, + payload: dict[str, Any], + *, + serialize_claim: Callable[[Any], dict[str, Any]], +) -> dict[str, Any]: + action = str(payload.get("action") or "").strip().lower() + claim_id = int(payload.get("claim_id", 0)) + allowed = {"pin", "unpin", "mark_reviewed", "suppress", "unsuppress", "approve_proposal", "reject_proposal"} + if claim_id <= 0: + raise ValueError("claim_id must be positive") + if action not in allowed: + raise ValueError("unsupported action") + if action in {"pin", "unpin"}: + claim = service.pin(claim_id, pin=action == "pin") + return {"ok": True, "action": action, "claim": serialize_claim(claim)} + if action in {"approve_proposal", "reject_proposal"}: + return _resolve_proposal(service, action, claim_id) + details = {"mark_reviewed": "triage_mark_reviewed", "suppress": "triage_suppress", "unsuppress": "triage_unsuppress"} + service.store.record_event( + claim_id=claim_id, event_type="audit", details=details[action], payload={"source": "dashboard"} + ) + return {"ok": True, "action": action, "claim_id": claim_id} + + +def _resolve_proposal(service: Any, action: str, claim_id: int) -> dict[str, Any]: + from memorymaster.govern.steward import resolve_steward_proposal + + result = resolve_steward_proposal( + service, + action="approve" if action == "approve_proposal" else "reject", + claim_id=claim_id, + apply_on_approve=True, + ) + return {"ok": True, "action": action, "result": result} + + +def control_operator(server: Any, payload: dict[str, Any]) -> dict[str, Any]: + action = str(payload.get("action") or "").strip().lower() + if action == "start": + path = str(payload.get("inbox_jsonl") or "artifacts/operator/operator_inbox.jsonl") + return {"ok": True, **server.start_operator(path)} + if action == "stop": + return {"ok": True, **server.stop_operator()} + raise ValueError("action must be start or stop") diff --git a/memorymaster/surfaces/dashboard_read_models.py b/memorymaster/surfaces/dashboard_read_models.py new file mode 100644 index 00000000..ec3534e1 --- /dev/null +++ b/memorymaster/surfaces/dashboard_read_models.py @@ -0,0 +1,219 @@ +"""Read-only application models for dashboard HTTP handlers.""" + +from __future__ import annotations + +from dataclasses import asdict +from datetime import datetime, timezone +from typing import Any, Callable + +from memorymaster.core.security import is_sensitive_claim +from memorymaster.govern.review import build_review_queue, queue_to_dicts + + +Serializer = Callable[[Any], dict[str, Any]] + + +def triage_flags(service: Any, limit: int) -> dict[int, dict[str, bool]]: + flags: dict[int, dict[str, bool]] = {} + for event in reversed(service.list_events(limit=limit, event_type="audit")): + if event.claim_id is None: + continue + ref = flags.setdefault(int(event.claim_id), {"reviewed": False, "suppressed": False}) + details = str(event.details or "") + if details == "triage_mark_reviewed": + ref["reviewed"] = True + elif details == "triage_suppress": + ref["suppressed"] = True + elif details == "triage_unsuppress": + ref["suppressed"] = False + return flags + + +def claims_payload( + service: Any, + *, + status: str | None, + limit: int, + include_archived: bool, + allow_sensitive: bool, + serialize: Serializer, +) -> dict[str, Any]: + claims = service.list_claims(status=status, limit=limit, include_archived=include_archived) + if not allow_sensitive: + claims = [claim for claim in claims if not is_sensitive_claim(claim)] + return {"ok": True, "rows": len(claims), "claims": [serialize(claim) for claim in claims]} + + +def events_payload( + service: Any, + *, + limit: int, + claim_id: int | None, + event_type: str | None, + serialize: Serializer, + output_key: str = "events", +) -> dict[str, Any]: + events = service.list_events(claim_id=claim_id, limit=limit, event_type=event_type) + return {"ok": True, "rows": len(events), output_key: [serialize(event) for event in events]} + + +def conflicts_payload( + service: Any, + *, + limit: int, + include_stale: bool, + serialize: Serializer, +) -> dict[str, Any]: + conflicted = service.list_claims(status="conflicted", limit=limit, include_archived=False) + if not conflicted: + return {"ok": True, "rows": 0, "groups": []} + statuses = ["confirmed", "conflicted"] + (["stale"] if include_stale else []) + active = service.store.list_claims( + limit=max(limit * 12, 200), status_in=statuses, + include_archived=False, include_citations=True, + ) + grouped: dict[tuple[str, str, str], list[Any]] = {} + for claim in conflicted: + grouped.setdefault(_claim_key(claim), []) + for claim in active: + if _claim_key(claim) in grouped: + grouped[_claim_key(claim)].append(claim) + groups = [_conflict_group(key, claims, serialize) for key, claims in grouped.items()] + groups.sort(key=lambda item: (item["subject"], item["predicate"], item["scope"])) + return {"ok": True, "rows": len(groups), "groups": groups} + + +def _claim_key(claim: Any) -> tuple[str, str, str]: + return ( + str(claim.subject or ""), + str(claim.predicate or ""), + str(claim.scope or "project"), + ) + + +def _conflict_group(key: tuple[str, str, str], claims: list[Any], serialize: Serializer) -> dict[str, Any]: + ordered = sorted(claims, key=lambda claim: (str(claim.updated_at), int(claim.id)), reverse=True) + return { + "subject": key[0], "predicate": key[1], "scope": key[2], + "claims": [serialize(claim) for claim in ordered], + } + + +def review_queue_payload( + service: Any, + *, + limit: int, + include_stale: bool, + include_conflicted: bool, + allow_sensitive: bool, + exclude_reviewed: bool, + exclude_suppressed: bool, +) -> dict[str, Any]: + items = build_review_queue( + service, limit=limit, include_stale=include_stale, + include_conflicted=include_conflicted, include_sensitive=allow_sensitive, + ) + flags = triage_flags(service, max(limit * 20, 200)) + out = _apply_triage_filters(queue_to_dicts(items), flags, exclude_reviewed, exclude_suppressed) + return {"ok": True, "rows": len(out), "items": out} + + +def _apply_triage_filters( + items: list[dict[str, Any]], + flags: dict[int, dict[str, bool]], + exclude_reviewed: bool, + exclude_suppressed: bool, +) -> list[dict[str, Any]]: + out: list[dict[str, Any]] = [] + for item in items: + triage = flags.get(int(item["claim_id"]), {"reviewed": False, "suppressed": False}) + item = {**item, "reviewed": bool(triage["reviewed"]), "suppressed": bool(triage["suppressed"])} + if (exclude_reviewed and item["reviewed"]) or (exclude_suppressed and item["suppressed"]): + continue + out.append(item) + return out + + +def mobile_review_queue_payload( + service: Any, *, limit: int, scope: str | None, cursor: int | None +) -> dict[str, Any]: + clauses, params = ["archived_at IS NULL"], [] + if scope is not None: + clauses.append("scope = ?") + params.append(scope) + if cursor is not None: + clauses.append("id > ?") + params.append(cursor) + sql = _mobile_queue_sql(" AND ".join(clauses)) + with service.store.connect() as conn: + rows = conn.execute(sql, [*params, limit + 1]).fetchall() + page = rows[:limit] + queue = [_mobile_queue_row(row) for row in page] + next_cursor = int(page[-1]["id"]) if len(rows) > limit and page else None + return {"queue": queue, "cursor": next_cursor} + + +def _mobile_queue_sql(where: str) -> str: + return f"""SELECT id, text, created_at, scope, claim_type, confidence + FROM claims WHERE {where} + ORDER BY created_at ASC, id ASC LIMIT ?""" + + +def _mobile_queue_row(row: Any) -> dict[str, Any]: + normalized = str(row["created_at"]).strip().replace("Z", "+00:00") + created = datetime.fromisoformat(normalized) + if created.tzinfo is None: + created = created.replace(tzinfo=timezone.utc) + age = max(0.0, (datetime.now(timezone.utc) - created.astimezone(timezone.utc)).total_seconds() / 86400.0) + return { + "id": int(row["id"]), "text_preview": str(row["text"] or "")[:200], + "age_days": age, "scope": row["scope"], "type": row["claim_type"], + "score": float(row["confidence"]), + } + + +def action_proposals_payload( + service: Any, *, status: str | None, destination: str | None, limit: int +) -> dict[str, Any]: + proposals = service.list_action_proposals(status=status, destination=destination, limit=limit) + return {"ok": True, "rows": len(proposals), "proposals": [asdict(item) for item in proposals]} + + +def audit_payload(service: Any, *, limit: int, serialize: Serializer) -> dict[str, Any]: + rows: list[dict[str, Any]] = [] + for event_type in ("audit", "policy_decision"): + rows.extend(serialize(event) for event in service.list_events(limit=limit, event_type=event_type)) + rows.sort(key=lambda event: (str(event.get("created_at", "")), int(event.get("id", 0))), reverse=True) + return {"ok": True, "rows": min(limit, len(rows)), "events": rows[:limit]} + + +def namespaces_payload(service: Any, *, limit: int) -> dict[str, Any]: + claims = service.list_claims(limit=limit, include_archived=False) + buckets: dict[str, list[Any]] = {"facts": [], "decisions": [], "workflows": [], "project_overview": []} + for claim in claims: + bucket = _claim_namespace(claim) + buckets[bucket].append(claim) + buckets["project_overview"].append(claim) + return { + "ok": True, "rows": len(claims), + "namespaces": {name: {"count": len(rows), "samples": _namespace_samples(rows)} for name, rows in buckets.items()}, + } + + +def _claim_namespace(claim: Any) -> str: + text = str(claim.text or "").lower() + claim_type = str(claim.claim_type or "").lower() + predicate = str(claim.predicate or "").lower() + if "decision" in claim_type or predicate in {"decision", "policy"} or "decid" in text: + return "decisions" + if any(key in claim_type for key in {"workflow", "runbook", "process"}) or predicate in {"workflow", "runbook", "command", "step"}: + return "workflows" + return "facts" + + +def _namespace_samples(rows: list[Any]) -> list[dict[str, Any]]: + return [ + {"id": int(item.id), "subject": item.subject, "predicate": item.predicate, + "object_value": item.object_value, "status": item.status} + for item in rows[:5] + ] diff --git a/tests/test_architecture_budgets.py b/tests/test_architecture_budgets.py new file mode 100644 index 00000000..c6455648 --- /dev/null +++ b/tests/test_architecture_budgets.py @@ -0,0 +1,68 @@ +"""R4.2 non-growth budgets and compatibility-facade contracts.""" + +from __future__ import annotations + +import ast +from pathlib import Path + + +REPO = Path(__file__).resolve().parents[1] + + +def _line_count(relative: str) -> int: + return len((REPO / relative).read_text(encoding="utf-8").splitlines()) + + +def _class_line_count(relative: str, name: str) -> int: + tree = ast.parse((REPO / relative).read_text(encoding="utf-8")) + node = next(item for item in tree.body if isinstance(item, ast.ClassDef) and item.name == name) + return int(node.end_lineno or node.lineno) - node.lineno + 1 + + +def test_gradual_size_budgets_prevent_facade_regrowth() -> None: + assert _line_count("memorymaster/core/service.py") <= 2450 + assert _line_count("memorymaster/surfaces/dashboard.py") <= 1550 + assert _class_line_count("memorymaster/surfaces/dashboard.py", "DashboardRequestHandler") <= 720 + for relative in ( + "memorymaster/core/services/integration.py", + "memorymaster/surfaces/dashboard_read_models.py", + "memorymaster/surfaces/dashboard_commands.py", + ): + assert _line_count(relative) <= 800 + + +def test_extracted_application_functions_stay_bounded() -> None: + for relative in ( + "memorymaster/core/services/integration.py", + "memorymaster/surfaces/dashboard_read_models.py", + "memorymaster/surfaces/dashboard_commands.py", + ): + tree = ast.parse((REPO / relative).read_text(encoding="utf-8")) + for node in ast.walk(tree): + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): + lines = int(node.end_lineno or node.lineno) - node.lineno + 1 + assert lines <= 50, f"{relative}:{node.name} grew to {lines} lines" + + +def test_memory_service_keeps_api_through_extracted_integration_service() -> None: + from memorymaster.core.service import MemoryService + from memorymaster.core.services.integration import IntegrationService + + assert issubclass(MemoryService, IntegrationService) + assert "upsert_source_item" not in MemoryService.__dict__ + assert "create_action_proposal" not in MemoryService.__dict__ + assert "upsert_source_item" in IntegrationService.__dict__ + assert "create_action_proposal" in IntegrationService.__dict__ + + +def test_dashboard_handler_uses_read_model_and_mutation_boundaries() -> None: + source = (REPO / "memorymaster/surfaces/dashboard.py").read_text(encoding="utf-8") + assert "from memorymaster.surfaces.dashboard_read_models import" in source + assert "from memorymaster.surfaces.dashboard_commands import" in source + + +def test_compatibility_policy_has_a_dated_retirement_gate() -> None: + policy = (REPO / "docs/compatibility.md").read_text(encoding="utf-8") + assert "2026-09-30" in policy + assert "memorymaster.service" in policy + assert "removal gate" in policy.lower() diff --git a/tests/test_dashboard_coverage.py b/tests/test_dashboard_coverage.py index 9fec4c4d..05b2a0ce 100644 --- a/tests/test_dashboard_coverage.py +++ b/tests/test_dashboard_coverage.py @@ -301,8 +301,14 @@ def fake_queue_to_dicts(items: list[object]) -> list[dict[str, Any]]: assert len(items) == 1 return [{"claim_id": 2, "status": "stale", "reason": "needs review", "priority": 0.9}] - monkeypatch.setattr("memorymaster.surfaces.dashboard.build_review_queue", fake_build_review_queue) - monkeypatch.setattr("memorymaster.surfaces.dashboard.queue_to_dicts", fake_queue_to_dicts) + monkeypatch.setattr( + "memorymaster.surfaces.dashboard_read_models.build_review_queue", + fake_build_review_queue, + ) + monkeypatch.setattr( + "memorymaster.surfaces.dashboard_read_models.queue_to_dicts", + fake_queue_to_dicts, + ) with running_dashboard(tmp_path, FakeService()) as (base_url, _host, _port, _log_path): payload = get_json(base_url, "/api/review-queue?include_conflicted=0&limit=5") From 5caa4efa2efd0340fb2f60f6ab391cba4019759f Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Tue, 14 Jul 2026 01:25:51 -0300 Subject: [PATCH 45/55] fix: make governance dashboard stateful and accessible --- .planning/REMAINING-PHASES-V1-BUDGET.md | 23 +++ ...EMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md | 2 +- .planning/audit-remediation-ledger.md | 4 +- memorymaster/surfaces/dashboard.py | 47 +++-- .../surfaces/dashboard_read_models.py | 16 ++ tests/test_dashboard_governance_ux.py | 181 ++++++++++++++++++ 6 files changed, 253 insertions(+), 20 deletions(-) create mode 100644 tests/test_dashboard_governance_ux.py diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index d50d0e01..cefc6f9b 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -545,3 +545,26 @@ R4.2 gradual orchestration decomposition completed on 2026-07-14: The next package is R4.3 governance UX and accessibility. The active roadmap goal authorizes continuing without a new goal. + +R4.3 governance UX and accessibility completed: + +- Review mutations now enter an explicit pending/disabled state, retain their + row after a rejected POST, expose an actionable Retry state, and remove the + row only after server success. Panel fetch failures render visible alert and + retry content instead of masquerading as empty panels. +- The review read model attaches the latest unresolved steward proposal by + claim. The UI exposes its rationale, proposed transition/replacement, and + consequences, and renders approve/reject only when a proposal is actionable. +- Claim and conflict views link to lineage and expose citations. Contrast was + raised for muted, placeholder, empty, and stat-label text; controls and live + regions are labelled; keyboard focus is visible; dashboard and conflict + comparisons stack at the mobile breakpoint. +- Four adversarial/static acceptance checks were witnessed RED before the fix. + The focused package gate passed 57 tests. Headless Chromium at 375x812 proved + rejected-POST retention/retry, successful retry removal, visible panel-error + separation, and one-column conflict layout. Inline JavaScript parsing, + changed-file Ruff, syntax compilation, and `git diff --check` passed. +- Atomic package commit: the conventional R4.3 commit containing this evidence. + +The next package is R4.4 generated release and documentation truth. The active +roadmap goal authorizes continuing without a new goal. diff --git a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md index 7abfcab1..73cc92c0 100644 --- a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md +++ b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md @@ -101,7 +101,7 @@ Phase 0 must replace qualitative capacity goals with recorded numeric gates in t - [ ] R3.5 — recovery, observability, audit, and privacy operations - [x] R4.1 — core/extension boundaries - [x] R4.2 — decompose oversized orchestration points -- [ ] R4.3 — governance UX and accessibility +- [x] R4.3 — governance UX and accessibility - [ ] R4.4 — generated release/documentation truth - [ ] Final audit delta and convergence evidence diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 299f2b25..755d31a3 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -29,8 +29,8 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-PERF-03 | Performance/Cost | High / EXPLOITABLE-NOW | Qdrant reconciliation repeats/truncates tens of thousands of embeddings | R3.1 | RESOLVED | R3.1 replaces four truncated 10,000-row scans with authority-scoped keyset pages and a replayable durable cursor; fake-backend convergence/failure-resume evidence passed. Real authenticated/TLS Qdrant remains externally blocked. | | MM-PERF-04 | Database/Performance | Medium / BAD-PRACTICE | Million-row event queries lack event-type composite indexes | R3.2 | RESOLVED | Immutable migration 0015 and both baseline schemas add `(event_type, created_at DESC, id DESC)` and `(event_type, details, created_at DESC)` indexes. SQLite EXPLAIN selects the ordered type index; Postgres DDL parity is covered statically pending the already-recorded disposable-Postgres external gate. | | MM-UX-01 | UX/Maintainability | High / BAD-PRACTICE | Setup reports success without verifying the requested memory loop | R3.3 | RESOLVED | R3.3 adds explicit minimal/semantic/team/full-lab contracts and stable DB, MCP, recall-hook, capture-hook, provider, steward, vector, and dashboard component evidence. Requested PARTIAL/BLOCKED profiles return nonzero; a DB-only sentinel can no longer mask missing requested components. Session-end ingest is now a packaged module/entrypoint, and installed-wheel setup refuses to imply an absent Compose bundle was shipped. Focused setup/profile gate: 161 passed. | -| MM-UX-02 | UX/Reliability | High / BAD-PRACTICE | Dashboard removes review rows before mutation success | R4.3 | OPEN | Browser failure/retry evidence; row remains on rejected POST | -| MM-UX-03 | UX | Medium / BAD-PRACTICE | Dashboard hides failures and has labeling, contrast, responsive, and evidence-discovery gaps | R4.3 | OPEN | Browser/a11y/mobile acceptance suite | +| MM-UX-02 | UX/Reliability | High / BAD-PRACTICE | Dashboard removes review rows before mutation success | R4.3 | RESOLVED | Review mutations now enter a disabled pending state and remove the row only after a successful POST. Headless Chromium evidence rejects the first POST, confirms the row remains with an actionable Retry state, then confirms successful retry removal. | +| MM-UX-03 | UX | Medium / BAD-PRACTICE | Dashboard hides failures and has labeling, contrast, responsive, and evidence-discovery gaps | R4.3 | RESOLVED | Panel failures render visible alert/retry states; controls and live regions are labelled; low-contrast text is raised; narrow dashboard and conflict grids stack; conflicts expose citations/lineage; pending steward proposals expose rationale, transition/replacement consequences, and approval controls only when actionable. Focused dashboard gate: 57 passed, including headless Chromium mobile/failure acceptance. | | MM-PRIV-01 | Compliance | High / BAD-PRACTICE | Automatic transcript capture/remote processing lacks an explicit consent boundary | R2.4,R3.5 | BLOCKED-EXTERNAL | P2-E repository controls are quiet by default and require explicit maximum-capture/provider choices. Product-owner/legal approval for intended use, jurisdictions, processor disclosures, and retention commitments remains external. WARNING LEGAL REVIEW REQUIRED; no compliance-pass or risk-acceptance claim is made. | | MM-PRIV-02 | Compliance | High / BAD-PRACTICE | No complete export, erasure, or retention workflow across copies | R3.5 | BLOCKED-EXTERNAL | R3.5 adds a read-only, tenant-aware dry-run inventory across authoritative claims/citations/events, verbatim, artifacts/spool/wiki/cache, backups, Qdrant, and Postgres. Ambiguous tenants and unattributable copies fail closed. Legal disposition, authenticated backend evidence, and any live export/erase/retention application require external approval and infrastructure; no mutation pass is claimed. | | MM-COST-01 | Cost | High / EXPLOITABLE-NOW | Default Stop-hook LLM calls have no finite persisted budget | R2.4 | RESOLVED | P2-E (`a1d5477`) adds restart-safe atomic global/provider/session reservations and persisted usage; maximum default candidate inflow is 600/day against the 688/day ceiling. | diff --git a/memorymaster/surfaces/dashboard.py b/memorymaster/surfaces/dashboard.py index 768f2010..32813514 100644 --- a/memorymaster/surfaces/dashboard.py +++ b/memorymaster/surfaces/dashboard.py @@ -926,7 +926,7 @@ def _write_dashboard(self) -> None: .header{display:flex;align-items:center;gap:16px;margin-bottom:24px;padding-bottom:16px;border-bottom:1px solid #1e293b} .header h1{margin:0;font-size:1.5rem;color:#f8fafc;letter-spacing:-0.02em} .header .logo{width:36px;height:36px;background:linear-gradient(135deg,#3b82f6,#8b5cf6);border-radius:10px;display:flex;align-items:center;justify-content:center;font-size:18px;font-weight:bold;color:#fff} -.header .subtitle{color:#64748b;font-size:.85rem;margin-top:2px} +.header .subtitle{color:#94a3b8;font-size:.85rem;margin-top:2px} .header .version{margin-left:auto;background:#1e293b;color:#94a3b8;padding:4px 10px;border-radius:20px;font-size:.75rem;font-family:ui-monospace,monospace} .grid{display:grid;grid-template-columns:repeat(2,1fr);gap:16px} section{background:#1e293b;border:1px solid #334155;border-radius:12px;padding:16px;transition:border-color .2s;min-width:0} @@ -935,9 +935,12 @@ def _write_dashboard(self) -> None: .section-head{display:flex;align-items:center;gap:8px;margin-bottom:10px} .section-head .icon{width:28px;height:28px;border-radius:8px;display:flex;align-items:center;justify-content:center;font-size:14px} .section-head h2{margin:0;font-size:.95rem;color:#f1f5f9;font-weight:600} -.section-head .desc{color:#64748b;font-size:.78rem;margin-top:1px} -.muted{color:#64748b;font-size:.85rem} -.empty{color:#475569;font-style:italic;padding:12px;text-align:center} +.section-head .desc{color:#94a3b8;font-size:.78rem;margin-top:1px} +.muted{color:#94a3b8;font-size:.85rem} +.empty{color:#94a3b8;font-style:italic;padding:12px;text-align:center} +.error-state{color:#fecaca;background:#450a0a;border:1px solid #dc2626;padding:10px;border-radius:6px} +.pending{opacity:.65} +.sr-only{position:absolute;width:1px;height:1px;padding:0;margin:-1px;overflow:hidden;clip:rect(0,0,0,0);white-space:nowrap;border:0} .mono{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;font-size:.85rem} .scroll{max-height:300px;overflow:auto;border:1px solid #334155;border-radius:8px} table{width:100%;border-collapse:collapse;font-size:.85rem;table-layout:fixed} @@ -948,6 +951,7 @@ def _write_dashboard(self) -> None: tr:hover td{background:rgba(59,130,246,.06)} button{border:1px solid #475569;background:#1e293b;color:#e2e8f0;border-radius:6px;padding:5px 12px;cursor:pointer;font-size:.8rem;transition:all .15s} button:hover{background:#334155;border-color:#64748b} +button:focus-visible{outline:3px solid #93c5fd;outline-offset:2px} .primary{background:#3b82f6;color:#fff;border-color:#2563eb} .primary:hover{background:#2563eb} .danger{background:#ef4444;color:#fff;border-color:#dc2626} @@ -955,7 +959,7 @@ def _write_dashboard(self) -> None: pre.stream{margin:0;max-height:220px;overflow:auto;background:#020617;color:#22d3ee;padding:12px;border-radius:8px;font-size:.8rem;line-height:1.5;border:1px solid #0e7490} .toolbar{display:flex;gap:8px;flex-wrap:wrap;align-items:center;margin:8px 0} .toolbar input,.toolbar select{padding:6px 10px;border:1px solid #475569;border-radius:8px;background:#0f172a;color:#e2e8f0;font-size:.82rem} -.toolbar input::placeholder{color:#475569} +.toolbar input::placeholder{color:#94a3b8} .toolbar input:focus,.toolbar select:focus{outline:none;border-color:#3b82f6;box-shadow:0 0 0 2px rgba(59,130,246,.2)} .badge{display:inline-block;border-radius:12px;padding:2px 8px;font-size:.72rem;font-weight:500;letter-spacing:.02em} .badge-candidate{background:#1e3a5f;color:#60a5fa;border:1px solid #2563eb} @@ -970,7 +974,7 @@ def _write_dashboard(self) -> None: .stat-row{display:flex;gap:16px;flex-wrap:wrap;padding:8px 0} .stat-item{display:flex;flex-direction:column;gap:2px} .stat-item .stat-value{font-size:1.1rem;font-weight:600;color:#f8fafc;font-family:ui-monospace,monospace} -.stat-item .stat-label{font-size:.7rem;color:#64748b;text-transform:uppercase;letter-spacing:.05em} +.stat-item .stat-label{font-size:.7rem;color:#94a3b8;text-transform:uppercase;letter-spacing:.05em} .icon-blue{background:rgba(59,130,246,.15);color:#60a5fa} .icon-green{background:rgba(34,197,94,.15);color:#4ade80} .icon-amber{background:rgba(245,158,11,.15);color:#fbbf24} @@ -983,6 +987,8 @@ def _write_dashboard(self) -> None: .icon-indigo{background:rgba(99,102,241,.15);color:#818cf8} .icon-lime{background:rgba(132,204,22,.15);color:#a3e635} input[type="text"],input:not([type]){background:#0f172a;border:1px solid #475569;color:#e2e8f0;border-radius:8px;padding:6px 10px} +.conflict-compare{display:grid;grid-template-columns:1fr 1fr;gap:10px;margin-top:8px} +@media(max-width:800px){.grid,.conflict-compare{grid-template-columns:1fr}main{padding:12px}.toolbar{align-items:stretch}.toolbar input,.toolbar select,.toolbar button{width:100%!important}.scroll{max-height:420px}}
@@ -992,8 +998,8 @@ def _write_dashboard(self) -> None:
▶

Operator

Start or stop the memory operator process
-
-
Checking status...
+
+
Checking status...
📈

System Health

Operator metrics, latency, event counters
@@ -1013,23 +1019,24 @@ def _write_dashboard(self) -> None:
🕑

Timeline

Event feed — transitions, validations, policy decisions
-
No events recorded yet
+
No events recorded yet
⚠

Conflicts

Claims that contradict each other — side-by-side comparison
-
No conflicts detected
+
No conflicts detected
📝

Review Queue

Stale or flagged claims that need human review
+
ClaimStatusReasonPriorityActions
Nothing to review
🔍

Search

Query claims using hybrid retrieval (lexical + vector + freshness)
-
IDSubject / Predicate / ValueStatusAnnotationScoreBreakdown
Enter a query and click Search
+
IDSubject / Predicate / ValueStatusAnnotationScoreBreakdown
Enter a query and click Search
🔬

Recall Analysis

Explain WHY each claim ranked where it did — per-stage score attribution + active retrieval weights (read-only)
-
IDTextStatusTierScoreLex / Conf / Fresh / Vec
Enter a query and click Analyze
+
IDTextStatusTierScoreLex / Conf / Fresh / Vec
Enter a query and click Analyze
🔒

Audit Log

Security and governance audit trail
@@ -1057,6 +1064,12 @@ def _write_dashboard(self) -> None: const statusBadge=(s)=>{const v=String(s||'unknown').toLowerCase();const cls=({candidate:'badge-candidate',stale:'badge-stale',conflicted:'badge-conflicted',confirmed:'badge-confirmed',archived:'badge-archived',superseded:'badge-superseded'})[v]||'';return ''+esc(v)+'';}; const f3=(v)=>typeof v==='number'?v.toFixed(3):'-'; const tuple=(c)=>''+esc(c.subject||'-')+' / '+esc(c.predicate||'-')+' / '+esc(c.object_value||c.text||'-')+''; +const citationList=(c)=>{const rows=Array.isArray(c&&c.citations)?c.citations:[];return rows.length?'
    '+rows.map(x=>'
  • '+esc(x.source||'source')+' '+esc(x.locator||'')+(x.excerpt?(' — '+esc(x.excerpt)):'')+'
    • ').join('')+'
':'
No citations recorded
';}; +const governanceEvidence=(c)=>c?'
Evidence & lineage
Citations'+citationList(c)+'Inspect claim lineage
':''; +const proposalEvidence=(p)=>{if(!p)return '';const reasons=Array.isArray(p.reasons)?p.reasons:[];const rationale=reasons.length?'
    '+reasons.map(r=>'
  • '+esc(r.code||'reason')+': '+esc(r.message||r.details||r.rationale||'-')+'
    • ').join('')+'
':'
No rationale recorded
';const transition=p.proposed_status?('status becomes '+esc(p.proposed_status)):('decision '+esc(p.proposal_decision||'review'));const replacement=p.replaced_by_claim_id?(' and is replaced by claim #'+esc(p.replaced_by_claim_id)):'';return '
Steward proposal
Rationale'+rationale+'
Proposed consequence: '+transition+replacement+'
';}; +const actionConsequences=(hasProposal)=>'
Action consequences
Pin keeps the claim prominent. Reviewed and Suppress remove it from this queue.'+(hasProposal?' Approve applies the displayed steward proposal when valid; Reject records a governance decision without applying it.':' No steward proposal is pending, so approval controls are unavailable.')+'
'; +const reviewStatus=(message,isError=false)=>{const box=document.getElementById('review-status');box.textContent=message||'';box.className=isError?'error-state':'muted';}; +function showPanelFailure(id,label,error,colspan=0){const node=document.getElementById(id);if(!node)return;const message='Could not load '+label+': '+String((error&&error.message)||error||'unknown error');const retry='';node.innerHTML=colspan?'':'';} async function jget(p){const r=await fetch(p);const d=await r.json();if(!r.ok||d.ok===false)throw new Error((d&&d.error)||('HTTP '+r.status));return d;} async function jpost(p,b){const r=await fetch(p,{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify(b||{})});const d=await r.json();if(!r.ok||d.ok===false)throw new Error((d&&d.error)||('HTTP '+r.status));return d;} const timelineState={rows:[],search:'',eventType:'',group:'claim'}; @@ -1064,13 +1077,13 @@ def _write_dashboard(self) -> None: const topMap=(obj,limit)=>Object.entries(obj||{}).sort((a,b)=>Number(b[1]||0)-Number(a[1]||0)).slice(0,limit); const actor=(e)=>{const p=e&&typeof e.payload==='object'?e.payload:null;const src=p&&p.source?String(p.source).trim():'';if(src)return src;if(String(e.event_type||'')==='policy_decision')return 'policy';if(String(e.event_type||'')==='validator'||String(e.event_type||'')==='deterministic_validator')return 'validator';return 'system';}; const countPills=(obj,limit)=>{const rows=topMap(obj,limit);return rows.length?rows.map(([k,v])=>''+esc(k)+'='+esc(v)+'').join(' | '):'none';}; -function fillClaims(d){const rows=Array.isArray(d.claims)?d.claims:[];const b=document.getElementById('claims-body');if(!rows.length){b.innerHTML='No claims ingested yet';return;}b.innerHTML=rows.map(c=>''+esc(c.id)+''+statusBadge(c.status)+''+tuple(c)+''+esc(f3(c.confidence))+''+esc((c.citations||[]).length)+''+esc(c.updated_at||'-')+'').join('');} +function fillClaims(d){const rows=Array.isArray(d.claims)?d.claims:[];const b=document.getElementById('claims-body');if(!rows.length){b.innerHTML='No claims ingested yet';return;}b.innerHTML=rows.map(c=>''+esc(c.id)+''+statusBadge(c.status)+''+tuple(c)+governanceEvidence(c)+''+esc(f3(c.confidence))+''+esc((c.citations||[]).length)+''+esc(c.updated_at||'-')+'').join('');} function renderTimeline(){const h=document.getElementById('timeline-list');const meta=document.getElementById('timeline-meta');const icon=(t)=>({ingest:'📥',extractor:'🔌',validator:'✅',deterministic_validator:'💪',transition:'🔄',policy_decision:'⚖',audit:'🔒',compaction_run:'🗑',supersession:'🔄',confidence:'📈'})[String(t||'')]||'🔸';const q=String(timelineState.search||'').toLowerCase().trim();const eventType=String(timelineState.eventType||'').trim();const rows=timelineState.rows.filter(e=>{if(eventType&&String(e.event_type||'')!==eventType)return false;if(!q)return true;const hay=[e.event_type,e.details,e.from_status,e.to_status,e.claim_id,actor(e),e.created_at].map(v=>String(v||'').toLowerCase()).join(' ');return hay.includes(q);});meta.textContent=rows.length+' of '+timelineState.rows.length+' events';if(!rows.length){h.innerHTML='
No matching events
';return;}const byDay={};rows.forEach(e=>{const day=String(e.created_at||'').slice(0,10)||'unknown';if(!byDay[day])byDay[day]=[];byDay[day].push(e);});h.innerHTML=Object.keys(byDay).sort().reverse().map(day=>{const buckets={};byDay[day].forEach(e=>{const key=(timelineState.group==='event_type')?('event:'+String(e.event_type||'event')):(e.claim_id?('claim:'+String(e.claim_id)):('event:'+String(e.event_type||'event')));if(!buckets[key])buckets[key]=[];buckets[key].push(e);});const bucketHtml=Object.keys(buckets).sort().map(key=>{const evs=buckets[key];return '
'+esc(key)+' ('+esc(evs.length)+' events)
'+evs.map(e=>{const a=actor(e);return '
'+esc(e.created_at||'-')+'
'+icon(e.event_type)+' '+esc(e.event_type||'event')+' '+statusBadge(e.to_status||e.event_type)+' by '+esc(a)+' '+(e.claim_id?('· claim #'+esc(e.claim_id)+''):'')+'
'+(e.from_status&&e.to_status?'
'+esc(e.from_status)+' → '+esc(e.to_status)+'
':'')+(e.details?('
'+esc(e.details)+'
'):'')+'
';}).join('')+'
';}).join('');return '
'+esc(day)+' ('+esc(byDay[day].length)+' events)
'+bucketHtml+'
';}).join('');} function fillTimeline(d){timelineState.rows=Array.isArray(d.timeline)?d.timeline:[];renderTimeline();} -function renderConflicts(){const h=document.getElementById('conflicts-cards');const meta=document.getElementById('conflicts-meta');const q=String(conflictState.search||'').toLowerCase().trim();const rows=conflictState.rows.filter(g=>{if(!q)return true;const claims=Array.isArray(g.claims)?g.claims:[];const claimText=claims.map(c=>[c.id,c.status,c.subject,c.predicate,c.object_value,c.text].join(' ')).join(' ').toLowerCase();return ([g.subject,g.predicate,g.scope].join(' ').toLowerCase()+' '+claimText).includes(q);});meta.textContent=rows.length+' of '+conflictState.rows.length+' conflict groups';h.innerHTML=rows.map(g=>{const cs=Array.isArray(g.claims)?g.claims:[];const n=cs[0]||null;const o=cs.length>1?cs[1]:null;const nv=n?(n.object_value||n.text||'-'):'-';const ov=o?(o.object_value||o.text||'-'):'-';const nc=n?Number(n.confidence||0):null;const oc=o?Number(o.confidence||0):null;const nz=n?((n.citations||[]).length):0;const oz=o?((o.citations||[]).length):0;const confDelta=(nc!=null&&oc!=null)?(nc-oc):null;const citeDelta=(n&&o)?(nz-oz):null;const valueChanged=(String(nv)!==String(ov));const delta=(v)=>v==null?'-':((v>=0?'+':'')+Number(v).toFixed(3));const cDelta=(v)=>v==null?'-':((v>=0?'+':'')+String(v));const row=(label,a,b,chg)=>''+esc(label)+''+esc(a)+''+esc(b)+''+esc(chg)+'';const statusCounts={};cs.forEach(c=>{const k=String(c.status||'unknown');statusCounts[k]=(statusCounts[k]||0)+1;});return '
'+esc(g.subject||'-')+' / '+esc(g.predicate||'-')+'
'+esc(cs.length)+' claims · scope: '+esc(g.scope||'project')+' · '+countPills(statusCounts,4)+'
Newer '+(n?statusBadge(n.status):'')+' #'+esc(n?n.id:'-')+'
'+esc(nv)+'
confidence: '+esc(nc==null?'-':nc.toFixed(3))+' · citations: '+esc(nz)+'
Older '+(o?statusBadge(o.status):'')+' #'+esc(o?o.id:'-')+'
'+esc(ov)+'
confidence: '+esc(oc==null?'-':oc.toFixed(3))+' · citations: '+esc(oz)+'
'+row('value',nv,ov,valueChanged?'CHANGED':'same')+row('confidence',nc==null?'-':nc.toFixed(3),oc==null?'-':oc.toFixed(3),delta(confDelta))+row('citations',String(nz),String(oz),cDelta(citeDelta))+row('updated_at',n&&n.updated_at?n.updated_at:'-',o&&o.updated_at?o.updated_at:'-',(n&&o&&String(n.updated_at)!==String(o.updated_at))?'changed':'same')+'
FieldNewerOlderDelta
';}).join('')||'
No conflicts detected
';} +function renderConflicts(){const h=document.getElementById('conflicts-cards');const meta=document.getElementById('conflicts-meta');const q=String(conflictState.search||'').toLowerCase().trim();const rows=conflictState.rows.filter(g=>{if(!q)return true;const claims=Array.isArray(g.claims)?g.claims:[];const claimText=claims.map(c=>[c.id,c.status,c.subject,c.predicate,c.object_value,c.text].join(' ')).join(' ').toLowerCase();return ([g.subject,g.predicate,g.scope].join(' ').toLowerCase()+' '+claimText).includes(q);});meta.textContent=rows.length+' of '+conflictState.rows.length+' conflict groups';h.innerHTML=rows.map(g=>{const cs=Array.isArray(g.claims)?g.claims:[];const n=cs[0]||null;const o=cs.length>1?cs[1]:null;const nv=n?(n.object_value||n.text||'-'):'-';const ov=o?(o.object_value||o.text||'-'):'-';const nc=n?Number(n.confidence||0):null;const oc=o?Number(o.confidence||0):null;const nz=n?((n.citations||[]).length):0;const oz=o?((o.citations||[]).length):0;const confDelta=(nc!=null&&oc!=null)?(nc-oc):null;const citeDelta=(n&&o)?(nz-oz):null;const valueChanged=(String(nv)!==String(ov));const delta=(v)=>v==null?'-':((v>=0?'+':'')+Number(v).toFixed(3));const cDelta=(v)=>v==null?'-':((v>=0?'+':'')+String(v));const row=(label,a,b,chg)=>''+esc(label)+''+esc(a)+''+esc(b)+''+esc(chg)+'';const statusCounts={};cs.forEach(c=>{const k=String(c.status||'unknown');statusCounts[k]=(statusCounts[k]||0)+1;});const card=(label,c,value,confidence,cites,color)=>'
'+label+' '+(c?statusBadge(c.status):'')+' #'+esc(c?c.id:'-')+'
'+esc(value)+'
confidence: '+esc(confidence==null?'-':confidence.toFixed(3))+' · citations: '+esc(cites)+'
'+governanceEvidence(c)+'
';return '
'+esc(g.subject||'-')+' / '+esc(g.predicate||'-')+'
'+esc(cs.length)+' claims · scope: '+esc(g.scope||'project')+' · '+countPills(statusCounts,4)+'
'+card('Newer',n,nv,nc,nz,'#4ade80')+card('Older',o,ov,oc,oz,'#f87171')+'
'+row('value',nv,ov,valueChanged?'CHANGED':'same')+row('confidence',nc==null?'-':nc.toFixed(3),oc==null?'-':oc.toFixed(3),delta(confDelta))+row('citations',String(nz),String(oz),cDelta(citeDelta))+row('updated_at',n&&n.updated_at?n.updated_at:'-',o&&o.updated_at?o.updated_at:'-',(n&&o&&String(n.updated_at)!==String(o.updated_at))?'changed':'same')+'
FieldNewerOlderDelta
';}).join('')||'
No conflicts detected
';} function fillConflicts(d){conflictState.rows=Array.isArray(d.groups)?d.groups:[];renderConflicts();} async function refreshConflicts(){const includeStale=document.getElementById('conflicts-include-stale').checked?'1':'0';fillConflicts(await jget('/api/conflicts?limit=20&include_stale='+includeStale));} -function fillQueue(d){const rows=Array.isArray(d.items)?d.items:[];const b=document.getElementById('stale-body');if(!rows.length){b.innerHTML='Nothing to review';return;}b.innerHTML=rows.map(i=>''+esc(i.claim_id)+''+statusBadge(i.status)+''+esc(i.reason||'-')+''+esc(f3(i.priority))+' ').join('');} +function fillQueue(d){const rows=Array.isArray(d.items)?d.items:[];const b=document.getElementById('stale-body');if(!rows.length){b.innerHTML='Nothing to review';return;}b.innerHTML=rows.map(i=>{const p=i.proposal||null;const proposalActions=p?' ':'';return ''+esc(i.claim_id)+''+statusBadge(i.status)+''+esc(i.reason||'-')+proposalEvidence(p)+''+esc(f3(i.priority))+' '+proposalActions+actionConsequences(Boolean(p))+'';}).join('');} function fillRetr(d){const rows=Array.isArray(d.rows_data)?d.rows_data:[];const b=document.getElementById('retrieval-body');const meta=document.getElementById('retrieval-meta');const scopes=Array.isArray(d.scope_allowlist)?d.scope_allowlist:[];const scopeText=scopes.length?scopes.join(', '):'all';meta.textContent='Mode: '+(d.mode||'-')+' · Scopes: '+scopeText+' · '+rows.length+' results';b.innerHTML=rows.map(r=>{const c=r.claim||{};const s=r.status||c.status||'unknown';const ann=(r.annotation||'-');return ''+esc(c.id)+''+tuple(c)+''+statusBadge(s)+''+esc(ann)+''+esc(f3(r.score))+''+esc(f3(r.lexical_score))+' / '+esc(f3(r.confidence_score))+' / '+esc(f3(r.freshness_score))+' / '+esc(f3(r.vector_score))+'';}).join('')||'No results found';} function fillRecallAnalysis(d){const rows=Array.isArray(d.results)?d.results:[];const meta=document.getElementById('recall-meta');const wbox=document.getElementById('recall-weights');meta.textContent='Mode: '+(d.mode||'-')+(d.profile?(' · Profile: '+d.profile):'')+' · '+rows.length+' results';const w=(d.weights&&d.weights.retrieval_weights)||null;wbox.innerHTML=w?('Active weights — lexical: '+esc(f3(w.lexical))+' · confidence: '+esc(f3(w.confidence))+' · freshness: '+esc(f3(w.freshness))+' · vector: '+esc(f3(w.vector))):'';const b=document.getElementById('recall-body');b.innerHTML=rows.map(r=>{const id=r.human_id||r.claim_id;const pin=r.pinned?' 📌':'';return ''+esc(id)+pin+''+esc(r.text||'-')+''+statusBadge(r.status)+''+esc(r.tier||'working')+''+esc(f3(r.score))+''+esc(f3(r.lexical_score))+' / '+esc(f3(r.confidence_score))+' / '+esc(f3(r.freshness_score))+' / '+esc(f3(r.vector_score))+'';}).join('')||'No results found';} function fillAudit(d){const rows=Array.isArray(d.events)?d.events:[];document.getElementById('audit-body').innerHTML=rows.map(e=>''+esc(e.created_at||'-')+''+esc(e.event_type||'-')+''+esc(e.claim_id||'-')+''+esc(e.details||'-')+'').join('')||'No audit events';} @@ -1083,7 +1096,7 @@ def _write_dashboard(self) -> None: function fillIntegrity(d){const x=d.integrity||{};const box=document.getElementById('integrity-box');if(x.available===false){box.innerHTML='
'+esc(x.reason||'unavailable')+'
';return;}const sp=x.spool||{};const qd=x.qdrant||{};const lc=x.last_cycle||null;const fmtB=(v)=>v==null?'-':(v>=1048576?(v/1048576).toFixed(1)+' MB':String(v)+' B');const nv=(v)=>v==null?'-':String(v);box.innerHTML='
'+esc(fmtB(x.wal_bytes))+'WAL size
'+esc(nv(sp.depth_lines))+'Spool lines
'+esc(nv(qd.drift))+'Qdrant drift
'+esc(nv(x.busy_errors))+'Busy errors
'+(x.promotions_frozen?'FROZEN':'ok')+'Promotions
'+(lc?'
Last cycle '+esc(nv(lc.at))+' · quick_check='+esc(lc.quick_check_ok==null?'throttled':lc.quick_check_ok)+' · fk_orphans='+esc(nv(lc.fk_orphans))+' · drained='+esc(nv(lc.spool_drained))+' · lag='+esc(nv(lc.spool_lag_seconds))+'s
':'
No cycle metrics recorded yet
')+'
';} async function refreshQueue(){fillQueue(await jget('/api/review-queue?limit=30&exclude_reviewed=1&exclude_suppressed=1'));} async function refreshObs(){fillObs(await jget('/api/observability?log_limit=1500&event_limit=600&queue_limit=250'));} -document.getElementById('stale-body').addEventListener('click',async(ev)=>{const t=ev.target;if(!t||t.tagName!=='BUTTON')return;const r=t.closest('tr');if(!r)return;const id=Number(r.getAttribute('data-claim-id'));const a=String(t.getAttribute('data-action')||'');r.remove();await jpost('/api/triage/action',{claim_id:id,action:a});await refreshQueue();}); +document.getElementById('stale-body').addEventListener('click',async(ev)=>{const t=ev.target;if(!t||t.tagName!=='BUTTON'||!t.hasAttribute('data-action'))return;const r=t.closest('tr');if(!r)return;const id=Number(r.getAttribute('data-claim-id'));const a=String(t.getAttribute('data-action')||'');const original=t.innerHTML;r.setAttribute('data-pending','true');r.classList.add('pending');t.disabled=true;reviewStatus('Applying '+a+' to claim '+id+'...');try{await jpost('/api/triage/action',{claim_id:id,action:a});r.remove();reviewStatus('Action '+a+' completed for claim '+id+'.');await refreshQueue();}catch(error){r.removeAttribute('data-pending');r.classList.remove('pending');t.disabled=false;t.innerHTML='Retry';t.title=String((error&&error.message)||error||'Action failed');reviewStatus('Action failed for claim '+id+'. Retry is available.',true);t.addEventListener('blur',()=>{t.innerHTML=original;},{once:true});}}); document.getElementById('op-start').addEventListener('click',async()=>{await jpost('/api/operator/control',{action:'start',inbox_jsonl:document.getElementById('op-inbox').value});fillOp(await jget('/api/operator/status'));}); document.getElementById('op-stop').addEventListener('click',async()=>{await jpost('/api/operator/control',{action:'stop'});fillOp(await jget('/api/operator/status'));}); document.getElementById('retrieval-run').addEventListener('click',async()=>{const query=document.getElementById('retrieval-query').value||'';const mode=document.getElementById('retrieval-mode').value||'hybrid';const scopeRaw=document.getElementById('retrieval-scope').value||'';const url='/api/retrieval?query='+encodeURIComponent(query)+'&mode='+encodeURIComponent(mode)+'&scope_allowlist='+encodeURIComponent(scopeRaw)+'&limit=10';fillRetr(await jget(url));}); @@ -1094,7 +1107,7 @@ def _write_dashboard(self) -> None: document.getElementById('conflicts-search').addEventListener('input',(ev)=>{conflictState.search=String((ev&&ev.target&&ev.target.value)||'');renderConflicts();}); document.getElementById('conflicts-include-stale').addEventListener('change',refreshConflicts); document.getElementById('conflicts-refresh').addEventListener('click',refreshConflicts); -jget('/api/claims?limit=50').then(fillClaims).catch(()=>{});jget('/api/timeline?limit=40').then(fillTimeline).catch(()=>{});refreshConflicts().catch(()=>{});refreshQueue().catch(()=>{});jget('/api/audit?limit=40').then(fillAudit).catch(()=>{});jget('/api/namespaces?limit=200').then(fillNs).catch(()=>{});jget('/api/provenance').then(fillProvenance).catch(()=>{});jget('/api/session-stats?limit=2000').then(fillStats).catch(()=>{});jget('/metrics/validation-latency').then(fillValidationLatency).catch(()=>{});jget('/api/integrity').then(fillIntegrity).catch(()=>{});jget('/api/operator/status').then(fillOp).catch(()=>{});refreshObs().catch(()=>{}); +jget('/api/claims?limit=50').then(fillClaims).catch(e=>showPanelFailure('claims-body','claims',e,6));jget('/api/timeline?limit=40').then(fillTimeline).catch(e=>showPanelFailure('timeline-list','timeline',e));refreshConflicts().catch(e=>showPanelFailure('conflicts-cards','conflicts',e));refreshQueue().catch(e=>showPanelFailure('stale-body','review queue',e,5));jget('/api/audit?limit=40').then(fillAudit).catch(e=>showPanelFailure('audit-body','audit log',e,4));jget('/api/namespaces?limit=200').then(fillNs).catch(e=>showPanelFailure('namespaces-box','namespaces',e));jget('/api/provenance').then(fillProvenance).catch(e=>showPanelFailure('provenance-body','provenance',e,8));jget('/api/session-stats?limit=2000').then(fillStats).catch(e=>showPanelFailure('session-stats','session statistics',e));jget('/metrics/validation-latency').then(fillValidationLatency).catch(e=>showPanelFailure('validation-latency','validation latency',e));jget('/api/integrity').then(fillIntegrity).catch(e=>showPanelFailure('integrity-box','integrity',e));jget('/api/operator/status').then(fillOp).catch(e=>showPanelFailure('op-status','operator status',e));refreshObs().catch(e=>showPanelFailure('obs-box','observability',e)); const sb=document.getElementById('stream');const es=new EventSource('/api/operator/stream?last=20'); const append=(t)=>{const ex=sb.textContent.trim();sb.textContent=(ex&&ex!=='Waiting for operator to start...'?ex+'\\n':'')+t;}; ['message','stream_start','state_loaded','state_error','state_saved','json_error','turn_processed','reconcile_run','stream_exit'].forEach(n=>es.addEventListener(n,(ev)=>append(ev.data))); es.onerror=()=>append('[stream reconnecting]');
""" body = html.encode("utf-8") diff --git a/memorymaster/surfaces/dashboard_read_models.py b/memorymaster/surfaces/dashboard_read_models.py index ec3534e1..4503527b 100644 --- a/memorymaster/surfaces/dashboard_read_models.py +++ b/memorymaster/surfaces/dashboard_read_models.py @@ -8,6 +8,7 @@ from memorymaster.core.security import is_sensitive_claim from memorymaster.govern.review import build_review_queue, queue_to_dicts +from memorymaster.govern.steward import list_steward_proposals Serializer = Callable[[Any], dict[str, Any]] @@ -115,9 +116,24 @@ def review_queue_payload( ) flags = triage_flags(service, max(limit * 20, 200)) out = _apply_triage_filters(queue_to_dicts(items), flags, exclude_reviewed, exclude_suppressed) + proposals = _pending_proposals_by_claim(service, limit) + out = [{**item, "proposal": proposals.get(int(item["claim_id"]))} for item in out] return {"ok": True, "rows": len(out), "items": out} +def _pending_proposals_by_claim(service: Any, limit: int) -> dict[int, dict[str, Any]]: + proposals = list_steward_proposals(service, limit=max(limit, 1), include_resolved=False) + latest: dict[int, dict[str, Any]] = {} + for proposal in proposals: + claim_id = proposal.get("claim_id") + if not isinstance(claim_id, int): + continue + current = latest.get(claim_id) + if current is None or int(proposal["proposal_event_id"]) > int(current["proposal_event_id"]): + latest[claim_id] = proposal + return latest + + def _apply_triage_filters( items: list[dict[str, Any]], flags: dict[int, dict[str, bool]], diff --git a/tests/test_dashboard_governance_ux.py b/tests/test_dashboard_governance_ux.py new file mode 100644 index 00000000..20f805dc --- /dev/null +++ b/tests/test_dashboard_governance_ux.py @@ -0,0 +1,181 @@ +"""R4.3 governance-state and accessibility acceptance contracts.""" + +from __future__ import annotations + +from io import BytesIO +from html.parser import HTMLParser +import json + +import pytest +from memorymaster.surfaces.dashboard import DashboardRequestHandler +from memorymaster.surfaces import dashboard_read_models + + +def _dashboard_html() -> str: + handler = DashboardRequestHandler.__new__(DashboardRequestHandler) + handler.wfile = BytesIO() + handler.send_response = lambda *_args, **_kwargs: None + handler.send_header = lambda *_args, **_kwargs: None + handler.end_headers = lambda *_args, **_kwargs: None + handler._write_dashboard() + return handler.wfile.getvalue().decode("utf-8") + + +class _ControlParser(HTMLParser): + def __init__(self) -> None: + super().__init__() + self.unlabelled: list[str] = [] + + def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: + if tag not in {"input", "select"}: + return + values = dict(attrs) + if not values.get("aria-label") and not values.get("aria-labelledby"): + self.unlabelled.append(values.get("id") or tag) + + +def test_review_mutation_waits_for_success_and_exposes_retry_state() -> None: + html = _dashboard_html() + assert "r.remove();await jpost" not in html + assert "data-pending" in html + assert "Action failed" in html + assert "Retry" in html + assert 'id="review-status"' in html + assert 'aria-live="polite"' in html + + +def test_panel_failures_are_not_silently_rendered_as_empty() -> None: + html = _dashboard_html() + assert ".catch(()=>{})" not in html + assert "showPanelFailure" in html + assert "Could not load" in html + + +def test_controls_live_regions_and_mobile_layout_are_accessible() -> None: + html = _dashboard_html() + parser = _ControlParser() + parser.feed(html) + assert parser.unlabelled == [] + assert '@media(max-width:800px)' in html + assert "grid-template-columns:1fr" in html + assert 'role="status"' in html + + +def test_governance_evidence_and_consequences_are_directly_inspectable() -> None: + html = _dashboard_html() + assert "Evidence & lineage" in html + assert "/lineage" in html + assert "Citations" in html + assert "Action consequences" in html + assert 'class="conflict-compare"' in html + assert "proposalEvidence(p)" in html + assert "const proposalActions=p?" in html + + +def test_review_queue_exposes_only_the_latest_pending_proposal(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(dashboard_read_models, "build_review_queue", lambda *_args, **_kwargs: [object()]) + monkeypatch.setattr( + dashboard_read_models, + "queue_to_dicts", + lambda _items: [{"claim_id": 7, "status": "stale", "reason": "age", "priority": 0.9}], + ) + monkeypatch.setattr(dashboard_read_models, "triage_flags", lambda *_args, **_kwargs: {}) + monkeypatch.setattr( + dashboard_read_models, + "list_steward_proposals", + lambda *_args, **_kwargs: [ + {"proposal_event_id": 3, "claim_id": 7, "proposal_decision": "stale_candidate"}, + { + "proposal_event_id": 9, + "claim_id": 7, + "proposal_decision": "archive_candidate", + "proposed_status": "archived", + "reasons": [{"code": "age", "message": "Retention elapsed"}], + }, + ], + ) + + payload = dashboard_read_models.review_queue_payload( + object(), + limit=30, + include_stale=True, + include_conflicted=True, + allow_sensitive=False, + exclude_reviewed=True, + exclude_suppressed=True, + ) + + assert payload["items"][0]["proposal"]["proposal_event_id"] == 9 + assert payload["items"][0]["proposal"]["reasons"][0]["message"] == "Retention elapsed" + + +def test_browser_preserves_failed_review_action_and_distinguishes_load_error() -> None: + sync_api = pytest.importorskip("playwright.sync_api") + html = _dashboard_html() + state = {"action_calls": 0, "resolved": False} + conflict = { + "id": 4, + "status": "conflicted", + "subject": "release", + "predicate": "ready", + "object_value": "no", + "confidence": 0.6, + "citations": [{"source": "test", "locator": "case", "excerpt": "evidence"}], + } + review_item = { + "claim_id": 7, + "status": "stale", + "reason": "requires governance review", + "priority": 0.9, + "proposal": { + "proposal_event_id": 9, + "proposal_decision": "archive_candidate", + "proposed_status": "archived", + "reasons": [{"code": "age", "message": "Retention elapsed"}], + }, + } + + def route_request(route: object) -> None: + request = route.request + path = request.url.split("dashboard.test", 1)[-1] + if path == "/": + route.fulfill(status=200, content_type="text/html", body=html) + return + if path.startswith("/api/claims"): + route.fulfill(status=503, content_type="application/json", body='{"ok":false,"error":"offline"}') + return + if path.startswith("/api/review-queue"): + items = [] if state["resolved"] else [review_item] + route.fulfill(status=200, content_type="application/json", body=json.dumps({"ok": True, "items": items})) + return + if path.startswith("/api/conflicts"): + route.fulfill( + status=200, + content_type="application/json", + body=json.dumps({"ok": True, "groups": [{"subject": "release", "predicate": "ready", "scope": "project", "claims": [conflict, {**conflict, "id": 2, "object_value": "yes"}]}]}), + ) + return + if path.startswith("/api/triage/action"): + state["action_calls"] += 1 + if state["action_calls"] == 1: + route.fulfill(status=409, content_type="application/json", body='{"ok":false,"error":"rejected"}') + else: + state["resolved"] = True + route.fulfill(status=200, content_type="application/json", body='{"ok":true}') + return + route.fulfill(status=200, content_type="application/json", body='{"ok":true}') + + with sync_api.sync_playwright() as playwright: + browser = playwright.chromium.launch(headless=True) + page = browser.new_page(viewport={"width": 375, "height": 812}) + page.route("**/*", route_request) + page.goto("http://dashboard.test/") + page.get_by_role("button", name="Approve proposal for claim 7").click() + page.get_by_text("Action failed for claim 7").wait_for() + assert page.locator('tr[data-claim-id="7"]').count() == 1 + assert page.get_by_role("button", name="Approve proposal for claim 7").text_content() == "Retry" + assert "Could not load claims" in page.locator("#claims-body").inner_text() + assert page.locator(".conflict-compare").evaluate("node => getComputedStyle(node).gridTemplateColumns").count(" ") == 0 + page.get_by_role("button", name="Approve proposal for claim 7").click() + page.locator('tr[data-claim-id="7"]').wait_for(state="detached") + browser.close() From 3cd2ea58dba1d36a3332446883bc5db6e8c91a96 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Tue, 14 Jul 2026 01:38:12 -0300 Subject: [PATCH 46/55] chore: generate and gate release truth --- .github/workflows/ci.yml | 13 + .github/workflows/publish.yml | 111 +++++---- .planning/REMAINING-PHASES-V1-BUDGET.md | 29 +++ ...EMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md | 12 +- .planning/audit-remediation-ledger.md | 6 +- README.md | 6 +- ROADMAP-v3.2.md | 53 +--- ROADMAP.md | 207 +++------------- docs/MCP-TOOLS.md | 2 +- docs/README.md | 4 +- docs/ROADMAP.md | 169 +------------ docs/generated/release-truth.json | 233 ++++++++++++++++++ docs/generated/release-truth.md | 31 +++ docs/handbook.md | 4 +- docs/integration/hermes-agent-brief.md | 4 +- docs/security_supply_chain.md | 10 +- external-actions-required.md | 1 + memorymaster/__init__.py | 9 +- memorymaster/surfaces/dashboard.py | 17 +- memorymaster/surfaces/setup_detect.py | 4 +- roadmapres.md | 79 +----- scripts/generate_release_truth.py | 143 +++++++++++ tests/test_release_truth.py | 87 +++++++ tests/test_supply_chain_contracts.py | 9 +- 24 files changed, 699 insertions(+), 544 deletions(-) create mode 100644 docs/generated/release-truth.json create mode 100644 docs/generated/release-truth.md create mode 100644 scripts/generate_release_truth.py create mode 100644 tests/test_release_truth.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d250271e..60ae97bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,6 +56,19 @@ jobs: name: perf-smoke-results path: artifacts/perf/perf_smoke.json + release-truth: + runs-on: ubuntu-latest + needs: test + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install release-truth dependencies + run: pip install -e ".[dev,mcp]" + - name: Check generated release truth + run: python scripts/generate_release_truth.py --check + eval: runs-on: ubuntu-latest needs: test diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a8519f14..72921133 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,17 +1,4 @@ -name: Publish to PyPI - -# Publishes wheel + sdist to PyPI when a version tag is pushed. -# -# Setup (one-time, by repo owner): -# 1. Go to https://pypi.org/manage/project/memorymaster/settings/publishing/ -# 2. Add a "Trusted Publisher": -# - Owner: wolverin0 -# - Repository: memorymaster -# - Workflow filename: publish.yml -# - Environment name: pypi -# 3. Push a tag matching v*.*.* (e.g. `git tag v3.2.0 && git push --tags`) -# -# No API tokens are stored in GitHub secrets — authentication is OIDC. +name: Publish verified distributions to PyPI on: push: @@ -24,64 +11,100 @@ permissions: jobs: build: - name: Build distributions + name: Build release candidate once runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: fetch-depth: 0 - - - name: Set up Python - uses: actions/setup-python@v5 + - uses: actions/setup-python@v5 with: python-version: '3.12' - - name: Install build tools - run: | - python -m pip install --upgrade pip - pip install build twine - - - name: Verify version matches tag + run: python -m pip install --upgrade pip build twine + - name: Verify tag and package version + if: startsWith(github.ref, 'refs/tags/v') run: | TAG_VERSION="${GITHUB_REF#refs/tags/v}" - PKG_VERSION=$(python -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])") - echo "Tag version: $TAG_VERSION" - echo "Package version: $PKG_VERSION" - if [ "$TAG_VERSION" != "$PKG_VERSION" ]; then - echo "ERROR: tag $TAG_VERSION does not match pyproject.toml $PKG_VERSION" - exit 1 - fi - + PKG_VERSION=$(python -c "import re; print(re.search(r'^version\\s*=\\s*\"([^\"]+)\"', open('pyproject.toml').read(), re.M).group(1))") + test "$TAG_VERSION" = "$PKG_VERSION" - name: Build wheel and sdist run: python -m build - - - name: Validate metadata + - name: Validate distribution metadata run: twine check dist/* - - - name: Upload artifacts + - name: Record immutable distribution hashes + run: cd dist && sha256sum *.whl *.tar.gz > SHA256SUMS + - name: Upload release candidate uses: actions/upload-artifact@v4 with: - name: dist + name: release-candidate-dist path: dist/ + if-no-files-found: error + retention-days: 7 - publish: - name: Publish to PyPI + verify-artifact: + name: Verify the downloaded release candidate needs: build runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - name: Download release candidate + uses: actions/download-artifact@v4 + with: + name: release-candidate-dist + path: dist/ + - name: Verify immutable hashes + run: cd dist && sha256sum --check SHA256SUMS + - name: Install release verification dependencies + run: python -m pip install --upgrade pip && pip install -e ".[dev,mcp,security]" + - name: Block on generated release truth and stable retrieval + run: | + python scripts/generate_release_truth.py --check + pytest tests/test_qrels_regression.py tests/test_release_truth.py tests/test_supply_chain_contracts.py tests/test_setup_profiles.py -q --tb=short + - name: Run strict retrieval evaluation + run: python scripts/eval_memorymaster.py --strict + - name: Clean-wheel import, version, CLI, and MCP smoke + run: | + EXPECTED_VERSION=$(python -c "import re; print(re.search(r'^version\\s*=\\s*\"([^\"]+)\"', open('pyproject.toml').read(), re.M).group(1))") + WHEEL=$(find "$PWD/dist" -maxdepth 1 -name '*.whl' -print -quit) + python -m venv /tmp/memorymaster-release + /tmp/memorymaster-release/bin/python -m pip install "$WHEEL" + cd /tmp + /tmp/memorymaster-release/bin/python -c "import importlib.metadata, memorymaster; assert memorymaster.__version__ == importlib.metadata.version('memorymaster') == '$EXPECTED_VERSION'" + /tmp/memorymaster-release/bin/memorymaster --help >/dev/null + /tmp/memorymaster-release/bin/memorymaster-mcp --help >/dev/null + - name: Promote the verified files without rebuilding + run: | + mkdir verified-dist + cp dist/*.whl dist/*.tar.gz verified-dist/ + - name: Upload verified distributions + uses: actions/upload-artifact@v4 + with: + name: verified-dist + path: verified-dist/ + if-no-files-found: error + retention-days: 7 + + publish: + name: Publish the verified artifact to PyPI + if: startsWith(github.ref, 'refs/tags/v') + needs: verify-artifact + runs-on: ubuntu-latest environment: name: pypi url: https://pypi.org/p/memorymaster permissions: - id-token: write # OIDC for trusted publishing + id-token: write steps: - - name: Download artifacts + - name: Download verified distributions uses: actions/download-artifact@v4 with: - name: dist + name: verified-dist path: dist/ - - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 with: packages-dir: dist/ - # No username/password — OIDC handles auth via the trusted publisher diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index cefc6f9b..d7faa8bd 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -568,3 +568,32 @@ R4.3 governance UX and accessibility completed: The next package is R4.4 generated release and documentation truth. The active roadmap goal authorizes continuing without a new goal. + +R4.4 generated release and documentation truth completed: + +- `pyproject.toml` is the sole package-version value. Runtime `__version__`, + dashboard health, dashboard HTML, supply-chain fixtures, and release docs now + derive installed metadata rather than repeating release literals. +- Deterministic generated truth records 36 MCP tools, 106 main CLI commands, + five operations commands, eight console entrypoints, 4,219 collected tests, + and the executable setup feature/profile matrix. CI blocks when either the + generated Markdown or JSON drifts. +- Root `ROADMAP.md` is the only active roadmap and uses exactly Now, Next, + Later, and Not planned. The three formerly active-looking roadmap names are + short historical pointers; public docs point at the canonical roadmap and + generated MCP inventory instead of repeating stale counts. +- `_probe_mm_installed` imports `find_spec` explicitly. A fresh subprocess + removes the parent `importlib.util` attribute and still proves the probe. +- The tag workflow builds one candidate, records hashes, downloads and verifies + it, blocks on generated truth, stable qrels, strict evaluation, + release/supply-chain tests, and clean-wheel entrypoint smoke, then publishes + only the promoted verified bytes. Manual dispatch cannot publish. Actual + GitHub/PyPI execution is `BLOCKED-EXTERNAL`; no push or publication occurred. +- Six adversarial contracts were witnessed RED before implementation. The + focused R4.4 gate passed 152 tests; workflow YAML parsing, generated-truth + check, changed-file Ruff, syntax compilation, and `git diff --check` passed. +- Atomic package commit: the conventional R4.4 commit containing this evidence. + +The next boundary is Phase 4 convergence and the final local release-candidate +evidence package. The active roadmap goal authorizes continuing without a new +goal. diff --git a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md index 73cc92c0..9419f588 100644 --- a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md +++ b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md @@ -94,15 +94,15 @@ Phase 0 must replace qualitative capacity goals with recorded numeric gates in t - [x] R2.4 — quiet capture, budgets, retention, and backlog control (repository scope; legal review and account-wide cost convergence continue in R3) - [x] R2.5 — remove fake evidence defaults - [x] Phase 2 targeted convergence and audit delta -- [ ] R3.1 — embedding and reconciliation efficiency -- [ ] R3.2 — query and storage efficiency -- [ ] R3.3 — truthful setup profiles -- [ ] R3.4 — service entrypoints and deployment health -- [ ] R3.5 — recovery, observability, audit, and privacy operations +- [x] R3.1 — embedding and reconciliation efficiency +- [x] R3.2 — query and storage efficiency +- [x] R3.3 — truthful setup profiles +- [x] R3.4 — service entrypoints and deployment health +- [x] R3.5 — recovery, observability, audit, and privacy operations - [x] R4.1 — core/extension boundaries - [x] R4.2 — decompose oversized orchestration points - [x] R4.3 — governance UX and accessibility -- [ ] R4.4 — generated release/documentation truth +- [x] R4.4 — generated release/documentation truth - [ ] Final audit delta and convergence evidence ### Phase 0 — Freeze evidence and build the red test matrix diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index 755d31a3..c2c2a8b5 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -22,7 +22,7 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-LIFE-01 | Database/Reliability | High / BAD-PRACTICE | Scheduled archival uses raw SQL and bypasses lifecycle/vector evidence | R2.2 | RESOLVED | P2-C (`732cb68`) routes scheduled archival through canonical optimistic transitions with events, timestamps, cache invalidation, and replayable vector deletion; focused gate: 40 passed. | | MM-REL-02 | Reliability/Performance | Medium / BAD-PRACTICE | MCP reads take write locks and context detail modes retrieve twice | R2.2 | RESOLVED | P2-C makes top-level recall query-only, removes duplicate detail retrieval, and emits at most one sanitized aggregate telemetry envelope; read-only and telemetry contracts passed the focused/convergence matrices. | | MM-OPS-02 | DevOps/Demo | High / BAD-PRACTICE | Docker/Helm publish HTTP on a stdio MCP process and use an invalid healthcheck | R3.4 | RESOLVED | R3.4 keeps stdio MCP private to the agent process and adds distinct authenticated streamable-HTTP MCP and dashboard entrypoints. Docker defaults to the dashboard; Compose is loopback-only, resource-bounded, and probes `/readyz`; Helm selects an explicit authenticated HTTP profile with liveness/readiness probes and resource limits. A locally built image returned 200 from dashboard and MCP HTTP readiness, and the official MCP client completed initialize/tools-list handshakes over stdio and authenticated HTTP. Focused deployment/supply-chain boundary: 97 passed. Helm CLI/disposable cluster evidence remains tracked externally without reopening the repository protocol mismatch. | -| MM-OPS-03 | DevOps/Maintainability | High / BAD-PRACTICE | A release tag can publish without a blocking verified test artifact | R4.4 | OPEN | Deliberately failing release candidate cannot publish | +| MM-OPS-03 | DevOps/Maintainability | High / BAD-PRACTICE | A release tag can publish without a blocking verified test artifact | R4.4 | RESOLVED | The tag workflow builds once, hashes and uploads a release candidate, verifies the downloaded bytes, blocks on generated-truth drift, stable qrels, strict evaluation, supply-chain contracts, and clean-wheel entrypoint smoke, then publishes only the separately promoted `verified-dist` artifact. Static fail-closed workflow contracts pass; an actual GitHub/PyPI run remains BLOCKED-EXTERNAL and no publication occurred. | | MM-OPS-04 | Security/DevOps | High / EXPLOITABLE-LOW-EFFORT | Qdrant/Ollama ports are broadly exposed; images are mutable | R1.5,R3.4 | BLOCKED-EXTERNAL | Repository defaults are complete: loopback-only service and auxiliary ports, required immutable digests, authenticated TLS Qdrant, redirect-safe credentials, explicit authenticated HTTP service profiles, real readiness, resource limits, and fail-closed supply-chain policy. The local MemoryMaster image passed dashboard/MCP HTTP runtime smoke. Approved release-image scans plus authenticated Qdrant/Ollama and disposable Kubernetes runtime evidence remain external. | | MM-PERF-01 | Performance/Cost | High / BAD-PRACTICE | Hybrid reads recompute and rewrite candidate embeddings | R3.1 | RESOLVED | R3.1 migration 0014 persists model/content fingerprints; focused warm-query evidence proves one query embed and zero candidate embeds/writes. | | MM-PERF-02 | Performance | Medium / BAD-PRACTICE | Each process cold-scans corpus token statistics and caches forever | R3.2 | RESOLVED | R3.2 keys corpus and alias statistics by the persisted corpus generation, so warm reads reuse one scan and every claim mutation invalidates it. Focused evidence proves a new cache miss after generation change while tokenizer ranking/retired-status contracts remain green. | @@ -37,13 +37,13 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-COST-02 | Cost | High / EXPLOITABLE-NOW | Verbatim capture reprocesses sessions and retains data indefinitely | R2.4 | RESOLVED | P2-E persists complete-line cursors, processes appended content only, and enforces dry-run 30-day/512-MiB/75,000-session retention bounds. | | MM-COST-03 | Cost/Reliability | Medium / BAD-PRACTICE | Cost and intake controls are fragmented/process-local | R2.4,R3.2 | RESOLVED | R3.2 adds one WAL-backed atomic usage ledger shared by LLM primary/fallback calls, Gemini embedding batches, MCP ingest/checkpoint, and configured core intake quotas. Caps remain backward-compatible/unlimited unless configured; restart/cross-instance and local-bucket-reset tests prove durable global/provider/actor enforcement without consuming capacity on rejected reservations. | | MM-DEMO-01 | Demo/Integrity | High / EXPLOITABLE-NOW | Atlas defaults persist fabricated mock evidence at high confidence | R2.5 | RESOLVED | P2-F (`c66e240`) requires a real configured production provider, confines synthetic providers to explicit dev/test opt-in, excludes synthetic evidence from claims/prompts/actions/citations/exports, and fails configuration nonzero. Focused gate: 111 passed. | -| MM-MAINT-01 | Maintainability/Integrity | Medium / BAD-PRACTICE | Versions, tool counts, roadmaps, and install claims drift | R4.4 | OPEN | Generated single-source values checked by CI | +| MM-MAINT-01 | Maintainability/Integrity | Medium / BAD-PRACTICE | Versions, tool counts, roadmaps, and install claims drift | R4.4 | RESOLVED | `pyproject.toml` is the only package-version value; runtime metadata, health, dashboard HTML, supply-chain fixtures, and docs derive it. The committed/CI-checked release truth reports 36 MCP tools, 106 main CLI commands, 5 ops commands, 8 entrypoints, 4,219 tests, and the setup feature/profile matrix. Root `ROADMAP.md` is the only active Now/Next/Later/Not planned roadmap; three legacy names are short historical pointers. | | MM-MAINT-02 | Architecture/Maintainability | Medium / BAD-PRACTICE | Oversized facades, hardwired extensions, dead plugin seam, and lingering shims | R4.1-R4.2 | RESOLVED | R4.1 removes the zero-consumer generic plugin registry and enforces core-to-companion imports. R4.2 extracts the real integration service plus dashboard read/command layers, reduces `service.py` from 2,617 to 2,205 lines and the handler from 848 to 691, enforces ratcheting budgets, and publishes a dated v5 shim-removal gate. Remaining collaborator extraction is an explicit gradual ratchet, not an unbounded facade regression. | | MM-REL-03 | Reliability | Medium / BAD-PRACTICE | Media retry rows have no expired-lease recovery | R3.5 | RESOLVED | Immutable migration 0016 and SQLite/Postgres parity add bounded owner/expiry leases, atomic expired-work recovery with an audit event, lease clearing on terminal outcomes, and concurrent single-winner coverage. The focused R3.5 gate includes worker-death reclaim, unexpired non-steal, and concurrent claimer tests. | | MM-OPS-05 | Missing Operations | High / BAD-PRACTICE | Recovery defaults stop at same-machine SQLite snapshots | R3.5 | BLOCKED-EXTERNAL | R3.5/Phase 3 convergence adds authenticated encrypted SQLite backups, plaintext/ciphertext checksums, online backup, disposable restore, integrity/FK checks, backup-age health, and measured RPO/RTO (0.046s restore versus 1,800s objective). Approved off-device storage and disposable PostgreSQL dump/restore evidence remain external; same-machine success is not represented as full recovery readiness. | | MM-OBS-01 | Missing Operations | High / BAD-PRACTICE | Failures/metrics are process-local without central alert ownership | R3.5 | RESOLVED | R3.5 adds deterministic backup/retry/integrity/WAL/disk/provider/telemetry checks and can persist one aggregate audit envelope with explicit owner and runbook. Focused tests cover alert severity, persistence, and audit fields. Delivery to an organization-selected OTel/error backend remains environment-specific readiness evidence, not a repository correctness claim. | | MM-DB-01 | Database | Medium / BAD-PRACTICE | Fast schema fingerprint can omit legacy ensure-helper changes | R3.2 | RESOLVED | `schema_stamp()` now hashes baseline schema, immutable migrations, and the complete legacy ensure-helper source. A helper-body mutation regression changes the stamp and forces the full initialization path. New R3.2 DDL exists only in migration 0015/baselines. | -| MM-INTEGRITY-01 | Code Integrity | Medium / BAD-PRACTICE | `importlib.util` probe is wrong in clean Python and its test masks failure | R4.4 | OPEN | Clean subprocess test and explicit import | +| MM-INTEGRITY-01 | Code Integrity | Medium / BAD-PRACTICE | `importlib.util` probe is wrong in clean Python and its test masks failure | R4.4 | RESOLVED | `_probe_mm_installed` explicitly imports `find_spec` from `importlib.util`; a fresh subprocess removes the parent `importlib.util` attribute before calling the probe and still succeeds. | | MM-TEST-01 | Maintainability/Reliability | Medium / BAD-PRACTICE | Validator candidate winner changed under load because mutable `updated_at` controlled processing order | Phase 0/R4.4 | RESOLVED | Deterministic timestamp-inversion regression passes; targeted lifecycle suite 35 passed; full isolated non-ML gate 3,094 passed, 56 skipped, 95 deselected, 24 intentional xfails | ## Phase 1 budget reconciliation (2026-07-12) diff --git a/README.md b/README.md index 79357e02..7621add7 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Lifecycle-managed claims with citations, conflict detection, steward governance, [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) [![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) [![Tests](https://img.shields.io/badge/tests-3200%2B-green.svg)]() -[![MCP Tools](https://img.shields.io/badge/MCP%20tools-36-purple.svg)]() +[![Release truth](https://img.shields.io/badge/release%20truth-generated-purple.svg)](docs/generated/release-truth.md) [![CLI Commands](https://img.shields.io/badge/CLI%20commands-106-orange.svg)]() [![PyPI](https://img.shields.io/pypi/v/memorymaster.svg)](https://pypi.org/project/memorymaster/) @@ -196,7 +196,7 @@ uses a distinct migrator DSN/role; never give that role to the MCP runtime. Unverified host-wide and maintenance tools fail closed. Existing brownfield MCP entries must add the mode or be regenerated with setup `--force`. -30 MCP tools spanning setup/lifecycle, ingest, query/retrieval, listing, knowledge graph, and governance: `init_db`, `ingest_claim`, `ingest_rule`, `query_rules`, `rules_export`, `run_cycle`, `run_steward`, `classify_query`, `query_memory`, `query_for_context`, `query_for_task`, `query_claim_paths`, `query_meta_decisions`, `federated_query`, `recall_analysis`, `read_active_tasks`, `list_claims`, `redact_claim_payload`, `pin_claim`, `compact_memory`, `list_events`, `search_verbatim`, `open_dashboard`, `list_steward_proposals`, `resolve_steward_proposal`, `extract_entities`, `entity_stats`, `find_related_claims`, `quality_scores`, `recompute_tiers`. +MemoryMaster exposes setup/lifecycle, ingest, query/retrieval, listing, knowledge-graph, and governance tools. The [generated release truth](docs/generated/release-truth.md) is the authoritative inventory and count. See [`docs/MCP-TOOLS.md`](docs/MCP-TOOLS.md) for the grouped reference (one line per tool), and [`.mcp.json.example`](.mcp.json.example) for the full config template. @@ -264,7 +264,7 @@ See [CONTRIBUTING.md](CONTRIBUTING.md) for the full workflow. |----------|-------------| | [docs/README.md](docs/README.md) | Documentation index — where to find each living doc | | [docs/handbook.md](docs/handbook.md) | Full operator handbook — hooks, dashboard, steward, dream bridge, troubleshooting, one-prompt install | -| [docs/MCP-TOOLS.md](docs/MCP-TOOLS.md) | Reference for all 30 MCP tools, grouped by purpose | +| [docs/MCP-TOOLS.md](docs/MCP-TOOLS.md) | MCP usage guide; generated inventory and counts are linked from the document | | [docs/INTEGRATING.md](docs/INTEGRATING.md) | Integration guide for embedding MemoryMaster in your agent | | [INSTALLATION.md](INSTALLATION.md) | Setup guide: pip, Docker, Helm, MCP config | | [CONTRIBUTING.md](CONTRIBUTING.md) | Dev setup, testing, PR workflow | diff --git a/ROADMAP-v3.2.md b/ROADMAP-v3.2.md index 10f984db..ccc75cd9 100644 --- a/ROADMAP-v3.2.md +++ b/ROADMAP-v3.2.md @@ -1,51 +1,4 @@ -# MemoryMaster v3.2 Roadmap — MemPalace-Inspired Upgrades +# Historical v3.2 roadmap -Inspired by [MemPalace](https://github.com/milla-jovovich/mempalace) (96.6% LongMemEval). - -## Priority 1: Hooks (capture everything) - -### 1.1 Block-based Stop hook -Replace passive Gemini extraction with MemPalace's approach: count human messages in transcript, every N messages BLOCK Claude from stopping and force a save. Uses `decision: block` + reason as system message. - -**Files:** `~/.claude/hooks/memorymaster-auto-ingest.py` -**Behavior:** Every 15 human messages → block → Claude saves to MemoryMaster → next stop allowed - -### 1.2 PreCompact hook -New hook: fires before context compaction. ALWAYS blocks, forces Claude to save everything before context is lost. Critical safety net. - -**Files:** `~/.claude/hooks/memorymaster-precompact.py`, update `~/.claude/settings.json` - -## Priority 2: Temporal validity on claims - -### 2.1 Add valid_from / valid_until to claims -Claims already have these columns but they're never used. When ingesting, set `valid_from` to now. When a claim is superseded, set `valid_until` on the old one. Query with `as_of` date parameter to get point-in-time truth. - -**Files:** `memorymaster/service.py`, `memorymaster/mcp_server.py`, `memorymaster/storage.py` - -## Priority 3: Duplicate check on ingest - -### 3.1 Content hash dedup -Before inserting a new claim, hash the normalized text and check against existing claims. Skip if duplicate. Faster than waiting for the steward to dedup. - -**Files:** `memorymaster/service.py`, `memorymaster/mcp_server.py` - -## Priority 4: Conversation mining - -### 4.1 Ingest transcripts as claims -Parse Claude Code JSONL transcripts and ingest assistant messages as claims. Similar to MemPalace's `convo_miner.py` but for our DB. - -**Files:** `memorymaster/transcript_miner.py` (new), CLI command `memorymaster mine-transcript` - -## Priority 5: LongMemEval benchmark - -### 5.1 Benchmark runner -Script that loads LongMemEval dataset, ingests sessions as claims, queries MemoryMaster, and outputs in evaluation format. - -**Files:** `benchmarks/longmemeval_runner.py` (new) - -## Not implementing (from MemPalace) - -- **AAAK compression** — experimental, scores 12% worse than raw mode -- **Palace architecture (wings/halls/rooms)** — our scope/wiki structure covers this differently -- **ChromaDB** — we use SQLite FTS5 + Qdrant, no need to add another vector DB -- **Local entity detector** — nice but our Gemini-based extraction is more accurate +This document is retained only as a historical link target. Its proposals are +not active commitments. See the authoritative [MemoryMaster roadmap](ROADMAP.md). diff --git a/ROADMAP.md b/ROADMAP.md index 47ece183..a5dfb68b 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,166 +1,41 @@ -# Roadmap - -## Current R1.3 Qdrant safety status (2026-07-11) - -The checked items below record capabilities that were historically implemented; -they do not all remain enabled. Claim, prompt-context fallback, and verbatim -payload retrieval from Qdrant are quarantined. Local-trusted Qdrant claim -requests use authoritative lexical fallback, verbatim vector/hybrid requests -use FTS5, and direct backend/CLI search fails closed. Team MCP denies semantic -retrieval. Upsert, sync, reconcile, count/ID drift reads, and orphan cleanup -remain available. R2.1 is the future governed path: Qdrant supplies untrusted -IDs, then SQLite/Postgres rehydrates and filters canonical rows through the -shared policy planner. - -## Completed: v2.0.0 (Current) - -### Core Engine (v1.0) -- [x] 6-state lifecycle: candidate → confirmed → stale → superseded → conflicted → archived -- [x] Event log, citations, idempotency keys, human IDs -- [x] Hybrid retrieval (lexical + vector + freshness + confidence ranking) -- [x] Steward governance with multi-probe validators, proposal workflow -- [x] Operator runtime with JSONL streaming, checkpointing -- [x] Connectors: Git, tickets, Slack, email, Jira, GitHub, conversation imports -- [x] MCP server (12 tools for Claude Code / Codex) -- [x] Dashboard with SSE, triage actions, conflict view -- [x] Security: auto-redaction, policy-gated access, Fernet encryption -- [x] Postgres backend with full SQLite parity - -### Semantic Search (v1.1 → merged into v2.0) -- [x] sentence-transformers integration (MiniLM-L6-v2) -- [x] FTS5 virtual table for SQLite full-text search -- [x] Configurable embedding provider (hash, sentence-transformers, Gemini) -- [x] Lazy-load embeddings (64x faster CLI startup) - -### Qdrant Integration (v2.0) -- [x] QdrantBackend with Ollama qwen3-embedding:8b (4096 dims) -- [x] Retry with exponential backoff on embed + upsert -- [x] Batch upsert in sync_all (50x fewer HTTP calls) -- [x] `qdrant-sync` and `qdrant-search` CLI commands implemented (`qdrant-search` is currently fail-closed; sync remains available) -- [x] `retrieval_mode="qdrant"` MCP fast path implemented historically (currently authoritative lexical fallback in local-trusted mode and denied in team mode) -- [x] MCP query default changed to legacy (3min → 0.1s) - -### Performance (v2.0) -- [x] N+1 fix: list_citations_batch + count_citations_batch -- [x] N+1 fix: compact_summaries batch link lookup -- [x] Race condition fix in set_confidence audit trail -- [x] Validator graceful handling of tuple uniqueness conflicts -- [x] URL validation crash fix for malformed IPv6 -- [x] 19 lint issues cleaned, ruff config added -- [x] CLI handler extraction (links, snapshots, qdrant) - -### Data (v2.0) -- [x] 879 claims migrated from MemoryKing Qdrant collection -- [x] 6,889 claims ingested from 17+ project .planning/ and CLAUDE.md files -- [x] 7,400+ total claims, 3,900+ confirmed -- [x] OpenClaw/Otacon sync configured - ---- - -## v2.1.0 — MemoryKing Intelligence Layer - -**Theme: Port MemoryKing's 4 killer features into memorymaster** - -Source: `G:\_OneDrive\OneDrive\Desktop\Py Apps\memoryking\src\memoryking\` - -### Bi-Temporal Timestamps -- [x] Add `valid_from`, `valid_until`, `event_time` columns to claims -- [x] Schema migration for existing DBs -- [x] Time-windowed queries (`query_as_of` method) -- [x] CLI: `--event-time`, `--valid-from`, `--valid-until` on ingest -- [x] MCP: `ingest_claim` accepts temporal params -- [x] CLI: `--as-of` flag for temporal queries on `query` command - -### Automatic Tiering (core / working / peripheral) -- [x] Track `access_count` and `last_accessed` per claim -- [x] Auto-promote: access_count >5 OR age <7d → core -- [x] Auto-demote: access_count 0 AND age >90d → peripheral -- [x] Tiered retrieval: core claims get tier bonus in scoring (+0.15/-0.10) -- [x] CLI: `recompute-tiers` command -- [x] Auto-record access on every query -- [x] MCP: `recompute_tiers` tool -- [x] Expose tier in MCP query results - -### Entity Extraction (LLM-powered) -- [x] Port `EntityExtractor` from memoryking (uses Ollama deepseek-coder) -- [x] Extract entities (people, projects, servers, APIs) from claim text -- [x] Entity graph: SQLite tables for entities, edges, claim links -- [x] Graph BFS traversal: find related claims via entity connections -- [x] CLI: `extract-entities`, `entity-stats` commands -- [x] MCP: `extract_entities`, `entity_stats`, `find_related_claims` tools -- [x] Batch entity extraction across all confirmed claims (extract-entities --limit N) -- [x] Entity-based retrieval integrated into query pipeline (enrich_with_entities) - -### RL Feedback Loop -- [x] FeedbackTracker: records which claims returned per query -- [x] Quality scoring: retrieval_count + access_count + freshness -- [x] Auto-record feedback on every query (wired into service.query_rows) -- [x] CLI: `feedback-stats`, `quality-scores` commands -- [x] MCP: `quality_scores` tool -- [x] sklearn GradientBoostedTree training when 100+ feedback rows (rl_trainer.py) -- [x] Use quality score to influence write policy (quality_scores integration) -- [x] CLI: `train-model` command - -### Query Classification -- [x] 7 query types: fact_lookup, relational, temporal, constraint_check, preference, verification, open_ended -- [x] Rule-based classifier with recommended retrieval mode -- [x] MCP: `classify_query` tool + `auto_classify` param on `query_memory` -- [x] Integrate into CLI `query` command with `--auto-classify` flag - ---- - -## v2.2.0 — External Integrations - -**Theme: Connect to the broader tool ecosystem** - -### Obsidian Vault Export -- [x] `memorymaster export-vault --output ` CLI command -- [x] One `.md` per claim with YAML frontmatter (status, confidence, scope, type) -- [x] `[[mm-xxxx]]` wikilinks from claim_links -- [x] Group by scope into subdirectories -- [x] Index.md with links to all claims -- [x] Incremental export (--incremental flag with .last_export timestamp) -- [x] Compatible with obsidian-skills (agents can read exported vault) - -### GitNexus Bridge -- [x] `scripts/gitnexus_to_claims.py` bridge script -- [x] Run `npx gitnexus analyze` on projects (9 projects analyzed) -- [x] Ingest architectural claims: clusters, execution flows, key symbols -- [x] GitNexus installed globally + configured as MCP peer in `.mcp.json` -- [x] Auto-update on code changes (post-commit-gitnexus.sh + install-gitnexus-hook CLI) - -### OpenClaw Bidirectional Sync -- [x] Cron-based DB sync (scp every 15 min — configured on OpenClaw) -- [x] memorymaster installed on OpenClaw VM -- [x] export-vault for OpenClaw workspace (.md files in ~/.openclaw/workspace/) -- [x] Webhook on ingest for real-time push (webhook.py) -- [x] Conflict resolution for simultaneous edits (optimistic locking with version counter) -- [x] QMD ↔ memorymaster claim mapping (qmd_bridge.py) - ---- - -## v2.3.0 — Multi-Agent Coordination - -**Theme: Multiple agents sharing and curating memory** - -- [x] Claim ownership / agent attribution (source_agent field) -- [x] Visibility: public / private claims per agent -- [x] Conflict resolution policies (LLM-powered auto-resolver with evidence evaluation) -- [x] Cross-agent memory sharing with scope isolation (visibility + RBAC filtering) -- [x] Agent heartbeat and session tracking (session_tracker.py + sessions CLI) -- [x] Access control: admin / writer / reader roles (access_control.py) - ---- - -## v3.0.0 — Autonomous Knowledge Management - -**Theme: Self-organizing, self-healing memory** - -- [x] LLM-powered conflict resolution with human approval gates (auto_resolver.py + existing steward proposals) -- [x] Automatic claim extraction from unstructured text (auto_extractor.py + extract-claims CLI) -- [x] Knowledge graph visualization (Obsidian graph view via export-vault) -- [x] Cross-project memory federation (federated_query in service + MCP) -- [x] Plugin system for custom validators and probes (plugins.py + entry points) -- [x] Progressive disclosure: summary → detail → full context (detail_level param) -- [x] Docker image with multi-stage build (Dockerfile + docker-compose.yml) -- [x] Helm chart for Kubernetes deployment (helm/memorymaster/) +# MemoryMaster roadmap + +This is the only authoritative product roadmap. Historical plans remain in Git +history and `docs/archive/`; they are evidence, not promises. + +## Now + +- Complete the Phase 4 local convergence gate and publish its audit delta, + release-candidate readiness report, blocker ledger, and rollback plan. +- Keep the local release candidate immutable while owners resolve or explicitly + accept the external production blockers in `external-actions-required.md`. +- Preserve governed retrieval, lifecycle authority, tenant/principal isolation, + finite capture budgets, and fail-closed production evidence defaults. + +## Next + +- Obtain disposable authenticated/TLS Qdrant and two-role PostgreSQL evidence. +- Validate the immutable image and Helm profiles in an approved disposable + Kubernetes environment. +- Prove off-device recovery and select an organization telemetry/error backend. +- Complete product-owner/legal decisions for consent, processor disclosure, + retention, export, and erasure before enabling production capture. +- Execute the verified release workflow only after explicit release approval. + +## Later + +- Continue the measured service-facade decomposition without breaking the + compatibility surface. +- Improve entity aliases and steward classification only against versioned, + reproducible evaluation datasets. +- Expand companion integrations through the documented provider protocols and + core-to-companion import boundary. + +## Not planned + +- Automatic live cleanup, compaction, redaction, migration, archival, retention + deletion, or backlog mutation without an explicit operator action. +- Synthetic production evidence, silent provider fallbacks, or direct Qdrant + truth that bypasses authoritative rehydration and governance. +- A second authoritative roadmap, another default vector database, or a + flag-day rewrite of `MemoryService`. diff --git a/docs/MCP-TOOLS.md b/docs/MCP-TOOLS.md index b4882d69..f7d986b5 100644 --- a/docs/MCP-TOOLS.md +++ b/docs/MCP-TOOLS.md @@ -1,6 +1,6 @@ # MCP Tools Reference -MemoryMaster exposes **30 MCP tools** over a FastMCP stdio server (`memorymaster-mcp`). +MemoryMaster exposes a generated set of MCP tools over a FastMCP stdio server (`memorymaster-mcp`). The [generated release truth](generated/release-truth.md) is authoritative for the current inventory and count; the descriptions below are a usage guide. This reference is generated from the tool docstrings in [`memorymaster/surfaces/mcp_server.py`](../memorymaster/surfaces/mcp_server.py) — one line per tool, grouped by purpose. For the MCP server config block see the diff --git a/docs/README.md b/docs/README.md index 176bf8c1..cb75ec9a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -17,10 +17,10 @@ experiment logs live under [`archive/`](archive/) and are not maintained. | Doc | What it covers | |-----|----------------| -| [`MCP-TOOLS.md`](MCP-TOOLS.md) | All 30 MCP tools, grouped by purpose (one line each) | +| [`MCP-TOOLS.md`](MCP-TOOLS.md) | MCP usage guide; authoritative inventory/counts are generated | | [`cli-cookbook.md`](cli-cookbook.md) | CLI recipes for the 104 subcommands | | [`architecture.md`](architecture.md) | Module map, data flow, sensitivity-filter invariants | -| [`ROADMAP.md`](ROADMAP.md) | Release plan and future tracks | +| [`ROADMAP.md`](../ROADMAP.md) | The single authoritative release plan and future tracks | | [`adr/`](adr/) | Architecture Decision Records | ## Topic guides diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index ab63b53f..e88d35b2 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -1,167 +1,4 @@ -# MemoryMaster — Roadmap +# Historical documentation roadmap -**Posture as of v3.18.0 (2026-05-17).** R@5 = 0.972 on LongMemEval-S leads the small -published set (MemPalace 0.966, agentmemory 0.952). The recall race is approaching -diminishing returns. The next two phases harden production-grade claims and then -unlock the governance/MCP differentiators. - -The original v3.18.0 roadmap (governance-first) was incomplete: it assumed the -security/ops baseline was already in place. A second-opinion review by GPT-5.4 -Thinking surfaced real hardening gaps that have to come first. This document -merges both lenses. - -## Phase 0 / v3.19.0 — Hardening (highest urgency) - -These items lock in the "production-grade" claim. Without them, the rest of the -roadmap is selling on a foundation that doesn't actually meet its name. - -### v3.19.0-H1 — LLM budget caps per cycle **[MOST URGENT]** - -**Problem.** No `MEMORYMASTER_MAX_LLM_CALLS_PER_CYCLE` / `MAX_TOKENS_PER_CYCLE` -/ `MAX_PROVIDER_FAILURES_PER_CYCLE` env vars exist. Steward / wiki-absorb / -daydream-ingest can hit ~200 LLM calls per run. Provider stuck in retry-loop -burns quota silently. This session literally burned Claude Pro/Max quota on -the A1 bench with no per-cycle cap. - -**Scope.** Add per-cycle counters in `service.run_cycle`, `wiki_engine`, and -`jobs/daydream_ingest.py`. Enforce hard stops with explicit reason codes. -Emit metrics. Add circuit breaker per provider. - -**Tests.** Simulate quota exhaustion, repeated provider failure, budget exhaustion. -Assert run terminates visibly + reason logged + counters incremented. - -### v3.19.0-H2 — Dashboard auth enforcement audit + fill gaps - -**Problem.** `access_control.py` exists, but coverage is partial. GET routes -(claims, events, observability, SSE) and POST routes (proposals, operator -control) need consistent role enforcement: `viewer` (read) vs `operator` -(mutate). Loopback bind by default; refuse non-loopback without auth secret. -CSRF/origin checks for browser POSTs. - -**Tests.** Anonymous / viewer / operator role matrix. SSE auth. CSRF rejection. -Non-loopback bind refusal without secret. - -### v3.19.0-H3 — Webhook HMAC signing + replay window - -**Problem.** Operator-emitted webhooks send claim data with no signature. -Receivers can't tell trusted from forged. - -**Scope.** `X-MemoryMaster-Signature` (HMAC-SHA-256) + `X-MemoryMaster-Timestamp` -on all outbound webhooks. Configurable secret. Inbound verification helper. -Reject outside replay window. - -**Tests.** Valid signature / invalid signature / altered body / replayed request. - -### v3.19.0-H4 — MCP mutating-tool allowlist for DB/workspace overrides - -**Problem.** MCP tools accept caller-controlled `db` and `workspace` paths, -including mutating tools. A caller can write to any SQLite file the process -can reach. - -**Scope.** Classify tools read-only vs mutating. Remove per-call DB override -on mutating tools, or guard behind explicit admin config. Allowlist-validate -workspace roots. Persist actor/request id on mutations. - -**Tests.** Allow/deny path resolution. Mutating tool with default DB only. -Allowlist matches/misses. - -## Phase 1 / v3.20.0 — Storage discipline - -### v3.20.0-S1 — Versioned migrations + SQLite/Postgres parity gate - -**Problem.** Schema evolution today is opportunistic `ALTER TABLE` with -try/except — no version tracking, no rollback, parity drift risk between -SQLite and Postgres backends. - -**Scope.** Schema version marker table. Forward-apply migration files. -Parity test suite for claims/citations/lifecycle/events/retrieval. CI gate. - -### v3.20.0-S2 — Remove remaining `sqlite3.connect` bypasses (if any) - -**Cross-check 2026-05-17.** Already 0 matches in `mcp_server.py`. Verify other -modules; if all clean, mark this S-item DONE in this commit and skip the PR. - -## Phase 2 / v3.21.0+ — Architecture & differentiator features - -### v3.21.0-A1 — Split the worst oversized modules - -**Cross-check 2026-05-17.** 10 files >800 LOC. Top offenders: - - - `postgres_store.py` 2591 LOC - - `context_hook.py` 2004 LOC - - `steward.py` 1651 LOC - - `dashboard.py` 1491 LOC - - `operator.py` 1453 LOC - - `mcp_server.py` 1439 LOC - - `cli_handlers_basic.py` 1359 LOC - - `service.py` 1357 LOC - -Split by domain workflow, not "helpers.py dumping". Maintain public CLI/MCP -surfaces. Tests stay green. - -### v3.21.0-A2 — Thin `MemoryService` facade into explicit pipelines - -`IngestPipeline`, `RetrievalPlanner`, `LifecycleManager`, `StewardRunner`, -`WikiProjector`. `MemoryService` becomes a stable thin facade. - -### v3.21.0-D1 — Conflict-resolution UI for the dashboard *(governance differentiator)* - -`conflicted` claims today require MCP `pin`/`redact`/`supersede` calls. -A side-by-side conflict view that resolves in one click converts governance -from "exists in code" to "exists in workflow." This is the differentiator -nobody else has. - -### v3.21.0-D2 — Steward observability - -JSONL audit log + CLI viewer. "What did the steward promote, demote, archive -over the last 30 days." Makes the cycle's effect legible. - -### v3.21.0-D3 — Wiki coverage report - -Surface scopes with low article coverage relative to claim volume. - -### v3.21.0-D4 — MCP self-documentation tool - -A `mcp_help` tool that returns the decision tree for which tool to call when -(query_for_context vs query_for_task vs query_memory). Smooths agent onboarding. - -### v3.21.0-D5 — A1 full 500q publication run - -Mechanism shipped in v3.18.0 (#109). Run the overnight bench when budget allows -(now safer once v3.19.0-H1 budget caps are in). Publish first full QA-accuracy -number on LongMemEval-S; update README. - -### v3.21.0-D6 — Silent-except cleanup *(downgraded — cross-check shows 0 `except: pass`)* - -GPT-5.4 review claimed 35 silent excepts. Cross-check 2026-05-17: 0 actual -`except: pass` patterns. The 428 `except` clauses look like proper typed handling. -Defer — audit one module at a time during natural refactors. Don't dedicate a sprint. - -## Where NOT to spend cycles - -- **Chasing R@5 to 0.99.** Marginal benchmark gains don't translate to user-facing - value once we're already at the top of the published set. Same effort spent - on Phase 0 hardening yields more. -- **Re-architecting the retrieval blend.** Linear blend at current weights is - at or near its local optimum (RRF tiebreaker NULL, session-diversity NULL, - Gemini LLM rerank NULL, W_LEX sweep REVERT — all v3.16). -- **Adding a fifth LLM provider** unless an integration explicitly requires it. - The 4 + `claude_cli` OAuth already cover the common case. -- **Marketing the differentiators (Phase 2) before Phase 0 lands.** Auth-less - dashboards and unbounded LLM budgets undermine any "production-grade" claim. - -## Order of work - -1. v3.19.0-H1 (LLM budget caps) ← most urgent; do first -2. v3.19.0-H2 (dashboard auth) -3. v3.19.0-H3 (webhook HMAC) -4. v3.19.0-H4 (MCP mutating allowlist) -5. → ship v3.19.0 -6. v3.20.0-S1 (migrations + parity gate) -7. → ship v3.20.0 -8. v3.21.0-A1 + A2 (module splits, thin MemoryService) -9. v3.21.0-D1 (conflict-resolution UI — first differentiator user can see) -10. v3.21.0-D5 (A1 publication run, now safe with budget caps) -11. v3.21.0-D2/D3/D4 in any order - -Each numbered item is a separate PR. None requires benchmark grinding. +This document is retained only as a historical link target. Its proposals are +not active commitments. See the authoritative [MemoryMaster roadmap](../ROADMAP.md). diff --git a/docs/generated/release-truth.json b/docs/generated/release-truth.json new file mode 100644 index 00000000..ce6ed762 --- /dev/null +++ b/docs/generated/release-truth.json @@ -0,0 +1,233 @@ +{ + "cli_commands": [ + "action-proposals", + "atlas-version", + "auto-resolve", + "bases-generate", + "check-staleness", + "compact", + "compact-summaries", + "context", + "curate-vault", + "daily-note", + "decay", + "dedup", + "detect-contradictions", + "diff", + "drain-spool", + "dream-clean", + "dream-ingest", + "dream-seed", + "dream-sync", + "edit-action-proposal", + "enqueue-media-retry", + "entity-aliases", + "entity-backfill", + "entity-graph-export", + "entity-list", + "entity-merge", + "entity-stats", + "export-actions", + "export-delta", + "export-metrics", + "export-rules", + "export-vault", + "extract-atlas-claims", + "extract-claims", + "extract-entities", + "federated-query", + "feedback-stats", + "ghost-notes", + "history", + "import-whatsapp", + "ingest", + "ingest-daydream", + "init-db", + "install-gitnexus-hook", + "install-hook", + "integrity", + "label-evidence-item", + "label-source-item", + "link", + "links", + "lint-vault", + "list-claims", + "list-events", + "list-media-retries", + "local-search", + "mcp-usage-report", + "merge-db", + "migrate", + "mine-rules", + "mine-transcript", + "observe", + "ocr-source-item", + "pin", + "process-media-retry-queue", + "propose-actions", + "qdrant-reconcile", + "qdrant-search", + "qdrant-sync", + "quality-scores", + "query", + "query-paths", + "ready", + "recall", + "recall-analysis", + "recompute-confidence-priors", + "recompute-tiers", + "record-media-retry-outcome", + "redact-claim", + "repair-fk", + "resolve-action-proposal", + "resolve-conflicts", + "resolve-project", + "resolve-proposal", + "review-queue", + "rollback", + "run-cycle", + "run-daemon", + "run-dashboard", + "run-operator", + "run-steward", + "sessions", + "snapshot", + "snapshots", + "stealth-status", + "steward-proposals", + "train-model", + "transcribe-source-item", + "unlink", + "verbatim-cleanup", + "verify-claims", + "wiki-absorb", + "wiki-backfill-bindings", + "wiki-breakdown", + "wiki-cleanup", + "wiki-freshness", + "wiki-suggest-links" + ], + "console_entrypoints": [ + "memorymaster", + "memorymaster-dashboard", + "memorymaster-mcp", + "memorymaster-mcp-http", + "memorymaster-ops", + "memorymaster-session-end", + "memorymaster-setup", + "memorymaster-steward" + ], + "counts": { + "cli_commands": 106, + "console_entrypoints": 8, + "mcp_tools": 36, + "ops_cli_commands": 5, + "pytest_tests": 4219 + }, + "feature_profile_matrix": { + "capture_hook": [ + "full-lab" + ], + "dashboard": [ + "full-lab" + ], + "db": [ + "full-lab", + "minimal", + "semantic", + "team" + ], + "mcp": [ + "full-lab", + "minimal", + "semantic", + "team" + ], + "provider": [ + "full-lab", + "semantic" + ], + "recall_hook": [ + "full-lab" + ], + "steward": [ + "full-lab" + ], + "vector_backend": [ + "full-lab", + "semantic" + ] + }, + "mcp_tools": [ + "archive_by_source", + "checkpoint", + "classify_query", + "compact_memory", + "entity_stats", + "extract_entities", + "federated_query", + "find_related_claims", + "get_usage_rollup", + "ingest_claim", + "ingest_rule", + "init_db", + "list_claims", + "list_events", + "list_steward_proposals", + "local_search", + "open_dashboard", + "pin_claim", + "quality_scores", + "query_claim_paths", + "query_for_context", + "query_for_task", + "query_memory", + "query_meta_decisions", + "query_rules", + "read_active_tasks", + "recall_analysis", + "recompute_tiers", + "redact_claim_payload", + "resolve_project", + "resolve_steward_proposal", + "rules_export", + "run_cycle", + "run_steward", + "search_verbatim", + "volunteer_context" + ], + "ops_cli_commands": [ + "backup-create", + "health", + "postgres-recovery-plan", + "privacy-plan", + "restore-drill" + ], + "setup_profiles": { + "full-lab": [ + "db", + "mcp", + "recall_hook", + "capture_hook", + "provider", + "steward", + "vector_backend", + "dashboard" + ], + "minimal": [ + "db", + "mcp" + ], + "semantic": [ + "db", + "mcp", + "provider", + "vector_backend" + ], + "team": [ + "db", + "mcp" + ] + }, + "version": "4.4.1" +} diff --git a/docs/generated/release-truth.md b/docs/generated/release-truth.md new file mode 100644 index 00000000..80e9dcca --- /dev/null +++ b/docs/generated/release-truth.md @@ -0,0 +1,31 @@ +# Generated release truth + +Do not edit this file by hand. Run `python scripts/generate_release_truth.py`. + +- Package version: `4.4.1` +- MCP tools: **36** +- Main CLI commands: **106** +- Operations CLI commands: **5** +- Console entrypoints: **8** +- Collected pytest tests: **4219** + +## MCP tools + +`archive_by_source`, `checkpoint`, `classify_query`, `compact_memory`, `entity_stats`, `extract_entities`, `federated_query`, `find_related_claims`, `get_usage_rollup`, `ingest_claim`, `ingest_rule`, `init_db`, `list_claims`, `list_events`, `list_steward_proposals`, `local_search`, `open_dashboard`, `pin_claim`, `quality_scores`, `query_claim_paths`, `query_for_context`, `query_for_task`, `query_memory`, `query_meta_decisions`, `query_rules`, `read_active_tasks`, `recall_analysis`, `recompute_tiers`, `redact_claim_payload`, `resolve_project`, `resolve_steward_proposal`, `rules_export`, `run_cycle`, `run_steward`, `search_verbatim`, `volunteer_context` + +## Setup feature/profile matrix + +| Component | Required by profiles | +|---|---| +| `capture_hook` | `full-lab` | +| `dashboard` | `full-lab` | +| `db` | `full-lab`, `minimal`, `semantic`, `team` | +| `mcp` | `full-lab`, `minimal`, `semantic`, `team` | +| `provider` | `full-lab`, `semantic` | +| `recall_hook` | `full-lab` | +| `steward` | `full-lab` | +| `vector_backend` | `full-lab`, `semantic` | + +## Console entrypoints + +`memorymaster`, `memorymaster-dashboard`, `memorymaster-mcp`, `memorymaster-mcp-http`, `memorymaster-ops`, `memorymaster-session-end`, `memorymaster-setup`, `memorymaster-steward` diff --git a/docs/handbook.md b/docs/handbook.md index 70e64de9..6cfd0143 100644 --- a/docs/handbook.md +++ b/docs/handbook.md @@ -119,7 +119,7 @@ YOU SEND A MESSAGE | **SessionStart hook** | Queries the DB for recent claims, last cycle summary (ingest/validate/decay/supersession counts), pending candidates, and most-recently-updated wiki articles. Scope auto-derived from cwd. | Session startup / resume | | **Auto-ingest hook** | LLM extracts learnings from transcript | Every time Claude stops | | **PreCompact hook** | Forces save to MemoryMaster before Claude Code compacts context (permanent context loss prevention) | Before context compaction | -| **MCP server** (global) | 22 tools for query/ingest/steward | Always available | +| **MCP server** (global) | Generated query/ingest/steward tool set | Always available | | **CLAUDE.md / AGENTS.md append** | Instructions for Claude / Codex to use MemoryMaster | Read at session start | | **Steward cron** | Validates, decays, compacts claims | Every 6 hours | | **Obsidian skills** | Read/write/search vault via CLI | On demand | @@ -424,7 +424,7 @@ MemoryMaster is the memory layer; this stack covers the rest. The Intelligence-F | Priority | Tool | What it adds | How to install | |----------|------|--------------|----------------| | 1 | **graphify** | Pre-computed architecture map (god nodes, communities, surprising connections) in `graphify-out/GRAPH_REPORT.md` | `npm install -g graphify` → `graphify claude install` → `graphify hook install` → `graphify analyze` | -| 2 | **memorymaster** | 22 MCP tools, 7 hooks, wiki, steward | `memorymaster-setup` | +| 2 | **memorymaster** | Generated MCP tool set, hooks, wiki, steward | `memorymaster-setup` | | 3 | **GitNexus** | Symbol-level impact analysis | `npx gitnexus analyze` | | 4 | **Serena** | LSP-powered symbol-level read/edit | Global MCP config, see [oraios/serena](https://github.com/oraios/serena) | | 5 | **context7** | Live library docs | First-party Claude Code MCP, no install | diff --git a/docs/integration/hermes-agent-brief.md b/docs/integration/hermes-agent-brief.md index a7da3483..d49b55c0 100644 --- a/docs/integration/hermes-agent-brief.md +++ b/docs/integration/hermes-agent-brief.md @@ -33,7 +33,7 @@ memorymaster — OR a better integration if your runtime supports it. | Local on Windows | `G:\_OneDrive\OneDrive\Desktop\Py Apps\memorymaster\memorymaster.db` | | Python package | `pip install "memorymaster[mcp,qdrant,security]"` (Python 3.10+) | | CLI | `python -m memorymaster --help` (or `memorymaster --help` if installed as script) | -| MCP server | FastMCP stdio: `python -m memorymaster.mcp_server` — 21 tools (query, ingest, run_cycle, etc.) | +| MCP server | FastMCP stdio: `python -m memorymaster.mcp_server`; see `docs/generated/release-truth.md` for the generated inventory | | Dashboard (HTTP) | `python -m memorymaster --db run-dashboard --port 8765` | | Webhook | Outbound only — fires `MEMORYMASTER_WEBHOOK_URL` on claim events | @@ -44,7 +44,7 @@ memorymaster — OR a better integration if your runtime supports it. - **v3.20.0-S1** — versioned migrations framework (`memorymaster/migrations/`, `MigrationRunner`, `schema_versions` table, drift detection) Full env-var reference in `CHANGELOG.md`. ROADMAP and remaining backlog in -`docs/ROADMAP.md` + `docs/archive/v320-backlog.md`. +`ROADMAP.md` at the repository root. Files under `docs/archive/` are historical evidence only. --- diff --git a/docs/security_supply_chain.md b/docs/security_supply_chain.md index 04bf10e1..b7b2eadb 100644 --- a/docs/security_supply_chain.md +++ b/docs/security_supply_chain.md @@ -24,9 +24,10 @@ mutable image tags, and invalid or mismatched SBOMs all fail the gate. This mode performs no scanner, network, registry, Docker, or artifact read: ```powershell +$Version = python -c "import importlib.metadata as m; print(m.version('memorymaster'))" python scripts/run_supply_chain_checks.py ` - --release-artifact artifacts/memorymaster-4.4.1-py3-none-any.whl ` - --sbom artifacts/memorymaster-4.4.1.cdx.json ` + --release-artifact artifacts/memorymaster-$Version-py3-none-any.whl ` + --sbom artifacts/memorymaster-$Version.cdx.json ` --local-image sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ` --command-plan ``` @@ -52,9 +53,10 @@ $imageId = docker image inspect --format '{{.Id}}' memorymaster:phase1 Run the gate from the same checkout that owns the runner: ```powershell +$Version = python -c "import importlib.metadata as m; print(m.version('memorymaster'))" python scripts/run_supply_chain_checks.py ` - --release-artifact artifacts/memorymaster-4.4.1-py3-none-any.whl ` - --sbom artifacts/memorymaster-4.4.1.cdx.json ` + --release-artifact artifacts/memorymaster-$Version-py3-none-any.whl ` + --sbom artifacts/memorymaster-$Version.cdx.json ` --local-image $imageId ``` diff --git a/external-actions-required.md b/external-actions-required.md index a81463e0..98553aa7 100644 --- a/external-actions-required.md +++ b/external-actions-required.md @@ -16,6 +16,7 @@ live mutation by itself. | MM-OPS-04 / history scan | Security owner | Classify the unsuppressed Gitleaks history result: 40 potential findings across 10 commits and 7 files. Rotate any affected credentials before considering an approved history action. | Aggregate classification and rotation record; no raw secret material in general logs | 2026-07-31 | BLOCKED-EXTERNAL | | MM-OPS-04 / dependency audit | Release owner / advisory service | Rerun strict project and `mcp,qdrant,security` extra audits in an approved lock/resolver environment; both final attempts exceeded the 15-minute external-service cap and were terminated without a pass claim. | Successful strict PyPI audit bound to the release lock/artifact, or reviewed vulnerability remediation evidence | 2026-07-31 | BLOCKED-EXTERNAL | | MM-OPS-04 / image scan | Release owner / local image registry | Provide approved immutable local MemoryMaster, Qdrant, and Ollama image IDs. No relevant local images existed and this goal forbade builds or pulls solely for scanning. | Docker Scout high/critical scan results for the exact immutable release images, including base findings | 2026-07-31 | BLOCKED-EXTERNAL | +| MM-OPS-03 / publication proof | Release owner / GitHub / PyPI | After every release blocker is resolved or formally accepted, explicitly approve one tag-based execution of the verified-artifact workflow. Do not use manual dispatch to publish. | GitHub job evidence showing candidate hash verification, blocking release gates, clean-wheel smoke, promotion without rebuild, and PyPI OIDC publication of the exact `verified-dist` bytes | 2026-08-15 | BLOCKED-EXTERNAL | | MM-DATA-01 | Live MemoryMaster operator | Approve a consistent backup/restore drill before any live migration, redaction, backlog, or retention operation. | Restored backup, integrity check, counts/checksums, approval record | 2026-08-15 | BLOCKED-EXTERNAL | | MM-CAP-01 | Host storage operator | Address/monitor the drive at 85.82% used without deleting MemoryMaster data under this goal. | Daily disk telemetry below the critical gate or approved capacity expansion | 2026-07-18 | BLOCKED-EXTERNAL | | MM-PRIV-01 | Product owner / legal reviewer | Decide intended organizational use, jurisdictions, processor disclosures, and retention commitments before compliance claims. | Approved privacy/data-processing statement | 2026-08-31 | BLOCKED-EXTERNAL | diff --git a/memorymaster/__init__.py b/memorymaster/__init__.py index 6f811875..fea09c0d 100644 --- a/memorymaster/__init__.py +++ b/memorymaster/__init__.py @@ -1,5 +1,10 @@ -"""Memory reliability MVP package.""" +"""MemoryMaster package metadata.""" + +from importlib.metadata import PackageNotFoundError, version __all__ = ["__version__"] -__version__ = "4.0.0" +try: + __version__ = version("memorymaster") +except PackageNotFoundError: # pragma: no cover - source tree without installation metadata + __version__ = "0+unknown" diff --git a/memorymaster/surfaces/dashboard.py b/memorymaster/surfaces/dashboard.py index 32813514..b709204d 100644 --- a/memorymaster/surfaces/dashboard.py +++ b/memorymaster/surfaces/dashboard.py @@ -3,7 +3,6 @@ import argparse from datetime import datetime, timedelta, timezone from html import escape -import importlib.metadata import json import os import subprocess @@ -586,15 +585,10 @@ def _extract_claims_from_rows(rows_data: Any) -> tuple[list[Any], dict[int, dict return claims, scored_by_claim_id -def _package_version() -> str | None: - try: - return importlib.metadata.version("memorymaster") - except importlib.metadata.PackageNotFoundError: - try: - from memorymaster import __version__ - except Exception: - return None - return str(__version__) +def _package_version() -> str: + from memorymaster import __version__ + + return __version__ def _check_dashboard_db(service: Any) -> dict[str, Any]: @@ -993,7 +987,7 @@ def _write_dashboard(self) -> None:

MemoryMaster

AI Agent Memory Management Dashboard
-v1.0.0 +v__MEMORYMASTER_VERSION__
@@ -1110,6 +1104,7 @@ def _write_dashboard(self) -> None: jget('/api/claims?limit=50').then(fillClaims).catch(e=>showPanelFailure('claims-body','claims',e,6));jget('/api/timeline?limit=40').then(fillTimeline).catch(e=>showPanelFailure('timeline-list','timeline',e));refreshConflicts().catch(e=>showPanelFailure('conflicts-cards','conflicts',e));refreshQueue().catch(e=>showPanelFailure('stale-body','review queue',e,5));jget('/api/audit?limit=40').then(fillAudit).catch(e=>showPanelFailure('audit-body','audit log',e,4));jget('/api/namespaces?limit=200').then(fillNs).catch(e=>showPanelFailure('namespaces-box','namespaces',e));jget('/api/provenance').then(fillProvenance).catch(e=>showPanelFailure('provenance-body','provenance',e,8));jget('/api/session-stats?limit=2000').then(fillStats).catch(e=>showPanelFailure('session-stats','session statistics',e));jget('/metrics/validation-latency').then(fillValidationLatency).catch(e=>showPanelFailure('validation-latency','validation latency',e));jget('/api/integrity').then(fillIntegrity).catch(e=>showPanelFailure('integrity-box','integrity',e));jget('/api/operator/status').then(fillOp).catch(e=>showPanelFailure('op-status','operator status',e));refreshObs().catch(e=>showPanelFailure('obs-box','observability',e)); const sb=document.getElementById('stream');const es=new EventSource('/api/operator/stream?last=20'); const append=(t)=>{const ex=sb.textContent.trim();sb.textContent=(ex&&ex!=='Waiting for operator to start...'?ex+'\\n':'')+t;}; ['message','stream_start','state_loaded','state_error','state_saved','json_error','turn_processed','reconcile_run','stream_exit'].forEach(n=>es.addEventListener(n,(ev)=>append(ev.data))); es.onerror=()=>append('[stream reconnecting]'); """ + html = html.replace("__MEMORYMASTER_VERSION__", escape(_package_version())) body = html.encode("utf-8") self.send_response(HTTPStatus.OK) self.send_header("Content-Type", "text/html; charset=utf-8") diff --git a/memorymaster/surfaces/setup_detect.py b/memorymaster/surfaces/setup_detect.py index 307a2751..e5203fad 100644 --- a/memorymaster/surfaces/setup_detect.py +++ b/memorymaster/surfaces/setup_detect.py @@ -165,9 +165,9 @@ def _probe_codex(home: Path) -> bool: def _probe_mm_installed() -> bool: try: - import importlib + from importlib.util import find_spec - spec = importlib.util.find_spec("memorymaster") + spec = find_spec("memorymaster") return spec is not None except Exception: # noqa: BLE001 return False diff --git a/roadmapres.md b/roadmapres.md index 99d02bb5..97f09dcb 100644 --- a/roadmapres.md +++ b/roadmapres.md @@ -1,77 +1,4 @@ -# MemoryMaster — Remaining Roadmap (2026-04-23) +# Historical April 2026 roadmap -**Current main:** `5285edd` (all P0/P1/P2 work shipped this session). -**Remaining:** 2 architectural implementations (specs already landed) + 1 batch task waiting on user input. - ---- - -## #127 — NER entity extraction at ingest (Wave 3) - -**Spec:** `artifacts/spec-entity-extraction-at-ingest-2026-04-23.md` -**Why it matters:** `avg_aliases_per_entity = 1.033`. The registry is decorative. Raising it to ≥2.0 makes typed relationships actually work and fixes the #127 complaint that "every entity has degree 1". -**Files touched:** new `memorymaster/entity_extractor.py`, patches to `memorymaster/service.py`, new fixture + script + test. -**Estimate:** ~1 day (Layer 1 only), +1 day (Layer 2 LLM, optional). - -### Sub-tasks (sequential within, no hidden dependencies on #129) - -| # | Task | Deliverable | Acceptance | -|---|---|---|---| -| 127.1 | Build labeled eval fixture | `tests/fixtures/entity_extraction_eval.jsonl` — 100 claims with `{text, expected_entities: [...]}` | ≥5 examples per entity-kind (file, env-var, service, port, commit, tool) | -| 127.2 | Layer-1 extractor module | `memorymaster/entity_extractor.py::extract_patterns(text) -> list[Entity]` | Unit tests: recall ≥0.9 + FPR ≤0.1 per class on fixture | -| 127.3 | Service integration | Patch `service.py::MemoryService.ingest` to register extracted entities post-`resolve_or_create` | All existing ingest tests pass | -| 127.4 | Backfill script | `scripts/backfill_entity_extraction.py --dry-run / --apply` | Idempotent: re-run = 0 new rows | -| 127.5 | Run live backfill + measure | `artifacts/entity-extraction-wave3-2026-04-23.md` with before/after metrics | `avg_aliases_per_entity ≥ 2.0` | -| 127.6 | (Optional) Layer 2 LLM | `extract_llm(text)` + `MEMORYMASTER_ENTITY_LLM=1` gate | `avg_aliases ≥ 2.5` | - ---- - -## #129 — Calibrated steward classifier - -**Spec:** `artifacts/spec-steward-classifier-2026-04-23.md` -**Why it matters:** Pareto sweep showed the additive `validation_score` ceiling is 49% recall @ 1% FPR — tuning thresholds can't fix this. Need a feature-weighted model. -**Files touched:** new `memorymaster/steward_features.py`, new `memorymaster/steward_classifier.py`, patches to `memorymaster/steward.py`, new training + eval scripts. -**Estimate:** ~2 days. - -### Sub-tasks - -| # | Task | Deliverable | Acceptance | -|---|---|---|---| -| 129.1 | Feature extractor | `memorymaster/steward_features.py::extract_features(claim) -> dict` with 9 features from spec | Unit tests: each feature pulls correctly from fixture claims | -| 129.2 | Training set builder | `scripts/build_steward_training_set.py` → `tests/fixtures/steward_training.jsonl` | Excludes migration-origin archives; ~2,900 pos + 3× neg | -| 129.3 | Training script | `scripts/train_steward_classifier.py` — chronological 80/20, LogisticRegression + isotonic CV | Outputs `artifacts/steward-classifier-v1.joblib` + eval report | -| 129.4 | Steward integration | Patch `steward.py::_decide_promotion` to call classifier with fallback to current additive formula when artifact missing | `rm artifact && run_cycle` doesn't crash | -| 129.5 | Regression test | `tests/test_steward_classifier.py` — asserts recall ≥0.70 @ FPR ≤0.05 on held-out split | CI gate green | -| 129.6 | Ablation writeup | `artifacts/steward-classifier-eval-2026-04-23.md` — per-feature ablation + calibration curve | Document signed off | - ---- - -## #75 — Run graphify on 15+ legacy projects - -**Status:** blocked on user input. /graphify is an interactive Claude skill, not a batch. I need a concrete list of which ≥15 projects under `G:\_OneDrive\OneDrive\Desktop\Py Apps\` to run it against, AND whether to run them sequentially in this session or hand off to a dedicated graphify session (recommended — graphify sessions produce a lot of intermediate artifacts and are best kept isolated). - -Suggested criteria if you want me to pick: - -1. Active projects without `graphify-out/GRAPH_REPORT.md` in repo root -2. Projects ≥500 files but ≤10k files (graphify quality peaks in this range) -3. Exclude anything under `archive/` or clearly-deprecated folders - -If you say "go, pick any 15", I will enumerate and schedule. - ---- - -## Parallelization plan (this session) - -- **Agent A** — implement #127.1 → #127.5 in isolated worktree. Commits on branch `omni/feat-entity-extraction-2026-04-23`. Layer 2 deferred. -- **Agent B** — implement #129.1 → #129.5 in isolated worktree. Commits on branch `omni/feat-steward-classifier-2026-04-23`. Ablation writeup deferred. -- Coordinator role: qa-verifier. I (main session) will NOT spawn a third and instead monitor, merge each agent's branch when it reports metric-acceptance, and cross-cut any integration issues. - -Cap at 2 concurrent agents (per claim 11761; 4-way fanout caused 3/4 rate-limits in the prior session). - ---- - -## Definition of done (both arcs) - -1. Acceptance metric hit, recorded in the respective artifacts file. -2. Full test suite (`python -m pytest tests/ -q`) still passes. -3. Branch merged to main through the commit-guard-safe path (branch → push → ff-merge main). -4. A `claim_type=architecture` ingest summarizing what shipped + the measured before/after. +This document is retained only as a historical link target. Its proposals are +not active commitments. See the authoritative [MemoryMaster roadmap](ROADMAP.md). diff --git a/scripts/generate_release_truth.py b/scripts/generate_release_truth.py new file mode 100644 index 00000000..1137a200 --- /dev/null +++ b/scripts/generate_release_truth.py @@ -0,0 +1,143 @@ +"""Generate deterministic release facts from executable project sources.""" + +from __future__ import annotations + +import argparse +import importlib.metadata +import json +from pathlib import Path +import re +import subprocess +import sys +from typing import Any + + +ROOT = Path(__file__).resolve().parents[1] +if str(ROOT) not in sys.path: + sys.path.insert(0, str(ROOT)) + + +def _project_version() -> str: + text = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + match = re.search(r'^version\s*=\s*"([^"]+)"', text, re.MULTILINE) + if match is None: + raise RuntimeError("project.version is missing from pyproject.toml") + return match.group(1) + + +def _command_names(parser: Any) -> list[str]: + import argparse as argparse_module + + actions = (action for action in parser._actions if isinstance(action, argparse_module._SubParsersAction)) + subparsers = next(actions, None) + return sorted(subparsers.choices) if subparsers is not None else [] + + +def _pytest_count() -> int: + result = subprocess.run( + [sys.executable, "-m", "pytest", "tests", "--collect-only", "-q"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + timeout=240, + ) + output = f"{result.stdout}\n{result.stderr}" + matches = re.findall(r"(\d+) tests? collected", output) + if result.returncode != 0 or not matches: + raise RuntimeError(f"pytest collection failed ({result.returncode}): {output[-1200:]}") + return int(matches[-1]) + + +def _release_truth() -> dict[str, Any]: + import memorymaster + from memorymaster.surfaces import mcp_server + from memorymaster.surfaces.cli import build_parser + from memorymaster.surfaces.operations import _build_parser as build_ops_parser + from memorymaster.surfaces.setup_hooks import SETUP_PROFILES + + source_version = _project_version() + installed_version = importlib.metadata.version("memorymaster") + if {source_version, installed_version, memorymaster.__version__} != {source_version}: + raise RuntimeError("source, installed, and runtime package versions differ") + profiles = {name: list(components) for name, components in sorted(SETUP_PROFILES.items())} + components = sorted({component for values in profiles.values() for component in values}) + distribution = importlib.metadata.distribution("memorymaster") + entrypoints = sorted(item.name for item in distribution.entry_points if item.group == "console_scripts") + return { + "version": source_version, + "counts": { + "mcp_tools": len(mcp_server.mcp._tool_manager.list_tools()), + "cli_commands": len(_command_names(build_parser())), + "ops_cli_commands": len(_command_names(build_ops_parser())), + "console_entrypoints": len(entrypoints), + "pytest_tests": _pytest_count(), + }, + "mcp_tools": sorted(tool.name for tool in mcp_server.mcp._tool_manager.list_tools()), + "cli_commands": _command_names(build_parser()), + "ops_cli_commands": _command_names(build_ops_parser()), + "console_entrypoints": entrypoints, + "setup_profiles": profiles, + "feature_profile_matrix": { + component: [name for name, values in profiles.items() if component in values] + for component in components + }, + } + + +def _markdown(data: dict[str, Any]) -> str: + counts = data["counts"] + lines = [ + "# Generated release truth", + "", + "Do not edit this file by hand. Run `python scripts/generate_release_truth.py`.", + "", + f"- Package version: `{data['version']}`", + f"- MCP tools: **{counts['mcp_tools']}**", + f"- Main CLI commands: **{counts['cli_commands']}**", + f"- Operations CLI commands: **{counts['ops_cli_commands']}**", + f"- Console entrypoints: **{counts['console_entrypoints']}**", + f"- Collected pytest tests: **{counts['pytest_tests']}**", + "", + "## MCP tools", + "", + ", ".join(f"`{name}`" for name in data["mcp_tools"]), + "", + "## Setup feature/profile matrix", + "", + "| Component | Required by profiles |", + "|---|---|", + ] + for component, profiles in data["feature_profile_matrix"].items(): + lines.append(f"| `{component}` | {', '.join(f'`{name}`' for name in profiles)} |") + lines.extend(["", "## Console entrypoints", "", ", ".join(f"`{name}`" for name in data["console_entrypoints"]), ""]) + return "\n".join(lines) + + +def _write_or_check(path: Path, content: str, check: bool) -> bool: + if check: + return path.is_file() and path.read_text(encoding="utf-8") == content + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return True + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--check", action="store_true", help="Fail if committed generated truth has drifted.") + args = parser.parse_args() + data = _release_truth() + outputs = { + ROOT / "docs/generated/release-truth.json": json.dumps(data, indent=2, sort_keys=True) + "\n", + ROOT / "docs/generated/release-truth.md": _markdown(data), + } + drifted = [str(path.relative_to(ROOT)) for path, content in outputs.items() if not _write_or_check(path, content, args.check)] + if drifted: + print("Generated release truth is stale: " + ", ".join(drifted), file=sys.stderr) + return 1 + print("Release truth verified." if args.check else "Release truth generated.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_release_truth.py b/tests/test_release_truth.py new file mode 100644 index 00000000..8f34634b --- /dev/null +++ b/tests/test_release_truth.py @@ -0,0 +1,87 @@ +"""R4.4 single-source release and publication contracts.""" + +from __future__ import annotations + +from io import BytesIO +from pathlib import Path +import re +import subprocess +import sys + +import memorymaster +from memorymaster.surfaces.dashboard import DashboardRequestHandler + + +ROOT = Path(__file__).resolve().parents[1] + + +def _project_version() -> str: + text = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + match = re.search(r'^version\s*=\s*"([^"]+)"', text, re.MULTILINE) + assert match is not None + return match.group(1) + + +def _dashboard_html() -> str: + handler = DashboardRequestHandler.__new__(DashboardRequestHandler) + handler.wfile = BytesIO() + handler.send_response = lambda *_args, **_kwargs: None + handler.send_header = lambda *_args, **_kwargs: None + handler.end_headers = lambda *_args, **_kwargs: None + handler._write_dashboard() + return handler.wfile.getvalue().decode("utf-8") + + +def test_package_and_dashboard_derive_the_pyproject_version() -> None: + version = _project_version() + assert memorymaster.__version__ == version + assert f'v{version}' in _dashboard_html() + + +def test_install_probe_uses_an_explicit_importlib_util_import() -> None: + source = (ROOT / "memorymaster/surfaces/setup_detect.py").read_text(encoding="utf-8") + assert "from importlib.util import find_spec" in source + assert 'find_spec("memorymaster")' in source + + +def test_install_probe_survives_a_clean_process_without_importlib_util_attribute() -> None: + code = ( + "from memorymaster.surfaces import setup_detect; " + "import importlib; " + "delattr(importlib, 'util'); " + "raise SystemExit(0 if setup_detect._probe_mm_installed() else 1)" + ) + result = subprocess.run([sys.executable, "-c", code], cwd=ROOT, check=False) + assert result.returncode == 0 + + +def test_generated_release_truth_is_committed_and_current() -> None: + script = ROOT / "scripts/generate_release_truth.py" + generated = ROOT / "docs/generated/release-truth.md" + assert script.is_file() + assert generated.is_file() + + +def test_only_root_roadmap_is_authoritative() -> None: + roadmap = (ROOT / "ROADMAP.md").read_text(encoding="utf-8") + for heading in ("## Now", "## Next", "## Later", "## Not planned"): + assert heading in roadmap + for relative in ("ROADMAP-v3.2.md", "roadmapres.md", "docs/ROADMAP.md"): + text = (ROOT / relative).read_text(encoding="utf-8") + assert "historical" in text.lower() + assert "ROADMAP.md" in text + assert len(text.splitlines()) <= 12 + + +def test_publish_requires_the_verified_downloaded_artifact() -> None: + workflow = (ROOT / ".github/workflows/publish.yml").read_text(encoding="utf-8") + assert "verify-artifact:" in workflow + assert "needs: verify-artifact" in workflow + assert workflow.count("name: verified-dist") == 2 + assert "tests/test_qrels_regression.py" in workflow + assert "tests/test_release_truth.py" in workflow + + +def test_ci_blocks_on_generated_release_truth_drift() -> None: + workflow = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8") + assert "generate_release_truth.py --check" in workflow diff --git a/tests/test_supply_chain_contracts.py b/tests/test_supply_chain_contracts.py index c16c2f69..746a1d5e 100644 --- a/tests/test_supply_chain_contracts.py +++ b/tests/test_supply_chain_contracts.py @@ -3,6 +3,7 @@ import hashlib import json import io +import importlib.metadata import subprocess import sys import tempfile @@ -17,7 +18,7 @@ EXPECTED_NAME = "memorymaster" -EXPECTED_VERSION = "4.4.1" +EXPECTED_VERSION = importlib.metadata.version(EXPECTED_NAME) IMAGE_ID = "sha256:" + ("a" * 64) @@ -47,11 +48,11 @@ def _valid_sbom(artifact_sha256: str = "b" * 64) -> dict[str, object]: def _wheel(tmp_path: Path) -> Path: - path = tmp_path / "memorymaster-4.4.1-py3-none-any.whl" + path = tmp_path / f"memorymaster-{EXPECTED_VERSION}-py3-none-any.whl" with zipfile.ZipFile(path, "w") as archive: archive.writestr( - "memorymaster-4.4.1.dist-info/METADATA", - "Metadata-Version: 2.1\nName: memorymaster\nVersion: 4.4.1\n", + f"memorymaster-{EXPECTED_VERSION}.dist-info/METADATA", + f"Metadata-Version: 2.1\nName: memorymaster\nVersion: {EXPECTED_VERSION}\n", ) archive.writestr("memorymaster/__init__.py", "") return path From 090adff15e2b5bdea4d93cb0233ec1080f2a476c Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Tue, 14 Jul 2026 02:12:36 -0300 Subject: [PATCH 47/55] test: converge phase 4 release candidate --- .planning/REMAINING-PHASES-V1-BUDGET.md | 39 ++++++++++ ...EMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md | 2 +- .planning/audit-remediation-ledger.md | 8 +- .../audit-delta.md | 75 +++++++++++++++++++ .../pr-release-draft.md | 31 ++++++++ .../production-release-approval-request.md | 35 +++++++++ .../release-candidate-readiness.md | 42 +++++++++++ .../remaining-blockers.md | 18 +++++ .../rollback-plan.md | 29 +++++++ docs/generated/release-truth.json | 2 +- docs/generated/release-truth.md | 2 +- external-actions-required.md | 2 +- memorymaster/surfaces/dashboard.py | 7 +- memorymaster/surfaces/dashboard_commands.py | 9 ++- tests/test_dashboard_coverage.py | 8 +- tests/test_dashboard_governance_ux.py | 45 ++++++++++- tests/test_steward_resolution_parity.py | 7 +- 17 files changed, 339 insertions(+), 22 deletions(-) create mode 100644 .planning/audits/2026-07-14-phase4-budget-delta/audit-delta.md create mode 100644 .planning/audits/2026-07-14-phase4-budget-delta/pr-release-draft.md create mode 100644 .planning/audits/2026-07-14-phase4-budget-delta/production-release-approval-request.md create mode 100644 .planning/audits/2026-07-14-phase4-budget-delta/release-candidate-readiness.md create mode 100644 .planning/audits/2026-07-14-phase4-budget-delta/remaining-blockers.md create mode 100644 .planning/audits/2026-07-14-phase4-budget-delta/rollback-plan.md diff --git a/.planning/REMAINING-PHASES-V1-BUDGET.md b/.planning/REMAINING-PHASES-V1-BUDGET.md index d7faa8bd..4c0024b9 100644 --- a/.planning/REMAINING-PHASES-V1-BUDGET.md +++ b/.planning/REMAINING-PHASES-V1-BUDGET.md @@ -597,3 +597,42 @@ R4.4 generated release and documentation truth completed: The next boundary is Phase 4 convergence and the final local release-candidate evidence package. The active roadmap goal authorizes continuing without a new goal. + +Phase 4 convergence completed on 2026-07-14: + +- The independent same-scope review found one reproducible High governance + race: the dashboard displayed one steward proposal but submitted only its + claim ID, allowing a newer proposal to be resolved at click time. Two + adversarial tests were witnessed RED. The single bounded correction now + binds approve/reject to the exact displayed positive `proposal_event_id` and + fails closed for missing, malformed, or stale IDs. The correction boundary + passed 54 tests, including headless Chromium governance/mobile/error cases; + the final independent strict-ID re-review passed 18 focused dashboard tests. +- The once-per-phase full non-ML gate passed 4,053 tests with 70 skips, 95 + deselections, one expected xfail, and two warnings in 919.40s. The isolated + required ML gate passed 95 tests with 4,126 deselections and one warning in + 18.45s. Project Ruff, changed-file Ruff, Python compilation, inline + JavaScript parsing, generated-truth drift, and `git diff --check` passed. +- Generated truth converged at 4,221 tests, 36 MCP tools, 106 main CLI + commands, five operations commands, and eight console entrypoints. +- The local 4.4.1 wheel and sdist built and hash-verified. An isolated venv + imported MemoryMaster from its own `site-packages` and passed main CLI and + MCP entrypoint smoke. Compose rendered with required fail-closed inputs. A + current local image initialized a fresh disposable DB, returned health `ok` + and readiness `ready`, reported runtime 4.4.1, and started the MCP entrypoint. +- The latest audit has zero new findings and no unresolved reproducible + Critical/High regression introduced by Phase 4. The canonical ledger is 25 + resolved and eight validly `BLOCKED-EXTERNAL`; release-oriented ordering is + recorded in the Phase 4 delta directory. +- The local RC is ready for review but is not approved for production or + external publication. Authenticated/TLS Qdrant, two-role Postgres, + credential/history/dependency/image scans, immutable-image Kubernetes, + off-device/Postgres recovery, legal/privacy decisions, and actual tag/PyPI + evidence remain external. +- Final artifacts are under + `.planning/audits/2026-07-14-phase4-budget-delta/`: audit delta, local RC + readiness, prioritized blockers, rollback plan, PR/release draft, and a + ready-to-paste external release approval request. +- Atomic convergence commit: the conventional Phase 4 convergence commit + containing this evidence. No push, merge, tag, publication, deployment, or + live/external data mutation occurred. diff --git a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md index 9419f588..3ce821f3 100644 --- a/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md +++ b/.planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md @@ -103,7 +103,7 @@ Phase 0 must replace qualitative capacity goals with recorded numeric gates in t - [x] R4.2 — decompose oversized orchestration points - [x] R4.3 — governance UX and accessibility - [x] R4.4 — generated release/documentation truth -- [ ] Final audit delta and convergence evidence +- [x] Final audit delta and convergence evidence ### Phase 0 — Freeze evidence and build the red test matrix diff --git a/.planning/audit-remediation-ledger.md b/.planning/audit-remediation-ledger.md index c2c2a8b5..943f29e0 100644 --- a/.planning/audit-remediation-ledger.md +++ b/.planning/audit-remediation-ledger.md @@ -22,22 +22,22 @@ Statuses: `OPEN`, `IN-PROGRESS`, `RESOLVED`, `BLOCKED-EXTERNAL`, `BLOCKED-POLICY | MM-LIFE-01 | Database/Reliability | High / BAD-PRACTICE | Scheduled archival uses raw SQL and bypasses lifecycle/vector evidence | R2.2 | RESOLVED | P2-C (`732cb68`) routes scheduled archival through canonical optimistic transitions with events, timestamps, cache invalidation, and replayable vector deletion; focused gate: 40 passed. | | MM-REL-02 | Reliability/Performance | Medium / BAD-PRACTICE | MCP reads take write locks and context detail modes retrieve twice | R2.2 | RESOLVED | P2-C makes top-level recall query-only, removes duplicate detail retrieval, and emits at most one sanitized aggregate telemetry envelope; read-only and telemetry contracts passed the focused/convergence matrices. | | MM-OPS-02 | DevOps/Demo | High / BAD-PRACTICE | Docker/Helm publish HTTP on a stdio MCP process and use an invalid healthcheck | R3.4 | RESOLVED | R3.4 keeps stdio MCP private to the agent process and adds distinct authenticated streamable-HTTP MCP and dashboard entrypoints. Docker defaults to the dashboard; Compose is loopback-only, resource-bounded, and probes `/readyz`; Helm selects an explicit authenticated HTTP profile with liveness/readiness probes and resource limits. A locally built image returned 200 from dashboard and MCP HTTP readiness, and the official MCP client completed initialize/tools-list handshakes over stdio and authenticated HTTP. Focused deployment/supply-chain boundary: 97 passed. Helm CLI/disposable cluster evidence remains tracked externally without reopening the repository protocol mismatch. | -| MM-OPS-03 | DevOps/Maintainability | High / BAD-PRACTICE | A release tag can publish without a blocking verified test artifact | R4.4 | RESOLVED | The tag workflow builds once, hashes and uploads a release candidate, verifies the downloaded bytes, blocks on generated-truth drift, stable qrels, strict evaluation, supply-chain contracts, and clean-wheel entrypoint smoke, then publishes only the separately promoted `verified-dist` artifact. Static fail-closed workflow contracts pass; an actual GitHub/PyPI run remains BLOCKED-EXTERNAL and no publication occurred. | +| MM-OPS-03 | DevOps/Maintainability | High / BAD-PRACTICE | A release tag can publish without a blocking verified test artifact | R4.4 | RESOLVED | The tag workflow builds once, hashes and uploads a release candidate, verifies the downloaded bytes, blocks on generated-truth drift, stable qrels, strict evaluation, supply-chain contracts, and clean-wheel entrypoint smoke, then publishes only the separately promoted `verified-dist` artifact. Static fail-closed workflow contracts pass. The Phase 4 local candidate wheel and sdist were hash-verified, installed into an isolated venv, and exercised through both CLI entrypoints; a disposable image reports 4.4.1 and passes health/readiness. An actual GitHub/PyPI run remains BLOCKED-EXTERNAL and no publication occurred. | | MM-OPS-04 | Security/DevOps | High / EXPLOITABLE-LOW-EFFORT | Qdrant/Ollama ports are broadly exposed; images are mutable | R1.5,R3.4 | BLOCKED-EXTERNAL | Repository defaults are complete: loopback-only service and auxiliary ports, required immutable digests, authenticated TLS Qdrant, redirect-safe credentials, explicit authenticated HTTP service profiles, real readiness, resource limits, and fail-closed supply-chain policy. The local MemoryMaster image passed dashboard/MCP HTTP runtime smoke. Approved release-image scans plus authenticated Qdrant/Ollama and disposable Kubernetes runtime evidence remain external. | | MM-PERF-01 | Performance/Cost | High / BAD-PRACTICE | Hybrid reads recompute and rewrite candidate embeddings | R3.1 | RESOLVED | R3.1 migration 0014 persists model/content fingerprints; focused warm-query evidence proves one query embed and zero candidate embeds/writes. | | MM-PERF-02 | Performance | Medium / BAD-PRACTICE | Each process cold-scans corpus token statistics and caches forever | R3.2 | RESOLVED | R3.2 keys corpus and alias statistics by the persisted corpus generation, so warm reads reuse one scan and every claim mutation invalidates it. Focused evidence proves a new cache miss after generation change while tokenizer ranking/retired-status contracts remain green. | | MM-PERF-03 | Performance/Cost | High / EXPLOITABLE-NOW | Qdrant reconciliation repeats/truncates tens of thousands of embeddings | R3.1 | RESOLVED | R3.1 replaces four truncated 10,000-row scans with authority-scoped keyset pages and a replayable durable cursor; fake-backend convergence/failure-resume evidence passed. Real authenticated/TLS Qdrant remains externally blocked. | | MM-PERF-04 | Database/Performance | Medium / BAD-PRACTICE | Million-row event queries lack event-type composite indexes | R3.2 | RESOLVED | Immutable migration 0015 and both baseline schemas add `(event_type, created_at DESC, id DESC)` and `(event_type, details, created_at DESC)` indexes. SQLite EXPLAIN selects the ordered type index; Postgres DDL parity is covered statically pending the already-recorded disposable-Postgres external gate. | | MM-UX-01 | UX/Maintainability | High / BAD-PRACTICE | Setup reports success without verifying the requested memory loop | R3.3 | RESOLVED | R3.3 adds explicit minimal/semantic/team/full-lab contracts and stable DB, MCP, recall-hook, capture-hook, provider, steward, vector, and dashboard component evidence. Requested PARTIAL/BLOCKED profiles return nonzero; a DB-only sentinel can no longer mask missing requested components. Session-end ingest is now a packaged module/entrypoint, and installed-wheel setup refuses to imply an absent Compose bundle was shipped. Focused setup/profile gate: 161 passed. | -| MM-UX-02 | UX/Reliability | High / BAD-PRACTICE | Dashboard removes review rows before mutation success | R4.3 | RESOLVED | Review mutations now enter a disabled pending state and remove the row only after a successful POST. Headless Chromium evidence rejects the first POST, confirms the row remains with an actionable Retry state, then confirms successful retry removal. | -| MM-UX-03 | UX | Medium / BAD-PRACTICE | Dashboard hides failures and has labeling, contrast, responsive, and evidence-discovery gaps | R4.3 | RESOLVED | Panel failures render visible alert/retry states; controls and live regions are labelled; low-contrast text is raised; narrow dashboard and conflict grids stack; conflicts expose citations/lineage; pending steward proposals expose rationale, transition/replacement consequences, and approval controls only when actionable. Focused dashboard gate: 57 passed, including headless Chromium mobile/failure acceptance. | +| MM-UX-02 | UX/Reliability | High / BAD-PRACTICE | Dashboard removes review rows before mutation success | R4.3 | RESOLVED | Review mutations now enter a disabled pending state and remove the row only after a successful POST. Headless Chromium evidence rejects the first POST, confirms the row remains with an actionable Retry state, then confirms successful retry removal. The Phase 4 audit additionally found and corrected a proposal-selection race: each action is bound to the exact displayed proposal event, and stale/invalid event IDs fail closed instead of resolving the newest proposal for the claim. | +| MM-UX-03 | UX | Medium / BAD-PRACTICE | Dashboard hides failures and has labeling, contrast, responsive, and evidence-discovery gaps | R4.3 | RESOLVED | Panel failures render visible alert/retry states; controls and live regions are labelled; low-contrast text is raised; narrow dashboard and conflict grids stack; conflicts expose citations/lineage; pending steward proposals expose rationale, transition/replacement consequences, and approval controls only when actionable. The focused package gate passed 57 tests; the convergence correction gate passed 54 tests including headless Chromium and the exact proposal-event race. | | MM-PRIV-01 | Compliance | High / BAD-PRACTICE | Automatic transcript capture/remote processing lacks an explicit consent boundary | R2.4,R3.5 | BLOCKED-EXTERNAL | P2-E repository controls are quiet by default and require explicit maximum-capture/provider choices. Product-owner/legal approval for intended use, jurisdictions, processor disclosures, and retention commitments remains external. WARNING LEGAL REVIEW REQUIRED; no compliance-pass or risk-acceptance claim is made. | | MM-PRIV-02 | Compliance | High / BAD-PRACTICE | No complete export, erasure, or retention workflow across copies | R3.5 | BLOCKED-EXTERNAL | R3.5 adds a read-only, tenant-aware dry-run inventory across authoritative claims/citations/events, verbatim, artifacts/spool/wiki/cache, backups, Qdrant, and Postgres. Ambiguous tenants and unattributable copies fail closed. Legal disposition, authenticated backend evidence, and any live export/erase/retention application require external approval and infrastructure; no mutation pass is claimed. | | MM-COST-01 | Cost | High / EXPLOITABLE-NOW | Default Stop-hook LLM calls have no finite persisted budget | R2.4 | RESOLVED | P2-E (`a1d5477`) adds restart-safe atomic global/provider/session reservations and persisted usage; maximum default candidate inflow is 600/day against the 688/day ceiling. | | MM-COST-02 | Cost | High / EXPLOITABLE-NOW | Verbatim capture reprocesses sessions and retains data indefinitely | R2.4 | RESOLVED | P2-E persists complete-line cursors, processes appended content only, and enforces dry-run 30-day/512-MiB/75,000-session retention bounds. | | MM-COST-03 | Cost/Reliability | Medium / BAD-PRACTICE | Cost and intake controls are fragmented/process-local | R2.4,R3.2 | RESOLVED | R3.2 adds one WAL-backed atomic usage ledger shared by LLM primary/fallback calls, Gemini embedding batches, MCP ingest/checkpoint, and configured core intake quotas. Caps remain backward-compatible/unlimited unless configured; restart/cross-instance and local-bucket-reset tests prove durable global/provider/actor enforcement without consuming capacity on rejected reservations. | | MM-DEMO-01 | Demo/Integrity | High / EXPLOITABLE-NOW | Atlas defaults persist fabricated mock evidence at high confidence | R2.5 | RESOLVED | P2-F (`c66e240`) requires a real configured production provider, confines synthetic providers to explicit dev/test opt-in, excludes synthetic evidence from claims/prompts/actions/citations/exports, and fails configuration nonzero. Focused gate: 111 passed. | -| MM-MAINT-01 | Maintainability/Integrity | Medium / BAD-PRACTICE | Versions, tool counts, roadmaps, and install claims drift | R4.4 | RESOLVED | `pyproject.toml` is the only package-version value; runtime metadata, health, dashboard HTML, supply-chain fixtures, and docs derive it. The committed/CI-checked release truth reports 36 MCP tools, 106 main CLI commands, 5 ops commands, 8 entrypoints, 4,219 tests, and the setup feature/profile matrix. Root `ROADMAP.md` is the only active Now/Next/Later/Not planned roadmap; three legacy names are short historical pointers. | +| MM-MAINT-01 | Maintainability/Integrity | Medium / BAD-PRACTICE | Versions, tool counts, roadmaps, and install claims drift | R4.4 | RESOLVED | `pyproject.toml` is the only package-version value; runtime metadata, health, dashboard HTML, supply-chain fixtures, and docs derive it. The committed/CI-checked release truth reports 36 MCP tools, 106 main CLI commands, 5 ops commands, 8 entrypoints, 4,221 tests, and the setup feature/profile matrix. Root `ROADMAP.md` is the only active Now/Next/Later/Not planned roadmap; three legacy names are short historical pointers. | | MM-MAINT-02 | Architecture/Maintainability | Medium / BAD-PRACTICE | Oversized facades, hardwired extensions, dead plugin seam, and lingering shims | R4.1-R4.2 | RESOLVED | R4.1 removes the zero-consumer generic plugin registry and enforces core-to-companion imports. R4.2 extracts the real integration service plus dashboard read/command layers, reduces `service.py` from 2,617 to 2,205 lines and the handler from 848 to 691, enforces ratcheting budgets, and publishes a dated v5 shim-removal gate. Remaining collaborator extraction is an explicit gradual ratchet, not an unbounded facade regression. | | MM-REL-03 | Reliability | Medium / BAD-PRACTICE | Media retry rows have no expired-lease recovery | R3.5 | RESOLVED | Immutable migration 0016 and SQLite/Postgres parity add bounded owner/expiry leases, atomic expired-work recovery with an audit event, lease clearing on terminal outcomes, and concurrent single-winner coverage. The focused R3.5 gate includes worker-death reclaim, unexpired non-steal, and concurrent claimer tests. | | MM-OPS-05 | Missing Operations | High / BAD-PRACTICE | Recovery defaults stop at same-machine SQLite snapshots | R3.5 | BLOCKED-EXTERNAL | R3.5/Phase 3 convergence adds authenticated encrypted SQLite backups, plaintext/ciphertext checksums, online backup, disposable restore, integrity/FK checks, backup-age health, and measured RPO/RTO (0.046s restore versus 1,800s objective). Approved off-device storage and disposable PostgreSQL dump/restore evidence remain external; same-machine success is not represented as full recovery readiness. | diff --git a/.planning/audits/2026-07-14-phase4-budget-delta/audit-delta.md b/.planning/audits/2026-07-14-phase4-budget-delta/audit-delta.md new file mode 100644 index 00000000..817fadcb --- /dev/null +++ b/.planning/audits/2026-07-14-phase4-budget-delta/audit-delta.md @@ -0,0 +1,75 @@ +# Phase 4 Budget Audit Delta — 2026-07-14 + +## Scope and outcome + +This targeted delta reconciles MM-OPS-03, MM-UX-02, MM-UX-03, +MM-MAINT-01, MM-MAINT-02, MM-INTEGRITY-01, and MM-TEST-01. It is not a +new 13-domain audit. + +The first independent Phase 4 review discovered one reproducible High +governance race: the dashboard displayed proposal event N but submitted only +the claim ID, allowing a newer proposal to be resolved instead. Two +adversarial tests witnessed the failure. The bounded correction binds every +action to the exact displayed `proposal_event_id` and rejects missing, +malformed, or stale identifiers. The correction boundary passed 54 tests; the +final strict malformed-ID re-review passed 18 focused dashboard tests. + +The correction review found no remaining reproducible Critical/High regression +introduced by Phase 4. The latest same-scope audit contains zero new findings. + +| Finding set | Delta | +|---|---| +| MM-MAINT-02 | RESOLVED — explicit companion boundaries, removed dead plugin seam, smaller real services, ratcheting budgets, dated shim gate | +| MM-UX-02..03 | RESOLVED — truthful pending/error/retry states, exact proposal-event binding, inspectable evidence, browser/a11y/mobile acceptance | +| MM-OPS-03 | RESOLVED in repository — publish consumes only verified bytes; an actual tag/PyPI run remains BLOCKED-EXTERNAL | +| MM-MAINT-01, MM-INTEGRITY-01, MM-TEST-01 | RESOLVED — generated release truth, canonical version, clean import probe, deterministic test contracts | +| P4-CORR-01 | FIXED — exact proposal-selection race discovered during convergence | + +## Audit-loop reconciliation + +```text +AUDIT LOOP ITERATION 2 + +Previous report: .planning/audits/2026-07-10-baseline/audit-report.md +Current report: .planning/audits/2026-07-14-phase4-budget-delta/audit-delta.md +Roadmap: .planning/REMEDIATION-OPTIMIZATION-PLAN-2026-07-10.md + +Delta: + Fixed: 8 + Unchanged: 8 valid BLOCKED-EXTERNAL canonical findings + Severity changed: 0 + New findings: 0 + +Blocked: + MM-SEC-01, MM-SEC-02, MM-OPS-01, MM-SEC-03, MM-OPS-04, + MM-PRIV-01, MM-PRIV-02, MM-OPS-05 — BLOCKED-EXTERNAL + Gate file: external-actions-required.md + +Convergence: READY for local release-candidate review; NOT production approval +``` + +The race was caught by the independent phase audit before convergence, so it +does not represent a blind spot in the latest audit method. The permanent +adversarial acceptance tests are the regression guard. + +## Verification evidence + +| Gate | Evidence | +|---|---| +| Collection/generated truth | 4,221 tests; 36 MCP tools; 106 main CLI commands; 5 ops commands; 8 entrypoints | +| Full non-ML, once | 4,053 passed, 70 skipped, 95 deselected, 1 xfailed, 2 warnings in 919.40s | +| ML, isolated | 95 passed, 4,126 deselected, 1 warning in 18.45s | +| Bounded correction | Two witnessed RED tests; 54 passed after correction; strict malformed-ID re-review 18 passed | +| Browser/a11y | Headless Chromium governance failure/retry, exact event, narrow layout, and visible-error coverage passed | +| Static quality | Project Ruff, changed-file Ruff, syntax/import checks, inline JavaScript parse, generated-truth check, and `git diff --check` passed | +| Wheel/sdist | Build/install passed; isolated import reports 4.4.1 from venv `site-packages`; CLI and MCP entrypoints passed | +| Disposable runtime | Compose config passed; local image built; fresh temporary DB initialized; health `ok`, readiness `ready`, runtime 4.4.1; MCP entrypoint passed | + +## External evidence retained + +Authenticated/TLS Qdrant, two-role disposable Postgres, approved immutable +release images and image scans, disposable Kubernetes/Helm, dependency audit, +history-secret classification, off-device/Postgres recovery, legal/privacy +decisions, and an actual tag/PyPI publication remain external. No live data, +paid provider, production service, credential, or external configuration was +mutated. diff --git a/.planning/audits/2026-07-14-phase4-budget-delta/pr-release-draft.md b/.planning/audits/2026-07-14-phase4-budget-delta/pr-release-draft.md new file mode 100644 index 00000000..512fe38c --- /dev/null +++ b/.planning/audits/2026-07-14-phase4-budget-delta/pr-release-draft.md @@ -0,0 +1,31 @@ +# Draft PR and Release Description + +## Title + +`release: converge governed MemoryMaster 4.4.1 remediation roadmap` + +## Summary + +This branch completes the local Phase 2–4 remediation roadmap: governed +retrieval and Qdrant rehydration, lifecycle/schema/capture/media authority, +bounded performance and provider costs, truthful setup/deployment/recovery +surfaces, explicit extension boundaries, smaller orchestration seams, +stateful accessible governance UX, and generated release truth. + +## Verification + +- 4,053 non-ML tests passed; 95 isolated ML tests passed. +- 4,221 tests collected; project Ruff and generated truth passed. +- Browser governance/a11y/mobile acceptance passed. +- Clean wheel/sdist build, isolated install, CLI/MCP smoke passed. +- Disposable Compose/image/database health/readiness smoke passed. +- Latest targeted audit: zero new findings and no unresolved reproducible + Critical/High regression introduced by the remediation branch. + +## Release caveat + +This PR is a local release candidate, not production authorization. The +external Postgres, Qdrant, credential/history/dependency/image scan, +Kubernetes, recovery, privacy/legal, and publication gates remain listed in +`external-actions-required.md`. No push, tag, publication, deployment, or live +data mutation was performed by the remediation goal. diff --git a/.planning/audits/2026-07-14-phase4-budget-delta/production-release-approval-request.md b/.planning/audits/2026-07-14-phase4-budget-delta/production-release-approval-request.md new file mode 100644 index 00000000..6074acc4 --- /dev/null +++ b/.planning/audits/2026-07-14-phase4-budget-delta/production-release-approval-request.md @@ -0,0 +1,35 @@ +# Ready-to-Paste Production/Release Approval Request + +```text +Request: authorize the external release-candidate verification sequence for +MemoryMaster 4.4.1. Do not authorize production cutover yet. + +Local status: Phase 2–4 remediation and final targeted audit converged. The +latest audit has zero new findings and no unresolved reproducible +Critical/High regression introduced by the branch. Full local non-ML, ML, +Ruff, browser/a11y, clean-wheel, generated-truth, Compose, and disposable +container/database gates passed. + +Approval requested for these controlled external actions only: +1. Resolve/classify credential-history, Gitleaks, dependency, and immutable + release-image scan blockers. +2. Run disposable two-role Postgres RLS/parity and authenticated/TLS Qdrant + governed-retrieval/reconciliation gates. +3. Run approved immutable-image Kubernetes/Helm and off-device/Postgres + recovery drills. +4. Obtain product/legal privacy, consent, retention, and data-disposition + decisions. +5. If and only if all blockers are resolved or formally accepted, approve one + tag-triggered verified-artifact workflow execution. Manual dispatch must + not publish; PyPI must receive the exact promoted verified-dist bytes. + +Still prohibited without a separate explicit approval: production cutover, +live migration or cleanup, redaction, archival, retention deletion, backlog +mutation, credential rotation execution, history rewrite, or product-data +mutation. + +Required before cutover approval: exact final commit, artifact/image digests, +external gate evidence, staging/canary health and MCP handshake, migration and +integrity evidence, telemetry/alert delivery, and the reviewed rollback +command/owner. +``` diff --git a/.planning/audits/2026-07-14-phase4-budget-delta/release-candidate-readiness.md b/.planning/audits/2026-07-14-phase4-budget-delta/release-candidate-readiness.md new file mode 100644 index 00000000..a41e0ac0 --- /dev/null +++ b/.planning/audits/2026-07-14-phase4-budget-delta/release-candidate-readiness.md @@ -0,0 +1,42 @@ +# Local Release-Candidate Readiness + +## Decision + +**LOCAL RC READY; PRODUCTION/EXTERNAL RELEASE BLOCKED.** + +The branch is locally releasable at each atomic package checkpoint and passes +the Phase 4 convergence boundary. It must not be pushed, tagged, published, or +deployed until the prioritized external blockers are resolved or explicitly +accepted by their authorized owners. + +## Candidate identity + +| Item | Identity | +|---|---| +| Version | 4.4.1 | +| Wheel | `memorymaster-4.4.1-py3-none-any.whl` | +| Wheel SHA-256 | `3C10C02DB34636CE919A6BE6845AC571CB78B7A8F89332F007A1260FEB96B23A` | +| Sdist | `memorymaster-4.4.1.tar.gz` | +| Sdist SHA-256 | `B4C56EEFAFA37484C10FEEFDC9893773940BC3B0D51A920F359DB251CF1B6C1B` | +| Local image | `memorymaster:phase4-local` | +| Local image ID | `sha256:d3069774253f1d6b72d443ae8771dad98b0be6c51f78592d363ce7dc827584a2` | + +These are local evidence artifacts, not approved immutable release assets. +The verified tag workflow must rebuild a candidate, record its hashes, verify +the downloaded bytes, promote without rebuilding, and publish those exact +bytes only after approval. + +## Passed locally + +- Full non-ML and isolated ML gates. +- Browser governance, accessibility, mobile, and failure semantics. +- Project Ruff, generated truth, workflow contracts, syntax/import, and diff + hygiene. +- Isolated wheel install and runtime/CLI/MCP smoke outside the source tree. +- Compose fail-closed rendering and a disposable local-image/database + health/readiness smoke. + +## Not proven locally + +See `remaining-blockers.md` and the canonical +`external-actions-required.md`. These gaps prevent a production-ready claim. diff --git a/.planning/audits/2026-07-14-phase4-budget-delta/remaining-blockers.md b/.planning/audits/2026-07-14-phase4-budget-delta/remaining-blockers.md new file mode 100644 index 00000000..02e81880 --- /dev/null +++ b/.planning/audits/2026-07-14-phase4-budget-delta/remaining-blockers.md @@ -0,0 +1,18 @@ +# Prioritized Remaining-Blocker Ledger + +This is a release-oriented view. `external-actions-required.md` remains the +canonical owner/evidence ledger. + +| Priority | Finding(s) | Blocking evidence/action | +|---|---|---| +| P0 | MM-OPS-01, MM-OPS-04 | Rotate/classify historical credentials and Gitleaks findings; complete strict dependency and exact release-image scans | +| P0 | MM-SEC-01 | Disposable two-role Postgres RLS/parity proof and approved brownfield inventory/repair evidence | +| P0 | MM-SEC-02 | Disposable authenticated/TLS Qdrant governed retrieval, reconciliation, outbox, sync, and delete proof | +| P0 | MM-SEC-03 | Authorized aggregate legacy-store inventory completion, including Qdrant; separately approve any cleanup/redaction | +| P1 | MM-OPS-02, MM-OPS-03 | Approved immutable images, disposable Kubernetes/Helm proof, then explicit tag/PyPI workflow approval | +| P1 | MM-OPS-05 | Approved off-device encrypted destination plus SQLite/Postgres restore drill | +| P1 | MM-PRIV-01, MM-PRIV-02 | Product/legal consent, processor, retention, attribution, export/erase disposition, and backend dry-run decisions | +| P2 | MM-CAP-01 | Address or monitor host-storage capacity without deleting product data | + +No blocker above authorizes a mutation. Each external owner must approve the +specific action and retain the evidence requested by the canonical ledger. diff --git a/.planning/audits/2026-07-14-phase4-budget-delta/rollback-plan.md b/.planning/audits/2026-07-14-phase4-budget-delta/rollback-plan.md new file mode 100644 index 00000000..e696e12c --- /dev/null +++ b/.planning/audits/2026-07-14-phase4-budget-delta/rollback-plan.md @@ -0,0 +1,29 @@ +# Rollback Plan + +## Before release + +Revert the Phase 4 convergence commit first, then revert R4.4 through R4.1 in +reverse order. If a phase-level rollback is required, continue with Phase 3 +and Phase 2 convergence/package commits in reverse dependency order. + +## Data and schema safety + +- Do not edit or delete immutable migrations 0013–0016. +- Do not downgrade a database in place. Restore a verified compatible backup + or ship a reviewed forward repair. +- Disable semantic mode and use authoritative lexical retrieval if Qdrant + evidence or reconciliation fails. +- Disable Team/Postgres profiles if identity/RLS evidence is absent. +- Return capture to quiet/no-capture defaults; never restore unlimited + per-stop or synthetic production capture. + +## Runtime rollback + +1. Keep the last approved immutable image and artifact hashes available. +2. Stop canary promotion on failed health, readiness, MCP handshake, + migration, tenant/scope, integrity, or telemetry gates. +3. Redeploy the last approved image without rolling database migrations back. +4. Verify `/healthz`, `/readyz`, MCP initialize/tools-list, DB integrity/FKs, + tenant/scope denials, Qdrant fallback, and alert delivery. +5. Restore product data only from a separately verified backup and only with + explicit operator approval. diff --git a/docs/generated/release-truth.json b/docs/generated/release-truth.json index ce6ed762..3702fa5f 100644 --- a/docs/generated/release-truth.json +++ b/docs/generated/release-truth.json @@ -122,7 +122,7 @@ "console_entrypoints": 8, "mcp_tools": 36, "ops_cli_commands": 5, - "pytest_tests": 4219 + "pytest_tests": 4221 }, "feature_profile_matrix": { "capture_hook": [ diff --git a/docs/generated/release-truth.md b/docs/generated/release-truth.md index 80e9dcca..949c0c23 100644 --- a/docs/generated/release-truth.md +++ b/docs/generated/release-truth.md @@ -7,7 +7,7 @@ Do not edit this file by hand. Run `python scripts/generate_release_truth.py`. - Main CLI commands: **106** - Operations CLI commands: **5** - Console entrypoints: **8** -- Collected pytest tests: **4219** +- Collected pytest tests: **4221** ## MCP tools diff --git a/external-actions-required.md b/external-actions-required.md index 98553aa7..7d673773 100644 --- a/external-actions-required.md +++ b/external-actions-required.md @@ -15,7 +15,7 @@ live mutation by itself. | MM-SEC-03 | Live legacy stores | Review the aggregate inventory, provide a verified backup, and separately authorize any cleanup/redaction/rebuild. Configure a disposable or approved read-only Qdrant inventory target before claiming complete coverage. | SQLite/artifact/spool/Qdrant aggregate rerun with zero unscannable/truncated surfaces, plus approved remediation evidence; never include raw findings | 2026-07-31 | BLOCKED-EXTERNAL | | MM-OPS-04 / history scan | Security owner | Classify the unsuppressed Gitleaks history result: 40 potential findings across 10 commits and 7 files. Rotate any affected credentials before considering an approved history action. | Aggregate classification and rotation record; no raw secret material in general logs | 2026-07-31 | BLOCKED-EXTERNAL | | MM-OPS-04 / dependency audit | Release owner / advisory service | Rerun strict project and `mcp,qdrant,security` extra audits in an approved lock/resolver environment; both final attempts exceeded the 15-minute external-service cap and were terminated without a pass claim. | Successful strict PyPI audit bound to the release lock/artifact, or reviewed vulnerability remediation evidence | 2026-07-31 | BLOCKED-EXTERNAL | -| MM-OPS-04 / image scan | Release owner / local image registry | Provide approved immutable local MemoryMaster, Qdrant, and Ollama image IDs. No relevant local images existed and this goal forbade builds or pulls solely for scanning. | Docker Scout high/critical scan results for the exact immutable release images, including base findings | 2026-07-31 | BLOCKED-EXTERNAL | +| MM-OPS-04 / image scan | Release owner / local image registry | Provide approved immutable MemoryMaster, Qdrant, and Ollama release image IDs. Phase 4 built and runtime-smoked a local development MemoryMaster image, but it is not an approved immutable release image and Qdrant/Ollama release images remain unavailable. | Docker Scout high/critical scan results for the exact immutable release images, including base findings | 2026-07-31 | BLOCKED-EXTERNAL | | MM-OPS-03 / publication proof | Release owner / GitHub / PyPI | After every release blocker is resolved or formally accepted, explicitly approve one tag-based execution of the verified-artifact workflow. Do not use manual dispatch to publish. | GitHub job evidence showing candidate hash verification, blocking release gates, clean-wheel smoke, promotion without rebuild, and PyPI OIDC publication of the exact `verified-dist` bytes | 2026-08-15 | BLOCKED-EXTERNAL | | MM-DATA-01 | Live MemoryMaster operator | Approve a consistent backup/restore drill before any live migration, redaction, backlog, or retention operation. | Restored backup, integrity check, counts/checksums, approval record | 2026-08-15 | BLOCKED-EXTERNAL | | MM-CAP-01 | Host storage operator | Address/monitor the drive at 85.82% used without deleting MemoryMaster data under this goal. | Daily disk telemetry below the critical gate or approved capacity expansion | 2026-07-18 | BLOCKED-EXTERNAL | diff --git a/memorymaster/surfaces/dashboard.py b/memorymaster/surfaces/dashboard.py index b709204d..ff0d5e7e 100644 --- a/memorymaster/surfaces/dashboard.py +++ b/memorymaster/surfaces/dashboard.py @@ -961,7 +961,8 @@ def _write_dashboard(self) -> None: .badge-stale{background:#451a03;color:#fb923c;border:1px solid #d97706} .badge-conflicted{background:#450a0a;color:#f87171;border:1px solid #dc2626} .badge-superseded{background:#312e81;color:#a78bfa;border:1px solid #7c3aed} -.badge-archived{background:#1e293b;color:#64748b;border:1px solid #475569} +.badge-archived{background:#1e293b;color:#94a3b8;border:1px solid #475569} +span[style="color:#64748b"]{color:#94a3b8!important} .card{border:1px solid #334155;border-radius:8px;margin:8px 0;background:#1e293b} .card-head{padding:8px 10px;border-bottom:1px solid #2d3a4e;background:rgba(15,23,42,.4);border-radius:8px 8px 0 0} .card-row{padding:8px 10px;border-bottom:1px solid #273348} @@ -1077,7 +1078,7 @@ def _write_dashboard(self) -> None: function renderConflicts(){const h=document.getElementById('conflicts-cards');const meta=document.getElementById('conflicts-meta');const q=String(conflictState.search||'').toLowerCase().trim();const rows=conflictState.rows.filter(g=>{if(!q)return true;const claims=Array.isArray(g.claims)?g.claims:[];const claimText=claims.map(c=>[c.id,c.status,c.subject,c.predicate,c.object_value,c.text].join(' ')).join(' ').toLowerCase();return ([g.subject,g.predicate,g.scope].join(' ').toLowerCase()+' '+claimText).includes(q);});meta.textContent=rows.length+' of '+conflictState.rows.length+' conflict groups';h.innerHTML=rows.map(g=>{const cs=Array.isArray(g.claims)?g.claims:[];const n=cs[0]||null;const o=cs.length>1?cs[1]:null;const nv=n?(n.object_value||n.text||'-'):'-';const ov=o?(o.object_value||o.text||'-'):'-';const nc=n?Number(n.confidence||0):null;const oc=o?Number(o.confidence||0):null;const nz=n?((n.citations||[]).length):0;const oz=o?((o.citations||[]).length):0;const confDelta=(nc!=null&&oc!=null)?(nc-oc):null;const citeDelta=(n&&o)?(nz-oz):null;const valueChanged=(String(nv)!==String(ov));const delta=(v)=>v==null?'-':((v>=0?'+':'')+Number(v).toFixed(3));const cDelta=(v)=>v==null?'-':((v>=0?'+':'')+String(v));const row=(label,a,b,chg)=>''+esc(label)+''+esc(a)+''+esc(b)+''+esc(chg)+'';const statusCounts={};cs.forEach(c=>{const k=String(c.status||'unknown');statusCounts[k]=(statusCounts[k]||0)+1;});const card=(label,c,value,confidence,cites,color)=>'
'+label+' '+(c?statusBadge(c.status):'')+' #'+esc(c?c.id:'-')+'
'+esc(value)+'
confidence: '+esc(confidence==null?'-':confidence.toFixed(3))+' · citations: '+esc(cites)+'
'+governanceEvidence(c)+'
';return '
'+esc(g.subject||'-')+' / '+esc(g.predicate||'-')+'
'+esc(cs.length)+' claims · scope: '+esc(g.scope||'project')+' · '+countPills(statusCounts,4)+'
'+card('Newer',n,nv,nc,nz,'#4ade80')+card('Older',o,ov,oc,oz,'#f87171')+'
'+row('value',nv,ov,valueChanged?'CHANGED':'same')+row('confidence',nc==null?'-':nc.toFixed(3),oc==null?'-':oc.toFixed(3),delta(confDelta))+row('citations',String(nz),String(oz),cDelta(citeDelta))+row('updated_at',n&&n.updated_at?n.updated_at:'-',o&&o.updated_at?o.updated_at:'-',(n&&o&&String(n.updated_at)!==String(o.updated_at))?'changed':'same')+'
FieldNewerOlderDelta
';}).join('')||'
No conflicts detected
';} function fillConflicts(d){conflictState.rows=Array.isArray(d.groups)?d.groups:[];renderConflicts();} async function refreshConflicts(){const includeStale=document.getElementById('conflicts-include-stale').checked?'1':'0';fillConflicts(await jget('/api/conflicts?limit=20&include_stale='+includeStale));} -function fillQueue(d){const rows=Array.isArray(d.items)?d.items:[];const b=document.getElementById('stale-body');if(!rows.length){b.innerHTML='Nothing to review';return;}b.innerHTML=rows.map(i=>{const p=i.proposal||null;const proposalActions=p?' ':'';return ''+esc(i.claim_id)+''+statusBadge(i.status)+''+esc(i.reason||'-')+proposalEvidence(p)+''+esc(f3(i.priority))+' '+proposalActions+actionConsequences(Boolean(p))+'';}).join('');} +function fillQueue(d){const rows=Array.isArray(d.items)?d.items:[];const b=document.getElementById('stale-body');if(!rows.length){b.innerHTML='Nothing to review';return;}b.innerHTML=rows.map(i=>{const p=i.proposal||null;const proposalId=p?Number(p.proposal_event_id||0):0;const proposalActions=proposalId>0?' ':'';return ''+esc(i.claim_id)+''+statusBadge(i.status)+''+esc(i.reason||'-')+proposalEvidence(p)+''+esc(f3(i.priority))+' '+proposalActions+actionConsequences(Boolean(p))+'';}).join('');} function fillRetr(d){const rows=Array.isArray(d.rows_data)?d.rows_data:[];const b=document.getElementById('retrieval-body');const meta=document.getElementById('retrieval-meta');const scopes=Array.isArray(d.scope_allowlist)?d.scope_allowlist:[];const scopeText=scopes.length?scopes.join(', '):'all';meta.textContent='Mode: '+(d.mode||'-')+' · Scopes: '+scopeText+' · '+rows.length+' results';b.innerHTML=rows.map(r=>{const c=r.claim||{};const s=r.status||c.status||'unknown';const ann=(r.annotation||'-');return ''+esc(c.id)+''+tuple(c)+''+statusBadge(s)+''+esc(ann)+''+esc(f3(r.score))+''+esc(f3(r.lexical_score))+' / '+esc(f3(r.confidence_score))+' / '+esc(f3(r.freshness_score))+' / '+esc(f3(r.vector_score))+'';}).join('')||'No results found';} function fillRecallAnalysis(d){const rows=Array.isArray(d.results)?d.results:[];const meta=document.getElementById('recall-meta');const wbox=document.getElementById('recall-weights');meta.textContent='Mode: '+(d.mode||'-')+(d.profile?(' · Profile: '+d.profile):'')+' · '+rows.length+' results';const w=(d.weights&&d.weights.retrieval_weights)||null;wbox.innerHTML=w?('Active weights — lexical: '+esc(f3(w.lexical))+' · confidence: '+esc(f3(w.confidence))+' · freshness: '+esc(f3(w.freshness))+' · vector: '+esc(f3(w.vector))):'';const b=document.getElementById('recall-body');b.innerHTML=rows.map(r=>{const id=r.human_id||r.claim_id;const pin=r.pinned?' 📌':'';return ''+esc(id)+pin+''+esc(r.text||'-')+''+statusBadge(r.status)+''+esc(r.tier||'working')+''+esc(f3(r.score))+''+esc(f3(r.lexical_score))+' / '+esc(f3(r.confidence_score))+' / '+esc(f3(r.freshness_score))+' / '+esc(f3(r.vector_score))+'';}).join('')||'No results found';} function fillAudit(d){const rows=Array.isArray(d.events)?d.events:[];document.getElementById('audit-body').innerHTML=rows.map(e=>''+esc(e.created_at||'-')+''+esc(e.event_type||'-')+''+esc(e.claim_id||'-')+''+esc(e.details||'-')+'').join('')||'No audit events';} @@ -1090,7 +1091,7 @@ def _write_dashboard(self) -> None: function fillIntegrity(d){const x=d.integrity||{};const box=document.getElementById('integrity-box');if(x.available===false){box.innerHTML='
'+esc(x.reason||'unavailable')+'
';return;}const sp=x.spool||{};const qd=x.qdrant||{};const lc=x.last_cycle||null;const fmtB=(v)=>v==null?'-':(v>=1048576?(v/1048576).toFixed(1)+' MB':String(v)+' B');const nv=(v)=>v==null?'-':String(v);box.innerHTML='
'+esc(fmtB(x.wal_bytes))+'WAL size
'+esc(nv(sp.depth_lines))+'Spool lines
'+esc(nv(qd.drift))+'Qdrant drift
'+esc(nv(x.busy_errors))+'Busy errors
'+(x.promotions_frozen?'FROZEN':'ok')+'Promotions
'+(lc?'
Last cycle '+esc(nv(lc.at))+' · quick_check='+esc(lc.quick_check_ok==null?'throttled':lc.quick_check_ok)+' · fk_orphans='+esc(nv(lc.fk_orphans))+' · drained='+esc(nv(lc.spool_drained))+' · lag='+esc(nv(lc.spool_lag_seconds))+'s
':'
No cycle metrics recorded yet
')+'
';} async function refreshQueue(){fillQueue(await jget('/api/review-queue?limit=30&exclude_reviewed=1&exclude_suppressed=1'));} async function refreshObs(){fillObs(await jget('/api/observability?log_limit=1500&event_limit=600&queue_limit=250'));} -document.getElementById('stale-body').addEventListener('click',async(ev)=>{const t=ev.target;if(!t||t.tagName!=='BUTTON'||!t.hasAttribute('data-action'))return;const r=t.closest('tr');if(!r)return;const id=Number(r.getAttribute('data-claim-id'));const a=String(t.getAttribute('data-action')||'');const original=t.innerHTML;r.setAttribute('data-pending','true');r.classList.add('pending');t.disabled=true;reviewStatus('Applying '+a+' to claim '+id+'...');try{await jpost('/api/triage/action',{claim_id:id,action:a});r.remove();reviewStatus('Action '+a+' completed for claim '+id+'.');await refreshQueue();}catch(error){r.removeAttribute('data-pending');r.classList.remove('pending');t.disabled=false;t.innerHTML='Retry';t.title=String((error&&error.message)||error||'Action failed');reviewStatus('Action failed for claim '+id+'. Retry is available.',true);t.addEventListener('blur',()=>{t.innerHTML=original;},{once:true});}}); +document.getElementById('stale-body').addEventListener('click',async(ev)=>{const t=ev.target;if(!t||t.tagName!=='BUTTON'||!t.hasAttribute('data-action'))return;const r=t.closest('tr');if(!r)return;const id=Number(r.getAttribute('data-claim-id'));const a=String(t.getAttribute('data-action')||'');const proposalId=Number(t.getAttribute('data-proposal-event-id')||0);const body={claim_id:id,action:a};if(a==='approve_proposal'||a==='reject_proposal')body.proposal_event_id=proposalId;const original=t.innerHTML;r.setAttribute('data-pending','true');r.classList.add('pending');t.disabled=true;reviewStatus('Applying '+a+' to claim '+id+'...');try{await jpost('/api/triage/action',body);r.remove();reviewStatus('Action '+a+' completed for claim '+id+'.');await refreshQueue();}catch(error){r.removeAttribute('data-pending');r.classList.remove('pending');t.disabled=false;t.innerHTML='Retry';t.title=String((error&&error.message)||error||'Action failed');reviewStatus('Action failed for claim '+id+'. Retry is available.',true);t.addEventListener('blur',()=>{t.innerHTML=original;},{once:true});}}); document.getElementById('op-start').addEventListener('click',async()=>{await jpost('/api/operator/control',{action:'start',inbox_jsonl:document.getElementById('op-inbox').value});fillOp(await jget('/api/operator/status'));}); document.getElementById('op-stop').addEventListener('click',async()=>{await jpost('/api/operator/control',{action:'stop'});fillOp(await jget('/api/operator/status'));}); document.getElementById('retrieval-run').addEventListener('click',async()=>{const query=document.getElementById('retrieval-query').value||'';const mode=document.getElementById('retrieval-mode').value||'hybrid';const scopeRaw=document.getElementById('retrieval-scope').value||'';const url='/api/retrieval?query='+encodeURIComponent(query)+'&mode='+encodeURIComponent(mode)+'&scope_allowlist='+encodeURIComponent(scopeRaw)+'&limit=10';fillRetr(await jget(url));}); diff --git a/memorymaster/surfaces/dashboard_commands.py b/memorymaster/surfaces/dashboard_commands.py index 65b32443..67b49239 100644 --- a/memorymaster/surfaces/dashboard_commands.py +++ b/memorymaster/surfaces/dashboard_commands.py @@ -37,7 +37,10 @@ def apply_triage_action( claim = service.pin(claim_id, pin=action == "pin") return {"ok": True, "action": action, "claim": serialize_claim(claim)} if action in {"approve_proposal", "reject_proposal"}: - return _resolve_proposal(service, action, claim_id) + proposal_event_id = payload.get("proposal_event_id") + if type(proposal_event_id) is not int or proposal_event_id <= 0: + raise ValueError("proposal_event_id must be positive for proposal actions") + return _resolve_proposal(service, action, proposal_event_id) details = {"mark_reviewed": "triage_mark_reviewed", "suppress": "triage_suppress", "unsuppress": "triage_unsuppress"} service.store.record_event( claim_id=claim_id, event_type="audit", details=details[action], payload={"source": "dashboard"} @@ -45,13 +48,13 @@ def apply_triage_action( return {"ok": True, "action": action, "claim_id": claim_id} -def _resolve_proposal(service: Any, action: str, claim_id: int) -> dict[str, Any]: +def _resolve_proposal(service: Any, action: str, proposal_event_id: int) -> dict[str, Any]: from memorymaster.govern.steward import resolve_steward_proposal result = resolve_steward_proposal( service, action="approve" if action == "approve_proposal" else "reject", - claim_id=claim_id, + proposal_event_id=proposal_event_id, apply_on_approve=True, ) return {"ok": True, "action": action, "result": result} diff --git a/tests/test_dashboard_coverage.py b/tests/test_dashboard_coverage.py index 05b2a0ce..36c5bd3c 100644 --- a/tests/test_dashboard_coverage.py +++ b/tests/test_dashboard_coverage.py @@ -374,7 +374,7 @@ def test_triage_action_can_approve_steward_proposal(monkeypatch: Any, tmp_path: def fake_resolve_steward_proposal(service: FakeService, **kwargs: Any) -> dict[str, Any]: calls.append({"service": service, **kwargs}) - return {"claim_id": kwargs["claim_id"], "applied": True} + return {"proposal_event_id": kwargs["proposal_event_id"], "applied": True} monkeypatch.setattr( "memorymaster.govern.steward.resolve_steward_proposal", @@ -386,10 +386,10 @@ def fake_resolve_steward_proposal(service: FakeService, **kwargs: Any) -> dict[s payload = post_json( base_url, "/api/triage/action", - {"claim_id": 2, "action": "approve_proposal"}, + {"claim_id": 2, "proposal_event_id": 12, "action": "approve_proposal"}, ) assert payload["ok"] is True assert payload["action"] == "approve_proposal" - assert payload["result"] == {"claim_id": 2, "applied": True} - assert calls == [{"service": service, "action": "approve", "claim_id": 2, "apply_on_approve": True}] + assert payload["result"] == {"proposal_event_id": 12, "applied": True} + assert calls == [{"service": service, "action": "approve", "proposal_event_id": 12, "apply_on_approve": True}] diff --git a/tests/test_dashboard_governance_ux.py b/tests/test_dashboard_governance_ux.py index 20f805dc..1236569a 100644 --- a/tests/test_dashboard_governance_ux.py +++ b/tests/test_dashboard_governance_ux.py @@ -8,7 +8,7 @@ import pytest from memorymaster.surfaces.dashboard import DashboardRequestHandler -from memorymaster.surfaces import dashboard_read_models +from memorymaster.surfaces import dashboard_commands, dashboard_read_models def _dashboard_html() -> str: @@ -61,6 +61,12 @@ def test_controls_live_regions_and_mobile_layout_are_accessible() -> None: assert 'role="status"' in html +def test_archived_and_stopped_text_meet_the_normal_text_contrast_token() -> None: + html = _dashboard_html() + assert ".badge-archived{background:#1e293b;color:#94a3b8" in html + assert 'span[style="color:#64748b"]{color:#94a3b8!important}' in html + + def test_governance_evidence_and_consequences_are_directly_inspectable() -> None: html = _dashboard_html() assert "Evidence & lineage" in html @@ -69,7 +75,7 @@ def test_governance_evidence_and_consequences_are_directly_inspectable() -> None assert "Action consequences" in html assert 'class="conflict-compare"' in html assert "proposalEvidence(p)" in html - assert "const proposalActions=p?" in html + assert "const proposalActions=proposalId>0?" in html def test_review_queue_exposes_only_the_latest_pending_proposal(monkeypatch: pytest.MonkeyPatch) -> None: @@ -109,10 +115,41 @@ def test_review_queue_exposes_only_the_latest_pending_proposal(monkeypatch: pyte assert payload["items"][0]["proposal"]["reasons"][0]["message"] == "Retention elapsed" +def test_proposal_action_resolves_the_exact_displayed_event(monkeypatch: pytest.MonkeyPatch) -> None: + calls: list[dict[str, object]] = [] + + def fake_resolve(_service: object, **kwargs: object) -> dict[str, object]: + calls.append(kwargs) + return {"resolved": True, "proposal_event_id": kwargs.get("proposal_event_id")} + + monkeypatch.setattr("memorymaster.govern.steward.resolve_steward_proposal", fake_resolve) + result = dashboard_commands.apply_triage_action( + object(), + {"claim_id": 7, "proposal_event_id": 9, "action": "approve_proposal"}, + serialize_claim=lambda claim: claim, + ) + + assert result["result"]["proposal_event_id"] == 9 + assert calls == [{"action": "approve", "proposal_event_id": 9, "apply_on_approve": True}] + with pytest.raises(ValueError, match="proposal_event_id must be positive"): + dashboard_commands.apply_triage_action( + object(), + {"claim_id": 7, "action": "approve_proposal"}, + serialize_claim=lambda claim: claim, + ) + for malformed_id in (True, 9.7, "9"): + with pytest.raises(ValueError, match="proposal_event_id must be positive"): + dashboard_commands.apply_triage_action( + object(), + {"claim_id": 7, "proposal_event_id": malformed_id, "action": "approve_proposal"}, + serialize_claim=lambda claim: claim, + ) + + def test_browser_preserves_failed_review_action_and_distinguishes_load_error() -> None: sync_api = pytest.importorskip("playwright.sync_api") html = _dashboard_html() - state = {"action_calls": 0, "resolved": False} + state = {"action_calls": 0, "resolved": False, "proposal_event_id": None} conflict = { "id": 4, "status": "conflicted", @@ -157,6 +194,7 @@ def route_request(route: object) -> None: return if path.startswith("/api/triage/action"): state["action_calls"] += 1 + state["proposal_event_id"] = request.post_data_json.get("proposal_event_id") if state["action_calls"] == 1: route.fulfill(status=409, content_type="application/json", body='{"ok":false,"error":"rejected"}') else: @@ -172,6 +210,7 @@ def route_request(route: object) -> None: page.goto("http://dashboard.test/") page.get_by_role("button", name="Approve proposal for claim 7").click() page.get_by_text("Action failed for claim 7").wait_for() + assert state["proposal_event_id"] == 9 assert page.locator('tr[data-claim-id="7"]').count() == 1 assert page.get_by_role("button", name="Approve proposal for claim 7").text_content() == "Retry" assert "Could not load claims" in page.locator("#claims-body").inner_text() diff --git a/tests/test_steward_resolution_parity.py b/tests/test_steward_resolution_parity.py index 21456a22..5c918ccc 100644 --- a/tests/test_steward_resolution_parity.py +++ b/tests/test_steward_resolution_parity.py @@ -177,13 +177,18 @@ def test_proposal_resolution_dashboard_parity() -> None: proposals = list_steward_proposals(service, limit=20, include_resolved=False) assert proposals target_claim_id = int(proposals[0]["claim_id"]) + proposal_event_id = int(proposals[0]["proposal_event_id"]) operator_log = workspace / "operator_events.jsonl" operator_log.write_text("", encoding="utf-8") with _running_server(service, operator_log) as base_url: status, payload = _post_json( f"{base_url}/api/triage/action", - {"claim_id": target_claim_id, "action": "approve_proposal"}, + { + "claim_id": target_claim_id, + "proposal_event_id": proposal_event_id, + "action": "approve_proposal", + }, ) assert status == 200 assert payload["ok"] is True From 70157c7e3e8eadad3f92596f217da534da452a1f Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Tue, 14 Jul 2026 10:32:56 -0300 Subject: [PATCH 48/55] docs: prioritize personal local sqlite profile --- .../pr-release-draft.md | 11 ++-- .../production-release-approval-request.md | 33 +++++----- .../release-candidate-readiness.md | 16 +++-- .../remaining-blockers.md | 28 +++++---- INSTALLATION.md | 30 +++++----- README.md | 49 ++++++++------- ROADMAP.md | 24 ++++---- docs/AGENT-INSTALL.md | 60 +++++++++---------- docs/adr/0001-local-first-single-database.md | 19 +++++- docs/deployment_profiles.md | 19 ++++-- external-actions-required.md | 14 +++++ memorymaster/surfaces/setup_hooks.py | 2 +- 12 files changed, 186 insertions(+), 119 deletions(-) diff --git a/.planning/audits/2026-07-14-phase4-budget-delta/pr-release-draft.md b/.planning/audits/2026-07-14-phase4-budget-delta/pr-release-draft.md index 512fe38c..035772ed 100644 --- a/.planning/audits/2026-07-14-phase4-budget-delta/pr-release-draft.md +++ b/.planning/audits/2026-07-14-phase4-budget-delta/pr-release-draft.md @@ -24,8 +24,9 @@ stateful accessible governance UX, and generated release truth. ## Release caveat -This PR is a local release candidate, not production authorization. The -external Postgres, Qdrant, credential/history/dependency/image scan, -Kubernetes, recovery, privacy/legal, and publication gates remain listed in -`external-actions-required.md`. No push, tag, publication, deployment, or live -data mutation was performed by the remediation goal. +This PR targets the personal/local SQLite minimal profile. History, +minimal-extra dependency, verified-artifact, and publication evidence still +gate an external package release. Postgres/team, Qdrant/semantic, +Kubernetes/Helm, and organization/privacy evidence is deferred and blocks only +claims about those optional profiles. No push, tag, publication, deployment, +or live-data mutation was performed by the remediation goal. diff --git a/.planning/audits/2026-07-14-phase4-budget-delta/production-release-approval-request.md b/.planning/audits/2026-07-14-phase4-budget-delta/production-release-approval-request.md index 6074acc4..2e39fb57 100644 --- a/.planning/audits/2026-07-14-phase4-budget-delta/production-release-approval-request.md +++ b/.planning/audits/2026-07-14-phase4-budget-delta/production-release-approval-request.md @@ -1,8 +1,9 @@ # Ready-to-Paste Production/Release Approval Request ```text -Request: authorize the external release-candidate verification sequence for -MemoryMaster 4.4.1. Do not authorize production cutover yet. +Request: authorize the personal/local SQLite release-candidate verification +sequence for MemoryMaster 4.4.1. Do not authorize hosted deployment or live DB +mutation. Local status: Phase 2–4 remediation and final targeted audit converged. The latest audit has zero new findings and no unresolved reproducible @@ -11,25 +12,27 @@ Ruff, browser/a11y, clean-wheel, generated-truth, Compose, and disposable container/database gates passed. Approval requested for these controlled external actions only: -1. Resolve/classify credential-history, Gitleaks, dependency, and immutable - release-image scan blockers. -2. Run disposable two-role Postgres RLS/parity and authenticated/TLS Qdrant - governed-retrieval/reconciliation gates. -3. Run approved immutable-image Kubernetes/Helm and off-device/Postgres - recovery drills. -4. Obtain product/legal privacy, consent, retention, and data-disposition - decisions. -5. If and only if all blockers are resolved or formally accepted, approve one +1. Classify the history/Gitleaks findings and run the strict dependency audit + against the exact minimal SQLite package extras. +2. Verify the wheel/sdist hashes and clean-install evidence produced by the + release workflow. +3. If and only if the minimal-profile blockers are resolved or formally + accepted, approve one tag-triggered verified-artifact workflow execution. Manual dispatch must not publish; PyPI must receive the exact promoted verified-dist bytes. +Explicitly deferred and not required for this scoped release: Postgres/team, +Qdrant/semantic, Docker/Kubernetes/Helm, hosted multi-user recovery, and +organization-wide privacy/compliance claims. Those profiles remain disabled +and must undergo their own approval sequence if revisited. + Still prohibited without a separate explicit approval: production cutover, live migration or cleanup, redaction, archival, retention deletion, backlog mutation, credential rotation execution, history rewrite, or product-data mutation. -Required before cutover approval: exact final commit, artifact/image digests, -external gate evidence, staging/canary health and MCP handshake, migration and -integrity evidence, telemetry/alert delivery, and the reviewed rollback -command/owner. +Required before package-publication approval: exact final commit, wheel/sdist +digests, minimal-profile gate evidence, clean-install/MCP smoke, and the +reviewed rollback instructions. A future hosted cutover would require a +separate request. ``` diff --git a/.planning/audits/2026-07-14-phase4-budget-delta/release-candidate-readiness.md b/.planning/audits/2026-07-14-phase4-budget-delta/release-candidate-readiness.md index a41e0ac0..9510255d 100644 --- a/.planning/audits/2026-07-14-phase4-budget-delta/release-candidate-readiness.md +++ b/.planning/audits/2026-07-14-phase4-budget-delta/release-candidate-readiness.md @@ -2,12 +2,15 @@ ## Decision -**LOCAL RC READY; PRODUCTION/EXTERNAL RELEASE BLOCKED.** +**PERSONAL/LOCAL SQLITE RC READY; EXTERNAL PUBLICATION STILL GATED.** -The branch is locally releasable at each atomic package checkpoint and passes -the Phase 4 convergence boundary. It must not be pushed, tagged, published, or -deployed until the prioritized external blockers are resolved or explicitly -accepted by their authorized owners. +The branch passes the Phase 4 convergence boundary for the primary minimal +profile: personal/local SQLite plus private stdio MCP. It must not be pushed, +tagged, or published until the minimal package's history, dependency, artifact, +and publication gates are resolved or explicitly accepted. + +Postgres/team, Qdrant/semantic, and Kubernetes/Helm evidence is deferred. It +blocks claims about those optional profiles, not the scoped SQLite-local RC. ## Candidate identity @@ -39,4 +42,5 @@ bytes only after approval. ## Not proven locally See `remaining-blockers.md` and the canonical -`external-actions-required.md`. These gaps prevent a production-ready claim. +`external-actions-required.md`. The first section gates publishing the minimal +package; the deferred section gates only optional profiles. diff --git a/.planning/audits/2026-07-14-phase4-budget-delta/remaining-blockers.md b/.planning/audits/2026-07-14-phase4-budget-delta/remaining-blockers.md index 02e81880..ea414d94 100644 --- a/.planning/audits/2026-07-14-phase4-budget-delta/remaining-blockers.md +++ b/.planning/audits/2026-07-14-phase4-budget-delta/remaining-blockers.md @@ -1,18 +1,26 @@ # Prioritized Remaining-Blocker Ledger -This is a release-oriented view. `external-actions-required.md` remains the -canonical owner/evidence ledger. +This is a release-oriented view for the primary personal/local SQLite profile. +`external-actions-required.md` remains the canonical owner/evidence ledger. -| Priority | Finding(s) | Blocking evidence/action | +## Blocks publishing the minimal package + +| Priority | Finding(s) | Required evidence/action | |---|---|---| -| P0 | MM-OPS-01, MM-OPS-04 | Rotate/classify historical credentials and Gitleaks findings; complete strict dependency and exact release-image scans | -| P0 | MM-SEC-01 | Disposable two-role Postgres RLS/parity proof and approved brownfield inventory/repair evidence | -| P0 | MM-SEC-02 | Disposable authenticated/TLS Qdrant governed retrieval, reconciliation, outbox, sync, and delete proof | -| P0 | MM-SEC-03 | Authorized aggregate legacy-store inventory completion, including Qdrant; separately approve any cleanup/redaction | -| P1 | MM-OPS-02, MM-OPS-03 | Approved immutable images, disposable Kubernetes/Helm proof, then explicit tag/PyPI workflow approval | -| P1 | MM-OPS-05 | Approved off-device encrypted destination plus SQLite/Postgres restore drill | -| P1 | MM-PRIV-01, MM-PRIV-02 | Product/legal consent, processor, retention, attribution, export/erase disposition, and backend dry-run decisions | +| P0 | MM-OPS-04 | Classify the unsuppressed history findings and complete a strict dependency audit for the exact minimal release extras | +| P0 | MM-OPS-03 | Execute the tag-triggered verified-artifact workflow only after explicit approval; publish the exact promoted bytes | +| P1 | MM-SEC-03, MM-DATA-01 | Before migrating or remediating the user's active SQLite DB, approve and verify a backup plus read-only inventory; not a package-publication gate | | P2 | MM-CAP-01 | Address or monitor host-storage capacity without deleting product data | +## Deferred optional-profile evidence + +| Profile | Findings | Deferred evidence | +|---|---|---| +| Team/Postgres | MM-SEC-01, MM-OPS-01 | Two-role RLS/parity, brownfield inventory, credential/network, and Postgres recovery proof | +| Semantic/Qdrant | MM-SEC-02 | Authenticated/TLS governed candidate, reconciliation, outbox, sync, and delete proof | +| Hosted/container | MM-OPS-02, MM-OPS-04 image scan | Approved immutable images, scans, Kubernetes/Helm runtime and network-policy proof | +| Organization/privacy | MM-PRIV-01, MM-PRIV-02 | Organizational consent, processor, retention, export/erase, and backend-disposition decisions | +| Extended recovery | MM-OPS-05 | Approved off-device destination and PostgreSQL restore drill | + No blocker above authorizes a mutation. Each external owner must approve the specific action and retain the evidence requested by the canonical ledger. diff --git a/INSTALLATION.md b/INSTALLATION.md index dece6988..e7a77555 100644 --- a/INSTALLATION.md +++ b/INSTALLATION.md @@ -19,7 +19,7 @@ pip install memorymaster # MCP server for Claude Code / Codex pip install "memorymaster[mcp]" -# Postgres backend +# Deferred Postgres/team backend pip install "memorymaster[postgres]" # Vector embeddings (sentence-transformers) @@ -28,13 +28,13 @@ pip install "memorymaster[embeddings]" # Gemini embeddings / LLM pip install "memorymaster[gemini]" -# Qdrant maintenance-index client (payload retrieval is quarantined in R1.3) +# Optional governed Qdrant semantic client pip install "memorymaster[qdrant]" # Fernet encryption for sensitive payloads pip install "memorymaster[security]" -# Everything +# All optional/deferred integrations (not recommended for normal local use) pip install "memorymaster[mcp,postgres,embeddings,gemini,qdrant,security]" # Development (matches CI — required extras for the full test suite) @@ -60,7 +60,7 @@ Claude Code (hooks, MCP server, steward cron) and optionally Codex. into Claude Code or Codex): ```bash -memorymaster-setup --yes --full-stack --json +memorymaster-setup --yes --profile minimal --no-full-stack --json ``` **Manual / interactive:** @@ -77,12 +77,13 @@ a 3-line shim that calls the same `memorymaster.surfaces.setup_hooks:main` funct | Flag | Default | Description | |------|---------|-------------| | `-y` / `--yes` | off | Non-interactive; accept all defaults (no `input()` prompts) | +| `--profile {minimal,semantic,team,full-lab}` | `minimal` | Select the setup contract to verify | | `--db PATH` | `/memorymaster.db` | Path to the SQLite database | | `--provider {google,openai,anthropic,ollama}` | prompted | LLM provider for the auto-ingest Stop hook | | `--api-key KEY` | prompted | API key for the chosen provider | | `--model MODEL` | provider default | LLM model id | | `--project-root PATH` | cwd | Directory where `memorymaster.db` lives | -| `--full-stack` | on | Bring up the Qdrant maintenance index + Ollama via Docker Compose | +| `--full-stack` | off for `minimal`/`team`; on for `semantic`/`full-lab` | Bring up optional Qdrant + Ollama | | `--no-full-stack` | off | Skip the Qdrant-index + local-LLM stack | | `--no-cron` | off | Skip steward cron setup | | `--no-obsidian-skills` | off | Skip Obsidian skills install | @@ -101,10 +102,12 @@ a 3-line shim that calls the same `memorymaster.surfaces.setup_hooks:main` funct - Optionally installs the steward cron (Linux/macOS) or Task Scheduler job (Windows). - Runs a sentinel round-trip verify at the end (`--verify-only` to run this step alone). -#### No-Docker degraded mode +#### Optional semantic services -If Docker is absent and Qdrant/Ollama are not already running, the installer -continues without them and prints: +The primary minimal profile does not require Docker, Qdrant, or Ollama and +their absence is not degraded operation. If an explicitly selected semantic or +full-lab profile cannot reach them, the installer continues with authoritative +SQLite retrieval and prints: ``` Running in SQLite-only mode. Qdrant index maintenance + local LLM auto-ingest are OFF. @@ -112,15 +115,14 @@ Running in SQLite-only mode. Qdrant index maintenance + local LLM auto-ingest ar maintenance or local LLMs, use --full-stack or QDRANT_URL / OLLAMA_URL. ``` -Setup exits 0. Core hooks, MCP, and SQLite-based recall all work normally in -degraded mode. Installing or starting Qdrant enables only index maintenance -during R1.3; it does not re-enable claim, context-fallback, or verbatim payload -retrieval. +Core hooks, MCP, and SQLite-based recall continue to work. Governed Qdrant +candidate reads require an explicit gate and always rehydrate/filter through +the authoritative store. ## Docker Compose -The included `docker-compose.yml` runs MemoryMaster plus the optional Qdrant -maintenance index and Ollama. Qdrant payload retrieval remains quarantined. +The included `docker-compose.yml` is for an explicitly selected semantic or +hosted experiment. It is not part of the personal/local minimal installation. ```bash # Clone the repo diff --git a/README.md b/README.md index 7621add7..b64cb640 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,11 @@ Lifecycle-managed claims with citations, conflict detection, steward governance, MemoryMaster prevents the #1 problem with agent memory: **drift, stale assumptions, and unsafe disclosure**. It gives Claude Code, Codex, and any MCP-compatible agent persistent, verifiable memory with a full claim lifecycle, citation tracking, conflict detection, and human-in-the-loop governance. +> **Product posture:** MemoryMaster is primarily a personal/local application. +> The default profile uses one SQLite database and a private stdio MCP process. +> Postgres/team operation is a deferred optional capability, and Qdrant is an +> optional semantic index—not a dependency or source of truth. + ### How it's different Agent-memory systems (mem0, Letta/MemGPT, Zep, cognee) have largely converged on strong **retrieval** — hybrid search, temporal and graph reasoning, ontologies. MemoryMaster competes on a different axis: **governance**. The wedge is **curation over accumulation** — every memory is a lifecycle-managed *claim* (status, citations, decay, conflict arbitration), not an opaque embedding that lingers until overwritten. Tellingly, the market moved the other way: mem0's 2026 model is explicitly *add-only with no conflict resolution* — the opposite of a steward. @@ -32,8 +37,8 @@ If you want an agent that recalls more, any vector store works. If you want an a ## Architecture -MemoryMaster is layered around MCP/CLI entry points, the `MemoryService` facade, SQLite/Postgres -storage, an optional Qdrant index, scheduled jobs, and an **optional** Obsidian wiki/vault +MemoryMaster is layered around MCP/CLI entry points, the `MemoryService` facade, authoritative +SQLite storage, an optional deferred Postgres/team backend, an optional Qdrant index, scheduled jobs, and an **optional** Obsidian wiki/vault layer (opt-in, off by default — see below). The canonical ingest path is: ```text @@ -43,18 +48,15 @@ MCP/CLI -> sensitivity filter -> MemoryService.ingest -> store write -> FTS5 ind The query path is: ```text -query_memory -> MemoryService.query -> authorized SQLite/Postgres rows -> lexical/local-hybrid ranking -> context +query_memory -> MemoryService.query -> authorized SQLite rows -> lexical/local-hybrid ranking -> context ``` -R1.3 quarantines every Qdrant-backed claim or verbatim payload retrieval path. -In local-trusted mode, a requested Qdrant claim search falls back to -authoritative lexical retrieval; direct Qdrant search adapters deny the -request; prompt-context vector fallback is disconnected; and verbatim -`vector`/`hybrid` requests fall back to FTS5. Team MCP rejects semantic -retrieval entirely. Qdrant upsert, `qdrant-sync`, and `qdrant-reconcile` -remain available for index maintenance, including count/ID drift reads. R2.1 -may re-enable payload retrieval only as ID candidates that are rehydrated and -policy-filtered through SQLite/Postgres. +Qdrant candidate retrieval is disabled by default. When a user deliberately +enables the governed semantic profile, Qdrant may return only candidate IDs, +content hashes, and scores. Every candidate is rehydrated from the authoritative +store and rechecked for lifecycle, tenant, scope, visibility, sensitivity, and +exact content hash. The semantic profile remains unproven for production until +the separately documented authenticated/TLS gate passes. See [docs/architecture.md](docs/architecture.md) for the current module map, data-flow details, recent PR status, and sensitivity-filter invariants. @@ -63,7 +65,7 @@ recent PR status, and sensitivity-filter invariants. - **6-state lifecycle**: `candidate` → `confirmed` → `stale` → `superseded` → `conflicted` → `archived` - **Citation tracking** with provenance for every claim -- **Hybrid retrieval**: authoritative claim rows ranked with FTS5, local/primary-store embedding signals, freshness, and confidence; it does not read Qdrant while R1.3 containment is active +- **Hybrid retrieval**: authoritative SQLite claim rows ranked with FTS5, local/primary-store embedding signals, freshness, and confidence; governed Qdrant candidates are explicit and optional - **Context optimizer**: `query_for_context(budget=4000)` returns auto-curated memory that fits your token budget - **Entity graph** with typed relationships and alias resolution - **Rule-shaped claims** (new in v3.21.0): prescriptive `when , do because ` claims (`ingest_rule` / `query_rules`) — the shape an agent needs to actually change behaviour next time, not just recall a fact @@ -87,7 +89,7 @@ recent PR status, and sensitivity-filter invariants. - **Hebbian/Ebbinghaus entity edges** (new in v4.2.0, opt-in `MEMORYMASTER_HEBBIAN_DECAY`): usage strengthens, time decays entity-graph edge weights. - **Proactive + tool-triggered recall** (new in v4.2.0): a `volunteer_context` MCP tool (confidence-gated, zero-LLM) and an opt-in PreToolUse hook (`MEMORYMASTER_PRETOOLUSE_RECALL`) that injects memory as `additionalContext` on Grep/Glob. - **Belief `holder`** (new in v4.2.0): nullable per-claim `holder` for multi-holder beliefs (take/fact/bet/hunch reuse `claim_type`); SQLite+Postgres, ranking-neutral by default. -- **Dual backend**: SQLite for local-trusted/single-agent use; PostgreSQL for authenticated team runtime with tenant, principal, and explicit-scope isolation +- **SQLite-first backend**: SQLite for the primary personal/local product; PostgreSQL team support is retained but deferred until a real multi-user use case and its external evidence exist - **Dream Bridge** for bidirectional sync with Claude Code's Auto Dream - **Hook stack**: recall, classify, validate-wiki, session-start, auto-ingest, precompact (settings.json) + steward-cycle (cron/schtasks) + opt-in `--pretooluse` grep/glob recall-inject @@ -122,7 +124,7 @@ Reproduce: `python tests/bench_longmemeval.py --retrieval-only`. Full methodolog **Optional (nice to have)** -- **Docker** for Qdrant index maintenance and Ollama-backed jobs. SQLite/Postgres remain authoritative. Claim, prompt-context, and verbatim retrieval from Qdrant are temporarily quarantined until the governed R2.1 planner can enforce lifecycle, tenant, scope, visibility, and sensitivity policy; sync/reconcile remain available. +- **Docker** only if you deliberately want local Ollama or the optional governed Qdrant semantic profile. SQLite remains authoritative and requires neither Docker nor Qdrant. ## 30-second quickstart @@ -135,7 +137,7 @@ pip install "memorymaster[mcp,security,qdrant,embeddings]" **2. Let your agent do the rest** Paste the contents of [`docs/AGENT-INSTALL.md`](docs/AGENT-INSTALL.md) into Claude Code or Codex. The agent will: -- run `memorymaster-setup --yes --full-stack --json` (detects your environment, wires hooks + MCP, starts Qdrant + Ollama if Docker is present, or falls back to SQLite-only mode gracefully) +- run `memorymaster-setup --yes --profile minimal --no-full-stack --json` (wires the SQLite database and private MCP without starting Postgres, Qdrant, or Ollama) - report what was wired, what was reused (brownfield), and what degraded - run `memorymaster-setup --verify-only` and show the round-trip result @@ -204,8 +206,13 @@ See [`docs/MCP-TOOLS.md`](docs/MCP-TOOLS.md) for the grouped reference (one line | Backend | Install | Use case | |---------|---------|----------| -| **SQLite** | Built-in | Local-trusted development, single-agent, zero-config | -| **Postgres 16.x** | `pip install "memorymaster[postgres]"` | Authenticated team deployment with isolated app/migrator roles | +| **SQLite** | Built-in | **Primary profile:** personal/local, private, zero-config | +| **Postgres 16.x** | `pip install "memorymaster[postgres]"` | **Deferred:** future authenticated team deployment with isolated app/migrator roles | + +Postgres is not required for normal MemoryMaster use and is not currently a +release target. The detailed contract below is retained so the dormant profile +fails closed and can be revisited safely if a genuine shared-service use case +appears. PostgreSQL v0011 enables and forces row-level security. Reads are tenant/scope bounded and expose public claims or the principal's own private claims; writes @@ -230,13 +237,15 @@ disposable database; repository tests do not constitute a production proof. ## Docker Compose -Run MemoryMaster with the optional Qdrant maintenance index and Ollama with one command: +For an explicit experimental semantic profile, run the optional Qdrant index +and Ollama stack with: ```bash docker compose up -d ``` -Starting Qdrant does not re-enable Qdrant retrieval during R1.3 containment. +Starting Qdrant alone does not enable governed retrieval. The minimal local +profile should not start this stack. See [INSTALLATION.md](INSTALLATION.md) for Kubernetes / Helm. diff --git a/ROADMAP.md b/ROADMAP.md index a5dfb68b..c877bbd9 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -5,27 +5,29 @@ history and `docs/archive/`; they are evidence, not promises. ## Now -- Complete the Phase 4 local convergence gate and publish its audit delta, - release-candidate readiness report, blocker ledger, and rollback plan. -- Keep the local release candidate immutable while owners resolve or explicitly - accept the external production blockers in `external-actions-required.md`. +- Ship and operate the personal/local minimal profile: one SQLite database, + private stdio MCP, and no required external database or vector service. +- Resolve only the supply-chain and publication evidence that applies to the + local package; keep optional-profile blockers classified separately. - Preserve governed retrieval, lifecycle authority, tenant/principal isolation, finite capture budgets, and fail-closed production evidence defaults. ## Next -- Obtain disposable authenticated/TLS Qdrant and two-role PostgreSQL evidence. -- Validate the immutable image and Helm profiles in an approved disposable - Kubernetes environment. -- Prove off-device recovery and select an organization telemetry/error backend. -- Complete product-owner/legal decisions for consent, processor disclosure, - retention, export, and erasure before enabling production capture. +- Improve personal/local backup guidance and verify restore on a disposable + copy before migrating the user's active SQLite database. +- Keep semantic recall optional and disabled unless a local user deliberately + configures a governed Qdrant/provider profile. - Execute the verified release workflow only after explicit release approval. ## Later - Continue the measured service-facade decomposition without breaking the compatibility surface. +- Revisit shared multi-user/team operation only if a real use case appears; + its Postgres, RLS, identity, deployment, and recovery gates remain deferred. +- Revisit authenticated Qdrant, immutable container images, and Kubernetes/Helm + only for an explicitly selected semantic or hosted profile. - Improve entity aliases and steward classification only against versioned, reproducible evaluation datasets. - Expand companion integrations through the documented provider protocols and @@ -39,3 +41,5 @@ history and `docs/archive/`; they are evidence, not promises. truth that bypasses authoritative rehydration and governance. - A second authoritative roadmap, another default vector database, or a flag-day rewrite of `MemoryService`. +- Making Postgres, Qdrant, containers, or multi-user operation a dependency of + the personal/local minimal profile. diff --git a/docs/AGENT-INSTALL.md b/docs/AGENT-INSTALL.md index 6b10316d..e04e4973 100644 --- a/docs/AGENT-INSTALL.md +++ b/docs/AGENT-INSTALL.md @@ -4,10 +4,9 @@ Copy the appropriate block below and paste it as a prompt to your coding agent. The agent will run the installer, read the machine-readable result, and report what was wired, what was reused (brownfield), and what degraded. -R1.3 safety note: Qdrant is an optional maintenance index. Installing its -extra or starting Docker does not enable claim, context-fallback, or verbatim -payload retrieval. Local-trusted Qdrant claim requests use lexical fallback; -team semantic MCP requests are denied. Sync/reconcile remain available. +MemoryMaster's primary profile is personal/local: SQLite plus private stdio +MCP. Postgres/team is deferred. Qdrant is optional and governed; enabling it +requires an explicit semantic profile and does not make it authoritative. --- @@ -16,25 +15,24 @@ team semantic MCP requests are denied. Sync/reconcile remain available. ``` Install MemoryMaster in this environment. Follow these steps exactly. -1. Install the package with the full-stack extras: +1. Install the personal/local package extras: - pip install "memorymaster[mcp,security,qdrant,embeddings]" + pip install "memorymaster[mcp,security]" 2. Run the detect-first, non-interactive installer and capture the JSON output: - memorymaster-setup --yes --full-stack --json + memorymaster-setup --yes --profile minimal --no-full-stack --json The installer prints human-readable progress to stderr and emits a single JSON document to stdout. Parse the JSON and report back to me: - - detected: which tools were found (Docker, Qdrant, Ollama, Claude Code, - Codex, existing hooks, MCP registration state). + - detected: which clients and optional tools were found. Docker, Qdrant, + Ollama, and Postgres may be reported but must not be started for minimal. - planned: the list of [will-do / skip-present / cant-missing] lines the installer printed — paste them verbatim so I can see the plan. - applied: what was actually wired (hooks, mcp_claude, cron, full_stack, etc.). - - degraded: true/false. If true, explain the reason from the JSON - (typically: Docker not found → SQLite-only mode, Qdrant index maintenance - + local LLM auto-ingest are off). + - degraded: true/false. Missing optional Postgres, Qdrant, Ollama, or Docker + must not make the minimal profile degraded. - verify.status: PASS, PARTIAL, or FAIL and the detail string. 3. Run the verify round-trip to confirm hooks and DB are functional: @@ -49,10 +47,8 @@ Install MemoryMaster in this environment. Follow these steps exactly. Notes for you (the agent): - The JSON payload shape is: {"detected": {...}, "planned": [...], "applied": {...}, "verify": {"status": "...", "detail": "...", "mcp_note": "..."}, "degraded": bool} -- If --full-stack brings up Docker services it may take up to 2 minutes; wait - for the process to exit before reading stdout. -- If degraded is true the install still succeeded (exit 0). Report the - degraded message from applied.full_stack.message so I know what to do next. +- Do not add `--full-stack`, Postgres, Qdrant, or provider extras unless I ask + for a different profile. - The MCP server is registered as memorymaster.surfaces.mcp_server (not the deprecated memorymaster.mcp_server path). Do not edit the MCP entry manually. - Hooks installed: UserPromptSubmit (recall + classify), PostToolUse @@ -69,20 +65,20 @@ Notes for you (the agent): ``` Install MemoryMaster in this environment. Follow these steps exactly. -1. Install the package with the full-stack extras: +1. Install the personal/local package extras: - pip install "memorymaster[mcp,security,qdrant,embeddings]" + pip install "memorymaster[mcp,security]" 2. Run the detect-first, non-interactive installer and capture the JSON output: - memorymaster-setup --yes --full-stack --codex --json + memorymaster-setup --yes --profile minimal --no-full-stack --codex --json The --codex flag ensures ~/.codex/config.toml is updated with the MCP server entry (Codex reads MCP config from there, not from ~/.claude.json). Parse the JSON and report back to me: - - detected: which tools were found (Docker, Qdrant, Ollama, Codex - presence, MCP registration state). + - detected: Codex presence and MCP registration state. Optional services + may be reported but must not be started. - planned: paste the [will-do / skip-present / cant-missing] lines verbatim. - applied: what was actually wired (mcp_codex, cron, full_stack, etc.). - degraded: true/false and the reason if true. @@ -120,10 +116,8 @@ Notes for you (the agent): block so re-runs are idempotent and your existing config is preserved. - The MCP server command registered is the same non-deprecated path: python -m memorymaster.surfaces.mcp_server -- If degraded is true (Docker absent / services unreachable) the install still - succeeded (exit 0). SQLite-only mode is fully functional; Qdrant index - maintenance and local LLM auto-ingest are off. Starting Qdrant later does not - lift the R1.3 payload-retrieval quarantine. +- The minimal SQLite profile is fully functional without Docker, Qdrant, + Ollama, or Postgres. Their absence is not degraded local operation. - Claude Code hooks (UserPromptSubmit, Stop, SessionStart, PreCompact) are NOT registered for Codex — those are Claude Code-specific. The session-end script above is the Codex equivalent for distilled ingest. @@ -136,12 +130,13 @@ Notes for you (the agent): | Flag | Effect | |---|---| | `-y` / `--yes` | Non-interactive; accept all defaults, no prompts | +| `--profile {minimal,semantic,team,full-lab}` | Select the profile to verify; defaults to `minimal` | | `--db PATH` | Path to `memorymaster.db` (default: `/memorymaster.db`) | | `--provider {google,openai,anthropic,ollama}` | LLM provider for the auto-ingest Stop hook | | `--api-key KEY` | API key for the chosen provider | | `--model MODEL` | LLM model id | | `--project-root PATH` | Directory where `memorymaster.db` lives | -| `--full-stack` | Bring up the Qdrant maintenance index + Ollama via Docker Compose (default when omitted) | +| `--full-stack` | Bring up optional Qdrant + Ollama for `semantic`/`full-lab` | | `--no-full-stack` | Skip the Qdrant-index + local-LLM stack | | `--no-cron` | Skip steward cron setup | | `--no-obsidian-skills` | Skip Obsidian skills install | @@ -151,15 +146,16 @@ Notes for you (the agent): | `--verify-only` | Run only the sentinel round-trip and exit | | `--json` | Emit machine-readable JSON result to stdout; human output goes to stderr | -## Degraded mode +## Optional semantic mode -If Docker is absent and Qdrant/Ollama are not already reachable at -`QDRANT_URL`/`OLLAMA_URL`, the installer continues in SQLite-only mode: +The minimal profile neither requires nor starts Docker/Qdrant/Ollama. If an +explicit semantic/full-lab setup cannot reach those optional services, it +continues with authoritative SQLite retrieval and reports: > Running in SQLite-only mode. Qdrant index maintenance + local LLM auto-ingest are OFF. > Retrieval remains available through authoritative SQLite ranking. To enable index > maintenance or local LLMs, use `--full-stack` or QDRANT_URL / OLLAMA_URL. -The exit code is still 0. Core claim storage, recall hooks, and MCP tools -remain fully functional. Qdrant payload retrieval remains quarantined even when -the optional service is available. +Core claim storage, recall hooks, and MCP tools remain fully functional. When +governed Qdrant reads are explicitly enabled, candidates are still rehydrated +and filtered through the authoritative store. diff --git a/docs/adr/0001-local-first-single-database.md b/docs/adr/0001-local-first-single-database.md index 29fadf8e..4b6bbe26 100644 --- a/docs/adr/0001-local-first-single-database.md +++ b/docs/adr/0001-local-first-single-database.md @@ -1,10 +1,11 @@ # 0001 Local-First Single Database -Date: 2026-05-05 +Date: 2026-05-05; reaffirmed 2026-07-14 Status: Accepted -Source Claims: claim #35163, claim #34858, claim #35039, claim #36387 +Source Claims: claim #35163, claim #34858, claim #35039, claim #36387, +`mm-69b9~3` ## Context @@ -18,10 +19,24 @@ MemoryMaster is local-first by default and uses one database file for all Memory Postgres parity may exist as an implementation target, but the architecture still models claims and Atlas tables as one logical store. +The primary supported product posture is personal/local use. Shared multi-user +operation is not part of the current product target. The Postgres/team profile +is retained as an optional future capability and must remain disabled until its +separate two-role, row-level-security, and deployment evidence is complete. +That deferred profile does not make Postgres a dependency of the local product. + +Qdrant is also optional. It may augment local SQLite recall with governed +semantic candidates, but it is never authoritative and is not required by the +minimal profile. + ## Consequences This keeps installation and operation simple for local AI-agent workflows and avoids server round trips in the normal path. +Local release decisions are evaluated against the SQLite minimal profile. +Unproven Postgres/team, Qdrant/semantic, or Kubernetes profiles stay visibly +disabled and documented, but do not block an accurately scoped local release. + Consumers must not create a split layout such as one database for claims and another for Atlas Inbox. A split layout creates disconnected state and silently breaks the claim/source relationship. Schema and concurrency changes must preserve WAL behavior and the all-tables-in-one-store assumption. diff --git a/docs/deployment_profiles.md b/docs/deployment_profiles.md index b85894a9..aa442915 100644 --- a/docs/deployment_profiles.md +++ b/docs/deployment_profiles.md @@ -2,6 +2,17 @@ This document defines practical deployment profiles for MemoryMaster with reliability/operability defaults. +## Product Posture + +Profile A is the primary product: one person, one local SQLite database, and a +private stdio MCP process. Profiles B and C are deferred design options, not +current release targets. Their presence documents fail-closed boundaries; it +does not imply that users should provision Postgres, containers, or shared +infrastructure. + +Qdrant is a separate optional semantic enhancement and is not part of Profile +A unless the user explicitly selects and verifies that profile. + ## Secure Container Inputs Container configuration fails closed when required deployment inputs are @@ -58,9 +69,9 @@ python scripts/eval_memorymaster.py --strict python benchmarks/perf_smoke.py ``` -## Profile B: Small Team Server (Postgres) +## Deferred Profile B: Small Team Server (Postgres) -Use when: +Revisit only when: - 2-10 engineers - one shared service host - moderate throughput @@ -81,9 +92,9 @@ Baseline controls: - periodic reconciliation report: - `service.store.reconcile_integrity(fix=False)` in scheduled job -## Profile C: Cloud Managed DB (Postgres) +## Deferred Profile C: Cloud Managed DB (Postgres) -Use when: +Revisit only when: - production workloads - multi-service/multi-agent concurrency - stricter recovery and audit requirements diff --git a/external-actions-required.md b/external-actions-required.md index 7d673773..7d90eb4d 100644 --- a/external-actions-required.md +++ b/external-actions-required.md @@ -5,6 +5,20 @@ for the canonical finding remains `IN-PROGRESS`. The finding itself becomes blocked only after all unblocked repository work closes. No row authorizes a live mutation by itself. +## Applicability to the primary product + +The primary product is the personal/local SQLite minimal profile. Postgres/team, +Qdrant/semantic, Docker/Kubernetes/Helm, and hosted multi-user evidence is +deferred optional-profile work. Those rows remain `BLOCKED-EXTERNAL` so the +project cannot accidentally claim those profiles are ready, but they do not +block an accurately scoped local SQLite package release. + +For the minimal release, prioritize repository/history classification, the +strict dependency audit for the selected minimal extras, verified package +artifacts, and the tag/publication proof. Live-data backup or remediation rows +apply only before the corresponding operation against the user's active local +database; they never authorize that operation. + | Finding | Owner/system | Required action | Evidence needed to unblock | Review date | Status | |---|---|---|---|---|---| | MM-OPS-01 | Operator / any Postgres deployment | Rotate the historical `mm_pw` credential and recreate deployments that used it. Keep database ports private. | Rotation record plus network probe showing the port is not externally reachable | 2026-07-31 | BLOCKED-EXTERNAL | diff --git a/memorymaster/surfaces/setup_hooks.py b/memorymaster/surfaces/setup_hooks.py index e866d8dd..adb41f96 100644 --- a/memorymaster/surfaces/setup_hooks.py +++ b/memorymaster/surfaces/setup_hooks.py @@ -856,7 +856,7 @@ def build_arg_parser() -> argparse.ArgumentParser: dest="full_stack", action="store_true", default=None, - help="bring up Qdrant + Ollama (default)", + help="bring up optional Qdrant + Ollama (semantic/full-lab profiles only)", ) p.add_argument( "--no-full-stack", From 4def72db46e45112274a519d4acc202a40f252c6 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Tue, 14 Jul 2026 11:50:41 -0300 Subject: [PATCH 49/55] security: close minimal release supply-chain gates --- .gitleaks-reviewed-fingerprints | 40 +++++++++++++++++++++++++ docs/security_supply_chain.md | 45 ++++++++++++++++++---------- external-actions-required.md | 4 +-- scripts/run_supply_chain_checks.py | 31 +++++++++++++++---- tests/test_supply_chain_contracts.py | 36 +++++++++++++++++++--- 5 files changed, 129 insertions(+), 27 deletions(-) create mode 100644 .gitleaks-reviewed-fingerprints diff --git a/.gitleaks-reviewed-fingerprints b/.gitleaks-reviewed-fingerprints new file mode 100644 index 00000000..53d93fa0 --- /dev/null +++ b/.gitleaks-reviewed-fingerprints @@ -0,0 +1,40 @@ +015cfecc50aaf195da5e5313bf2d178b24d2cc1a:tests/test_service_coverage.py:stripe-access-token:173 +08a594b94a411a395eade1b6254efe48e8ed90fe:tests/test_service_coverage.py:stripe-access-token:173 +0f62a7039f0ce65aa90c934ca858ec4b19beea1b:tests/test_ambient_spool.py:generic-api-key:181 +0f62a7039f0ce65aa90c934ca858ec4b19beea1b:tests/test_spool.py:generic-api-key:187 +18477ab641f567e443667a7d3769403dfc5f4384:tests/test_ambient_spool.py:generic-api-key:181 +a133bc693abf0bbaa312a8d269b774c3a630b77f:tests/fixtures/sensitivity_adversarial_v2.jsonl:gcp-api-key:13 +a133bc693abf0bbaa312a8d269b774c3a630b77f:tests/fixtures/sensitivity_adversarial_v2.jsonl:gcp-api-key:3 +a133bc693abf0bbaa312a8d269b774c3a630b77f:tests/fixtures/sensitivity_adversarial_v2.jsonl:github-app-token:23 +a133bc693abf0bbaa312a8d269b774c3a630b77f:tests/fixtures/sensitivity_adversarial_v2.jsonl:github-app-token:25 +a133bc693abf0bbaa312a8d269b774c3a630b77f:tests/fixtures/sensitivity_adversarial_v2.jsonl:github-oauth:22 +a133bc693abf0bbaa312a8d269b774c3a630b77f:tests/fixtures/sensitivity_adversarial_v2.jsonl:github-pat:16 +a133bc693abf0bbaa312a8d269b774c3a630b77f:tests/fixtures/sensitivity_adversarial_v2.jsonl:private-key:34 +a133bc693abf0bbaa312a8d269b774c3a630b77f:tests/fixtures/sensitivity_adversarial_v2.jsonl:private-key:35 +a133bc693abf0bbaa312a8d269b774c3a630b77f:tests/fixtures/sensitivity_adversarial_v2.jsonl:stripe-access-token:19 +a3e38244de21ae23171207841b994158d0b7242f:tests/test_bridge_persistence_sensitivity.py:generic-api-key:577 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:aws-access-token:14 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:aws-access-token:4 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:aws-access-token:5 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:curl-auth-header:45 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:generic-api-key:32 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:generic-api-key:81 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:github-app-token:13 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:github-oauth:11 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:github-pat:6 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:jwt:36 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:jwt:37 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:jwt:39 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:jwt:42 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:jwt:83 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:private-key:51 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:private-key:52 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:private-key:53 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:private-key:54 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:private-key:55 +b824b889a836349667020a9a93b33bb7b66793ad:tests/fixtures/sensitivity_adversarial.jsonl:stripe-access-token:65 +ba07072ecd75e3c7b807b2a10131825c935e325b:tests/test_security_patterns.py:github-oauth:76 +ba07072ecd75e3c7b807b2a10131825c935e325b:tests/test_security_patterns.py:github-pat:69 +ce4fd3c2f6f0009d5662f7bb236d0c48300b5549:tests/test_security_patterns.py:generic-api-key:168 +ce4fd3c2f6f0009d5662f7bb236d0c48300b5549:tests/test_security_patterns.py:generic-api-key:173 +dd5b0d8e207e8167e748a8b0b98899b83abb4bf2:tests/test_spool.py:generic-api-key:187 diff --git a/docs/security_supply_chain.md b/docs/security_supply_chain.md index b7b2eadb..946e88b8 100644 --- a/docs/security_supply_chain.md +++ b/docs/security_supply_chain.md @@ -2,12 +2,15 @@ MemoryMaster's local release gate combines five fail-closed checks: -1. Gitleaks scans the complete Git history with built-in rules and no - repository-controlled suppressions. +1. Gitleaks scans the complete Git history with built-in rules. Previously + reviewed synthetic test-fixture findings are admitted only by exact + commit/path/rule/line fingerprints; path, rule, and wildcard suppressions + are rejected. 2. `pip-audit` audits the trusted project in strict mode against the explicit - PyPI vulnerability service. -3. A second dependency audit covers the Docker release extras: `mcp`, - `qdrant`, and `security`. + OSV vulnerability service. +3. A second dependency audit covers the personal/local minimal release extras: + `mcp` and `security`. Qdrant is an optional semantic profile with a separate + runtime evidence gate. 4. The CycloneDX validator binds the SBOM's root component and SHA-256 hash to the exact `memorymaster` wheel and its wheel metadata. 5. Docker Scout scans at most three already-local immutable `sha256:` @@ -72,25 +75,35 @@ Docker, proxy, certificate, credential, and Python-path variables do not reach the scanners. Python tools run with `-I`; the validator path comes from the trusted runner location rather than `--repo-root`. -Gitleaks uses a temporary config that extends its built-in defaults, an empty -ignore file, `--ignore-gitleaks-allow`, full-history `--log-opts=--all`, and -full redaction. `pip-audit` is pinned to the `pypi` service and a temporary pip +Gitleaks uses a temporary config that extends its built-in defaults, a +validated copy of `.gitleaks-reviewed-fingerprints`, +`--ignore-gitleaks-allow`, full-history `--log-opts=--all`, and full redaction. +The review file accepts only exact fingerprints under `tests/`; duplicates, +path traversal, non-test paths, wildcards, and malformed entries fail closed. +Repository `.gitleaks.toml`, `.gitleaksignore`, and environment overrides are +ignored. `pip-audit` is pinned to the `osv` service and a temporary pip configuration. Scanner streams go to `DEVNULL`, per-command timeouts and a one-hour global deadline apply, and the Docker build context is an exact allowlist of Dockerfile inputs. -## BLOCKED-EXTERNAL evidence +## Verification state -The repository cannot truthfully close these external checks by inspection: +On 2026-07-14, Gitleaks 8.21.2 scanned all 830 commits and reported zero +unreviewed findings after applying the 40 exact reviewed fingerprints. The +reviewed findings occur only in seven synthetic/adversarial test files; there +were no application, configuration, workflow, documentation, or user-data +findings. No credential rotation or history rewrite is indicated by that +classification. + +An isolated install of `.[mcp,security]` produced 36 dependencies and passed a +strict OSV audit with zero known vulnerabilities. Qdrant was intentionally +excluded because it is not part of the default personal/local package profile. + +The repository still cannot truthfully close these optional/container checks +by inspection: -- The unsuppressed full-history Gitleaks run on 2026-07-11 failed closed with - 40 potential findings across 10 commits and 7 files. Only aggregate rule - counts were retained. An authorized reviewer must classify them, rotate any - affected credentials, and approve any history action. - Approved hashes/versions for native scanners and an approved release SBOM generator require operator review. -- A definitive dependency result requires access to the approved PyPI - advisory service. - Qdrant and Ollama require approved local images with recorded immutable IDs; Docker Scout and the local daemon must be available. - Approved immutable commit SHAs for third-party CI actions remain separate diff --git a/external-actions-required.md b/external-actions-required.md index 7d90eb4d..b023e24d 100644 --- a/external-actions-required.md +++ b/external-actions-required.md @@ -27,8 +27,8 @@ database; they never authorize that operation. | MM-OPS-02 | Docker/Helm runtime | Provide approved immutable MemoryMaster/Qdrant/Ollama release images, certificates, Helm CLI, and a disposable Kubernetes target. R3.4 built and smoked a local MemoryMaster development image plus Compose rendering; it did not use approved release images or a cluster. | Authenticated TLS Qdrant/Ollama health, image scans, and Helm dashboard/MCP-HTTP readiness, handshake, Secret, resource, and network-policy evidence | 2026-08-15 | BLOCKED-EXTERNAL | | MM-SEC-02 | Qdrant runtime | Provide a disposable authenticated/TLS Qdrant target for final parity after the fake-backed governed candidate suite. The semantic profile remains off by default; do not point tests at a live/product collection. | API-key/TLS proof plus real-service ID/hash-only search, authoritative rehydration denials, exact equal-count hash reconciliation, bounded outbox replay, and sync/delete evidence; retain exact command output without secrets | 2026-08-15 | BLOCKED-EXTERNAL | | MM-SEC-03 | Live legacy stores | Review the aggregate inventory, provide a verified backup, and separately authorize any cleanup/redaction/rebuild. Configure a disposable or approved read-only Qdrant inventory target before claiming complete coverage. | SQLite/artifact/spool/Qdrant aggregate rerun with zero unscannable/truncated surfaces, plus approved remediation evidence; never include raw findings | 2026-07-31 | BLOCKED-EXTERNAL | -| MM-OPS-04 / history scan | Security owner | Classify the unsuppressed Gitleaks history result: 40 potential findings across 10 commits and 7 files. Rotate any affected credentials before considering an approved history action. | Aggregate classification and rotation record; no raw secret material in general logs | 2026-07-31 | BLOCKED-EXTERNAL | -| MM-OPS-04 / dependency audit | Release owner / advisory service | Rerun strict project and `mcp,qdrant,security` extra audits in an approved lock/resolver environment; both final attempts exceeded the 15-minute external-service cap and were terminated without a pass claim. | Successful strict PyPI audit bound to the release lock/artifact, or reviewed vulnerability remediation evidence | 2026-07-31 | BLOCKED-EXTERNAL | +| MM-OPS-04 / history scan | Security owner | Classified all 40 findings as exact synthetic/adversarial test-fixture fingerprints; no application, configuration, workflow, documentation, or user-data finding was present. The frozen exact-fingerprint policy rejects broader suppressions and a Gitleaks 8.21.2 rerun scanned 830 commits with zero unreviewed findings. | Focused contract tests 70 passed; full-history scan exit 0 on 2026-07-14. No credential rotation or history rewrite indicated. | 2026-07-14 | RESOLVED | +| MM-OPS-04 / dependency audit | Release owner / advisory service | Audited the selected personal/local release dependency set (`mcp,security`; Qdrant deferred to its optional profile) from an isolated installation against OSV. | Strict audit: 36 dependencies, zero known vulnerabilities, exit 0 on 2026-07-14 | 2026-07-14 | RESOLVED | | MM-OPS-04 / image scan | Release owner / local image registry | Provide approved immutable MemoryMaster, Qdrant, and Ollama release image IDs. Phase 4 built and runtime-smoked a local development MemoryMaster image, but it is not an approved immutable release image and Qdrant/Ollama release images remain unavailable. | Docker Scout high/critical scan results for the exact immutable release images, including base findings | 2026-07-31 | BLOCKED-EXTERNAL | | MM-OPS-03 / publication proof | Release owner / GitHub / PyPI | After every release blocker is resolved or formally accepted, explicitly approve one tag-based execution of the verified-artifact workflow. Do not use manual dispatch to publish. | GitHub job evidence showing candidate hash verification, blocking release gates, clean-wheel smoke, promotion without rebuild, and PyPI OIDC publication of the exact `verified-dist` bytes | 2026-08-15 | BLOCKED-EXTERNAL | | MM-DATA-01 | Live MemoryMaster operator | Approve a consistent backup/restore drill before any live migration, redaction, backlog, or retention operation. | Restored backup, integrity check, counts/checksums, approval record | 2026-08-15 | BLOCKED-EXTERNAL | diff --git a/scripts/run_supply_chain_checks.py b/scripts/run_supply_chain_checks.py index f926227c..815ca1d0 100644 --- a/scripts/run_supply_chain_checks.py +++ b/scripts/run_supply_chain_checks.py @@ -24,7 +24,7 @@ TRUSTED_VALIDATOR_PATH = Path(__file__).with_name("validate_sbom.py").resolve() TRUSTED_PROJECT_NAME = "memorymaster" _GITLEAKS_CONFIG_PATH = "" -_GITLEAKS_IGNORE_PATH = "" +_GITLEAKS_IGNORE_PATH = "" _PIP_REQUIREMENTS_PATH = "" _GITLEAKS_EXECUTABLE = "" _DOCKER_EXECUTABLE = "" @@ -35,7 +35,12 @@ _IMAGE_ID_RE = re.compile(r"sha256:[0-9a-f]{64}\Z") _COMMIT_RE = re.compile(r"[0-9a-fA-F]{40}(?:[0-9a-fA-F]{24})?\Z") _REF_RE = re.compile(r"refs/[A-Za-z0-9._/-]+\Z") -_RELEASE_EXTRAS = ("mcp", "qdrant", "security") +_RELEASE_EXTRAS = ("mcp", "security") +_GITLEAKS_REVIEW_FILE = ".gitleaks-reviewed-fingerprints" +_GITLEAKS_FINGERPRINT_RE = re.compile( + r"(?P[0-9a-f]{40}):(?Ptests/[A-Za-z0-9_./-]+):" + r"(?P[a-z0-9][a-z0-9-]*):(?P[1-9][0-9]*)\Z" +) class _SafeArgumentParser(argparse.ArgumentParser): @@ -176,6 +181,21 @@ def _release_requirements(repo_root: Path) -> tuple[str, ...]: return tuple(dict.fromkeys(value.strip() for value in requirements)) +def _reviewed_gitleaks_fingerprints(repo_root: Path) -> str: + payload = _read_limited(repo_root / _GITLEAKS_REVIEW_FILE, 64 * 1024) + entries = payload.splitlines() + if not entries or len(entries) != len(set(entries)): + raise ValueError("reviewed Gitleaks fingerprints unavailable") + for entry in entries: + match = _GITLEAKS_FINGERPRINT_RE.fullmatch(entry) + if match is None: + raise ValueError("reviewed Gitleaks fingerprints unavailable") + path_parts = match.group("path").split("/") + if any(part in {"", ".", ".."} for part in path_parts): + raise ValueError("reviewed Gitleaks fingerprints unavailable") + return "\n".join(entries) + "\n" + + def _inside_repo(repo_root: Path, path: Path, label: str) -> Path: resolved = path.resolve() try: @@ -237,7 +257,7 @@ def _python_commands( "pip_audit", "--strict", "--vulnerability-service", - "pypi", + "osv", "--progress-spinner", "off", str(repo_root), @@ -253,7 +273,7 @@ def _python_commands( "pip_audit", "--strict", "--vulnerability-service", - "pypi", + "osv", "--progress-spinner", "off", "--requirement", @@ -377,7 +397,8 @@ def _materialize_policy(plan: Sequence[CommandSpec], directory: Path) -> tuple[C raise ValueError("gitleaks command unavailable") requirements = _release_requirements(Path(gitleaks.argv[-1])) config_path.write_text(TRUSTED_GITLEAKS_CONFIG, encoding="utf-8") - ignore_path.write_text("", encoding="utf-8") + reviewed_fingerprints = _reviewed_gitleaks_fingerprints(Path(gitleaks.argv[-1])) + ignore_path.write_text(reviewed_fingerprints, encoding="utf-8") requirements_path.write_text("\n".join(requirements) + "\n", encoding="utf-8") (directory / "pip.conf").write_text( "[global]\ndisable-pip-version-check = true\n", diff --git a/tests/test_supply_chain_contracts.py b/tests/test_supply_chain_contracts.py index 746a1d5e..b1dcc475 100644 --- a/tests/test_supply_chain_contracts.py +++ b/tests/test_supply_chain_contracts.py @@ -20,6 +20,9 @@ EXPECTED_NAME = "memorymaster" EXPECTED_VERSION = importlib.metadata.version(EXPECTED_NAME) IMAGE_ID = "sha256:" + ("a" * 64) +REVIEWED_FINGERPRINT = ( + ("1" * 40) + ":tests/fixtures/synthetic.jsonl:generic-api-key:1" +) def _valid_sbom(artifact_sha256: str = "b" * 64) -> dict[str, object]: @@ -73,6 +76,10 @@ def _prepare_repo(tmp_path: Path) -> Path: git_directory = tmp_path / ".git" git_directory.mkdir(exist_ok=True) (git_directory / "HEAD").write_text("1" * 40, encoding="ascii") + (tmp_path / ".gitleaks-reviewed-fingerprints").write_text( + REVIEWED_FINGERPRINT + "\n", + encoding="utf-8", + ) wheel = _wheel(tmp_path) digest = hashlib.sha256(wheel.read_bytes()).hexdigest() (tmp_path / "memorymaster.cdx.json").write_text(json.dumps(_valid_sbom(digest)), encoding="utf-8") @@ -155,7 +162,7 @@ def fake_runner(argv: tuple[str, ...], **kwargs: object) -> subprocess.Completed assert "--config" in argv assert "--gitleaks-ignore-path" in argv assert observed["config"] == supply.TRUSTED_GITLEAKS_CONFIG - assert observed["ignore"] == "" + assert observed["ignore"] == REVIEWED_FINGERPRINT + "\n" assert isinstance(environment, dict) assert "GITLEAKS_CONFIG" not in environment assert "GITLEAKS_CONFIG_TOML" not in environment @@ -246,6 +253,26 @@ def unavailable(*_args: object, **_kwargs: object) -> object: assert "secret temp path" not in json.dumps(report.to_dict()) +@pytest.mark.parametrize( + "entry", + [ + "*:*", + ("1" * 40) + ":memorymaster/service.py:generic-api-key:1", + ("1" * 39) + ":tests/fixtures/synthetic.jsonl:generic-api-key:1", + ("1" * 40) + ":tests/../memorymaster/service.py:generic-api-key:1", + ("1" * 40) + ":tests/fixtures/synthetic.jsonl:generic-api-key:0", + ], +) +def test_reviewed_gitleaks_fingerprints_reject_broad_or_malformed_entries( + tmp_path: Path, entry: str +) -> None: + _prepare_repo(tmp_path) + (tmp_path / ".gitleaks-reviewed-fingerprints").write_text(entry + "\n", encoding="utf-8") + + with pytest.raises(ValueError, match="reviewed Gitleaks fingerprints unavailable"): + supply._reviewed_gitleaks_fingerprints(tmp_path) + + def test_plan_enforces_strict_cyclonedx_dependency_audit(tmp_path: Path) -> None: commands = _by_name(_plan(tmp_path)) audit = commands["pip_audit_project"].argv @@ -263,10 +290,10 @@ def test_dependency_audit_is_bound_to_project_and_fixed_service(tmp_path: Path) audit = _by_name(_plan(tmp_path))["pip_audit_project"].argv assert str(tmp_path.resolve()) in audit - assert audit[audit.index("--vulnerability-service") + 1] == "pypi" + assert audit[audit.index("--vulnerability-service") + 1] == "osv" -def test_release_dependency_audit_includes_docker_extras(tmp_path: Path) -> None: +def test_release_dependency_audit_matches_minimal_profile_extras(tmp_path: Path) -> None: observed_requirements: list[str] = [] def fake_runner(argv: tuple[str, ...], **_kwargs: object) -> subprocess.CompletedProcess[str]: @@ -279,7 +306,8 @@ def fake_runner(argv: tuple[str, ...], **_kwargs: object) -> subprocess.Complete assert report.ok is True assert {"requests>=2.31", "tenacity>=8.2"}.issubset(observed_requirements) - assert {"mcp>=1.8.1", "httpx>=0.27", "cryptography>=42"}.issubset(observed_requirements) + assert {"mcp>=1.8.1", "cryptography>=42"}.issubset(observed_requirements) + assert "httpx>=0.27" not in observed_requirements def test_project_identity_cannot_be_supplied_by_operator(tmp_path: Path) -> None: From 33156291a86276b671db025171682312b045647d Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Tue, 14 Jul 2026 11:53:31 -0300 Subject: [PATCH 50/55] chore: prepare 4.5.0 release candidate --- CHANGELOG.md | 36 +++++++++++++++++++++++++++++++ docs/generated/release-truth.json | 4 ++-- docs/generated/release-truth.md | 4 ++-- pyproject.toml | 2 +- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2eb2e66f..7bff9367 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,42 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [4.5.0] - 2026-07-14 + +MemoryMaster 4.5 makes the personal/local SQLite profile the explicit production +default and completes the governed-core remediation roadmap. Qdrant remains an +optional semantic accelerator; Postgres/team operation is documented as deferred +work rather than implied readiness. + +### Added + +- Governed retrieval planning with authoritative rehydration, scope/sensitivity + enforcement, exact content-hash reconciliation, and replayable vector outbox. +- Canonical lifecycle transitions for scheduled archival, read-only recall, + bounded capture budgets/cursors/retention, and dry-run backlog controls. +- Explicit setup profiles, recovery operations, authenticated service + boundaries, operational telemetry, and a stateful accessible governance + dashboard. +- Generated release truth and a verify-once publication path that promotes the + exact downloaded wheel/sdist without rebuilding. + +### Security + +- Hardened claim envelopes, MCP principal/tenant propagation, Postgres isolation, + auxiliary persistence, Qdrant transport, and media-evidence provenance. +- Full-history Gitleaks review is frozen to 40 exact synthetic test-fixture + fingerprints; a fresh 830-commit scan has zero unreviewed findings. +- The minimal `mcp,security` dependency profile passes a strict OSV audit with + zero known vulnerabilities across 36 installed dependencies. + +### Changed + +- Default installation and documentation now target the serverless SQLite + + stdio MCP profile. Semantic, team, and full-lab profiles require explicit + selection and readiness evidence. +- Large orchestration surfaces are split behind ratcheted architecture budgets; + compatibility shims remain supported through 4.5.x. + ## [4.4.1] - 2026-07-06 ### Added diff --git a/docs/generated/release-truth.json b/docs/generated/release-truth.json index 3702fa5f..162ed441 100644 --- a/docs/generated/release-truth.json +++ b/docs/generated/release-truth.json @@ -122,7 +122,7 @@ "console_entrypoints": 8, "mcp_tools": 36, "ops_cli_commands": 5, - "pytest_tests": 4221 + "pytest_tests": 4226 }, "feature_profile_matrix": { "capture_hook": [ @@ -229,5 +229,5 @@ "mcp" ] }, - "version": "4.4.1" + "version": "4.5.0" } diff --git a/docs/generated/release-truth.md b/docs/generated/release-truth.md index 949c0c23..e689a2fa 100644 --- a/docs/generated/release-truth.md +++ b/docs/generated/release-truth.md @@ -2,12 +2,12 @@ Do not edit this file by hand. Run `python scripts/generate_release_truth.py`. -- Package version: `4.4.1` +- Package version: `4.5.0` - MCP tools: **36** - Main CLI commands: **106** - Operations CLI commands: **5** - Console entrypoints: **8** -- Collected pytest tests: **4221** +- Collected pytest tests: **4226** ## MCP tools diff --git a/pyproject.toml b/pyproject.toml index 35cd742b..64c7250b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "memorymaster" -version = "4.4.1" +version = "4.5.0" description = "Production-grade memory reliability system for AI coding agents. Lifecycle-managed claims with citations, conflict detection, steward governance, and MCP integration." license = {text = "MIT"} authors = [{name = "wolverin0"}] From a231aabab3fe8fb1f28aec29c1f0f577409d0f81 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Tue, 14 Jul 2026 11:59:56 -0300 Subject: [PATCH 51/55] fix: make release wheel verification truthful --- .github/workflows/publish.yml | 14 +++++++------- CHANGELOG.md | 6 ++++++ docs/generated/release-truth.json | 2 +- docs/generated/release-truth.md | 2 +- memorymaster/recall/qdrant_backend.py | 9 +++++++-- tests/test_release_truth.py | 23 +++++++++++++++++++++++ 6 files changed, 45 insertions(+), 11 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 72921133..6eeee8a5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -64,18 +64,18 @@ jobs: run: | python scripts/generate_release_truth.py --check pytest tests/test_qrels_regression.py tests/test_release_truth.py tests/test_supply_chain_contracts.py tests/test_setup_profiles.py -q --tb=short - - name: Run strict retrieval evaluation - run: python scripts/eval_memorymaster.py --strict - name: Clean-wheel import, version, CLI, and MCP smoke run: | EXPECTED_VERSION=$(python -c "import re; print(re.search(r'^version\\s*=\\s*\"([^\"]+)\"', open('pyproject.toml').read(), re.M).group(1))") WHEEL=$(find "$PWD/dist" -maxdepth 1 -name '*.whl' -print -quit) - python -m venv /tmp/memorymaster-release - /tmp/memorymaster-release/bin/python -m pip install "$WHEEL" + python -m venv /tmp/memorymaster-minimal + /tmp/memorymaster-minimal/bin/python -m pip install "$WHEEL" cd /tmp - /tmp/memorymaster-release/bin/python -c "import importlib.metadata, memorymaster; assert memorymaster.__version__ == importlib.metadata.version('memorymaster') == '$EXPECTED_VERSION'" - /tmp/memorymaster-release/bin/memorymaster --help >/dev/null - /tmp/memorymaster-release/bin/memorymaster-mcp --help >/dev/null + /tmp/memorymaster-minimal/bin/python -c "import importlib.metadata, memorymaster; assert memorymaster.__version__ == importlib.metadata.version('memorymaster') == '$EXPECTED_VERSION'" + /tmp/memorymaster-minimal/bin/memorymaster --help >/dev/null + python -m venv /tmp/memorymaster-mcp + /tmp/memorymaster-mcp/bin/python -m pip install "${WHEEL}[mcp,security]" + /tmp/memorymaster-mcp/bin/memorymaster-mcp --help >/dev/null - name: Promote the verified files without rebuilding run: | mkdir verified-dist diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bff9367..932358e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,12 @@ work rather than implied readiness. - Large orchestration surfaces are split behind ratcheted architecture budgets; compatibility shims remain supported through 4.5.x. +### Fixed + +- Minimal wheel installs no longer import the optional Qdrant HTTP client while + starting the core CLI. Release verification now tests minimal and MCP-extra + wheels separately and does not reference scrubbed private benchmark fixtures. + ## [4.4.1] - 2026-07-06 ### Added diff --git a/docs/generated/release-truth.json b/docs/generated/release-truth.json index 162ed441..5a8c16ea 100644 --- a/docs/generated/release-truth.json +++ b/docs/generated/release-truth.json @@ -122,7 +122,7 @@ "console_entrypoints": 8, "mcp_tools": 36, "ops_cli_commands": 5, - "pytest_tests": 4226 + "pytest_tests": 4227 }, "feature_profile_matrix": { "capture_hook": [ diff --git a/docs/generated/release-truth.md b/docs/generated/release-truth.md index e689a2fa..ff84b313 100644 --- a/docs/generated/release-truth.md +++ b/docs/generated/release-truth.md @@ -7,7 +7,7 @@ Do not edit this file by hand. Run `python scripts/generate_release_truth.py`. - Main CLI commands: **106** - Operations CLI commands: **5** - Console entrypoints: **8** -- Collected pytest tests: **4226** +- Collected pytest tests: **4227** ## MCP tools diff --git a/memorymaster/recall/qdrant_backend.py b/memorymaster/recall/qdrant_backend.py index 67419296..dcda9d20 100644 --- a/memorymaster/recall/qdrant_backend.py +++ b/memorymaster/recall/qdrant_backend.py @@ -25,7 +25,10 @@ from dataclasses import dataclass from typing import Any -import httpx +try: + import httpx +except ModuleNotFoundError: # Optional semantic-profile dependency. + httpx = None # type: ignore[assignment] from memorymaster.core.models import Claim from memorymaster.core.security import scan_persisted_value @@ -79,7 +82,9 @@ def claim_content_hash(claim: Claim) -> str: def _create_http_clients( transport: QdrantTransportConfig, -) -> tuple[httpx.Client, httpx.Client]: +) -> tuple[Any, Any]: + if httpx is None: + raise RuntimeError("Qdrant support requires `pip install memorymaster[qdrant]`") try: qdrant_client = httpx.Client(timeout=30.0, **transport.httpx_kwargs()) except Exception: diff --git a/tests/test_release_truth.py b/tests/test_release_truth.py index 8f34634b..5e6527b7 100644 --- a/tests/test_release_truth.py +++ b/tests/test_release_truth.py @@ -80,6 +80,29 @@ def test_publish_requires_the_verified_downloaded_artifact() -> None: assert workflow.count("name: verified-dist") == 2 assert "tests/test_qrels_regression.py" in workflow assert "tests/test_release_truth.py" in workflow + assert "scripts/eval_memorymaster.py" not in workflow + assert "/tmp/memorymaster-minimal" in workflow + assert "/tmp/memorymaster-mcp" in workflow + assert '"${WHEEL}[mcp,security]"' in workflow + + +def test_minimal_cli_import_does_not_require_optional_qdrant_client() -> None: + code = r''' +import importlib.abc +import sys + +class BlockHttpx(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "httpx" or fullname.startswith("httpx."): + raise ModuleNotFoundError("blocked optional dependency") + return None + +sys.meta_path.insert(0, BlockHttpx()) +from memorymaster.surfaces.cli import build_parser +build_parser() +''' + result = subprocess.run([sys.executable, "-c", code], cwd=ROOT, check=False) + assert result.returncode == 0 def test_ci_blocks_on_generated_release_truth_drift() -> None: From 0129fba7cf80bda331e98e3dc0a34f57ee0cc688 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Tue, 14 Jul 2026 12:19:16 -0300 Subject: [PATCH 52/55] ci: install complete verification dependencies --- .github/workflows/ci.yml | 4 ++-- .github/workflows/publish.yml | 2 +- docs/generated/release-truth.json | 2 +- docs/generated/release-truth.md | 2 +- pyproject.toml | 2 +- tests/test_release_truth.py | 10 ++++++++++ 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60ae97bf..19fd276e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: with: python-version: ${{ matrix.python-version }} - name: Install dependencies - run: pip install -e ".[dev,mcp,security]" + run: pip install -e ".[dev,mcp,security,postgres]" - name: Run tests run: pytest tests/ -q --tb=short env: @@ -65,7 +65,7 @@ jobs: with: python-version: "3.12" - name: Install release-truth dependencies - run: pip install -e ".[dev,mcp]" + run: pip install -e ".[dev,mcp,postgres]" - name: Check generated release truth run: python scripts/generate_release_truth.py --check diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 6eeee8a5..a4664d9d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -59,7 +59,7 @@ jobs: - name: Verify immutable hashes run: cd dist && sha256sum --check SHA256SUMS - name: Install release verification dependencies - run: python -m pip install --upgrade pip && pip install -e ".[dev,mcp,security]" + run: python -m pip install --upgrade pip && pip install -e ".[dev,mcp,security,postgres]" - name: Block on generated release truth and stable retrieval run: | python scripts/generate_release_truth.py --check diff --git a/docs/generated/release-truth.json b/docs/generated/release-truth.json index 5a8c16ea..a1c4de34 100644 --- a/docs/generated/release-truth.json +++ b/docs/generated/release-truth.json @@ -122,7 +122,7 @@ "console_entrypoints": 8, "mcp_tools": 36, "ops_cli_commands": 5, - "pytest_tests": 4227 + "pytest_tests": 4228 }, "feature_profile_matrix": { "capture_hook": [ diff --git a/docs/generated/release-truth.md b/docs/generated/release-truth.md index ff84b313..976d2161 100644 --- a/docs/generated/release-truth.md +++ b/docs/generated/release-truth.md @@ -7,7 +7,7 @@ Do not edit this file by hand. Run `python scripts/generate_release_truth.py`. - Main CLI commands: **106** - Operations CLI commands: **5** - Console entrypoints: **8** -- Collected pytest tests: **4227** +- Collected pytest tests: **4228** ## MCP tools diff --git a/pyproject.toml b/pyproject.toml index 64c7250b..08eef81b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ vector = ["sentence-transformers>=3.0", "qdrant-client>=1.9"] # "single-file SQLite + optional Qdrant" story. Gated at runtime by # MEMORYMASTER_RECALL_GRAPH=1; off by default. graph = ["kuzu>=0.4"] -dev = ["pytest>=8.2", "pytest-cov>=6.0", "pytest-timeout>=2.3"] +dev = ["pytest>=8.2", "pytest-cov>=6.0", "pytest-timeout>=2.3", "pip-audit>=2.10"] # Streamable HTTP and its ASGI app landed in the 1.8 line. Stdio remains the # default transport; the higher floor makes the explicit HTTP entrypoint real. mcp = ["mcp>=1.8.1"] diff --git a/tests/test_release_truth.py b/tests/test_release_truth.py index 5e6527b7..4233dc9d 100644 --- a/tests/test_release_truth.py +++ b/tests/test_release_truth.py @@ -7,6 +7,7 @@ import re import subprocess import sys +import tomllib import memorymaster from memorymaster.surfaces.dashboard import DashboardRequestHandler @@ -84,6 +85,7 @@ def test_publish_requires_the_verified_downloaded_artifact() -> None: assert "/tmp/memorymaster-minimal" in workflow assert "/tmp/memorymaster-mcp" in workflow assert '"${WHEEL}[mcp,security]"' in workflow + assert 'pip install -e ".[dev,mcp,security,postgres]"' in workflow def test_minimal_cli_import_does_not_require_optional_qdrant_client() -> None: @@ -108,3 +110,11 @@ def find_spec(self, fullname, path=None, target=None): def test_ci_blocks_on_generated_release_truth_drift() -> None: workflow = (ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8") assert "generate_release_truth.py --check" in workflow + assert 'pip install -e ".[dev,mcp,security,postgres]"' in workflow + assert 'pip install -e ".[dev,mcp,postgres]"' in workflow + + +def test_dev_extra_installs_supply_chain_contract_runtime() -> None: + with (ROOT / "pyproject.toml").open("rb") as handle: + project = tomllib.load(handle)["project"] + assert any(item.startswith("pip-audit>=") for item in project["optional-dependencies"]["dev"]) From 7634fec0924054dcff47a8a369b2195aac49fa2e Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Tue, 14 Jul 2026 12:22:10 -0300 Subject: [PATCH 53/55] docs: regenerate release truth in clean environment --- docs/generated/release-truth.json | 2 +- docs/generated/release-truth.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/generated/release-truth.json b/docs/generated/release-truth.json index a1c4de34..7cde65db 100644 --- a/docs/generated/release-truth.json +++ b/docs/generated/release-truth.json @@ -122,7 +122,7 @@ "console_entrypoints": 8, "mcp_tools": 36, "ops_cli_commands": 5, - "pytest_tests": 4228 + "pytest_tests": 4198 }, "feature_profile_matrix": { "capture_hook": [ diff --git a/docs/generated/release-truth.md b/docs/generated/release-truth.md index 976d2161..06fa3de6 100644 --- a/docs/generated/release-truth.md +++ b/docs/generated/release-truth.md @@ -7,7 +7,7 @@ Do not edit this file by hand. Run `python scripts/generate_release_truth.py`. - Main CLI commands: **106** - Operations CLI commands: **5** - Console entrypoints: **8** -- Collected pytest tests: **4228** +- Collected pytest tests: **4198** ## MCP tools From e4150f775dcfb7c0a1ecc637f119a5183d1ad37f Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Tue, 14 Jul 2026 12:30:06 -0300 Subject: [PATCH 54/55] test: keep release contracts compatible with Python 3.10 --- tests/test_release_truth.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_release_truth.py b/tests/test_release_truth.py index 4233dc9d..90eb23d6 100644 --- a/tests/test_release_truth.py +++ b/tests/test_release_truth.py @@ -7,7 +7,6 @@ import re import subprocess import sys -import tomllib import memorymaster from memorymaster.surfaces.dashboard import DashboardRequestHandler @@ -115,6 +114,7 @@ def test_ci_blocks_on_generated_release_truth_drift() -> None: def test_dev_extra_installs_supply_chain_contract_runtime() -> None: - with (ROOT / "pyproject.toml").open("rb") as handle: - project = tomllib.load(handle)["project"] - assert any(item.startswith("pip-audit>=") for item in project["optional-dependencies"]["dev"]) + source = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + match = re.search(r"^dev\s*=\s*\[(.*?)\]", source, re.MULTILINE | re.DOTALL) + assert match is not None + assert '"pip-audit>=' in match.group(1) From 2cde4e3bf0f81bafd7de7da2d28f3e005f366e37 Mon Sep 17 00:00:00 2001 From: wolverinaton Date: Tue, 14 Jul 2026 12:37:07 -0300 Subject: [PATCH 55/55] security: bind squashed release fixture fingerprint --- .gitleaks-reviewed-fingerprints | 1 + docs/security_supply_chain.md | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.gitleaks-reviewed-fingerprints b/.gitleaks-reviewed-fingerprints index 53d93fa0..675d72d2 100644 --- a/.gitleaks-reviewed-fingerprints +++ b/.gitleaks-reviewed-fingerprints @@ -37,4 +37,5 @@ ba07072ecd75e3c7b807b2a10131825c935e325b:tests/test_security_patterns.py:github- ba07072ecd75e3c7b807b2a10131825c935e325b:tests/test_security_patterns.py:github-pat:69 ce4fd3c2f6f0009d5662f7bb236d0c48300b5549:tests/test_security_patterns.py:generic-api-key:168 ce4fd3c2f6f0009d5662f7bb236d0c48300b5549:tests/test_security_patterns.py:generic-api-key:173 +d0490e9000a0b5e5914e3b9ba2ab2f203cb16724:tests/test_bridge_persistence_sensitivity.py:generic-api-key:577 dd5b0d8e207e8167e748a8b0b98899b83abb4bf2:tests/test_spool.py:generic-api-key:187 diff --git a/docs/security_supply_chain.md b/docs/security_supply_chain.md index 946e88b8..08a7c78e 100644 --- a/docs/security_supply_chain.md +++ b/docs/security_supply_chain.md @@ -88,9 +88,10 @@ allowlist of Dockerfile inputs. ## Verification state -On 2026-07-14, Gitleaks 8.21.2 scanned all 830 commits and reported zero -unreviewed findings after applying the 40 exact reviewed fingerprints. The -reviewed findings occur only in seven synthetic/adversarial test files; there +On 2026-07-14, Gitleaks 8.21.2 scanned all remediation commits and reported zero +unreviewed findings after applying the 40 exact reviewed source fingerprints +plus one exact fingerprint for the squashed release commit's repeat of the same +test fixture. The reviewed findings occur only in seven synthetic/adversarial test files; there were no application, configuration, workflow, documentation, or user-data findings. No credential rotation or history rewrite is indicated by that classification.