From 7b2087a11a0e2130e8508c2bb01b1532ac2d0e96 Mon Sep 17 00:00:00 2001 From: anduin9527 Date: Sun, 23 Aug 2026 12:18:10 +0800 Subject: [PATCH 01/18] feat(memory): freeze M2 episode contracts Signed-off-by: anduin9527 --- docs/development/gap/mainline.md | 61 +- docs/development/plan/plan-20260819.md | 78 +- docs/development/tracing/memory.md | 166 ++- src/internal/ai/memory/canonical.rs | 696 ++++++++++ src/internal/ai/memory/domain.rs | 552 ++++++++ src/internal/ai/memory/mod.rs | 14 + src/internal/ai/memory/validation.rs | 1691 ++++++++++++++++++++++++ src/internal/ai/mod.rs | 3 + 8 files changed, 3148 insertions(+), 113 deletions(-) create mode 100644 src/internal/ai/memory/canonical.rs create mode 100644 src/internal/ai/memory/domain.rs create mode 100644 src/internal/ai/memory/mod.rs create mode 100644 src/internal/ai/memory/validation.rs diff --git a/docs/development/gap/mainline.md b/docs/development/gap/mainline.md index b89015dd0..5517add5c 100644 --- a/docs/development/gap/mainline.md +++ b/docs/development/gap/mainline.md @@ -264,7 +264,7 @@ M0 Trace、M1 Fact、M2 Episode、M3 Skill/Profile/Policy 不是本文新增的 - **目标**：给 agent「编辑前先读相关 sealed intent/decision」的读面——确定性（无 embedding）加性打分器 + 检索态叠加（current/superseded/abandoned/stale）+ 可审计的 ContextBundle / selection receipt。这是 mainline 量化价值集中处（eval CF-IF delta）。作为 `memory.md` recall 在 sealed intent 上的**具体落地**，而非平行系统。 - **范围**：在本地 sealed + 已验证 team projection 上建读面（query/files/current 三模）；先以 scope、visibility、trust、sensitivity 和 ACL 过滤，再按 `scope.rs` 文件重叠 + subsystem + title/what/why/decision 关键词 + open risk/followup + recency + same-thread + supersession lineage 打分，保持确定性、无 embedding。检索态分类器（stale 由 age/file-churn，superseded 由 lineage）。 - - 新增 versioned sibling `IntentContextSelectionReceiptV1`，**不扩展**现有带 raw content/attachment、`deny_unknown_fields` 的 ContextFrame wire schema。receipt 默认 local-only，关联可选 frame id，但仅记录 `query_hash`（非 raw query）、as-of、code commit、branch、private AI/team ref head、projection/index watermark、config/publication/redaction policy hash、selector/scorer/weights/render version、selected public/local object id + score/reason/order、omission reason、token budget、bundle hash。`recorded_at` / UUID 不进入 canonical selection hash。 + - 复用共享上下文层的 `ContextSelectionReceiptV1` 与本地 append-only `context_selection_receipt` 账本，**不扩展**现有带 raw content/attachment、`deny_unknown_fields` 的 ContextFrame wire schema。receipt 默认 local-only，可关联 frame id；固定 envelope 为 schema/source kind、repository/digest key、principal/query HMAC、effective time、code commit/full branch ref、private AI/team source heads、projection watermarks、policy hash、selector version、selected/omissions、token budget、bundle hash、reproducibility state 与 recorded time。raw principal/query/content 不入账；`recorded_at` / UUID 不进入 canonical selection hash。 - 对同一规范化输入快照，receipt 必须得到相同 selected IDs、顺序、reason codes 和 bundle hash；若 policy/watermark/source object 缺失或 stale，则返回 `non_reproducible` / `stale`，不得静默换用别的对象。它只承诺选择与渲染输入可审计/可重放，不承诺外部 provider 的完整 prompt/输出逐字节一致。 - CLI 是初始入口。C9 建立真实 default-deny McpAuthorizer、覆盖 tools/list 和所有 calls 前，**不注册** read 或 mutating intent MCP tool；C9 完成后，MCP 才能走 `code.md` C6 的 `libra code --stdio`，仍不用 `memory.md` 提议的 `libra mcp --stdio`。 - **依赖**：ML-02（有 sealed 记录可检索）；projection（has-current）；**与 memory.md recall/inject 收敛**（§9）。 @@ -508,7 +508,7 @@ flowchart LR | Projection | `projection/rebuild.rs` 可从 formal objects 重建线程投影并事务化物化。 | 缺 private/local 与 validated-team publication 分开的 watermark、跨 actor fold、pin/read-model 专用索引。 | | Decision | live runtime 有 `ai_final_decision.summary_json` 和 git-internal Decision/MCP；`agent_run::MergeDecision` 是 schema scaffold。 | ML-03 必须改 live Decision 平面，不得建在 `agent_run`。 | | Hooks | `hooks/runtime.rs` 当前主责是 capture/ingest；`HookTarget::AgentTraces` 已写 traces。 | ML-08 是反向只读注入，需要独立设计，不应改写捕获语义。 | -| ContextFrame / ContextSnapshot | 已保存局部选择、来源、token、附件或任意 context data，但无 query/ranking/policy/index receipt；部分载体含 raw content。 | ML-05 应新建 local-only `IntentContextSelectionReceiptV1`，不能扩展/传播 raw ContextFrame。 | +| ContextFrame / ContextSnapshot | 已保存局部选择、来源、token、附件或任意 context data，但无 query/ranking/policy/index receipt；部分载体含 raw content。 | ML-05 复用 local-only `ContextSelectionReceiptV1`，不能扩展/传播 raw ContextFrame。 | | MCP authz | production `authz: None` 时 server allow-all，且部分 ContextFrame call 没有 authz。 | C9 default-deny/全工具覆盖是任何 intent MCP surface 的硬前置；本计划 P0/P1 走 CLI。 | ### 12.3 可执行最小纵切 @@ -576,7 +576,7 @@ rg -n "ImportActorLog|knownImportedActorEventType|ActorLogAcceptedEvent" /Volume - `sealed/` — `IntentSealedEventV1`（内容寻址 blob，文件名即对象 id）。 - `pin/` — `IntentPinV1`。 - `decision/` — 已有 `decision` 对象；本计划新增 `DecisionV1` schema version（或扩展现有 schema），必须保证旧 reader 跳过未知字段。 -- `context-receipt/` — local-only `IntentContextSelectionReceiptV1`；它不是 ContextFrame 的 wire 变体，也不属于 `intent-team` publication。 +- `context-receipt/` — local-only `ContextSelectionReceiptV1`；由共享上下文层拥有，不是 ContextFrame 的 wire 变体，也不属于 `intent-team` publication。 `IntentSealedEventV1` 最小字段（对应 mainline `IntentSealedEvent` + v0.3 审计字段）： @@ -739,37 +739,40 @@ TeamIntentRevocationV1 是独立的 team-only tombstone；公开 ref 只携带 它只移除 future team read/injection，不能宣称物理删除 remote、existing clone 或 durable storage tier。 -IntentContextSelectionReceiptV1 是 local-only ML-05 audit object。它不包含 raw -query 或未选中的 sensitive content，也不是 intent-team record。 +ContextSelectionReceiptV1 是 Memory 与 mainline 共用的 local-only selection audit object。以下 envelope +也是两条读取链路的唯一 JSON 合同；它不包含 raw query、可逆 principal 或未选中的 sensitive content， +也不是 intent-team record。 ~~~json { - "schema_version": "libra.intent.context-receipt.v1", - "frame_id": "", - "intent_id": "", - "query": {"mode": "current|files|query", "hash": "sha256:", "as_of": ""}, - "snapshot": { - "code_commit": "", - "branch": "", - "ai_ref_head": "", - "team_ref_head": "", - "projection_built_from": "", - "index_manifest_hash": "sha256:", - "config_policy_hash": "sha256:" - }, - "selector": {"id": "intent-v1", "version": "1", "weights_hash": "sha256:"}, + "receipt_id": "", + "schema_version": 1, + "source_kind": "intent", + "repository_id": "", + "digest_key_id": "", + "principal_hmac": "hmac-sha256::", + "query_hmac": "hmac-sha256::", + "effective_at": "", + "code_commit": "", + "full_branch_ref": "refs/heads/", + "source_heads": {"private_ai": "", "validated_team": ""}, + "projection_watermarks": {"private_ai": "", "validated_team": ""}, + "policy_hash": "sha256:", + "selector_version": "intent-v1", "selected": [ - {"object_id": "", "kind": "sealed_intent|decision", "score": 0, "reasons": ["file_overlap"]} + {"object_id": "", "revision_oid": "", "summary_key": "", "order": 0, + "reason_codes": ["file_overlap"], "score_components": {"file_overlap": 1}} ], - "omissions": [{"object_id": "", "reason": "budget|scope|trust|stale"}], - "budget": {"limit_tokens": 0, "selected_tokens": 0}, - "redaction_policy_hash": "sha256:", - "render_version": "1", - "bundle_hash": "sha256:" + "omissions": [{"reason_code": "budget|scope|trust|stale", "count": 0}], + "token_budget": 0, + "bundle_hash": "sha256:", + "reproducibility_state": "reproducible|stale|expired|non_reproducible", + "recorded_at": "", + "frame_id": "" } ~~~ -Canonical receipt/bundle hash 排除 recorded_at 和 generated UUID。引用 missing、 +Canonical receipt/bundle hash 排除 `recorded_at` 和 `receipt_id`。引用 missing、 untrusted 或 stale source 的 receipt 必须报告该状态，不能静默换用其他 object。 #### 12.6.2 CLI 命令面（新增 `src/command/intent.rs`，在 `src/cli.rs` 注册为 `Commands::Intent`） @@ -832,7 +835,7 @@ C9 完成后，MCP 面才可在 code.md C6 的 libra code --stdio 上注册， - `ai_intent_coverage` — coverage 状态：commit_sha, coverage_status（local-covered/team-covered/skipped/uncovered）, source_pin_id, publication_record_id, skip_reason, baseline。 - `ai_intent_team_publication` — 已验证 manifest：remote, publication_id, ref_head, policy_hash, redaction_policy_hash, publisher, received_at, validation_state, built_from。 - `ai_intent_team_record` — team record / tombstone 的只读投影：record_id, kind, intent_id, content_hash, visibility, review_state, sensitivity, trust, publication_id, revoked_at；未知字段/状态不得落表。 -- `ai_intent_context_receipt` — local-only receipt：receipt_id, query_hash, code_commit, private_ai_ref_head, team_ref_head, projection_built_from, selector_version, policy_hash, bundle_hash, reproducibility_state。 +- `context_selection_receipt` — local-only shared receipt：receipt_id、schema/source kind、repository/digest key、principal/query HMAC、effective time、code commit/full branch ref、source heads、projection watermarks、selector/policy、selected/omissions、token budget、bundle hash、reproducibility state、recorded_at 与可选 frame_id。 - 扩展 `ai_final_decision` 表或新增 `ai_decision_v1` 投影表，把 `alternatives` 从 JSON 字符串拆成结构化列/表；保留 `summary_json` 作为叠加字段。 - 扩展 `ai_index_*` 或新增 `ai_intent_retrieval_index` 用于文件/关键词反向索引；必须分别记录 private local 和 validated-team `built_from` watermark / index manifest，不能把 imported team record 混回 AI_REF projection。 @@ -1062,7 +1065,7 @@ team projection 已就绪，并且明确拒绝 raw AI history；**不得**宣称 | 不走 git notes 走 history.rs | N/A（mainline 用 notes） | `notes.rs` SQLite 侧表 + 无 Note ConfigKind | ✅ §5 决策成立 | | traces 传输的 lease/tracking 机件可复用 | N/A | `agent/push.rs:30-83` force-with-lease | ✅ 仅复用机件；raw AI_REF mirror 已明确禁止 | | AI_REF 不可直接成为团队平面 | N/A | `history.rs` 混合对象 + hook `ai_session/raw_hook_events` | ✅ 新增 intent-team allow-list/redaction/manifest 边界 | -| ContextFrame 不是可发布 retrieval receipt | N/A | frame 可带 raw content/attachment，缺 policy/ranking/watermark | ✅ 新增 local-only ContextReceiptV1 | +| ContextFrame 不是可发布 retrieval receipt | N/A | frame 可带 raw content/attachment，缺 policy/ranking/watermark | ✅ 复用 local-only `ContextSelectionReceiptV1` | | MCP C6 不等于授权 | N/A | production authz None/allow-all，部分调用无 gate | ✅ C9 设为所有 intent MCP tool 的前置 | | 决策缺 alternatives | mainline `IntentSummary.rejected` | `phase4.rs:500-524` 仅 rationale | ✅ ML-03 方向正确 | | hook 只捕获 vs SessionStart/TurnStart 只读注入 | `hooks/dispatcher.go:16-22,91-101,296-417` | `hooks/runtime.rs` ingest only | ✅ ML-08 方向正确；本次补充 TurnStart 轻量提醒边界 | @@ -1080,7 +1083,7 @@ team projection 已就绪，并且明确拒绝 raw AI history；**不得**宣称 | 无 auto-sync 命令列表 | 各命令新鲜度门禁不一致 | §12.6.9 | | 无 mainline→Libra 工作流映射 | 误造 turn/actor-log 平行平面 | §12.6.7 | | 原稿把 AI_REF 当 intent-only remote ref | raw Run/tool/context/session 可能被团队复制，lease 不能弥补授权/脱敏缺口 | §5/ML-01/§12.6.1/§12.8 改为 ML-01a safe rail → ML-02 → ML-01b approved publication | -| 原稿缺 ContextBundle/receipt | 无法解释为何选中这组意图，且 raw ContextFrame 可能泄露 | ML-05 + §12.6.1 增 local-only ContextReceiptV1、重放/缺失 fail-loud 测试 | +| 原稿缺 ContextBundle/receipt | 无法解释为何选中这组意图，且 raw ContextFrame 可能泄露 | ML-05 + §12.6.1 复用 local-only `ContextSelectionReceiptV1`、重放/缺失 fail-loud 测试 | | 原稿把 C6 当 MCP 安全边界 | production authz allow-all，mutating seal/pin tool 可能无授权暴露 | §8/§12.6.3 改为 C9 default-deny/全覆盖前不注册任何 intent MCP tool | | 矩阵遗漏 receipt 行 | 静默欠覆盖 | §3 补 ContextBundle/receipt 行，计数改 36 | | `check` phase-2 无 schema | Hub `last_check` 无法落地 | §12.6.2 `check` + judgment event 说明 | diff --git a/docs/development/plan/plan-20260819.md b/docs/development/plan/plan-20260819.md index b1c9828fb..8143d485c 100644 --- a/docs/development/plan/plan-20260819.md +++ b/docs/development/plan/plan-20260819.md @@ -68,7 +68,7 @@ flowchart TB end subgraph S2["② 安全编译"] direction LR - D["MemoryWriter
解析 · 鉴权 · 脱敏"] --> E["EpisodeCompiler
Observation + Inference"] --> F["自动信任门
身份 · 证据 · schema · policy"] + D["MemoryWriter
解析 · 鉴权 · 脱敏"] --> E["EpisodeCompiler
Observation + Inference"] --> F["研发历程准入
证据 · schema · root · code anchor"] end subgraph S3["③ 权威历史与读取投影"] direction LR @@ -82,7 +82,7 @@ flowchart TB C --> D F --> G I --> J - H -. "仅 Confirmed Task revision：唤醒父 Intent" .-> C + H -. "新 Task Episode revision：唤醒父 Intent" .-> C H -. "投影失败" .-> R["标记 stale；rebuild 前不做安全敏感注入"] R --> I ``` @@ -93,9 +93,9 @@ flowchart TB 2. 触发器只提交受信任的 root kind / root ID、source ref OID 与已认证 principal，不把原始会话直接交给模型。 3. `MemoryWriter` 内部先解析有界来源窗口、鉴权和脱敏，再调用 `EpisodeCompiler`。输入指纹只基于脱敏后的规范输入。 4. 编译器只能提议 Episode 内容，不能决定 scope、namespace、path、root identity、`note_id` 或 ref。 -5. Writer 从终态事实机械填充 completion、时间、related IDs 与 code anchor；自动信任门验证来源信任级别、EvidenceRef locator/digest、脱敏报告、schema、policy 和 prompt-injection reason code。通过才 Confirmed；未通过的经历仍以 Quarantined revision 保存，但不自动注入。 +5. Writer 从终态事实机械填充 completion、时间、related IDs 与 code anchor，并在已授权来源前提下验证来源归属、EvidenceRef locator/digest、脱敏报告、schema、root、code anchor 与 CompileRecord。通过后自动 Confirmed；无效输入返回有类型错误且不写入，不按成功/失败、trust 标签或推断置信度筛除经历。 6. Writer 校验后追加不可变 revision 和事件，以 CAS 推进 repo Memory ref；同一幂等键命中时返回既有 revision，不追加事件。 -7. 新 Confirmed Task revision 会由 Memory ref observer 唤醒对应的已终态父 Intent；输入指纹包含固定 Task revision 集，重复通知幂等，Intent revision 不再反向触发自己。 +7. 新 Task Episode revision 会由 Memory ref observer 唤醒对应的已终态父 Intent；输入指纹包含固定 Task revision 集，重复通知幂等，Intent revision 不再反向触发自己。 8. SQLite 保存快速读取投影。投影水位与 ref 不一致时，自动注入暂停，重建器从权威历史恢复。 9. 检索返回紧凑 Episode；只有 Agent 需要核查时，才沿 `EvidenceRef` 展开现有原始数据。 @@ -129,11 +129,11 @@ flowchart TB 固定 Cell： ```text -Repo / default / episodic.tasks. -Repo / default / episodic.intents. +Repo / default / episodic.tasks.r- +Repo / default / episodic.intents.r- ``` -Intent Episode 的 `MemoryNote.links` 固定每个贡献 Task Episode 的 `note_id + revision_oid`。因此之后 Task Episode 修订不会悄悄改写旧的 Intent 摘要；下一次 Intent 编译会显式产生新 revision。 +Intent Episode 的 `MemoryNote.links` 为每个贡献 Task Episode 固定一个 `kind=Supports` 的 `note_id + revision_oid`。因此之后 Task Episode 修订不会悄悄改写旧的 Intent 摘要；下一次 Intent 编译会显式产生新 revision。 `content_digest` 包含 Episode 的语义字段和 code anchor；排除 `revision_oid`、`parents`、`EvidenceRef` 中的存储 OID/hash 与 link 的 target revision OID。完整 JSON blob 的对象 OID仍覆盖全部字节。该规则同时修正文档中“明确包含 effective commit”与“排除任何 Git OID”的冲突。 @@ -150,7 +150,7 @@ refs/heads/libra/memory/repo ``` - `MemoryNote` revision 与 `MemoryEvent` 都是普通 JSON blob，不向 `git-internal::ObjectType` 增加新枚举。 -- 首次自动写入连续追加 `Created`、`Confirmed`；新来源版本连续追加 `Revised`、`Confirmed`。这里的 Confirmed 表示“来源、schema、策略和编译记录完整，可被读取”，不表示每条 Inference 都是客观真理。 +- 首次自动写入连续追加 `Created`、`Confirmed`；新来源版本连续追加 `Revised`、`Confirmed`。这里的 Confirmed 只表示“授权来源、schema、证据、根绑定、代码锚点和编译记录结构有效，可被读取”，不表示任务成功或每条 Inference 都是客观真理。 - Observation 与 Inference 的边界由结构表达；推断置信度不影响是否保存该研发经历。 - 失败、取消和未改代码的结果照常写入。模型错误、schema 错误或授权失败不产生半条 note，只保留有界作业诊断。 @@ -370,6 +370,7 @@ FTS 表固定 `tokenize='unicode61 remove_diacritics 2'`；`bm25()` 的列权重 | 2026-08-20 | 新 reviewer R3 | 把现有 base CI 与 CodeQL 从“开工日再看”改为具名 workflow/job/ref/SHA/success predicate 的外部 D-01 合同 | 新增 DEP-M2-CI-02 | 依赖登记、M2-15、任务审计、review log | | 2026-08-20 | 新 reviewer R4 | 闭合发布点等待远端门的 `remote-pending` 状态，并把四平台探针显式列入所有消费卡依赖 | 无拆卡 | DEP-M2-CI-02、M2-14C、M2-15、任务审计、review log | | 2026-08-21 | 模板 v2.1 生效（ER-08/C 组与 G-07a 改为「每卡 `patch + 1` + 工具链 lockfile + `gh` 触发 release」） | 登记模板 v2.1 迁移例外：本计划在 v2.1 生效前成稿，按模板「模板版本与迁移政策」增量迁移，未触碰卡保持 v2 口径。继续按 REL-M2-01 的「批量发布组收口时一次 bump」执行（REL-M2-01 全体成员 M2-01, M2-01K, M2-02, M2-02F, M2-02R, M2-03..M2-14, M2-14C 推送不 bump、M2-15 唯一发布点）；下次触碰 REL-M2-01 发布模型时迁到本版 ER-08（每卡 `patch + 1` + 版本面五处 + 工具链刷新 `Cargo.lock` + `gh release create`）。M2-15 沿用「本计划不创建/推送 release tag，D-02 为 N/A，由 maintainer 后续 release 流程拥有」口径，作为本次例外的一部分。同批将全体成员卡 `C/D coverage from` 由 `self` 更正为 `M2-15`（D-01 不继承）：此为 v2 既有口径（成员卡 `Release write set: N/A`、REL-M2-01 登记「不 bump/tag」、版本面本由 M2-15 收口）的归属更正，不改变成员卡发布行为、不构成对成员卡的规范性修改，不触发逐卡 v2.1 迁移 | 无新卡 | 模板版本与迁移政策、REL-M2-01、全体成员卡 `C/D coverage from`、任务卡字段默认值、M2-15、发布分组与并发窗口 | +| 2026-08-22 | 用户实现评审 | M2 Episode 改用授权来源前提与确定性准入，删除 `AutoEpisodeTrustGateV1` 三态摄入；任务结局、trust 标签与推断置信度不再筛除研发经历，Quarantined 只保留给写入后处置 | M2-01/M2-07/M2-09 语义修订，不拆卡 | 核心链路、ADR-M2-06/07、guardrails、任务卡、测试矩阵、风险表 | ## 已决议设计决策 @@ -418,20 +419,20 @@ FTS 表固定 `tokenize='unicode61 remove_diacritics 2'`；`bm25()` 的列权重 - **Consequences:** 终态写入不等待 LLM；有界 job 状态保证最终收敛。 - **Revisit when:** 终态事件模型发生兼容性变更。 -### ADR-M2-06：自动确认表示可用性，不表示推断真理 +### ADR-M2-06：M2 Episode 使用授权来源前提与确定性准入 - **Status:** Accepted -- **Context:** M2 前提是 Agent 全自动，人不参与审核；当前 `memory.md` 对 LLM 产物默认限制为 Draft。 -- **Decision:** Writer 内置确定性的 `AutoEpisodeTrustGateV1`。输入为 authenticated principal、source trust labels、EvidenceRef locator/digest 验证、redaction/injection scan report、compiler allowlist、schema/root/code anchor、ACL/sensitivity/policy version；输出为 `confirmed | quarantined | rejected` 与稳定 reason codes。只有全部门通过才连续写 `Created|Revised + Confirmed`；不可信外部片段、疑似 prompt injection、未解析 locator 或 SecretLike 进入 Quarantined/Rejected，绝不自动注入。Inference 自带置信度与证据，Confirmed 只表示“通过自动使用门”，不表示推断是真理。 -- **Alternatives considered:** 人工 Trust Gate；所有模型输出直接视为 Verified。前者不满足自动化边界，后者混淆可用性和事实真值。 -- **Consequences:** M2-01 必须修订 `memory.md` 的 producer policy；M2-07 实现 trust gate。失败经历仍可保留为 Quarantined revision，读取端按 state/trust/confidence 展示，默认注入只接受 Confirmed。 -- **Revisit when:** 自动 policy gate 的离线误用率超过 benchmark 门槛。 +- **Context:** M2 前提是 Agent 全自动且用户不逐条审核；当前 Agent 能读取的研发来源已经由用户启动的任务、仓库身份和现有访问控制授权。成功、失败、取消和低置信度推断都属于需要保留的研发历程。 +- **Decision:** 首版 M2 不实现 `AutoEpisodeTrustGateV1` 三态摄入。Writer 对已授权且 pinned 的来源执行确定性研发历程准入：来源归属、EvidenceRef locator/digest、脱敏、schema/root/code anchor、CompileRecord 与 compiler 配置全部有效时，连续写 `Created|Revised + Confirmed`；任一无效返回有类型错误且不写入 Memory ref。`trust` 与 confidence 只记录 provenance/认识论状态，不决定是否摄入；`Quarantined` 保留给写入后发现对象损坏、来源失效或投毒时的显式处置。 +- **Alternatives considered:** 人工 Trust Gate；按来源 trust/置信度自动分流 confirmed/quarantined/rejected。前者不满足自动化边界，后者会把研发经历的价值与来源标签或模型置信度混为一谈。 +- **Consequences:** M2-01 不再定义 gate 三态类型；M2-07 实现来源、证据、脱敏和结构准入，不实现 intake trust classifier。失败与低置信度经历可自动 Confirmed，但仍保留明确 completion、trust、confidence 和 EvidenceRef，供后续 Agent 自行判断。 +- **Revisit when:** 需要接入未受当前 Agent 授权边界约束的外部来源，或 benchmark 证明自动摄入造成不可接受的持续误用。 ### ADR-M2-07：自动作业使用每 root 单行 generation + lease/fence - **Status:** Accepted - **Context:** 终态写入不能被 LLM 延迟阻塞，也不能因进程退出漏掉编译。 -- **Decision:** `memory_compile_job` 每个 `(scope, root_kind, root_id)` 最多一行。外部事实触发只接受声明的 Task/Intent 终态；另有一条内部依赖观测：从 Memory ref 看到新 Confirmed Task revision 时，只唤醒其已终态父 Intent，Intent revision 自身不再触发编译。Task 的规范输入指纹绑定 terminal source OID；Intent 的规范输入指纹绑定 terminal Intent OID + 有序贡献 Task revision OID（缺失项用稳定 marker）。只有指纹变化才递增 observed generation。`memory_compile_observer_state` 分别记录每个 scope/source ref 已完整扫描的 first-parent OID；observer 在同一 SQLite transaction 内幂等 upsert job 后推进水位。lease winner 处理到目标 generation，按 fence 推进 processed generation。 +- **Decision:** `memory_compile_job` 每个 `(scope, root_kind, root_id)` 最多一行。外部事实触发只接受声明的 Task/Intent 终态；另有一条内部依赖观测：从 Memory ref 看到新 Task Episode revision 时，只唤醒其已终态父 Intent，Intent revision 自身不再触发编译。Task 的规范输入指纹绑定 terminal source OID；Intent 的规范输入指纹绑定 terminal Intent OID + 有序贡献 Task revision OID（缺失项用稳定 marker）。只有指纹变化才递增 observed generation。`memory_compile_observer_state` 分别记录每个 scope/source ref 已完整扫描的 first-parent OID；observer 在同一 SQLite transaction 内幂等 upsert job 后推进水位。lease winner 处理到目标 generation，按 fence 推进 processed generation。 - **Alternatives considered:** 内存队列；每次触发一行永久 job；同步调用模型。三者分别会丢任务、无限增长或阻塞事实写入。 - **Consequences:** job 状态有界；终态事件 + CompileRecord 可用于 repair。 - **Revisit when:** Libra 建立通用持久作业调度 Module。 @@ -485,9 +486,9 @@ FTS 表固定 `tokenize='unicode61 remove_diacritics 2'`；`bm25()` 的列权重 - **GC-M2-07 本地存储：** 首个切片不把 Memory branch、Confidential 正文、job、receipt 或 input HMAC 进入 remote durable tier；普通 push/fetch/mirror 明确排除 `refs/heads/libra/memory/*`。 - **GC-M2-08 错误边界：** 公开失败使用稳定 `LBR-MEMORY-*`；job 内只持久化稳定 code 和脱敏摘要，不存 provider 原始响应。 - **GC-M2-09 FTS 同步：** `memory_episode_search_doc` 与 external-content FTS5 表通过同一投影 transaction 更新；禁止单独把 FTS 当事实源修补。 -- **GC-M2-10 核心覆盖：** 100% line + region 口径固定为 `domain.rs`、`canonical.rs`、`validation.rs`、`state.rs`、`trust.rs`、`applicability.rs`、`selector.rs`、`job_state.rs` 的纯逻辑；I/O Adapter、SeaORM entity、CLI glue 另走集成测试，并在 coverage 配置中具名说明。 +- **GC-M2-10 核心覆盖：** 100% line + region 口径固定为 `domain.rs`、`canonical.rs`、`validation.rs`、`state.rs`、`admission.rs`、`applicability.rs`、`selector.rs`、`job_state.rs` 的纯逻辑；I/O Adapter、SeaORM entity、CLI glue 另走集成测试，并在 coverage 配置中具名说明。 - **GC-M2-11 证据定位：** 每个可注入 claim 必须有至少一个可解析且已授权的 `EvidenceRefV1` locator，解析结果的 redacted canonical fragment digest 必须匹配；只指向整段 Session 的宽引用不合格。 -- **GC-M2-12 自动信任：** compiler 输出属于不可信 proposal；只有 `AutoEpisodeTrustGateV1` 的 confirmed 结果可进入默认注入，Quarantined 仍保留历史但不得因相关度高越过门禁。 +- **GC-M2-12 自动准入：** compiler 只能提议内容；Writer 必须从已授权 pinned source 机械覆盖可信字段并验证证据、脱敏、schema、root、code anchor 与 CompileRecord，验证通过后自动 Confirmed，失败不写入。任务结局、trust 标签和推断置信度不得成为摄入过滤条件。 - **GC-M2-13 keyed digest：** idempotency、principal、query 与 source-input HMAC 只能调用 ADR-M2-11 的 typed purpose API；禁止自定义 label、直接读取 seed、用普通 hash 代替或在密钥不可用时静默生成新 key。 ## 执行检查必备需求（强制） @@ -513,7 +514,7 @@ flowchart TB M203 --> M204["M2-04 MemoryWriter 权威历史"] M204 --> M205["M2-05 本地 Memory branch 保护"] M205 --> M206["M2-06 投影 replay / rebuild"] - M206 --> M207["M2-07 source / redaction / trust gate"] + M206 --> M207["M2-07 source / redaction / admission"] M207 --> M208["M2-08 observer / generation job"] M208 --> M209["M2-09 Task Episode compiler"] M209 --> M210["M2-10 Intent Episode compiler"] @@ -634,7 +635,7 @@ flowchart TB **Lifecycle / Acceptance:** `in-progress` / 空 -**Description:** 在首个持久化实现之前，把 `EpisodePayloadV1`、自动 producer policy、canonical digest 和稳定 Cell 规则写入单一规范，并实现不依赖 I/O 的 Rust 领域类型与校验器。 +**Description:** 在首个持久化实现之前，把 `EpisodePayloadV1`、授权来源前提与确定性准入、canonical digest 和稳定 Cell 规则写入单一规范，并实现不依赖 I/O 的 Rust 领域类型与校验器。 **Out of scope:** SQLite、对象写入、模型调用和 CLI；分别由 M2-02/M2-02F/M2-02R、M2-04、M2-09/10、M2-13 承接。 @@ -642,14 +643,14 @@ flowchart TB | 事实 | 证据 | |---|---| -| `memory.md` 已补齐 Episode payload、自动信任门、稳定 Cell 与共享 selection receipt 合同 | `docs/development/tracing/memory.md:504-633,1380-1407`、`docs/development/gap/mainline.md:267,742-780` | +| `memory.md` 已补齐 Episode payload、授权来源/准入边界、稳定 Cell 与共享 selection receipt 合同 | `docs/development/tracing/memory.md:504-633,1380-1407`、`docs/development/gap/mainline.md:267,742-780` | | crate-private `memory` Module 已落 I/O-free 领域类型、校验器与 canonical digest | `src/internal/ai/mod.rs`、`src/internal/ai/memory/{mod.rs,domain.rs,validation.rs,canonical.rs}` | -| 纯逻辑 focused 门共 28 个用例全绿；仓库级 all-target/all-feature clippy 零告警 | 2026-08-20 Rust 1.97.1：domain 4/4、canonical 3/3、validation 21/21；`cargo clippy --all-targets --all-features -- -D warnings` exit 0 | +| 纯逻辑 focused 门共 32 个用例全绿；M2-01 生产代码 clippy 零告警 | 2026-08-22 Rust 1.97.1：domain 4/4、canonical 4/4、validation 24/24；`cargo clippy --lib -- -D warnings` exit 0。仓库级 all-target/all-feature clippy 当前被本卡未修改的 `session/jsonl.rs` 两处 `drop_non_drop` 阻塞，不计作本卡通过证据 | **Acceptance criteria:** - [x] `memory.md` 定义 `EpisodePayloadV1`、Task/Intent 两级语义，并逐字段标明“trusted source 机械填充”或“compiler proposal”。 -- [x] 文档冻结 `AutoEpisodeTrustGateV1` 的三态输出、稳定 reason codes、默认注入条件及其与 Inference 真值的区别。 +- [x] 文档冻结 M2 的授权来源前提与确定性准入：成功/失败、trust 标签和推断置信度不筛除经历；来源归属、证据、脱敏、schema/root/code anchor/CompileRecord 无效时不写入。 - [x] `memory.md` 与 `mainline.md` 将选择回执统一为 ADR-M2-10 的 `ContextSelectionReceiptV1`；删除平行 Rust 类型/SQLite 表的设计入口，明确 `ContextFrame` 不变。 - [x] `MemoryNoteV1` / `MemoryEventV1` / `EpisodePayloadV1` / `EvidenceRefV1` / `CompileRecordV1` 有显式 version gate：已知版本忽略 additive unknown fields，未知不兼容版本拒绝，未知 enum/state 始终拒绝。 - [x] canonical JSON 与 `content_digest` 字段清单有跨 SHA-1/SHA-256 仓库算法无关的 golden vectors。 @@ -660,9 +661,9 @@ flowchart TB **Verification:** - [x] `source .env.test && cargo test --lib internal::ai::memory::domain`（4 passed） -- [x] `source .env.test && cargo test --lib internal::ai::memory::canonical`（3 passed） -- [x] `source .env.test && cargo test --lib internal::ai::memory::validation`（21 passed；完整 `internal::ai::memory` 28/28） -- [x] `source .env.test && cargo test --doc internal::ai::memory`（命令通过；本过滤范围 0 doctests） +- [x] `source .env.test && cargo test --lib internal::ai::memory::canonical`（4 passed） +- [x] `source .env.test && cargo test --lib internal::ai::memory::validation`（24 passed；完整 `internal::ai::memory` 32/32） +- [x] `source .env.test && cargo clippy --lib -- -D warnings` **Dependencies:** 无。 @@ -1210,13 +1211,13 @@ flowchart TB **Granularity:** `type=implementation; axis=projection replay; recovery=drop affected projection scope and rebuild forward; complete=yes; self-contained=yes; AC=8/8; VER=5/8; landing=3; prod-files=8; scope=M; deps=M2-05; writeset=serialized-at-M2-05; release=batch-release child; split-from=N/A; exception=N/A` -### Task M2-07：实现来源、证据与自动信任边界 +### Task M2-07：实现来源、证据与自动准入边界 **Task type:** `implementation` **Lifecycle / Acceptance:** `pending` / 空 -**Description:** 实现 Writer 内部的安全编译边界：把 trusted root + pinned source ref 解析为有界、鉴权、脱敏且带精确 EvidenceRef 的 source，并用 `AutoEpisodeTrustGateV1` 对 compiler proposal 给出确定性状态。 +**Description:** 实现 Writer 内部的安全编译边界：把 trusted root + pinned source ref 解析为有界、鉴权、脱敏且带精确 EvidenceRef 的 source，并对 compiler proposal 执行不含价值判断的确定性研发历程准入。 **Out of scope:** LLM prompt、job lease、Memory 写入状态机；由 M2-09/10、M2-08、M2-04 承接。 @@ -1235,7 +1236,7 @@ flowchart TB - [ ] 对象数、总字节、单项字节、session 片段数和 token estimate 均有硬上限；遗漏项进入稳定 omission 清单。 - [ ] raw bytes 经过 secret + Memory PII/private-marker policy 后才能构造 `RedactedEpisodeSource`；外部代码无法任意构造该类型。crate-private `EpisodeCompiler` Interface 只接收该类型与冻结配置，返回不可信 proposal 或 typed failure，不持有 storage/ref handle。 - [ ] resolver 为实际使用的 fragment 生成 `source_ref_oid + typed locator + redacted fragment_digest`，并在写前重新解析/比对；整段 Session 宽引用拒绝。 -- [ ] `AutoEpisodeTrustGateV1` 只接受 allowlisted compiler、合法 locator/digest、通过 redaction/injection scan、ACL/sensitivity/schema/root/code policy 的 proposal，并返回 confirmed/quarantined/rejected + reason codes；untrusted instruction、secret echo 与 unsupported observation 只能 Quarantined/Rejected，不能 Confirmed。 +- [ ] Writer 只接受受配置约束的 compiler、合法 locator/digest、完成 redaction 且满足来源归属、schema/root/code anchor/CompileRecord 合同的 proposal；通过后自动 Confirmed，无效时返回稳定有类型错误且不写入。Done/Failed/Cancelled、changed/unchanged/unknown、来源 trust 标签和推断 confidence 均不得成为摄入过滤条件。 - [ ] source manifest 只记录规范 ID/OID、locators、anchors、limits、omissions、trust/redaction/policy versions；ordinary hash/job/log/error/object 的 secret probe 为零，需要去重时使用 repo-local HMAC。 **Verification:** @@ -1245,13 +1246,13 @@ flowchart TB - [ ] `source .env.test && cargo test --test memory_episode_test source_pinned_ref_reachability` - [ ] `source .env.test && cargo test --test memory_episode_test source_principal_scope_denied` - [ ] `source .env.test && cargo test --test memory_episode_test source_secret_probe_zero_leak` -- [ ] `source .env.test && cargo test --test memory_episode_test trust_gate_untrusted_injection_quarantined` +- [ ] `source .env.test && cargo test --test memory_episode_test episode_admission_preserves_outcomes_and_confidence` **Dependencies:** M2-06（投影状态与 ref-direct diagnostic reader）。 **Deliverables:** N/A。 -**Implementation write set:** `src/internal/ai/memory/{source.rs,evidence.rs,redaction.rs,trust.rs,limits.rs,compiler/mod.rs}`、`src/internal/ai/projection/index.rs`（只增受限读取 helper）、`src/internal/ai/observed_agents/redaction.rs`（共享规则扩展）、`tests/memory_episode_test.rs`、`tests/fixtures/memory/`。 +**Implementation write set:** `src/internal/ai/memory/{source.rs,evidence.rs,redaction.rs,admission.rs,limits.rs,compiler/mod.rs}`、`src/internal/ai/projection/index.rs`（只增受限读取 helper）、`src/internal/ai/observed_agents/redaction.rs`（共享规则扩展）、`tests/memory_episode_test.rs`、`tests/fixtures/memory/`。 **Release write set:** N/A。 @@ -1275,7 +1276,7 @@ flowchart TB **C/D coverage from:** `M2-15`（C 组版本面覆盖与 D-02 由唯一发布点 M2-15 收口；**D-01 不继承**——本卡自己会推送 main/贡献分支，自行取远端 CI 证据） -**Granularity:** `type=implementation; axis=compiler safety boundary; recovery=revert resolver+redactor+trust gate and keep writer deterministic-only; complete=yes; self-contained=yes; AC=8/8; VER=6/8; landing=3; prod-files=9; scope=M; deps=M2-06; writeset=serialized-at-M2-06; release=batch-release child; split-from=N/A; exception=N/A` +**Granularity:** `type=implementation; axis=compiler safety boundary; recovery=revert resolver+redactor+admission and keep writer deterministic-only; complete=yes; self-contained=yes; AC=8/8; VER=6/8; landing=3; prod-files=9; scope=M; deps=M2-06; writeset=serialized-at-M2-06; release=batch-release child; split-from=N/A; exception=N/A` ### Task M2-08：接入终态触发与可恢复 generation job @@ -1297,7 +1298,7 @@ flowchart TB **Acceptance criteria:** - [ ] 外部事实触发集合精确为 `TaskEventKind::{Done,Failed,Cancelled}` 与 `IntentEventKind::{Completed,Cancelled}`；代码 commit 和其它研发事件不触发。 -- [ ] observer 分别从已存 cursor 到 pinned `libra/intent` 与 `libra/memory/repo` head 做有界 first-parent 增量扫描；Intent ref 发现终态，Memory ref 只接受 Confirmed Task revision 并唤醒其已终态父 Intent，忽略 Intent revision，因而不会形成自循环；head 非后代、对象损坏或超预算时 fail loud。 +- [ ] observer 分别从已存 cursor 到 pinned `libra/intent` 与 `libra/memory/repo` head 做有界 first-parent 增量扫描；Intent ref 发现终态，Memory ref 接受完成确定性准入的 Task Episode revision 并唤醒其已终态父 Intent，忽略 Intent revision，因而不会形成自循环；head 非后代、对象损坏或超预算时 fail loud。 - [ ] 同一短 SQLite transaction 内按规范输入指纹幂等 upsert root job，再推进对应 observer cursor；Task 指纹绑定 terminal source OID，Intent 指纹绑定 terminal Intent OID + 有序 Task revision OID/缺失 marker；相同指纹重扫不递增 generation，任一 observer 崩溃可在下次启动补齐。 - [ ] 同一 root 只有一个有效 lease holder；fenced-out runner 不能推进 processed generation、覆盖结果或释放新 owner 的 lease。 - [ ] runner 对 pinned target generation 解析来源并调用注入的 compiler Adapter；本卡用 deterministic fake 验证调度，M2-09/10 再接真实 Task/Intent Adapter；新 generation 到来时在 deadline 内继续或留下 dirty 状态。 @@ -1369,7 +1370,7 @@ flowchart TB - [ ] decisions、failed attempts、unresolved 每项使用 `EpisodeClaimV1`，无 EvidenceRef 或 observation/inference 标记不合法。 - [ ] completion status/time、goal、related IDs、code-change status/anchors 与 Cell 由 trusted source 机械注入；模型字段包含这些键时拒绝 proposal。 - [ ] compiler/model/prompt/rules/policy/input manifest 进入完整 CompileRecord；provider 原始响应不持久化。 -- [ ] Done/Failed/Cancelled 与 changed/unchanged/unknown 的正交组合都能产生 proposal；首次/修订 generation 经 trust gate 进入 Confirmed 或 Quarantined，重复 generation 零新事件。 +- [ ] Done/Failed/Cancelled 与 changed/unchanged/unknown 的正交组合都能产生 proposal；首次/修订 generation 通过确定性准入后进入 Confirmed，无效 proposal 返回有类型错误，重复 generation 零新事件。 - [ ] malformed JSON、超限数组、未知 enum、无证据 claim、trusted-field injection、secret echo 与 provider timeout 有 deterministic fixture 和稳定错误路径。 **Verification:** @@ -1882,7 +1883,7 @@ flowchart TB | Migration | 三张迁移的 up/idempotent/down guard/fresh-vs-upgrade/old-reader preflight | `source .env.test && scripts/run-test-filter-nonempty.sh --test db_migration_test --filter memory_core_old_reader_rejects_migrated_schema`；同样执行 `memory_fts_old_reader_rejects_migrated_schema` 与 `context_receipt_old_reader_rejects_migrated_schema` | | FTS capability | bundled/release linkage、external-content、MATCH、BM25 ASC、四平台 | `source .env.test && LIBRA_SKIP_WEB_BUILD=1 scripts/run-test-filter-nonempty.sh --release --test fts5_capability_test --filter sqlite_fts5_release_capability` | | Projection | incremental/rebuild equivalence/corruption/stale | `source .env.test && scripts/run-test-filter-nonempty.sh --test memory_episode_test --filter projection_rebuild_equivalence` | -| Source/security | pinned ref、authz、bounds、precise locator、secret/PII、HMAC、trust gate | `source .env.test && scripts/run-test-filter-nonempty.sh --test memory_episode_test --filter source_secret_probe_zero_leak` | +| Source/security | pinned ref、authz、bounds、precise locator、secret/PII、HMAC、Episode admission | `source .env.test && scripts/run-test-filter-nonempty.sh --test memory_episode_test --filter source_secret_probe_zero_leak` | | Job/recovery | exact terminal triggers、双 observer cursor、Task→Intent 依赖唤醒、generation、lease/fence、crash/repair | `source .env.test && scripts/run-test-filter-nonempty.sh --test memory_episode_test --filter observer_task_revision_requeues_parent_intent` | | Compiler | completion/code-change matrix、root binding、unsupported claim、Task→Intent revision pins | `source .env.test && LIBRA_ENABLE_TEST_PROVIDER=1 scripts/run-test-filter-nonempty.sh --features test-provider --test memory_episode_test --filter task_episode_completion_code_change_matrix`；`source .env.test && LIBRA_ENABLE_TEST_PROVIDER=1 scripts/run-test-filter-nonempty.sh --features test-provider --test memory_episode_test --filter intent_episode_pins_task_revisions` | | Retrieval | filters、BM25 ASC/weights、五态 applicability、ties、evidence authz | `source .env.test && scripts/run-test-filter-nonempty.sh --test memory_episode_test --filter reader_git_applicability_path_matrix` | @@ -1906,7 +1907,7 @@ flowchart TB | M2-04 | MemoryWriter §4.2.1 | memory writer/store/tree | error codes | writer integration | | M2-05 | local-only owned ref + op/push/fetch risk | shared owned-ref classifier + branch/op/transport callsites | local branch/ref boundary | branch/op/transport integration | | M2-06 | projection/rebuild contract | replay/projection | memory.md | projection matrix | -| M2-07 | existing AI graph + RedactedBytes | source/evidence/redaction/trust gate | privacy/limits/trust | source/security/trust matrix | +| M2-07 | existing AI graph + RedactedBytes | source/evidence/redaction/admission | privacy/limits/admission | source/security/admission matrix | | M2-08 | exact terminal events + export job pattern | dual observer cursor/dependency wake/job/runner adapter | auto trigger/recovery | observer/job matrix | | M2-09 | Task Episode design | compiler/task + prompt | CompileRecord | task compiler fixtures | | M2-10 | Intent→Task graph | compiler/intent | pinned link semantics | intent compiler fixtures | @@ -1953,7 +1954,7 @@ flowchart TB | 并发首次创建双 note | 高：一个 root 两个权威摘要 | Cell idempotency + CAS winner recheck | M2-04 | | ref CAS 与 projection 原子边界不清 | 高：半提交或水位失真 | 通用 transaction primitive + fault injection；权威审计只用同一历史内的 MemoryEvent/CompileRecord | M2-03,M2-04,M2-06 | | terminal fact 成功但 job observe 丢失 | 高：自动 Memory 漏记 | 持久 source cursor 增量重扫并在 job upsert 后原子推进 | M2-02,M2-08 | -| Task Episode 晚到或修订后父 Intent 不再编译 | 高：需求迭代摘要永久陈旧 | Memory ref observer 只传播 Confirmed Task→已终态 Intent，输入指纹去重并忽略 Intent revision | M2-08,M2-10 | +| Task Episode 晚到或修订后父 Intent 不再编译 | 高：需求迭代摘要永久陈旧 | Memory ref observer 传播完成确定性准入的 Task Episode→已终态 Intent，输入指纹去重并忽略 Intent revision | M2-08,M2-10 | | Intent 聚合读取漂移 Task head | 中：历史不可重放 | link 固定 note_id+revision_oid | M2-10 | | op restore 回退 Memory ref | 高：破坏 event sourcing | shared owned-ref classifier | M2-05 | | FTS5 在某 release target 未启用 | 高：公开命令不可用 | bundled build contract + 四平台 capability required gate | M2-02F,M2-14C,M2-15 | @@ -1981,7 +1982,7 @@ flowchart TB ## 兼容与文档收口 -- [ ] `docs/development/tracing/memory.md` 已包含 Episode payload、自动 producer policy、digest、writer/source/job/recall/injection 精确合同。 +- [ ] `docs/development/tracing/memory.md` 已包含 Episode payload、授权来源与准入策略、digest、writer/source/job/recall/injection 精确合同。 - [ ] `docs/development/plan/plan-long.md` 日期计划索引与 MEM-01/02 实际子状态已同步。 - [ ] `docs/commands/memory.md` 与 `docs/commands/zh-CN/memory.md` 已同步。 - [ ] `docs/development/commands/memory.md` 已同步。 @@ -2003,6 +2004,9 @@ Result 只允许 `PASS` 或 `FAIL`；最近一轮必须为 `PASS` 才能开工 | R3 | R2 修订后的冻结计划 | FAIL | base CI / CodeQL D-01 只承诺开工日提取，没有在依赖表和 M2-15 固定 workflow、job、PR ref、head SHA 与成功判据 | 新增 DEP-M2-CI-02；保留开工日提取作为漂移校验 | 独立 reviewer 对 SHA-256 `1c7348ad…86e0` 的只读复审，2026-08-20；其余 R2 六项均确认关闭 | | R4 | R3 修订后的冻结计划 | FAIL | M2-15 推送后等待/失败状态误用 `locally-accepted`；M2-14C/M2-15 消费 DEP-M2-CI-01 却未在全部依赖字段登记 | 发布点改用 `remote-pending`；两卡 Dependencies/Granularity/审计行统一显式依赖 | 独立 reviewer 对 SHA-256 `ee06a355…e5bb4` 的只读复审，2026-08-20；DEP-M2-CI-02 精确合同确认通过 | | R5 | R4 修订后的冻结计划 | PASS | 无 | 无遗留 P2 | 独立 reviewer 对 SHA-256 `a52c554b…e04` 的只读复审，2026-08-20；状态机、依赖、任务审计、C/D 字段与 `plan-long.md` 索引全部一致 | +| R6 | M2-01 实现与用户准入语义修订 | FAIL | canonical digest 使用整对象黑名单投影；Episode golden 过滤器零命中；EpisodeCompiler 可使用 Namespace 去重 | 改为显式 v1 白名单投影；落精确 Episode golden；Namespace 仅允许 Consolidation/Onboard；补 135 组合生产路径矩阵 | `gpt-5.6-sol` / xhigh 只读实现审查，2026-08-22；无 P0、无范围越界或虚假测试 | +| R7 | R6 修订后的 M2-01 | FAIL | 无 P0/P1 | 修正 `ResolvedMemoryView.view_hash` 与固定 receipt envelope 的最后一处 P2 文案冲突，并清理 Phase A 旧去重表述 | 同一 `gpt-5.6-sol` / xhigh reviewer 复审，2026-08-22 | +| R8 | R7 修订后的 M2-01 | PASS | 无 | P0/P1/P2 全部关闭 | 同一 `gpt-5.6-sol` / xhigh reviewer 最终复审，2026-08-22；确认无计划偏离、过度防御、范围越界、虚假或零命中测试 | ## 非目标与延后项 diff --git a/docs/development/tracing/memory.md b/docs/development/tracing/memory.md index 0a606cc0a..ecc1b55e3 100644 --- a/docs/development/tracing/memory.md +++ b/docs/development/tracing/memory.md @@ -493,7 +493,7 @@ metrics 不进入上表的权威 namespace：高频遥测会放大 Git 历史、 - view 解析必须 fail-closed：任何参与层出现未知 policy、watermark 不匹配、principal 不可验证或 scope 编码失败时，该层不得进入 view；安全敏感读取不得静默降级到更宽的 Repo / Global 层。 - SessionStart 冻结初始 view；branch、worktree 或 principal 改变时必须重新解析并产生新的 `view_hash`。旧 ContextReceipt 继续引用旧 view，不被当前状态覆盖。 - `memory status` 可按当前状态即时重算 view，并展示每层 source OID、新鲜度和排除原因；session attach / refresh 只进入本地有界 access audit，不推进 memory ref。 -- `ContextReceipt.as_of` 必须包含 `view_hash` 与完整 layer snapshot。selector 不得在一次注入过程中重新读取当前 HEAD 或 wall clock 后偷换 view。 +- `ContextSelectionReceiptV1` 通过 `code_commit` / `full_branch_ref`、`source_heads`、`projection_watermarks` 与 `policy_hash` 记录构成该冻结 view 的输入；debug/status 可按这些字段重算并对照 `view_hash`，回执不另设平行的 `as_of` envelope。selector 不得在一次注入过程中重新读取当前 HEAD 或 wall clock 后偷换 view。 ## 4. 对象模型 @@ -519,6 +519,7 @@ Memory 遵循与 Libra 其余部分**相同的快照（Snapshot）/ 事件（Eve | `lifecycle` | enum | `Replacement`（覆盖式）/ `Accretive`（累加式） | | `body` | `String` | 被记住的陈述（允许 Markdown，保持简短） | | `rationale` | `Option` | 可选的「为何重要」/「从何而来」说明 | +| `episode` | `Option` | M2 研发历程负载（§4.1.3）；规范 JSON 始终输出该键，无负载时为 `null` | | `evidence_refs` | `Vec` | 指向 `Evidence`、`Run`、`Decision`、commit OID 的指针，用以佐证该条记忆 | | `links` | `Vec` | 显式的 sibling / supports / prerequisite / contradicts / supersedes 链接 | | `entities` | `Vec` | 可选的结构化实体 mention；用于 alias 消歧与可重建实体索引，不创建第二套实体真源 | @@ -541,7 +542,7 @@ Memory 遵循与 Libra 其余部分**相同的快照（Snapshot）/ 事件（Eve - `MemoryNote` JSON 必须使用稳定字段名与向后兼容的 serde 策略。新增字段只能 additive，旧 reader 必须忽略未知字段；删除或改变字段语义必须 bump `schema_version` 并提供迁移 / rebuild 逻辑。 - `revision_oid` 是 `MemoryNote` blob 写入后得到的 Git OID，只存在于 tree path、`MemoryEvent`、返回 envelope 与投影中，**不得**序列化进 `MemoryNote` 正文。否则会出现“正文包含自身哈希”的不可解自引用。 - canonical JSON 必须固定 UTF-8、字段序、数字与时间格式；`content_digest` 的输入排除该字段本身。读取时同时校验 blob OID 和 `content_digest`，任一不符均视为损坏。 -- **canonical payload 的字段清单（`content_digest` 的唯一规范来源）。** 为消除「digest 不含任何存储 OID」与 `parents` / `evidence_refs` 含 Git OID 之间的冲突，digest 输入被固定为一组**明确列出的内容字段**：`schema_version`、`note_id`、`namespace`、`path`、`kind`、`scope`、`visibility`、`acl_policy_id`、`lifecycle`、`body`、`rationale`、`links`（links 中仅取 `kind` + `target_note_id`，不含 `target_revision_oid`）、`entities`、`tags`、`confidence`、`trust`、`sensitivity`、`valid_from`、`valid_until`、`effective_from_commit`、`effective_until_commit`、`expires_at`、`author`、`created_at`、`compile_record`。**排除**：`content_digest` 自身、`revision_oid`、`evidence_refs`、`parents` 以及任何 Git OID——这些是存储布局 / 引用字段，与正文内容分离，由 blob OID 与事件引用承载完整性。字段清单本身纳入 `schema_version` 的版本化语义：清单变更必须 bump schema 并提供迁移 / rebuild。 +- **canonical payload 的字段清单（`content_digest` 的唯一规范来源）。** digest 输入固定为 `schema_version`、`note_id`、`namespace`、`path`、`kind`、`scope`、`visibility`、`acl_policy_id`、`lifecycle`、`body`、`rationale`、`episode`、`evidence_refs` 的语义投影、`links` 的语义投影、`entities` 的语义投影、`tags`、`confidence`、`trust`、`sensitivity`、`valid_from`、`valid_until`、`effective_from_commit`、`effective_until_commit`、`expires_at`、`author`、`created_at`、`compile_record`。代码适用性 OID（包括 `effective_*_commit`、Episode code anchor 与 `code_range.commit_oid`）属于语义字段并参与 digest。**排除**：`content_digest` 自身、正文中不存在的 `revision_oid`、`parents`、`EvidenceRef.source_ref_oid`、`EvidenceRef.fragment_digest` 与 `MemoryLink.target_revision_oid`；这些字段用于存储定位或完整性复核。EvidenceRef 的 source plane、kind、object ID、locator、visibility、captured time 与可选 code commit 仍参与 digest。canonical writer 固定 UTF-8、对象键 Unicode 码点序、数组顺序、整数与 RFC 3339 时间格式；字段清单或投影规则变化必须 bump `schema_version` 并提供迁移 / rebuild。 - 一个 `MemoryNote` 快照回答的是**「agent 在这一版本相信什么？」**，且永不被改写。 - 撤销、取代或遗忘一条记忆都是一个**事件（Event）**，而非对快照的就地编辑；cache prune 不改变 note 生命周期。 - 对同一 `note_id` 而言，`namespace`、`scope`、`path` 在逻辑上不可变。要移动一条记忆，应写一条新 note 并取代旧的（§10.2）。 @@ -558,7 +559,8 @@ Memory 遵循与 Libra 其余部分**相同的快照（Snapshot）/ 事件（Eve | 字段 | 类型 | 含义 | |---|---|---| -| `origin` | enum | `Explicit` / `PromotedFromAnchor` / `DistilledFromFrame` / `Classifier` / `Consolidation` / `Onboard` / `BranchFork` / `Import` / `Coordinator`（MEM-06 协调写入，additive，见 §19.4） | +| `schema_version` | `u32` | 编译记录 schema；第一版固定为 `1` | +| `origin` | enum | `Explicit` / `PromotedFromAnchor` / `DistilledFromFrame` / `Classifier` / `Consolidation` / `Onboard` / `BranchFork` / `Import` / `Coordinator` / `EpisodeCompiler` | | `producer` | `String` | 生产者标识与版本，如 `libra-memory/0.19.0` 或 `consolidation-job/1` | | `rules_version` | `u32` | 确定性规则集（worthiness 正则、路径验证、redaction 策略）的版本 | | `prompt_version` | `Option` | 参与生产的 LLM prompt 模板版本；纯确定性路径为 `None` | @@ -572,7 +574,7 @@ Memory 遵循与 Libra 其余部分**相同的快照（Snapshot）/ 事件（Eve - 写入事务（§4.2.1）第 1 步即校验编译记录完整性：`origin` 与调用入口不符、`input_hashes` 为空或幂等键缺失，一律 fail-closed 拒绝写入。 - 幂等键去重只作用于**新建**（`Created`）：默认按 `idempotency_scope = Cell` 去重，即同一 `(scope, namespace, path)` cell 内同键重复摄入不产生新 note，直接返回既有 `note_id` 且不追加新事件（与 §4.2 的 event 幂等语义一致）；**不同 path 的目标互不干扰**，调用方显式指定的新路径绝不会被同内容去重静默丢弃（§7.4 纪律）。仅 consolidation / onboard 等聚合入口可显式使用 `idempotency_scope = Namespace`（键不含 path），且必须在编译记录中标注。显式 `revise` / `move` 针对既有 `note_id`，不受其约束。`memory_note_index` 的幂等唯一索引按 §5.2 相应区分两种 scope。 -- LLM 参与生产的 note（`prompt_version` / `model_id` 非空）默认最高只能进入 `Draft`；`trust` 上限沿 §7.3 规则，不因编译记录存在而放宽。 +- LLM 参与生产的普通知识 note（`prompt_version` / `model_id` 非空）默认最高只能进入 `Draft`；`trust` 上限沿 §7.3 规则，不因编译记录存在而放宽。首版 M2 Episode 是单独的全自动研发历程路径：它只消费当前 Agent 已经通过用户启动的任务、仓库身份与现有访问控制获准读取的来源，不要求用户逐条二次确认，也不按任务成功/失败、来源 trust 标签或推断置信度决定是否保留。Writer 在持久化前执行 §4.1.3 的确定性研发历程准入；通过后连续追加 `Created|Revised + Confirmed`，失败则返回有类型错误且不写入 Memory ref。compiler 自报的身份、来源范围或准入结果不生效。 - 发现某个 producer / prompt / model 版本产出系统性坏记忆时，必须能按编译记录批量定位受影响 note 并 quarantine 或重新编译——这是把编译记录设为硬门槛的直接回报。 - 编译记录是 note 正文的一部分，随 blob 不可变、可随投影重建；`memory_note_index` 投影为此新增 `origin` 与 `idempotency_key` 列（§5.2），存储创建版本的键以支撑去重与批量召回。 - 每个 revision 的 producer / prompt / model / policy / input fingerprint 进入 `memory_revision_index` 投影；只在 note 级保存创建 origin 无法定位后续坏 revision，因此不能满足批量 quarantine / recompile 要求。 @@ -582,12 +584,54 @@ Memory 遵循与 Libra 其余部分**相同的快照（Snapshot）/ 事件（Eve 为使授权、provenance 与图扩展可实现，以下 supporting types 必须 versioned，不能留成无约束 JSON： -- `EvidenceRef` 至少包含 `schema_version`、`source_plane`、`kind`、`object_id`、`content_hash`、`visibility`、可选 `captured_at` / `code_commit`。解析或授权失败只会降低 trust / 排除候选，绝不能通过远程 URL 即时抓取来“补齐证据”。 +- `EvidenceRefV1` 包含 `schema_version`、`source_plane`、`kind`、`object_id`、`source_ref_oid`、封闭的 `locator`、脱敏规范片段的 `fragment_digest`、`visibility` 与可选 `captured_at` / `code_commit`。locator 只允许 `object`、`event_seq`、`json_pointer`、`session_fragment`、`tool_call`、`code_range` 六种形态；Session 必须给有界序号区间，工具调用必须指定 invocation/output part，代码必须固定 commit、仓库相对路径和行范围。resolver 在授权和脱敏后重算 fragment digest；解析、授权或 digest 比对失败会隔离/排除候选，不通过网络即时补取来源。 - `MemoryEntityMention` 至少包含 `schema_version`、`canonical_key`、`display_name`、`aliases`、`role`（subject / object / topic）、`resolution_confidence` 与 evidence 指针。`canonical_key` 是 repository-local 的规范键，不得编码 actor PII；alias 冲突只产生 merge proposal，不自动改写真源。 - `MemoryLink` 至少包含 `kind`、`target_note_id`、可选 `target_revision_oid`、`evidence_refs` 与可选 `valid_from` / `valid_until`。link 归属于 source `MemoryNote` revision，其变化通过 source note 的新 revision 表达；需要独立 review / revoke / supersede 生命周期的关系必须建模为单独 semantic note，不能把嵌入式 link 扩成第二套事件系统。读取 link 前先对 target 重新执行 scope / ACL / sensitivity 检查；禁止先图扩展后过滤，否则计数、路径和 timing 都会泄露私有节点。 - `acl_policy_id` 指向 canonical policy snapshot hash。policy 至少定义可读/可写 principal、允许的 namespace/scope、自动确认上限、远端存储许可、retention 与 prompt 注入许可。policy 缺失、未知或 hash 不匹配时，mutating path 与 prompt injection fail-closed。 - `manifest.json` 固定当前 ref 的 scope、schema version、last_event_seq、policy snapshot hash 与 writer version。它不得携带 secret 或 actor PII；actor ref 使用 principal HMAC / opaque ID。 +#### 4.1.3 `EpisodePayloadV1` —— M2 研发历程负载 + +M2 从现有 Intent / Task / Run / Evidence / Decision / PatchSet / Session 与代码提交编译两级研发历程： + +- **任务研发历程摘要（Task Episode Summary）**：汇总一个终态 Task 下的多次 Run、决策、失败尝试、未决项和代码结果。 +- **需求迭代摘要（Intent Iteration Summary）**：汇总一个终态 Intent 下已经固定 revision 的多个 Task Episode；每个贡献 Task 都必须有且只有一个 `kind=Supports` 的 `MemoryLink`，以 Task 的稳定 `note_id + revision_oid` 固定具体版本，后续 Task 修订只会触发新的 Intent revision。 + +`MemoryWriter` 从受信任触发器和已授权 source window 机械填充身份与边界字段；`EpisodeCompiler` 只提议自然语言内容： + +| 字段 | 所有者 | 约束 | +|---|---|---| +| `schema_version` | Writer | 固定为 `1` | +| `root_kind` / `root_id` | Writer | `task \| intent`；编译器回显必须完全一致 | +| `related_intent_ids` / `related_task_ids` / `related_run_ids` | Writer | 排序、去重、有界；Intent Episode 至少包含一个贡献 Task | +| `started_at` / `ended_at` | Writer | 从来源事件时间计算，`started_at <= ended_at` | +| `completion_status` | Writer | `completed \| failed \| cancelled`，来自终态事实 | +| `code_change_status` | Writer | `changed \| unchanged \| unknown`，与任务成败正交 | +| `code` | Writer | 可选 base/result commit、完整 branch ref、排序去重后的仓库相对路径 | +| `goal` | Writer | 从已授权来源机械生成 observation，保持来源目标语义并带 EvidenceRef；compiler 不得输出或覆盖 | +| `summary` | compiler proposal | 固定为 inference，带置信度与 EvidenceRef | +| `observations` | compiler proposal | 每项固定为 observation，不带置信度，EvidenceRef 非空 | +| `inferences` | compiler proposal | 每项固定为 inference，必须带置信度与 EvidenceRef | +| `decisions` / `failed_attempts` / `unresolved` | compiler proposal | 每项显式标记 observation 或 inference，并带 EvidenceRef | +| `omissions` | Writer | 记录每个有界集合被裁掉的条目数 | + +`EpisodeClaimV1` 固定为 `epistemic_status + claim + confidence? + evidence_refs`。Observation 禁止携带 confidence；Inference 必须携带 confidence。校验只能证明引用存在、可见且定位/digest 匹配，不能把自然语言蕴含声明为形式化证明。 + +首个仓库切片固定 `kind=Episodic`、`scope=Repo`、`visibility=RepoLocal`、`lifecycle=Accretive`、`namespace=default`，并使用： + +```text +episodic.tasks.r- +episodic.intents.r- +``` + +root ID 采用精确 UTF-8 字节，拒绝空值、首尾空白、控制字符和超过 120 bytes 的输入。稳定 `note_id` 使用冻结的 UUIDv5 namespace `f2b4d3a0-1c9e-4f75-8d20-2a6b7c8d9e01` 与 name bytes ` + NUL + root_id` 生成，独立于仓库 SHA-1 / SHA-256 object format。存在 result commit 时 `effective_from_commit=result_oid`，否则使用 base OID；`effective_until_commit=None`。`episode` 有值当且仅当 `CompileRecord.origin=EpisodeCompiler`，避免普通 note 冒充自动编译结果或 Episode 丢失编译来源。 + +解析器在反序列化前执行原始 JSON 字节上限：`CompileRecord <= 32 KiB`、`MemoryNote <= 256 KiB`、`MemoryEvent <= 128 KiB`、`EpisodePayload <= 64 KiB`、`EvidenceRef <= 16 KiB`。结构内上限为：`body <= 16 KiB`、单个自然语言/路径项 `<= 4 KiB`、标识字段 `<= 512 bytes`、每个可变集合 `<= 128` 项、单个 Session fragment `<= 256` 个序号；线性 note 的 `parents <= 1`。超限先按稳定规则裁剪并记录 omissions，仍超限则拒绝写入。 + +所有 v1 reader 采用同一兼容规则：`MemoryNoteV1`、`MemoryEventV1`、`EpisodePayloadV1`、`EvidenceRefV1` 与 `CompileRecordV1` 在 `schema_version=1` 时忽略 additive unknown fields；未知 schema version、未知 enum、未知 lifecycle/action/state 一律拒绝。 + +首版 M2 Episode 不定义 `AutoEpisodeTrustGateV1`，也不产生 `confirmed | quarantined | rejected` 三态摄入结果。它采用**授权来源前提**：当前 Agent 可读取的 Task / Intent / Run / Session 等研发来源，已经由用户启动的任务、仓库身份和现有访问控制授权；Memory 不再要求用户逐条批准，也不对成功、失败、取消、未改代码或低置信度推断作价值判断。Writer 仍执行确定性的**研发历程准入**：验证来源属于 pinned ref 与当前仓库/工作区、EvidenceRef locator/digest 可解析、脱敏已经完成、schema/root/code anchor/CompileRecord 合法且 compiler 实现受配置约束。全部有效时机械追加 `Created|Revised + Confirmed`；任一无效时返回稳定的解析、授权、脱敏或合同错误且不写入 Memory ref。`trust` 与 confidence 继续作为 provenance/认识论标签保存，不是 M2 摄入门槛；`Quarantined` 保留给写入后发现对象损坏、来源失效或投毒时的显式处置。 + ### 4.2 `MemoryEvent` —— 事件 [E] 针对某个 `MemoryNote` 的只追加（append-only）生命周期记录。 @@ -975,40 +1019,54 @@ CREATE TABLE memory_embedding_cache ( created_at TEXT NOT NULL ); --- 注入回执账本（§8.6）。注意：这是本地 append-only 审计账本， +-- 共享上下文选择回执账本（§8.6）。这是本地 append-only 审计账本， -- **不是**投影——它记录读取时刻的选择，无法也无须从 Git 历史重建； -- rebuild 不触碰它，保留策略负责有界修剪。 -CREATE TABLE memory_context_receipt ( +CREATE TABLE context_selection_receipt ( receipt_id TEXT PRIMARY KEY, - emitted_at TEXT NOT NULL, - scope_key TEXT NOT NULL, - view_hash TEXT NOT NULL, - source_refs_json TEXT NOT NULL, - projection_watermarks_json TEXT NOT NULL, - as_of_commit TEXT, + schema_version INTEGER NOT NULL, + source_kind TEXT NOT NULL, + repository_id TEXT NOT NULL, + digest_key_id TEXT NOT NULL, + principal_hmac TEXT NOT NULL, + query_hmac TEXT NOT NULL, effective_at TEXT NOT NULL, + code_commit TEXT, + full_branch_ref TEXT, + source_heads_json TEXT NOT NULL, + projection_watermarks_json TEXT NOT NULL, + policy_hash TEXT NOT NULL, selector_version TEXT NOT NULL, - rules_version INTEGER NOT NULL, - index_version TEXT NOT NULL, - policy_version TEXT NOT NULL, - query_hmac TEXT, token_budget INTEGER NOT NULL, - tokens_used INTEGER NOT NULL, selected_json TEXT NOT NULL, - dropped_json TEXT NOT NULL, - bundle_hash TEXT NOT NULL + omissions_json TEXT NOT NULL, + bundle_hash TEXT NOT NULL, + reproducibility_state TEXT NOT NULL, + frame_id TEXT, + recorded_at TEXT NOT NULL +); +CREATE INDEX idx_context_selection_receipt_repository_time + ON context_selection_receipt(repository_id, recorded_at); +CREATE INDEX idx_context_selection_receipt_time + ON context_selection_receipt(recorded_at); + +-- 每仓回执保留水位。ReceiptStore 与回执追加/裁剪在同一短事务更新它； +-- pruned_before 是已经删除的最晚 recorded_at，用来区分 expired 与 not_found。 +CREATE TABLE context_selection_receipt_retention ( + repository_id TEXT PRIMARY KEY, + pruned_before TEXT, + last_pruned_at TEXT, + retained_rows INTEGER NOT NULL DEFAULT 0 CHECK (retained_rows >= 0) ); -CREATE INDEX idx_memory_receipt_time - ON memory_context_receipt(emitted_at); ``` -`memory_head`、`memory_path_summary`、`memory_note_index`、`memory_revision_index`、`memory_link_index`、`memory_entity_index`、`memory_taxonomy_node`、`memory_projection_state` 是可重建投影。`memory_classifier_cache` 与 `memory_embedding_cache`（§8.7）是可丢弃 cache；`memory_access_stats` 与 `memory_context_receipt` 是本地有界账本，不能从 Git 历史重建，`rebuild` 不触碰它们。删除账本会降低本地可观测性，但不能改变 live memory 语义。 +`memory_head`、`memory_path_summary`、`memory_note_index`、`memory_revision_index`、`memory_link_index`、`memory_entity_index`、`memory_taxonomy_node`、`memory_projection_state` 是可重建投影。`memory_classifier_cache` 与 `memory_embedding_cache`（§8.7）是可丢弃 cache；`memory_access_stats`、`context_selection_receipt` 与 `context_selection_receipt_retention` 是本地有界账本及其保留水位，不能从 Git 历史重建，`rebuild` 不触碰它们。删除账本会降低本地可观测性，但不能改变 live memory 语义。 查询实现必须始终带上 `scope_key` 与 `namespace`，禁止只按 `path` 做全局查询后在内存中过滤。跨 scope / namespace 的检索只能由显式 `--all-namespaces` 或策略允许的 scope fallback 触发，并且必须在结果中保留原始 `scope` 与 `namespace`，防止 prompt 注入时发生来源混淆。 `list_prefix` 与 `summarize` 不得执行无上限扫描。实现应使用规范化后的 path 前缀范围查询和 keyset pagination，并设置默认 `LIMIT`（建议 100 条 summary、50 条 note）与硬上限。因为 SQL `LIKE` 的转义与 collation 容易引入前缀越界，推荐存储 canonical `path_key` / `parent_path` 后做复合索引范围查询；复杂度表述统一为 O(log n + k)，其中 k 是有界返回量。 -在访问模式上还有一条对齐约定值得明确：Memory 的投影表用 SeaORM entity 来访问（与同样可重建的 `ai_index_*` 投影一致），而不采用 `agent_session` / `agent_checkpoint` / `agent_usage_stats` 那种**故意**保持的 raw-SQL、无 entity 风格。原因在于：`agent_*` 那批表是外部捕获的独立账本，而 Memory 的这些表是 git 真源（`refs/libra/memory/...`）的可重建投影，本质与 `ai_index_*` 同类，因而对齐 `ai_index_*` 的 SeaORM 模式。账本例外是 `memory_access_stats` 与 `memory_context_receipt`（§8.6）：二者都不是可重建投影，沿用 raw-SQL 账本模式，不配 entity。 +在访问模式上还有一条对齐约定值得明确：Memory 的投影表用 SeaORM entity 来访问（与同样可重建的 `ai_index_*` 投影一致），而不采用 `agent_session` / `agent_checkpoint` / `agent_usage_stats` 那种**故意**保持的 raw-SQL、无 entity 风格。原因在于：`agent_*` 那批表是外部捕获的独立账本，而 Memory 的这些表是 git 真源（`refs/libra/memory/...`）的可重建投影，本质与 `ai_index_*` 同类，因而对齐 `ai_index_*` 的 SeaORM 模式。账本例外是 `memory_access_stats` 与 `context_selection_receipt`（§8.6）：二者都不是可重建投影，沿用 raw-SQL 账本模式，不配 entity。 ### 5.3 ClientStorage 分层 @@ -1239,6 +1297,12 @@ LLM 输出必须按 schema 校验：未知 enum、未知 namespace、非法 path #### 7.5.1 Trust Gate —— Draft → Confirmed 的显式晋升阶段（A4） +本节适用于 M1 事实、规则、技能和其它需要晋升的普通知识流。首版 M2 Episode +按 §4.1.3 的“授权来源前提 + 研发历程准入”直接自动确认，不进入 +`pass | fail | needs-human` 三态，也不因为任务失败、来源 trust 标签或推断置信度 +较低而转入 Draft / Quarantined。二者共享脱敏、EvidenceRef、CompileRecord、 +MemoryWriter 和写入后隔离能力，但摄入语义不同。 + 沿 §0.0.4（fava-trails Trust Gate）与 §0.2，`Draft → Confirmed` 不是自动 完成，而必须经过一个命名的 **Trust Gate** 阶段，把「确定性规则 + 可选 LLM 评审」统一为一条可审计的晋升路径： @@ -1331,29 +1395,33 @@ memory.get(scope, namespace, path) -> Vec `scope`、`confidence`、`trust` 以及一个简短的证据指针。agent 被告知： 记忆只是指引，当前的源文件 / 命令输出会覆盖陈旧的记忆。 -### 8.6 注入回执（`ContextReceipt`，Phase C 硬性门槛） +### 8.6 上下文选择回执（`ContextSelectionReceiptV1`，简称 `ContextReceipt`，Phase C 硬性门槛） -每次 `with_memory(...)` 注入（§8.5）以及引擎内召回（§8.2 / §8.3）完成后，必须产出一份 `ContextReceipt`；写不出完整回执，该次注入 / 召回按失败处理（fail-closed）。直接路径 get（§8.1）与调用方驱动原语（§8.4）是确定性查询、不含引擎侧选择，不产回执。 +每次 `with_memory(...)` 注入（§8.5）以及引擎内召回（§8.2 / §8.3）完成后，必须产出一份共享 `ContextSelectionReceiptV1`；写不出完整回执，该次注入 / 召回按失败处理（fail-closed）。直接路径 get（§8.1）与调用方驱动原语（§8.4）是确定性查询、不含引擎侧选择，不产回执。该类型由 `src/internal/ai/context_budget/receipt.rs` 的共享上下文层拥有，Memory 与 mainline 都写入同一账本；现有 `ContextFrame` wire schema 保持不变。 | 字段 | 含义 | |---|---| -| `receipt_id` | 回执标识 | -| `emitted_at` | 产出时间 | +| `receipt_id` | UUIDv7 回执标识 | +| `schema_version` / `source_kind` | 回执版本与调用来源（Memory / intent / hook 等） | +| `repository_id` / `digest_key_id` | 规范仓库身份与仓库本地 keyed-digest generation | +| `principal_hmac` / `query_hmac` | 已认证主体与规范化查询/过滤/K/排序输入的域分离 HMAC；不保存 raw principal/query | +| `effective_at` | 本次选择冻结的有效时间；重放不得重新读取当前 wall clock | | `selected` | 选中的 note `revision_oid` / path summary key 列表，每项附 reason code、稳定 score 分量与顺序 | -| `dropped` | 因预算 / 门禁被丢弃项及 reason code | -| `token_budget` / `tokens_used` | 预算与实际用量 | -| `as_of` | `ResolvedMemoryView.view_hash`、解析时所有参与 scope 的 memory ref OID、各自 projection watermark 与 code commit / full branch ref | -| `versions` | selector / rules（编译规则集）/ index（投影 schema）/ policy 版本 | -| `effective_at` | 选择时冻结的时间，用于 TTL / recency；重放不得读取当前 wall clock | -| `query_hmac` | 可选，本地密钥派生的查询 HMAC；不得保存 raw query 或可跨仓库关联的普通 hash | +| `omissions` | 因授权、状态、适用性、预算或资源上限被排除项的稳定 reason code 与有界计数 | +| `token_budget` | 共享上下文层冻结的预算 | +| `code_commit` / `full_branch_ref` | 本次选择使用的代码锚点 | +| `source_heads` / `projection_watermarks` | `ResolvedMemoryView` 的各来源 ref OID 与投影水位 | +| `policy_hash` / `selector_version` | 授权/选择合同版本 | | `bundle_hash` | 注入渲染块的规范化哈希 | +| `reproducibility_state` | `reproducible \| stale \| expired \| non_reproducible` | +| `recorded_at` / `frame_id?` | 本地记录时间与可选 ContextFrame 关联；均不把 raw frame 写入回执 | 规则（承 §0.0.2 与 §0.0.11）： -- **同输入同选择可验证。** 固定所有参与 scope 的 source ref OID / projection watermark、code anchor、`effective_at`、query HMAC 对应的调用输入、同一版本组与预算，重放必须得到相同 selected IDs、顺序、reason codes 与 `bundle_hash`；`receipt_id`、`emitted_at` 等非确定字段不进入 canonical hash。缺失对象、policy 或 index snapshot 时返回 `stale / non-reproducible`，不静默 fallback。该承诺只覆盖选择和渲染输入，不承诺 provider 输出逐字节一致。 -- **回执是本地审计账本，不是投影。** 存入本地 append-only 表 `memory_context_receipt`（§5.2）；它记录的是读取时刻的选择，无法也无须从 Git 历史重建，因此明确豁免于「删表可 rebuild」条款（§13.1），并按保留策略有界修剪。回执、query hash 与 selected IDs 可能泄露工作意图，默认 local-only、不进团队 ref；任何共享另做 visibility / retention / threat-model 审查。 +- **同输入同选择可验证。** 固定所有参与 scope 的 source ref OID / projection watermark、code anchor、`effective_at`、query HMAC 对应的调用输入、同一版本组与预算，重放必须得到相同 selected IDs、顺序、reason codes 与 `bundle_hash`；`receipt_id`、`recorded_at` 等非确定字段不进入 canonical hash。缺失对象、policy 或 index snapshot 时返回 `stale / non-reproducible`，不静默 fallback。该承诺只覆盖选择和渲染输入，不承诺 provider 输出逐字节一致。 +- **回执是本地审计账本，不是投影。** 存入本地 append-only 表 `context_selection_receipt`（§5.2）；它记录读取时刻的选择，无法也无须从 Git 历史重建，因此明确豁免于「删表可 rebuild」条款（§13.1）。默认每仓保留 30 天且最多 10,000 行，追加时按 `recorded_at` 索引修剪；缺失 UUIDv7 内嵌时间早于 `pruned_before` 返回 `expired`。回执中的 HMAC 与 selected IDs 仍可能暴露工作模式，默认 local-only、不进团队 ref；任何共享另做 visibility / retention / threat-model 审查。 - **单一 receipt 原语。** Rust 类型与 mainline ML-05 / ML-08 共用同一定义（§0.0.10）；本文与 mainline 各自的注入管线写同一张回执面，不得分叉出两种 schema。 -- `memory inspect-injection` 从回执读取并重放展示，而非从当前投影反推。被预算丢弃的项直接记录在 receipt 的 `dropped` 中，不再为每次读取追加 `PromptTrimmed` 权威事件。 +- `memory inspect-injection` 从回执读取并重放展示，而非从当前投影反推。被预算丢弃的项直接记录在 receipt 的 `omissions` 中，不再为每次读取追加 `PromptTrimmed` 权威事件。 ### 8.7 混合检索通道（Hybrid Retrieval Channels） @@ -1704,7 +1772,7 @@ session JSONL 模型，只有 Draft / Confirmed / Revoked / Superseded。Memory 5. 高置信度的 procedural 规则最后才保留，除非其本身就长于整个 预算；此时它们会被替换为其路径摘要与一条 direct-get 提示。 -丢弃行为记录在 `ContextReceipt.dropped` 与本地有界 audit 中，以便审计；它是读取决策，不追加权威 `MemoryEvent`，避免每个 turn 都推进 memory ref。 +丢弃行为记录在 `ContextReceipt.omissions` 与本地有界 audit 中，以便审计；它是读取决策，不追加权威 `MemoryEvent`，避免每个 turn 都推进 memory ref。 ## 12. CLI 命令面 @@ -1808,7 +1876,7 @@ MCP stdio 独占 stdin/stdout：在 stdio 模式下，**不得**输出 banner、 - **无编译记录不落盘。** 任何入口产生的 `MemoryNote` 缺少完整 `CompileRecord`（§4.1.1）必须在写入事务第 1 步被拒绝；同幂等键的重复摄入不得产生第二条 note。这是 Phase A 的硬性退出门槛（§0.2）。 - **注入必须留回执。** 任何 prompt 注入或引擎内召回若未能写出完整 `ContextReceipt`（§8.6），该次操作按失败处理（fail-closed）；`inspect-injection` 只从回执重放，不从当前投影反推。回执默认 local-only，未经 visibility / retention 审查不得成批越过 MCP 边界。这是 Phase C 的硬性退出门槛（§0.2）。 - **写入只有一个 seam。** 所有 mutating Adapter 必须通过 `MemoryWriter`；任何绕过 CompileRecord、authorization、事件状态机、CAS 或投影事务而直接写 ref / tree / SQLite 的路径都属于发布阻断。worktree checkout 状态不能充当 writer lock。 -- **session view 必须冻结且可解释。** recall / injection 使用的 `ResolvedMemoryView.view_hash`、source ref OID、watermark、policy 与 code anchor 必须进入 status / receipt；branch、worktree 或 principal 改变后不得继续使用旧 view。 +- **session view 必须冻结且可解释。** `ResolvedMemoryView.view_hash` 进入 status；构成该 view 的 source heads、projection watermarks、policy hash 与 code anchor 进入 receipt，并可供 status/debug 重算对照。branch、worktree 或 principal 改变后不得继续使用旧 view。 - **OID 不自引用。** `MemoryNote` 正文不含 `revision_oid`；写入后 OID 只出现在 event / envelope / tree path / projection。读取必须同时验证 Git OID 与 canonical `content_digest`。 - **权威历史线性。** 每个 memory ref 只接受 first-parent 单父 commit 与严格递增 `event_seq`；跨 ref merge/cherry-pick 必须经语义 writer 重新追加，不能直接导入任意 Git merge DAG。 - **投影不可越权。** prompt 注入、MCP 只读返回与 CLI recall 都必须先解析 actor、repo、branch、worktree 和 namespace policy；`private:` 只能被同一 actor 或授权 reviewer 读取。 @@ -1844,7 +1912,7 @@ agent 捕获使用的 `2026050303_agent_capture.sql` 迁移属于同一模式， - `memory_classifier_cache`（§5.2，可选，带 TTL） - `memory_embedding_cache`（§8.7，可选向量通道缓存，可丢弃） - `memory_access_stats`（§5.2，本地账本，不参与 rebuild） -- `memory_context_receipt`（§8.6，账本类：append-only、豁免 rebuild、按保留策略有界修剪） +- `context_selection_receipt`（§8.6，共享账本：append-only、豁免 rebuild、按保留策略有界修剪） 只有 `memory_head`、`memory_path_summary`、`memory_note_index`、`memory_revision_index`、`memory_link_index`、`memory_entity_index`、`memory_taxonomy_node`、`memory_projection_state` 可由 `libra memory rebuild` 从 `refs/libra/memory/...` 重建。classifier cache 与 embedding cache 可直接丢弃；access stats 与 receipt 是本地账本，不参与 rebuild。 @@ -1854,7 +1922,7 @@ agent 捕获使用的 `2026050303_agent_capture.sql` 迁移属于同一模式， 重建的 `ai_index_*` 投影同模式），而外部捕获的 `agent_session` / `agent_checkpoint` / `agent_usage_stats` 是**故意**采用 raw-SQL、不配 entity 的。 Memory 之所以对齐 `ai_index_*` 的 SeaORM 模式，是因为它的表本身就是 git 真源的 -可重建投影。`memory_access_stats` 与 `memory_context_receipt` 是例外：二者与 +可重建投影。`memory_access_stats` 与 `context_selection_receipt` 是例外：二者与 `agent_*` 同为本地账本，沿 raw-SQL 账本模式访问（§5.2、§8.6）。 ## 15. 分阶段路线图 @@ -1874,7 +1942,8 @@ Memory 之所以对齐 `ai_index_*` 的 SeaORM 模式，是因为它的表本身 `materialize` 生成 §5.5 的只读、脱敏 Markdown 投影，`consolidate` 触发一次排期归并（§10.5）。 - 不引入分类器——调用方必须自行提供 `namespace` 与 `path`。 - `CompileRecord` 类型与写入侧强制校验（§4.1.1）：所有入口必须携带 - 编译记录，幂等键在 `(scope, namespace)` 内去重。 + 编译记录；默认在 `(scope, namespace, path)` Cell 内去重，只有 + Consolidation / Onboard 可显式使用 Namespace 级跨路径去重。 退出门槛：所有 Phase A 写入 Adapter 都无法绕过 `MemoryWriter`；路径聚合与 `memory status` 可用；物化投影对同一 `view_hash` 产生相同 manifest / 文件集合，且不可见正文不落盘、raw HTML 默认禁用、frontmatter 与 URL 经转义 / 净化；投影重建在语义行、watermark、排序与 canonical digest 上稳定；OID 无自引用；损坏点阻止对应 scope watermark 前进；**每条已写入 note 都携带完整编译记录， 缺失即拒绝写入，同幂等键重复摄入不产生新 note——硬性门槛，见 §0.2**；**写路径成本探针**：单条 note 写入（blob + event + commit + ref CAS + 投影表事务）的端到端延迟与新增对象数量有确定预算，并纳入 §16 的确定性探针，防止投影写放大失控。 @@ -1893,9 +1962,12 @@ Memory 之所以对齐 `ai_index_*` 的 SeaORM 模式，是因为它的表本身 idempotency 当作语义去重。 - 带「编辑后投影」（redacted projection）语义的 `forget` API。 -退出门槛：任何未经评审、形似 secret、外部不可信、或处于隔离（quarantine） -状态的 note，都不能进入 prompt 注入；entity alias 歧义不能触发静默合并； -Working 缓冲有 TTL / auto-evict；任何 Draft→Confirmed 都经 Trust Gate（§7.5.1）。 +退出门槛：普通知识流中任何未经评审、形似 secret、外部不可信、或处于隔离 +（quarantine）状态的 note，都不能进入 prompt 注入；entity alias 歧义不能触发 +静默合并；Working 缓冲有 TTL / auto-evict；普通知识的任何 Draft→Confirmed +都经 Trust Gate（§7.5.1）。M2 Episode 不经过 Draft 晋升：它按 §4.1.3 的授权 +来源前提与确定性研发历程准入自动 Confirmed，trust/confidence 仅作为来源与认识论 +标签保存。 ### Phase C — 分类、召回与注入（2–3 周） @@ -1905,7 +1977,7 @@ Working 缓冲有 TTL / auto-evict；任何 Draft→Confirmed 都经 Trust Gate 两跳查询；授权先于图扩展，每个 target 重新鉴权。 - prompt 期注入（§11.7），使用 `ProjectMemory` 与 `MemoryAnchor` 预算分段。 -- `ContextReceipt` 产出与 `memory_context_receipt` 账本（§8.6）： +- `ContextSelectionReceiptV1` 产出与 `context_selection_receipt` 账本（§8.6）： 注入与引擎内召回写不出完整回执即按失败处理。 - 用于可观测性的 `libra memory inspect-injection`（从回执重放）与 **`libra code --debug-memory`（G14，Phase C 硬交付）**：每 turn 逐字打印 @@ -2052,7 +2124,7 @@ Memory 只有在配齐有针对性的回归覆盖后才发布： - 注入回执：每次注入与引擎内召回都写出 `ContextReceipt`；固定 `as_of` 快照重放选择得到相同 selected 集合与 bundle hash；缺失快照返回 stale / non-reproducible；删除全部投影表并 rebuild 后回执账本不受 - 影响、也不被重建；预算丢弃只进入 `ContextReceipt.dropped` 与按 + 影响、也不被重建；预算丢弃只进入 `ContextReceipt.omissions` 与按 `receipt_id` 互链的本地 audit，不追加 `PromptTrimmed` 权威事件。 - onboarding：cold、warm 与 meta-only 刷新产出确定性的路径，且不会改写 无关的 namespace。 diff --git a/src/internal/ai/memory/canonical.rs b/src/internal/ai/memory/canonical.rs new file mode 100644 index 000000000..85d45d2ae --- /dev/null +++ b/src/internal/ai/memory/canonical.rs @@ -0,0 +1,696 @@ +use chrono::{DateTime, Utc}; +use serde::Serialize; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use super::{ + super::context_budget::MemoryAnchorConfidence, + domain::{ + ActorKind, ActorRefV1, CodeChangeStatus, CompileOriginV1, CompileRecordV1, + CompletionStatus, EpisodeClaimV1, EpisodeCodeContextV1, EpisodeOmissionsV1, + EpisodePayloadV1, EpisodeRootKind, EpistemicStatus, EvidenceKind, EvidenceLocatorV1, + EvidenceRefV1, EvidenceSourcePlane, EvidenceVisibility, IdempotencyScopeV1, + MemoryContractError, MemoryEntityMentionV1, MemoryEntityRole, MemoryKind, MemoryLifecycle, + MemoryLinkKind, MemoryLinkV1, MemoryNoteV1, MemoryScopeV1, MemorySensitivity, MemoryTrust, + MemoryVisibility, ToolCallPart, + }, +}; + +#[derive(Serialize)] +struct CanonicalMemoryNoteV1<'a> { + schema_version: u32, + note_id: &'a Uuid, + namespace: &'a str, + path: &'a str, + kind: MemoryKind, + scope: &'a MemoryScopeV1, + visibility: MemoryVisibility, + acl_policy_id: &'a str, + lifecycle: MemoryLifecycle, + body: &'a str, + rationale: &'a Option, + episode: Option>, + evidence_refs: Vec>, + links: Vec>, + entities: Vec>, + tags: &'a [String], + confidence: MemoryAnchorConfidence, + trust: MemoryTrust, + sensitivity: MemorySensitivity, + valid_from: &'a Option>, + valid_until: &'a Option>, + effective_from_commit: &'a Option, + effective_until_commit: &'a Option, + expires_at: &'a Option>, + author: CanonicalActorRefV1<'a>, + created_at: &'a DateTime, + compile_record: CanonicalCompileRecordV1<'a>, +} + +impl<'a> From<&'a MemoryNoteV1> for CanonicalMemoryNoteV1<'a> { + fn from(note: &'a MemoryNoteV1) -> Self { + Self { + schema_version: note.schema_version, + note_id: ¬e.note_id, + namespace: ¬e.namespace, + path: ¬e.path, + kind: note.kind, + scope: ¬e.scope, + visibility: note.visibility, + acl_policy_id: ¬e.acl_policy_id, + lifecycle: note.lifecycle, + body: ¬e.body, + rationale: ¬e.rationale, + episode: note.episode.as_ref().map(Into::into), + evidence_refs: note.evidence_refs.iter().map(Into::into).collect(), + links: note.links.iter().map(Into::into).collect(), + entities: note.entities.iter().map(Into::into).collect(), + tags: ¬e.tags, + confidence: note.confidence, + trust: note.trust, + sensitivity: note.sensitivity, + valid_from: ¬e.valid_from, + valid_until: ¬e.valid_until, + effective_from_commit: ¬e.effective_from_commit, + effective_until_commit: ¬e.effective_until_commit, + expires_at: ¬e.expires_at, + author: (¬e.author).into(), + created_at: ¬e.created_at, + compile_record: (¬e.compile_record).into(), + } + } +} + +#[derive(Serialize)] +struct CanonicalEpisodePayloadV1<'a> { + schema_version: u32, + root_kind: EpisodeRootKind, + root_id: &'a str, + related_intent_ids: &'a [String], + related_task_ids: &'a [String], + related_run_ids: &'a [String], + started_at: &'a Option>, + ended_at: &'a Option>, + goal: CanonicalEpisodeClaimV1<'a>, + completion_status: CompletionStatus, + code_change_status: CodeChangeStatus, + summary: CanonicalEpisodeClaimV1<'a>, + observations: Vec>, + inferences: Vec>, + decisions: Vec>, + failed_attempts: Vec>, + unresolved: Vec>, + code: CanonicalEpisodeCodeContextV1<'a>, + omissions: CanonicalEpisodeOmissionsV1, +} + +impl<'a> From<&'a EpisodePayloadV1> for CanonicalEpisodePayloadV1<'a> { + fn from(payload: &'a EpisodePayloadV1) -> Self { + Self { + schema_version: payload.schema_version, + root_kind: payload.root_kind, + root_id: &payload.root_id, + related_intent_ids: &payload.related_intent_ids, + related_task_ids: &payload.related_task_ids, + related_run_ids: &payload.related_run_ids, + started_at: &payload.started_at, + ended_at: &payload.ended_at, + goal: (&payload.goal).into(), + completion_status: payload.completion_status, + code_change_status: payload.code_change_status, + summary: (&payload.summary).into(), + observations: payload.observations.iter().map(Into::into).collect(), + inferences: payload.inferences.iter().map(Into::into).collect(), + decisions: payload.decisions.iter().map(Into::into).collect(), + failed_attempts: payload.failed_attempts.iter().map(Into::into).collect(), + unresolved: payload.unresolved.iter().map(Into::into).collect(), + code: (&payload.code).into(), + omissions: (&payload.omissions).into(), + } + } +} + +#[derive(Serialize)] +struct CanonicalEpisodeClaimV1<'a> { + epistemic_status: EpistemicStatus, + claim: &'a str, + confidence: &'a Option, + evidence_refs: Vec>, +} + +impl<'a> From<&'a EpisodeClaimV1> for CanonicalEpisodeClaimV1<'a> { + fn from(claim: &'a EpisodeClaimV1) -> Self { + Self { + epistemic_status: claim.epistemic_status, + claim: &claim.claim, + confidence: &claim.confidence, + evidence_refs: claim.evidence_refs.iter().map(Into::into).collect(), + } + } +} + +#[derive(Serialize)] +struct CanonicalEpisodeCodeContextV1<'a> { + base_oid: &'a Option, + result_oid: &'a Option, + branch_ref: &'a Option, + paths: &'a [String], +} + +impl<'a> From<&'a EpisodeCodeContextV1> for CanonicalEpisodeCodeContextV1<'a> { + fn from(code: &'a EpisodeCodeContextV1) -> Self { + Self { + base_oid: &code.base_oid, + result_oid: &code.result_oid, + branch_ref: &code.branch_ref, + paths: &code.paths, + } + } +} + +#[derive(Serialize)] +struct CanonicalEpisodeOmissionsV1 { + related_run_ids: u32, + observations: u32, + inferences: u32, + decisions: u32, + failed_attempts: u32, + unresolved: u32, +} + +impl From<&EpisodeOmissionsV1> for CanonicalEpisodeOmissionsV1 { + fn from(omissions: &EpisodeOmissionsV1) -> Self { + Self { + related_run_ids: omissions.related_run_ids, + observations: omissions.observations, + inferences: omissions.inferences, + decisions: omissions.decisions, + failed_attempts: omissions.failed_attempts, + unresolved: omissions.unresolved, + } + } +} + +#[derive(Serialize)] +struct CanonicalEvidenceRefV1<'a> { + schema_version: u32, + source_plane: EvidenceSourcePlane, + kind: EvidenceKind, + object_id: &'a str, + locator: CanonicalEvidenceLocatorV1<'a>, + visibility: EvidenceVisibility, + captured_at: &'a Option>, + code_commit: &'a Option, +} + +impl<'a> From<&'a EvidenceRefV1> for CanonicalEvidenceRefV1<'a> { + fn from(evidence: &'a EvidenceRefV1) -> Self { + Self { + schema_version: evidence.schema_version, + source_plane: evidence.source_plane, + kind: evidence.kind, + object_id: &evidence.object_id, + locator: (&evidence.locator).into(), + visibility: evidence.visibility, + captured_at: &evidence.captured_at, + code_commit: &evidence.code_commit, + } + } +} + +#[derive(Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum CanonicalEvidenceLocatorV1<'a> { + Object, + EventSeq { + event_seq: u64, + }, + JsonPointer { + pointer: &'a str, + }, + SessionFragment { + start_seq: u64, + end_seq: u64, + }, + ToolCall { + invocation_id: &'a str, + part: ToolCallPart, + }, + CodeRange { + commit_oid: &'a str, + path: &'a str, + start_line: u32, + end_line: u32, + }, +} + +impl<'a> From<&'a EvidenceLocatorV1> for CanonicalEvidenceLocatorV1<'a> { + fn from(locator: &'a EvidenceLocatorV1) -> Self { + match locator { + EvidenceLocatorV1::Object => Self::Object, + EvidenceLocatorV1::EventSeq { event_seq } => Self::EventSeq { + event_seq: *event_seq, + }, + EvidenceLocatorV1::JsonPointer { pointer } => Self::JsonPointer { pointer }, + EvidenceLocatorV1::SessionFragment { start_seq, end_seq } => Self::SessionFragment { + start_seq: *start_seq, + end_seq: *end_seq, + }, + EvidenceLocatorV1::ToolCall { + invocation_id, + part, + } => Self::ToolCall { + invocation_id, + part: *part, + }, + EvidenceLocatorV1::CodeRange { + commit_oid, + path, + start_line, + end_line, + } => Self::CodeRange { + commit_oid, + path, + start_line: *start_line, + end_line: *end_line, + }, + } + } +} + +#[derive(Serialize)] +struct CanonicalMemoryLinkV1<'a> { + kind: MemoryLinkKind, + target_note_id: &'a Uuid, + evidence_refs: Vec>, + valid_from: &'a Option>, + valid_until: &'a Option>, +} + +impl<'a> From<&'a MemoryLinkV1> for CanonicalMemoryLinkV1<'a> { + fn from(link: &'a MemoryLinkV1) -> Self { + Self { + kind: link.kind, + target_note_id: &link.target_note_id, + evidence_refs: link.evidence_refs.iter().map(Into::into).collect(), + valid_from: &link.valid_from, + valid_until: &link.valid_until, + } + } +} + +#[derive(Serialize)] +struct CanonicalMemoryEntityMentionV1<'a> { + schema_version: u32, + canonical_key: &'a str, + display_name: &'a str, + aliases: &'a [String], + role: MemoryEntityRole, + resolution_confidence: MemoryAnchorConfidence, + evidence_refs: Vec>, +} + +impl<'a> From<&'a MemoryEntityMentionV1> for CanonicalMemoryEntityMentionV1<'a> { + fn from(entity: &'a MemoryEntityMentionV1) -> Self { + Self { + schema_version: entity.schema_version, + canonical_key: &entity.canonical_key, + display_name: &entity.display_name, + aliases: &entity.aliases, + role: entity.role, + resolution_confidence: entity.resolution_confidence, + evidence_refs: entity.evidence_refs.iter().map(Into::into).collect(), + } + } +} + +#[derive(Serialize)] +struct CanonicalActorRefV1<'a> { + kind: ActorKind, + principal_id: &'a str, +} + +impl<'a> From<&'a ActorRefV1> for CanonicalActorRefV1<'a> { + fn from(actor: &'a ActorRefV1) -> Self { + Self { + kind: actor.kind, + principal_id: &actor.principal_id, + } + } +} + +#[derive(Serialize)] +struct CanonicalCompileRecordV1<'a> { + schema_version: u32, + origin: CompileOriginV1, + producer: &'a str, + rules_version: u32, + prompt_version: &'a Option, + model_id: &'a Option, + policy_version: &'a str, + input_hashes: &'a [String], + idempotency_key: &'a str, + idempotency_scope: IdempotencyScopeV1, +} + +impl<'a> From<&'a CompileRecordV1> for CanonicalCompileRecordV1<'a> { + fn from(record: &'a CompileRecordV1) -> Self { + Self { + schema_version: record.schema_version, + origin: record.origin, + producer: &record.producer, + rules_version: record.rules_version, + prompt_version: &record.prompt_version, + model_id: &record.model_id, + policy_version: &record.policy_version, + input_hashes: &record.input_hashes, + idempotency_key: &record.idempotency_key, + idempotency_scope: record.idempotency_scope, + } + } +} + +pub(super) fn memory_note_content_digest_v1( + note: &MemoryNoteV1, +) -> Result { + let payload = memory_note_canonical_payload_v1(note)?; + Ok(format!("sha256:{}", hex::encode(Sha256::digest(payload)))) +} + +pub(super) fn verify_memory_note_content_digest_v1( + note: &MemoryNoteV1, +) -> Result<(), MemoryContractError> { + if note.content_digest != memory_note_content_digest_v1(note)? { + return Err(MemoryContractError::InvalidField { + field: "MemoryNote.content_digest", + }); + } + Ok(()) +} + +pub(super) fn memory_note_canonical_payload_v1( + note: &MemoryNoteV1, +) -> Result, MemoryContractError> { + let value = serde_json::to_value(CanonicalMemoryNoteV1::from(note)).map_err(|_| { + MemoryContractError::InvalidJson { + object: "MemoryNote canonical payload", + } + })?; + + let mut bytes = Vec::new(); + write_canonical_json(&value, &mut bytes)?; + Ok(bytes) +} + +fn write_canonical_json(value: &Value, output: &mut Vec) -> Result<(), MemoryContractError> { + match value { + Value::Null => output.extend_from_slice(b"null"), + Value::Bool(value) => output.extend_from_slice(if *value { b"true" } else { b"false" }), + Value::Number(number) => { + if !number.is_i64() && !number.is_u64() { + return Err(MemoryContractError::InvalidField { + field: "MemoryNote canonical number", + }); + } + output.extend_from_slice(number.to_string().as_bytes()); + } + Value::String(value) => { + let encoded = + serde_json::to_vec(value).map_err(|_| MemoryContractError::InvalidJson { + object: "MemoryNote canonical string", + })?; + output.extend_from_slice(&encoded); + } + Value::Array(values) => { + output.push(b'['); + for (index, value) in values.iter().enumerate() { + if index > 0 { + output.push(b','); + } + write_canonical_json(value, output)?; + } + output.push(b']'); + } + Value::Object(object) => { + output.push(b'{'); + let mut entries = object.iter().collect::>(); + entries.sort_by_key(|(key, _)| *key); + for (index, (key, value)) in entries.into_iter().enumerate() { + if index > 0 { + output.push(b','); + } + let encoded_key = + serde_json::to_vec(key).map_err(|_| MemoryContractError::InvalidJson { + object: "MemoryNote canonical key", + })?; + output.extend_from_slice(&encoded_key); + output.push(b':'); + write_canonical_json(value, output)?; + } + output.push(b'}'); + } + } + + Ok(()) +} + +#[cfg(test)] +mod tests { + use chrono::{TimeZone, Utc}; + + use super::{ + super::{ + super::context_budget::MemoryAnchorConfidence, + domain::{ + CodeChangeStatus, CompileOriginV1, CompletionStatus, EpisodeClaimV1, + EpisodeCodeContextV1, EpisodeOmissionsV1, EpisodePayloadV1, EpisodeRootKind, + EpistemicStatus, + }, + validation::parse_memory_note_v1, + }, + memory_note_content_digest_v1, verify_memory_note_content_digest_v1, + }; + + const SHA1_OID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const SHA256_OID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + + fn note_fixture() -> super::super::domain::MemoryNoteV1 { + let value = serde_json::json!({ + "schema_version": 1, + "note_id": "98809d1c-f0cd-5e98-84b8-c1dddf5aeb19", + "content_digest": "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc", + "namespace": "default", + "path": "episodic.tasks.r-7461736b2d3432", + "kind": "episodic", + "scope": { "type": "repo" }, + "visibility": "repo_local", + "acl_policy_id": "repo-policy-v1", + "lifecycle": "accretive", + "body": "The retry clock caused two failed attempts.", + "rationale": "Keep the failure chain available to later agents.", + "episode": null, + "evidence_refs": [{ + "schema_version": 1, + "source_plane": "agent_runtime", + "kind": "task", + "object_id": "task-42", + "source_ref_oid": SHA1_OID, + "locator": { "type": "object" }, + "fragment_digest": "sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "visibility": "repo_local", + "captured_at": null, + "code_commit": null + }], + "links": [{ + "kind": "supports", + "target_note_id": "760369f7-ba78-541a-9aae-4e899154530b", + "target_revision_oid": SHA1_OID, + "evidence_refs": [], + "valid_from": null, + "valid_until": null + }], + "entities": [], + "parents": [SHA1_OID], + "tags": ["retry"], + "confidence": "high", + "trust": "repo_evidence", + "sensitivity": "internal", + "valid_from": null, + "valid_until": null, + "effective_from_commit": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", + "effective_until_commit": null, + "expires_at": null, + "author": { "kind": "agent", "principal_id": "agent:test" }, + "created_at": "2026-08-20T09:00:00Z", + "compile_record": { + "schema_version": 1, + "origin": "explicit", + "producer": "libra-memory/1", + "rules_version": 1, + "prompt_version": null, + "model_id": null, + "policy_version": "repo-policy-v1", + "input_hashes": ["sha256:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"], + "idempotency_key": "hmac-sha256:key-1:ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff", + "idempotency_scope": "cell" + } + }); + + let mut note: super::super::domain::MemoryNoteV1 = + serde_json::from_value(value).expect("fixture has a valid MemoryNote shape"); + note.content_digest = memory_note_content_digest_v1(¬e).expect("fixture canonicalizes"); + + parse_memory_note_v1(&serde_json::to_vec(¬e).expect("fixture serializes")) + .expect("fixture is a valid MemoryNote") + } + + fn episode_note_fixture() -> super::super::domain::MemoryNoteV1 { + let mut note = note_fixture(); + let evidence_ref = note.evidence_refs[0].clone(); + let observation = EpisodeClaimV1 { + epistemic_status: EpistemicStatus::Observation, + claim: "the retry test failed twice".to_string(), + confidence: None, + evidence_refs: vec![evidence_ref.clone()], + }; + let inference = EpisodeClaimV1 { + epistemic_status: EpistemicStatus::Inference, + claim: "the retry clock is probably nondeterministic".to_string(), + confidence: Some(MemoryAnchorConfidence::High), + evidence_refs: vec![evidence_ref], + }; + note.episode = Some(EpisodePayloadV1 { + schema_version: 1, + root_kind: EpisodeRootKind::Task, + root_id: "task-42".to_string(), + related_intent_ids: vec!["intent-9".to_string()], + related_task_ids: vec!["task-42".to_string()], + related_run_ids: vec!["run-1".to_string(), "run-2".to_string()], + started_at: Utc.with_ymd_and_hms(2026, 8, 20, 8, 0, 0).single(), + ended_at: Utc.with_ymd_and_hms(2026, 8, 20, 9, 0, 0).single(), + goal: observation.clone(), + completion_status: CompletionStatus::Completed, + code_change_status: CodeChangeStatus::Changed, + summary: inference.clone(), + observations: vec![observation], + inferences: vec![inference], + decisions: Vec::new(), + failed_attempts: Vec::new(), + unresolved: Vec::new(), + code: EpisodeCodeContextV1 { + base_oid: Some(SHA1_OID.to_string()), + result_oid: Some("bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb".to_string()), + branch_ref: Some("refs/heads/feature/retry".to_string()), + paths: vec!["src/retry.rs".to_string()], + }, + omissions: EpisodeOmissionsV1::default(), + }); + note.compile_record.origin = CompileOriginV1::EpisodeCompiler; + note.compile_record.prompt_version = Some("episode-v1".to_string()); + note.compile_record.model_id = Some("synthetic-model".to_string()); + note + } + + #[test] + fn content_digest_has_a_storage_oid_independent_golden_vector() { + let note = note_fixture(); + let digest = memory_note_content_digest_v1(¬e).expect("note canonicalizes"); + assert_eq!( + digest, + "sha256:74c9998dbdda299afe9b90d80ad3c5493534b4d9f3a32fd3e33ccf6d733e004b", + ); + + let mut different_storage = note.clone(); + different_storage.parents[0] = SHA256_OID.to_string(); + different_storage.evidence_refs[0].source_ref_oid = SHA256_OID.to_string(); + different_storage.evidence_refs[0].fragment_digest = format!("sha256:{}", "1".repeat(64)); + different_storage.links[0].target_revision_oid = Some(SHA256_OID.to_string()); + assert_eq!( + digest, + memory_note_content_digest_v1(&different_storage).expect("note canonicalizes"), + ); + + let mut different_body = note.clone(); + different_body.body.push_str(" A third attempt succeeded."); + assert_ne!( + digest, + memory_note_content_digest_v1(&different_body).expect("note canonicalizes"), + ); + + let mut different_code_anchor = note; + different_code_anchor.effective_from_commit = Some("2".repeat(40)); + assert_ne!( + digest, + memory_note_content_digest_v1(&different_code_anchor).expect("note canonicalizes"), + ); + } + + #[test] + fn semantic_code_anchor_has_sha1_and_sha256_golden_vectors() { + let sha1_note = note_fixture(); + assert_eq!( + memory_note_content_digest_v1(&sha1_note).expect("SHA-1 note canonicalizes"), + "sha256:74c9998dbdda299afe9b90d80ad3c5493534b4d9f3a32fd3e33ccf6d733e004b", + ); + + let mut sha256_note = sha1_note; + sha256_note.effective_from_commit = Some(SHA256_OID.to_string()); + assert_eq!( + memory_note_content_digest_v1(&sha256_note).expect("SHA-256 note canonicalizes"), + "sha256:a4be4edf791a7773d7316ece36caac545f868224d8d13d7fb49063d00b1f653b", + ); + } + + #[test] + fn content_digest_verification_detects_semantic_tampering() { + let mut note = note_fixture(); + note.content_digest = memory_note_content_digest_v1(¬e).expect("note canonicalizes"); + verify_memory_note_content_digest_v1(¬e).expect("matching digest is valid"); + + note.body.push_str(" Tampered after hashing."); + assert!(verify_memory_note_content_digest_v1(¬e).is_err()); + } + + #[test] + fn canonical_episode_payload_golden() { + let note = episode_note_fixture(); + let digest = memory_note_content_digest_v1(¬e).expect("Episode note canonicalizes"); + assert_eq!( + digest, + "sha256:a6112fef9230e538eff93eca6c9449009728145054571c9a2c0bb8b14f52b4e9", + ); + + let mut different_evidence_storage = note.clone(); + different_evidence_storage.parents[0] = SHA256_OID.to_string(); + different_evidence_storage.evidence_refs[0].source_ref_oid = SHA256_OID.to_string(); + different_evidence_storage.evidence_refs[0].fragment_digest = + format!("sha256:{}", "2".repeat(64)); + different_evidence_storage.links[0].target_revision_oid = Some(SHA256_OID.to_string()); + let claim = &mut different_evidence_storage + .episode + .as_mut() + .expect("fixture carries Episode") + .summary; + claim.evidence_refs[0].source_ref_oid = SHA256_OID.to_string(); + claim.evidence_refs[0].fragment_digest = format!("sha256:{}", "3".repeat(64)); + assert_eq!( + digest, + memory_note_content_digest_v1(&different_evidence_storage) + .expect("Episode note canonicalizes"), + ); + + let mut different_summary = note; + different_summary + .episode + .as_mut() + .expect("fixture carries Episode") + .summary + .claim + .push_str(" A controllable clock should fix it."); + assert_ne!( + digest, + memory_note_content_digest_v1(&different_summary).expect("Episode note canonicalizes"), + ); + } +} diff --git a/src/internal/ai/memory/domain.rs b/src/internal/ai/memory/domain.rs new file mode 100644 index 000000000..d2d34e566 --- /dev/null +++ b/src/internal/ai/memory/domain.rs @@ -0,0 +1,552 @@ +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use uuid::Uuid; + +use super::super::context_budget::MemoryAnchorConfidence; + +const EPISODE_NAMESPACE: &str = "default"; +const MAX_EPISODE_ROOT_ID_BYTES: usize = 120; + +// This UUID is part of the persisted identity contract. Changing it would give +// an existing Task Episode a different note ID. +const EPISODE_NOTE_NAMESPACE_V1: Uuid = Uuid::from_bytes([ + 0xf2, 0xb4, 0xd3, 0xa0, 0x1c, 0x9e, 0x4f, 0x75, 0x8d, 0x20, 0x2a, 0x6b, 0x7c, 0x8d, 0x9e, 0x01, +]); + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum EpisodeRootKind { + Task, + Intent, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum EvidenceSourcePlane { + AgentRuntime, + Session, + Git, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum EvidenceKind { + Intent, + Task, + Run, + Evidence, + Decision, + PatchSet, + Session, + ToolCall, + Code, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum EvidenceVisibility { + Private, + RepoLocal, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum ToolCallPart { + Invocation, + Output, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum EpistemicStatus { + Observation, + Inference, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct EpisodeClaimV1 { + pub(crate) epistemic_status: EpistemicStatus, + pub(crate) claim: String, + pub(crate) confidence: Option, + pub(crate) evidence_refs: Vec, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum CompletionStatus { + Completed, + Failed, + Cancelled, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum CodeChangeStatus { + Changed, + Unchanged, + Unknown, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct EpisodeCodeContextV1 { + pub(crate) base_oid: Option, + pub(crate) result_oid: Option, + pub(crate) branch_ref: Option, + pub(crate) paths: Vec, +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct EpisodeOmissionsV1 { + pub(crate) related_run_ids: u32, + pub(crate) observations: u32, + pub(crate) inferences: u32, + pub(crate) decisions: u32, + pub(crate) failed_attempts: u32, + pub(crate) unresolved: u32, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct EpisodePayloadV1 { + pub(crate) schema_version: u32, + pub(crate) root_kind: EpisodeRootKind, + pub(crate) root_id: String, + pub(crate) related_intent_ids: Vec, + pub(crate) related_task_ids: Vec, + pub(crate) related_run_ids: Vec, + pub(crate) started_at: Option>, + pub(crate) ended_at: Option>, + pub(crate) goal: EpisodeClaimV1, + pub(crate) completion_status: CompletionStatus, + pub(crate) code_change_status: CodeChangeStatus, + pub(crate) summary: EpisodeClaimV1, + pub(crate) observations: Vec, + pub(crate) inferences: Vec, + pub(crate) decisions: Vec, + pub(crate) failed_attempts: Vec, + pub(crate) unresolved: Vec, + pub(crate) code: EpisodeCodeContextV1, + pub(crate) omissions: EpisodeOmissionsV1, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TrustedEpisodeFieldsV1 { + pub(crate) root: EpisodeRoot, + pub(crate) related_intent_ids: Vec, + pub(crate) related_task_ids: Vec, + pub(crate) related_run_ids: Vec, + pub(crate) started_at: Option>, + pub(crate) ended_at: Option>, + pub(crate) goal: EpisodeClaimV1, + pub(crate) completion_status: CompletionStatus, + pub(crate) code_change_status: CodeChangeStatus, + pub(crate) code: EpisodeCodeContextV1, + pub(crate) omissions: EpisodeOmissionsV1, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum MemoryKind { + Procedural, + Semantic, + Episodic, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(tag = "type", content = "value", rename_all = "snake_case")] +pub(crate) enum MemoryScopeV1 { + Repo, + Branch(String), + Worktree(String), + Actor(String), + Global, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum MemoryVisibility { + Private, + RepoLocal, + TeamCandidate, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum MemoryLifecycle { + Replacement, + Accretive, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum MemoryTrust { + Verified, + RepoEvidence, + UserAsserted, + ExternalUntrusted, + Inferred, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum MemorySensitivity { + Public, + Internal, + Confidential, + SecretLike, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum ActorKind { + Human, + Agent, + System, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct ActorRefV1 { + pub(crate) kind: ActorKind, + pub(crate) principal_id: String, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum CompileOriginV1 { + Explicit, + PromotedFromAnchor, + DistilledFromFrame, + Classifier, + Consolidation, + Onboard, + BranchFork, + Import, + Coordinator, + EpisodeCompiler, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum IdempotencyScopeV1 { + Cell, + Namespace, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct CompileRecordV1 { + pub(crate) schema_version: u32, + pub(crate) origin: CompileOriginV1, + pub(crate) producer: String, + pub(crate) rules_version: u32, + pub(crate) prompt_version: Option, + pub(crate) model_id: Option, + pub(crate) policy_version: String, + pub(crate) input_hashes: Vec, + pub(crate) idempotency_key: String, + pub(crate) idempotency_scope: IdempotencyScopeV1, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum MemoryLinkKind { + Sibling, + Supports, + Prerequisite, + Contradicts, + Supersedes, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct MemoryLinkV1 { + pub(crate) kind: MemoryLinkKind, + pub(crate) target_note_id: Uuid, + pub(crate) target_revision_oid: Option, + pub(crate) evidence_refs: Vec, + pub(crate) valid_from: Option>, + pub(crate) valid_until: Option>, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum MemoryEntityRole { + Subject, + Object, + Topic, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct MemoryEntityMentionV1 { + pub(crate) schema_version: u32, + pub(crate) canonical_key: String, + pub(crate) display_name: String, + pub(crate) aliases: Vec, + pub(crate) role: MemoryEntityRole, + pub(crate) resolution_confidence: MemoryAnchorConfidence, + pub(crate) evidence_refs: Vec, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct MemoryNoteV1 { + pub(crate) schema_version: u32, + pub(crate) note_id: Uuid, + pub(crate) content_digest: String, + pub(crate) namespace: String, + pub(crate) path: String, + pub(crate) kind: MemoryKind, + pub(crate) scope: MemoryScopeV1, + pub(crate) visibility: MemoryVisibility, + pub(crate) acl_policy_id: String, + pub(crate) lifecycle: MemoryLifecycle, + pub(crate) body: String, + pub(crate) rationale: Option, + pub(crate) episode: Option, + pub(crate) evidence_refs: Vec, + pub(crate) links: Vec, + pub(crate) entities: Vec, + pub(crate) parents: Vec, + pub(crate) tags: Vec, + pub(crate) confidence: MemoryAnchorConfidence, + pub(crate) trust: MemoryTrust, + pub(crate) sensitivity: MemorySensitivity, + pub(crate) valid_from: Option>, + pub(crate) valid_until: Option>, + pub(crate) effective_from_commit: Option, + pub(crate) effective_until_commit: Option, + pub(crate) expires_at: Option>, + pub(crate) author: ActorRefV1, + pub(crate) created_at: DateTime, + pub(crate) compile_record: CompileRecordV1, +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum MemoryEventAction { + Created, + Revised, + Confirmed, + Quarantined, + Superseded, + Revoked, + Forgotten, + TaxonomyExpanded, + Consolidated, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct MemoryEventV1 { + pub(crate) schema_version: u32, + pub(crate) event_id: Uuid, + pub(crate) event_seq: u64, + pub(crate) note_id: Option, + pub(crate) revision_oid: Option, + pub(crate) namespace: Option, + pub(crate) target_path: Option, + pub(crate) action: MemoryEventAction, + pub(crate) reason_code: Option, + pub(crate) actor: ActorRefV1, + pub(crate) at: DateTime, + pub(crate) evidence_refs: Vec, + pub(crate) next_note_id: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub(crate) enum EvidenceLocatorV1 { + Object, + EventSeq { + event_seq: u64, + }, + JsonPointer { + pointer: String, + }, + SessionFragment { + start_seq: u64, + end_seq: u64, + }, + ToolCall { + invocation_id: String, + part: ToolCallPart, + }, + CodeRange { + commit_oid: String, + path: String, + start_line: u32, + end_line: u32, + }, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct EvidenceRefV1 { + pub(crate) schema_version: u32, + pub(crate) source_plane: EvidenceSourcePlane, + pub(crate) kind: EvidenceKind, + pub(crate) object_id: String, + pub(crate) source_ref_oid: String, + pub(crate) locator: EvidenceLocatorV1, + pub(crate) fragment_digest: String, + pub(crate) visibility: EvidenceVisibility, + pub(crate) captured_at: Option>, + pub(crate) code_commit: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct EpisodeRoot { + kind: EpisodeRootKind, + id: String, + path: String, + note_id: Uuid, +} + +#[derive(Debug, Error, Eq, PartialEq)] +pub(crate) enum MemoryContractError { + #[error("episode root ID cannot be empty")] + EmptyEpisodeRootId, + #[error("episode root ID must not have surrounding whitespace or control characters")] + NonCanonicalEpisodeRootId, + #[error("episode root ID exceeds the {max_bytes}-byte limit")] + EpisodeRootIdTooLong { max_bytes: usize }, + #[error("unsupported {object} schema version {version}")] + UnsupportedSchemaVersion { object: &'static str, version: u32 }, + #[error("invalid {field}")] + InvalidField { field: &'static str }, + #[error("invalid {object} JSON")] + InvalidJson { object: &'static str }, +} + +impl EpisodeRoot { + pub(crate) fn task(id: impl Into) -> Result { + Self::new(EpisodeRootKind::Task, id.into()) + } + + pub(crate) fn intent(id: impl Into) -> Result { + Self::new(EpisodeRootKind::Intent, id.into()) + } + + fn new(kind: EpisodeRootKind, id: String) -> Result { + if id.is_empty() { + return Err(MemoryContractError::EmptyEpisodeRootId); + } + if id.len() > MAX_EPISODE_ROOT_ID_BYTES { + return Err(MemoryContractError::EpisodeRootIdTooLong { + max_bytes: MAX_EPISODE_ROOT_ID_BYTES, + }); + } + if id.trim() != id || id.chars().any(char::is_control) { + return Err(MemoryContractError::NonCanonicalEpisodeRootId); + } + + let encoded_id = hex::encode(id.as_bytes()); + let (identity_prefix, path_prefix) = match kind { + EpisodeRootKind::Task => (b"task\0".as_slice(), "episodic.tasks"), + EpisodeRootKind::Intent => (b"intent\0".as_slice(), "episodic.intents"), + }; + let mut identity_name = identity_prefix.to_vec(); + identity_name.extend_from_slice(id.as_bytes()); + + Ok(Self { + kind, + path: format!("{path_prefix}.r-{encoded_id}"), + note_id: Uuid::new_v5(&EPISODE_NOTE_NAMESPACE_V1, &identity_name), + id, + }) + } + + pub(crate) fn kind(&self) -> EpisodeRootKind { + self.kind + } + + pub(crate) fn id(&self) -> &str { + &self.id + } + + pub(crate) fn namespace(&self) -> &'static str { + EPISODE_NAMESPACE + } + + pub(crate) fn path(&self) -> &str { + &self.path + } + + pub(crate) fn note_id(&self) -> Uuid { + self.note_id + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn task_root_has_stable_cell_identity() { + let root = EpisodeRoot::task("task-42").expect("synthetic task id is valid"); + + assert_eq!(root.kind(), EpisodeRootKind::Task); + assert_eq!(root.id(), "task-42"); + assert_eq!(root.namespace(), "default"); + assert_eq!(root.path(), "episodic.tasks.r-7461736b2d3432"); + assert_eq!( + root.note_id().to_string(), + "98809d1c-f0cd-5e98-84b8-c1dddf5aeb19", + ); + assert_eq!( + EpisodeRoot::task(""), + Err(MemoryContractError::EmptyEpisodeRootId), + ); + } + + #[test] + fn intent_root_has_a_distinct_stable_cell_identity() { + let root = EpisodeRoot::intent("intent-9").expect("synthetic intent id is valid"); + + assert_eq!(root.kind(), EpisodeRootKind::Intent); + assert_eq!(root.id(), "intent-9"); + assert_eq!(root.namespace(), "default"); + assert_eq!(root.path(), "episodic.intents.r-696e74656e742d39"); + assert_eq!( + root.note_id().to_string(), + "760369f7-ba78-541a-9aae-4e899154530b", + ); + } + + #[test] + fn root_ids_reject_ambiguous_or_unbounded_values() { + assert_eq!( + EpisodeRoot::task(" task-42"), + Err(MemoryContractError::NonCanonicalEpisodeRootId), + ); + assert_eq!( + EpisodeRoot::task("task-42\n"), + Err(MemoryContractError::NonCanonicalEpisodeRootId), + ); + assert_eq!( + EpisodeRoot::task("x".repeat(121)), + Err(MemoryContractError::EpisodeRootIdTooLong { max_bytes: 120 }), + ); + } + + #[test] + fn contract_errors_have_stable_redacted_messages() { + assert_eq!( + MemoryContractError::UnsupportedSchemaVersion { + object: "MemoryNote", + version: 2, + } + .to_string(), + "unsupported MemoryNote schema version 2", + ); + assert_eq!( + MemoryContractError::InvalidJson { + object: "EpisodePayload", + } + .to_string(), + "invalid EpisodePayload JSON", + ); + assert_eq!( + MemoryContractError::InvalidField { + field: "EpisodePayload.summary", + } + .to_string(), + "invalid EpisodePayload.summary", + ); + } +} diff --git a/src/internal/ai/memory/mod.rs b/src/internal/ai/memory/mod.rs new file mode 100644 index 000000000..68437d16a --- /dev/null +++ b/src/internal/ai/memory/mod.rs @@ -0,0 +1,14 @@ +#![allow( + dead_code, + reason = "M2-01 freezes I/O-free contracts before the M2-04 writer consumes them" +)] + +//! Versioned Agent Memory domain contracts. +//! +//! This module intentionally exposes only validated, I/O-free domain values. +//! Storage, projection, compilation, and command adapters are implemented by +//! later plan slices and must not become alternate write seams. + +mod canonical; +mod domain; +mod validation; diff --git a/src/internal/ai/memory/validation.rs b/src/internal/ai/memory/validation.rs new file mode 100644 index 000000000..3a22a912e --- /dev/null +++ b/src/internal/ai/memory/validation.rs @@ -0,0 +1,1691 @@ +use super::domain::{ + CompileOriginV1, CompileRecordV1, EpisodeClaimV1, EpisodeCodeContextV1, EpisodePayloadV1, + EpisodeRoot, EpisodeRootKind, EpistemicStatus, EvidenceLocatorV1, EvidenceRefV1, + IdempotencyScopeV1, MemoryContractError, MemoryEventAction, MemoryEventV1, MemoryKind, + MemoryLifecycle, MemoryLinkKind, MemoryNoteV1, MemoryScopeV1, MemoryVisibility, + TrustedEpisodeFieldsV1, +}; + +const MEMORY_SCHEMA_VERSION_V1: u32 = 1; +const MAX_COMPILE_RECORD_BYTES: usize = 32 * 1024; +const MAX_MEMORY_NOTE_BYTES: usize = 256 * 1024; +const MAX_MEMORY_EVENT_BYTES: usize = 128 * 1024; +const MAX_EVIDENCE_REF_BYTES: usize = 16 * 1024; +const MAX_IDENTIFIER_BYTES: usize = 512; +const MAX_PATH_BYTES: usize = 4 * 1024; +const MAX_SESSION_FRAGMENT_ITEMS: u64 = 256; +const MAX_EPISODE_TEXT_BYTES: usize = 4 * 1024; +const MAX_EPISODE_COLLECTION_ITEMS: usize = 128; +const MAX_EPISODE_PAYLOAD_BYTES: usize = 64 * 1024; +const MAX_MEMORY_NOTE_BODY_BYTES: usize = 16 * 1024; +const MAX_MEMORY_NOTE_PARENTS: usize = 1; + +pub(super) fn parse_compile_record_v1( + bytes: &[u8], +) -> Result { + validate_input_size(bytes, MAX_COMPILE_RECORD_BYTES, "CompileRecord.size")?; + let record = serde_json::from_slice(bytes).map_err(|_| MemoryContractError::InvalidJson { + object: "CompileRecord", + })?; + validate_compile_record(&record)?; + Ok(record) +} + +pub(super) fn parse_memory_note_v1(bytes: &[u8]) -> Result { + validate_input_size(bytes, MAX_MEMORY_NOTE_BYTES, "MemoryNote.size")?; + let note = serde_json::from_slice(bytes).map_err(|_| MemoryContractError::InvalidJson { + object: "MemoryNote", + })?; + validate_memory_note(¬e)?; + super::canonical::verify_memory_note_content_digest_v1(¬e)?; + Ok(note) +} + +pub(super) fn parse_memory_event_v1(bytes: &[u8]) -> Result { + validate_input_size(bytes, MAX_MEMORY_EVENT_BYTES, "MemoryEvent.size")?; + let event = serde_json::from_slice(bytes).map_err(|_| MemoryContractError::InvalidJson { + object: "MemoryEvent", + })?; + validate_memory_event(&event)?; + Ok(event) +} + +pub(super) fn parse_episode_payload_v1( + bytes: &[u8], + trusted: &TrustedEpisodeFieldsV1, +) -> Result { + validate_input_size(bytes, MAX_EPISODE_PAYLOAD_BYTES, "EpisodePayload.size")?; + let payload = serde_json::from_slice(bytes).map_err(|_| MemoryContractError::InvalidJson { + object: "EpisodePayload", + })?; + validate_episode_payload(&payload, trusted)?; + Ok(payload) +} + +pub(super) fn parse_evidence_ref_v1(bytes: &[u8]) -> Result { + validate_input_size(bytes, MAX_EVIDENCE_REF_BYTES, "EvidenceRef.size")?; + let evidence_ref = + serde_json::from_slice(bytes).map_err(|_| MemoryContractError::InvalidJson { + object: "EvidenceRef", + })?; + validate_evidence_ref(&evidence_ref)?; + Ok(evidence_ref) +} + +pub(super) fn validate_episode_payload( + payload: &EpisodePayloadV1, + trusted: &TrustedEpisodeFieldsV1, +) -> Result<(), MemoryContractError> { + validate_episode_payload_shape(payload)?; + if payload.root_kind != trusted.root.kind() + || payload.root_id != trusted.root.id() + || payload.related_intent_ids != trusted.related_intent_ids + || payload.related_task_ids != trusted.related_task_ids + || payload.related_run_ids != trusted.related_run_ids + || payload.started_at != trusted.started_at + || payload.ended_at != trusted.ended_at + || payload.goal != trusted.goal + || payload.completion_status != trusted.completion_status + || payload.code_change_status != trusted.code_change_status + || payload.code != trusted.code + || payload.omissions != trusted.omissions + { + return Err(MemoryContractError::InvalidField { + field: "EpisodePayload.trusted_fields", + }); + } + + Ok(()) +} + +fn validate_episode_payload_shape(payload: &EpisodePayloadV1) -> Result<(), MemoryContractError> { + if payload.schema_version != MEMORY_SCHEMA_VERSION_V1 { + return Err(MemoryContractError::UnsupportedSchemaVersion { + object: "EpisodePayload", + version: payload.schema_version, + }); + } + if payload + .started_at + .zip(payload.ended_at) + .is_some_and(|(start, end)| start > end) + { + return Err(MemoryContractError::InvalidField { + field: "EpisodePayload.time_range", + }); + } + validate_related_ids(payload)?; + validate_code_context(&payload.code)?; + + validate_episode_claim(&payload.goal)?; + if payload.goal.epistemic_status != EpistemicStatus::Observation { + return Err(MemoryContractError::InvalidField { + field: "EpisodePayload.goal", + }); + } + validate_episode_claim(&payload.summary)?; + if payload.summary.epistemic_status != EpistemicStatus::Inference { + return Err(MemoryContractError::InvalidField { + field: "EpisodePayload.summary", + }); + } + validate_claim_collection(&payload.observations, Some(EpistemicStatus::Observation))?; + validate_claim_collection(&payload.inferences, Some(EpistemicStatus::Inference))?; + validate_claim_collection(&payload.decisions, None)?; + validate_claim_collection(&payload.failed_attempts, None)?; + validate_claim_collection(&payload.unresolved, None)?; + + let encoded = serde_json::to_vec(payload).map_err(|_| MemoryContractError::InvalidField { + field: "EpisodePayload.encoding", + })?; + if encoded.len() > MAX_EPISODE_PAYLOAD_BYTES { + return Err(MemoryContractError::InvalidField { + field: "EpisodePayload.size", + }); + } + + Ok(()) +} + +fn validate_compile_record(record: &CompileRecordV1) -> Result<(), MemoryContractError> { + if record.schema_version != MEMORY_SCHEMA_VERSION_V1 { + return Err(MemoryContractError::UnsupportedSchemaVersion { + object: "CompileRecord", + version: record.schema_version, + }); + } + if !is_bounded_nonempty(&record.producer, MAX_IDENTIFIER_BYTES) + || record.rules_version == 0 + || !is_bounded_nonempty(&record.policy_version, MAX_IDENTIFIER_BYTES) + || record.input_hashes.is_empty() + || record.input_hashes.len() > MAX_EPISODE_COLLECTION_ITEMS + || record + .input_hashes + .iter() + .any(|hash| !is_sha256_digest(hash) && !is_hmac_sha256_digest(hash)) + || record + .input_hashes + .windows(2) + .any(|pair| pair[0] >= pair[1]) + || !is_hmac_sha256_digest(&record.idempotency_key) + || record + .prompt_version + .as_deref() + .is_some_and(|value| !is_bounded_nonempty(value, MAX_IDENTIFIER_BYTES)) + || record + .model_id + .as_deref() + .is_some_and(|value| !is_bounded_nonempty(value, MAX_IDENTIFIER_BYTES)) + || record.prompt_version.is_some() != record.model_id.is_some() + || (record.origin == CompileOriginV1::EpisodeCompiler && record.model_id.is_none()) + || (record.idempotency_scope == IdempotencyScopeV1::Namespace + && !matches!( + record.origin, + CompileOriginV1::Consolidation | CompileOriginV1::Onboard + )) + { + return Err(MemoryContractError::InvalidField { + field: "CompileRecord", + }); + } + + Ok(()) +} + +fn validate_memory_note(note: &MemoryNoteV1) -> Result<(), MemoryContractError> { + if note.schema_version != MEMORY_SCHEMA_VERSION_V1 { + return Err(MemoryContractError::UnsupportedSchemaVersion { + object: "MemoryNote", + version: note.schema_version, + }); + } + if !is_sha256_digest(¬e.content_digest) + || !is_valid_memory_scope(¬e.scope) + || !is_bounded_nonempty(¬e.namespace, MAX_IDENTIFIER_BYTES) + || !is_bounded_nonempty(¬e.path, MAX_PATH_BYTES) + || !is_bounded_nonempty(¬e.acl_policy_id, MAX_IDENTIFIER_BYTES) + || note.body.is_empty() + || note.body.len() > MAX_MEMORY_NOTE_BODY_BYTES + || note + .rationale + .as_deref() + .is_some_and(|value| value.len() > MAX_EPISODE_TEXT_BYTES) + || !is_bounded_nonempty(¬e.author.principal_id, MAX_IDENTIFIER_BYTES) + || note.evidence_refs.len() > MAX_EPISODE_COLLECTION_ITEMS + || note.links.len() > MAX_EPISODE_COLLECTION_ITEMS + || note.entities.len() > MAX_EPISODE_COLLECTION_ITEMS + || note.parents.len() > MAX_MEMORY_NOTE_PARENTS + || note.tags.len() > MAX_EPISODE_COLLECTION_ITEMS + || note + .valid_from + .zip(note.valid_until) + .is_some_and(|(from, until)| from > until) + || note + .effective_from_commit + .as_deref() + .is_some_and(|oid| !is_git_oid(oid)) + || note + .effective_until_commit + .as_deref() + .is_some_and(|oid| !is_git_oid(oid)) + || note.parents.iter().any(|oid| !is_git_oid(oid)) + || note + .tags + .iter() + .any(|tag| !is_bounded_nonempty(tag, MAX_IDENTIFIER_BYTES)) + || note.tags.windows(2).any(|pair| pair[0] >= pair[1]) + { + return Err(MemoryContractError::InvalidField { + field: "MemoryNote", + }); + } + validate_compile_record(¬e.compile_record)?; + for evidence_ref in ¬e.evidence_refs { + validate_evidence_ref(evidence_ref)?; + } + for link in ¬e.links { + if link.evidence_refs.len() > MAX_EPISODE_COLLECTION_ITEMS + || link + .target_revision_oid + .as_deref() + .is_some_and(|oid| !is_git_oid(oid)) + || link + .valid_from + .zip(link.valid_until) + .is_some_and(|(from, until)| from > until) + { + return Err(MemoryContractError::InvalidField { + field: "MemoryNote.links", + }); + } + for evidence_ref in &link.evidence_refs { + validate_evidence_ref(evidence_ref)?; + } + } + for entity in ¬e.entities { + if entity.schema_version != MEMORY_SCHEMA_VERSION_V1 + || !is_bounded_nonempty(&entity.canonical_key, MAX_IDENTIFIER_BYTES) + || !is_bounded_nonempty(&entity.display_name, MAX_EPISODE_TEXT_BYTES) + || entity.aliases.len() > MAX_EPISODE_COLLECTION_ITEMS + || entity + .aliases + .iter() + .any(|alias| !is_bounded_nonempty(alias, MAX_EPISODE_TEXT_BYTES)) + || entity.aliases.windows(2).any(|pair| pair[0] >= pair[1]) + || entity.evidence_refs.is_empty() + || entity.evidence_refs.len() > MAX_EPISODE_COLLECTION_ITEMS + { + return Err(MemoryContractError::InvalidField { + field: "MemoryNote.entities", + }); + } + for evidence_ref in &entity.evidence_refs { + validate_evidence_ref(evidence_ref)?; + } + } + + if note.episode.is_some() != (note.compile_record.origin == CompileOriginV1::EpisodeCompiler) { + return Err(MemoryContractError::InvalidField { + field: "MemoryNote.episode_origin", + }); + } + if let Some(payload) = ¬e.episode { + validate_episode_payload_shape(payload)?; + let root = match payload.root_kind { + EpisodeRootKind::Task => EpisodeRoot::task(&payload.root_id), + EpisodeRootKind::Intent => EpisodeRoot::intent(&payload.root_id), + }?; + let expected_effective_commit = payload + .code + .result_oid + .as_ref() + .or(payload.code.base_oid.as_ref()); + if note.kind != MemoryKind::Episodic + || note.scope != MemoryScopeV1::Repo + || note.visibility != MemoryVisibility::RepoLocal + || note.lifecycle != MemoryLifecycle::Accretive + || note.namespace != root.namespace() + || note.path != root.path() + || note.note_id != root.note_id() + || note.valid_from.is_some() + || note.valid_until.is_some() + || note.expires_at.is_some() + || note.effective_from_commit.as_ref() != expected_effective_commit + || note.effective_until_commit.is_some() + || note.compile_record.origin != CompileOriginV1::EpisodeCompiler + { + return Err(MemoryContractError::InvalidField { + field: "MemoryNote.episode_envelope", + }); + } + validate_intent_task_links(note, payload)?; + } + + Ok(()) +} + +fn validate_memory_event(event: &MemoryEventV1) -> Result<(), MemoryContractError> { + if event.schema_version != MEMORY_SCHEMA_VERSION_V1 { + return Err(MemoryContractError::UnsupportedSchemaVersion { + object: "MemoryEvent", + version: event.schema_version, + }); + } + if event.event_seq == 0 + || !is_bounded_nonempty(&event.actor.principal_id, MAX_IDENTIFIER_BYTES) + || event + .namespace + .as_deref() + .is_some_and(|value| !is_bounded_nonempty(value, MAX_IDENTIFIER_BYTES)) + || event + .target_path + .as_deref() + .is_some_and(|value| !is_bounded_nonempty(value, MAX_PATH_BYTES)) + || event + .reason_code + .as_deref() + .is_some_and(|value| !is_bounded_nonempty(value, MAX_IDENTIFIER_BYTES)) + || event.evidence_refs.len() > MAX_EPISODE_COLLECTION_ITEMS + || event + .revision_oid + .as_deref() + .is_some_and(|oid| !is_git_oid(oid)) + { + return Err(MemoryContractError::InvalidField { + field: "MemoryEvent", + }); + } + for evidence_ref in &event.evidence_refs { + validate_evidence_ref(evidence_ref)?; + } + + match event.action { + MemoryEventAction::TaxonomyExpanded => { + if event.namespace.as_deref().is_none_or(str::is_empty) + || event.target_path.as_deref().is_none_or(str::is_empty) + || event.note_id.is_some() + || event.revision_oid.is_some() + || event.next_note_id.is_some() + { + return Err(MemoryContractError::InvalidField { + field: "MemoryEvent.taxonomy_target", + }); + } + } + _ if event.note_id.is_none() + || event.revision_oid.is_none() + || event.namespace.is_some() + || event.target_path.is_some() + || (event.next_note_id.is_some() && event.action != MemoryEventAction::Superseded) => + { + return Err(MemoryContractError::InvalidField { + field: "MemoryEvent.note_target", + }); + } + _ => {} + } + + Ok(()) +} + +pub(super) fn validate_episode_claim(claim: &EpisodeClaimV1) -> Result<(), MemoryContractError> { + if claim.claim.is_empty() || claim.claim.len() > MAX_EPISODE_TEXT_BYTES { + return Err(MemoryContractError::InvalidField { + field: "EpisodeClaim.claim", + }); + } + if claim.evidence_refs.is_empty() || claim.evidence_refs.len() > MAX_EPISODE_COLLECTION_ITEMS { + return Err(MemoryContractError::InvalidField { + field: "EpisodeClaim.evidence_refs", + }); + } + match (claim.epistemic_status, claim.confidence) { + (EpistemicStatus::Observation, None) | (EpistemicStatus::Inference, Some(_)) => {} + _ => { + return Err(MemoryContractError::InvalidField { + field: "EpisodeClaim.confidence", + }); + } + } + for evidence_ref in &claim.evidence_refs { + validate_evidence_ref(evidence_ref)?; + } + + Ok(()) +} + +pub(super) fn validate_evidence_ref( + evidence_ref: &EvidenceRefV1, +) -> Result<(), MemoryContractError> { + if evidence_ref.schema_version != MEMORY_SCHEMA_VERSION_V1 { + return Err(MemoryContractError::UnsupportedSchemaVersion { + object: "EvidenceRef", + version: evidence_ref.schema_version, + }); + } + if !is_bounded_nonempty(&evidence_ref.object_id, MAX_IDENTIFIER_BYTES) { + return Err(MemoryContractError::InvalidField { + field: "EvidenceRef.object_id", + }); + } + if !is_git_oid(&evidence_ref.source_ref_oid) { + return Err(MemoryContractError::InvalidField { + field: "EvidenceRef.source_ref_oid", + }); + } + if !is_sha256_digest(&evidence_ref.fragment_digest) { + return Err(MemoryContractError::InvalidField { + field: "EvidenceRef.fragment_digest", + }); + } + if let Some(code_commit) = &evidence_ref.code_commit + && !is_git_oid(code_commit) + { + return Err(MemoryContractError::InvalidField { + field: "EvidenceRef.code_commit", + }); + } + + match &evidence_ref.locator { + EvidenceLocatorV1::Object => {} + EvidenceLocatorV1::EventSeq { event_seq } if *event_seq > 0 => {} + EvidenceLocatorV1::JsonPointer { pointer } if is_valid_json_pointer(pointer) => {} + EvidenceLocatorV1::SessionFragment { start_seq, end_seq } + if start_seq > &0 + && start_seq <= end_seq + && end_seq - start_seq < MAX_SESSION_FRAGMENT_ITEMS => {} + EvidenceLocatorV1::ToolCall { invocation_id, .. } + if is_bounded_nonempty(invocation_id, MAX_IDENTIFIER_BYTES) => {} + EvidenceLocatorV1::CodeRange { + commit_oid, + path, + start_line, + end_line, + } if is_git_oid(commit_oid) + && is_repo_relative_path(path) + && start_line > &0 + && start_line <= end_line + && evidence_ref + .code_commit + .as_ref() + .is_none_or(|code_commit| code_commit == commit_oid) => {} + _ => { + return Err(MemoryContractError::InvalidField { + field: "EvidenceRef.locator", + }); + } + } + if !locator_matches_source(evidence_ref) { + return Err(MemoryContractError::InvalidField { + field: "EvidenceRef.source_locator", + }); + } + + Ok(()) +} + +fn is_git_oid(value: &str) -> bool { + matches!(value.len(), 40 | 64) && value.bytes().all(is_lower_hex_digit) +} + +fn is_sha256_digest(value: &str) -> bool { + value + .strip_prefix("sha256:") + .is_some_and(|digest| digest.len() == 64 && digest.bytes().all(is_lower_hex_digit)) +} + +fn is_hmac_sha256_digest(value: &str) -> bool { + let Some(rest) = value.strip_prefix("hmac-sha256:") else { + return false; + }; + let Some((key_id, digest)) = rest.split_once(':') else { + return false; + }; + is_bounded_nonempty(key_id, 128) && digest.len() == 64 && digest.bytes().all(is_lower_hex_digit) +} + +fn is_repo_relative_path(path: &str) -> bool { + !path.is_empty() + && path.len() <= MAX_PATH_BYTES + && !path.starts_with('/') + && !path.contains('\\') + && !path.chars().any(char::is_control) + && !path + .split('/') + .any(|segment| segment.is_empty() || segment == "." || segment == "..") +} + +fn is_lower_hex_digit(byte: u8) -> bool { + byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte) +} + +fn is_bounded_nonempty(value: &str, max_bytes: usize) -> bool { + !value.is_empty() && value.len() <= max_bytes && !value.chars().any(char::is_control) +} + +fn is_valid_memory_scope(scope: &MemoryScopeV1) -> bool { + match scope { + MemoryScopeV1::Repo | MemoryScopeV1::Global => true, + MemoryScopeV1::Branch(branch_ref) => is_valid_branch_ref(branch_ref), + MemoryScopeV1::Worktree(id) | MemoryScopeV1::Actor(id) => { + is_bounded_nonempty(id, MAX_IDENTIFIER_BYTES) + } + } +} + +fn is_valid_json_pointer(pointer: &str) -> bool { + if !pointer.starts_with('/') + || pointer.len() > MAX_PATH_BYTES + || pointer.chars().any(char::is_control) + { + return false; + } + + let mut chars = pointer.chars(); + while let Some(character) = chars.next() { + if character == '~' && !matches!(chars.next(), Some('0' | '1')) { + return false; + } + } + true +} + +fn validate_input_size( + bytes: &[u8], + max_bytes: usize, + field: &'static str, +) -> Result<(), MemoryContractError> { + if bytes.len() > max_bytes { + return Err(MemoryContractError::InvalidField { field }); + } + Ok(()) +} + +fn validate_related_ids(payload: &EpisodePayloadV1) -> Result<(), MemoryContractError> { + for (field, ids) in [ + ( + "EpisodePayload.related_intent_ids", + &payload.related_intent_ids, + ), + ("EpisodePayload.related_task_ids", &payload.related_task_ids), + ("EpisodePayload.related_run_ids", &payload.related_run_ids), + ] { + if ids.len() > MAX_EPISODE_COLLECTION_ITEMS + || ids + .iter() + .any(|id| !is_bounded_nonempty(id, MAX_IDENTIFIER_BYTES)) + || ids.windows(2).any(|pair| pair[0] >= pair[1]) + { + return Err(MemoryContractError::InvalidField { field }); + } + } + + let root_is_present = match payload.root_kind { + EpisodeRootKind::Task => payload + .related_task_ids + .iter() + .any(|id| id == &payload.root_id), + EpisodeRootKind::Intent => payload + .related_intent_ids + .iter() + .any(|id| id == &payload.root_id), + }; + if !root_is_present { + return Err(MemoryContractError::InvalidField { + field: "EpisodePayload.root_relation", + }); + } + if payload.root_kind == EpisodeRootKind::Intent && payload.related_task_ids.is_empty() { + return Err(MemoryContractError::InvalidField { + field: "EpisodePayload.related_task_ids", + }); + } + + Ok(()) +} + +fn validate_intent_task_links( + note: &MemoryNoteV1, + payload: &EpisodePayloadV1, +) -> Result<(), MemoryContractError> { + if payload.root_kind != EpisodeRootKind::Intent { + return Ok(()); + } + + for task_id in &payload.related_task_ids { + let task_note_id = EpisodeRoot::task(task_id)?.note_id(); + let mut matching_links = note.links.iter().filter(|link| { + link.target_note_id == task_note_id && link.kind == MemoryLinkKind::Supports + }); + let Some(task_link) = matching_links.next() else { + return Err(MemoryContractError::InvalidField { + field: "MemoryNote.intent_task_links", + }); + }; + if matching_links.next().is_some() || task_link.target_revision_oid.is_none() { + return Err(MemoryContractError::InvalidField { + field: "MemoryNote.intent_task_links", + }); + } + } + + Ok(()) +} + +fn validate_code_context(code: &EpisodeCodeContextV1) -> Result<(), MemoryContractError> { + if code.base_oid.as_deref().is_some_and(|oid| !is_git_oid(oid)) + || code + .result_oid + .as_deref() + .is_some_and(|oid| !is_git_oid(oid)) + || code + .branch_ref + .as_deref() + .is_some_and(|branch_ref| !is_valid_branch_ref(branch_ref)) + || code.paths.len() > MAX_EPISODE_COLLECTION_ITEMS + || code.paths.iter().any(|path| !is_repo_relative_path(path)) + || code.paths.windows(2).any(|pair| pair[0] >= pair[1]) + { + return Err(MemoryContractError::InvalidField { + field: "EpisodePayload.code", + }); + } + + Ok(()) +} + +fn is_valid_branch_ref(branch_ref: &str) -> bool { + let Some(name) = branch_ref.strip_prefix("refs/heads/") else { + return false; + }; + is_bounded_nonempty(name, MAX_PATH_BYTES) && crate::utils::util::is_valid_refname(branch_ref) +} + +fn validate_claim_collection( + claims: &[EpisodeClaimV1], + required_status: Option, +) -> Result<(), MemoryContractError> { + if claims.len() > MAX_EPISODE_COLLECTION_ITEMS { + return Err(MemoryContractError::InvalidField { + field: "EpisodePayload.claims", + }); + } + for claim in claims { + validate_episode_claim(claim)?; + if required_status.is_some_and(|status| claim.epistemic_status != status) { + return Err(MemoryContractError::InvalidField { + field: "EpisodePayload.claim_status", + }); + } + } + + Ok(()) +} + +fn locator_matches_source(evidence_ref: &EvidenceRefV1) -> bool { + use super::domain::{EvidenceKind, EvidenceSourcePlane}; + + matches!( + ( + evidence_ref.source_plane, + evidence_ref.kind, + &evidence_ref.locator, + ), + ( + EvidenceSourcePlane::AgentRuntime, + EvidenceKind::Intent + | EvidenceKind::Task + | EvidenceKind::Run + | EvidenceKind::Evidence + | EvidenceKind::Decision + | EvidenceKind::PatchSet, + EvidenceLocatorV1::Object + | EvidenceLocatorV1::EventSeq { .. } + | EvidenceLocatorV1::JsonPointer { .. }, + ) | ( + EvidenceSourcePlane::Session, + EvidenceKind::Session, + EvidenceLocatorV1::SessionFragment { .. }, + ) | ( + EvidenceSourcePlane::AgentRuntime | EvidenceSourcePlane::Session, + EvidenceKind::ToolCall, + EvidenceLocatorV1::ToolCall { .. }, + ) | ( + EvidenceSourcePlane::Git, + EvidenceKind::Code, + EvidenceLocatorV1::CodeRange { .. } + ) + ) +} + +#[cfg(test)] +mod tests { + use chrono::{TimeZone, Utc}; + + use super::{ + super::{ + super::context_budget::MemoryAnchorConfidence, + domain::{ + CodeChangeStatus, CompletionStatus, EpisodeClaimV1, EpisodeCodeContextV1, + EpisodeOmissionsV1, EpisodePayloadV1, EpisodeRoot, EpistemicStatus, EvidenceKind, + EvidenceLocatorV1, EvidenceRefV1, EvidenceSourcePlane, EvidenceVisibility, + MemoryContractError, MemoryNoteV1, MemoryTrust, ToolCallPart, + TrustedEpisodeFieldsV1, + }, + }, + validate_episode_claim, validate_episode_payload, validate_evidence_ref, + }; + + const SOURCE_OID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + const CODE_OID: &str = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"; + const FRAGMENT_DIGEST: &str = + "sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"; + + fn evidence( + source_plane: EvidenceSourcePlane, + kind: EvidenceKind, + locator: EvidenceLocatorV1, + ) -> EvidenceRefV1 { + EvidenceRefV1 { + schema_version: 1, + source_plane, + kind, + object_id: "object-1".to_string(), + source_ref_oid: SOURCE_OID.to_string(), + locator, + fragment_digest: FRAGMENT_DIGEST.to_string(), + visibility: EvidenceVisibility::RepoLocal, + captured_at: None, + code_commit: None, + } + } + + fn task_payload_fixture() -> (EpisodePayloadV1, TrustedEpisodeFieldsV1) { + let root = EpisodeRoot::task("task-42").expect("synthetic task id is valid"); + let evidence_ref = evidence( + EvidenceSourcePlane::AgentRuntime, + EvidenceKind::Evidence, + EvidenceLocatorV1::EventSeq { event_seq: 7 }, + ); + let observation = EpisodeClaimV1 { + epistemic_status: EpistemicStatus::Observation, + claim: "the retry test failed twice".to_string(), + confidence: None, + evidence_refs: vec![evidence_ref.clone()], + }; + let inference = EpisodeClaimV1 { + epistemic_status: EpistemicStatus::Inference, + claim: "the retry clock is probably nondeterministic".to_string(), + confidence: Some(MemoryAnchorConfidence::High), + evidence_refs: vec![evidence_ref], + }; + let code = EpisodeCodeContextV1 { + base_oid: Some(SOURCE_OID.to_string()), + result_oid: Some(CODE_OID.to_string()), + branch_ref: Some("refs/heads/feature/retry".to_string()), + paths: vec!["src/retry.rs".to_string()], + }; + let started_at = Utc + .with_ymd_and_hms(2026, 8, 20, 8, 0, 0) + .single() + .expect("synthetic timestamp is valid"); + let ended_at = Utc + .with_ymd_and_hms(2026, 8, 20, 9, 0, 0) + .single() + .expect("synthetic timestamp is valid"); + let trusted = TrustedEpisodeFieldsV1 { + root: root.clone(), + related_intent_ids: vec!["intent-9".to_string()], + related_task_ids: vec!["task-42".to_string()], + related_run_ids: vec!["run-1".to_string(), "run-2".to_string()], + started_at: Some(started_at), + ended_at: Some(ended_at), + goal: observation.clone(), + completion_status: CompletionStatus::Completed, + code_change_status: CodeChangeStatus::Changed, + code: code.clone(), + omissions: EpisodeOmissionsV1::default(), + }; + let payload = EpisodePayloadV1 { + schema_version: 1, + root_kind: root.kind(), + root_id: root.id().to_string(), + related_intent_ids: trusted.related_intent_ids.clone(), + related_task_ids: trusted.related_task_ids.clone(), + related_run_ids: trusted.related_run_ids.clone(), + started_at: trusted.started_at, + ended_at: trusted.ended_at, + goal: trusted.goal.clone(), + completion_status: trusted.completion_status, + code_change_status: trusted.code_change_status, + summary: inference.clone(), + observations: vec![observation], + inferences: vec![inference], + decisions: Vec::new(), + failed_attempts: Vec::new(), + unresolved: Vec::new(), + code, + omissions: trusted.omissions.clone(), + }; + + (payload, trusted) + } + + fn compile_record_json() -> serde_json::Value { + serde_json::json!({ + "schema_version": 1, + "origin": "explicit", + "producer": "libra-memory/1", + "rules_version": 1, + "prompt_version": null, + "model_id": null, + "policy_version": "repo-policy-v1", + "input_hashes": [FRAGMENT_DIGEST], + "idempotency_key": "hmac-sha256:key-1:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", + "idempotency_scope": "cell" + }) + } + + fn mark_episode_compiler(note: &mut serde_json::Value) { + note["compile_record"]["origin"] = serde_json::json!("episode_compiler"); + note["compile_record"]["prompt_version"] = serde_json::json!("episode-v1"); + note["compile_record"]["model_id"] = serde_json::json!("synthetic-model"); + } + + fn memory_note_json() -> serde_json::Value { + let mut note = serde_json::json!({ + "schema_version": 1, + "note_id": "98809d1c-f0cd-5e98-84b8-c1dddf5aeb19", + "content_digest": FRAGMENT_DIGEST, + "namespace": "default", + "path": "episodic.tasks.r-7461736b2d3432", + "kind": "episodic", + "scope": { "type": "repo" }, + "visibility": "repo_local", + "acl_policy_id": "repo-policy-v1", + "lifecycle": "accretive", + "body": "The retry clock caused two failed attempts.", + "rationale": null, + "episode": null, + "evidence_refs": [], + "links": [], + "entities": [], + "parents": [], + "tags": ["retry"], + "confidence": "high", + "trust": "repo_evidence", + "sensitivity": "internal", + "valid_from": null, + "valid_until": null, + "effective_from_commit": CODE_OID, + "effective_until_commit": null, + "expires_at": null, + "author": { "kind": "agent", "principal_id": "agent:test" }, + "created_at": "2026-08-20T09:00:00Z", + "compile_record": compile_record_json() + }); + refresh_note_digest(&mut note); + note + } + + fn refresh_note_digest(note: &mut serde_json::Value) { + let typed: MemoryNoteV1 = + serde_json::from_value(note.clone()).expect("note fixture has a valid wire shape"); + let digest = super::super::canonical::memory_note_content_digest_v1(&typed) + .expect("note fixture canonicalizes"); + note["content_digest"] = serde_json::json!(digest); + } + + fn memory_event_json() -> serde_json::Value { + serde_json::json!({ + "schema_version": 1, + "event_id": "018fe3c4-4c00-7000-8000-000000000001", + "event_seq": 1, + "note_id": "98809d1c-f0cd-5e98-84b8-c1dddf5aeb19", + "revision_oid": SOURCE_OID, + "namespace": null, + "target_path": null, + "action": "created", + "reason_code": null, + "actor": { "kind": "agent", "principal_id": "agent:test" }, + "at": "2026-08-20T09:00:00Z", + "evidence_refs": [], + "next_note_id": null + }) + } + + fn intent_episode_note_json() -> serde_json::Value { + let (mut payload, _) = task_payload_fixture(); + let intent_root = EpisodeRoot::intent("intent-9").expect("synthetic intent id is valid"); + let task_root = EpisodeRoot::task("task-42").expect("synthetic task id is valid"); + payload.root_kind = intent_root.kind(); + payload.root_id = intent_root.id().to_string(); + payload.related_intent_ids = vec![intent_root.id().to_string()]; + + let mut note = memory_note_json(); + note["note_id"] = serde_json::json!(intent_root.note_id()); + note["path"] = serde_json::json!(intent_root.path()); + note["episode"] = serde_json::to_value(payload).expect("Episode serializes"); + note["links"] = serde_json::json!([{ + "kind": "supports", + "target_note_id": task_root.note_id(), + "target_revision_oid": SOURCE_OID, + "evidence_refs": [], + "valid_from": null, + "valid_until": null + }]); + mark_episode_compiler(&mut note); + refresh_note_digest(&mut note); + note + } + + #[test] + fn evidence_ref_accepts_all_six_bounded_locators() { + let cases = [ + evidence( + EvidenceSourcePlane::AgentRuntime, + EvidenceKind::Task, + EvidenceLocatorV1::Object, + ), + evidence( + EvidenceSourcePlane::AgentRuntime, + EvidenceKind::Evidence, + EvidenceLocatorV1::EventSeq { event_seq: 7 }, + ), + evidence( + EvidenceSourcePlane::AgentRuntime, + EvidenceKind::Task, + EvidenceLocatorV1::JsonPointer { + pointer: "/goal".to_string(), + }, + ), + evidence( + EvidenceSourcePlane::Session, + EvidenceKind::Session, + EvidenceLocatorV1::SessionFragment { + start_seq: 3, + end_seq: 8, + }, + ), + evidence( + EvidenceSourcePlane::Session, + EvidenceKind::ToolCall, + EvidenceLocatorV1::ToolCall { + invocation_id: "call-2".to_string(), + part: ToolCallPart::Output, + }, + ), + evidence( + EvidenceSourcePlane::Git, + EvidenceKind::Code, + EvidenceLocatorV1::CodeRange { + commit_oid: CODE_OID.to_string(), + path: "src/lib.rs".to_string(), + start_line: 10, + end_line: 14, + }, + ), + ]; + + for evidence_ref in cases { + validate_evidence_ref(&evidence_ref).expect("bounded synthetic locator is valid"); + } + } + + #[test] + fn evidence_ref_rejects_unbounded_or_ambiguous_locators() { + let mut cases = Vec::new(); + + let mut unknown_version = evidence( + EvidenceSourcePlane::AgentRuntime, + EvidenceKind::Task, + EvidenceLocatorV1::Object, + ); + unknown_version.schema_version = 2; + cases.push(unknown_version); + + let mut uppercase_oid = evidence( + EvidenceSourcePlane::AgentRuntime, + EvidenceKind::Task, + EvidenceLocatorV1::Object, + ); + uppercase_oid.source_ref_oid = "A".repeat(40); + cases.push(uppercase_oid); + + cases.push(evidence( + EvidenceSourcePlane::AgentRuntime, + EvidenceKind::Evidence, + EvidenceLocatorV1::EventSeq { event_seq: 0 }, + )); + cases.push(evidence( + EvidenceSourcePlane::AgentRuntime, + EvidenceKind::Task, + EvidenceLocatorV1::JsonPointer { + pointer: "goal".to_string(), + }, + )); + cases.push(evidence( + EvidenceSourcePlane::AgentRuntime, + EvidenceKind::Task, + EvidenceLocatorV1::JsonPointer { + pointer: "/bad~2escape".to_string(), + }, + )); + cases.push(evidence( + EvidenceSourcePlane::Session, + EvidenceKind::Session, + EvidenceLocatorV1::SessionFragment { + start_seq: 1, + end_seq: 258, + }, + )); + cases.push(evidence( + EvidenceSourcePlane::Session, + EvidenceKind::ToolCall, + EvidenceLocatorV1::ToolCall { + invocation_id: String::new(), + part: ToolCallPart::Invocation, + }, + )); + cases.push(evidence( + EvidenceSourcePlane::Git, + EvidenceKind::Code, + EvidenceLocatorV1::CodeRange { + commit_oid: CODE_OID.to_string(), + path: "../src/lib.rs".to_string(), + start_line: 10, + end_line: 14, + }, + )); + cases.push(evidence( + EvidenceSourcePlane::Git, + EvidenceKind::Code, + EvidenceLocatorV1::CodeRange { + commit_oid: CODE_OID.to_string(), + path: "src/lib.rs".to_string(), + start_line: 14, + end_line: 10, + }, + )); + let mut mismatched_commit = evidence( + EvidenceSourcePlane::Git, + EvidenceKind::Code, + EvidenceLocatorV1::CodeRange { + commit_oid: CODE_OID.to_string(), + path: "src/lib.rs".to_string(), + start_line: 10, + end_line: 14, + }, + ); + mismatched_commit.code_commit = Some(SOURCE_OID.to_string()); + cases.push(mismatched_commit); + cases.push(evidence( + EvidenceSourcePlane::AgentRuntime, + EvidenceKind::Session, + EvidenceLocatorV1::SessionFragment { + start_seq: 1, + end_seq: 2, + }, + )); + + for evidence_ref in cases { + assert!( + validate_evidence_ref(&evidence_ref).is_err(), + "invalid locator was accepted: {evidence_ref:?}", + ); + } + } + + #[test] + fn claims_separate_observations_from_inferences() { + let evidence_ref = evidence( + EvidenceSourcePlane::AgentRuntime, + EvidenceKind::Evidence, + EvidenceLocatorV1::EventSeq { event_seq: 7 }, + ); + let observation = EpisodeClaimV1 { + epistemic_status: EpistemicStatus::Observation, + claim: "the retry test failed twice".to_string(), + confidence: None, + evidence_refs: vec![evidence_ref.clone()], + }; + let inference = EpisodeClaimV1 { + epistemic_status: EpistemicStatus::Inference, + claim: "the retry clock is probably nondeterministic".to_string(), + confidence: Some(MemoryAnchorConfidence::High), + evidence_refs: vec![evidence_ref], + }; + + validate_episode_claim(&observation).expect("direct observation is valid"); + validate_episode_claim(&inference).expect("evidence-backed inference is valid"); + + let mut invalid_observation = observation.clone(); + invalid_observation.confidence = Some(MemoryAnchorConfidence::Low); + assert!(validate_episode_claim(&invalid_observation).is_err()); + + let mut invalid_inference = inference.clone(); + invalid_inference.confidence = None; + assert!(validate_episode_claim(&invalid_inference).is_err()); + + let mut unsupported = observation; + unsupported.evidence_refs.clear(); + assert!(validate_episode_claim(&unsupported).is_err()); + } + + #[test] + fn payload_rejects_compiler_overrides_of_trusted_task_fields() { + let (payload, trusted) = task_payload_fixture(); + + validate_episode_payload(&payload, &trusted).expect("trusted Task payload is valid"); + + let mut overridden = payload; + overridden.completion_status = CompletionStatus::Failed; + assert!(validate_episode_payload(&overridden, &trusted).is_err()); + + let (mut overridden, trusted) = task_payload_fixture(); + overridden.related_run_ids.pop(); + assert!(validate_episode_payload(&overridden, &trusted).is_err()); + + let (mut overridden, trusted) = task_payload_fixture(); + overridden.started_at = None; + assert!(validate_episode_payload(&overridden, &trusted).is_err()); + + let (mut overridden, trusted) = task_payload_fixture(); + overridden.code.result_oid = Some(SOURCE_OID.to_string()); + assert!(validate_episode_payload(&overridden, &trusted).is_err()); + } + + #[test] + fn payload_rejects_git_invalid_full_branch_refs() { + for branch_ref in [ + "refs/heads/foo/.bar", + "refs/heads/foo/bar.lock/baz", + r"refs/heads/foo\bar", + ] { + let (mut payload, mut trusted) = task_payload_fixture(); + payload.code.branch_ref = Some(branch_ref.to_string()); + trusted.code = payload.code.clone(); + + assert!( + validate_episode_payload(&payload, &trusted).is_err(), + "Git-invalid full branch ref was accepted: {branch_ref}", + ); + } + } + + #[test] + fn payload_rejects_compiler_override_of_trusted_goal() { + let (mut payload, trusted) = task_payload_fixture(); + payload.goal.claim = "compiler replaced the trusted goal".to_string(); + + assert!(validate_episode_payload(&payload, &trusted).is_err()); + } + + #[test] + fn payload_rejects_compiler_override_of_trusted_omissions() { + let (mut payload, trusted) = task_payload_fixture(); + payload.omissions.failed_attempts = 1; + + assert!(validate_episode_payload(&payload, &trusted).is_err()); + } + + #[test] + fn intent_payload_requires_at_least_one_contributing_task() { + let (mut payload, mut trusted) = task_payload_fixture(); + let root = EpisodeRoot::intent("intent-9").expect("synthetic intent id is valid"); + trusted.root = root.clone(); + payload.root_kind = root.kind(); + payload.root_id = root.id().to_string(); + + validate_episode_payload(&payload, &trusted).expect("Intent payload has a Task input"); + + payload.related_task_ids.clear(); + trusted.related_task_ids.clear(); + assert!(validate_episode_payload(&payload, &trusted).is_err()); + } + + #[test] + fn intent_episode_note_pins_each_contributing_task_revision_once() { + use super::parse_memory_note_v1; + + let note = intent_episode_note_json(); + parse_memory_note_v1(&serde_json::to_vec(¬e).expect("note serializes")) + .expect("Intent Episode pins its contributing Task revision"); + + let mut with_sibling = note.clone(); + let mut sibling = with_sibling["links"][0].clone(); + sibling["kind"] = serde_json::json!("sibling"); + with_sibling["links"] + .as_array_mut() + .expect("links fixture is an array") + .push(sibling); + refresh_note_digest(&mut with_sibling); + parse_memory_note_v1( + &serde_json::to_vec(&with_sibling).expect("note with sibling serializes"), + ) + .expect("a non-Supports relation to the same Task remains valid"); + + let mut missing = note.clone(); + missing["links"] = serde_json::json!([]); + refresh_note_digest(&mut missing); + assert!( + parse_memory_note_v1(&serde_json::to_vec(&missing).expect("note serializes")).is_err(), + ); + + let mut duplicate = note.clone(); + duplicate["links"] = serde_json::json!([note["links"][0], note["links"][0]]); + refresh_note_digest(&mut duplicate); + assert!( + parse_memory_note_v1(&serde_json::to_vec(&duplicate).expect("note serializes")) + .is_err(), + ); + + let mut floating = note; + floating["links"][0]["target_revision_oid"] = serde_json::Value::Null; + refresh_note_digest(&mut floating); + assert!( + parse_memory_note_v1(&serde_json::to_vec(&floating).expect("note serializes")).is_err(), + ); + } + + #[test] + fn payload_rejects_noncanonical_trusted_ids() { + let (mut payload, mut trusted) = task_payload_fixture(); + payload.related_run_ids = vec!["run-\n".to_string()]; + trusted.related_run_ids = payload.related_run_ids.clone(); + + assert!(validate_episode_payload(&payload, &trusted).is_err()); + + let (mut payload, mut trusted) = task_payload_fixture(); + payload.related_run_ids = vec!["r".repeat(513)]; + trusted.related_run_ids = payload.related_run_ids.clone(); + assert!(validate_episode_payload(&payload, &trusted).is_err()); + } + + #[test] + fn episode_and_evidence_v1_gate_versions_enums_and_additive_fields() { + use super::{parse_episode_payload_v1, parse_evidence_ref_v1}; + + let (payload, trusted) = task_payload_fixture(); + let mut payload_json = serde_json::to_value(&payload).expect("payload serializes"); + payload_json + .as_object_mut() + .expect("payload is an object") + .insert("future_hint".to_string(), serde_json::json!(true)); + parse_episode_payload_v1( + &serde_json::to_vec(&payload_json).expect("payload JSON serializes"), + &trusted, + ) + .expect("additive field is ignored by a v1 reader"); + + payload_json["schema_version"] = serde_json::json!(2); + assert!( + parse_episode_payload_v1( + &serde_json::to_vec(&payload_json).expect("payload JSON serializes"), + &trusted, + ) + .is_err(), + ); + payload_json["schema_version"] = serde_json::json!(1); + payload_json["root_kind"] = serde_json::json!("future_root"); + assert!( + parse_episode_payload_v1( + &serde_json::to_vec(&payload_json).expect("payload JSON serializes"), + &trusted, + ) + .is_err(), + ); + + let evidence_ref = evidence( + EvidenceSourcePlane::AgentRuntime, + EvidenceKind::Task, + EvidenceLocatorV1::Object, + ); + let mut evidence_json = serde_json::to_value(evidence_ref).expect("evidence serializes"); + evidence_json + .as_object_mut() + .expect("evidence is an object") + .insert("future_hint".to_string(), serde_json::json!(true)); + parse_evidence_ref_v1( + &serde_json::to_vec(&evidence_json).expect("evidence JSON serializes"), + ) + .expect("additive field is ignored by a v1 reader"); + evidence_json["schema_version"] = serde_json::json!(2); + assert!( + parse_evidence_ref_v1( + &serde_json::to_vec(&evidence_json).expect("evidence JSON serializes"), + ) + .is_err(), + ); + } + + #[test] + fn note_event_and_compile_record_gate_versions_enums_and_additive_fields() { + use super::{parse_compile_record_v1, parse_memory_event_v1, parse_memory_note_v1}; + + let mut compile = compile_record_json(); + compile["future_hint"] = serde_json::json!(true); + parse_compile_record_v1(&serde_json::to_vec(&compile).expect("compile JSON serializes")) + .expect("additive CompileRecord field is ignored"); + compile["schema_version"] = serde_json::json!(2); + assert!( + parse_compile_record_v1( + &serde_json::to_vec(&compile).expect("compile JSON serializes"), + ) + .is_err(), + ); + compile["schema_version"] = serde_json::json!(1); + compile["origin"] = serde_json::json!("future_origin"); + assert!( + parse_compile_record_v1( + &serde_json::to_vec(&compile).expect("compile JSON serializes"), + ) + .is_err(), + ); + + let mut note = memory_note_json(); + note["future_hint"] = serde_json::json!(true); + super::parse_memory_note_v1(&serde_json::to_vec(¬e).expect("note JSON serializes")) + .expect("additive MemoryNote field is ignored"); + note["schema_version"] = serde_json::json!(2); + assert!( + super::parse_memory_note_v1(&serde_json::to_vec(¬e).expect("note JSON serializes")) + .is_err(), + ); + note["schema_version"] = serde_json::json!(1); + note["kind"] = serde_json::json!("future_kind"); + assert!( + parse_memory_note_v1(&serde_json::to_vec(¬e).expect("note JSON serializes")) + .is_err(), + ); + + let mut event = memory_event_json(); + event["future_hint"] = serde_json::json!(true); + parse_memory_event_v1(&serde_json::to_vec(&event).expect("event JSON serializes")) + .expect("additive MemoryEvent field is ignored"); + event["schema_version"] = serde_json::json!(2); + assert!( + parse_memory_event_v1(&serde_json::to_vec(&event).expect("event JSON serializes")) + .is_err(), + ); + event["schema_version"] = serde_json::json!(1); + event["action"] = serde_json::json!("updated"); + assert!( + parse_memory_event_v1(&serde_json::to_vec(&event).expect("event JSON serializes")) + .is_err(), + ); + } + + #[test] + fn memory_event_actions_require_unambiguous_target_shapes() { + use super::parse_memory_event_v1; + + let lifecycle = memory_event_json(); + parse_memory_event_v1(&serde_json::to_vec(&lifecycle).expect("event serializes")) + .expect("lifecycle event has a note target"); + + let mut taxonomy = memory_event_json(); + taxonomy["action"] = serde_json::json!("taxonomy_expanded"); + taxonomy["namespace"] = serde_json::json!("default"); + taxonomy["target_path"] = serde_json::json!("episodic.tasks"); + assert!( + parse_memory_event_v1(&serde_json::to_vec(&taxonomy).expect("event serializes")) + .is_err(), + ); + taxonomy["note_id"] = serde_json::Value::Null; + taxonomy["revision_oid"] = serde_json::Value::Null; + parse_memory_event_v1(&serde_json::to_vec(&taxonomy).expect("event serializes")) + .expect("taxonomy event has only a taxonomy target"); + + let mut mixed_lifecycle = memory_event_json(); + mixed_lifecycle["namespace"] = serde_json::json!("default"); + assert!( + parse_memory_event_v1( + &serde_json::to_vec(&mixed_lifecycle).expect("event serializes"), + ) + .is_err(), + ); + + let mut invalid_successor = memory_event_json(); + invalid_successor["next_note_id"] = + serde_json::json!("760369f7-ba78-541a-9aae-4e899154530b"); + assert!( + parse_memory_event_v1( + &serde_json::to_vec(&invalid_successor).expect("event serializes"), + ) + .is_err(), + ); + invalid_successor["action"] = serde_json::json!("superseded"); + parse_memory_event_v1(&serde_json::to_vec(&invalid_successor).expect("event serializes")) + .expect("Superseded may identify its successor"); + } + + #[test] + fn episode_note_envelope_rejects_wall_clock_validity() { + use super::parse_memory_note_v1; + + let (payload, _) = task_payload_fixture(); + let mut note = memory_note_json(); + note["episode"] = serde_json::to_value(payload).expect("Episode serializes"); + mark_episode_compiler(&mut note); + refresh_note_digest(&mut note); + parse_memory_note_v1(&serde_json::to_vec(¬e).expect("note JSON serializes")) + .expect("fixed Episode envelope is valid"); + + note["expires_at"] = serde_json::json!("2026-09-01T00:00:00Z"); + refresh_note_digest(&mut note); + assert!( + parse_memory_note_v1(&serde_json::to_vec(¬e).expect("note JSON serializes")) + .is_err(), + ); + } + + #[test] + fn episode_compiler_origin_requires_an_episode_payload() { + use super::parse_memory_note_v1; + + let mut note = memory_note_json(); + mark_episode_compiler(&mut note); + refresh_note_digest(&mut note); + assert!( + parse_memory_note_v1(&serde_json::to_vec(¬e).expect("note JSON serializes")) + .is_err(), + ); + } + + #[test] + fn memory_note_parser_rejects_content_digest_mismatch() { + use super::parse_memory_note_v1; + + let mut note = memory_note_json(); + note["body"] = serde_json::json!("tampered after the digest was computed"); + + assert!( + parse_memory_note_v1(&serde_json::to_vec(¬e).expect("note JSON serializes")) + .is_err(), + ); + } + + #[test] + fn top_level_parsers_reject_oversized_input_before_deserialize() { + use super::{ + MAX_COMPILE_RECORD_BYTES, MAX_EPISODE_PAYLOAD_BYTES, MAX_EVIDENCE_REF_BYTES, + MAX_MEMORY_EVENT_BYTES, MAX_MEMORY_NOTE_BYTES, parse_compile_record_v1, + parse_episode_payload_v1, parse_evidence_ref_v1, parse_memory_event_v1, + parse_memory_note_v1, + }; + + let (_, trusted) = task_payload_fixture(); + let cases = [ + ( + parse_compile_record_v1(&vec![b' '; MAX_COMPILE_RECORD_BYTES + 1]).map(|_| ()), + "CompileRecord.size", + ), + ( + parse_memory_note_v1(&vec![b' '; MAX_MEMORY_NOTE_BYTES + 1]).map(|_| ()), + "MemoryNote.size", + ), + ( + parse_memory_event_v1(&vec![b' '; MAX_MEMORY_EVENT_BYTES + 1]).map(|_| ()), + "MemoryEvent.size", + ), + ( + parse_episode_payload_v1(&vec![b' '; MAX_EPISODE_PAYLOAD_BYTES + 1], &trusted) + .map(|_| ()), + "EpisodePayload.size", + ), + ( + parse_evidence_ref_v1(&vec![b' '; MAX_EVIDENCE_REF_BYTES + 1]).map(|_| ()), + "EvidenceRef.size", + ), + ]; + + for (result, field) in cases { + assert_eq!(result, Err(MemoryContractError::InvalidField { field })); + } + } + + #[test] + fn memory_note_rejects_oversized_metadata() { + use super::parse_memory_note_v1; + + let mut too_many_tags = memory_note_json(); + let tags = (0..129) + .map(|index| format!("tag-{index:03}")) + .collect::>(); + too_many_tags["tags"] = serde_json::to_value(tags).expect("tags serialize"); + refresh_note_digest(&mut too_many_tags); + assert!( + parse_memory_note_v1( + &serde_json::to_vec(&too_many_tags).expect("note JSON serializes"), + ) + .is_err(), + ); + + let mut long_namespace = memory_note_json(); + long_namespace["namespace"] = serde_json::json!("n".repeat(4 * 1024 + 1)); + refresh_note_digest(&mut long_namespace); + assert!( + parse_memory_note_v1( + &serde_json::to_vec(&long_namespace).expect("note JSON serializes"), + ) + .is_err(), + ); + } + + #[test] + fn memory_note_rejects_oversized_scope_identity() { + use super::parse_memory_note_v1; + + let mut note = memory_note_json(); + note["scope"] = serde_json::json!({ + "type": "actor", + "value": "a".repeat(513), + }); + refresh_note_digest(&mut note); + + assert!( + parse_memory_note_v1(&serde_json::to_vec(¬e).expect("note JSON serializes")) + .is_err(), + ); + } + + #[test] + fn memory_event_rejects_oversized_evidence_collection() { + use super::parse_memory_event_v1; + + let evidence_ref = evidence( + EvidenceSourcePlane::AgentRuntime, + EvidenceKind::Evidence, + EvidenceLocatorV1::EventSeq { event_seq: 7 }, + ); + let mut event = memory_event_json(); + event["evidence_refs"] = + serde_json::to_value(vec![evidence_ref; 129]).expect("evidence serializes"); + + assert!( + parse_memory_event_v1(&serde_json::to_vec(&event).expect("event JSON serializes")) + .is_err(), + ); + } + + #[test] + fn evidence_ref_rejects_oversized_locator_metadata() { + let evidence_ref = evidence( + EvidenceSourcePlane::AgentRuntime, + EvidenceKind::Task, + EvidenceLocatorV1::JsonPointer { + pointer: format!("/{}", "p".repeat(4 * 1024)), + }, + ); + + assert!(validate_evidence_ref(&evidence_ref).is_err()); + } + + #[test] + fn compile_record_rejects_oversized_metadata() { + use super::parse_compile_record_v1; + + let mut record = compile_record_json(); + record["producer"] = serde_json::json!("p".repeat(4 * 1024 + 1)); + + assert!( + parse_compile_record_v1( + &serde_json::to_vec(&record).expect("compile record serializes"), + ) + .is_err(), + ); + + let mut unsafe_key_id = compile_record_json(); + unsafe_key_id["idempotency_key"] = + serde_json::json!(format!("hmac-sha256:key\n1:{}", "f".repeat(64))); + assert!( + parse_compile_record_v1( + &serde_json::to_vec(&unsafe_key_id).expect("compile record serializes"), + ) + .is_err(), + ); + } + + #[test] + fn namespace_idempotency_is_reserved_for_aggregate_compilers() { + use super::parse_compile_record_v1; + + let mut episode = compile_record_json(); + episode["origin"] = serde_json::json!("episode_compiler"); + episode["prompt_version"] = serde_json::json!("episode-v1"); + episode["model_id"] = serde_json::json!("synthetic-model"); + episode["idempotency_scope"] = serde_json::json!("namespace"); + assert!( + parse_compile_record_v1( + &serde_json::to_vec(&episode).expect("compile record serializes"), + ) + .is_err(), + "Episode compilation must not deduplicate across Cells", + ); + + let mut consolidation = compile_record_json(); + consolidation["origin"] = serde_json::json!("consolidation"); + consolidation["idempotency_scope"] = serde_json::json!("namespace"); + parse_compile_record_v1( + &serde_json::to_vec(&consolidation).expect("compile record serializes"), + ) + .expect("consolidation may explicitly deduplicate within a namespace"); + } + + #[test] + fn episode_contract_does_not_filter_development_outcomes_or_provenance() { + use super::parse_memory_note_v1; + + for completion_status in [ + CompletionStatus::Completed, + CompletionStatus::Failed, + CompletionStatus::Cancelled, + ] { + for code_change_status in [ + CodeChangeStatus::Changed, + CodeChangeStatus::Unchanged, + CodeChangeStatus::Unknown, + ] { + for trust in [ + MemoryTrust::Verified, + MemoryTrust::RepoEvidence, + MemoryTrust::UserAsserted, + MemoryTrust::ExternalUntrusted, + MemoryTrust::Inferred, + ] { + for confidence in [ + MemoryAnchorConfidence::Low, + MemoryAnchorConfidence::Medium, + MemoryAnchorConfidence::High, + ] { + let (mut payload, mut trusted) = task_payload_fixture(); + payload.completion_status = completion_status; + trusted.completion_status = completion_status; + payload.code_change_status = code_change_status; + trusted.code_change_status = code_change_status; + payload.summary.confidence = Some(confidence); + payload.inferences[0].confidence = Some(confidence); + validate_episode_payload(&payload, &trusted).expect( + "completion, code-change, trust and confidence are not intake filters", + ); + + let mut note = memory_note_json(); + note["episode"] = + serde_json::to_value(payload).expect("Episode serializes"); + note["trust"] = serde_json::to_value(trust).expect("trust serializes"); + note["confidence"] = + serde_json::to_value(confidence).expect("confidence serializes"); + mark_episode_compiler(&mut note); + refresh_note_digest(&mut note); + + parse_memory_note_v1( + &serde_json::to_vec(¬e).expect("MemoryNote serializes"), + ) + .expect("provenance labels do not block a valid M2 Episode contract"); + } + } + } + } + } +} diff --git a/src/internal/ai/mod.rs b/src/internal/ai/mod.rs index 329c5851d..732f55ad8 100644 --- a/src/internal/ai/mod.rs +++ b/src/internal/ai/mod.rs @@ -112,6 +112,9 @@ pub mod intentspec; pub mod libra_vcs; // Model Context Protocol server exposing Libra to MCP-aware clients. pub mod mcp; +// Versioned Agent Memory contracts. Storage and compiler implementations stay +// behind the small crate-private `memory` surface. +pub(crate) mod memory; // Adapter that lets agents participate as nodes in the workflow DAG. pub mod node_adapter; // Phase 0/1/2 orchestrator: intent -> plan -> execute pipeline. From 72d0895d57f7e1d20d436f8dd741b2150149a7f9 Mon Sep 17 00:00:00 2001 From: anduin9527 Date: Mon, 24 Aug 2026 02:39:56 +0800 Subject: [PATCH 02/18] feat(memory): add repository keyed digest Signed-off-by: anduin9527 --- docs/development/plan/plan-20260819.md | 56 +- docs/development/tracing/memory.md | 11 + docs/error-codes.md | 2 + src/command/config.rs | 119 +- src/internal/ai/keyed_digest.rs | 1561 ++++++++++++++++++++++++ src/internal/ai/mod.rs | 1 + src/internal/ai/session/jsonl.rs | 2 - src/internal/config.rs | 51 +- src/internal/config_ownership.rs | 8 + src/internal/vault.rs | 78 +- src/utils/error.rs | 20 +- tests/SERIAL_REGISTRY.tsv | 1 + tests/command/config_test.rs | 160 +++ 13 files changed, 1979 insertions(+), 91 deletions(-) create mode 100644 src/internal/ai/keyed_digest.rs diff --git a/docs/development/plan/plan-20260819.md b/docs/development/plan/plan-20260819.md index 8143d485c..66b211cb6 100644 --- a/docs/development/plan/plan-20260819.md +++ b/docs/development/plan/plan-20260819.md @@ -468,7 +468,7 @@ FTS 表固定 `tokenize='unicode61 remove_diacritics 2'`；`bm25()` 的列权重 - **Status:** Accepted - **Context:** `CompileRecord.idempotency_key`、receipt 的 principal/query digest 与 source 去重需要 keyed HMAC；如果各模块自行取密钥、换代或静默重建，会产生重复 note、不可重放回执或敏感摘要泄漏。 -- **Decision:** M2-01K 在 `src/internal/ai/keyed_digest.rs` 提供唯一 crate-private `RepositoryKeyedDigest`。它通过现有 repository vault + encrypted local config 的固定条目 `memory.keyed_digest.v1` 保存一次生成的 32-byte seed、`generation=1` 与随机 `key_id`；只在确认仓库尚无 Memory 对象/receipt 时生成。每个用途用 ring HKDF-SHA256 和固定 info `libra/memory/idempotency/v1`、`libra/memory/principal/v1`、`libra/memory/query/v1`、`libra/memory/source-input/v1` 派生独立 HMAC-SHA256 key，输出 envelope 总带 `key_id`。首个切片不自动轮换；密钥缺失、解密失败或 generation 未知时不生成替代 key，新 Memory 写入/receipt 追加 fail-closed 为 `LBR-MEMORY-DIGEST-KEY-UNAVAILABLE`，旧 receipt replay 标为 `non_reproducible`，已有权威对象仍可按 OID 只读诊断。 +- **Decision:** M2-01K 在 `src/internal/ai/keyed_digest.rs` 提供唯一 crate-private `RepositoryKeyedDigest`。它通过现有 repository vault + encrypted local config 的固定条目 `memory.keyed_digest.v1` 保存一次生成的 32-byte seed、`generation=1` 与随机 UUIDv4 `key_id`；只在确认仓库尚无 Memory 对象/receipt 时生成。派生配方固定为 `HKDF-Extract(SHA-256, salt="libra/memory/keyed-digest/salt/v1", IKM=seed)`，再按用途执行 `HKDF-Expand(PRK, info, 32)`，最后以所得 key 计算 `HMAC-SHA-256(input)`。四个固定 info 为 `libra/memory/idempotency/v1`、`libra/memory/principal/v1`、`libra/memory/query/v1`、`libra/memory/source-input/v1`；调用方不能自定义 label。输出 envelope 总带 `version/key_id/purpose/digest`。首个切片不自动轮换；密钥缺失、解密失败或 generation 未知时不生成替代 key，新 Memory 写入/receipt 追加统一 fail-closed 为 `LBR-MEMORY-001`，旧 receipt replay 标为 `non_reproducible`，已有权威对象仍可按 OID 只读诊断。 - **Alternatives considered:** 用 repository ID 直接 hash；每个模块保存自己的 secret；密钥丢失后自动重建。它们分别不能隐藏低熵输入、制造多套生命周期，或破坏跨重启幂等。 - **Consequences:** seed/key 只存在于现有加密本地配置，不进入 SQLite 明文字段、Git 对象、日志、remote tier 或 CI artifact；未来轮换必须独立 migration 同时定义跨 generation 幂等和 receipt replay。 - **Revisit when:** 需要显式 rekey、跨设备 Memory publication 或 repository vault 生命周期变化。 @@ -606,7 +606,7 @@ flowchart TB | 任务 | type | axis | recovery | complete | self-contained | AC | VER | landing / prod-files | scope | deps | writeset | release | split-from | exception | |---|---|---|---|---|---|---|---|---|---|---|---|---|---|---| | M2-01 | implementation | contract/domain | revert | yes | yes | 8/8 | 4/8 | 2/5 | M | none | no-overlap | batch child | N/A | N/A | -| M2-01K | implementation | repository keyed-digest lifecycle | forward-only | yes | yes | 7/8 | 4/8 | 3/4 | M | M2-01 | serialized | batch child | N/A | N/A | +| M2-01K | implementation | repository keyed-digest lifecycle | forward-only | yes | yes | 8/8 | 4/8 | 3/8 | M | M2-01 | serialized | batch child | N/A | N/A | | M2-02 | migration | core projection/job schema | forward-only | yes | yes | 8/8 | 6/8 | 4/12 | M | M2-01K | serialized | batch child | N/A | N/A | | M2-02F | migration | FTS5 build/schema capability | forward-only | yes | yes | 8/8 | 6/8 | 4/10 | M | M2-02,DEP-M2-CI-01 | serialized | batch child | M2-02 | N/A | | M2-02R | migration | receipt ledger/retention | forward-only | yes | yes | 8/8 | 6/8 | 3/8 | M | M2-02F | serialized | batch child | M2-02 | N/A | @@ -625,7 +625,7 @@ flowchart TB | M2-14C | implementation | scoped coverage/CI gate | revert | yes | yes | 8/8 | 6/8 | 4/8 | M | M2-14,DEP-M2-CI-01 | serialized | batch child | M2-14 | N/A | | M2-15 | release | release closure | immutable-release | yes | yes | 9/12 | 6/12 | 0/0 | S | M2-14C,DEP-M2-CI-01,DEP-M2-CI-02 | serialized | batch point | N/A | N/A | -**Verification 新增标记：** M2-01、M2-01K、M2-02、M2-02F、M2-02R、M2-04..M2-14 的 Verification 中所有具名测试函数/target 都标记为 `(new)`，并由同卡创建或注册、同步 `tests/INDEX.md`；M2-03 只有 `linear_ref_transaction` 为 `(new)`，其余三个 filter 已存在；M2-14C/M2-15 复用前置测试，M2-14C 新增的两个 scripts 标记为 `(new)`。这一集中标记适用于下列每张卡，不把零命中视为通过。 +**Verification 新增标记：** M2-01、M2-02、M2-02F、M2-02R、M2-04..M2-14 的 Verification 中所有具名测试函数/target 都标记为 `(new)`，并由同卡创建或注册、同步 `tests/INDEX.md`；M2-01K 的接口是 crate-private，具名验证放在 `src/internal/ai/keyed_digest.rs` 的模块测试中，不创建无法访问该接口的集成 target，也不登记 `tests/INDEX.md`；M2-03 只有 `linear_ref_transaction` 为 `(new)`，其余三个 filter 已存在；M2-14C/M2-15 复用前置测试，M2-14C 新增的两个 scripts 标记为 `(new)`。这一集中标记适用于下列每张卡，不把零命中视为通过。 ## 任务卡 @@ -633,7 +633,7 @@ flowchart TB **Task type:** `implementation` -**Lifecycle / Acceptance:** `in-progress` / 空 +**Lifecycle / Acceptance:** `in-progress` / `locally-accepted` **Description:** 在首个持久化实现之前，把 `EpisodePayloadV1`、授权来源前提与确定性准入、canonical digest 和稳定 Cell 规则写入单一规范，并实现不依赖 I/O 的 Rust 领域类型与校验器。 @@ -645,7 +645,7 @@ flowchart TB |---|---| | `memory.md` 已补齐 Episode payload、授权来源/准入边界、稳定 Cell 与共享 selection receipt 合同 | `docs/development/tracing/memory.md:504-633,1380-1407`、`docs/development/gap/mainline.md:267,742-780` | | crate-private `memory` Module 已落 I/O-free 领域类型、校验器与 canonical digest | `src/internal/ai/mod.rs`、`src/internal/ai/memory/{mod.rs,domain.rs,validation.rs,canonical.rs}` | -| 纯逻辑 focused 门共 32 个用例全绿；M2-01 生产代码 clippy 零告警 | 2026-08-22 Rust 1.97.1：domain 4/4、canonical 4/4、validation 24/24；`cargo clippy --lib -- -D warnings` exit 0。仓库级 all-target/all-feature clippy 当前被本卡未修改的 `session/jsonl.rs` 两处 `drop_non_drop` 阻塞，不计作本卡通过证据 | +| 纯逻辑 focused 门共 32 个用例全绿；M2-01 生产代码 clippy 零告警 | 2026-08-22 Rust 1.97.1：domain 4/4、canonical 4/4、validation 24/24；`cargo clippy --lib -- -D warnings` exit 0。2026-08-24 的 stacked tree 已在同一 Rust 1.97.1 容器通过 `cargo clippy --all-targets --all-features -- -D warnings`；C 组全量覆盖仍由 M2-15 承接 | **Acceptance criteria:** @@ -699,40 +699,52 @@ flowchart TB **Task type:** `implementation` -**Lifecycle / Acceptance:** `pending` / 空 +**Lifecycle / Acceptance:** `in-progress` / `locally-accepted` **Description:** 实现 ADR-M2-11 的单一 repository keyed-digest provider，为后续幂等键、身份/query 摘要和 source-input 指纹提供稳定、域分离且 fail-closed 的 HMAC 基础。 **Out of scope:** 显式密钥轮换、跨设备 key publication、Memory 写入和 receipt schema；轮换/publication 延后，消费者由 M2-04、M2-07、M2-02R/M2-12 承接。 -**Current evidence:** 仓库已有 repository vault、encrypted local config 与 ring HKDF；当前 AI/Memory 还没有共享 keyed-digest helper。证据：`src/internal/vault.rs:59-62,698+`、`src/internal/config.rs:1381-1504`，以及核对日 `rg -n "principal_hmac|query_hmac|RepositoryKeyedDigest" src` 零实现命中。 +**Current evidence:** + +| 事实 | 证据 | +|---|---| +| crate-private `RepositoryKeyedDigest` 已隐藏 seed、派生 key、固定 HKDF label、vault/config 生命周期与进程缓存，只向调用方开放闭集用途和 bytes | `src/internal/ai/keyed_digest.rs`、`src/internal/ai/mod.rs` | +| repository-local seed 通过现有 vault 加密保存；并发首次初始化、跨重启稳定、缓存后删除/替换拒绝、旧 vault key fallback、损坏状态与密钥丢失均有真实 SQLite/vault 测试；持久状态漂移会毒化同一缓存项已经发出的 handle | 2026-08-24 Rust 1.97.1：`internal::ai::keyed_digest` 10/10、vault legacy compatibility 1/1、stable error 2/2、cache ownership 1/1 | +| 四个固定 HKDF/HMAC 向量、已有 Memory ref/receipt 时拒绝替代 key、非空 secret probe 的三个具名计划过滤器均实际命中并通过 | 2026-08-24 修订后：三个 `--exact` 过滤器各 1/1；secret probe 用已知 derived key 独立复算 HMAC，并以非空控制样本逐项检查 Git 对象、SQLite/config/vault、operation log、渲染面及“无远端 adapter”结构边界；完整命令见本卡 Verification | +| `memory.keyed_digest.v1` 的公开配置写入口已关闭，只读保持脱敏 | 2026-08-24：owner-managed 精确命令测试 1/1；`config_test::test_config_` 相关回归 55/55，覆盖 set/add/unset/import/remove-section/rename-section、encrypted row、plaintext 损坏行及人类/JSON 读取形式 | +| 本卡 A 组与 family-child 静态门已通过 | 2026-08-24 修订后：`cargo +nightly fmt --all --check` exit 0；`cargo clippy --all-targets --all-features -- -D warnings` exit 0；`compat_serial_registry` 7/7 | +| `cargo test --all` 的 9 个失败不由本卡引入 | 当前树 4473 passed / 9 failed；其中 2 个锁竞争用例单独运行通过，另 7 个在同服务器、同工具链的未修改 `7b2087a11` 基线上逐项复现相同失败；独立 sol/max 归因 review 为 PASS。C 组全量测试仍由 M2-15 在修复基线后统一取得 | +| 独立 Standards / Spec 终审已收敛 | 2026-08-24 两位 sol/max reviewer 对最终 live diff 分别返回 PASS；此前提出的 owner-only 写边界、plaintext 读取脱敏、缓存持久状态复核与旧 handle 毒化、rollback 次级失败传播，以及具名非空 secret surface probe 均已实现并通过固定容器门禁 | **Acceptance criteria:** -- [ ] `RepositoryKeyedDigest` 是 crate-private 深接口，只接受闭集 `DigestPurpose` 与 bytes，调用方不能读取 seed 或传自定义 HKDF label。 -- [ ] 新仓库在确认 Memory ref/receipt 均不存在后生成一次 32-byte seed、`generation=1` 与随机 `key_id`，经现有 vault 加密写入 repository-local config 的 `memory.keyed_digest.v1`；并发初始化只保留一个 winner。 -- [ ] 四个 purpose 使用 ADR-M2-11 固定 HKDF info 派生不同 HMAC-SHA256 key；同 key/purpose/input 跨重启稳定，不同 purpose 输出不同。 -- [ ] 输出 envelope 携带 version/key_id/purpose/digest；日志、错误、Debug 与序列化永不暴露 seed/derived key。 -- [ ] 已存在 Memory/receipt 时 key 缺失、解密失败或 generation 未知统一 fail-closed，不自动换 key；新写入返回稳定错误，旧 receipt replay 标记不可重放。 -- [ ] seed、derived key 和可逆 principal/query 不进入 Git 对象、SQLite 明文、remote tier、operation log 或测试 artifact；secret probe 零命中。 -- [ ] `memory.md`、`docs/error-codes.md` 与 keyed-digest rustdoc 同步 key owner、无自动轮换和丢失恢复语义。 +- [x] `RepositoryKeyedDigest` 是 crate-private 深接口，只接受闭集 `DigestPurpose` 与 bytes，调用方不能读取 seed 或传自定义 HKDF label。 +- [x] 新仓库在确认 Memory ref/receipt 均不存在后生成一次 32-byte seed、`generation=1` 与随机 `key_id`，经现有 vault 加密写入 repository-local config 的 `memory.keyed_digest.v1`；并发初始化只保留一个 winner。 +- [x] `memory.keyed_digest.v1` 只能由 keyed-digest owner 初始化；公开 config set/add/unset/import/remove-section/rename-section 均不能创建、替换、移动或删除它；即使持久行损坏为 plaintext，所有 get/get-all/get-regexp/list 与 JSON 输出也只能显示脱敏占位。 +- [x] 四个 purpose 使用 ADR-M2-11 固定 HKDF info 派生不同 HMAC-SHA256 key；同 key/purpose/input 跨重启稳定，不同 purpose 输出不同。 +- [x] 输出 envelope 携带 version/key_id/purpose/digest；日志、错误、Debug 与序列化永不暴露 seed/derived key。 +- [x] 已存在 Memory/receipt 时 key 缺失、解密失败、generation 未知，或进程缓存与持久配置不一致时由 provider 统一 fail-closed 且不自动换 key；发现不一致会毒化已发出的进程内 handle，使其后续 digest 同样失败；稳定错误供后续 writer 使用，`memory.md` 已冻结旧 receipt replay 的 `non_reproducible` 承接合同。 +- [x] seed、derived key 和可逆 principal/query 不进入 Git 对象、SQLite 明文、remote tier、operation log 或受测渲染面；secret probe 对每个声明表面做具名、非空的结构或控制断言并零命中。 +- [x] `memory.md`、`docs/error-codes.md` 与 keyed-digest rustdoc 同步 key owner、无自动轮换和丢失恢复语义。 **Verification:** -- [ ] `source .env.test && cargo test --lib internal::ai::keyed_digest` -- [ ] `source .env.test && cargo test --test memory_episode_test keyed_digest_domains_are_distinct` -- [ ] `source .env.test && cargo test --test memory_episode_test keyed_digest_missing_existing_repo_fails_closed` -- [ ] `source .env.test && cargo test --test memory_episode_test keyed_digest_secret_probe_zero_leak` +- [x] `source .env.test && LIBRA_SKIP_WEB_BUILD=1 cargo test --lib internal::ai::keyed_digest`（10 passed） +- [x] `source .env.test && LIBRA_SKIP_WEB_BUILD=1 cargo test --lib internal::ai::keyed_digest::tests::keyed_digest_domains_are_distinct_and_match_frozen_vectors -- --exact`（1 passed） +- [x] `source .env.test && LIBRA_SKIP_WEB_BUILD=1 cargo test --lib internal::ai::keyed_digest::tests::keyed_digest_missing_existing_repo_fails_closed -- --exact`（1 passed） +- [x] `source .env.test && LIBRA_SKIP_WEB_BUILD=1 cargo test --lib internal::ai::keyed_digest::tests::keyed_digest_secret_probe_zero_leak -- --exact`（1 passed） +- [x] `source .env.test && LIBRA_SKIP_WEB_BUILD=1 cargo test --test command_test command::config_test::test_config_memory_keyed_digest_is_owner_managed -- --exact`（1 passed） **Dependencies:** M2-01（消费 versioned digest envelope 与错误合同）。 **Deliverables:** N/A。 -**Implementation write set:** `src/internal/ai/keyed_digest.rs`、`src/internal/ai/mod.rs`、`src/internal/{vault.rs,config.rs}` 的最小加密配置 helper、`tests/memory_episode_test.rs`、`docs/development/tracing/memory.md`、`docs/error-codes.md`、`tests/INDEX.md`。 +**Implementation write set:** `src/internal/ai/keyed_digest.rs`、`src/internal/ai/mod.rs`、`src/internal/{vault.rs,config.rs}` 的最小加密配置 helper、`src/internal/config_ownership.rs` 的 process-cache 登记、`src/command/config.rs` 的 owner-only 公开写边界、`src/utils/error.rs` 的稳定错误码、`tests/command/config_test.rs` 与 `tests/SERIAL_REGISTRY.tsv` 的命令回归、`docs/development/tracing/memory.md`、`docs/error-codes.md`；其余验证使用 keyed-digest 模块测试。另含 `src/internal/ai/session/jsonl.rs` 两处 test-only 冗余 `drop(&mut closure)` 删除，用于修复 Rust 1.97.1 `drop_non_drop` 的既有 all-target Clippy 阻塞，不改变运行时行为。 **Release write set:** N/A。 -**Files likely touched:** 4 个生产文件以内、一个集成 target 与随附文档/索引。 +**Files likely touched:** 8 个生产路径（其中 `session/jsonl.rs` 仅 test-only 两行机械清理）、1 个命令测试路径、串行测试登记与随附文档；crate-private 合同由模块测试覆盖。 **Docs and compatibility impact:** `memory.md` 和 error codes 同步；无公开命令或 wire schema。 @@ -742,7 +754,7 @@ flowchart TB **Security and privacy:** 本卡是 key 生命周期边界；测试使用临时 vault/合成输入，任何 plaintext seed 命中都阻断。 -**Performance budget:** key material 每进程至多解密一次并放入不可序列化、无 `Debug` 的私有有界缓存；单次 HKDF/HMAC 为 O(input bytes)，不访问网络。 +**Performance budget:** key material 每进程至多解密一次并放入不可序列化、无 `Debug` 的私有有界缓存；单次摘要只增加两次进程内原子有效性检查并执行 O(input bytes) 的 HMAC，不访问 SQLite 或网络。 **Estimated scope:** `M` @@ -752,7 +764,7 @@ flowchart TB **C/D coverage from:** `M2-15`（C 组版本面覆盖与 D-02 由唯一发布点 M2-15 收口；**D-01 不继承**——本卡自己会推送 main/贡献分支，自行取远端 CI 证据） -**Granularity:** `type=implementation; axis=repository keyed-digest lifecycle; recovery=retain encrypted seed and repair forward once referenced; complete=yes; self-contained=yes; AC=7/8; VER=4/8; landing=3; prod-files=4; scope=M; deps=M2-01; writeset=serialized-at-M2-01; release=batch-release child; split-from=N/A; exception=N/A` +**Granularity:** `type=implementation; axis=repository keyed-digest lifecycle; recovery=retain encrypted seed and repair forward once referenced; complete=yes; self-contained=yes; AC=8/8; VER=4/8; landing=3; prod-files=8; scope=M; deps=M2-01; writeset=serialized-at-M2-01; release=batch-release child; split-from=N/A; exception=N/A` ### Task M2-02：增加 Memory / Episode SQLite schema diff --git a/docs/development/tracing/memory.md b/docs/development/tracing/memory.md index ecc1b55e3..33b3d4533 100644 --- a/docs/development/tracing/memory.md +++ b/docs/development/tracing/memory.md @@ -580,6 +580,17 @@ Memory 遵循与 Libra 其余部分**相同的快照（Snapshot）/ 事件（Eve - 每个 revision 的 producer / prompt / model / policy / input fingerprint 进入 `memory_revision_index` 投影；只在 note 级保存创建 origin 无法定位后续坏 revision，因此不能满足批量 quarantine / recompile 要求。 - `input_hashes` 只能基于已经 redacted、canonicalized 的输入。对可能仍具可识别性的用户文本或 secret-like token，使用 repository-local keyed HMAC 并在字段中标明算法/key generation；普通 SHA-256 不能被当作匿名化。 +##### 仓库级密钥摘要提供器（Repository Keyed Digest） + +Memory 的幂等键、主体摘要、查询摘要和来源输入摘要共用一个仓库级密钥所有者 `RepositoryKeyedDigest`。调用方只能选择 `Idempotency`、`Principal`、`Query`、`SourceInput` 四种封闭用途并提交字节输入；seed、派生 key 和 HKDF label 都不暴露给调用方。 + +- 固定配置条目为 `memory.keyed_digest.v1`。其中保存 `schema_version=1`、`generation=1`、随机 UUIDv4 `key_id` 与 32-byte seed，经现有 repository vault 加密后写入 repository-local `config_kv`。该条目由 keyed-digest provider 单独管理：`libra config` 可以只读查看其脱敏占位，但 set/add/unset、import、remove-section 与 rename-section 都不能创建、替换、移动或删除它。 +- 生成前必须在同一个 SQLite 写事务中确认 `libra/memory/*` 本地分支和 `context_selection_receipt` 行都不存在。并发首次调用由 SQLite 写锁串行化，只允许一个加密条目成为 winner；一旦出现 Memory ref 或 receipt，缺失条目不得自动补发新 seed。 +- 密码学配方固定为 `HKDF-Extract(SHA-256, salt="libra/memory/keyed-digest/salt/v1", IKM=seed)`，再对所选用途执行 `HKDF-Expand(PRK, info, 32)`，最后计算 `HMAC-SHA-256(input)`。四个 info 依次为 `libra/memory/idempotency/v1`、`libra/memory/principal/v1`、`libra/memory/query/v1`、`libra/memory/source-input/v1`。配方或 label 变化必须引入新的 generation 和迁移，不能静默修改。 +- 返回值固定携带 `version`、`key_id`、`purpose` 与 `digest`。seed、派生 key、raw principal/query 不进入 envelope、Debug、错误、日志、Git 对象、SQLite 明文字段、远端层或 CI artifact。 +- 配置缺失、重复、被明文保存、无法解密、schema/generation 未知、repository vault key 不可用，或已缓存 provider 观察到持久配置被删除/替换时，统一停止新的 digest-bearing 写入，并使用稳定错误 `LBR-MEMORY-001`。恢复方式是修复原加密条目或以独立迁移显式引入新 generation；旧 receipt 在原 key 无法恢复时标记 `non_reproducible`。 +- provider 按 canonical repository DB path 与不可变 repository ID 放入私有有界进程缓存，同一进程每仓至多解密一次。每次重新获取 provider 时读取该配置行并校验密文 SHA-256 指纹；一旦发现持久配置被删除或替换，共享有效性标记会立即毒化同一缓存项此前发出的全部 handle，后续摘要统一返回 `LBR-MEMORY-001`，修复配置后需要重启进程才能重新载入。真正的摘要热路径只执行两次进程内有效性检查与本地 O(input bytes) 的 HMAC，不访问 SQLite 或网络。 + #### 4.1.2 引用、链接与 policy 最小契约 为使授权、provenance 与图扩展可实现，以下 supporting types 必须 versioned，不能留成无约束 JSON： diff --git a/docs/error-codes.md b/docs/error-codes.md index aba80d3da..cc10e5763 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -88,6 +88,7 @@ structured report is always present. | `128` | `LBR-REPO-001` | `repo` | Not inside a Libra repository | running repo commands outside `.libra` | | `128` | `LBR-REPO-002` | `repo` | Repository metadata is corrupt or incompatible | missing DB, corrupted metadata | | `128` | `LBR-REPO-003` | `repo` | Repository state blocks the operation | no commits yet, detached state mismatch, missing configured remote | +| `128` | `LBR-MEMORY-001` | `repo` | Repository Memory digest key is missing, invalid, or cannot be decrypted | missing encrypted `memory.keyed_digest.v1`, duplicate/plaintext entry, unsupported generation, unavailable repository vault key, or cached/persisted key mismatch | | `128` | `LBR-WORKTREE-001` | `repo` | Pagination cursor is malformed, foreign, or expired | `libra worktree doctor --cursor ` | | `128` | `LBR-WORKTREE-002` | `repo` | A worktree/workspace scope is corrupt or unreadable, so the diagnosis would be incomplete | `libra worktree doctor` where a `workspace_record` row or the worktree registry cannot be read | | `128` | `LBR-CONFIG-001` | `config` | Global config DB schema is newer than this Libra binary supports | `pull`, `push`, `fetch`, `clone`, or `cloud` would otherwise silently ignore global storage config | @@ -163,6 +164,7 @@ structured report is always present. | `LBR-REPO-001` | Not inside a Libra repository | | `LBR-REPO-002` | Repository metadata is corrupt or incompatible | | `LBR-REPO-003` | Repository state blocks the operation | +| `LBR-MEMORY-001` | Repository Memory digest key is missing, invalid, or cannot be decrypted; restore the original encrypted entry or repair the repository vault before writing new Memory data | | `LBR-WORKTREE-001` | The pagination cursor is malformed or expired; drop it and re-read the first page | | `LBR-WORKTREE-002` | A worktree/workspace scope is corrupt or unreadable; repair it before trusting any diagnostic report | diff --git a/src/command/config.rs b/src/command/config.rs index 4fb5b9d65..8472d7dd4 100644 --- a/src/command/config.rs +++ b/src/command/config.rs @@ -13,7 +13,10 @@ use tokio::sync::Mutex; use crate::{ internal::{ - config::{ConfigKv, ConfigKvEntry, is_sensitive_key, is_vault_internal_key}, + config::{ + ConfigKv, ConfigKvEntry, is_memory_owned_config_key, is_sensitive_key, + is_vault_internal_key, + }, db::{create_database, establish_connection, get_db_conn_instance}, upgrade::settings::{ UPGRADE_MODE_KEY, UpgradeMode, UpgradeSettingsError, read_mode as read_upgrade_mode, @@ -1424,6 +1427,8 @@ async fn handle_set( .with_exit_code(1)); } + reject_memory_owned_config_mutation(key, "written")?; + // `--encrypt` and `--plaintext` are mutually exclusive. config.md (line 77) // classifies this as a CLI usage error (exit 2 in fine mode, 129 in // coarse) — route through `command_usage` so the category matches. @@ -1675,6 +1680,9 @@ async fn render_get_value( scope: ConfigScope, _use_cascade: bool, ) -> CliResult { + if is_memory_owned_config_key(&entry.key) { + return Ok("".to_string()); + } if !entry.encrypted { return Ok(entry.value.clone()); } @@ -1691,6 +1699,21 @@ async fn render_get_value( Ok(decrypted) } +fn render_list_value(entry: &ConfigKvEntry, name_only: bool) -> Option { + if name_only { + return None; + } + if entry.encrypted || is_memory_owned_config_key(&entry.key) { + return Some("".to_string()); + } + let plaintext_warning = if is_sensitive_key(&entry.key) { + " [PLAINTEXT]" + } else { + "" + }; + Some(format!("{}{plaintext_warning}", entry.value)) +} + /// Value type for `--type`/`--bool`/`--int`/`--path` canonicalization on read. #[derive(Clone, Copy, Debug, PartialEq, Eq)] enum ConfigValueType { @@ -2231,20 +2254,9 @@ async fn handle_list( if is_upgrade_namespace_key(&e.key) { continue; } - let plaintext_warning = if !e.encrypted && is_sensitive_key(&e.key) { - " [PLAINTEXT]" - } else { - "" - }; entries.push(ConfigListEntry { key: e.key.clone(), - value: if name_only { - None - } else if e.encrypted { - Some("".to_string()) - } else { - Some(format!("{}{plaintext_warning}", e.value)) - }, + value: render_list_value(&e, name_only), origin: if show_origin { Some(scope_name(s).to_string()) } else { @@ -2303,24 +2315,11 @@ async fn handle_list( .into_iter() // Reserved namespace: suppress any legacy SQLite `upgrade.*` rows. .filter(|e| !is_upgrade_namespace_key(&e.key)) - .map(|e| { - let plaintext_warning = if !e.encrypted && is_sensitive_key(&e.key) { - " [PLAINTEXT]" - } else { - "" - }; - ConfigListEntry { - key: e.key.clone(), - value: if name_only { - None - } else if e.encrypted { - Some("".to_string()) - } else { - Some(format!("{}{plaintext_warning}", e.value)) - }, - origin: None, - encrypted: Some(e.encrypted), - } + .map(|e| ConfigListEntry { + key: e.key.clone(), + value: render_list_value(&e, name_only), + origin: None, + encrypted: Some(e.encrypted), }) .collect(); @@ -2426,6 +2425,8 @@ async fn handle_unset( scope: ConfigScope, output: &OutputConfig, ) -> CliResult<()> { + reject_memory_owned_config_mutation(key, "removed")?; + let count = if all { ScopedConfig::unset_all(scope, key) .await @@ -2474,6 +2475,19 @@ fn config_write_cli_error(message: impl Into) -> CliError { .with_exit_code(128) } +/// Keep repository Memory key material behind its single owner seam. Reads +/// continue through the normal redacted config path; every public mutation is +/// rejected before it can alter row cardinality, encryption state, or value. +fn reject_memory_owned_config_mutation(key: &str, action: &str) -> CliResult<()> { + if !is_memory_owned_config_key(key) { + return Ok(()); + } + Err(CliError::failure(format!( + "configuration key '{key}' is managed by Libra Memory and cannot be {action} through `libra config`" + )) + .with_stable_code(StableErrorCode::RepoStateInvalid)) +} + /// Whether `key` belongs to git section `section`, using Git's section / /// subsection identity rather than a raw prefix. A fully-qualified key splits /// as `section.[subsection.]name` (section = before the FIRST dot, name = after @@ -2547,6 +2561,9 @@ async fn handle_remove_section( .with_exit_code(128), ); } + for key in &keys { + reject_memory_owned_config_mutation(key, "removed")?; + } let mut removed = 0usize; for key in &keys { @@ -2618,6 +2635,20 @@ async fn handle_rename_section( ); } + let destination: Vec = source + .iter() + .map(|entry| { + let name = entry.key.strip_prefix(&old_prefix).unwrap_or(&entry.key); + format!("{new}.{name}") + }) + .collect(); + for entry in &source { + reject_memory_owned_config_mutation(&entry.key, "moved")?; + } + for key in &destination { + reject_memory_owned_config_mutation(key, "created")?; + } + // Refuse to write into a destination section that already exists, so every // re-added key is fresh (preserving the source's exact value + encrypted // flag, and avoiding silent multi-value merges). @@ -2632,12 +2663,10 @@ async fn handle_rename_section( // encrypted source row into it or land a key under a vault/secret namespace // (which direct `set --system` also rejects). if scope == ConfigScope::System { - for e in &source { - let name = e.key.strip_prefix(&old_prefix).unwrap_or(&e.key); - let new_key = format!("{new}.{name}"); + for (e, new_key) in source.iter().zip(&destination) { if e.encrypted || new_key.to_ascii_lowercase().starts_with("vault.") - || is_sensitive_key(&new_key) + || is_sensitive_key(new_key) { return Err(CliError::command_usage( "vault-encrypted secrets are not supported in --system scope", @@ -2649,12 +2678,8 @@ async fn handle_rename_section( } } - for e in &source { - // Exact members all begin with `{old}.`; the remainder is the key name - // under the section (which itself may contain dots for nested names). - let name = e.key.strip_prefix(&old_prefix).unwrap_or(&e.key); - let new_key = format!("{new}.{name}"); - ConfigKv::add_with_conn(&txn, &new_key, &e.value, e.encrypted) + for (e, new_key) in source.iter().zip(&destination) { + ConfigKv::add_with_conn(&txn, new_key, &e.value, e.encrypted) .await .map_err(|err| config_write_cli_error(format!("failed to write '{new_key}': {err}")))?; } @@ -3063,6 +3088,7 @@ async fn import_git_config(scope: ConfigScope) -> Result Result Result)" )); } + if ignored_memory_owned > 0 { + emit_warning(format!( + "ignored {ignored_memory_owned} Memory-owned configuration entries" + )); + } Ok(ConfigImportSummary { scope: scope_name(scope), imported, skipped_duplicates: skipped, ignored_invalid, - ignored_reserved, + ignored_reserved: ignored_reserved + ignored_memory_owned, auto_encrypted, collapsed_multivalue_warnings: collapsed_warnings, }) diff --git a/src/internal/ai/keyed_digest.rs b/src/internal/ai/keyed_digest.rs new file mode 100644 index 000000000..672cbaca2 --- /dev/null +++ b/src/internal/ai/keyed_digest.rs @@ -0,0 +1,1561 @@ +#![allow( + dead_code, + reason = "M2-01K lands the keyed-digest owner before later Memory writers consume it" +)] + +//! Repository-scoped keyed digests for Agent Memory. +//! +//! Callers choose one closed [`DigestPurpose`] and pass bytes. This module +//! owns repository pinning, encrypted seed persistence, domain separation, +//! first-writer-wins initialization, and the process cache. It never exposes +//! the repository seed, derived HMAC keys, or custom HKDF labels. + +use std::{ + collections::HashMap, + fmt, + path::{Path, PathBuf}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; + +use once_cell::sync::Lazy; +use ring::{digest, hkdf, hmac}; +use sea_orm::{ConnectionTrait, DatabaseConnection, Statement}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use tokio::sync::Mutex; +use uuid::Uuid; + +use crate::{ + internal::{ + config::{ConfigKv, MEMORY_KEYED_DIGEST_CONFIG_KEY}, + db, vault, + workspace::RepoIdentity, + }, + utils::error::StableErrorCode, +}; + +const DIGEST_VERSION: u8 = 1; +const DERIVED_KEY_BYTES: usize = 32; +const HKDF_SALT: &[u8] = b"libra/memory/keyed-digest/salt/v1"; +const MEMORY_REF_PREFIX: &str = "libra/memory/"; +const RECEIPT_TABLE: &str = "context_selection_receipt"; +const PERSISTED_SCHEMA_VERSION: u8 = 1; +const PERSISTED_GENERATION: u32 = 1; +const PROCESS_CACHE_CAPACITY: usize = 64; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct RepositoryCacheKey { + canonical_db_path: PathBuf, + repository_id: String, +} + +static REPOSITORY_KEYED_DIGEST_CACHE: Lazy< + Mutex>>, +> = Lazy::new(|| Mutex::new(HashMap::new())); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum DigestPurpose { + Idempotency, + Principal, + Query, + SourceInput, +} + +impl DigestPurpose { + const fn info(self) -> &'static [u8] { + match self { + Self::Idempotency => b"libra/memory/idempotency/v1", + Self::Principal => b"libra/memory/principal/v1", + Self::Query => b"libra/memory/query/v1", + Self::SourceInput => b"libra/memory/source-input/v1", + } + } + + const fn index(self) -> usize { + match self { + Self::Idempotency => 0, + Self::Principal => 1, + Self::Query => 2, + Self::SourceInput => 3, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub(crate) struct KeyedDigestEnvelope { + version: u8, + key_id: Uuid, + purpose: DigestPurpose, + digest: String, +} + +impl KeyedDigestEnvelope { + pub(crate) const fn key_id(&self) -> Uuid { + self.key_id + } + + pub(crate) const fn purpose(&self) -> DigestPurpose { + self.purpose + } + + pub(crate) const fn version(&self) -> u8 { + self.version + } + + pub(crate) fn digest_hex(&self) -> &str { + &self.digest + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum KeyedDigestErrorKind { + RepositoryUnavailable, + RepositoryIdentityInvalid, + StateQueryFailed, + MissingAfterDurableUse, + PlaintextConfig, + DuplicateConfig, + VaultKeyUnavailable, + CiphertextInvalid, + PayloadInvalid, + UnsupportedSchema, + UnsupportedGeneration, + RandomUnavailable, + PersistFailed, + PersistedStateChanged, + CacheCapacity, + Derivation, +} + +impl fmt::Display for KeyedDigestErrorKind { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + let label = match self { + Self::RepositoryUnavailable => "repository database unavailable", + Self::RepositoryIdentityInvalid => "repository identity missing or ambiguous", + Self::StateQueryFailed => "repository Memory state could not be inspected", + Self::MissingAfterDurableUse => "key is missing after durable Memory state exists", + Self::PlaintextConfig => "stored key payload is not marked encrypted", + Self::DuplicateConfig => "multiple stored key payloads exist", + Self::VaultKeyUnavailable => "repository vault key unavailable", + Self::CiphertextInvalid => "stored key payload cannot be decrypted", + Self::PayloadInvalid => "stored key payload is malformed", + Self::UnsupportedSchema => "stored key schema version is unsupported", + Self::UnsupportedGeneration => "stored key generation is unsupported", + Self::RandomUnavailable => "secure randomness unavailable", + Self::PersistFailed => "encrypted key payload could not be persisted", + Self::PersistedStateChanged => "persisted key changed after it was cached", + Self::CacheCapacity => "process cache capacity reached", + Self::Derivation => "HKDF key derivation failed", + }; + formatter.write_str(label) + } +} + +#[derive(Debug, Error)] +#[error( + "repository Memory digest key is unavailable ({kind}); restore 'memory.keyed_digest.v1' from repository-local encrypted configuration or repair the repository vault{rollback_context}" +)] +pub(crate) struct KeyedDigestError { + kind: KeyedDigestErrorKind, + rollback_context: &'static str, +} + +impl KeyedDigestError { + const fn new(kind: KeyedDigestErrorKind) -> Self { + Self { + kind, + rollback_context: "", + } + } + + const fn with_rollback_failure(mut self) -> Self { + self.rollback_context = "; repository transaction rollback also failed, so inspect repository state before retrying"; + self + } + + pub(crate) const fn kind(&self) -> KeyedDigestErrorKind { + self.kind + } + + pub(crate) const fn stable_code(&self) -> StableErrorCode { + StableErrorCode::MemoryDigestKeyUnavailable + } +} + +#[derive(Serialize, Deserialize)] +struct PersistedDigestKeyV1 { + schema_version: u8, + generation: u32, + key_id: Uuid, + seed_hex: String, +} + +struct HmacSha256KeyLength; + +impl hkdf::KeyType for HmacSha256KeyLength { + fn len(&self) -> usize { + DERIVED_KEY_BYTES + } +} + +pub(crate) struct RepositoryKeyedDigest { + key_id: Uuid, + keys: [hmac::Key; 4], + persisted_config_fingerprint: [u8; 32], + valid: AtomicBool, +} + +impl fmt::Debug for RepositoryKeyedDigest { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("RepositoryKeyedDigest") + .field("key_id", &self.key_id) + .finish_non_exhaustive() + } +} + +impl RepositoryKeyedDigest { + pub(crate) async fn load_or_initialize( + repository_db_path: &Path, + ) -> Result, KeyedDigestError> { + let canonical_db_path = tokio::fs::canonicalize(repository_db_path) + .await + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::RepositoryUnavailable))?; + let database = db::get_db_conn_instance_for_path(&canonical_db_path) + .await + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::RepositoryUnavailable))?; + let repository_id = repository_id(&database).await?; + let cache_key = RepositoryCacheKey { + canonical_db_path: canonical_db_path.clone(), + repository_id, + }; + + // Holding this async mutex across the cold load is intentional: it is + // the single-flight guard that prevents duplicate decrypts for one + // repository. Loads are local-only and bounded; normal processes own + // one repository identity. + let mut cache = REPOSITORY_KEYED_DIGEST_CACHE.lock().await; + if let Some(provider) = cache.get(&cache_key) { + validate_cached_provider(&database, provider).await?; + return Ok(Arc::clone(provider)); + } + if cache.len() >= PROCESS_CACHE_CAPACITY { + return Err(KeyedDigestError::new(KeyedDigestErrorKind::CacheCapacity)); + } + + let provider = + Arc::new(Self::load_or_initialize_uncached(&database, &canonical_db_path).await?); + cache.insert(cache_key, Arc::clone(&provider)); + Ok(provider) + } + + async fn load_or_initialize_uncached( + database: &DatabaseConnection, + repository_db_path: &Path, + ) -> Result { + let transaction = db::begin_write_transaction(database) + .await + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::StateQueryFailed))?; + + let result = + Self::load_or_initialize_in_transaction(&transaction, repository_db_path).await; + match result { + Ok(provider) => { + transaction + .commit() + .await + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::PersistFailed))?; + Ok(provider) + } + Err(error) => match transaction.rollback().await { + Ok(()) => Err(error), + Err(_) => Err(error.with_rollback_failure()), + }, + } + } + + async fn load_or_initialize_in_transaction( + database: &C, + repository_db_path: &Path, + ) -> Result { + let rows = ConfigKv::get_all_with_conn(database, MEMORY_KEYED_DIGEST_CONFIG_KEY) + .await + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::StateQueryFailed))?; + match rows.as_slice() { + [row] => load_persisted_provider(database, repository_db_path, row).await, + [] => { + ensure_initialization_is_eligible(database).await?; + initialize_provider(database, repository_db_path).await + } + _ => Err(KeyedDigestError::new(KeyedDigestErrorKind::DuplicateConfig)), + } + } + + fn from_seed( + key_id: Uuid, + seed: [u8; 32], + persisted_config_fingerprint: [u8; 32], + ) -> Result { + let salt = hkdf::Salt::new(hkdf::HKDF_SHA256, HKDF_SALT); + let pseudo_random_key = salt.extract(&seed); + let mut derived_keys = Vec::with_capacity(4); + + for purpose in [ + DigestPurpose::Idempotency, + DigestPurpose::Principal, + DigestPurpose::Query, + DigestPurpose::SourceInput, + ] { + let info = [purpose.info()]; + let output = pseudo_random_key + .expand(&info, HmacSha256KeyLength) + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::Derivation))?; + let mut key_bytes = [0_u8; DERIVED_KEY_BYTES]; + output + .fill(&mut key_bytes) + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::Derivation))?; + derived_keys.push(hmac::Key::new(hmac::HMAC_SHA256, &key_bytes)); + } + + let keys: [hmac::Key; 4] = derived_keys + .try_into() + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::Derivation))?; + Ok(Self { + key_id, + keys, + persisted_config_fingerprint, + valid: AtomicBool::new(true), + }) + } + + pub(crate) const fn key_id(&self) -> Uuid { + self.key_id + } + + fn invalidate(&self) { + self.valid.store(false, Ordering::Release); + } + + fn ensure_valid(&self) -> Result<(), KeyedDigestError> { + if self.valid.load(Ordering::Acquire) { + Ok(()) + } else { + Err(KeyedDigestError::new( + KeyedDigestErrorKind::PersistedStateChanged, + )) + } + } + + pub(crate) fn digest( + &self, + purpose: DigestPurpose, + input: &[u8], + ) -> Result { + self.ensure_valid()?; + let tag = hmac::sign(&self.keys[purpose.index()], input); + let envelope = KeyedDigestEnvelope { + version: DIGEST_VERSION, + key_id: self.key_id, + purpose, + digest: hex::encode(tag.as_ref()), + }; + // A concurrent mismatch observation can invalidate the shared handle + // while HMAC is running. Check again before releasing the envelope. + self.ensure_valid()?; + Ok(envelope) + } +} + +fn config_fingerprint(ciphertext_hex: &str) -> [u8; 32] { + digest::digest(&digest::SHA256, ciphertext_hex.as_bytes()) + .as_ref() + .try_into() + // INVARIANT: ring's SHA-256 algorithm always emits exactly 32 bytes. + .expect("SHA-256 output length is fixed at 32 bytes") +} + +async fn validate_cached_provider( + database: &C, + provider: &RepositoryKeyedDigest, +) -> Result<(), KeyedDigestError> { + provider.ensure_valid()?; + let validation = match ConfigKv::get_all_with_conn(database, MEMORY_KEYED_DIGEST_CONFIG_KEY) + .await + { + Err(_) => Err(KeyedDigestError::new( + KeyedDigestErrorKind::StateQueryFailed, + )), + Ok(rows) => match rows.as_slice() { + [] => Err(KeyedDigestError::new( + KeyedDigestErrorKind::MissingAfterDurableUse, + )), + [row] if !row.encrypted => { + Err(KeyedDigestError::new(KeyedDigestErrorKind::PlaintextConfig)) + } + [row] if config_fingerprint(&row.value) != provider.persisted_config_fingerprint => { + Err(KeyedDigestError::new( + KeyedDigestErrorKind::PersistedStateChanged, + )) + } + [_] => Ok(()), + _ => Err(KeyedDigestError::new(KeyedDigestErrorKind::DuplicateConfig)), + }, + }; + if validation.is_err() { + provider.invalidate(); + } + validation +} + +async fn repository_id(database: &DatabaseConnection) -> Result { + let identity = RepoIdentity::resolve(database) + .await + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::RepositoryIdentityInvalid))?; + let rows = ConfigKv::get_all_with_conn(database, "libra.repoid") + .await + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::RepositoryIdentityInvalid))?; + let [row] = rows.as_slice() else { + return Err(KeyedDigestError::new( + KeyedDigestErrorKind::RepositoryIdentityInvalid, + )); + }; + if row.encrypted { + return Err(KeyedDigestError::new( + KeyedDigestErrorKind::RepositoryIdentityInvalid, + )); + } + Ok(identity.as_str().to_owned()) +} + +async fn ensure_initialization_is_eligible( + database: &C, +) -> Result<(), KeyedDigestError> { + let backend = database.get_database_backend(); + let memory_ref = database + .query_one_raw(Statement::from_sql_and_values( + backend, + "SELECT 1 FROM reference + WHERE kind = 'Branch' AND remote IS NULL + AND (name = ? OR name LIKE ?) + LIMIT 1", + [ + "libra/memory/repo".into(), + format!("{MEMORY_REF_PREFIX}%").into(), + ], + )) + .await + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::StateQueryFailed))?; + if memory_ref.is_some() { + return Err(KeyedDigestError::new( + KeyedDigestErrorKind::MissingAfterDurableUse, + )); + } + + let receipt_table = database + .query_one_raw(Statement::from_sql_and_values( + backend, + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ? LIMIT 1", + [RECEIPT_TABLE.into()], + )) + .await + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::StateQueryFailed))?; + if receipt_table.is_some() { + let receipt = database + .query_one_raw(Statement::from_string( + backend, + format!("SELECT 1 FROM {RECEIPT_TABLE} LIMIT 1"), + )) + .await + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::StateQueryFailed))?; + if receipt.is_some() { + return Err(KeyedDigestError::new( + KeyedDigestErrorKind::MissingAfterDurableUse, + )); + } + } + Ok(()) +} + +async fn initialize_provider( + database: &C, + repository_db_path: &Path, +) -> Result { + use ring::rand::{SecureRandom, SystemRandom}; + + let unseal_key = vault::load_unseal_key_for_db_path_with_conn(repository_db_path, database) + .await + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::VaultKeyUnavailable))?; + let mut seed = [0_u8; 32]; + SystemRandom::new() + .fill(&mut seed) + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::RandomUnavailable))?; + let key_id = Uuid::new_v4(); + let payload = PersistedDigestKeyV1 { + schema_version: PERSISTED_SCHEMA_VERSION, + generation: PERSISTED_GENERATION, + key_id, + seed_hex: hex::encode(seed), + }; + let plaintext = serde_json::to_vec(&payload) + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::PayloadInvalid))?; + let ciphertext = vault::encrypt_token(&unseal_key, &plaintext) + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::CiphertextInvalid))?; + let ciphertext_hex = hex::encode(ciphertext); + let inserted = ConfigKv::insert_vault_internal_if_absent_with_conn( + database, + MEMORY_KEYED_DIGEST_CONFIG_KEY, + &ciphertext_hex, + ) + .await + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::PersistFailed))?; + if !inserted { + return Err(KeyedDigestError::new(KeyedDigestErrorKind::PersistFailed)); + } + RepositoryKeyedDigest::from_seed(key_id, seed, config_fingerprint(&ciphertext_hex)) +} + +async fn load_persisted_provider( + database: &C, + repository_db_path: &Path, + row: &crate::internal::config::ConfigKvEntry, +) -> Result { + if !row.encrypted { + return Err(KeyedDigestError::new(KeyedDigestErrorKind::PlaintextConfig)); + } + let unseal_key = vault::load_unseal_key_for_db_path_with_conn(repository_db_path, database) + .await + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::VaultKeyUnavailable))?; + let ciphertext = hex::decode(&row.value) + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::CiphertextInvalid))?; + let plaintext = vault::decrypt_token(&unseal_key, &ciphertext) + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::CiphertextInvalid))?; + let payload: PersistedDigestKeyV1 = serde_json::from_str(&plaintext) + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::PayloadInvalid))?; + if payload.schema_version != PERSISTED_SCHEMA_VERSION { + return Err(KeyedDigestError::new( + KeyedDigestErrorKind::UnsupportedSchema, + )); + } + if payload.generation != PERSISTED_GENERATION { + return Err(KeyedDigestError::new( + KeyedDigestErrorKind::UnsupportedGeneration, + )); + } + if payload.key_id.get_version_num() != 4 + || payload.seed_hex.len() != 64 + || !payload + .seed_hex + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(KeyedDigestError::new(KeyedDigestErrorKind::PayloadInvalid)); + } + let seed_bytes = hex::decode(payload.seed_hex) + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::PayloadInvalid))?; + let seed: [u8; 32] = seed_bytes + .try_into() + .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::PayloadInvalid))?; + RepositoryKeyedDigest::from_seed(payload.key_id, seed, config_fingerprint(&row.value)) +} + +#[cfg(test)] +async fn reset_digest_cache_for_tests() { + REPOSITORY_KEYED_DIGEST_CACHE.lock().await.clear(); +} + +#[cfg(test)] +mod tests { + use std::{ + collections::{BTreeMap, HashSet}, + ffi::OsString, + fs, + path::{Path, PathBuf}, + time::Duration, + }; + + use ring::hmac; + use sea_orm::{ConnectionTrait, Statement}; + use serial_test::serial; + use tempfile::TempDir; + use uuid::Uuid; + use walkdir::WalkDir; + + use super::{ + DigestPurpose, KeyedDigestError, KeyedDigestErrorKind, PERSISTED_GENERATION, + PERSISTED_SCHEMA_VERSION, PersistedDigestKeyV1, RepositoryKeyedDigest, config_fingerprint, + reset_digest_cache_for_tests, + }; + use crate::{ + internal::{ + config::{self, ConfigKv, MEMORY_KEYED_DIGEST_CONFIG_KEY as CONFIG_KEY}, + db, vault, + }, + utils::error::StableErrorCode, + }; + + const TEST_VAULT_KEY: [u8; 32] = [0x42; 32]; + + struct TestRepository { + _repo: TempDir, + _home: TempDir, + previous_test_home: Option, + repo_root: PathBuf, + home_root: PathBuf, + repo_id: String, + vault_key: Vec, + db_path: PathBuf, + } + + impl TestRepository { + async fn new() -> Self { + Self::new_inner(false).await + } + + async fn new_with_real_vault() -> Self { + Self::new_inner(true).await + } + + async fn new_inner(real_vault: bool) -> Self { + let repo = tempfile::tempdir().expect("temporary repository must be created"); + let home = tempfile::tempdir().expect("temporary home must be created"); + let previous_test_home = std::env::var_os("LIBRA_TEST_HOME"); + // SAFETY: every test using this fixture is marked `#[serial]`, and + // the previous value is restored when the fixture is dropped. + unsafe { std::env::set_var("LIBRA_TEST_HOME", home.path()) }; + + let storage = repo.path().join(".libra"); + tokio::fs::create_dir_all(&storage) + .await + .expect("repository storage must be created"); + let db_path = storage.join("libra.db"); + let conn = db::create_database(&db_path.to_string_lossy()) + .await + .expect("repository database must be created"); + let repo_id = Uuid::new_v4().to_string(); + ConfigKv::set_with_conn(&conn, "libra.repoid", &repo_id, false) + .await + .expect("repository identity must be stored"); + + let vault_key = if real_vault { + vault::init_vault(&storage) + .await + .expect("repository vault must initialize") + .0 + } else { + TEST_VAULT_KEY.to_vec() + }; + + let key_path = home.path().join(".libra/vault-keys").join(&repo_id); + tokio::fs::create_dir_all( + key_path + .parent() + .expect("vault key path must have a parent"), + ) + .await + .expect("vault key directory must be created"); + tokio::fs::write(&key_path, hex::encode(&vault_key)) + .await + .expect("repository vault key must be stored"); + + Self { + repo_root: repo.path().to_path_buf(), + home_root: home.path().to_path_buf(), + repo_id, + vault_key, + _repo: repo, + _home: home, + previous_test_home, + db_path, + } + } + + async fn connection(&self) -> sea_orm::DatabaseConnection { + db::get_db_conn_instance_for_path(&self.db_path) + .await + .expect("repository database must open") + } + + async fn set_digest_config(&self, value: &str, encrypted: bool) { + let conn = self.connection().await; + ConfigKv::set_with_conn(&conn, CONFIG_KEY, value, encrypted) + .await + .expect("digest config must be stored"); + } + + async fn add_digest_config(&self, value: &str, encrypted: bool) { + let conn = self.connection().await; + ConfigKv::add_with_conn(&conn, CONFIG_KEY, value, encrypted) + .await + .expect("additional digest config must be stored"); + } + + async fn store_payload(&self, payload: &PersistedDigestKeyV1) { + let plaintext = serde_json::to_vec(payload).expect("payload must serialize"); + let ciphertext = + vault::encrypt_token(&self.vault_key, &plaintext).expect("payload must encrypt"); + self.set_digest_config(&hex::encode(ciphertext), true).await; + } + + async fn insert_memory_ref(&self) { + let conn = self.connection().await; + conn.execute_raw(Statement::from_string( + conn.get_database_backend(), + "INSERT INTO reference (name, kind, `commit`, remote, worktree_id) \ + VALUES ('libra/memory/repo', 'Branch', NULL, NULL, NULL)", + )) + .await + .expect("Memory ref must be inserted"); + } + + async fn insert_receipt(&self) { + let conn = self.connection().await; + let backend = conn.get_database_backend(); + conn.execute_raw(Statement::from_string( + backend, + "CREATE TABLE context_selection_receipt (id INTEGER PRIMARY KEY)", + )) + .await + .expect("receipt table must be created"); + conn.execute_raw(Statement::from_string( + backend, + "INSERT INTO context_selection_receipt (id) VALUES (1)", + )) + .await + .expect("receipt must be inserted"); + } + + async fn remove_home_vault_key(&self) { + tokio::fs::remove_file(self.home_root.join(".libra/vault-keys").join(&self.repo_id)) + .await + .expect("home vault key must be removed"); + } + + async fn cleanup(&self) { + reset_digest_cache_for_tests().await; + db::reset_db_conn_instance_for_path(&self.db_path).await; + } + } + + impl Drop for TestRepository { + fn drop(&mut self) { + // SAFETY: the fixture's tests are serialized and this restores the + // process environment to the value observed during construction. + unsafe { + match self.previous_test_home.take() { + Some(value) => std::env::set_var("LIBRA_TEST_HOME", value), + None => std::env::remove_var("LIBRA_TEST_HOME"), + } + } + } + } + + fn inventory_files(root: &Path) -> BTreeMap> { + let mut inventory = BTreeMap::new(); + for entry in WalkDir::new(root) { + let entry = entry.expect("persistence surface must be walkable"); + if entry.file_type().is_file() { + let relative_path = entry + .path() + .strip_prefix(root) + .expect("inventory entry must remain below its root") + .to_path_buf(); + inventory.insert( + relative_path, + fs::read(entry.path()).expect("persistence surface must be readable"), + ); + } + } + inventory + } + + struct WorkingDirectoryGuard(PathBuf); + + impl WorkingDirectoryGuard { + fn enter(path: &std::path::Path) -> Self { + let previous = std::env::current_dir().expect("working directory must resolve"); + std::env::set_current_dir(path).expect("working directory must change"); + Self(previous) + } + } + + impl Drop for WorkingDirectoryGuard { + fn drop(&mut self) { + std::env::set_current_dir(&self.0).expect("working directory must be restored"); + } + } + + #[test] + fn keyed_digest_domains_are_distinct_and_match_frozen_vectors() { + let seed: [u8; 32] = std::array::from_fn(|index| index as u8); + let provider = RepositoryKeyedDigest::from_seed( + Uuid::parse_str("123e4567-e89b-42d3-a456-426614174000") + .expect("fixed UUIDv4 must parse"), + seed, + config_fingerprint("frozen-test-vector"), + ) + .expect("fixed seed must construct a provider"); + + let input = b"libra memory vector"; + let cases = [ + ( + DigestPurpose::Idempotency, + "c113c6657398689378eda6ca44806a480c1bd02be1861edaf16fb4206a5cba4e", + ), + ( + DigestPurpose::Principal, + "a2cdadc40035b5ea669a39256e3011f4e5d06110f72cb1dcef15b909b02a3428", + ), + ( + DigestPurpose::Query, + "4635c232c8e0d3fe583958bd98270695ef87e0548a7da2b8e3acb90c34126243", + ), + ( + DigestPurpose::SourceInput, + "829132f11015ee8e47dba36fc775f76faa1f70feb8f393be8a0af54e8f4c4b49", + ), + ]; + + for &(purpose, expected) in &cases { + let digest = provider + .digest(purpose, input) + .expect("a fresh fixed-vector provider must remain valid"); + assert_eq!(digest.digest_hex(), expected); + assert_eq!(digest.purpose(), purpose); + assert_eq!(digest.key_id(), provider.key_id()); + assert_eq!(digest.version(), 1); + } + + let distinct: HashSet<_> = cases + .into_iter() + .map(|(purpose, _)| { + provider + .digest(purpose, input) + .expect("a fresh fixed-vector provider must remain valid") + .digest_hex() + .to_string() + }) + .collect(); + assert_eq!(distinct.len(), 4); + + let envelope = provider + .digest(DigestPurpose::Query, input) + .expect("a fresh fixed-vector provider must remain valid"); + let json = serde_json::to_value(&envelope).expect("envelope must serialize"); + assert_eq!(json["version"], 1); + assert_eq!(json["key_id"], provider.key_id().to_string()); + assert_eq!(json["purpose"], "query"); + assert_eq!(json["digest"], envelope.digest_hex()); + assert!(json.get("seed").is_none()); + } + + #[test] + fn keyed_digest_error_contract_is_stable_and_actionable() { + let error = KeyedDigestError::new(KeyedDigestErrorKind::UnsupportedGeneration); + assert_eq!( + error.to_string(), + "repository Memory digest key is unavailable (stored key generation is unsupported); \ + restore 'memory.keyed_digest.v1' from repository-local encrypted configuration or \ + repair the repository vault" + ); + assert_eq!( + error.stable_code(), + StableErrorCode::MemoryDigestKeyUnavailable + ); + assert_eq!(error.stable_code().as_str(), "LBR-MEMORY-001"); + + let rollback_error = + KeyedDigestError::new(KeyedDigestErrorKind::PersistFailed).with_rollback_failure(); + assert_eq!(rollback_error.kind(), KeyedDigestErrorKind::PersistFailed); + assert_eq!( + rollback_error.stable_code(), + StableErrorCode::MemoryDigestKeyUnavailable + ); + assert!( + rollback_error + .to_string() + .contains("repository transaction rollback also failed") + ); + assert!( + rollback_error + .to_string() + .contains("inspect repository state before retrying") + ); + } + + #[tokio::test] + #[serial] + async fn encrypted_key_survives_provider_reload() { + reset_digest_cache_for_tests().await; + let repository = TestRepository::new_with_real_vault().await; + + let first = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect("first load must initialize the repository digest key"); + let first_digest = first + .digest(DigestPurpose::Query, b"same query") + .expect("fresh provider must digest"); + + reset_digest_cache_for_tests().await; + let second = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect("second load must decrypt the persisted repository digest key"); + let second_digest = second + .digest(DigestPurpose::Query, b"same query") + .expect("reloaded provider must digest"); + + assert_eq!(second.key_id(), first.key_id()); + assert_eq!(second_digest, first_digest); + + let rows = ConfigKv::get_all_with_conn(&repository.connection().await, CONFIG_KEY) + .await + .expect("digest key rows must be readable"); + assert_eq!(rows.len(), 1); + assert!(rows[0].encrypted); + assert!(config::is_sensitive_key(CONFIG_KEY)); + assert!(config::is_vault_internal_key(CONFIG_KEY)); + + repository.cleanup().await; + } + + #[tokio::test] + #[serial] + async fn cached_provider_rejects_deleted_or_replaced_persisted_key() { + reset_digest_cache_for_tests().await; + { + let repository = TestRepository::new().await; + let cached = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect("first load must populate the process cache"); + + ConfigKv::unset_all_with_conn(&repository.connection().await, CONFIG_KEY) + .await + .expect("fixture must delete the persisted owner row"); + let deleted_error = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect_err("a cache hit must reject a deleted persisted key"); + assert_eq!( + deleted_error.kind(), + KeyedDigestErrorKind::MissingAfterDurableUse + ); + let stale_handle_error = cached + .digest(DigestPurpose::Query, b"must not sign after deletion") + .expect_err("a previously issued handle must be poisoned after deletion"); + assert_eq!( + stale_handle_error.kind(), + KeyedDigestErrorKind::PersistedStateChanged + ); + + repository.cleanup().await; + } + + { + let repository = TestRepository::new().await; + let cached = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect("first load must populate the process cache"); + let replacement_id = Uuid::new_v4(); + assert_ne!(replacement_id, cached.key_id()); + repository + .store_payload(&PersistedDigestKeyV1 { + schema_version: PERSISTED_SCHEMA_VERSION, + generation: PERSISTED_GENERATION, + key_id: replacement_id, + seed_hex: hex::encode([0x7c_u8; 32]), + }) + .await; + + let replaced_error = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect_err("a cache hit must reject a replaced persisted key"); + assert_eq!( + replaced_error.kind(), + KeyedDigestErrorKind::PersistedStateChanged + ); + let stale_handle_error = cached + .digest(DigestPurpose::Query, b"must not sign after replacement") + .expect_err("a previously issued handle must be poisoned after replacement"); + assert_eq!( + stale_handle_error.kind(), + KeyedDigestErrorKind::PersistedStateChanged + ); + + repository.cleanup().await; + } + } + + #[tokio::test] + #[serial] + async fn concurrent_initialization_keeps_one_persisted_winner() { + reset_digest_cache_for_tests().await; + let repository = TestRepository::new().await; + let mut tasks = Vec::new(); + + for _ in 0..32 { + let db_path = repository.db_path.clone(); + tasks.push(tokio::spawn(async move { + let conn = db::open_connection_without_schema_management( + db_path.to_string_lossy().as_ref(), + Duration::from_secs(30), + ) + .await + .expect("independent repository connection must open"); + RepositoryKeyedDigest::load_or_initialize_uncached(&conn, &db_path) + .await + .map(|provider| provider.key_id()) + })); + } + + let mut key_ids = Vec::new(); + for task in tasks { + key_ids.push( + task.await + .expect("initializer task must not panic") + .expect("initializer must converge on the winner"), + ); + } + assert!(key_ids.iter().all(|key_id| *key_id == key_ids[0])); + + let rows = ConfigKv::get_all_with_conn(&repository.connection().await, CONFIG_KEY) + .await + .expect("digest key rows must be readable"); + assert_eq!(rows.len(), 1); + assert!(rows[0].encrypted); + + repository.cleanup().await; + } + + #[tokio::test] + #[serial] + async fn keyed_digest_missing_existing_repo_fails_closed() { + reset_digest_cache_for_tests().await; + { + let repository = TestRepository::new().await; + repository.insert_memory_ref().await; + let error = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect_err("a Memory ref must prevent silent key replacement"); + assert_eq!(error.kind(), KeyedDigestErrorKind::MissingAfterDurableUse); + assert_eq!( + error.stable_code(), + StableErrorCode::MemoryDigestKeyUnavailable + ); + let rows = ConfigKv::get_all_with_conn(&repository.connection().await, CONFIG_KEY) + .await + .expect("digest rows must be inspectable"); + assert!(rows.is_empty()); + repository.cleanup().await; + } + + { + let repository = TestRepository::new().await; + repository.insert_receipt().await; + let error = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect_err("a selection receipt must prevent silent key replacement"); + assert_eq!(error.kind(), KeyedDigestErrorKind::MissingAfterDurableUse); + let rows = ConfigKv::get_all_with_conn(&repository.connection().await, CONFIG_KEY) + .await + .expect("digest rows must be inspectable"); + assert!(rows.is_empty()); + repository.cleanup().await; + } + } + + #[tokio::test] + #[serial] + async fn invalid_persisted_states_fail_closed_without_replacement() { + reset_digest_cache_for_tests().await; + { + let repository = TestRepository::new().await; + repository.set_digest_config("plaintext-seed", false).await; + let error = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect_err("plaintext digest config must be rejected"); + assert_eq!(error.kind(), KeyedDigestErrorKind::PlaintextConfig); + repository.cleanup().await; + } + + { + let repository = TestRepository::new().await; + repository.set_digest_config("00", true).await; + repository.add_digest_config("11", true).await; + let error = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect_err("duplicate digest config must be rejected"); + assert_eq!(error.kind(), KeyedDigestErrorKind::DuplicateConfig); + repository.cleanup().await; + } + + { + let repository = TestRepository::new().await; + repository.set_digest_config("not-hex", true).await; + let error = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect_err("invalid ciphertext must be rejected"); + assert_eq!(error.kind(), KeyedDigestErrorKind::CiphertextInvalid); + repository.cleanup().await; + } + + { + let repository = TestRepository::new().await; + let valid_payload = serde_json::to_vec(&PersistedDigestKeyV1 { + schema_version: PERSISTED_SCHEMA_VERSION, + generation: PERSISTED_GENERATION, + key_id: Uuid::new_v4(), + seed_hex: hex::encode([0x19_u8; 32]), + }) + .expect("payload must serialize"); + let wrong_ciphertext = vault::encrypt_token(&[0x99_u8; 32], &valid_payload) + .expect("payload must encrypt with the wrong key"); + repository + .set_digest_config(&hex::encode(wrong_ciphertext), true) + .await; + let error = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect_err("AES-GCM authentication failure must be rejected"); + assert_eq!(error.kind(), KeyedDigestErrorKind::CiphertextInvalid); + repository.cleanup().await; + } + + { + let repository = TestRepository::new().await; + let malformed_ciphertext = vault::encrypt_token(&repository.vault_key, b"{}") + .expect("malformed payload must still encrypt"); + repository + .set_digest_config(&hex::encode(malformed_ciphertext), true) + .await; + let error = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect_err("malformed decrypted JSON must be rejected"); + assert_eq!(error.kind(), KeyedDigestErrorKind::PayloadInvalid); + repository.cleanup().await; + } + + { + let repository = TestRepository::new().await; + repository + .store_payload(&PersistedDigestKeyV1 { + schema_version: PERSISTED_SCHEMA_VERSION, + generation: PERSISTED_GENERATION, + key_id: Uuid::parse_str("018f6f77-20c3-7d61-9d9b-94b63ce9a243") + .expect("fixed UUIDv7 must parse"), + seed_hex: hex::encode([0x28_u8; 32]), + }) + .await; + let error = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect_err("non-v4 key identity must be rejected"); + assert_eq!(error.kind(), KeyedDigestErrorKind::PayloadInvalid); + repository.cleanup().await; + } + + for (schema_version, generation, expected) in [ + ( + 2, + PERSISTED_GENERATION, + KeyedDigestErrorKind::UnsupportedSchema, + ), + ( + PERSISTED_SCHEMA_VERSION, + 2, + KeyedDigestErrorKind::UnsupportedGeneration, + ), + ] { + let repository = TestRepository::new().await; + repository + .store_payload(&PersistedDigestKeyV1 { + schema_version, + generation, + key_id: Uuid::new_v4(), + seed_hex: hex::encode([0x24_u8; 32]), + }) + .await; + let error = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect_err("unsupported persisted metadata must be rejected"); + assert_eq!(error.kind(), expected); + let rows = ConfigKv::get_all_with_conn(&repository.connection().await, CONFIG_KEY) + .await + .expect("digest rows must be inspectable"); + assert_eq!(rows.len(), 1, "invalid state must not be replaced"); + repository.cleanup().await; + } + } + + #[tokio::test] + #[serial] + async fn repository_identity_and_state_queries_fail_closed() { + reset_digest_cache_for_tests().await; + let absent_root = tempfile::tempdir().expect("temporary root must exist"); + let absent_db = absent_root.path().join("missing.db"); + let error = RepositoryKeyedDigest::load_or_initialize(&absent_db) + .await + .expect_err("an absent repository database must be rejected"); + assert_eq!(error.kind(), KeyedDigestErrorKind::RepositoryUnavailable); + + { + let repository = TestRepository::new().await; + ConfigKv::unset_all_with_conn(&repository.connection().await, "libra.repoid") + .await + .expect("repository identity must be removable for the corruption probe"); + let error = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect_err("missing repository identity must be rejected"); + assert_eq!( + error.kind(), + KeyedDigestErrorKind::RepositoryIdentityInvalid + ); + repository.cleanup().await; + } + + { + let repository = TestRepository::new().await; + let conn = repository.connection().await; + conn.execute_raw(Statement::from_string( + conn.get_database_backend(), + "DROP TABLE reference", + )) + .await + .expect("reference table must be removable for the corruption probe"); + let error = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect_err("unreadable Memory state must fail closed"); + assert_eq!(error.kind(), KeyedDigestErrorKind::StateQueryFailed); + repository.cleanup().await; + } + } + + #[tokio::test] + #[serial] + async fn missing_vault_key_and_wrong_working_directory_fail_closed() { + reset_digest_cache_for_tests().await; + { + let repository = TestRepository::new().await; + repository + .store_payload(&PersistedDigestKeyV1 { + schema_version: PERSISTED_SCHEMA_VERSION, + generation: PERSISTED_GENERATION, + key_id: Uuid::new_v4(), + seed_hex: hex::encode([0x35_u8; 32]), + }) + .await; + repository.remove_home_vault_key().await; + let error = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect_err("missing vault key must fail closed"); + assert_eq!(error.kind(), KeyedDigestErrorKind::VaultKeyUnavailable); + repository.cleanup().await; + } + + { + let repository = TestRepository::new().await; + let unrelated = tempfile::tempdir().expect("unrelated working directory must exist"); + let _cwd = WorkingDirectoryGuard::enter(unrelated.path()); + let provider = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect("explicit repository path must not depend on cwd"); + assert_eq!(provider.key_id().get_version_num(), 4); + repository.cleanup().await; + } + } + + #[tokio::test] + #[serial] + async fn keyed_digest_secret_probe_zero_leak() { + reset_digest_cache_for_tests().await; + let repository = TestRepository::new_with_real_vault().await; + let object_root = repository.repo_root.join(".libra/objects"); + fs::create_dir_all(&object_root).expect("object surface must be created"); + fs::write( + object_root.join("secret-probe-control"), + b"non-secret object control", + ) + .expect("object control must be populated"); + let object_inventory_before = inventory_files(&object_root); + assert_eq!( + object_inventory_before + .get(Path::new("secret-probe-control")) + .map(Vec::as_slice), + Some(b"non-secret object control".as_slice()), + "Git-object surface control must be present before provider use" + ); + + let conn = repository.connection().await; + conn.execute_raw(Statement::from_sql_and_values( + conn.get_database_backend(), + "INSERT INTO operation + (op_id, repo_id, view_id, command_name, description, actor, + args_digest, start_ts, end_ts, status) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + [ + "secret-probe-operation-control".into(), + repository.repo_id.clone().into(), + "secret-probe-view".into(), + "memory-keyed-digest-probe".into(), + "non-secret operation-log control".into(), + "test-actor".into(), + "non-secret-args-digest".into(), + 1_i64.into(), + 2_i64.into(), + "succeeded".into(), + ], + )) + .await + .expect("operation-log control must be populated"); + + let known_seed = [0xa5_u8; 32]; + let known_seed_hex = hex::encode(known_seed); + let known_principal_derived_key = + hex::decode("877eaf4c522e3f5080b2914c17462c794bc465349936550910c49d1cefda5cf2") + .expect("independently calculated principal HKDF vector must decode"); + let known_query_derived_key = + hex::decode("f5bb7c87932fc33ec7cb98ce063edf8011fc991f77a794a69d554790fc4c2cd6") + .expect("independently calculated query HKDF vector must decode"); + let known_principal_derived_key_hex = hex::encode(&known_principal_derived_key); + let known_query_derived_key_hex = hex::encode(&known_query_derived_key); + repository + .store_payload(&PersistedDigestKeyV1 { + schema_version: PERSISTED_SCHEMA_VERSION, + generation: PERSISTED_GENERATION, + key_id: Uuid::parse_str("123e4567-e89b-42d3-a456-426614174000") + .expect("fixed UUIDv4 must parse"), + seed_hex: known_seed_hex.clone(), + }) + .await; + + let provider = RepositoryKeyedDigest::load_or_initialize(&repository.db_path) + .await + .expect("encrypted test seed must load"); + let reversible_principal = "alice@example.com"; + let reversible_query = "why did checkout retry after the lock error?"; + let principal_envelope = provider + .digest(DigestPurpose::Principal, reversible_principal.as_bytes()) + .expect("valid provider must digest the principal probe"); + let query_envelope = provider + .digest(DigestPurpose::Query, reversible_query.as_bytes()) + .expect("valid provider must digest the query probe"); + let expected_principal = hmac::sign( + &hmac::Key::new(hmac::HMAC_SHA256, &known_principal_derived_key), + reversible_principal.as_bytes(), + ); + let expected_query = hmac::sign( + &hmac::Key::new(hmac::HMAC_SHA256, &known_query_derived_key), + reversible_query.as_bytes(), + ); + assert_eq!( + principal_envelope.digest_hex(), + hex::encode(expected_principal.as_ref()), + "principal derived-key marker must be the key actually used" + ); + assert_eq!( + query_envelope.digest_hex(), + hex::encode(expected_query.as_ref()), + "query derived-key marker must be the key actually used" + ); + let rendered_surfaces = [ + format!("{provider:?}"), + format!("{principal_envelope:?}"), + format!("{query_envelope:?}"), + serde_json::to_string(&principal_envelope).expect("principal envelope must serialize"), + serde_json::to_string(&query_envelope).expect("query envelope must serialize"), + KeyedDigestError::new(KeyedDigestErrorKind::CiphertextInvalid).to_string(), + ]; + assert!(!rendered_surfaces.is_empty()); + for rendered in &rendered_surfaces { + assert!( + !rendered.is_empty(), + "every declared render surface must exist" + ); + for forbidden in [ + known_seed_hex.as_str(), + known_principal_derived_key_hex.as_str(), + known_query_derived_key_hex.as_str(), + reversible_principal, + reversible_query, + ] { + assert!( + !rendered.contains(forbidden), + "secret marker leaked into a rendered surface" + ); + } + for forbidden in [ + known_seed.as_slice(), + known_principal_derived_key.as_slice(), + known_query_derived_key.as_slice(), + ] { + assert!( + !rendered + .as_bytes() + .windows(forbidden.len()) + .any(|window| window == forbidden), + "raw key bytes leaked into a rendered surface" + ); + } + } + + let object_inventory_after = inventory_files(&object_root); + assert_eq!( + object_inventory_after, object_inventory_before, + "keyed-digest use must not add or alter Git-object bytes" + ); + + let operation_rows = conn + .query_all_raw(Statement::from_string( + conn.get_database_backend(), + "SELECT op_id || char(31) || repo_id || char(31) || view_id || char(31) || + command_name || char(31) || description || char(31) || actor || char(31) || + COALESCE(args_digest, '') + FROM operation ORDER BY op_id", + )) + .await + .expect("operation-log surface must be readable"); + assert_eq!( + operation_rows.len(), + 1, + "operation-log probe must inspect its populated control row" + ); + let operation_surface = operation_rows[0] + .try_get_by_index::(0) + .expect("operation-log control must remain textual"); + assert!( + operation_surface.contains("secret-probe-operation-control"), + "operation-log control marker must be present" + ); + for forbidden in [ + known_seed_hex.as_str(), + known_principal_derived_key_hex.as_str(), + known_query_derived_key_hex.as_str(), + reversible_principal, + reversible_query, + ] { + assert!( + !operation_surface.contains(forbidden), + "secret marker leaked into the populated operation log" + ); + } + + // This owner has no remote-tier dependency or adapter. Keep that + // architectural boundary executable until a later task deliberately + // adds remote publication with its own secret contract. + let production_source = include_str!("keyed_digest.rs") + .split_once("\n#[cfg(test)]\n") + .expect("module must keep production and test sections distinct") + .0; + assert!( + production_source.contains("impl RepositoryKeyedDigest"), + "remote-tier guard must inspect the populated production module" + ); + for remote_adapter in [ + "RemoteStorage", + "ClientStorage", + "D1Client", + "reqwest::", + "utils::storage::remote", + ] { + assert!( + !production_source.contains(remote_adapter), + "keyed-digest owner unexpectedly gained remote adapter {remote_adapter}" + ); + } + + let mut repository_files = Vec::new(); + for entry in WalkDir::new(&repository.repo_root) { + let entry = entry.expect("repository surface must be walkable"); + if entry.file_type().is_file() { + repository_files.push(( + entry.path().to_path_buf(), + fs::read(entry.path()).expect("repository surface must be readable"), + )); + } + } + assert!( + repository_files.iter().any(|(_, bytes)| bytes + .windows(CONFIG_KEY.len()) + .any(|window| window == CONFIG_KEY.as_bytes())), + "probe must inspect a populated config database" + ); + assert!( + repository_files.len() >= 2, + "probe must inspect both repository and vault persistence surfaces" + ); + for (path, bytes) in &repository_files { + for forbidden in [ + known_seed.as_slice(), + known_principal_derived_key.as_slice(), + known_query_derived_key.as_slice(), + ] { + assert!( + !bytes + .windows(forbidden.len()) + .any(|window| window == forbidden), + "raw key material leaked into {}", + path.display() + ); + } + let rendered = String::from_utf8_lossy(bytes); + for forbidden in [ + known_seed_hex.as_str(), + known_principal_derived_key_hex.as_str(), + known_query_derived_key_hex.as_str(), + reversible_principal, + reversible_query, + ] { + assert!( + !rendered.contains(forbidden), + "secret marker leaked into {}", + path.display() + ); + } + } + + let home_key = fs::read_to_string( + repository + .home_root + .join(".libra/vault-keys") + .join(&repository.repo_id), + ) + .expect("control vault key must be readable"); + assert_eq!(home_key, hex::encode(&repository.vault_key)); + for forbidden in [ + known_seed_hex.as_str(), + known_principal_derived_key_hex.as_str(), + known_query_derived_key_hex.as_str(), + reversible_principal, + reversible_query, + ] { + assert!(!home_key.contains(forbidden)); + } + let vault_db = repository.repo_root.join(".libra/vault.db"); + assert!( + vault_db.is_file(), + "probe must include a real repository vault" + ); + assert!( + fs::metadata(vault_db) + .expect("vault metadata must be readable") + .len() + > 0 + ); + + let rows = ConfigKv::get_all_with_conn(&repository.connection().await, CONFIG_KEY) + .await + .expect("digest row must be readable"); + assert_eq!(rows.len(), 1); + assert!(rows[0].encrypted); + for forbidden in [ + known_seed_hex.as_str(), + known_principal_derived_key_hex.as_str(), + known_query_derived_key_hex.as_str(), + reversible_principal, + reversible_query, + ] { + assert!(!rows[0].value.contains(forbidden)); + } + + repository.cleanup().await; + } +} diff --git a/src/internal/ai/mod.rs b/src/internal/ai/mod.rs index 732f55ad8..cd4c1f408 100644 --- a/src/internal/ai/mod.rs +++ b/src/internal/ai/mod.rs @@ -114,6 +114,7 @@ pub mod libra_vcs; pub mod mcp; // Versioned Agent Memory contracts. Storage and compiler implementations stay // behind the small crate-private `memory` surface. +pub(crate) mod keyed_digest; pub(crate) mod memory; // Adapter that lets agents participate as nodes in the workflow DAG. pub mod node_adapter; diff --git a/src/internal/ai/session/jsonl.rs b/src/internal/ai/session/jsonl.rs index b085351d5..6f66a2c87 100644 --- a/src/internal/ai/session/jsonl.rs +++ b/src/internal/ai/session/jsonl.rs @@ -7350,7 +7350,6 @@ mod tests { payload: Value::Null, }); } - drop(append); let replay = CodeWorkflowReplay { events, gaps: Vec::new(), @@ -7437,7 +7436,6 @@ mod tests { payload: Value::Null, }); } - drop(append); assert_eq!(events.len(), EVENT_COUNT); let latest_attempt = latest_attempt.expect("at least one retry attempt"); let latest_attempt_event = latest_attempt_event.expect("latest retry event"); diff --git a/src/internal/config.rs b/src/internal/config.rs index fecdbf96b..6acece69b 100644 --- a/src/internal/config.rs +++ b/src/internal/config.rs @@ -27,7 +27,7 @@ use std::{collections::HashSet, mem::swap, path::Path}; use anyhow::{Context, Result, anyhow}; use sea_orm::{ ActiveValue::Set, ColumnTrait, ConnectionTrait, DatabaseConnection, EntityTrait, ModelTrait, - QueryFilter, QueryOrder, entity::ActiveModelTrait, + QueryFilter, QueryOrder, Statement, entity::ActiveModelTrait, }; use crate::{ @@ -264,6 +264,36 @@ impl ConfigKv { Ok(()) } + /// Insert one repository-owned vault-internal value if the key is absent. + /// + /// This deliberately has narrower semantics than [`Self::set_with_conn`]: + /// it never updates, never creates plaintext, and never accepts an ordinary + /// user configuration key. Callers must already hold the repository write + /// transaction when first-writer-wins initialization is required. + pub(crate) async fn insert_vault_internal_if_absent_with_conn( + db: &C, + key: &str, + value: &str, + ) -> Result { + if !is_vault_internal_key(key) { + return Err(anyhow!( + "refusing internal encrypted insert for non-vault key '{key}'" + )); + } + + let result = db + .execute_raw(Statement::from_sql_and_values( + db.get_database_backend(), + "INSERT INTO config_kv (key, value, encrypted) \ + SELECT ?, ?, 1 \ + WHERE NOT EXISTS (SELECT 1 FROM config_kv WHERE key = ?)", + [key.into(), value.into(), key.into()], + )) + .await + .context("failed to conditionally insert vault-internal config")?; + Ok(result.rows_affected() == 1) + } + /// Add a value for a key (allows duplicates, for multi-value keys). /// /// Enforces same-key-same-state: if existing entries for this key have a @@ -1891,6 +1921,18 @@ fn git_config_key_matches(stored: &str, requested: &str) -> bool { // Sensitive key detection // ───────────────────────────────────────────────────────────────────────────── +/// Repository-local encrypted seed owned exclusively by Agent Memory. +/// +/// Public config commands may read this key in redacted form, but only the +/// keyed-digest provider may create or mutate it. Keeping the spelling in this +/// module prevents the owner and the CLI guard from drifting apart. +pub(crate) const MEMORY_KEYED_DIGEST_CONFIG_KEY: &str = "memory.keyed_digest.v1"; + +/// Returns `true` for configuration state whose lifecycle belongs to Memory. +pub(crate) fn is_memory_owned_config_key(key: &str) -> bool { + key.eq_ignore_ascii_case(MEMORY_KEYED_DIGEST_CONFIG_KEY) +} + /// Returns `true` if the key holds sensitive material that should be /// encrypted and redacted by default. /// @@ -1918,7 +1960,11 @@ pub fn is_sensitive_key(key: &str) -> bool { if lower.ends_with(".privkey") { return true; } - if lower == "vault.unsealkey" || lower == "vault.roottoken" || lower == "vault.roottoken_enc" { + if lower == "vault.unsealkey" + || lower == "vault.roottoken" + || lower == "vault.roottoken_enc" + || is_memory_owned_config_key(key) + { return true; } @@ -1960,6 +2006,7 @@ pub fn is_vault_internal_key(key: &str) -> bool { || lower == "vault.unsealkey" || lower == "vault.roottoken" || lower == "vault.roottoken_enc" + || is_memory_owned_config_key(key) // `libra auth` token records: unset via config would be an unaudited // logout outside the owner API. || lower.starts_with("auth.token.") diff --git a/src/internal/config_ownership.rs b/src/internal/config_ownership.rs index 874c21f5f..968cc8d66 100644 --- a/src/internal/config_ownership.rs +++ b/src/internal/config_ownership.rs @@ -342,6 +342,14 @@ pub const CODE_AGENT_TABLE_OWNERSHIP: &[(&str, ConfigOwner)] = &[ /// guard scans for `static NAME: …OnceLock|LazyLock|Mutex|RwLock` in those /// namespaces and fails when a new one is missing here. pub const CODE_AGENT_PROCESS_CACHES: &[(&str, &str)] = &[ + ( + "REPOSITORY_KEYED_DIGEST_CACHE", + "keyed by canonical repository database path plus immutable repository id; bounded to 64 entries", + ), + ( + "CURRENT_PROCESS_OWNER_IDENTITY", + "process PID/start-time/boot identity; input-independent and holds no repository state", + ), ( "WORKSPACE_CONTEXT_CACHE", "keyed by canonical workspace PathBuf — per-workdir, no cross-scope reuse", diff --git a/src/internal/vault.rs b/src/internal/vault.rs index 5d01d028d..eb9fc687f 100644 --- a/src/internal/vault.rs +++ b/src/internal/vault.rs @@ -35,7 +35,10 @@ use libvault::{ errors::RvError, storage::{Backend, BackendEntry, sql::sqlite::SqliteBackend}, }; -use sea_orm::sqlx::{AssertSqlSafe, SqlitePool, query_scalar, sqlite::SqliteConnectOptions}; +use sea_orm::{ + ConnectionTrait, + sqlx::{AssertSqlSafe, SqlitePool, query_scalar, sqlite::SqliteConnectOptions}, +}; use serde_json::Value; use crate::utils::util::try_get_storage_path; @@ -700,18 +703,62 @@ pub async fn load_unseal_key_for_scope(scope: &str) -> Option> { /// This is used when callers need to resolve local secrets for an explicit /// repository target instead of the current working directory repository. pub async fn load_unseal_key_for_db_path(db_path: &Path) -> Option> { - if let Ok(repo_id) = repo_id_for_db_path(db_path).await - && let Some(hex_key) = load_unseal_key_from_home_for_repo_id(&repo_id).await - { - return hex::decode(hex_key).ok(); + use crate::internal::db::get_db_conn_instance_for_path; + let conn = get_db_conn_instance_for_path(db_path).await.ok()?; + load_unseal_key_for_db_path_with_conn(db_path, &conn) + .await + .ok() +} + +/// Load a repository vault key without acquiring another database connection. +/// +/// Transactional callers use this form so SQLite's single-writer lock is not +/// held while a second pool connection waits on the same repository. The +/// repository ID selects the home-directory key. A legacy `vault.unsealkey` +/// row remains readable for repositories created before home-key storage. +pub(crate) async fn load_unseal_key_for_db_path_with_conn( + db_path: &Path, + db: &C, +) -> Result> { + use crate::internal::config::ConfigKv; + + let repo_ids = ConfigKv::get_all_with_conn(db, "libra.repoid") + .await + .with_context(|| { + format!( + "failed to read repository identity from '{}'", + db_path.display() + ) + })?; + if repo_ids.len() > 1 { + return Err(anyhow!("multiple libra.repoid values are present")); } - use crate::internal::{config::ConfigKv, db::get_db_conn_instance_for_path}; - let conn = get_db_conn_instance_for_path(db_path).await.ok()?; - let entry = ConfigKv::get_with_conn(&conn, "vault.unsealkey") + if let Some(repo_id) = repo_ids.first() { + if repo_id.encrypted || repo_id.value.trim().is_empty() { + return Err(anyhow!("libra.repoid is invalid")); + } + if let Some(hex_key) = load_unseal_key_from_home_for_repo_id(repo_id.value.trim()).await { + return decode_unseal_key(&hex_key); + } + } + + let legacy = ConfigKv::get_all_with_conn(db, "vault.unsealkey") .await - .ok()??; - hex::decode(entry.value).ok() + .context("failed to read legacy repository vault key")?; + let [entry] = legacy.as_slice() else { + return Err(anyhow!("repository vault key is missing or ambiguous")); + }; + if entry.encrypted { + return Err(anyhow!( + "legacy repository vault key cannot be self-encrypted" + )); + } + decode_unseal_key(&entry.value) +} + +fn decode_unseal_key(encoded: &str) -> Result> { + hex::decode(encoded.trim()).context("repository vault key is not valid hex") } /// Load global unseal key from `~/.libra/vault-unseal-key`. @@ -1013,17 +1060,6 @@ async fn current_repo_id() -> Result { .ok_or_else(|| anyhow!("libra.repoid not set — was the repo initialized?")) } -async fn repo_id_for_db_path(db_path: &Path) -> Result { - use crate::internal::{config::ConfigKv, db::get_db_conn_instance_for_path}; - let conn = get_db_conn_instance_for_path(db_path) - .await - .context("failed to open repository config database")?; - ConfigKv::get_with_conn(&conn, "libra.repoid") - .await? - .map(|e| e.value) - .ok_or_else(|| anyhow!("libra.repoid not set — was the repo initialized?")) -} - fn unseal_key_path_for_repo_id(repo_id: &str) -> Result { let home = vault_home_dir().ok_or_else(|| anyhow!("cannot determine home directory"))?; Ok(home.join(".libra").join("vault-keys").join(repo_id)) diff --git a/src/utils/error.rs b/src/utils/error.rs index 6f34677f3..2a1152f52 100644 --- a/src/utils/error.rs +++ b/src/utils/error.rs @@ -194,6 +194,9 @@ pub enum StableErrorCode { RepoNotFound, RepoCorrupt, RepoStateInvalid, + /// The repository-local Memory keyed-digest seed is missing, malformed, + /// uses an unsupported generation, or cannot be decrypted. + MemoryDigestKeyUnavailable, /// Global config database schema is newer than this Libra binary supports. ConfigSchemaFuture, ConflictUnresolved, @@ -355,6 +358,7 @@ impl StableErrorCode { Self::RepoNotFound => "LBR-REPO-001", Self::RepoCorrupt => "LBR-REPO-002", Self::RepoStateInvalid => "LBR-REPO-003", + Self::MemoryDigestKeyUnavailable => "LBR-MEMORY-001", Self::ConfigSchemaFuture => "LBR-CONFIG-001", Self::ConflictUnresolved => "LBR-CONFLICT-001", Self::ConflictOperationBlocked => "LBR-CONFLICT-002", @@ -420,6 +424,7 @@ impl StableErrorCode { Self::RepoNotFound | Self::RepoCorrupt | Self::RepoStateInvalid + | Self::MemoryDigestKeyUnavailable | Self::WorktreeCursorInvalid | Self::WorktreeScopeCorrupt => CliErrorCategory::Repo, Self::ConfigSchemaFuture | Self::UpgradeSettingsInvalid => CliErrorCategory::Config, @@ -524,6 +529,9 @@ impl StableErrorCode { Self::RepoStateInvalid => { "Repository state prevents the requested operation from proceeding." } + Self::MemoryDigestKeyUnavailable => { + "Repository Memory digest key is missing, invalid, or cannot be decrypted." + } Self::ConfigSchemaFuture => { "Global config database schema is newer than this Libra binary supports." } @@ -2126,8 +2134,8 @@ mod tests { /// would invalidate every downstream pin without tripping any /// test until end-to-end JSON harness assertions caught it. /// - /// Enumerate all 23 non-agent variants so a new addition trips both this - /// list and the `as_str` impl's exhaustive match. + /// Pin the general-purpose and domain-specific variants alongside the + /// agent-code table below; the `as_str` impl remains exhaustive. #[test] fn stable_error_code_as_str_pins_each_variant() { assert_eq!(StableErrorCode::CliUnknownCommand.as_str(), "LBR-CLI-001"); @@ -2136,6 +2144,10 @@ mod tests { assert_eq!(StableErrorCode::RepoNotFound.as_str(), "LBR-REPO-001"); assert_eq!(StableErrorCode::RepoCorrupt.as_str(), "LBR-REPO-002"); assert_eq!(StableErrorCode::RepoStateInvalid.as_str(), "LBR-REPO-003"); + assert_eq!( + StableErrorCode::MemoryDigestKeyUnavailable.as_str(), + "LBR-MEMORY-001", + ); assert_eq!( StableErrorCode::ConfigSchemaFuture.as_str(), "LBR-CONFIG-001", @@ -2287,6 +2299,10 @@ mod tests { StableErrorCode::RepoStateInvalid.category(), CliErrorCategory::Repo, ); + assert_eq!( + StableErrorCode::MemoryDigestKeyUnavailable.category(), + CliErrorCategory::Repo, + ); // The worktree-doctor pair reuses `repo` deliberately (§C.13) — // re-bucketing either one would change `fine_exit_code()` for // scripts that branch on the doctor's refusals. diff --git a/tests/SERIAL_REGISTRY.tsv b/tests/SERIAL_REGISTRY.tsv index 3cb1f09e2..db449a5ba 100644 --- a/tests/SERIAL_REGISTRY.tsv +++ b/tests/SERIAL_REGISTRY.tsv @@ -211,6 +211,7 @@ test_config_list_defaults_to_local_scope_without_global_entries lane:cwd changes test_config_list_gpg_keys_outputs_configured_key_namespaces lane:cwd changes the process working directory (ChangeDirGuard/set_current_dir) test_config_list_name_only lane:cwd changes the process working directory (ChangeDirGuard/set_current_dir) test_config_list_ssh_keys_outputs_configured_public_keys lane:cwd changes the process working directory (ChangeDirGuard/set_current_dir) +test_config_memory_keyed_digest_is_owner_managed lane:cwd changes cwd and process HOME through the command integration harness test_config_null_terminated_output lane:cwd changes the process working directory (ChangeDirGuard/set_current_dir) test_config_remove_and_rename_section lane:cwd changes the process working directory (ChangeDirGuard/set_current_dir) test_config_rename_section_preserves_multivalue_order lane:cwd changes the process working directory (ChangeDirGuard/set_current_dir) diff --git a/tests/command/config_test.rs b/tests/command/config_test.rs index b72b101b3..1c777cab8 100644 --- a/tests/command/config_test.rs +++ b/tests/command/config_test.rs @@ -986,6 +986,166 @@ async fn test_config_set_plaintext_on_vault_internal_key_is_failure() { ); } +#[tokio::test] +#[serial] +async fn test_config_memory_keyed_digest_is_owner_managed() { + use libra::internal::config::ConfigKv; + + const KEY: &str = "memory.keyed_digest.v1"; + const CONTROL_VALUE: &str = "owner-encrypted-control"; + + let temp = tempdir().unwrap(); + test::setup_with_new_libra_in(temp.path()).await; + let _guard = test::ChangeDirGuard::new(temp.path()); + + for command in [ + &["config", "set", KEY, "replacement"][..], + &["config", "--add", KEY, "additional"][..], + &["config", "unset", KEY][..], + &["config", "--unset-all", KEY][..], + &["config", "--remove-section", "memory.keyed_digest"][..], + &[ + "config", + "--rename-section", + "memory.keyed_digest", + "archive.keyed_digest", + ][..], + ] { + ConfigKv::set(KEY, CONTROL_VALUE, true) + .await + .expect("owner-controlled fixture row must be stored"); + + let output = run_libra_command(command, temp.path()); + assert!( + !output.status.success(), + "public mutation must reject {command:?}" + ); + assert!( + String::from_utf8_lossy(&output.stderr).contains("managed by Libra Memory"), + "rejection must name the owning subsystem for {command:?}: {}", + String::from_utf8_lossy(&output.stderr) + ); + + let rows = ConfigKv::get_all(KEY) + .await + .expect("owner row must remain readable"); + assert_eq!(rows.len(), 1, "{command:?} changed row cardinality"); + assert_eq!(rows[0].value, CONTROL_VALUE, "{command:?} changed value"); + assert!(rows[0].encrypted, "{command:?} changed encryption state"); + } + + // A section rename must not create the owner key when it is absent. + ConfigKv::unset_all(KEY) + .await + .expect("fixture owner row must be removable internally"); + ConfigKv::set("source.keyed_digest.v1", "rename-control", false) + .await + .expect("rename source must be stored"); + let rename_into_owner = run_libra_command( + &[ + "config", + "--rename-section", + "source.keyed_digest", + "memory.keyed_digest", + ], + temp.path(), + ); + assert!(!rename_into_owner.status.success()); + assert!(String::from_utf8_lossy(&rename_into_owner.stderr).contains("managed by Libra Memory")); + assert!( + ConfigKv::get_all(KEY).await.unwrap().is_empty(), + "rename must not create the owner key" + ); + assert_eq!( + ConfigKv::get("source.keyed_digest.v1") + .await + .unwrap() + .map(|entry| entry.value) + .as_deref(), + Some("rename-control"), + "rejected rename must preserve its source" + ); + + // Git import follows the existing reserved-key convention: ignore the + // owner key, warn, and continue importing unrelated entries. + let git_init = Command::new("git") + .arg("init") + .current_dir(temp.path()) + .output() + .expect("git init must run"); + assert!(git_init.status.success()); + let git_set = Command::new("git") + .args(["config", KEY, "imported-replacement"]) + .current_dir(temp.path()) + .output() + .expect("git config must run"); + assert!( + git_set.status.success(), + "Git must accept the owner-key fixture: {}", + String::from_utf8_lossy(&git_set.stderr) + ); + let import = run_libra_command(&["config", "import"], temp.path()); + assert_cli_success(&import, "config import with an owner key"); + assert!( + String::from_utf8_lossy(&import.stderr).contains("Memory-owned"), + "import must report the ignored owner key: {}", + String::from_utf8_lossy(&import.stderr) + ); + assert!( + ConfigKv::get_all(KEY).await.unwrap().is_empty(), + "import must not create the owner key" + ); + + // Reads remain available, but the encrypted payload never leaves the CLI. + ConfigKv::set(KEY, CONTROL_VALUE, true) + .await + .expect("owner row must be restored for read probe"); + let read = run_libra_command(&["config", "get", KEY], temp.path()); + assert_cli_success(&read, "read owner-managed key"); + let stdout = String::from_utf8_lossy(&read.stdout); + assert!( + stdout.contains(""), + "owner key must be redacted: {stdout}" + ); + assert!(!stdout.contains(CONTROL_VALUE)); + + let reveal = run_libra_command(&["config", "get", "--reveal", KEY], temp.path()); + assert!(!reveal.status.success(), "owner key must not be revealable"); + + // Corruption must not turn the config command into a secret exfiltration + // path. Redaction is an ownership rule, independent of the row flag. + const PLAINTEXT_CORRUPT: &str = + r#"{"seed_hex":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}"#; + ConfigKv::unset_all(KEY) + .await + .expect("encrypted fixture row must be removable internally"); + ConfigKv::set(KEY, PLAINTEXT_CORRUPT, false) + .await + .expect("plaintext corruption fixture must be stored"); + for command in [ + &["config", "get", KEY][..], + &["config", "--get-all", KEY][..], + &["config", "--get-regexp", r"^memory\.keyed_digest\.v1$"][..], + &["config", "--list"][..], + &["config", "--list", "--show-origin"][..], + &["config", "--json", "get", KEY][..], + &["config", "--json", "--get-regexp", KEY][..], + &["config", "--json", "list"][..], + ] { + let output = run_libra_command(command, temp.path()); + assert_cli_success(&output, "read corrupt owner-managed key"); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + stdout.contains(""), + "{command:?} must render a redacted placeholder: {stdout}" + ); + assert!( + !stdout.contains(PLAINTEXT_CORRUPT) && !stdout.contains("aaaaaaaaaaaaaaaa"), + "{command:?} leaked the corrupt plaintext row: {stdout}" + ); + } +} + #[tokio::test] #[serial] async fn test_config_set_read_failure_does_not_silently_skip_existing_state_check() { From 6c8830f7e27f0242f53c55c0af611f5a0d4f4c4b Mon Sep 17 00:00:00 2001 From: anduin9527 Date: Mon, 24 Aug 2026 10:14:12 +0800 Subject: [PATCH 03/18] feat(memory): add core projection schema Signed-off-by: anduin9527 --- docs/development/commands/maintenance.md | 2 +- docs/development/plan/plan-20260819.md | 54 +- docs/development/tracing/memory.md | 506 +++++++-- sql/migrations/2026082401_memory_core.sql | 403 ++++++++ .../2026082401_memory_core_down.sql | 45 + sql/migrations/README.md | 1 + src/command/maintenance.rs | 91 ++ src/internal/ai/keyed_digest.rs | 135 ++- src/internal/ai/memory/job_sql.rs | 964 ++++++++++++++++++ src/internal/ai/memory/mod.rs | 1 + src/internal/db.rs | 31 +- src/internal/db/migration.rs | 12 +- src/internal/model/memory_episode_path.rs | 19 + src/internal/model/memory_head.rs | 40 + src/internal/model/memory_link_index.rs | 28 + src/internal/model/memory_model_tests.rs | 50 + src/internal/model/memory_note_index.rs | 30 + src/internal/model/memory_path_summary.rs | 25 + src/internal/model/memory_projection_state.rs | 20 + src/internal/model/memory_revision_index.rs | 26 + src/internal/model/mod.rs | 9 + tests/agent_capture_migration_test.rs | 4 +- tests/command/worktree_isolation_test.rs | 21 +- tests/db_migration_test.rs | 568 +++++++++-- 24 files changed, 2884 insertions(+), 201 deletions(-) create mode 100644 sql/migrations/2026082401_memory_core.sql create mode 100644 sql/migrations/2026082401_memory_core_down.sql create mode 100644 src/internal/ai/memory/job_sql.rs create mode 100644 src/internal/model/memory_episode_path.rs create mode 100644 src/internal/model/memory_head.rs create mode 100644 src/internal/model/memory_link_index.rs create mode 100644 src/internal/model/memory_model_tests.rs create mode 100644 src/internal/model/memory_note_index.rs create mode 100644 src/internal/model/memory_path_summary.rs create mode 100644 src/internal/model/memory_projection_state.rs create mode 100644 src/internal/model/memory_revision_index.rs diff --git a/docs/development/commands/maintenance.md b/docs/development/commands/maintenance.md index cccd3fff7..177ff786a 100644 --- a/docs/development/commands/maintenance.md +++ b/docs/development/commands/maintenance.md @@ -35,7 +35,7 @@ flowchart TD ## GC 类型化根清单（W2 §C.4.3） -- `GC_OBJECT_SOURCE_INVENTORY`（`src/command/maintenance.rs`,版本 1）登记**每一个**可含对象库 OID 的持久存储:traced root(refs、双侧 reflog、全 scope sequencer/rebase/bisect 行、全 worktree index 各 stage、全 gitdir held-autostash + merge/revert/rebase-aux sidecar + FETCH_HEAD、stash reflog、`notes.blob`、`operation_view_ref.target_oid`、`agent_checkpoint.*`)或 documented non-root(advisory 校验键、可重建缓存、obliteration 墓碑、layer 磁盘身份)。 +- `GC_OBJECT_SOURCE_INVENTORY`（`src/command/maintenance.rs`,版本 1）登记**每一个**可含对象库 OID 的持久存储:traced root(refs、双侧 reflog、全 scope sequencer/rebase/bisect 行、全 worktree index 各 stage、全 gitdir held-autostash + merge/revert/rebase-aux sidecar + FETCH_HEAD、stash reflog、`notes.blob`、`operation_view_ref.target_oid`、`agent_checkpoint.*`)或 documented non-root(advisory 校验键、Memory 可重建投影与作业/observer 水位、obliteration 墓碑、layer 磁盘身份)。Memory SQLite 行不保活对象；后续权威 Memory ref/object collector 负责版本历史的可达性，投影由 replay 修复。 - 守卫测试 `gc_object_source_inventory_covers_every_oid_column`(db_migration_test)扫描 live schema 的 OID 形态列,未登记即失败——新表新列不可能静默漏根。 - 由此 W0 的多 worktree gc/repack skip 已解除;任何根读取失败仍 fail-closed(绝不在部分根集上 prune)。本切片同时修复了单 worktree 下 note blob / undo 快照 / AI checkpoint 对象会被 prune 的既有数据丢失洞。 diff --git a/docs/development/plan/plan-20260819.md b/docs/development/plan/plan-20260819.md index 66b211cb6..8ee018572 100644 --- a/docs/development/plan/plan-20260819.md +++ b/docs/development/plan/plan-20260819.md @@ -177,7 +177,7 @@ erDiagram int last_event_seq int schema_version string policy_version - string rebuilt_at + int rebuilt_at } MEMORY_HEAD { string scope_key PK @@ -253,8 +253,8 @@ erDiagram | `memory_episode_path` | 可重建投影 | `(note_id, revision_oid, code_path)` | 精确路径和路径前缀过滤 | | `memory_episode_search_doc` | 可重建投影 | integer `rowid`；唯一 `(note_id, revision_oid)` | 保存 root / 时间 / completion / code-change 等结构化列及外部内容 FTS 的正文源 | | `memory_episode_fts` | 可重建 FTS5 | 与 search doc 共用 `rowid` | 倒排索引 `goal/summary/decisions/failed_attempts/unresolved`，使用 `bm25()` 排序 | -| `memory_compile_job` | 有界本地运行状态 | 唯一 `(scope_key, root_kind, root_id)` | 保存 observed/processed generation、最新 terminal source object OID、规范输入指纹、lease owner/fence、retry 状态；输入指纹不变时重放不递增 generation | -| `memory_compile_observer_state` | 有界本地运行状态 | `(scope_key, source_ref_name)` | 分别保存已完整扫描的 `libra/intent` 与 `libra/memory/repo` first-parent OID；前者发现 Task/Intent 终态，后者只用已确认 Task revision 唤醒已终态父 Intent；和 job upsert 在同一 SQLite transaction 推进 | +| `memory_compile_job` | 有界本地运行状态 | 唯一 `(scope_key, root_kind, root_id)` | 保存 observed/processed generation、最新 terminal source object OID、`SourceInputFingerprint(version/key_id/digest)`、持久 lease fence、owner/deadline、retry 与稳定错误；完整 fingerprint 与 source OID 都不变时重放不递增 generation | +| `memory_compile_observer_state` | 有界本地运行状态 | `(scope_key, source_ref_name)` | 分别保存已完整扫描的 `libra/intent` 与 `libra/memory/repo` first-parent OID；crate-private batch transaction 在全部 job 幂等 upsert 后 CAS 推进水位，空 batch 也可推进；前者发现 Task/Intent 终态，后者只用已确认 Task revision 唤醒已终态父 Intent | | `context_selection_receipt` | 有界本地审计账本 | UUIDv7 `receipt_id` | 共享 Memory/mainline 的选择回执；保存 source kind、view/source heads、水位、策略与 selector 版本、`digest_key_id`、选中/省略项、预算、bundle hash 和重放状态，不保存 raw query/body；`recorded_at` 有索引 | | `context_selection_receipt_retention` | 有界本地账本元数据 | `repository_id` | 保存 `pruned_before` 与最近 prune 时间；默认保留 30 天且每仓最多 10,000 行，append 时用索引删除过期/超额最旧行 | @@ -770,7 +770,7 @@ flowchart TB **Task type:** `migration` -**Lifecycle / Acceptance:** `pending` / 空 +**Lifecycle / Acceptance:** `in-progress` / `locally-accepted` **Description:** 用一个版本化迁移在现有 `.libra/libra.db` 中增加 Memory 核心可重建投影、每 root 单行的编译作业状态和 source observer 水位。 @@ -780,39 +780,45 @@ flowchart TB | 事实 | 证据 | |---|---| -| 最新 migration 为 `2026081301` | `sql/migrations/README.md:169-172` | +| 开工日最新 migration 为 `2026081301`；按实际撰写日期，本卡使用 `2026082401_memory_core` | `sql/migrations/README.md` registry 与命名规则 | | migration 需 include_str 集中注册并测试重复运行 | `sql/migrations/README.md:90-114` | -| model registry 无 memory entity | `src/internal/model/mod.rs:1-42` | +| 开工时 model registry 无 Memory entity；本卡已注册 7 个可重建投影 entity | `src/internal/model/mod.rs`、`src/internal/model/memory_*.rs` | +| 两轮 sol/max readiness review 已把缺失的三张表、FK/状态约束、typed fingerprint 与 crate-private transaction seam 收敛为唯一合同 | `docs/development/tracing/memory.md` §5.2/§5.2.1；2026-08-24 reviewer 记录 | +| `2026082401_memory_core` 已创建 7 张可重建投影表和 2 张有界作业/observer 状态表；fresh/upgrade/SQL-body idempotent/空表 down 与非空 fail-closed 均通过；完整 12 索引、复合 FK、两种幂等唯一约束与 job 状态约束有判别断言 | 2026-08-24 固定容器：3 个强化后的 `memory_episode_*` 用例通过；完整 `db_migration_test` 74/75，唯一失败为既有 SQLite 并发锁偶发项，具名单跑 1/1 通过 | +| source-input 指纹、旧 reader、7 个 SeaORM PK、observer/job 原子事务、同输入跨 cursor no-op、非零 processed generation 保留与 inflight lease 均有实际命中的测试 | 2026-08-24：`memory_` 12/12、`source_input_fingerprint` 1/1、`observer_job_` 4/4 | +| 新 OID 列全部登记为可重建 `IndexOnly` 或有界运行状态 `NonRoot`，SQLite 不成为对象保活真源 | `gc_object_source_inventory_covers_every_oid_column` 1/1；`docs/development/commands/maintenance.md` | +| 迁移 tip 传播与静态门通过 | `agent_capture_migration_test` 9/9；两个 worktree 具名回归各 1/1；`cargo +nightly fmt --all --check`、`cargo clippy --all-targets --all-features -- -D warnings` exit 0 | +| 独立 Standards / Spec 终审已收敛 | 2026-08-24 两位 sol/max reviewer 最终均 PASS；Spec 首轮提出的完整约束判别、跨 cursor no-op/非零进度与直接 SQL 幂等 3 个 P2 已补测并在二审关闭 | **Acceptance criteria:** -- [ ] forward migration 创建 `memory_head/path_summary/note_index/revision_index/link_index/projection_state/episode_path`，字段与本文 ER/表职责一致。 -- [ ] Cell、note/revision/link、scope watermark 与 Episode path 的主键、唯一键、CHECK 和热查询索引由 SQLite 约束。 -- [ ] `memory_compile_job` 每个 `(scope_key, root_kind, root_id)` 最多一行，并保存 source object OID、规范输入指纹、generation、lease/fence、retry 与稳定错误码。 -- [ ] `memory_compile_observer_state` 每个 `(scope_key, source_ref_name)` 一行，能分别跟踪 Intent 事实 ref 与 Memory ref，并与 job upsert 在同一事务推进 pinned first-parent 水位。 -- [ ] 可重建投影表均有 SeaORM entity；job/observer 由单一具名 raw-SQL Module 拥有。 -- [ ] `builtin_migrations()`、migration registry 文档、model registry 和 schema version 测试同步。 -- [ ] migration 重复 up 无变化，fresh database 与从上一版升级后的 schema 相同；前一 registry tip 的 reader 打开已迁移 DB 时在 schema preflight 明确拒绝，不能误读新表。 -- [ ] down 只允许所有新表为空；存在 projection/job/observer 数据时稳定拒绝，修复只以前滚 migration 完成。 +- [x] forward migration 创建 `memory_head/path_summary/note_index/revision_index/link_index/projection_state/episode_path`，字段与本文 ER/表职责一致。 +- [x] Cell、note/revision/link、scope watermark 与 Episode path 的主键、唯一键、CHECK 和热查询索引由 SQLite 约束。 +- [x] `memory_compile_job` 每个 `(scope_key, root_kind, root_id)` 最多一行，并保存 source object OID、typed SourceInput fingerprint 的 version/key ID/digest、generation、持久 fence、lease、retry 与稳定错误码。 +- [x] `memory_compile_observer_state` 每个 `(scope_key, source_ref_name)` 一行，能分别跟踪 Intent 事实 ref 与 Memory ref，并与 job upsert 在同一事务推进 pinned first-parent 水位。 +- [x] 可重建投影表均有 SeaORM entity；job/observer 由单一具名 raw-SQL Module 拥有。 +- [x] `builtin_migrations()`、migration registry 文档、model registry 和 schema version 测试同步。 +- [x] migration 重复 up 无变化，fresh database 与从上一版升级后的 schema 相同；前一 registry tip 的 reader 打开已迁移 DB 时在 schema preflight 明确拒绝，不能误读新表。 +- [x] down 只允许所有新表为空；存在 projection/job/observer 数据时稳定拒绝，修复只以前滚 migration 完成。 **Verification:** -- [ ] `source .env.test && cargo test --test db_migration_test memory_episode_schema` -- [ ] `source .env.test && cargo test --test db_migration_test memory_episode_schema_idempotent` -- [ ] `source .env.test && cargo test --test db_migration_test memory_episode_down_guard` -- [ ] `source .env.test && cargo test --test db_migration_test memory_core_old_reader_rejects_migrated_schema` -- [ ] `source .env.test && cargo test --lib internal::model::memory` -- [ ] `source .env.test && cargo test --test memory_episode_test observer_job_schema_transaction` +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --test db_migration_test memory_episode_schema` +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --test db_migration_test memory_episode_schema_idempotent` +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --test db_migration_test memory_episode_down_guard` +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --lib internal::db::tests::memory_core_old_reader_rejects_migrated_schema` +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --lib internal::model::memory_model_tests::memory_entities_expose_expected_primary_keys` +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --lib internal::ai::memory::job_sql::tests::observer_job_schema_transaction` **Dependencies:** M2-01K（全串行前置已冻结类型/schema 与 repository keyed-digest 基础）。 **Deliverables:** N/A。 -**Implementation write set:** `sql/migrations/20260819*_memory_core*`、`src/internal/db/migration.rs`、`src/internal/model/memory_*`、`src/internal/model/mod.rs`、`src/internal/ai/memory/job_sql.rs`、`tests/db_migration_test.rs`、`tests/memory_episode_test.rs`、`tests/INDEX.md`、`sql/migrations/README.md`。 +**Implementation write set:** `sql/migrations/2026082401_memory_core{,_down}.sql`、`src/internal/db/migration.rs`、`src/internal/db.rs` 的 compatibility classifier seam、`src/internal/model/memory_*`（含 `memory_model_tests.rs` 的非空 entity/PK 映射测试）、`src/internal/model/mod.rs`、`src/internal/ai/{keyed_digest.rs,memory/{mod.rs,job_sql.rs}}`、`src/command/maintenance.rs` 的 GC source inventory、`tests/db_migration_test.rs`、受 registry tip pin 影响的既有 migration tests、`sql/migrations/README.md`、`docs/development/commands/maintenance.md`、`memory.md` 与本计划。 **Release write set:** N/A。 -**Files likely touched:** migration up/down、8 个 entity 文件以内、registry、job SQL owner、tests/index。 +**Files likely touched:** migration up/down、7 个 entity、registry、job SQL owner、GC source inventory、既有 migration tests 与相关设计/维护文档；本卡不新增公开 API 或 integration target。 **Docs and compatibility impact:** migration README、测试索引；公开命令文档 N/A。 @@ -2019,6 +2025,10 @@ Result 只允许 `PASS` 或 `FAIL`；最近一轮必须为 `PASS` 才能开工 | R6 | M2-01 实现与用户准入语义修订 | FAIL | canonical digest 使用整对象黑名单投影；Episode golden 过滤器零命中；EpisodeCompiler 可使用 Namespace 去重 | 改为显式 v1 白名单投影；落精确 Episode golden；Namespace 仅允许 Consolidation/Onboard；补 135 组合生产路径矩阵 | `gpt-5.6-sol` / xhigh 只读实现审查，2026-08-22；无 P0、无范围越界或虚假测试 | | R7 | R6 修订后的 M2-01 | FAIL | 无 P0/P1 | 修正 `ResolvedMemoryView.view_hash` 与固定 receipt envelope 的最后一处 P2 文案冲突，并清理 Phase A 旧去重表述 | 同一 `gpt-5.6-sol` / xhigh reviewer 复审，2026-08-22 | | R8 | R7 修订后的 M2-01 | PASS | 无 | P0/P1/P2 全部关闭 | 同一 `gpt-5.6-sol` / xhigh reviewer 最终复审，2026-08-22；确认无计划偏离、过度防御、范围越界、虚假或零命中测试 | +| R9 | M2-01K 最终实现 | PASS | 无 | Standards / Spec 既有 findings 全部关闭 | 两位 sol/max reviewer 对最终 live diff 独立终审，2026-08-24；key owner、fail-closed、缓存毒化与非空 secret probe 均确认通过 | +| R10 | M2-02 schema / transaction 合同 | PASS | 无 | readiness 三轮提出的表职责、约束、指纹、cursor/generation/lease 与 down 语义均已冻结 | 两位 sol/max reviewer 对开工合同独立终审，2026-08-24 | +| R11 | M2-02 最终实现首轮 | FAIL | 无 | 3 个测试判别 P2：完整约束集合、同输入跨 cursor/非零 processed generation、直接重复 up SQL | Standards reviewer PASS；Spec reviewer 只读审查，2026-08-24；无生产缺陷、范围越界或虚假通过 | +| R12 | R11 测试增强后的 M2-02 | PASS | 无 | R11 的 3 个 P2 全部关闭 | 同一 Spec reviewer 二次复审 PASS；`memory_episode_*` 3/3、`observer_job_` 4/4，2026-08-24 | ## 非目标与延后项 diff --git a/docs/development/tracing/memory.md b/docs/development/tracing/memory.md index 33b3d4533..d52c1e287 100644 --- a/docs/development/tracing/memory.md +++ b/docs/development/tracing/memory.md @@ -20,7 +20,7 @@ Libra 在 [`object-model.md`](../../ai/object-model.md) 与 ## 执行摘要 -Memory 是 Libra 的 VCS-native 长期知识层：让 agent 跨 run / thread / branch 记住事物，且不会像 `CLAUDE.md` 扁平大块文件那样污染上下文或不可审计。核心机制是快照（`MemoryNote`）/ 事件（`MemoryEvent`）/ 投影（SQLite）三层，历史真源落在 `refs/libra/memory/*` 的普通 Git 线性历史上。每一条写入都必须携带 `CompileRecord`（可复现），每一次注入都必须产出 `ContextReceipt`（可重放）；写入统一经 `MemoryWriter` 单一 seam，读取统一经冻结的 `ResolvedMemoryView`。安全性以 fail-closed 为底线：Draft / Quarantined / SecretLike 永不注入 prompt，MCP 工具在 C9 default-deny authorizer 落地前不注册。本设计对应长期路线图 `MEM-01..05`，并行协调 `MEM-06` 见本文 §19（见下方追溯表与 [`plan-long.md`](../plan/plan-long.md)）。 +Memory 是 Libra 的 VCS-native 长期知识层：让 agent 跨 run / thread / branch 记住事物，且不会像 `CLAUDE.md` 扁平大块文件那样污染上下文或不可审计。核心机制是快照（`MemoryNote`）/ 事件（`MemoryEvent`）/ 投影（SQLite）三层，历史真源落在 `refs/heads/libra/memory/*` 的普通 Git 线性历史上。每一条写入都必须携带 `CompileRecord`（可复现），每一次注入都必须产出 `ContextReceipt`（可重放）；写入统一经 `MemoryWriter` 单一 seam，读取统一经冻结的 `ResolvedMemoryView`。安全性以 fail-closed 为底线：Draft / Quarantined / SecretLike 永不注入 prompt，MCP 工具在 C9 default-deny authorizer 落地前不注册。本设计对应长期路线图 `MEM-01..05`，并行协调 `MEM-06` 见本文 §19（见下方追溯表与 [`plan-long.md`](../plan/plan-long.md)）。 ### 目录 @@ -273,7 +273,7 @@ Libra 的映射为： 本轮多维复核同时识别并修正了八个原草案中的实现阻断；后文规范均以这些结论为准： -1. **ref 前缀冲突**：Git 不能同时保存 `refs/libra/memory` 与 `refs/libra/memory/...`；规范根 ref 改为 `refs/libra/memory/repo`，其它 scope 使用其兄弟 ref。 +1. **ref 前缀冲突**：Git 不能同时保存 `refs/heads/libra/memory` 与 `refs/heads/libra/memory/...`；规范根 ref 改为 `refs/heads/libra/memory/repo`，其它 scope 使用其兄弟 ref。 2. **OID 自引用**：`MemoryNote` 正文不能包含“自身 blob OID”再参与该 OID 的计算；`revision_oid` 改为写入后派生的 envelope / event 字段，不属于序列化正文。 3. **事件总序**：普通 Git merge 的 DAG 拓扑不是全序；每个 memory ref 必须保持 first-parent 线性历史，并以单调 `event_seq` 重放。跨 ref 合并由 `libra memory merge/cherry-pick` 重新验证并追加，不直接接受任意 merge commit。 4. **读写分离**：`SessionAttached`、`PromptTrimmed`、`last_used_at`、`use_count` 属本地访问遥测，不写入权威 memory ref；否则一次读取也会争抢 CAS，并破坏“投影可从 note/event 完全重建”的定义。 @@ -321,7 +321,7 @@ perstate 证明 Git-native memory 的第一性产品价值并不来自新的后 - 取代 `ContextFrame`、`ContextSnapshot` 或 `MemoryAnchor`。Memory 是一个对它们形成补充的新层——见 §3.1。 - 提供向量 / embedding 搜索引擎。基于路径的召回是默认方案；embedding 索引可作为后续扩展。 -- 在基础实现中提供图数据库。时序图与实体图可以叠加在 Memory 的事件流之上，但历史层面的真源仍然是 `refs/libra/memory*` 上的普通 Git 历史。 +- 在基础实现中提供图数据库。时序图与实体图可以叠加在 Memory 的事件流之上，但历史层面的真源仍然是 `refs/heads/libra/memory*` 上的普通 Git 历史。 - 静默存储 secret、私有数据或不可信的网络主张。入库时必须先对敏感度（sensitivity）与可信度（trust）进行分类，任何记忆才能成为可进入 prompt 的内容。 - 跨仓库联邦化记忆。Memory 是**按仓库（per-repo）**的构造，就像 `.libra/` 状态那样。跨仓库联邦留待将来的设计。 - 与完整聊天历史持久化竞争。Memory 存储的是**蒸馏后的、可复用的事实**，而不是原始 transcript。Transcript 已经存放在 `.libra/sessions/*.jsonl` 与 `git-internal` 的 AI 历史中。 @@ -795,15 +795,15 @@ ContextReceipt (local ledger) --selected--> MemoryHead[L] / MemoryNote[S] 如 §4 开头所述，Memory 的字节是自定义 JSON blob，存活在自己的 `libra/memory*` ref 上，与内部 AgentRuntime 的对象历史分离。后者位于孤儿分支 `libra/intent`（常量 `AI_REF`，`src/internal/ai/history.rs:92`），承载 git-internal 的 typed AI 对象（Intent/Plan/...）；而外部 agent 捕获位于 `traces`（常量 `TRACES_BRANCH`，`src/internal/branch.rs:42`，文档中写作 `refs/libra/traces`）。Memory 自己的 ref 命名沿用同一约定： ```text -refs/libra/intent # 现有 AI 工作流对象的孤儿分支（Intent/Plan/...，归 AgentRuntime） -refs/libra/memory/repo # 新增：仓库共享 memory（NEW） -refs/libra/memory/global # 新增：仅在显式启用 repo-local global policy 时使用（NEW） -refs/libra/memory/branch/ # 新增：分支作用域 memory（NEW） -refs/libra/memory/worktree/ # 新增：worktree 作用域 memory（NEW） -refs/libra/memory/actor/ # 新增：actor 私有 memory；不得用可逆 PII 作 ref 名（NEW） +refs/heads/libra/intent # 现有 AI 工作流对象的孤儿分支；SQLite 名称为 libra/intent +refs/heads/libra/memory/repo # 新增：仓库共享 memory；SQLite 名称为 libra/memory/repo +refs/heads/libra/memory/global # 新增：仅在显式启用 repo-local global policy 时使用（NEW） +refs/heads/libra/memory/branch/ # 新增：分支作用域 memory（NEW） +refs/heads/libra/memory/worktree/ # 新增：worktree 作用域 memory（NEW） +refs/heads/libra/memory/actor/ # 新增：actor 私有 memory；不得用可逆 PII 作 ref 名（NEW） ``` -Git 的 file/dir ref 规则禁止同时存在 `refs/libra/memory` 和 `refs/libra/memory/*`，因此不存在无后缀的 memory ref。`Global` 在本设计中仍是**当前仓库内的逻辑作用域**，不等于跨仓库用户档案；真正跨仓库 federation 仍不在范围内。 +Git 的 file/dir ref 规则禁止同时存在 `refs/heads/libra/memory` 和 `refs/heads/libra/memory/*`，因此不存在无后缀的 memory ref。`Global` 在本设计中仍是**当前仓库内的逻辑作用域**，不等于跨仓库用户档案；真正跨仓库 federation 仍不在范围内。 一个「memory commit」就是一个普通的 Git commit，其 tree 中包含： @@ -821,7 +821,7 @@ manifest.json # schema、scope、las Memory 的逻辑 key 不得直接拼接进 Git ref 或 tree path：branch name、namespace、actor ref 与动态 path 段都可能包含 `/`、`..`、控制字符、Unicode 归一化差异或大小写冲突。实现必须使用一套稳定、可逆、跨平台大小写安全的编码： - `scope_key` 的逻辑编码形如 `repo`、`global`、`branch:`、`worktree:`、`actor:`；branch 使用规范化 full refname（如 `refs/heads/main`），而不是短名或当前 commit OID。分支 rename 是显式 scope migration，不能静默产生一份新 memory 或丢失旧 scope。 -- `refs/libra/memory/branch/` 使用 validated canonical refname 的跨平台编码。actor ref 使用不可逆 principal hash，避免 ref 枚举泄露账号或邮箱。 +- `refs/heads/libra/memory/branch/` 使用 validated canonical refname 的跨平台编码。actor ref 使用不可逆 principal hash，避免 ref 枚举泄露账号或邮箱。 - tree path 中的 ``、``、``、`` 必须只包含 `[A-Za-z0-9._-]`；namespace 如 `private:` 必须编码为安全 segment。 - 解码后必须拒绝空 segment、`.`、`..`、绝对路径、反斜杠、NUL、控制字符以及超过长度上限的 segment。 - macOS / Windows 上不得依赖文件系统大小写行为区分两条 memory key；编码后的 segment 必须规范化为大小写不敏感仍不冲突的形式，或在写入前显式检测冲突并 fail-closed。 @@ -830,37 +830,90 @@ Memory 的逻辑 key 不得直接拼接进 Git ref 或 tree path：branch name ### 5.2 SQLite 投影表 -这些投影表通过**新建一个版本化迁移** `sql/migrations/YYYYMMDDNN_memory.sql` 引入，前向 DDL 必须幂等（`CREATE TABLE IF NOT EXISTS ...`），并可配套一份 `*_down.sql` 回滚脚本；**不要**把它们追加进 bootstrap 文件 `sql/sqlite_20260309_init.sql`。这与外部捕获采用迁移 `2026050303_agent_capture.sql` 的方式属于同一模式。 +M2-02 使用版本化迁移 `sql/migrations/2026082401_memory_core.sql` 与 +`2026082401_memory_core_down.sql` 引入核心 schema。前向 DDL 必须幂等 +（`CREATE TABLE/INDEX IF NOT EXISTS ...`）；**不要**把它们追加进 bootstrap +文件 `sql/sqlite_20260309_init.sql`。这与外部捕获采用迁移 +`2026050303_agent_capture.sql` 的方式属于同一模式。 + +该迁移只创建七张可重建投影表 `memory_head`、`memory_path_summary`、 +`memory_note_index`、`memory_revision_index`、`memory_link_index`、 +`memory_projection_state`、`memory_episode_path`，以及两张有界本地运行状态表 +`memory_compile_job`、`memory_compile_observer_state`。FTS 与 search document、 +selection receipt、entity/taxonomy/access statistics、classifier/embedding cache +分别由 M2-02F、M2-02R 或后续卡片引入，不能被这次 core migration 提前创建。 + +投影中的对象时间使用 canonical UTC RFC 3339 `TEXT`；SQLite 只检查字段存在， +Writer / Rebuilder 负责严格解析。projection rebuild、job、lease、retry 与 observer +的运行时间使用 Unix epoch milliseconds `INTEGER`，便于在本地事务中比较 deadline。所有 keyed +source-input fingerprint 拆列保存 `version/key_id/digest`；`purpose=source_input` +由不可绕过的 `SourceInputFingerprint` 类型固定，不在表中重复存字符串。 +迁移本身不得初始化、读取或轮换 repository keyed-digest seed。 下面以普通 `CREATE TABLE` 形式给出表结构，落地时请置于上述迁移文件中并加上 `IF NOT EXISTS` 幂等保护： ```sql -- 每条存活逻辑 note 的当前 head。 CREATE TABLE memory_head ( - scope_key TEXT NOT NULL, - namespace TEXT NOT NULL, - path TEXT NOT NULL, - note_id TEXT NOT NULL, - latest_revision_oid TEXT NOT NULL, - live_revision_oid TEXT, - latest_action TEXT NOT NULL, + scope_key TEXT NOT NULL CHECK (length(CAST(scope_key AS BLOB)) BETWEEN 1 AND 512), + namespace TEXT NOT NULL CHECK (length(CAST(namespace AS BLOB)) BETWEEN 1 AND 512), + path TEXT NOT NULL CHECK (length(CAST(path AS BLOB)) BETWEEN 1 AND 4096), + note_id TEXT NOT NULL CHECK ( + length(note_id) = 36 + AND substr(note_id, 9, 1) = '-' + AND substr(note_id, 14, 1) = '-' + AND substr(note_id, 19, 1) = '-' + AND substr(note_id, 24, 1) = '-' + AND length(replace(note_id, '-', '')) = 32 + AND replace(note_id, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + latest_revision_oid TEXT NOT NULL CHECK ( + length(latest_revision_oid) IN (40,64) + AND latest_revision_oid NOT GLOB '*[^0-9a-f]*' + ), + live_revision_oid TEXT CHECK ( + live_revision_oid IS NULL + OR (length(live_revision_oid) IN (40,64) + AND live_revision_oid NOT GLOB '*[^0-9a-f]*') + ), + -- taxonomy_expanded 没有 note/revision identity，只更新 taxonomy projection， + -- 因而不会成为某条 note 的 latest_action。 + latest_action TEXT NOT NULL CHECK (latest_action IN ( + 'created','revised','confirmed','quarantined','superseded', + 'revoked','forgotten','consolidated' + )), latest_review_state TEXT NOT NULL CHECK (latest_review_state IN ('draft','confirmed','quarantined','revoked','superseded','forgotten')), - kind TEXT NOT NULL, - lifecycle TEXT NOT NULL, - confidence TEXT NOT NULL, - trust TEXT NOT NULL, - sensitivity TEXT NOT NULL, - visibility TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('procedural','semantic','episodic')), + lifecycle TEXT NOT NULL CHECK (lifecycle IN ('replacement','accretive')), + confidence TEXT NOT NULL CHECK (confidence IN ('low','medium','high')), + trust TEXT NOT NULL CHECK (trust IN ('verified','repo_evidence','user_asserted','external_untrusted','inferred')), + sensitivity TEXT NOT NULL CHECK (sensitivity IN ('public','internal','confidential','secret_like')), + visibility TEXT NOT NULL CHECK (visibility IN ('private','repo_local','team_candidate')), acl_policy_id TEXT NOT NULL, valid_from TEXT, valid_until TEXT, - effective_from_commit TEXT, - effective_until_commit TEXT, + effective_from_commit TEXT CHECK ( + effective_from_commit IS NULL + OR (length(effective_from_commit) IN (40,64) + AND effective_from_commit NOT GLOB '*[^0-9a-f]*') + ), + effective_until_commit TEXT CHECK ( + effective_until_commit IS NULL + OR (length(effective_until_commit) IN (40,64) + AND effective_until_commit NOT GLOB '*[^0-9a-f]*') + ), expires_at TEXT, rank_hint INTEGER NOT NULL DEFAULT 0, - last_event_seq INTEGER NOT NULL, + last_event_seq INTEGER NOT NULL CHECK (last_event_seq >= 1), updated_at TEXT NOT NULL, - PRIMARY KEY (scope_key, namespace, path, note_id) + PRIMARY KEY (scope_key, namespace, path, note_id), + UNIQUE (note_id), + FOREIGN KEY (scope_key, namespace, path, note_id) + REFERENCES memory_note_index(scope_key, namespace, path, note_id), + FOREIGN KEY (scope_key, namespace, note_id, latest_revision_oid) + REFERENCES memory_revision_index(scope_key, namespace, note_id, revision_oid), + FOREIGN KEY (scope_key, namespace, note_id, live_revision_oid) + REFERENCES memory_revision_index(scope_key, namespace, note_id, revision_oid) ); CREATE INDEX idx_memory_head_lookup ON memory_head(scope_key, namespace, path, latest_review_state); @@ -870,13 +923,13 @@ CREATE INDEX idx_memory_head_path_prefix -- 每条路径的当前聚合。这是 summarize()、prompt 注入、 -- 以及分类法下钻的快路径。 CREATE TABLE memory_path_summary ( - scope_key TEXT NOT NULL, - namespace TEXT NOT NULL, - path TEXT NOT NULL, - confirmed_count INTEGER NOT NULL DEFAULT 0, - quarantined_count INTEGER NOT NULL DEFAULT 0, - child_count INTEGER NOT NULL DEFAULT 0, - prefix_count INTEGER NOT NULL DEFAULT 0, + scope_key TEXT NOT NULL CHECK (length(CAST(scope_key AS BLOB)) BETWEEN 1 AND 512), + namespace TEXT NOT NULL CHECK (length(CAST(namespace AS BLOB)) BETWEEN 1 AND 512), + path TEXT NOT NULL CHECK (length(CAST(path AS BLOB)) BETWEEN 1 AND 4096), + confirmed_count INTEGER NOT NULL DEFAULT 0 CHECK (confirmed_count >= 0), + quarantined_count INTEGER NOT NULL DEFAULT 0 CHECK (quarantined_count >= 0), + child_count INTEGER NOT NULL DEFAULT 0 CHECK (child_count >= 0), + prefix_count INTEGER NOT NULL DEFAULT 0 CHECK (prefix_count >= 0), preview TEXT NOT NULL DEFAULT '', last_changed_at TEXT NOT NULL, PRIMARY KEY (scope_key, namespace, path) @@ -886,69 +939,289 @@ CREATE INDEX idx_memory_path_summary_prefix -- 反向索引：note_id -> head 行，用于 O(1) 回答「这条 note 在哪里？」。 CREATE TABLE memory_note_index ( - note_id TEXT PRIMARY KEY, - scope_key TEXT NOT NULL, - namespace TEXT NOT NULL, - path TEXT NOT NULL, - kind TEXT NOT NULL, - lifecycle TEXT NOT NULL, - review_state TEXT NOT NULL, - confidence TEXT NOT NULL, - trust TEXT NOT NULL, - sensitivity TEXT NOT NULL, - visibility TEXT NOT NULL, + note_id TEXT PRIMARY KEY CHECK ( + length(note_id) = 36 + AND substr(note_id, 9, 1) = '-' + AND substr(note_id, 14, 1) = '-' + AND substr(note_id, 19, 1) = '-' + AND substr(note_id, 24, 1) = '-' + AND length(replace(note_id, '-', '')) = 32 + AND replace(note_id, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + scope_key TEXT NOT NULL CHECK (length(CAST(scope_key AS BLOB)) BETWEEN 1 AND 512), + namespace TEXT NOT NULL CHECK (length(CAST(namespace AS BLOB)) BETWEEN 1 AND 512), + path TEXT NOT NULL CHECK (length(CAST(path AS BLOB)) BETWEEN 1 AND 4096), + kind TEXT NOT NULL CHECK (kind IN ('procedural','semantic','episodic')), + lifecycle TEXT NOT NULL CHECK (lifecycle IN ('replacement','accretive')), + review_state TEXT NOT NULL CHECK (review_state IN ('draft','confirmed','quarantined','revoked','superseded','forgotten')), + confidence TEXT NOT NULL CHECK (confidence IN ('low','medium','high')), + trust TEXT NOT NULL CHECK (trust IN ('verified','repo_evidence','user_asserted','external_untrusted','inferred')), + sensitivity TEXT NOT NULL CHECK (sensitivity IN ('public','internal','confidential','secret_like')), + visibility TEXT NOT NULL CHECK (visibility IN ('private','repo_local','team_candidate')), acl_policy_id TEXT NOT NULL, - origin TEXT NOT NULL, + origin TEXT NOT NULL CHECK (origin IN ( + 'explicit','promoted_from_anchor','distilled_from_frame','classifier', + 'consolidation','onboard','branch_fork','import','coordinator','episode_compiler' + )), idempotency_key TEXT NOT NULL, - idempotency_scope TEXT NOT NULL DEFAULT 'cell', - created_at TEXT NOT NULL + idempotency_scope TEXT NOT NULL DEFAULT 'cell' + CHECK (idempotency_scope IN ('cell','namespace')), + created_at TEXT NOT NULL, + UNIQUE (scope_key, namespace, path, note_id), + UNIQUE (note_id, scope_key, namespace) ); -- 幂等键去重（§4.1.1）：默认 Cell scope 在 (scope, namespace, path) 内唯一； -- Namespace scope 的聚合入口（consolidation / onboard）在 (scope, namespace) 内唯一。 -CREATE UNIQUE INDEX idx_memory_note_idempotency - ON memory_note_index(scope_key, namespace, path, idempotency_key); +CREATE UNIQUE INDEX idx_memory_note_idempotency_cell + ON memory_note_index(scope_key, namespace, path, idempotency_key) + WHERE idempotency_scope = 'cell'; CREATE UNIQUE INDEX idx_memory_note_idempotency_ns ON memory_note_index(scope_key, namespace, idempotency_key) WHERE idempotency_scope = 'namespace'; -- revision 级 provenance / 影响面索引；可由每个 MemoryNote.compile_record 重建。 CREATE TABLE memory_revision_index ( - revision_oid TEXT PRIMARY KEY, - note_id TEXT NOT NULL, - scope_key TEXT NOT NULL, - namespace TEXT NOT NULL, - origin TEXT NOT NULL, + revision_oid TEXT PRIMARY KEY CHECK ( + length(revision_oid) IN (40,64) + AND revision_oid NOT GLOB '*[^0-9a-f]*' + ), + note_id TEXT NOT NULL CHECK ( + length(note_id) = 36 + AND substr(note_id, 9, 1) = '-' + AND substr(note_id, 14, 1) = '-' + AND substr(note_id, 19, 1) = '-' + AND substr(note_id, 24, 1) = '-' + AND length(replace(note_id, '-', '')) = 32 + AND replace(note_id, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + scope_key TEXT NOT NULL CHECK (length(CAST(scope_key AS BLOB)) BETWEEN 1 AND 512), + namespace TEXT NOT NULL CHECK (length(CAST(namespace AS BLOB)) BETWEEN 1 AND 512), + origin TEXT NOT NULL CHECK (origin IN ( + 'explicit','promoted_from_anchor','distilled_from_frame','classifier', + 'consolidation','onboard','branch_fork','import','coordinator','episode_compiler' + )), producer TEXT NOT NULL, - rules_version INTEGER NOT NULL, + rules_version INTEGER NOT NULL CHECK (rules_version > 0), prompt_version TEXT, model_id TEXT, policy_version TEXT NOT NULL, - input_fingerprints_json TEXT NOT NULL, - created_at TEXT NOT NULL + input_fingerprints_json TEXT NOT NULL CHECK ( + json_valid(input_fingerprints_json) + AND json_type(input_fingerprints_json) = 'array' + ), + created_at TEXT NOT NULL, + UNIQUE (note_id, revision_oid), + UNIQUE (scope_key, namespace, note_id, revision_oid), + FOREIGN KEY (note_id, scope_key, namespace) + REFERENCES memory_note_index(note_id, scope_key, namespace) ON DELETE CASCADE ); +CREATE INDEX idx_memory_revision_note + ON memory_revision_index(note_id, created_at, revision_oid); CREATE INDEX idx_memory_revision_producer - ON memory_revision_index(producer, prompt_version, model_id, policy_version); + ON memory_revision_index(scope_key, namespace, producer, prompt_version, model_id, policy_version); -- 派生的链接索引。历史真相是 MemoryNote.links。 CREATE TABLE memory_link_index ( - source_scope_key TEXT NOT NULL, - source_namespace TEXT NOT NULL, - source_note_id TEXT NOT NULL, - source_revision_oid TEXT NOT NULL, - target_note_id TEXT NOT NULL, - target_revision_oid TEXT, - link_kind TEXT NOT NULL, + source_scope_key TEXT NOT NULL CHECK (length(CAST(source_scope_key AS BLOB)) BETWEEN 1 AND 512), + source_namespace TEXT NOT NULL CHECK (length(CAST(source_namespace AS BLOB)) BETWEEN 1 AND 512), + source_note_id TEXT NOT NULL CHECK ( + length(source_note_id) = 36 + AND substr(source_note_id, 9, 1) = '-' + AND substr(source_note_id, 14, 1) = '-' + AND substr(source_note_id, 19, 1) = '-' + AND substr(source_note_id, 24, 1) = '-' + AND length(replace(source_note_id, '-', '')) = 32 + AND replace(source_note_id, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + source_revision_oid TEXT NOT NULL CHECK ( + length(source_revision_oid) IN (40,64) + AND source_revision_oid NOT GLOB '*[^0-9a-f]*' + ), + target_note_id TEXT NOT NULL CHECK ( + length(target_note_id) = 36 + AND substr(target_note_id, 9, 1) = '-' + AND substr(target_note_id, 14, 1) = '-' + AND substr(target_note_id, 19, 1) = '-' + AND substr(target_note_id, 24, 1) = '-' + AND length(replace(target_note_id, '-', '')) = 32 + AND replace(target_note_id, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + target_revision_oid TEXT CHECK ( + target_revision_oid IS NULL + OR (length(target_revision_oid) IN (40,64) + AND target_revision_oid NOT GLOB '*[^0-9a-f]*') + ), + link_kind TEXT NOT NULL CHECK (link_kind IN ('sibling','supports','prerequisite','contradicts','supersedes')), source_path TEXT NOT NULL, target_path TEXT NOT NULL, - evidence_refs_json TEXT NOT NULL, + evidence_refs_json TEXT NOT NULL CHECK ( + json_valid(evidence_refs_json) + AND json_type(evidence_refs_json) = 'array' + ), valid_from TEXT, valid_until TEXT, - PRIMARY KEY (source_revision_oid, target_note_id, link_kind) + PRIMARY KEY (source_revision_oid, target_note_id, link_kind), + FOREIGN KEY (source_scope_key, source_namespace, source_path, source_note_id) + REFERENCES memory_note_index(scope_key, namespace, path, note_id) ON DELETE CASCADE, + FOREIGN KEY (source_scope_key, source_namespace, source_note_id, source_revision_oid) + REFERENCES memory_revision_index(scope_key, namespace, note_id, revision_oid) ON DELETE CASCADE, + FOREIGN KEY (target_note_id) + REFERENCES memory_note_index(note_id) ON DELETE CASCADE, + FOREIGN KEY (target_note_id, target_revision_oid) + REFERENCES memory_revision_index(note_id, revision_oid) ); CREATE INDEX idx_memory_link_source ON memory_link_index(source_scope_key, source_namespace, source_note_id); CREATE INDEX idx_memory_link_target - ON memory_link_index(source_scope_key, source_namespace, target_note_id, link_kind); + ON memory_link_index(target_note_id, target_revision_oid); + +-- 每个 memory ref 的已构建水位线。安全敏感读取要求 +-- projected_ref_oid == current_ref_oid。 +CREATE TABLE memory_projection_state ( + scope_key TEXT PRIMARY KEY CHECK (length(CAST(scope_key AS BLOB)) BETWEEN 1 AND 512), + projected_ref_oid TEXT NOT NULL CHECK ( + length(projected_ref_oid) IN (40,64) + AND projected_ref_oid NOT GLOB '*[^0-9a-f]*' + ), + last_event_seq INTEGER NOT NULL CHECK (last_event_seq >= 0), + schema_version INTEGER NOT NULL CHECK (schema_version > 0), + policy_version TEXT NOT NULL, + rebuilt_at INTEGER NOT NULL CHECK (rebuilt_at >= 0) +); + +-- Episode revision -> code path。Git path 使用大小写敏感的 BINARY 排序； +-- prefix query 通过 code_path 的有界 range 扫描，不使用默认 ASCII-insensitive LIKE。 +CREATE TABLE memory_episode_path ( + note_id TEXT NOT NULL CHECK ( + length(note_id) = 36 + AND substr(note_id, 9, 1) = '-' + AND substr(note_id, 14, 1) = '-' + AND substr(note_id, 19, 1) = '-' + AND substr(note_id, 24, 1) = '-' + AND length(replace(note_id, '-', '')) = 32 + AND replace(note_id, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + revision_oid TEXT NOT NULL CHECK ( + length(revision_oid) IN (40,64) + AND revision_oid NOT GLOB '*[^0-9a-f]*' + ), + code_path TEXT NOT NULL CHECK ( + length(CAST(code_path AS BLOB)) BETWEEN 1 AND 4096 + AND substr(code_path, 1, 1) <> '/' + AND instr(code_path, char(92)) = 0 + ), + PRIMARY KEY (note_id, revision_oid, code_path), + FOREIGN KEY (note_id, revision_oid) + REFERENCES memory_revision_index(note_id, revision_oid) ON DELETE CASCADE +); +CREATE INDEX idx_memory_episode_path_code + ON memory_episode_path(code_path, note_id, revision_oid); + +-- 每个 Task / Intent root 最多一行的有界编译状态。M2-02 只冻结 schema +-- 与 observation transaction；lease runner / retry state transition 属于 M2-08。 +CREATE TABLE memory_compile_job ( + scope_key TEXT NOT NULL CHECK (length(CAST(scope_key AS BLOB)) BETWEEN 1 AND 512), + root_kind TEXT NOT NULL CHECK (root_kind IN ('task','intent')), + root_id TEXT NOT NULL CHECK ( + length(root_id) > 0 AND length(CAST(root_id AS BLOB)) <= 120 + ), + terminal_source_oid TEXT NOT NULL CHECK ( + length(terminal_source_oid) IN (40,64) + AND terminal_source_oid NOT GLOB '*[^0-9a-f]*' + ), + input_fingerprint_version INTEGER NOT NULL CHECK (input_fingerprint_version = 1), + input_fingerprint_key_id TEXT NOT NULL CHECK ( + length(input_fingerprint_key_id) = 36 + AND substr(input_fingerprint_key_id, 9, 1) = '-' + AND substr(input_fingerprint_key_id, 14, 1) = '-' + AND substr(input_fingerprint_key_id, 19, 1) = '-' + AND substr(input_fingerprint_key_id, 24, 1) = '-' + AND length(replace(input_fingerprint_key_id, '-', '')) = 32 + AND replace(input_fingerprint_key_id, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + input_fingerprint_digest TEXT NOT NULL CHECK ( + length(input_fingerprint_digest) = 64 + AND input_fingerprint_digest NOT GLOB '*[^0-9a-f]*' + ), + observed_generation INTEGER NOT NULL DEFAULT 1 CHECK (observed_generation >= 1), + processed_generation INTEGER NOT NULL DEFAULT 0 CHECK (processed_generation >= 0), + state TEXT NOT NULL DEFAULT 'dirty' + CHECK (state IN ('idle','dirty','inflight','failed')), + lease_owner TEXT, + lease_fence INTEGER NOT NULL DEFAULT 0 CHECK (lease_fence >= 0), + lease_expires_at INTEGER, + retry_count INTEGER NOT NULL DEFAULT 0 CHECK (retry_count >= 0), + next_retry_at INTEGER, + last_error_code TEXT CHECK ( + last_error_code IS NULL + OR (length(last_error_code) = 14 + AND last_error_code GLOB 'LBR-MEMORY-[0-9][0-9][0-9]') + ), + last_error_summary TEXT CHECK ( + last_error_summary IS NULL + OR length(CAST(last_error_summary AS BLOB)) <= 1024 + ), + created_at INTEGER NOT NULL CHECK (created_at >= 0), + updated_at INTEGER NOT NULL CHECK (updated_at >= created_at), + PRIMARY KEY (scope_key, root_kind, root_id), + CHECK (processed_generation <= observed_generation), + CHECK ( + (lease_owner IS NULL AND lease_expires_at IS NULL) + OR + (lease_owner IS NOT NULL AND lease_expires_at IS NOT NULL AND lease_fence > 0) + ), + CHECK (lease_expires_at IS NULL OR lease_expires_at >= 0), + CHECK (next_retry_at IS NULL OR next_retry_at >= 0), + CHECK (last_error_summary IS NULL OR last_error_code IS NOT NULL), + CHECK ( + (state = 'idle' + AND processed_generation = observed_generation + AND lease_owner IS NULL AND lease_expires_at IS NULL + AND retry_count = 0 AND next_retry_at IS NULL + AND last_error_code IS NULL AND last_error_summary IS NULL) + OR + (state = 'dirty' + AND processed_generation < observed_generation + AND lease_owner IS NULL AND lease_expires_at IS NULL) + OR + (state = 'inflight' + AND processed_generation < observed_generation + AND lease_owner IS NOT NULL AND lease_expires_at IS NOT NULL + AND next_retry_at IS NULL) + OR + (state = 'failed' + AND processed_generation < observed_generation + AND lease_owner IS NULL AND lease_expires_at IS NULL + AND retry_count > 0 AND next_retry_at IS NULL + AND last_error_code IS NOT NULL) + ), + CHECK (next_retry_at IS NULL OR state = 'dirty'), + CHECK ( + retry_count > 0 + OR (next_retry_at IS NULL + AND last_error_code IS NULL AND last_error_summary IS NULL) + ) +); +CREATE INDEX idx_memory_compile_job_runnable + ON memory_compile_job(state, next_retry_at, lease_expires_at, updated_at); +CREATE INDEX idx_memory_compile_job_scope_generation + ON memory_compile_job(scope_key, observed_generation, processed_generation); + +-- 每个 source ref 的 first-parent 增量扫描水位。名称使用 reference 表中的 +-- Branch.name，而不是展示用 full refname。 +CREATE TABLE memory_compile_observer_state ( + scope_key TEXT NOT NULL CHECK (length(CAST(scope_key AS BLOB)) BETWEEN 1 AND 512), + source_ref_name TEXT NOT NULL + CHECK (source_ref_name IN ('libra/intent','libra/memory/repo')), + scanned_through_oid TEXT NOT NULL CHECK ( + length(scanned_through_oid) IN (40,64) + AND scanned_through_oid NOT GLOB '*[^0-9a-f]*' + ), + updated_at INTEGER NOT NULL CHECK (updated_at >= 0), + PRIMARY KEY (scope_key, source_ref_name) +); + +-- 以下均为后续长期方案，不属于 2026082401_memory_core。 -- 实体 mention / alias 反向索引。历史真相是 MemoryNote.entities； -- canonical_key 与 alias_key 均为经过规范化的 repository-local key。 @@ -984,17 +1257,6 @@ CREATE TABLE memory_taxonomy_node ( CREATE INDEX idx_memory_taxonomy_parent ON memory_taxonomy_node(scope_key, namespace, parent_path); --- 每个 memory ref 的已构建水位线。安全敏感读取要求 --- projected_ref_oid == current_ref_oid。 -CREATE TABLE memory_projection_state ( - scope_key TEXT PRIMARY KEY, - projected_ref_oid TEXT NOT NULL, - last_event_seq INTEGER NOT NULL, - schema_version INTEGER NOT NULL, - policy_version TEXT NOT NULL, - rebuilt_at TEXT NOT NULL -); - -- 本地访问统计，不是 Git 投影，不参与默认 deterministic ranking。 CREATE TABLE memory_access_stats ( scope_key TEXT NOT NULL, @@ -1071,13 +1333,65 @@ CREATE TABLE context_selection_receipt_retention ( ); ``` -`memory_head`、`memory_path_summary`、`memory_note_index`、`memory_revision_index`、`memory_link_index`、`memory_entity_index`、`memory_taxonomy_node`、`memory_projection_state` 是可重建投影。`memory_classifier_cache` 与 `memory_embedding_cache`（§8.7）是可丢弃 cache；`memory_access_stats`、`context_selection_receipt` 与 `context_selection_receipt_retention` 是本地有界账本及其保留水位，不能从 Git 历史重建，`rebuild` 不触碰它们。删除账本会降低本地可观测性，但不能改变 live memory 语义。 +#### 5.2.1 M2-02 observation transaction + +M2-02 的 `job_sql` 是 crate-private 深 Module。调用方提交一次 pinned scan 的 +完整结果，不能直接拼表名、SQL 或 generation update。最小 Interface 为： + +```rust,ignore +record_observation_batch( + db, + ObservationBatch { + scope_key, + source_ref_name, + expected_cursor, + scanned_through_oid, + roots, + }, +) -> Result +``` + +`ObservedRoot` 只接收已经通过 M2-01 constructor 的 `EpisodeRoot`、已解析的 +`ObjectHash` 与 purpose-specific `SourceInputFingerprint`。后者是 +`KeyedDigestEnvelope` 的私有 newtype，只公开 `version/key_id/digest` getter； +通用 envelope 或任意自定义 purpose 不能进入 `job_sql`。同一 batch 内 root key +必须唯一；空 `roots` 合法，用于记录“该扫描区间没有终态 root”。 + +Module 使用现有 SQLite writer-lock helper 开启一个短事务，并按固定顺序执行： + +1. 读取 `(scope_key, source_ref_name)` 当前 cursor，比较 `expected_cursor`； +2. 幂等登记 batch 内全部 root job； +3. CAS 创建或推进 `scanned_through_oid`； +4. commit。任一步或 commit 失败都整批回滚。 + +generation 比较完整 `(version, key_id, digest)`： + +- 不存在 job 时创建 `observed_generation=1`、`processed_generation=0`、`state=dirty`； +- fingerprint 与 `terminal_source_oid` 均相同时不改 job 的 generation、lease、retry、error 或 timestamp； +- fingerprint 相同而 source OID 不同时返回稳定的 source-mismatch 错误并整批回滚； +- fingerprint 变化时只把 `observed_generation` 加一，更新 source/fingerprint，保持 `processed_generation` 与递增过的 `lease_fence`；`state='inflight'` 时原样保留 lease 与 state，其余状态转为 `dirty` 并清除旧 generation 的 retry/error。M2-02 不读取当前时间，也不判断 `lease_expires_at` 是否已过期；租约接管由 M2-08 负责。 + +cursor 校验必须先拒绝“`expected_cursor == scanned_through_oid` 且 roots 非空”的 +非法 batch，再进入幂等判断。其余使用 compare-and-swap 语义：无行只接受 `expected_cursor=None`；当前值等于 +expected 时正常推进；当前值已经等于 `scanned_through_oid` 时返回 +`AlreadyRecorded` 且不再碰 job，用于恢复“数据库已提交但调用方未收到成功”； +其它差异返回 `CursorConflict`。`expected_cursor == scanned_through_oid` 只允许空 +batch。first-parent 后代关系与扫描预算由 M2-08 observer 在调用前验证，SQL +Module 不读取 Git 对象。 + +本卡只实现上述 observation transaction。ref 扫描、terminal 判断、canonical +input 构造、lease acquire/renew/takeover、runner、compiler、retry scheduling 与 +processed-generation completion 均属于 M2-08。down migration 仅在九张 M2-02 +表全部为空时按 `observer/job → episode_path/link/head → revision → note → projection_state/path_summary` +的依赖顺序删除；任何表有行都以稳定错误拒绝，不能级联丢状态。 + +`memory_head`、`memory_path_summary`、`memory_note_index`、`memory_revision_index`、`memory_link_index`、`memory_projection_state`、`memory_episode_path` 是 M2-02 的可重建投影。`memory_compile_job` 与 `memory_compile_observer_state` 是有界本地运行状态。后续的 `memory_entity_index`、`memory_taxonomy_node` 仍是可重建投影；`memory_classifier_cache` 与 `memory_embedding_cache`（§8.7）是可丢弃 cache；`memory_access_stats`、`context_selection_receipt` 与 `context_selection_receipt_retention` 是本地有界账本及其保留水位，不能从 Git 历史重建，`rebuild` 不触碰它们。删除账本会降低本地可观测性，但不能改变 live memory 语义。 查询实现必须始终带上 `scope_key` 与 `namespace`，禁止只按 `path` 做全局查询后在内存中过滤。跨 scope / namespace 的检索只能由显式 `--all-namespaces` 或策略允许的 scope fallback 触发，并且必须在结果中保留原始 `scope` 与 `namespace`，防止 prompt 注入时发生来源混淆。 `list_prefix` 与 `summarize` 不得执行无上限扫描。实现应使用规范化后的 path 前缀范围查询和 keyset pagination，并设置默认 `LIMIT`（建议 100 条 summary、50 条 note）与硬上限。因为 SQL `LIKE` 的转义与 collation 容易引入前缀越界，推荐存储 canonical `path_key` / `parent_path` 后做复合索引范围查询；复杂度表述统一为 O(log n + k)，其中 k 是有界返回量。 -在访问模式上还有一条对齐约定值得明确：Memory 的投影表用 SeaORM entity 来访问（与同样可重建的 `ai_index_*` 投影一致），而不采用 `agent_session` / `agent_checkpoint` / `agent_usage_stats` 那种**故意**保持的 raw-SQL、无 entity 风格。原因在于：`agent_*` 那批表是外部捕获的独立账本，而 Memory 的这些表是 git 真源（`refs/libra/memory/...`）的可重建投影，本质与 `ai_index_*` 同类，因而对齐 `ai_index_*` 的 SeaORM 模式。账本例外是 `memory_access_stats` 与 `context_selection_receipt`（§8.6）：二者都不是可重建投影，沿用 raw-SQL 账本模式，不配 entity。 +在访问模式上还有一条对齐约定值得明确：Memory 的投影表用 SeaORM entity 来访问（与同样可重建的 `ai_index_*` 投影一致），而不采用 `agent_session` / `agent_checkpoint` / `agent_usage_stats` 那种**故意**保持的 raw-SQL、无 entity 风格。原因在于：`agent_*` 那批表是外部捕获的独立账本，而 Memory 的这些表是 git 真源（`refs/heads/libra/memory/...`）的可重建投影，本质与 `ai_index_*` 同类，因而对齐 `ai_index_*` 的 SeaORM 模式。账本例外是 `memory_access_stats` 与 `context_selection_receipt`（§8.6）：二者都不是可重建投影，沿用 raw-SQL 账本模式，不配 entity。 ### 5.3 ClientStorage 分层 @@ -1092,7 +1406,7 @@ CREATE TABLE context_selection_receipt_retention ( ### 5.4 远端发布与传输边界 -第一版所有 `refs/libra/memory/*` 都是 **local-only**，不会进入默认 push/fetch/clone，也不提供 `memory push`。这里的 `Repo` 表示同一仓库 / 关联 worktree 的本地共享 scope，不表示团队远端可见。原因是普通 Git ref 传输只有并发保护，没有 record-level ACL、redaction manifest、revocation propagation 或 remote principal authorization。 +第一版所有 `refs/heads/libra/memory/*` 都是 **local-only**，不会进入默认 push/fetch/clone，也不提供 `memory push`。这里的 `Repo` 表示同一仓库 / 关联 worktree 的本地共享 scope，不表示团队远端可见。原因是普通 Git ref 传输只有并发保护，没有 record-level ACL、redaction manifest、revocation propagation 或 remote principal authorization。 未来若增加团队 Memory publication，必须另行定义类似 `mainline.md` ML-01 的 allow-list manifest、隔离 tracking ref、ingress validation、lease、visibility/trust/sensitivity policy 与 tombstone 传播；不能直接 mirror 本地 memory ref。portable sealed intent / pin 仍由 `refs/libra/intent-team` 负责，Memory 不复制其 transport stack。该约束也意味着 `Confidential`、actor-private note、receipt、access stats 和原始 compile input HMAC 永不进入团队 publication。 @@ -1462,13 +1776,13 @@ query `scope = Branch("main")` 的 note 存放在 `libra/memory/branch/`（git ref 全名写作 -`refs/libra/memory/branch/`）。切换用户的工作分支 +`refs/heads/libra/memory/branch/`）。切换用户的工作分支 （通过 `libra switch`）会隐式切换被查询的作用域： ```text libra switch experiment -> agent 从 libra/memory/branch/ 读取 Branch 作用域， - 并按显式 precedence 合并 refs/libra/memory/repo 的 Repo 作用域 + 并按显式 precedence 合并 refs/heads/libra/memory/repo 的 Repo 作用域 ``` 这解决了 §2.1 中描述的「上下文污染」失效模式。 @@ -1504,7 +1818,7 @@ libra memory blame # 当前 head 由谁、于何时设定 ``` 这些都是 Libra 既有 `log` / `diff` / `blame` 命令之上的薄封装（thin shim）， -作用域限定在 `refs/libra/memory/...`。 +作用域限定在 `refs/heads/libra/memory/...`。 ### 9.3 merge 与 rebase @@ -1593,7 +1907,7 @@ libra memory revoke --reason "..." **Working 层（A3，四层巩固）。** 沿 §0.0.4（agentmemory 四层巩固）与 §0.2 决策， 归并需要一条显式、有界的 **Working 摄入缓冲**：原始 observation 只在本地 有界 intake 缓冲中作为 compiler 输入（§7.1 的 intake audit），**不写入 -`refs/libra/memory/*`，也不新增 note kind**——与「先编译，再使用」一致。 +`refs/heads/libra/memory/*`，也不新增 note kind**——与「先编译，再使用」一致。 Working 层不是 `MemoryNote` 快照，只是 intake 到确认之间的中间态： - 只保留经 redaction 的 observation 摘要 + 来源 + 脱敏输入 HMAC + TTL； @@ -1925,9 +2239,9 @@ agent 捕获使用的 `2026050303_agent_capture.sql` 迁移属于同一模式， - `memory_access_stats`（§5.2，本地账本，不参与 rebuild） - `context_selection_receipt`（§8.6，共享账本：append-only、豁免 rebuild、按保留策略有界修剪） -只有 `memory_head`、`memory_path_summary`、`memory_note_index`、`memory_revision_index`、`memory_link_index`、`memory_entity_index`、`memory_taxonomy_node`、`memory_projection_state` 可由 `libra memory rebuild` 从 `refs/libra/memory/...` 重建。classifier cache 与 embedding cache 可直接丢弃；access stats 与 receipt 是本地账本，不参与 rebuild。 +只有 `memory_head`、`memory_path_summary`、`memory_note_index`、`memory_revision_index`、`memory_link_index`、`memory_entity_index`、`memory_taxonomy_node`、`memory_projection_state` 可由 `libra memory rebuild` 从 `refs/heads/libra/memory/...` 重建。classifier cache 与 embedding cache 可直接丢弃；access stats 与 receipt 是本地账本，不参与 rebuild。 -协调 namespace（§19.3）的**路径占用索引**（§19.4 claim 跨 task 重叠检查）属于可重建投影：它由 `coordination.*` 活 claim 的规范化 `body` 路径范围派生，随 `refs/libra/memory/...` 事件重放重建，删除后可随 rebuild 恢复；其结构与实体索引同模式（见 §19.4 设计边界的索引规模与性能预算）。`CoordinationView`（§19.5）是读取时对该索引的投影视图，不新增独立存储。 +协调 namespace（§19.3）的**路径占用索引**（§19.4 claim 跨 task 重叠检查）属于可重建投影：它由 `coordination.*` 活 claim 的规范化 `body` 路径范围派生，随 `refs/heads/libra/memory/...` 事件重放重建，删除后可随 rebuild 恢复；其结构与实体索引同模式（见 §19.4 设计边界的索引规模与性能预算）。`CoordinationView`（§19.5）是读取时对该索引的投影视图，不新增独立存储。 值得一提的是模式选择上的对比：Memory 的投影表用 SeaORM entity 建模（与同样可 重建的 `ai_index_*` 投影同模式），而外部捕获的 `agent_session` / @@ -1943,7 +2257,7 @@ Memory 之所以对齐 `ai_index_*` 的 SeaORM 模式，是因为它的表本身 - 在 `src/internal/ai/memory/` 中定义 `MemoryNote` / `MemoryEvent` 的 Rust 类型。 - 建立唯一的 `MemoryWriter` Module 与 Adapter contract；Phase A 的 CLI 写入只通过该 seam，writer 内部完成对象持久化、事件序号、CAS 与投影事务。 -- 对 repo memory ref（`refs/libra/memory/repo`）的写入 / 读取；每个 commit 单父、`event_seq` 连续、CAS 有界重试。 +- 对 repo memory ref（`refs/heads/libra/memory/repo`）的写入 / 读取；每个 commit 单父、`event_seq` 连续、CAS 有界重试。 - SQLite 投影（§5.2）+ Sea-ORM entity： `memory_head`、`memory_path_summary`、`memory_note_index`、 `memory_revision_index`、`memory_link_index`、`memory_entity_index`、 @@ -2185,7 +2499,7 @@ Memory 只有在配齐有针对性的回归覆盖后才发布： > 本节曾于 2026-08-10 拆分为独立文档 `docs/development/tracing/memory-coordination.md`，现合并回本文（2026-08-10）。与主 Memory 设计共享的所有原语、约束与验收口径以本文为准。 -本设计为本文（主 Memory 设计）的**相邻但正交**扩展：多个 Agent 在同一仓库中**并行执行开发工作**时，通过 Memory 通道相互通信与协调。它与持久知识共享同一套基础设施（`MemoryWriter` seam、`refs/libra/memory/*`、事件系统、CAS、Working 缓冲），但语义、生命周期与授权目标不同。本设计**不新增存储层、不新增第二套并发机制**，完全构建在本文 §3（概念模型）/ §4（对象模型）/ §5（存储布局）之上：一个保留 namespace（`coordination`）+ 一个 `MemoryCoordinator` 入口 + 一个 `CoordinationView` 投影，并复用 Working 缓冲（§10.5）与 Trust Gate（§7.5.1）。`CompileRecord.origin` 相应新增 `Coordinator` 值（§4.1.1 的 additive 扩展）。 +本设计为本文（主 Memory 设计）的**相邻但正交**扩展：多个 Agent 在同一仓库中**并行执行开发工作**时，通过 Memory 通道相互通信与协调。它与持久知识共享同一套基础设施（`MemoryWriter` seam、`refs/heads/libra/memory/*`、事件系统、CAS、Working 缓冲），但语义、生命周期与授权目标不同。本设计**不新增存储层、不新增第二套并发机制**，完全构建在本文 §3（概念模型）/ §4（对象模型）/ §5（存储布局）之上：一个保留 namespace（`coordination`）+ 一个 `MemoryCoordinator` 入口 + 一个 `CoordinationView` 投影，并复用 Working 缓冲（§10.5）与 Trust Gate（§7.5.1）。`CompileRecord.origin` 相应新增 `Coordinator` 值（§4.1.1 的 additive 扩展）。 ### 19.1 开发者问题 @@ -2240,7 +2554,7 @@ coordination.conflicts. # 变更面冲突声明（accretive， | `body` | 有界正文：声明的文件/路径范围、移交的交付物摘要、进度的一行状态 | | `evidence_refs` | 移交/冲突可选引用 commit OID / Run / Evidence | -**存储位置（与 §19.2 / §19.4 / §19.5 / §19.7 / §19.8 一致）。** 协调条目（claim / release / handoff / progress / conflict-declare / sync-point）**直接写入共享 memory ref**（`refs/libra/memory/...` 下 Repo scope 的 `coordination` namespace，经 `MemoryWriter.MemoryCoordinator` 与 ref CAS），**不是** Working 缓冲中的 ephemeral 状态： +**存储位置（与 §19.2 / §19.4 / §19.5 / §19.7 / §19.8 一致）。** 协调条目（claim / release / handoff / progress / conflict-declare / sync-point）**直接写入共享 memory ref**（`refs/heads/libra/memory/...` 下 Repo scope 的 `coordination` namespace，经 `MemoryWriter.MemoryCoordinator` 与 ref CAS），**不是** Working 缓冲中的 ephemeral 状态： - 短 TTL 只控制 `CoordinationView` 的可见性与默认召回排除（§19.6）；历史事件与 note 保留在 ref 上可 `libra memory log` 审计（§10.7），这也满足 §19.8「从 refs 可重建」的验收。 - Working 缓冲（§10.5）只承接**巩固（consolidation）**的输入与输出：归并作业读取近期协调条目 + 原始 observation 摘要，产出候选 semantic / procedural note 并经 Trust Gate 确认；协调条目本身不落 Working 缓冲。若 claim / handoff 只存在本地缓冲，跨进程不可见、`CoordinationView` 只能看到本地声明、单写者赢也无法跨进程由 CAS 保证——那将整体破坏 §19.1 的协调目标。 @@ -2317,7 +2631,7 @@ MEM-06 **不新增存储层、不新增第二套并发机制**，完全构建在 - **过期不毒化**：claim 的 TTL 过期后从 `CoordinationView` 排除，历史仍在 `log` 可审计；不因过期条目拒绝新 claim。 - **冲突声明**：两个 agent 并行改重叠路径，`conflict_declare` 触发 `contradicts` 链接并进入隔离；`CoordinationView` 显示未解冲突。 - **不越权**：`SecretLike` / `Confidential` 不进协调通道；`actor` 不信任自报。 -- **无第二真源**：协调条目直接写入共享 memory ref、仍从 `refs/libra/memory/*` 可重建；Working 缓冲只承载 consolidation 输入/输出、可丢弃；`MemoryCoordinator` 不绕过 `MemoryWriter`。 +- **无第二真源**：协调条目直接写入共享 memory ref、仍从 `refs/heads/libra/memory/*` 可重建；Working 缓冲只承载 consolidation 输入/输出、可丢弃；`MemoryCoordinator` 不绕过 `MemoryWriter`。 - **与并行工作区衔接**：跨 worktree（§9.1 / [`libra-worktree-architecture.md`](../libra-worktree-architecture.md)）的 agent 都能读到同一 repo 级协调（Repo scope 共享），worktree 级协调按需隔离。 ### 19.9 分阶段与优先级 @@ -2376,7 +2690,7 @@ MEM-06 依赖 MEM-01（存储/隐私）与 MEM-03（Trust Gate / 巩固），建 ### 20.3 保留的强项（不修改） -1. Snapshot/Event/Projection 三层模型 + `refs/libra/memory/*` 线性历史 + CAS 并发（§0.1、§4）。 +1. Snapshot/Event/Projection 三层模型 + `refs/heads/libra/memory/*` 线性历史 + CAS 并发（§0.1、§4）。 2. CompileRecord / ContextReceipt 的确定性可重放承诺（§0.2、§8.6）。 3. 分层 scope（Actor→Worktree→Branch→Repo→Global）与 fail-closed 读取（§3.4）。 4. 遗忘/脱敏语义、MCP C9 门禁、`SecretLike` 边界（§10.6、§13）。 diff --git a/sql/migrations/2026082401_memory_core.sql b/sql/migrations/2026082401_memory_core.sql new file mode 100644 index 000000000..252f90736 --- /dev/null +++ b/sql/migrations/2026082401_memory_core.sql @@ -0,0 +1,403 @@ +-- M2-02: core Agent Memory projections and bounded compiler observation state. +-- +-- The seven projection tables are rebuildable from the repository Memory ref. +-- The job and observer tables are local coordination state. FTS, receipts, +-- replay, writer objects, and compiler execution belong to later migrations. + +CREATE TABLE IF NOT EXISTS `memory_note_index` ( + `note_id` TEXT PRIMARY KEY CHECK ( + length(`note_id`) = 36 + AND substr(`note_id`, 9, 1) = '-' + AND substr(`note_id`, 14, 1) = '-' + AND substr(`note_id`, 19, 1) = '-' + AND substr(`note_id`, 24, 1) = '-' + AND length(replace(`note_id`, '-', '')) = 32 + AND replace(`note_id`, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + `scope_key` TEXT NOT NULL + CHECK (length(CAST(`scope_key` AS BLOB)) BETWEEN 1 AND 512), + `namespace` TEXT NOT NULL + CHECK (length(CAST(`namespace` AS BLOB)) BETWEEN 1 AND 512), + `path` TEXT NOT NULL + CHECK (length(CAST(`path` AS BLOB)) BETWEEN 1 AND 4096), + `kind` TEXT NOT NULL + CHECK (`kind` IN ('procedural','semantic','episodic')), + `lifecycle` TEXT NOT NULL + CHECK (`lifecycle` IN ('replacement','accretive')), + `review_state` TEXT NOT NULL + CHECK (`review_state` IN ('draft','confirmed','quarantined','revoked','superseded','forgotten')), + `confidence` TEXT NOT NULL + CHECK (`confidence` IN ('low','medium','high')), + `trust` TEXT NOT NULL + CHECK (`trust` IN ('verified','repo_evidence','user_asserted','external_untrusted','inferred')), + `sensitivity` TEXT NOT NULL + CHECK (`sensitivity` IN ('public','internal','confidential','secret_like')), + `visibility` TEXT NOT NULL + CHECK (`visibility` IN ('private','repo_local','team_candidate')), + `acl_policy_id` TEXT NOT NULL, + `origin` TEXT NOT NULL CHECK (`origin` IN ( + 'explicit','promoted_from_anchor','distilled_from_frame','classifier', + 'consolidation','onboard','branch_fork','import','coordinator','episode_compiler' + )), + `idempotency_key` TEXT NOT NULL, + `idempotency_scope` TEXT NOT NULL DEFAULT 'cell' + CHECK (`idempotency_scope` IN ('cell','namespace')), + `created_at` TEXT NOT NULL, + UNIQUE (`scope_key`, `namespace`, `path`, `note_id`), + UNIQUE (`note_id`, `scope_key`, `namespace`) +); + +CREATE UNIQUE INDEX IF NOT EXISTS `idx_memory_note_idempotency_cell` + ON `memory_note_index`(`scope_key`, `namespace`, `path`, `idempotency_key`) + WHERE `idempotency_scope` = 'cell'; +CREATE UNIQUE INDEX IF NOT EXISTS `idx_memory_note_idempotency_ns` + ON `memory_note_index`(`scope_key`, `namespace`, `idempotency_key`) + WHERE `idempotency_scope` = 'namespace'; + +CREATE TABLE IF NOT EXISTS `memory_revision_index` ( + `revision_oid` TEXT PRIMARY KEY CHECK ( + length(`revision_oid`) IN (40,64) + AND `revision_oid` NOT GLOB '*[^0-9a-f]*' + ), + `note_id` TEXT NOT NULL CHECK ( + length(`note_id`) = 36 + AND substr(`note_id`, 9, 1) = '-' + AND substr(`note_id`, 14, 1) = '-' + AND substr(`note_id`, 19, 1) = '-' + AND substr(`note_id`, 24, 1) = '-' + AND length(replace(`note_id`, '-', '')) = 32 + AND replace(`note_id`, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + `scope_key` TEXT NOT NULL + CHECK (length(CAST(`scope_key` AS BLOB)) BETWEEN 1 AND 512), + `namespace` TEXT NOT NULL + CHECK (length(CAST(`namespace` AS BLOB)) BETWEEN 1 AND 512), + `origin` TEXT NOT NULL CHECK (`origin` IN ( + 'explicit','promoted_from_anchor','distilled_from_frame','classifier', + 'consolidation','onboard','branch_fork','import','coordinator','episode_compiler' + )), + `producer` TEXT NOT NULL, + `rules_version` INTEGER NOT NULL CHECK (`rules_version` > 0), + `prompt_version` TEXT, + `model_id` TEXT, + `policy_version` TEXT NOT NULL, + `input_fingerprints_json` TEXT NOT NULL CHECK ( + json_valid(`input_fingerprints_json`) + AND json_type(`input_fingerprints_json`) = 'array' + ), + `created_at` TEXT NOT NULL, + UNIQUE (`note_id`, `revision_oid`), + UNIQUE (`scope_key`, `namespace`, `note_id`, `revision_oid`), + FOREIGN KEY (`note_id`, `scope_key`, `namespace`) + REFERENCES `memory_note_index`(`note_id`, `scope_key`, `namespace`) + ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS `idx_memory_revision_note` + ON `memory_revision_index`(`note_id`, `created_at`, `revision_oid`); +CREATE INDEX IF NOT EXISTS `idx_memory_revision_producer` + ON `memory_revision_index`( + `scope_key`, `namespace`, `producer`, `prompt_version`, `model_id`, `policy_version` + ); + +CREATE TABLE IF NOT EXISTS `memory_path_summary` ( + `scope_key` TEXT NOT NULL + CHECK (length(CAST(`scope_key` AS BLOB)) BETWEEN 1 AND 512), + `namespace` TEXT NOT NULL + CHECK (length(CAST(`namespace` AS BLOB)) BETWEEN 1 AND 512), + `path` TEXT NOT NULL + CHECK (length(CAST(`path` AS BLOB)) BETWEEN 1 AND 4096), + `confirmed_count` INTEGER NOT NULL DEFAULT 0 CHECK (`confirmed_count` >= 0), + `quarantined_count` INTEGER NOT NULL DEFAULT 0 CHECK (`quarantined_count` >= 0), + `child_count` INTEGER NOT NULL DEFAULT 0 CHECK (`child_count` >= 0), + `prefix_count` INTEGER NOT NULL DEFAULT 0 CHECK (`prefix_count` >= 0), + `preview` TEXT NOT NULL DEFAULT '', + `last_changed_at` TEXT NOT NULL, + PRIMARY KEY (`scope_key`, `namespace`, `path`) +); + +CREATE INDEX IF NOT EXISTS `idx_memory_path_summary_prefix` + ON `memory_path_summary`(`scope_key`, `namespace`, `path`); + +CREATE TABLE IF NOT EXISTS `memory_projection_state` ( + `scope_key` TEXT PRIMARY KEY + CHECK (length(CAST(`scope_key` AS BLOB)) BETWEEN 1 AND 512), + `projected_ref_oid` TEXT NOT NULL CHECK ( + length(`projected_ref_oid`) IN (40,64) + AND `projected_ref_oid` NOT GLOB '*[^0-9a-f]*' + ), + `last_event_seq` INTEGER NOT NULL CHECK (`last_event_seq` >= 0), + `schema_version` INTEGER NOT NULL CHECK (`schema_version` > 0), + `policy_version` TEXT NOT NULL, + `rebuilt_at` INTEGER NOT NULL CHECK (`rebuilt_at` >= 0) +); + +CREATE TABLE IF NOT EXISTS `memory_head` ( + `scope_key` TEXT NOT NULL + CHECK (length(CAST(`scope_key` AS BLOB)) BETWEEN 1 AND 512), + `namespace` TEXT NOT NULL + CHECK (length(CAST(`namespace` AS BLOB)) BETWEEN 1 AND 512), + `path` TEXT NOT NULL + CHECK (length(CAST(`path` AS BLOB)) BETWEEN 1 AND 4096), + `note_id` TEXT NOT NULL CHECK ( + length(`note_id`) = 36 + AND substr(`note_id`, 9, 1) = '-' + AND substr(`note_id`, 14, 1) = '-' + AND substr(`note_id`, 19, 1) = '-' + AND substr(`note_id`, 24, 1) = '-' + AND length(replace(`note_id`, '-', '')) = 32 + AND replace(`note_id`, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + `latest_revision_oid` TEXT NOT NULL CHECK ( + length(`latest_revision_oid`) IN (40,64) + AND `latest_revision_oid` NOT GLOB '*[^0-9a-f]*' + ), + `live_revision_oid` TEXT CHECK ( + `live_revision_oid` IS NULL + OR (length(`live_revision_oid`) IN (40,64) + AND `live_revision_oid` NOT GLOB '*[^0-9a-f]*') + ), + `latest_action` TEXT NOT NULL CHECK (`latest_action` IN ( + 'created','revised','confirmed','quarantined','superseded', + 'revoked','forgotten','consolidated' + )), + `latest_review_state` TEXT NOT NULL CHECK ( + `latest_review_state` IN ('draft','confirmed','quarantined','revoked','superseded','forgotten') + ), + `kind` TEXT NOT NULL + CHECK (`kind` IN ('procedural','semantic','episodic')), + `lifecycle` TEXT NOT NULL + CHECK (`lifecycle` IN ('replacement','accretive')), + `confidence` TEXT NOT NULL + CHECK (`confidence` IN ('low','medium','high')), + `trust` TEXT NOT NULL + CHECK (`trust` IN ('verified','repo_evidence','user_asserted','external_untrusted','inferred')), + `sensitivity` TEXT NOT NULL + CHECK (`sensitivity` IN ('public','internal','confidential','secret_like')), + `visibility` TEXT NOT NULL + CHECK (`visibility` IN ('private','repo_local','team_candidate')), + `acl_policy_id` TEXT NOT NULL, + `valid_from` TEXT, + `valid_until` TEXT, + `effective_from_commit` TEXT CHECK ( + `effective_from_commit` IS NULL + OR (length(`effective_from_commit`) IN (40,64) + AND `effective_from_commit` NOT GLOB '*[^0-9a-f]*') + ), + `effective_until_commit` TEXT CHECK ( + `effective_until_commit` IS NULL + OR (length(`effective_until_commit`) IN (40,64) + AND `effective_until_commit` NOT GLOB '*[^0-9a-f]*') + ), + `expires_at` TEXT, + `rank_hint` INTEGER NOT NULL DEFAULT 0, + `last_event_seq` INTEGER NOT NULL CHECK (`last_event_seq` >= 1), + `updated_at` TEXT NOT NULL, + PRIMARY KEY (`scope_key`, `namespace`, `path`, `note_id`), + UNIQUE (`note_id`), + FOREIGN KEY (`scope_key`, `namespace`, `path`, `note_id`) + REFERENCES `memory_note_index`(`scope_key`, `namespace`, `path`, `note_id`), + FOREIGN KEY (`scope_key`, `namespace`, `note_id`, `latest_revision_oid`) + REFERENCES `memory_revision_index`(`scope_key`, `namespace`, `note_id`, `revision_oid`), + FOREIGN KEY (`scope_key`, `namespace`, `note_id`, `live_revision_oid`) + REFERENCES `memory_revision_index`(`scope_key`, `namespace`, `note_id`, `revision_oid`) +); + +CREATE INDEX IF NOT EXISTS `idx_memory_head_lookup` + ON `memory_head`(`scope_key`, `namespace`, `path`, `latest_review_state`); +CREATE INDEX IF NOT EXISTS `idx_memory_head_path_prefix` + ON `memory_head`(`scope_key`, `namespace`, `path`); + +CREATE TABLE IF NOT EXISTS `memory_link_index` ( + `source_scope_key` TEXT NOT NULL + CHECK (length(CAST(`source_scope_key` AS BLOB)) BETWEEN 1 AND 512), + `source_namespace` TEXT NOT NULL + CHECK (length(CAST(`source_namespace` AS BLOB)) BETWEEN 1 AND 512), + `source_note_id` TEXT NOT NULL CHECK ( + length(`source_note_id`) = 36 + AND substr(`source_note_id`, 9, 1) = '-' + AND substr(`source_note_id`, 14, 1) = '-' + AND substr(`source_note_id`, 19, 1) = '-' + AND substr(`source_note_id`, 24, 1) = '-' + AND length(replace(`source_note_id`, '-', '')) = 32 + AND replace(`source_note_id`, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + `source_revision_oid` TEXT NOT NULL CHECK ( + length(`source_revision_oid`) IN (40,64) + AND `source_revision_oid` NOT GLOB '*[^0-9a-f]*' + ), + `target_note_id` TEXT NOT NULL CHECK ( + length(`target_note_id`) = 36 + AND substr(`target_note_id`, 9, 1) = '-' + AND substr(`target_note_id`, 14, 1) = '-' + AND substr(`target_note_id`, 19, 1) = '-' + AND substr(`target_note_id`, 24, 1) = '-' + AND length(replace(`target_note_id`, '-', '')) = 32 + AND replace(`target_note_id`, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + `target_revision_oid` TEXT CHECK ( + `target_revision_oid` IS NULL + OR (length(`target_revision_oid`) IN (40,64) + AND `target_revision_oid` NOT GLOB '*[^0-9a-f]*') + ), + `link_kind` TEXT NOT NULL CHECK ( + `link_kind` IN ('sibling','supports','prerequisite','contradicts','supersedes') + ), + `source_path` TEXT NOT NULL, + `target_path` TEXT NOT NULL, + `evidence_refs_json` TEXT NOT NULL CHECK ( + json_valid(`evidence_refs_json`) + AND json_type(`evidence_refs_json`) = 'array' + ), + `valid_from` TEXT, + `valid_until` TEXT, + PRIMARY KEY (`source_revision_oid`, `target_note_id`, `link_kind`), + FOREIGN KEY (`source_scope_key`, `source_namespace`, `source_path`, `source_note_id`) + REFERENCES `memory_note_index`(`scope_key`, `namespace`, `path`, `note_id`) + ON DELETE CASCADE, + FOREIGN KEY (`source_scope_key`, `source_namespace`, `source_note_id`, `source_revision_oid`) + REFERENCES `memory_revision_index`(`scope_key`, `namespace`, `note_id`, `revision_oid`) + ON DELETE CASCADE, + FOREIGN KEY (`target_note_id`) + REFERENCES `memory_note_index`(`note_id`) ON DELETE CASCADE, + FOREIGN KEY (`target_note_id`, `target_revision_oid`) + REFERENCES `memory_revision_index`(`note_id`, `revision_oid`) +); + +CREATE INDEX IF NOT EXISTS `idx_memory_link_source` + ON `memory_link_index`(`source_scope_key`, `source_namespace`, `source_note_id`); +CREATE INDEX IF NOT EXISTS `idx_memory_link_target` + ON `memory_link_index`(`target_note_id`, `target_revision_oid`); + +CREATE TABLE IF NOT EXISTS `memory_episode_path` ( + `note_id` TEXT NOT NULL CHECK ( + length(`note_id`) = 36 + AND substr(`note_id`, 9, 1) = '-' + AND substr(`note_id`, 14, 1) = '-' + AND substr(`note_id`, 19, 1) = '-' + AND substr(`note_id`, 24, 1) = '-' + AND length(replace(`note_id`, '-', '')) = 32 + AND replace(`note_id`, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + `revision_oid` TEXT NOT NULL CHECK ( + length(`revision_oid`) IN (40,64) + AND `revision_oid` NOT GLOB '*[^0-9a-f]*' + ), + `code_path` TEXT NOT NULL CHECK ( + length(CAST(`code_path` AS BLOB)) BETWEEN 1 AND 4096 + AND substr(`code_path`, 1, 1) <> '/' + AND instr(`code_path`, char(92)) = 0 + ), + PRIMARY KEY (`note_id`, `revision_oid`, `code_path`), + FOREIGN KEY (`note_id`, `revision_oid`) + REFERENCES `memory_revision_index`(`note_id`, `revision_oid`) + ON DELETE CASCADE +); + +CREATE INDEX IF NOT EXISTS `idx_memory_episode_path_code` + ON `memory_episode_path`(`code_path`, `note_id`, `revision_oid`); + +CREATE TABLE IF NOT EXISTS `memory_compile_job` ( + `scope_key` TEXT NOT NULL + CHECK (length(CAST(`scope_key` AS BLOB)) BETWEEN 1 AND 512), + `root_kind` TEXT NOT NULL CHECK (`root_kind` IN ('task','intent')), + `root_id` TEXT NOT NULL CHECK ( + length(`root_id`) > 0 AND length(CAST(`root_id` AS BLOB)) <= 120 + ), + `terminal_source_oid` TEXT NOT NULL CHECK ( + length(`terminal_source_oid`) IN (40,64) + AND `terminal_source_oid` NOT GLOB '*[^0-9a-f]*' + ), + `input_fingerprint_version` INTEGER NOT NULL + CHECK (`input_fingerprint_version` = 1), + `input_fingerprint_key_id` TEXT NOT NULL CHECK ( + length(`input_fingerprint_key_id`) = 36 + AND substr(`input_fingerprint_key_id`, 9, 1) = '-' + AND substr(`input_fingerprint_key_id`, 14, 1) = '-' + AND substr(`input_fingerprint_key_id`, 19, 1) = '-' + AND substr(`input_fingerprint_key_id`, 24, 1) = '-' + AND length(replace(`input_fingerprint_key_id`, '-', '')) = 32 + AND replace(`input_fingerprint_key_id`, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + `input_fingerprint_digest` TEXT NOT NULL CHECK ( + length(`input_fingerprint_digest`) = 64 + AND `input_fingerprint_digest` NOT GLOB '*[^0-9a-f]*' + ), + `observed_generation` INTEGER NOT NULL DEFAULT 1 + CHECK (`observed_generation` >= 1), + `processed_generation` INTEGER NOT NULL DEFAULT 0 + CHECK (`processed_generation` >= 0), + `state` TEXT NOT NULL DEFAULT 'dirty' + CHECK (`state` IN ('idle','dirty','inflight','failed')), + `lease_owner` TEXT, + `lease_fence` INTEGER NOT NULL DEFAULT 0 CHECK (`lease_fence` >= 0), + `lease_expires_at` INTEGER, + `retry_count` INTEGER NOT NULL DEFAULT 0 CHECK (`retry_count` >= 0), + `next_retry_at` INTEGER, + `last_error_code` TEXT CHECK ( + `last_error_code` IS NULL + OR (length(`last_error_code`) = 14 + AND `last_error_code` GLOB 'LBR-MEMORY-[0-9][0-9][0-9]') + ), + `last_error_summary` TEXT CHECK ( + `last_error_summary` IS NULL + OR length(CAST(`last_error_summary` AS BLOB)) <= 1024 + ), + `created_at` INTEGER NOT NULL CHECK (`created_at` >= 0), + `updated_at` INTEGER NOT NULL CHECK (`updated_at` >= `created_at`), + PRIMARY KEY (`scope_key`, `root_kind`, `root_id`), + CHECK (`processed_generation` <= `observed_generation`), + CHECK ( + (`lease_owner` IS NULL AND `lease_expires_at` IS NULL) + OR (`lease_owner` IS NOT NULL AND `lease_expires_at` IS NOT NULL AND `lease_fence` > 0) + ), + CHECK (`lease_expires_at` IS NULL OR `lease_expires_at` >= 0), + CHECK (`next_retry_at` IS NULL OR `next_retry_at` >= 0), + CHECK (`last_error_summary` IS NULL OR `last_error_code` IS NOT NULL), + CHECK ( + (`state` = 'idle' + AND `processed_generation` = `observed_generation` + AND `lease_owner` IS NULL AND `lease_expires_at` IS NULL + AND `retry_count` = 0 AND `next_retry_at` IS NULL + AND `last_error_code` IS NULL AND `last_error_summary` IS NULL) + OR + (`state` = 'dirty' + AND `processed_generation` < `observed_generation` + AND `lease_owner` IS NULL AND `lease_expires_at` IS NULL) + OR + (`state` = 'inflight' + AND `processed_generation` < `observed_generation` + AND `lease_owner` IS NOT NULL AND `lease_expires_at` IS NOT NULL + AND `next_retry_at` IS NULL) + OR + (`state` = 'failed' + AND `processed_generation` < `observed_generation` + AND `lease_owner` IS NULL AND `lease_expires_at` IS NULL + AND `retry_count` > 0 AND `next_retry_at` IS NULL + AND `last_error_code` IS NOT NULL) + ), + CHECK (`next_retry_at` IS NULL OR `state` = 'dirty'), + CHECK ( + `retry_count` > 0 + OR (`next_retry_at` IS NULL + AND `last_error_code` IS NULL AND `last_error_summary` IS NULL) + ) +); + +CREATE INDEX IF NOT EXISTS `idx_memory_compile_job_runnable` + ON `memory_compile_job`(`state`, `next_retry_at`, `lease_expires_at`, `updated_at`); +CREATE INDEX IF NOT EXISTS `idx_memory_compile_job_scope_generation` + ON `memory_compile_job`(`scope_key`, `observed_generation`, `processed_generation`); + +CREATE TABLE IF NOT EXISTS `memory_compile_observer_state` ( + `scope_key` TEXT NOT NULL + CHECK (length(CAST(`scope_key` AS BLOB)) BETWEEN 1 AND 512), + `source_ref_name` TEXT NOT NULL + CHECK (`source_ref_name` IN ('libra/intent','libra/memory/repo')), + `scanned_through_oid` TEXT NOT NULL CHECK ( + length(`scanned_through_oid`) IN (40,64) + AND `scanned_through_oid` NOT GLOB '*[^0-9a-f]*' + ), + `updated_at` INTEGER NOT NULL CHECK (`updated_at` >= 0), + PRIMARY KEY (`scope_key`, `source_ref_name`) +); diff --git a/sql/migrations/2026082401_memory_core_down.sql b/sql/migrations/2026082401_memory_core_down.sql new file mode 100644 index 000000000..a7cc9fbd0 --- /dev/null +++ b/sql/migrations/2026082401_memory_core_down.sql @@ -0,0 +1,45 @@ +-- Rollback of 2026082401_memory_core. +-- +-- Every table in this migration must be empty. Once a projection or compiler +-- observation exists, repair moves forward; rollback must not erase it. + +CREATE TABLE IF NOT EXISTS `memory_core_down_guard` ( + `blocked` INTEGER NOT NULL, + CONSTRAINT `memory_core_down_guard_empty` CHECK (`blocked` = 0) +); + +INSERT INTO `memory_core_down_guard` (`blocked`) +SELECT + (SELECT COUNT(*) FROM `memory_compile_observer_state`) + + (SELECT COUNT(*) FROM `memory_compile_job`) + + (SELECT COUNT(*) FROM `memory_episode_path`) + + (SELECT COUNT(*) FROM `memory_link_index`) + + (SELECT COUNT(*) FROM `memory_head`) + + (SELECT COUNT(*) FROM `memory_revision_index`) + + (SELECT COUNT(*) FROM `memory_note_index`) + + (SELECT COUNT(*) FROM `memory_projection_state`) + + (SELECT COUNT(*) FROM `memory_path_summary`); + +DROP TABLE `memory_core_down_guard`; + +DROP TABLE IF EXISTS `memory_compile_observer_state`; +DROP INDEX IF EXISTS `idx_memory_compile_job_scope_generation`; +DROP INDEX IF EXISTS `idx_memory_compile_job_runnable`; +DROP TABLE IF EXISTS `memory_compile_job`; +DROP INDEX IF EXISTS `idx_memory_episode_path_code`; +DROP TABLE IF EXISTS `memory_episode_path`; +DROP INDEX IF EXISTS `idx_memory_link_target`; +DROP INDEX IF EXISTS `idx_memory_link_source`; +DROP TABLE IF EXISTS `memory_link_index`; +DROP INDEX IF EXISTS `idx_memory_head_path_prefix`; +DROP INDEX IF EXISTS `idx_memory_head_lookup`; +DROP TABLE IF EXISTS `memory_head`; +DROP INDEX IF EXISTS `idx_memory_revision_producer`; +DROP INDEX IF EXISTS `idx_memory_revision_note`; +DROP TABLE IF EXISTS `memory_revision_index`; +DROP INDEX IF EXISTS `idx_memory_note_idempotency_ns`; +DROP INDEX IF EXISTS `idx_memory_note_idempotency_cell`; +DROP TABLE IF EXISTS `memory_note_index`; +DROP TABLE IF EXISTS `memory_projection_state`; +DROP INDEX IF EXISTS `idx_memory_path_summary_prefix`; +DROP TABLE IF EXISTS `memory_path_summary`; diff --git a/sql/migrations/README.md b/sql/migrations/README.md index ccb80ba33..f4976c716 100644 --- a/sql/migrations/README.md +++ b/sql/migrations/README.md @@ -170,6 +170,7 @@ helpers in `db.rs`. Subsequent CEXes have populated this directory. | `2026080402` | `agent_usage_runtime_attribution` | `2026080402_agent_usage_runtime_attribution{,_down}.sql` (plan-20260715 W2-12: durable repository, turn, and replay-event attribution for runtime usage.) | | `2026080403` | `agent_usage_event_session_scope` | `2026080403_agent_usage_event_session_scope{,_down}.sql` (W2-12 follow-up: replay event IDs are unique within the durable session, permitting the same browser command ID in independent sessions.) | | `2026081301` | `approved_permission_provenance` | `2026081301_approved_permission_provenance{,_down}.sql` (plan-20260715 W4-07: Always-approval provenance columns; empty backfill; `project_id` not rewritten; down fail-closed with provenance or linked HEAD evidence.) | +| `2026082401` | `memory_core` | `2026082401_memory_core{,_down}.sql` (M2-02: rebuildable Memory projections plus bounded per-root compiler job and source-observer state; FTS5 and receipt storage land in later migrations.) | All registered migrations are loaded via `include_str!`. New migrations must follow the same pattern — inline SQL strings in `builtin_migrations()` are no diff --git a/src/command/maintenance.rs b/src/command/maintenance.rs index 3ccee549b..3fd6a18ac 100644 --- a/src/command/maintenance.rs +++ b/src/command/maintenance.rs @@ -2699,6 +2699,42 @@ pub struct GcObjectSource { pub note: &'static str, } +const fn memory_projection_oid( + location: &'static str, + column: &'static str, + note: &'static str, +) -> GcObjectSource { + GcObjectSource { + origin: GcSourceOrigin::Column, + location, + column, + status: GcSourceStatus::IndexOnly, + kind: GcStorageKind::SqliteColumn, + schema: "2026082401_memory_core rebuildable projection", + read_bound: "not read by GC; projection replay removes stale rows", + corruption: GcCorruptionPolicy::LenientSkip, + note, + } +} + +const fn memory_runtime_oid( + location: &'static str, + column: &'static str, + note: &'static str, +) -> GcObjectSource { + GcObjectSource { + origin: GcSourceOrigin::Column, + location, + column, + status: GcSourceStatus::NonRoot, + kind: GcStorageKind::SqliteColumn, + schema: "2026082401_memory_core bounded runtime state", + read_bound: "not read by GC; one bounded row per compiler root or source cursor", + corruption: GcCorruptionPolicy::NotApplicable, + note, + } +} + /// §C.4.3: the FILE-backed half of the inventory. Each entry names the /// collector or gate that implements its classification, so a reader can /// check the claim rather than trust it. @@ -3254,6 +3290,61 @@ pub const GC_OBJECT_SOURCE_INVENTORY: &[GcObjectSource] = &[ corruption: GcCorruptionPolicy::LenientSkip, note: "cache validity key (the tip is ref-anchored); rebuilt on demand", }, + memory_projection_oid( + "memory_revision_index", + "revision_oid", + "Memory revision lookup cache; the future authoritative Memory ref/object graph owns reachability, while replay can discard this row", + ), + memory_projection_oid( + "memory_projection_state", + "projected_ref_oid", + "Memory projection watermark; it records which authoritative ref tip was replayed and never keeps that tip alive", + ), + memory_projection_oid( + "memory_head", + "latest_revision_oid", + "Latest Memory revision cache; authoritative events and the Memory ref own reachability, and projection replay repairs this pointer", + ), + memory_projection_oid( + "memory_head", + "live_revision_oid", + "Effective Memory revision cache; authoritative events and the Memory ref own reachability, and projection replay repairs this pointer", + ), + memory_projection_oid( + "memory_head", + "effective_from_commit", + "Code-applicability cache derived from the authoritative Memory revision; the code ref/reflog and Memory authority, not SQLite, define retention", + ), + memory_projection_oid( + "memory_head", + "effective_until_commit", + "Code-applicability cache derived from the authoritative Memory revision; the code ref/reflog and Memory authority, not SQLite, define retention", + ), + memory_projection_oid( + "memory_link_index", + "source_revision_oid", + "Rebuildable Memory link index source; the authoritative Memory event graph owns the revision and replay can discard this row", + ), + memory_projection_oid( + "memory_link_index", + "target_revision_oid", + "Optional pinned Memory link target in a rebuildable index; the authoritative event graph owns retention and replay repairs the row", + ), + memory_projection_oid( + "memory_episode_path", + "revision_oid", + "Rebuildable Episode path lookup keyed by a Memory revision; it accelerates filtering and contributes no reachability authority", + ), + memory_runtime_oid( + "memory_compile_job", + "terminal_source_oid", + "Compiler input cursor into an authoritative Task or Intent source ref; bounded job state is recoverable by observation and is not an object root", + ), + memory_runtime_oid( + "memory_compile_observer_state", + "scanned_through_oid", + "First-parent observer watermark into an authoritative source ref; rescanning repairs it and the cursor itself keeps no object alive", + ), GcObjectSource { origin: GcSourceOrigin::Column, location: "object_obliteration", diff --git a/src/internal/ai/keyed_digest.rs b/src/internal/ai/keyed_digest.rs index 672cbaca2..f50765db9 100644 --- a/src/internal/ai/keyed_digest.rs +++ b/src/internal/ai/keyed_digest.rs @@ -111,6 +111,84 @@ impl KeyedDigestEnvelope { } } +/// Purpose-locked fingerprint for a compiler root's canonical source inputs. +/// +/// The wrapper deliberately has no `Debug` or serialization implementation: +/// job persistence writes its three validated parts explicitly, and routine +/// diagnostics must not print the digest. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct SourceInputFingerprint(KeyedDigestEnvelope); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum SourceInputFingerprintErrorKind { + UnsupportedVersion, + InvalidKeyId, + InvalidDigest, +} + +#[derive(Debug, Error)] +#[error("persisted source-input fingerprint is invalid ({kind:?})")] +pub(crate) struct SourceInputFingerprintError { + kind: SourceInputFingerprintErrorKind, +} + +impl SourceInputFingerprintError { + const fn new(kind: SourceInputFingerprintErrorKind) -> Self { + Self { kind } + } + + pub(crate) const fn kind(&self) -> SourceInputFingerprintErrorKind { + self.kind + } +} + +impl SourceInputFingerprint { + pub(crate) fn from_parts( + version: u8, + key_id: Uuid, + digest: String, + ) -> Result { + if version != DIGEST_VERSION { + return Err(SourceInputFingerprintError::new( + SourceInputFingerprintErrorKind::UnsupportedVersion, + )); + } + if key_id.get_version_num() != 4 { + return Err(SourceInputFingerprintError::new( + SourceInputFingerprintErrorKind::InvalidKeyId, + )); + } + if digest.len() != 64 + || !digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + return Err(SourceInputFingerprintError::new( + SourceInputFingerprintErrorKind::InvalidDigest, + )); + } + + Ok(Self(KeyedDigestEnvelope { + version, + key_id, + purpose: DigestPurpose::SourceInput, + digest, + })) + } + + pub(crate) const fn version(&self) -> u8 { + self.0.version() + } + + pub(crate) const fn key_id(&self) -> Uuid { + self.0.key_id() + } + + pub(crate) fn digest_hex(&self) -> &str { + self.0.digest_hex() + } +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum KeyedDigestErrorKind { RepositoryUnavailable, @@ -368,6 +446,14 @@ impl RepositoryKeyedDigest { self.ensure_valid()?; Ok(envelope) } + + pub(crate) fn source_input_fingerprint( + &self, + input: &[u8], + ) -> Result { + self.digest(DigestPurpose::SourceInput, input) + .map(SourceInputFingerprint) + } } fn config_fingerprint(ciphertext_hex: &str) -> [u8; 32] { @@ -586,7 +672,8 @@ mod tests { use super::{ DigestPurpose, KeyedDigestError, KeyedDigestErrorKind, PERSISTED_GENERATION, - PERSISTED_SCHEMA_VERSION, PersistedDigestKeyV1, RepositoryKeyedDigest, config_fingerprint, + PERSISTED_SCHEMA_VERSION, PersistedDigestKeyV1, RepositoryKeyedDigest, + SourceInputFingerprint, SourceInputFingerprintErrorKind, config_fingerprint, reset_digest_cache_for_tests, }; use crate::{ @@ -852,6 +939,52 @@ mod tests { assert!(json.get("seed").is_none()); } + #[test] + fn source_input_fingerprint_is_purpose_locked_and_round_trips_parts() { + let key_id = Uuid::parse_str("123e4567-e89b-42d3-a456-426614174000") + .expect("fixed UUIDv4 must parse"); + let provider = RepositoryKeyedDigest::from_seed( + key_id, + [0x24; 32], + config_fingerprint("source-input-fingerprint"), + ) + .expect("fixed seed must construct a provider"); + + let fingerprint = provider + .source_input_fingerprint(b"task-42 terminal inputs") + .expect("valid provider produces a source-input fingerprint"); + assert_eq!(fingerprint.version(), 1); + assert_eq!(fingerprint.key_id(), key_id); + assert_eq!(fingerprint.digest_hex().len(), 64); + + let restored = SourceInputFingerprint::from_parts( + fingerprint.version(), + fingerprint.key_id(), + fingerprint.digest_hex().to_owned(), + ) + .expect("persisted source-input parts must round-trip"); + assert!(restored == fingerprint); + + let Err(unsupported_version) = + SourceInputFingerprint::from_parts(2, key_id, "a".repeat(64)) + else { + panic!("unsupported versions must fail closed"); + }; + assert_eq!( + unsupported_version.kind(), + SourceInputFingerprintErrorKind::UnsupportedVersion + ); + + let Err(invalid_digest) = SourceInputFingerprint::from_parts(1, key_id, "A".repeat(64)) + else { + panic!("non-canonical digest text must fail closed"); + }; + assert_eq!( + invalid_digest.kind(), + SourceInputFingerprintErrorKind::InvalidDigest + ); + } + #[test] fn keyed_digest_error_contract_is_stable_and_actionable() { let error = KeyedDigestError::new(KeyedDigestErrorKind::UnsupportedGeneration); diff --git a/src/internal/ai/memory/job_sql.rs b/src/internal/ai/memory/job_sql.rs new file mode 100644 index 000000000..ad549b0b6 --- /dev/null +++ b/src/internal/ai/memory/job_sql.rs @@ -0,0 +1,964 @@ +use std::{ + collections::HashSet, + time::{SystemTime, UNIX_EPOCH}, +}; + +use git_internal::hash::ObjectHash; +use sea_orm::{ConnectionTrait, DatabaseConnection, DatabaseTransaction, Statement}; +use thiserror::Error; + +use super::domain::{EpisodeRoot, EpisodeRootKind}; +use crate::internal::{ai::keyed_digest::SourceInputFingerprint, db}; + +const INTENT_SOURCE_REF: &str = "libra/intent"; +const MEMORY_SOURCE_REF: &str = "libra/memory/repo"; + +pub(crate) struct ObservedRoot { + root: EpisodeRoot, + terminal_source_oid: ObjectHash, + input_fingerprint: SourceInputFingerprint, +} + +impl ObservedRoot { + pub(crate) const fn new( + root: EpisodeRoot, + terminal_source_oid: ObjectHash, + input_fingerprint: SourceInputFingerprint, + ) -> Self { + Self { + root, + terminal_source_oid, + input_fingerprint, + } + } + + fn kind_label(&self) -> &'static str { + match self.root.kind() { + EpisodeRootKind::Task => "task", + EpisodeRootKind::Intent => "intent", + } + } +} + +pub(crate) struct ObservationBatch { + scope_key: String, + source_ref_name: String, + expected_cursor: Option, + scanned_through_oid: ObjectHash, + roots: Vec, +} + +impl ObservationBatch { + pub(crate) fn new( + scope_key: impl Into, + source_ref_name: impl Into, + expected_cursor: Option, + scanned_through_oid: ObjectHash, + roots: Vec, + ) -> Result { + let scope_key = scope_key.into(); + if !valid_scope_key(&scope_key) { + return Err(RecordObservationError::new( + RecordObservationErrorKind::InvalidScope, + )); + } + + let source_ref_name = source_ref_name.into(); + if !matches!( + source_ref_name.as_str(), + INTENT_SOURCE_REF | MEMORY_SOURCE_REF + ) { + return Err(RecordObservationError::new( + RecordObservationErrorKind::InvalidSourceRef, + )); + } + + if expected_cursor == Some(scanned_through_oid) && !roots.is_empty() { + return Err(RecordObservationError::new( + RecordObservationErrorKind::InvalidCursorRange, + )); + } + + let mut root_keys = HashSet::with_capacity(roots.len()); + for root in &roots { + if !root_keys.insert((root.kind_label(), root.root.id())) { + return Err(RecordObservationError::new( + RecordObservationErrorKind::DuplicateRoot, + )); + } + } + + Ok(Self { + scope_key, + source_ref_name, + expected_cursor, + scanned_through_oid, + roots, + }) + } +} + +fn valid_scope_key(scope_key: &str) -> bool { + !scope_key.is_empty() + && scope_key.len() <= 512 + && scope_key.trim() == scope_key + && !scope_key.chars().any(char::is_control) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ObservationBatchOutcome { + Recorded { observed_roots: usize }, + AlreadyRecorded, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum RecordObservationErrorKind { + InvalidScope, + InvalidSourceRef, + InvalidCursorRange, + DuplicateRoot, + CursorConflict, + SourceMismatch, + Storage, +} + +#[derive(Debug, Error)] +#[error("Memory compiler observation failed ({kind:?})")] +pub(crate) struct RecordObservationError { + kind: RecordObservationErrorKind, +} + +impl RecordObservationError { + const fn new(kind: RecordObservationErrorKind) -> Self { + Self { kind } + } + + fn storage() -> Self { + Self::new(RecordObservationErrorKind::Storage) + } + + pub(crate) const fn kind(&self) -> RecordObservationErrorKind { + self.kind + } +} + +pub(crate) async fn record_observation_batch( + database: &DatabaseConnection, + batch: ObservationBatch, +) -> Result { + let transaction = db::begin_write_transaction(database) + .await + .map_err(|_| RecordObservationError::storage())?; + match record_observation_batch_in_transaction(&transaction, batch).await { + Ok(outcome) => { + transaction + .commit() + .await + .map_err(|_| RecordObservationError::storage())?; + Ok(outcome) + } + Err(error) => { + let _ = transaction.rollback().await; + Err(error) + } + } +} + +async fn record_observation_batch_in_transaction( + transaction: &DatabaseTransaction, + batch: ObservationBatch, +) -> Result { + let scanned_through_oid = batch.scanned_through_oid.to_string(); + let expected_cursor = batch.expected_cursor.map(|cursor| cursor.to_string()); + let current_cursor = + read_observer_cursor(transaction, &batch.scope_key, &batch.source_ref_name).await?; + + if current_cursor.as_deref() == Some(scanned_through_oid.as_str()) { + return Ok(ObservationBatchOutcome::AlreadyRecorded); + } + + let cursor_matches = match (current_cursor.as_deref(), expected_cursor.as_deref()) { + (None, None) => true, + (Some(current), Some(expected)) => current == expected, + _ => false, + }; + if !cursor_matches { + return Err(RecordObservationError::new( + RecordObservationErrorKind::CursorConflict, + )); + } + + let observed_at = epoch_millis()?; + let observed_roots = batch.roots.len(); + for root in batch.roots { + observe_root(transaction, &batch.scope_key, root, observed_at).await?; + } + + match current_cursor { + None => { + insert_observer_cursor( + transaction, + &batch.scope_key, + &batch.source_ref_name, + &scanned_through_oid, + observed_at, + ) + .await? + } + Some(current) => { + advance_observer_cursor( + transaction, + &batch.scope_key, + &batch.source_ref_name, + ¤t, + &scanned_through_oid, + observed_at, + ) + .await? + } + } + + Ok(ObservationBatchOutcome::Recorded { observed_roots }) +} + +fn epoch_millis() -> Result { + let elapsed = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| RecordObservationError::storage())?; + i64::try_from(elapsed.as_millis()).map_err(|_| RecordObservationError::storage()) +} + +async fn read_observer_cursor( + transaction: &DatabaseTransaction, + scope_key: &str, + source_ref_name: &str, +) -> Result, RecordObservationError> { + transaction + .query_one_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "SELECT scanned_through_oid FROM memory_compile_observer_state + WHERE scope_key = ? AND source_ref_name = ?", + [scope_key.into(), source_ref_name.into()], + )) + .await + .map_err(|_| RecordObservationError::storage())? + .map(|row| { + row.try_get("", "scanned_through_oid") + .map_err(|_| RecordObservationError::storage()) + }) + .transpose() +} + +async fn insert_observer_cursor( + transaction: &DatabaseTransaction, + scope_key: &str, + source_ref_name: &str, + scanned_through_oid: &str, + observed_at: i64, +) -> Result<(), RecordObservationError> { + transaction + .execute_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "INSERT INTO memory_compile_observer_state + (scope_key, source_ref_name, scanned_through_oid, updated_at) + VALUES (?, ?, ?, ?)", + [ + scope_key.into(), + source_ref_name.into(), + scanned_through_oid.into(), + observed_at.into(), + ], + )) + .await + .map_err(|_| RecordObservationError::storage())?; + Ok(()) +} + +async fn advance_observer_cursor( + transaction: &DatabaseTransaction, + scope_key: &str, + source_ref_name: &str, + expected_cursor: &str, + scanned_through_oid: &str, + observed_at: i64, +) -> Result<(), RecordObservationError> { + let result = transaction + .execute_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "UPDATE memory_compile_observer_state + SET scanned_through_oid = ?, updated_at = ? + WHERE scope_key = ? AND source_ref_name = ? AND scanned_through_oid = ?", + [ + scanned_through_oid.into(), + observed_at.into(), + scope_key.into(), + source_ref_name.into(), + expected_cursor.into(), + ], + )) + .await + .map_err(|_| RecordObservationError::storage())?; + if result.rows_affected() != 1 { + return Err(RecordObservationError::new( + RecordObservationErrorKind::CursorConflict, + )); + } + Ok(()) +} + +struct StoredJob { + terminal_source_oid: String, + input_fingerprint_version: i64, + input_fingerprint_key_id: String, + input_fingerprint_digest: String, + observed_generation: i64, + state: String, +} + +async fn observe_root( + transaction: &DatabaseTransaction, + scope_key: &str, + root: ObservedRoot, + observed_at: i64, +) -> Result<(), RecordObservationError> { + let root_kind = root.kind_label(); + let root_id = root.root.id(); + let existing = transaction + .query_one_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "SELECT terminal_source_oid, input_fingerprint_version, + input_fingerprint_key_id, input_fingerprint_digest, + observed_generation, state + FROM memory_compile_job + WHERE scope_key = ? AND root_kind = ? AND root_id = ?", + [scope_key.into(), root_kind.into(), root_id.into()], + )) + .await + .map_err(|_| RecordObservationError::storage())? + .map(|row| { + Ok::<_, RecordObservationError>(StoredJob { + terminal_source_oid: row + .try_get("", "terminal_source_oid") + .map_err(|_| RecordObservationError::storage())?, + input_fingerprint_version: row + .try_get("", "input_fingerprint_version") + .map_err(|_| RecordObservationError::storage())?, + input_fingerprint_key_id: row + .try_get("", "input_fingerprint_key_id") + .map_err(|_| RecordObservationError::storage())?, + input_fingerprint_digest: row + .try_get("", "input_fingerprint_digest") + .map_err(|_| RecordObservationError::storage())?, + observed_generation: row + .try_get("", "observed_generation") + .map_err(|_| RecordObservationError::storage())?, + state: row + .try_get("", "state") + .map_err(|_| RecordObservationError::storage())?, + }) + }) + .transpose()?; + + let terminal_source_oid = root.terminal_source_oid.to_string(); + let fingerprint_version = i64::from(root.input_fingerprint.version()); + let fingerprint_key_id = root.input_fingerprint.key_id().to_string(); + let fingerprint_digest = root.input_fingerprint.digest_hex(); + + let Some(existing) = existing else { + transaction + .execute_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "INSERT INTO memory_compile_job ( + scope_key, root_kind, root_id, terminal_source_oid, + input_fingerprint_version, input_fingerprint_key_id, + input_fingerprint_digest, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + [ + scope_key.into(), + root_kind.into(), + root_id.into(), + terminal_source_oid.into(), + fingerprint_version.into(), + fingerprint_key_id.into(), + fingerprint_digest.into(), + observed_at.into(), + observed_at.into(), + ], + )) + .await + .map_err(|_| RecordObservationError::storage())?; + return Ok(()); + }; + + let same_fingerprint = existing.input_fingerprint_version == fingerprint_version + && existing.input_fingerprint_key_id == fingerprint_key_id + && existing.input_fingerprint_digest == fingerprint_digest; + if same_fingerprint { + if existing.terminal_source_oid == terminal_source_oid { + return Ok(()); + } + return Err(RecordObservationError::new( + RecordObservationErrorKind::SourceMismatch, + )); + } + + let next_generation = existing + .observed_generation + .checked_add(1) + .ok_or_else(RecordObservationError::storage)?; + let (sql, values) = if existing.state == "inflight" { + ( + "UPDATE memory_compile_job + SET terminal_source_oid = ?, input_fingerprint_version = ?, + input_fingerprint_key_id = ?, input_fingerprint_digest = ?, + observed_generation = ?, updated_at = ? + WHERE scope_key = ? AND root_kind = ? AND root_id = ?", + vec![ + terminal_source_oid.into(), + fingerprint_version.into(), + fingerprint_key_id.into(), + fingerprint_digest.into(), + next_generation.into(), + observed_at.into(), + scope_key.into(), + root_kind.into(), + root_id.into(), + ], + ) + } else { + ( + "UPDATE memory_compile_job + SET terminal_source_oid = ?, input_fingerprint_version = ?, + input_fingerprint_key_id = ?, input_fingerprint_digest = ?, + observed_generation = ?, state = 'dirty', + lease_owner = NULL, lease_expires_at = NULL, + retry_count = 0, next_retry_at = NULL, + last_error_code = NULL, last_error_summary = NULL, + updated_at = ? + WHERE scope_key = ? AND root_kind = ? AND root_id = ?", + vec![ + terminal_source_oid.into(), + fingerprint_version.into(), + fingerprint_key_id.into(), + fingerprint_digest.into(), + next_generation.into(), + observed_at.into(), + scope_key.into(), + root_kind.into(), + root_id.into(), + ], + ) + }; + let result = transaction + .execute_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + sql, + values, + )) + .await + .map_err(|_| RecordObservationError::storage())?; + if result.rows_affected() != 1 { + return Err(RecordObservationError::storage()); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use git_internal::hash::ObjectHash; + use sea_orm::{ConnectionTrait, DatabaseConnection, Statement}; + use tempfile::TempDir; + use uuid::Uuid; + + use super::{ + super::domain::EpisodeRoot, ObservationBatch, ObservationBatchOutcome, ObservedRoot, + RecordObservationErrorKind, record_observation_batch, + }; + use crate::internal::{ai::keyed_digest::SourceInputFingerprint, db}; + + struct JobSnapshot { + terminal_source_oid: String, + fingerprint_digest: String, + observed_generation: i64, + processed_generation: i64, + state: String, + lease_owner: Option, + lease_fence: i64, + lease_expires_at: Option, + retry_count: i64, + next_retry_at: Option, + last_error_code: Option, + last_error_summary: Option, + updated_at: i64, + } + + async fn memory_database() -> (TempDir, DatabaseConnection) { + let directory = tempfile::tempdir().expect("temporary repository database directory"); + let path = directory.path().join("libra.db"); + let connection = db::create_database(&path.to_string_lossy()) + .await + .expect("current Libra schema must initialize"); + (directory, connection) + } + + fn oid(label: &[u8]) -> ObjectHash { + ObjectHash::new(label) + } + + fn fingerprint(fill: char) -> SourceInputFingerprint { + SourceInputFingerprint::from_parts( + 1, + Uuid::parse_str("123e4567-e89b-42d3-a456-426614174000") + .expect("fixed UUIDv4 must parse"), + fill.to_string().repeat(64), + ) + .expect("synthetic lowercase source-input fingerprint must validate") + } + + fn observed_task( + id: &str, + terminal_source_oid: ObjectHash, + input_fingerprint: SourceInputFingerprint, + ) -> ObservedRoot { + ObservedRoot::new( + EpisodeRoot::task(id).expect("synthetic task root must validate"), + terminal_source_oid, + input_fingerprint, + ) + } + + fn observed_intent( + id: &str, + terminal_source_oid: ObjectHash, + input_fingerprint: SourceInputFingerprint, + ) -> ObservedRoot { + ObservedRoot::new( + EpisodeRoot::intent(id).expect("synthetic intent root must validate"), + terminal_source_oid, + input_fingerprint, + ) + } + + async fn job_snapshot( + connection: &DatabaseConnection, + root_kind: &str, + root_id: &str, + ) -> Option { + connection + .query_one_raw(Statement::from_sql_and_values( + connection.get_database_backend(), + "SELECT terminal_source_oid, input_fingerprint_digest, + observed_generation, processed_generation, state, + lease_owner, lease_fence, lease_expires_at, retry_count, + next_retry_at, last_error_code, last_error_summary, updated_at + FROM memory_compile_job + WHERE scope_key = ? AND root_kind = ? AND root_id = ?", + ["repo".into(), root_kind.into(), root_id.into()], + )) + .await + .expect("query compiler job") + .map(|row| JobSnapshot { + terminal_source_oid: row.try_get("", "terminal_source_oid").unwrap(), + fingerprint_digest: row.try_get("", "input_fingerprint_digest").unwrap(), + observed_generation: row.try_get("", "observed_generation").unwrap(), + processed_generation: row.try_get("", "processed_generation").unwrap(), + state: row.try_get("", "state").unwrap(), + lease_owner: row.try_get("", "lease_owner").unwrap(), + lease_fence: row.try_get("", "lease_fence").unwrap(), + lease_expires_at: row.try_get("", "lease_expires_at").unwrap(), + retry_count: row.try_get("", "retry_count").unwrap(), + next_retry_at: row.try_get("", "next_retry_at").unwrap(), + last_error_code: row.try_get("", "last_error_code").unwrap(), + last_error_summary: row.try_get("", "last_error_summary").unwrap(), + updated_at: row.try_get("", "updated_at").unwrap(), + }) + } + + async fn observer_cursor(connection: &DatabaseConnection) -> Option { + connection + .query_one_raw(Statement::from_sql_and_values( + connection.get_database_backend(), + "SELECT scanned_through_oid FROM memory_compile_observer_state + WHERE scope_key = ? AND source_ref_name = ?", + ["repo".into(), "libra/intent".into()], + )) + .await + .expect("query observer cursor") + .map(|row| row.try_get("", "scanned_through_oid").unwrap()) + } + + #[tokio::test] + async fn observer_job_schema_transaction() { + let (_directory, connection) = memory_database().await; + let cursor_one = oid(b"observer-cursor-one"); + let cursor_two = oid(b"observer-cursor-two"); + let cursor_three = oid(b"observer-cursor-three"); + let task_source_one = oid(b"task-source-one"); + let task_source_two = oid(b"task-source-two"); + let intent_source = oid(b"intent-source"); + + let first = ObservationBatch::new( + "repo", + "libra/intent", + None, + cursor_one, + vec![ + observed_task("task-1", task_source_one, fingerprint('a')), + observed_intent("intent-1", intent_source, fingerprint('b')), + ], + ) + .expect("first observation batch must validate"); + assert_eq!( + record_observation_batch(&connection, first) + .await + .expect("first batch must commit"), + ObservationBatchOutcome::Recorded { observed_roots: 2 } + ); + assert_eq!( + observer_cursor(&connection).await, + Some(cursor_one.to_string()) + ); + + let initial_task = job_snapshot(&connection, "task", "task-1") + .await + .expect("task job must exist"); + assert_eq!( + initial_task.terminal_source_oid, + task_source_one.to_string() + ); + assert_eq!(initial_task.fingerprint_digest, "a".repeat(64)); + assert_eq!(initial_task.observed_generation, 1); + assert_eq!(initial_task.processed_generation, 0); + assert_eq!(initial_task.state, "dirty"); + assert_eq!(initial_task.lease_fence, 0); + + let retry = ObservationBatch::new( + "repo", + "libra/intent", + None, + cursor_one, + vec![observed_task("task-1", task_source_one, fingerprint('a'))], + ) + .expect("commit-outcome retry batch must validate"); + assert_eq!( + record_observation_batch(&connection, retry) + .await + .expect("commit-outcome retry must succeed"), + ObservationBatchOutcome::AlreadyRecorded + ); + assert_eq!( + job_snapshot(&connection, "task", "task-1") + .await + .expect("task job remains") + .updated_at, + initial_task.updated_at, + "AlreadyRecorded must not touch jobs" + ); + + connection + .execute_unprepared( + "UPDATE memory_compile_job + SET retry_count = 2, next_retry_at = 5, + last_error_code = 'LBR-MEMORY-101', + last_error_summary = 'old generation' + WHERE scope_key = 'repo' AND root_kind = 'task' AND root_id = 'task-1'", + ) + .await + .expect("seed retry diagnostics"); + + let changed = ObservationBatch::new( + "repo", + "libra/intent", + Some(cursor_one), + cursor_two, + vec![observed_task("task-1", task_source_two, fingerprint('c'))], + ) + .expect("changed observation batch must validate"); + assert_eq!( + record_observation_batch(&connection, changed) + .await + .expect("changed batch must commit"), + ObservationBatchOutcome::Recorded { observed_roots: 1 } + ); + let changed_task = job_snapshot(&connection, "task", "task-1") + .await + .expect("changed task job must exist"); + assert_eq!( + changed_task.terminal_source_oid, + task_source_two.to_string() + ); + assert_eq!(changed_task.fingerprint_digest, "c".repeat(64)); + assert_eq!(changed_task.observed_generation, 2); + assert_eq!(changed_task.processed_generation, 0); + assert_eq!(changed_task.state, "dirty"); + assert_eq!(changed_task.retry_count, 0); + assert_eq!(changed_task.next_retry_at, None); + assert_eq!(changed_task.last_error_code, None); + assert_eq!(changed_task.last_error_summary, None); + + let empty = ObservationBatch::new( + "repo", + "libra/intent", + Some(cursor_two), + cursor_three, + vec![], + ) + .expect("empty scan batch must validate"); + assert_eq!( + record_observation_batch(&connection, empty) + .await + .expect("empty scan must advance"), + ObservationBatchOutcome::Recorded { observed_roots: 0 } + ); + assert_eq!( + observer_cursor(&connection).await, + Some(cursor_three.to_string()) + ); + } + + #[tokio::test] + async fn observer_job_same_input_across_cursor_preserves_completed_progress() { + let (_directory, connection) = memory_database().await; + let cursor_one = oid(b"same-input-cursor-one"); + let cursor_two = oid(b"same-input-cursor-two"); + let cursor_three = oid(b"same-input-cursor-three"); + let source_one = oid(b"same-input-source-one"); + let source_two = oid(b"same-input-source-two"); + + let initial = ObservationBatch::new( + "repo", + "libra/intent", + None, + cursor_one, + vec![observed_task("task-progress", source_one, fingerprint('a'))], + ) + .expect("initial progress batch validates"); + record_observation_batch(&connection, initial) + .await + .expect("initial progress batch commits"); + connection + .execute_unprepared( + "UPDATE memory_compile_job + SET processed_generation = observed_generation, state = 'idle' + WHERE scope_key = 'repo' AND root_kind = 'task' + AND root_id = 'task-progress'", + ) + .await + .expect("mark the first generation processed"); + let completed = job_snapshot(&connection, "task", "task-progress") + .await + .expect("completed job exists"); + assert_eq!(completed.observed_generation, 1); + assert_eq!(completed.processed_generation, 1); + assert_eq!(completed.state, "idle"); + + let same_input = ObservationBatch::new( + "repo", + "libra/intent", + Some(cursor_one), + cursor_two, + vec![observed_task("task-progress", source_one, fingerprint('a'))], + ) + .expect("same-input next interval validates"); + assert_eq!( + record_observation_batch(&connection, same_input) + .await + .expect("same-input next interval advances only the cursor"), + ObservationBatchOutcome::Recorded { observed_roots: 1 } + ); + let unchanged = job_snapshot(&connection, "task", "task-progress") + .await + .expect("same-input job remains"); + assert_eq!(unchanged.observed_generation, 1); + assert_eq!(unchanged.processed_generation, 1); + assert_eq!(unchanged.state, "idle"); + assert_eq!(unchanged.terminal_source_oid, source_one.to_string()); + assert_eq!(unchanged.fingerprint_digest, "a".repeat(64)); + assert_eq!(unchanged.updated_at, completed.updated_at); + assert_eq!( + observer_cursor(&connection).await, + Some(cursor_two.to_string()) + ); + + let changed_input = ObservationBatch::new( + "repo", + "libra/intent", + Some(cursor_two), + cursor_three, + vec![observed_task("task-progress", source_two, fingerprint('b'))], + ) + .expect("changed-input next interval validates"); + record_observation_batch(&connection, changed_input) + .await + .expect("changed input schedules the next generation"); + let changed = job_snapshot(&connection, "task", "task-progress") + .await + .expect("changed-input job remains"); + assert_eq!(changed.observed_generation, 2); + assert_eq!( + changed.processed_generation, 1, + "observing generation two must retain completed generation one" + ); + assert_eq!(changed.state, "dirty"); + assert_eq!(changed.terminal_source_oid, source_two.to_string()); + assert_eq!(changed.fingerprint_digest, "b".repeat(64)); + } + + #[tokio::test] + async fn observer_job_rejects_conflicts_atomically() { + let (_directory, connection) = memory_database().await; + let cursor_one = oid(b"atomic-cursor-one"); + let cursor_two = oid(b"atomic-cursor-two"); + let other_cursor = oid(b"atomic-other-cursor"); + let source_one = oid(b"atomic-source-one"); + let source_two = oid(b"atomic-source-two"); + + let initial = ObservationBatch::new( + "repo", + "libra/intent", + None, + cursor_one, + vec![observed_task("task-1", source_one, fingerprint('a'))], + ) + .expect("initial batch validates"); + record_observation_batch(&connection, initial) + .await + .expect("initial batch commits"); + + let mismatch = ObservationBatch::new( + "repo", + "libra/intent", + Some(cursor_one), + cursor_two, + vec![ + observed_task("task-2", source_two, fingerprint('b')), + observed_task("task-1", source_two, fingerprint('a')), + ], + ) + .expect("mismatch batch validates structurally"); + let Err(error) = record_observation_batch(&connection, mismatch).await else { + panic!("same fingerprint with another source OID must fail"); + }; + assert_eq!(error.kind(), RecordObservationErrorKind::SourceMismatch); + assert!( + job_snapshot(&connection, "task", "task-2").await.is_none(), + "earlier writes in the failed batch must roll back" + ); + assert_eq!( + observer_cursor(&connection).await, + Some(cursor_one.to_string()) + ); + + let conflict = ObservationBatch::new( + "repo", + "libra/intent", + Some(other_cursor), + cursor_two, + vec![observed_task("task-3", source_two, fingerprint('c'))], + ) + .expect("cursor-conflict batch validates structurally"); + let Err(error) = record_observation_batch(&connection, conflict).await else { + panic!("stale expected cursor must fail"); + }; + assert_eq!(error.kind(), RecordObservationErrorKind::CursorConflict); + assert!(job_snapshot(&connection, "task", "task-3").await.is_none()); + + let Err(error) = ObservationBatch::new( + "repo", + "libra/intent", + Some(cursor_two), + cursor_two, + vec![observed_task("task-4", source_two, fingerprint('d'))], + ) else { + panic!("non-empty zero-width scan must be rejected before SQL"); + }; + assert_eq!(error.kind(), RecordObservationErrorKind::InvalidCursorRange); + + let Err(error) = ObservationBatch::new( + "repo", + "libra/intent", + Some(cursor_one), + cursor_two, + vec![ + observed_task("task-5", source_one, fingerprint('e')), + observed_task("task-5", source_one, fingerprint('e')), + ], + ) else { + panic!("duplicate roots must be rejected before SQL"); + }; + assert_eq!(error.kind(), RecordObservationErrorKind::DuplicateRoot); + + let Err(error) = ObservationBatch::new( + "repo", + "refs/heads/main", + Some(cursor_one), + cursor_two, + vec![], + ) else { + panic!("unsupported source refs must be rejected"); + }; + assert_eq!(error.kind(), RecordObservationErrorKind::InvalidSourceRef); + } + + #[tokio::test] + async fn observer_job_preserves_inflight_lease_on_new_generation() { + let (_directory, connection) = memory_database().await; + let cursor_one = oid(b"lease-cursor-one"); + let cursor_two = oid(b"lease-cursor-two"); + let source_one = oid(b"lease-source-one"); + let source_two = oid(b"lease-source-two"); + + let initial = ObservationBatch::new( + "repo", + "libra/intent", + None, + cursor_one, + vec![observed_task("task-lease", source_one, fingerprint('a'))], + ) + .expect("initial lease fixture validates"); + record_observation_batch(&connection, initial) + .await + .expect("initial lease fixture commits"); + connection + .execute_unprepared( + "UPDATE memory_compile_job + SET state = 'inflight', lease_owner = 'runner-a', lease_fence = 7, + lease_expires_at = 0, retry_count = 1, + last_error_code = 'LBR-MEMORY-102', + last_error_summary = 'pinned generation diagnostic' + WHERE scope_key = 'repo' AND root_kind = 'task' AND root_id = 'task-lease'", + ) + .await + .expect("seed an inflight lease whose deadline is already past"); + + let changed = ObservationBatch::new( + "repo", + "libra/intent", + Some(cursor_one), + cursor_two, + vec![observed_task("task-lease", source_two, fingerprint('b'))], + ) + .expect("new generation validates"); + record_observation_batch(&connection, changed) + .await + .expect("new generation commits without judging lease time"); + + let job = job_snapshot(&connection, "task", "task-lease") + .await + .expect("lease job remains"); + assert_eq!(job.observed_generation, 2); + assert_eq!(job.processed_generation, 0); + assert_eq!(job.terminal_source_oid, source_two.to_string()); + assert_eq!(job.fingerprint_digest, "b".repeat(64)); + assert_eq!(job.state, "inflight"); + assert_eq!(job.lease_owner.as_deref(), Some("runner-a")); + assert_eq!(job.lease_fence, 7); + assert_eq!(job.lease_expires_at, Some(0)); + assert_eq!(job.retry_count, 1); + assert_eq!(job.last_error_code.as_deref(), Some("LBR-MEMORY-102")); + assert_eq!( + job.last_error_summary.as_deref(), + Some("pinned generation diagnostic") + ); + } +} diff --git a/src/internal/ai/memory/mod.rs b/src/internal/ai/memory/mod.rs index 68437d16a..a051012cc 100644 --- a/src/internal/ai/memory/mod.rs +++ b/src/internal/ai/memory/mod.rs @@ -11,4 +11,5 @@ mod canonical; mod domain; +mod job_sql; mod validation; diff --git a/src/internal/db.rs b/src/internal/db.rs index 1a1be4ccb..57fe3c887 100644 --- a/src/internal/db.rs +++ b/src/internal/db.rs @@ -487,25 +487,29 @@ async fn inspect_database_schema_for_connection( let latest = migration::latest_builtin_schema_version() .map_err(|err| IOError::other(format!("Failed to inspect built-in migrations: {err}")))?; + Ok(classify_schema_compatibility(current, latest)) +} + +fn classify_schema_compatibility(current: Option, latest: Option) -> SchemaCompatibility { match (current, latest) { - (_, None) => Ok(SchemaCompatibility::Compatible { + (_, None) => SchemaCompatibility::Compatible { current_version: current, latest_version: latest, - }), - (Some(current), Some(latest)) if current == latest => Ok(SchemaCompatibility::Compatible { + }, + (Some(current), Some(latest)) if current == latest => SchemaCompatibility::Compatible { current_version: Some(current), latest_version: Some(latest), - }), + }, (Some(current), Some(latest)) if current > latest => { - Ok(SchemaCompatibility::UnsupportedFuture { + SchemaCompatibility::UnsupportedFuture { current_version: current, latest_version: Some(latest), - }) + } } - (current, Some(latest)) => Ok(SchemaCompatibility::UpgradeRequired { + (current, Some(latest)) => SchemaCompatibility::UpgradeRequired { current_version: current, latest_version: latest, - }), + }, } } @@ -873,6 +877,17 @@ mod tests { reference::{self, ConfigKind}, }; + #[test] + fn memory_core_old_reader_rejects_migrated_schema() { + assert_eq!( + classify_schema_compatibility(Some(2026082401), Some(2026081301)), + SchemaCompatibility::UnsupportedFuture { + current_version: 2026082401, + latest_version: Some(2026081301), + } + ); + } + /// TestDbPath is a helper struct create and delete test database file struct TestDbPath(String); impl Drop for TestDbPath { diff --git a/src/internal/db/migration.rs b/src/internal/db/migration.rs index 6b72dce79..3d7490ce7 100644 --- a/src/internal/db/migration.rs +++ b/src/internal/db/migration.rs @@ -1381,6 +1381,14 @@ pub fn builtin_migrations() -> Vec { "../../../sql/migrations/2026081301_approved_permission_provenance_down.sql" ), ), + // M2-02: rebuildable Agent Memory projections plus bounded compiler + // job/observer state. FTS and receipts intentionally land separately. + sql_migration( + 2026082401, + "memory_core", + include_str!("../../../sql/migrations/2026082401_memory_core.sql"), + include_str!("../../../sql/migrations/2026082401_memory_core_down.sql"), + ), ] } @@ -1828,9 +1836,9 @@ mod tests { // `builtin_migrations()` so silent registry regressions surface // here in addition to `tests/db_migration_test.rs`. let runner = builtin_runner().expect("CEX-12.5 builtin registry must build clean"); - assert_eq!(runner.len(), 55); + assert_eq!(runner.len(), 56); assert!(!runner.is_empty()); - assert_eq!(runner.max_registered_version(), Some(2026081301)); + assert_eq!(runner.max_registered_version(), Some(2026082401)); } #[test] diff --git a/src/internal/model/memory_episode_path.rs b/src/internal/model/memory_episode_path.rs new file mode 100644 index 000000000..36a6b4620 --- /dev/null +++ b/src/internal/model/memory_episode_path.rs @@ -0,0 +1,19 @@ +//! SeaORM entity for rebuildable Episode revision-to-code-path links. + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, Eq, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "memory_episode_path")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub note_id: String, + #[sea_orm(primary_key, auto_increment = false)] + pub revision_oid: String, + #[sea_orm(primary_key, auto_increment = false)] + pub code_path: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/internal/model/memory_head.rs b/src/internal/model/memory_head.rs new file mode 100644 index 000000000..e49d40bf8 --- /dev/null +++ b/src/internal/model/memory_head.rs @@ -0,0 +1,40 @@ +//! SeaORM entity for the rebuildable current Memory note heads. + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, Eq, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "memory_head")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub scope_key: String, + #[sea_orm(primary_key, auto_increment = false)] + pub namespace: String, + #[sea_orm(primary_key, auto_increment = false)] + pub path: String, + #[sea_orm(primary_key, auto_increment = false)] + pub note_id: String, + pub latest_revision_oid: String, + pub live_revision_oid: Option, + pub latest_action: String, + pub latest_review_state: String, + pub kind: String, + pub lifecycle: String, + pub confidence: String, + pub trust: String, + pub sensitivity: String, + pub visibility: String, + pub acl_policy_id: String, + pub valid_from: Option, + pub valid_until: Option, + pub effective_from_commit: Option, + pub effective_until_commit: Option, + pub expires_at: Option, + pub rank_hint: i64, + pub last_event_seq: i64, + pub updated_at: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/internal/model/memory_link_index.rs b/src/internal/model/memory_link_index.rs new file mode 100644 index 000000000..76103e774 --- /dev/null +++ b/src/internal/model/memory_link_index.rs @@ -0,0 +1,28 @@ +//! SeaORM entity for rebuildable fixed-revision Memory links. + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, Eq, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "memory_link_index")] +pub struct Model { + pub source_scope_key: String, + pub source_namespace: String, + pub source_note_id: String, + #[sea_orm(primary_key, auto_increment = false)] + pub source_revision_oid: String, + #[sea_orm(primary_key, auto_increment = false)] + pub target_note_id: String, + pub target_revision_oid: Option, + #[sea_orm(primary_key, auto_increment = false)] + pub link_kind: String, + pub source_path: String, + pub target_path: String, + pub evidence_refs_json: String, + pub valid_from: Option, + pub valid_until: Option, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/internal/model/memory_model_tests.rs b/src/internal/model/memory_model_tests.rs new file mode 100644 index 000000000..05e2b5120 --- /dev/null +++ b/src/internal/model/memory_model_tests.rs @@ -0,0 +1,50 @@ +use sea_orm::{Iterable, PrimaryKeyTrait}; + +use super::{ + memory_episode_path, memory_head, memory_link_index, memory_note_index, memory_path_summary, + memory_projection_state, memory_revision_index, +}; + +fn key_names

() -> Vec +where + P: Iterable + std::fmt::Debug, +{ + P::iter().map(|key| format!("{key:?}")).collect() +} + +#[test] +fn memory_entities_expose_expected_primary_keys() { + assert_eq!( + key_names::(), + ["ScopeKey", "Namespace", "Path", "NoteId"] + ); + assert_eq!( + key_names::(), + ["ScopeKey", "Namespace", "Path"] + ); + assert_eq!(key_names::(), ["NoteId"]); + assert_eq!( + key_names::(), + ["RevisionOid"] + ); + assert_eq!( + key_names::(), + ["SourceRevisionOid", "TargetNoteId", "LinkKind"] + ); + assert_eq!( + key_names::(), + ["ScopeKey"] + ); + assert_eq!( + key_names::(), + ["NoteId", "RevisionOid", "CodePath"] + ); + + assert!(!memory_head::PrimaryKey::auto_increment()); + assert!(!memory_path_summary::PrimaryKey::auto_increment()); + assert!(!memory_note_index::PrimaryKey::auto_increment()); + assert!(!memory_revision_index::PrimaryKey::auto_increment()); + assert!(!memory_link_index::PrimaryKey::auto_increment()); + assert!(!memory_projection_state::PrimaryKey::auto_increment()); + assert!(!memory_episode_path::PrimaryKey::auto_increment()); +} diff --git a/src/internal/model/memory_note_index.rs b/src/internal/model/memory_note_index.rs new file mode 100644 index 000000000..6e75b88ab --- /dev/null +++ b/src/internal/model/memory_note_index.rs @@ -0,0 +1,30 @@ +//! SeaORM entity for the rebuildable Memory note reverse index. + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, Eq, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "memory_note_index")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub note_id: String, + pub scope_key: String, + pub namespace: String, + pub path: String, + pub kind: String, + pub lifecycle: String, + pub review_state: String, + pub confidence: String, + pub trust: String, + pub sensitivity: String, + pub visibility: String, + pub acl_policy_id: String, + pub origin: String, + pub idempotency_key: String, + pub idempotency_scope: String, + pub created_at: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/internal/model/memory_path_summary.rs b/src/internal/model/memory_path_summary.rs new file mode 100644 index 000000000..696d76662 --- /dev/null +++ b/src/internal/model/memory_path_summary.rs @@ -0,0 +1,25 @@ +//! SeaORM entity for rebuildable Memory path summaries. + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, Eq, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "memory_path_summary")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub scope_key: String, + #[sea_orm(primary_key, auto_increment = false)] + pub namespace: String, + #[sea_orm(primary_key, auto_increment = false)] + pub path: String, + pub confirmed_count: i64, + pub quarantined_count: i64, + pub child_count: i64, + pub prefix_count: i64, + pub preview: String, + pub last_changed_at: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/internal/model/memory_projection_state.rs b/src/internal/model/memory_projection_state.rs new file mode 100644 index 000000000..7a6816a10 --- /dev/null +++ b/src/internal/model/memory_projection_state.rs @@ -0,0 +1,20 @@ +//! SeaORM entity for per-scope Memory projection watermarks. + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, Eq, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "memory_projection_state")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub scope_key: String, + pub projected_ref_oid: String, + pub last_event_seq: i64, + pub schema_version: i64, + pub policy_version: String, + pub rebuilt_at: i64, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/internal/model/memory_revision_index.rs b/src/internal/model/memory_revision_index.rs new file mode 100644 index 000000000..8442e5f00 --- /dev/null +++ b/src/internal/model/memory_revision_index.rs @@ -0,0 +1,26 @@ +//! SeaORM entity for rebuildable Memory revision provenance. + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, Eq, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "memory_revision_index")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = false)] + pub revision_oid: String, + pub note_id: String, + pub scope_key: String, + pub namespace: String, + pub origin: String, + pub producer: String, + pub rules_version: i64, + pub prompt_version: Option, + pub model_id: Option, + pub policy_version: String, + pub input_fingerprints_json: String, + pub created_at: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/internal/model/mod.rs b/src/internal/model/mod.rs index 142efec36..022df80d6 100644 --- a/src/internal/model/mod.rs +++ b/src/internal/model/mod.rs @@ -23,6 +23,13 @@ pub mod config; pub mod config_kv; pub mod layer; pub mod layer_path; +pub mod memory_episode_path; +pub mod memory_head; +pub mod memory_link_index; +pub mod memory_note_index; +pub mod memory_path_summary; +pub mod memory_projection_state; +pub mod memory_revision_index; pub mod metadata_kv; pub mod object_index; pub mod object_obliteration; @@ -41,5 +48,7 @@ pub mod sparse_view; pub mod working_dirty; pub mod working_dirty_meta; +#[cfg(test)] +mod memory_model_tests; #[cfg(test)] mod reference_test; diff --git a/tests/agent_capture_migration_test.rs b/tests/agent_capture_migration_test.rs index ecd586f2e..44dc5842c 100644 --- a/tests/agent_capture_migration_test.rs +++ b/tests/agent_capture_migration_test.rs @@ -205,7 +205,7 @@ async fn capture_workspace_scope_migration_preserves_legacy_unknown_and_fences_d // Rollback returns every rolled-back version, newest first; the two // agent-usage migrations (2026080402/2026080403) sit on top of the W4 // scope migration and ride along without touching agent_session. - vec![2026081301, 2026080403, 2026080402, 2026080401] + vec![2026082401, 2026081301, 2026080403, 2026080402, 2026080401] ); // This focused migration fixture intentionally does not install the @@ -237,7 +237,7 @@ async fn capture_workspace_scope_migration_preserves_legacy_unknown_and_fences_d .expect("upgrade legacy capture row to W4 scope schema"), // Ascending application order: W4 scope first, then the two // agent-usage migrations re-applied on top of it. - vec![2026080401, 2026080402, 2026080403, 2026081301] + vec![2026080401, 2026080402, 2026080403, 2026081301, 2026082401] ); let row = conn .query_one_raw(Statement::from_string( diff --git a/tests/command/worktree_isolation_test.rs b/tests/command/worktree_isolation_test.rs index a6ed00b13..19940d950 100644 --- a/tests/command/worktree_isolation_test.rs +++ b/tests/command/worktree_isolation_test.rs @@ -5896,9 +5896,9 @@ async fn worktree_commands_apply_capability_marker_before_registry_io() { assert_eq!( rolled, vec![ - 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, 2026073004, - 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, 2026072501, - 2026072403, 2026072402, 2026072401 + 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, + 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, + 2026072501, 2026072403, 2026072402, 2026072401 ] ); conn.close().await.expect("close"); @@ -8555,25 +8555,24 @@ async fn worktree_doctor_does_not_upgrade_a_behind_schema_repository() { let db = main.join(".libra").join("libra.db"); let db_url = format!("sqlite://{}?mode=rwc", db.display()); - // Use the real down migration rather than deleting its ledger row. W4's - // schema adds physical columns/triggers, so merely removing the version - // would turn the next ordinary migration run into a duplicate-column - // failure instead of representing a repository that is genuinely behind. + // Use the real down migration rather than deleting its ledger row. The + // Memory core migration creates physical tables, so undoing it represents + // a repository that is genuinely one migration behind. let conn = Database::connect(&db_url) .await .expect("open repository db"); assert_eq!( builtin_runner() .expect("builtin runner") - .rollback_to(&conn, 2026073101) + .rollback_to(&conn, 2026081301) .await .expect("roll back newest migration"), - vec![2026081301, 2026080403, 2026080402, 2026080401] + vec![2026082401] ); conn.close().await.expect("close repository db"); assert!( - sqlite_max_schema_version(&db) < 2026080401, - "2026080401 must be the NEWEST migration for this test to leave one \ + sqlite_max_schema_version(&db) < 2026082401, + "2026082401 must be the NEWEST migration for this test to leave one \ pending — retarget it at the new newest migration" ); let before = std::fs::read(&db).expect("db before"); diff --git a/tests/db_migration_test.rs b/tests/db_migration_test.rs index 1ce034386..bb0655321 100644 --- a/tests/db_migration_test.rs +++ b/tests/db_migration_test.rs @@ -54,7 +54,7 @@ fn builtin_migrations_register_current_schema_migrations() { 2026071405, 2026071406, 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, - 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301 + 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 ] ); assert_eq!( @@ -115,13 +115,460 @@ fn builtin_migrations_register_current_schema_migrations() { "agent_usage_runtime_attribution", "agent_usage_event_session_scope", "approved_permission_provenance", + "memory_core", ] ); let runner = builtin_runner().expect("builtin registry must build clean"); assert!(!runner.is_empty()); - assert_eq!(runner.len(), 55); - assert_eq!(runner.max_registered_version(), Some(2026081301)); + assert_eq!(runner.len(), 56); + assert_eq!(runner.max_registered_version(), Some(2026082401)); +} + +const MEMORY_CORE_TABLES: [&str; 9] = [ + "memory_compile_job", + "memory_compile_observer_state", + "memory_episode_path", + "memory_head", + "memory_link_index", + "memory_note_index", + "memory_path_summary", + "memory_projection_state", + "memory_revision_index", +]; + +async fn memory_core_schema_snapshot(conn: &DatabaseConnection) -> Vec<(String, String, String)> { + conn.query_all_raw(Statement::from_string( + conn.get_database_backend(), + "SELECT type, name, sql FROM sqlite_master + WHERE (type = 'table' AND name LIKE 'memory_%') + OR (type = 'index' AND name LIKE 'idx_memory_%') + ORDER BY type, name" + .to_string(), + )) + .await + .expect("read Memory schema") + .into_iter() + .map(|row| { + ( + row.try_get("", "type").expect("schema object type"), + row.try_get("", "name").expect("schema object name"), + row.try_get("", "sql").expect("schema object DDL"), + ) + }) + .collect() +} + +#[tokio::test] +async fn memory_episode_schema() { + let (_dir, url, _path) = fresh_db_url(); + let conn = connect(&url).await; + run_builtin_migrations(&conn) + .await + .expect("apply Memory core migration"); + + let snapshot = memory_core_schema_snapshot(&conn).await; + let tables: Vec<&str> = snapshot + .iter() + .filter(|(kind, _, _)| kind == "table") + .map(|(_, name, _)| name.as_str()) + .collect(); + assert_eq!(tables, MEMORY_CORE_TABLES); + + let indexes: Vec<&str> = snapshot + .iter() + .filter(|(kind, _, _)| kind == "index") + .map(|(_, name, _)| name.as_str()) + .collect(); + assert_eq!( + indexes, + vec![ + "idx_memory_compile_job_runnable", + "idx_memory_compile_job_scope_generation", + "idx_memory_episode_path_code", + "idx_memory_head_lookup", + "idx_memory_head_path_prefix", + "idx_memory_link_source", + "idx_memory_link_target", + "idx_memory_note_idempotency_cell", + "idx_memory_note_idempotency_ns", + "idx_memory_path_summary_prefix", + "idx_memory_revision_note", + "idx_memory_revision_producer", + ], + "the Memory migration must expose the complete reviewed hot-query index set" + ); + + let foreign_keys_enabled: i64 = conn + .query_one_raw(Statement::from_string( + conn.get_database_backend(), + "PRAGMA foreign_keys".to_string(), + )) + .await + .expect("query foreign key mode") + .expect("foreign key pragma row") + .try_get("", "foreign_keys") + .expect("foreign key pragma value"); + assert_eq!(foreign_keys_enabled, 1); + + let invalid_uuid = conn + .execute_unprepared( + "INSERT INTO memory_note_index ( + note_id, scope_key, namespace, path, kind, lifecycle, review_state, + confidence, trust, sensitivity, visibility, acl_policy_id, origin, + idempotency_key, created_at + ) VALUES ( + 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx', 'repo', 'default', + 'episodic.tasks.invalid', 'episodic', 'replacement', 'confirmed', + 'high', 'repo_evidence', 'internal', 'repo_local', 'default', + 'episode_compiler', 'invalid-uuid', '2026-08-24T00:00:00Z' + )", + ) + .await; + assert!( + invalid_uuid.is_err(), + "UUID-shaped non-hex text must be rejected" + ); + + let invalid_source_ref = conn + .execute_unprepared( + "INSERT INTO memory_compile_observer_state ( + scope_key, source_ref_name, scanned_through_oid, updated_at + ) VALUES ( + 'repo', 'refs/heads/main', + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1 + )", + ) + .await; + assert!( + invalid_source_ref.is_err(), + "observer source ref names must remain a closed set" + ); + + conn.execute_unprepared( + "INSERT INTO memory_note_index ( + note_id, scope_key, namespace, path, kind, lifecycle, review_state, + confidence, trust, sensitivity, visibility, acl_policy_id, origin, + idempotency_key, idempotency_scope, created_at + ) VALUES ( + '123e4567-e89b-42d3-a456-426614174000', 'repo', 'default', + 'episodic.tasks.one', 'episodic', 'replacement', 'confirmed', + 'high', 'repo_evidence', 'internal', 'repo_local', 'default', + 'episode_compiler', 'same-cell', 'cell', '2026-08-24T00:00:00Z' + )", + ) + .await + .expect("seed a valid Memory note"); + conn.execute_unprepared( + "INSERT INTO memory_revision_index ( + revision_oid, note_id, scope_key, namespace, origin, producer, + rules_version, policy_version, input_fingerprints_json, created_at + ) VALUES ( + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + '123e4567-e89b-42d3-a456-426614174000', 'repo', 'default', + 'episode_compiler', 'test', 1, 'v1', '[]', '2026-08-24T00:00:00Z' + )", + ) + .await + .expect("seed a valid Memory revision"); + + let invalid_composite_revision = conn + .execute_unprepared( + "INSERT INTO memory_head ( + scope_key, namespace, path, note_id, latest_revision_oid, + latest_action, latest_review_state, kind, lifecycle, confidence, + trust, sensitivity, visibility, acl_policy_id, last_event_seq, updated_at + ) VALUES ( + 'repo', 'default', 'episodic.tasks.one', + '123e4567-e89b-42d3-a456-426614174000', + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 'created', 'confirmed', 'episodic', 'replacement', 'high', + 'repo_evidence', 'internal', 'repo_local', 'default', 1, + '2026-08-24T00:00:00Z' + )", + ) + .await; + assert!( + invalid_composite_revision.is_err(), + "a Memory head cannot pair a valid note with another revision OID" + ); + + let duplicate_cell_key = conn + .execute_unprepared( + "INSERT INTO memory_note_index ( + note_id, scope_key, namespace, path, kind, lifecycle, review_state, + confidence, trust, sensitivity, visibility, acl_policy_id, origin, + idempotency_key, idempotency_scope, created_at + ) VALUES ( + '223e4567-e89b-42d3-a456-426614174000', 'repo', 'default', + 'episodic.tasks.one', 'episodic', 'replacement', 'confirmed', + 'high', 'repo_evidence', 'internal', 'repo_local', 'default', + 'episode_compiler', 'same-cell', 'cell', '2026-08-24T00:00:00Z' + )", + ) + .await; + assert!( + duplicate_cell_key.is_err(), + "cell idempotency must reject the same key in the same path" + ); + + conn.execute_unprepared( + "INSERT INTO memory_note_index ( + note_id, scope_key, namespace, path, kind, lifecycle, review_state, + confidence, trust, sensitivity, visibility, acl_policy_id, origin, + idempotency_key, idempotency_scope, created_at + ) VALUES ( + '323e4567-e89b-42d3-a456-426614174000', 'repo', 'default', + 'semantic.alpha', 'semantic', 'accretive', 'confirmed', + 'high', 'repo_evidence', 'internal', 'repo_local', 'default', + 'onboard', 'same-namespace', 'namespace', '2026-08-24T00:00:00Z' + )", + ) + .await + .expect("seed a namespace-scoped idempotency key"); + let duplicate_namespace_key = conn + .execute_unprepared( + "INSERT INTO memory_note_index ( + note_id, scope_key, namespace, path, kind, lifecycle, review_state, + confidence, trust, sensitivity, visibility, acl_policy_id, origin, + idempotency_key, idempotency_scope, created_at + ) VALUES ( + '423e4567-e89b-42d3-a456-426614174000', 'repo', 'default', + 'semantic.beta', 'semantic', 'accretive', 'confirmed', + 'high', 'repo_evidence', 'internal', 'repo_local', 'default', + 'onboard', 'same-namespace', 'namespace', '2026-08-24T00:00:00Z' + )", + ) + .await; + assert!( + duplicate_namespace_key.is_err(), + "namespace idempotency must reject the same key across paths" + ); + + let invalid_job_state = conn + .execute_unprepared( + "INSERT INTO memory_compile_job ( + scope_key, root_kind, root_id, terminal_source_oid, + input_fingerprint_version, input_fingerprint_key_id, + input_fingerprint_digest, state, created_at, updated_at + ) VALUES ( + 'repo', 'task', 'task-invalid-state', + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 1, + '123e4567-e89b-42d3-a456-426614174000', + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 'inflight', 1, 1 + )", + ) + .await; + assert!( + invalid_job_state.is_err(), + "an inflight compiler job must carry a fenced lease" + ); +} + +#[tokio::test] +async fn memory_episode_schema_idempotent() { + let (_fresh_dir, fresh_url, _fresh_path) = fresh_db_url(); + let fresh = connect(&fresh_url).await; + let fresh_applied = run_builtin_migrations(&fresh) + .await + .expect("apply full registry to fresh database"); + assert_eq!(fresh_applied.last(), Some(&2026082401)); + assert!( + run_builtin_migrations(&fresh) + .await + .expect("repeat full registry") + .is_empty(), + "second migration run must be a no-op" + ); + let fresh_snapshot = memory_core_schema_snapshot(&fresh).await; + let repeated_up = builtin_migrations() + .into_iter() + .last() + .expect("built-in registry contains Memory core migration"); + fresh + .execute_raw(Statement::from_string( + fresh.get_database_backend(), + repeated_up.up, + )) + .await + .expect("the Memory core SQL body itself must be idempotent"); + assert_eq!( + memory_core_schema_snapshot(&fresh).await, + fresh_snapshot, + "directly repeating the additive up SQL must not change the schema" + ); + + let (_upgrade_dir, upgrade_url, _upgrade_path) = fresh_db_url(); + let upgrade = connect(&upgrade_url).await; + let migrations = builtin_migrations(); + let (memory_core, previous) = migrations + .split_last() + .expect("built-in registry contains Memory core migration"); + assert_eq!(memory_core.version, 2026082401); + let mut old_runner = MigrationRunner::new(); + old_runner + .extend(previous.iter().cloned()) + .expect("register previous migration tip"); + old_runner + .run_pending(&upgrade) + .await + .expect("apply previous migration tip"); + assert_eq!( + old_runner.current_version(&upgrade).await.unwrap(), + Some(2026081301) + ); + assert_eq!( + run_builtin_migrations(&upgrade) + .await + .expect("upgrade old database"), + vec![2026082401] + ); + assert_eq!(memory_core_schema_snapshot(&upgrade).await, fresh_snapshot); +} + +#[tokio::test] +async fn memory_episode_down_guard() { + let (_dir, url, _path) = fresh_db_url(); + let conn = connect(&url).await; + let runner = builtin_runner().expect("builtin runner"); + runner + .run_pending(&conn) + .await + .expect("apply Memory core migration"); + + conn.execute_unprepared("PRAGMA foreign_keys = OFF") + .await + .expect("disable foreign keys for isolated down-guard fixtures"); + + let fixtures = [ + ( + "memory_head", + "INSERT INTO memory_head ( + scope_key, namespace, path, note_id, latest_revision_oid, + latest_action, latest_review_state, kind, lifecycle, confidence, + trust, sensitivity, visibility, acl_policy_id, last_event_seq, updated_at + ) VALUES ( + 'repo','default','episodic.tasks.a','123e4567-e89b-42d3-a456-426614174000', + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','created','confirmed','episodic', + 'replacement','high','repo_evidence','internal','repo_local','default',1, + '2026-08-24T00:00:00Z' + )", + ), + ( + "memory_path_summary", + "INSERT INTO memory_path_summary ( + scope_key, namespace, path, last_changed_at + ) VALUES ('repo','default','episodic.tasks','2026-08-24T00:00:00Z')", + ), + ( + "memory_note_index", + "INSERT INTO memory_note_index ( + note_id, scope_key, namespace, path, kind, lifecycle, review_state, + confidence, trust, sensitivity, visibility, acl_policy_id, origin, + idempotency_key, created_at + ) VALUES ( + '123e4567-e89b-42d3-a456-426614174000','repo','default','episodic.tasks.a', + 'episodic','replacement','confirmed','high','repo_evidence','internal', + 'repo_local','default','episode_compiler','fixture','2026-08-24T00:00:00Z' + )", + ), + ( + "memory_revision_index", + "INSERT INTO memory_revision_index ( + revision_oid, note_id, scope_key, namespace, origin, producer, + rules_version, policy_version, input_fingerprints_json, created_at + ) VALUES ( + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + '123e4567-e89b-42d3-a456-426614174000','repo','default', + 'episode_compiler','test',1,'v1','[]','2026-08-24T00:00:00Z' + )", + ), + ( + "memory_link_index", + "INSERT INTO memory_link_index ( + source_scope_key, source_namespace, source_note_id, source_revision_oid, + target_note_id, link_kind, source_path, target_path, evidence_refs_json + ) VALUES ( + 'repo','default','123e4567-e89b-42d3-a456-426614174000', + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + '223e4567-e89b-42d3-a456-426614174000','supports', + 'episodic.tasks.a','episodic.tasks.b','[]' + )", + ), + ( + "memory_projection_state", + "INSERT INTO memory_projection_state ( + scope_key, projected_ref_oid, last_event_seq, schema_version, + policy_version, rebuilt_at + ) VALUES ( + 'repo','aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',0,1,'v1',1 + )", + ), + ( + "memory_episode_path", + "INSERT INTO memory_episode_path (note_id, revision_oid, code_path) + VALUES ( + '123e4567-e89b-42d3-a456-426614174000', + 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','src/lib.rs' + )", + ), + ( + "memory_compile_job", + "INSERT INTO memory_compile_job ( + scope_key, root_kind, root_id, terminal_source_oid, + input_fingerprint_version, input_fingerprint_key_id, + input_fingerprint_digest, created_at, updated_at + ) VALUES ( + 'repo','task','task-1','aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',1, + '123e4567-e89b-42d3-a456-426614174000', + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb',1,1 + )", + ), + ( + "memory_compile_observer_state", + "INSERT INTO memory_compile_observer_state ( + scope_key, source_ref_name, scanned_through_oid, updated_at + ) VALUES ( + 'repo','libra/intent','aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa',1 + )", + ), + ]; + + for (table, insert) in fixtures { + conn.execute_unprepared(insert) + .await + .unwrap_or_else(|error| panic!("seed {table}: {error}")); + let error = runner.rollback_to(&conn, 2026081301).await.unwrap_err(); + assert!( + format!("{error:#}").contains("memory_core_down_guard_empty"), + "{table} data must fail the stable down guard: {error:#}" + ); + assert_eq!( + runner.current_version(&conn).await.unwrap(), + Some(2026082401) + ); + conn.execute_unprepared(&format!("DELETE FROM `{table}`")) + .await + .unwrap_or_else(|error| panic!("clear {table}: {error}")); + } + + conn.execute_unprepared("PRAGMA foreign_keys = ON") + .await + .expect("restore foreign key enforcement"); + assert_eq!( + runner + .rollback_to(&conn, 2026081301) + .await + .expect("empty Memory core schema may roll back"), + vec![2026082401] + ); + for table in MEMORY_CORE_TABLES { + assert!( + !table_exists(&conn, table).await, + "down left {table} behind" + ); + } } // --------------------------------------------------------------------------- @@ -194,9 +641,9 @@ async fn usage_event_session_scope_rollback_refuses_duplicate_event_ids() { "rollback error must provide collision remediation: {error:#}" ); // `rollback_to` applies each down in its own claim transaction (see - // `MigrationRunner::rollback_to` phase 2), so the intervening - // 2026081301 provenance down (empty provenance, no linked HEADs in - // this fixture) rolls back cleanly BEFORE 2026080403's guard refuses. + // `MigrationRunner::rollback_to` phase 2), so the intervening 2026082401 + // Memory core and 2026081301 provenance downs (both empty in this fixture) + // roll back cleanly BEFORE 2026080403's guard refuses. // The migration under test — the session-scope event migration — must // remain applied; the runner's partial-plan semantics leave the // version at exactly that refused step. @@ -1204,7 +1651,7 @@ async fn run_builtin_migrations_applies_current_builtin_registry() { 2026071405, 2026071406, 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, - 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301 + 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 ] ); assert!(table_exists(&conn, "schema_versions").await); @@ -1398,7 +1845,7 @@ async fn agent_subagent_content_up_down_up_and_nonempty_guard() { 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, - 2026080401, 2026080402, 2026080403, 2026081301 + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 ] ); conn.execute_raw(Statement::from_string( @@ -1463,7 +1910,7 @@ async fn agent_subagent_content_up_down_up_and_nonempty_guard() { 2026071406, 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, - 2026073101, 2026080401, 2026080402, 2026080403, 2026081301 + 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 ] ); assert!(table_exists(&conn, "agent_subagent_content_claim").await); @@ -1567,7 +2014,7 @@ async fn existing_agent_subagent_1406_schema_upgrades_to_replication() { 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, - 2026080401, 2026080402, 2026080403, 2026081301 + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 ] ); let claim = conn @@ -1701,7 +2148,7 @@ async fn evolved_agent_subagent_1406_columns_upgrade_idempotently() { 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, - 2026080401, 2026080402, 2026080403, 2026081301 + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 ] ); let cursor = conn @@ -1745,11 +2192,11 @@ async fn agent_import_identity_tombstone_up_down_up_round_trip() { assert_eq!( rolled, vec![ - 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, 2026073004, - 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, 2026072501, - 2026072403, 2026072402, 2026072401, 2026072304, 2026072303, 2026072302, 2026072301, - 2026072201, 2026072101, 2026071901, 2026071407, 2026071406, 2026071405, 2026071404, - 2026071403, 2026071402 + 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, + 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, + 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303, 2026072302, + 2026072301, 2026072201, 2026072101, 2026071901, 2026071407, 2026071406, 2026071405, + 2026071404, 2026071403, 2026071402 ] ); assert!(!table_exists(&conn, "agent_import_identity").await); @@ -1768,7 +2215,7 @@ async fn agent_import_identity_tombstone_up_down_up_round_trip() { 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, - 2026080403, 2026081301 + 2026080403, 2026081301, 2026082401 ] ); assert!(table_exists(&conn, "agent_import_identity").await); @@ -1799,10 +2246,11 @@ async fn existing_agent_tombstone_1403_schema_upgrades_to_compat_barrier() { assert_eq!( rolled, vec![ - 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, 2026073004, - 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, 2026072501, - 2026072403, 2026072402, 2026072401, 2026072304, 2026072303, 2026072302, 2026072301, - 2026072201, 2026072101, 2026071901, 2026071407, 2026071406, 2026071405, 2026071404 + 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, + 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, + 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303, 2026072302, + 2026072301, 2026072201, 2026072101, 2026071901, 2026071407, 2026071406, 2026071405, + 2026071404 ] ); assert!(table_exists(&conn, "agent_import_tombstone").await); @@ -1820,7 +2268,8 @@ async fn existing_agent_tombstone_1403_schema_upgrades_to_compat_barrier() { 2026071404, 2026071405, 2026071406, 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, - 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301 + 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, + 2026082401 ] ); assert!(trigger_exists(&conn, "agent_tombstone_block_session_insert").await); @@ -2165,14 +2614,14 @@ async fn approved_permission_up_down_up_round_trip() { assert_eq!( rolled, vec![ - 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, 2026073004, - 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, 2026072501, - 2026072403, 2026072402, 2026072401, 2026072304, 2026072303, 2026072302, 2026072301, - 2026072201, 2026072101, 2026071901, 2026071407, 2026071406, 2026071405, 2026071404, - 2026071403, 2026071402, 2026071401, 2026071301, 2026070803, 2026070802, 2026070801, - 2026070701, 2026070601, 2026070501, 2026070401, 2026070301, 2026070202, 2026070201, - 2026062301, 2026061401, 2026060801, 2026060401, 2026060201, 2026053101, 2026052301, - 2026050801, 2026050601 + 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, + 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, + 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303, 2026072302, + 2026072301, 2026072201, 2026072101, 2026071901, 2026071407, 2026071406, 2026071405, + 2026071404, 2026071403, 2026071402, 2026071401, 2026071301, 2026070803, 2026070802, + 2026070801, 2026070701, 2026070601, 2026070501, 2026070401, 2026070301, 2026070202, + 2026070201, 2026062301, 2026061401, 2026060801, 2026060401, 2026060201, 2026053101, + 2026052301, 2026050801, 2026050601 ] ); assert!( @@ -2203,7 +2652,7 @@ async fn approved_permission_up_down_up_round_trip() { 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, - 2026080403, 2026081301 + 2026080403, 2026081301, 2026082401 ] ); assert!(table_exists(&conn, "approved_permission").await); @@ -3075,9 +3524,9 @@ async fn legacy_layer_rows_with_linked_fail_migration() { .await .expect("rollback layer scope"), vec![ - 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, 2026073004, - 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, 2026072501, - 2026072403, 2026072402, 2026072401, 2026072304, 2026072303 + 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, + 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, + 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303 ] ); conn.execute_raw(Statement::from_string( @@ -3130,7 +3579,7 @@ async fn legacy_layer_rows_with_linked_fail_migration() { vec![ 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, - 2026073101, 2026080401, 2026080402, 2026080403, 2026081301 + 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 ] ); let row = conn @@ -3407,9 +3856,9 @@ async fn sparse_migration_projects_last_wins_toggle() { .await .expect("rollback sparse scope"), vec![ - 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, 2026073004, - 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, 2026072501, - 2026072403, 2026072402, 2026072401, 2026072304 + 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, + 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, + 2026072501, 2026072403, 2026072402, 2026072401, 2026072304 ] ); // Duplicate legacy rows: stale `true` (lower id) then effective `false` @@ -3445,7 +3894,7 @@ async fn sparse_migration_projects_last_wins_toggle() { vec![ 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, - 2026080401, 2026080402, 2026080403, 2026081301 + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 ] ); let row = conn @@ -3480,9 +3929,9 @@ async fn legacy_sparse_state_with_linked_requires_adopt_or_clear() { .await .expect("rollback sparse scope"), vec![ - 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, 2026073004, - 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, 2026072501, - 2026072403, 2026072402, 2026072401, 2026072304 + 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, + 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, + 2026072501, 2026072403, 2026072402, 2026072401, 2026072304 ] ); conn.execute_raw(Statement::from_string( @@ -3525,7 +3974,7 @@ async fn legacy_sparse_state_with_linked_requires_adopt_or_clear() { vec![ 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, - 2026080401, 2026080402, 2026080403, 2026081301 + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 ] ); assert!(column_exists(&conn, "sparse_view", "worktree_id").await); @@ -3746,9 +4195,9 @@ async fn worktree_registry_v2_capability_marker_round_trip() { .await .expect("rollback capability marker"), vec![ - 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, 2026073004, - 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, 2026072501, - 2026072403, 2026072402, 2026072401 + 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, + 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, + 2026072501, 2026072403, 2026072402, 2026072401 ] ); assert!(!table_exists(&conn, "worktree_registry_capability").await); @@ -3759,7 +4208,7 @@ async fn worktree_registry_v2_capability_marker_round_trip() { vec![ 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, - 2026080402, 2026080403, 2026081301 + 2026080402, 2026080403, 2026081301, 2026082401 ] ); assert!(table_exists(&conn, "worktree_registry_capability").await); @@ -3882,7 +4331,7 @@ async fn registry_v2_down_migration_rejects_nonterminal_state() { vec![ 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, - 2026080403, 2026081301 + 2026080403, 2026081301, 2026082401 ] ); } @@ -3964,7 +4413,8 @@ async fn workspace_record_down_migration_rejects_nonterminal_state() { runner.run_pending(&conn).await.expect("re-apply"), vec![ 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, - 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301 + 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, + 2026082401 ] ); @@ -4456,7 +4906,7 @@ async fn registry_v3_rollback_refuses_live_generations() { .current_version(&conn) .await .expect("current version"), - Some(2026081301), + Some(2026082401), "and the schema is untouched" ); } @@ -4503,7 +4953,7 @@ async fn registry_v3_rollback_allows_absent_generations() { assert_eq!( rolled, vec![ - 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005 + 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005 ], "{label}: exactly v3 rolled back" ); @@ -4532,7 +4982,7 @@ async fn registry_v3_rollback_allows_unreadable_registry() { .await .expect("an unparseable registry does not block the rollback"), vec![ - 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005 + 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005 ] ); } @@ -4828,8 +5278,8 @@ async fn stash_generation_fence_up_down_up_round_trip() { ); assert_eq!( runner.current_version(&conn).await.expect("version"), - Some(2026081301), - "the runtime usage migrations are the newest migrations — retarget this test when a newer one lands" + Some(2026082401), + "the Memory core migration is the newest migration — retarget this test when a newer one lands" ); let rolled = runner @@ -4838,7 +5288,9 @@ async fn stash_generation_fence_up_down_up_round_trip() { .expect("down to the previous version"); assert_eq!( rolled, - vec![2026081301, 2026080403, 2026080402, 2026080401, 2026073101], + vec![ + 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101 + ], "the runtime attribution, scope, and fence migrations roll back in order" ); assert!( @@ -4920,7 +5372,7 @@ async fn approved_permission_provenance_migration() { // Up: empty provenance backfill; project_id unchanged. assert_eq!( runner.run_pending(&conn).await.expect("apply W4-07"), - vec![2026081301] + vec![2026081301, 2026082401] ); assert!(column_exists(&conn, "approved_permission", "source_worktree_id").await); assert!(column_exists(&conn, "approved_permission", "source_session_id").await); @@ -5012,7 +5464,7 @@ async fn approved_permission_provenance_migration() { assert_eq!( runner.run_pending(&conn).await.expect("re-up"), - vec![2026081301] + vec![2026081301, 2026082401] ); let kept_after: i64 = conn .query_one_raw(Statement::from_string( @@ -5047,7 +5499,7 @@ async fn approved_permission_old_reader_rejects_migrated_schema() { .await .expect("read tip") .expect("applied tip"); - assert_eq!(current, 2026081301); + assert_eq!(current, 2026082401); // An old binary whose registry tip is still 2026080403 would see this // repository as UnsupportedFuture. Prove the refuse path on repository // DBs (not global config.db) by planting a version above this binary. From 7985982cf610893648fe8328a410675819de14a4 Mon Sep 17 00:00:00 2001 From: anduin9527 Date: Mon, 24 Aug 2026 19:11:30 +0800 Subject: [PATCH 04/18] feat(memory): add FTS5 episode search projection Signed-off-by: anduin9527 --- .github/workflows/memory-portability.yml | 64 ++ docs/development/plan/plan-20260819.md | 53 +- docs/development/tracing/memory.md | 48 +- .../2026082402_memory_fts_search.sql | 73 ++ .../2026082402_memory_fts_search_down.sql | 22 + sql/migrations/README.md | 1 + src/command/maintenance.rs | 11 + src/internal/ai/memory/fts_sql.rs | 792 ++++++++++++++++++ src/internal/ai/memory/mod.rs | 1 + src/internal/db.rs | 11 + src/internal/db/migration.rs | 13 +- .../model/memory_episode_search_doc.rs | 27 + src/internal/model/memory_model_tests.rs | 9 +- src/internal/model/mod.rs | 1 + src/internal/mutable_state_ownership.rs | 119 ++- tests/INDEX.md | 1 + tests/agent_capture_migration_test.rs | 8 +- tests/command/worktree_isolation_test.rs | 16 +- tests/db_migration_test.rs | 540 ++++++++++-- tests/fts5_capability_test.rs | 104 +++ 20 files changed, 1788 insertions(+), 126 deletions(-) create mode 100644 .github/workflows/memory-portability.yml create mode 100644 sql/migrations/2026082402_memory_fts_search.sql create mode 100644 sql/migrations/2026082402_memory_fts_search_down.sql create mode 100644 src/internal/ai/memory/fts_sql.rs create mode 100644 src/internal/model/memory_episode_search_doc.rs create mode 100644 tests/fts5_capability_test.rs diff --git a/.github/workflows/memory-portability.yml b/.github/workflows/memory-portability.yml new file mode 100644 index 000000000..9541300d7 --- /dev/null +++ b/.github/workflows/memory-portability.yml @@ -0,0 +1,64 @@ +name: Memory SQLite portability + +on: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + memory-fts5-probe: + name: memory-fts5-probe (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - platform: linux-amd64 + runner: ubuntu-latest + target: x86_64-unknown-linux-gnu + - platform: linux-arm64 + runner: ubuntu-24.04-arm + target: aarch64-unknown-linux-gnu + - platform: macos-arm64 + runner: macos-latest + target: aarch64-apple-darwin + - platform: windows-amd64 + runner: windows-latest + target: x86_64-pc-windows-msvc + env: + LIBRA_SKIP_WEB_BUILD: "1" + EXPECTED_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + MATRIX_TARGET: ${{ matrix.target }} + steps: + - name: Checkout reviewed head + uses: actions/checkout@v5 + with: + repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }} + ref: ${{ github.event.pull_request.head.sha || github.sha }} + submodules: recursive + persist-credentials: false + + - name: Set up release Rust target + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: stable + target: ${{ matrix.target }} + + - name: Verify tested commit + shell: bash + run: | + actual_sha="$(git rev-parse HEAD)" + if [[ "$actual_sha" != "$EXPECTED_SHA" ]]; then + echo "::error::checked out $actual_sha but expected reviewed head $EXPECTED_SHA" + exit 1 + fi + + - name: Probe release-linked FTS5 + shell: bash + run: | + cargo test --locked --release \ + --features keyring \ + --target "$MATRIX_TARGET" \ + --test fts5_capability_test -- --nocapture diff --git a/docs/development/plan/plan-20260819.md b/docs/development/plan/plan-20260819.md index 8ee018572..d1cc260fe 100644 --- a/docs/development/plan/plan-20260819.md +++ b/docs/development/plan/plan-20260819.md @@ -279,7 +279,7 @@ query + principal + current code commit FTS 表固定 `tokenize='unicode61 remove_diacritics 2'`；`bm25()` 的列权重按 `goal=8, summary=5, decisions=4, failed_attempts=3, unresolved=2`，SQLite 分值越小越相关，所以使用 `ASC`。权重、tokenizer、query normalization 或 tie-break 任一变化都必须提升 selector version 并更新 benchmark baseline。 -采用 FTS5/BM25 的原因：Libra 已使用 SQLite，数据规模以单仓研发历史为边界；FTS5 提供本地倒排索引，`bm25()` 提供无需模型和网络的确定性词项排序。Letta Code 与 Rekal 提供了 Git-backed / 版本关联的工程结构参考；`ldbd-sqlite-fts-baseline` 验证了 SQLite WAL、结构化 scope 过滤、FTS5、`bm25()` 与稳定回退可组成轻量基线。它们只作为实现证据，不替代本计划的 Libra 对象与权限契约。 +采用 FTS5/BM25 的原因：Libra 已使用 SQLite，数据规模以单仓研发历史为边界；FTS5 提供本地倒排索引，`bm25()` 提供无需模型和网络的确定性词项排序。Letta Code 与 Rekal 提供了 Git-backed / 版本关联的工程结构参考；`ldbd-sqlite-fts-baseline` 验证了 SQLite WAL、结构化 scope 过滤、FTS5、`bm25()` 与稳定排序可组成轻量基线。它们只作为实现证据，不替代本计划的 Libra 对象与权限契约。 代码适用性固定为五态：`exact`（current=result）、`descendant_unchanged`（result 是 current 祖先且所有 Episode path 的 tree entry 未变）、`descendant_path_changed`（祖先关系成立但至少一个相关 path 已变）、`diverged`（不在同一祖先链）、`unknown`（anchor/path 缺失、对象损坏或超出 512 paths / 2,048 commits 的预算）。默认自动注入只接收前两态；第三态可检索但带 stale 警告，后两态只允许显式诊断查询。路径检查比较 `result_oid` 与 current tree 上受限路径的 entry OID，不全量扫描仓库。 @@ -530,7 +530,7 @@ flowchart TB | ID | direction | 类型 | 对象 | Owner | 产物与可用性判据 | 证据 | 超时与失败策略 | |---|---|---|---|---|---|---|---| -| DEP-M2-CI-01 | incoming | 外部 D-01 执行面 | GitHub Actions 四平台 FTS5 probe | M2-02F / repository Actions | 新增 `.github/workflows/memory-portability.yml`；`pull_request` + `workflow_dispatch` 触发；job ID `memory-fts5-probe` 在 Linux amd64/arm64、macOS arm64、Windows amd64 对同一 head SHA 运行 `cargo test --locked --release --test fts5_capability_test -- --nocapture`；无 secrets、R2 upload、tag 或 release artifact | `gh run view --json headSha,event,status,conclusion,jobs,url`；四个 matrix job 均 `success`，`headSha` 等于被审分支/PR SHA | push 后 24h 内允许因平台故障重跑一次；仍无四个绿色 job 时 M2-02F 保持 `Lifecycle=blocked / Acceptance=locally-accepted`，不得启动 M2-11 reader；不改用 tag-only `release.yml` | +| DEP-M2-CI-01 | incoming | 外部 D-01 执行面 | GitHub Actions 四平台 FTS5 probe | M2-02F / repository Actions | 新增 `.github/workflows/memory-portability.yml`；`pull_request` + `workflow_dispatch` 触发；job ID `memory-fts5-probe` 在 Linux amd64/arm64、macOS arm64、Windows amd64 checkout 同一 PR head SHA，并运行 `cargo test --locked --release --features keyring --target --test fts5_capability_test -- --nocapture`；无 secrets、R2 upload、tag 或 release artifact | `gh run view --json headSha,event,status,conclusion,jobs,url`；四个 matrix job 均 `success`，`headSha` 等于被审分支/PR SHA | push 后 24h 内允许因平台故障重跑一次；仍无四个绿色 job 时 M2-02F 保持 `Lifecycle=blocked / Acceptance=locally-accepted`，不得启动 M2-11 reader；不改用 tag-only `release.yml` | | DEP-M2-CI-02 | incoming | 外部 D-01 执行面 | 现有 base CI + CodeQL | M2-15 / repository Actions | 对 base ref `main` 的 `pull_request`：`.github/workflows/base.yml` 的 job ID `format/clippy/web-check/compat-web-e2e/owner-liveness-macos/redundancy/test/command-tests/network-remotes` 全部运行；`.github/workflows/codeql.yml` 的 job `analyze` 在 `actions`、`rust` matrix 全部运行。每个 run 的 `headSha` 必须等于被审 PR head SHA，所有适用 job/matrix conclusion 必须为 `success`，不得以缺失、取消或旧 SHA 代替 | `gh pr checks `；分别用 `gh run view --json workflowName,event,headBranch,headSha,status,conclusion,jobs,url` 保存 base/CodeQL run ID、URL、job 名与结论；开工/发布日再从 workflow `jobs:` 节点机械比对，发现漂移先修计划 | C 组推送后进入 `Acceptance=remote-pending`；24h 内允许对基础设施失败重跑一次。仍有缺失/失败 job 时 M2-15 保持 `Lifecycle=blocked / Acceptance=remote-pending`，按失败所属前置卡新增 FIX 卡并以前滚 commit 修复；不 force-push、不降低 required check | | DEP-M2-OUT-01 | outgoing | 跨计划 handoff | mainline ML-05/ML-08 | 下一份 mainline 日期计划 / mentor | 复用 `ContextSelectionReceiptV1`、`context_selection_receipt`、`RepositoryKeyedDigest` 与 shared selector；不得重建平行类型/表 | 本计划 ADR-M2-10/11、M2-01、M2-01K、M2-02R、M2-12；核对 2026-08-20 | 不阻塞 M2；接收方未启动时保留 Memory owner，并在 `mainline.md` 标记 pending handoff | @@ -844,49 +844,62 @@ flowchart TB **Task type:** `migration` -**Lifecycle / Acceptance:** `pending` / 空 +**Lifecycle / Acceptance:** `in-progress` / `locally-accepted` **Description:** 让 Libra 的实际 release 二进制在四个目标平台都具备同一 FTS5 能力，并用独立迁移增加 Episode external-content 搜索表。 **Out of scope:** 排序/适用性 reader 和 benchmark 调权；由 M2-11、M2-14 承接。 -**Current evidence:** 当前 crate 通过 SeaORM/sqlx-sqlite 使用 SQLite，但仓库没有 FTS5 probe 或 feature contract；现有 `.github/workflows/release.yml` 只由 `v*` tag 触发且包含 R2 上传，不能作为无副作用 PR 门。证据：`Cargo.toml:66-70`、`.github/workflows/release.yml:1-8,100+`，以及核对日 `rg -n "FTS5|bm25\\(" src sql tests` 零实现命中。 +**Current evidence:** + +| 事实 | 证据 | +|---|---| +| SeaORM/sqlx-sqlite 已经通过 `libsqlite3-sys 0.30.1` 的 `bundled` feature 编译 SQLite，其 build script 明确设置 `SQLITE_ENABLE_FTS5`；本卡无需再增加直接依赖或修改 lockfile | `cargo tree -e features -i libsqlite3-sys`；Cargo registry 中 `libsqlite3-sys-0.30.1/build.rs` | +| `2026082402_memory_fts_search` 已创建单份正文表、external-content FTS5 虚拟表、稳定 rowid、复合外键与两个过滤索引 | `sql/migrations/2026082402_memory_fts_search{,_down}.sql`；`memory_episode_fts_*` 3/3 | +| `fts_sql` 由调用方持有 `MemoryWriteTransaction`；该不可伪造包装保证 SQLite 在任何读取前取得写锁。更新/删除先用旧正文删除 posting，再更新正文并写新 posting，任一步失败都由整个事务回滚 | `src/internal/ai/memory/fts_sql.rs`；模块测试 6/6（含自然语言部分词项召回与双连接写竞争） | +| 实际链接的 SQLite 已通过 create/insert/参数绑定 MATCH/五列加权 `bm25()` ASC/`unicode61 remove_diacritics 2` 探测 | `fts5_capability_test::sqlite_fts5_release_capability` 1/1（debug）；release+keyring 复验列于 Verification | +| 新 migration tip 已传播到受影响的 migration/doctor 测试；完整 migration target 77/78，唯一失败为既有 SQLite 并发锁偶发项，具名单跑 1/1 通过 | 2026-08-24 固定容器证据 | +| `cargo test --all` 已运行：lib target 4484/4495；其中 migration registry 与 mutable-state ownership 两项由本卡修复后具名单跑通过，余下 8 项集中在既有全局 cwd/进程隔离、bwrap 环境与 Web 状态测试，未被本卡写集触碰 | 2026-08-24 固定容器输出；本卡 focused 复验见 Verification | +| 四平台无副作用 workflow 已落地；PR head SHA checkout、四个 release target 和绿色 job 仍需 push 后取远端 D-01 证据 | `.github/workflows/memory-portability.yml`；DEP-M2-CI-01 | **Acceptance criteria:** -- [ ] build contract 固定为 release binary 自带可验证的 FTS5；任一目标缺失时启用 `libsqlite3-sys` bundled SQLite，而非降级成 `LIKE` 全表扫描。 -- [ ] migration 创建 `memory_episode_search_doc`、external-content `memory_episode_fts` 与一一对应 integer rowid。 -- [ ] FTS schema 固定 `unicode61 remove_diacritics 2` 和五列顺序，列权重由 ADR-M2-08/本文召回合同拥有。 -- [ ] 具名 `fts_sql` Module 在单一事务维护 search doc 与 FTS，不允许投影调用方散落 trigger/SQL 字符串。 -- [ ] 参数绑定和受限 query normalization 拒绝畸形 MATCH 表达式，不把用户输入拼接进 SQL。 -- [ ] migration up/idempotent/down/fresh-vs-upgrade 通过，已有核心 projection 数据不丢失；前一 registry tip 的 reader 对已迁移 DB 在 schema preflight 明确拒绝。 -- [ ] 本机以 release profile 链接与 Libra 相同 SQLite feature 的 capability target 能 create/insert/MATCH/`bm25()` 并断言分数升序语义。 +- [x] build contract 固定为 release binary 自带可验证的 FTS5；当前传递依赖已启用 bundled SQLite/FTS5，不增加第二份直接依赖，也不降级成 `LIKE` 全表扫描。 +- [x] migration 创建 `memory_episode_search_doc`、external-content `memory_episode_fts` 与一一对应 integer rowid；`ended_at` 允许未结束来源使用 `NULL`。 +- [x] FTS schema 固定 `unicode61 remove_diacritics 2`、五列顺序及 `goal/summary/decisions/failed_attempts/unresolved = 8/5/4/3/2` 权重合同。 +- [x] 具名 `fts_sql` Module 只接受调用方持有且已预先取得 SQLite 写锁的 `MemoryWriteTransaction`，按「旧 posting 删除 → content update/delete → 新 posting 写入」维护一致性；不使用 trigger，不自行 commit。 +- [x] 参数绑定和版本化 plain-text normalization 提取连续 Unicode 字母/数字词项，以 ASCII 大小写折叠键去重、其它 Unicode 词项按原文去重，再以受控 `OR` 连接并逐词做 literal quoting；拒绝空输入、控制字符、超长 term/query 与过多去重后词项，不让调用者提交 FTS 运算符或 SQL 片段，也不内置特定语言的停用词表。 +- [x] migration up/idempotent/down/fresh-vs-upgrade 通过，已有核心 projection 数据不丢失；前一 registry tip 的 reader 对已迁移 DB 在 schema preflight 明确拒绝。 +- [x] 本机实际链接的 SQLite capability target 能 create/insert/MATCH/`bm25()` 并断言分数升序语义；release profile + `keyring` 复验仍列在本卡 Verification 门中。 - [ ] `memory-portability.yml` 按 DEP-M2-CI-01 在同一 branch/PR SHA 跑四平台 probe 且无发布副作用；缺一绿色平台时本卡保持模板规定的 `locally-accepted` 并阻止后续 reader 开工。 **Verification:** -- [ ] `source .env.test && cargo test --test db_migration_test memory_episode_fts_schema` -- [ ] `source .env.test && cargo test --test memory_episode_test sqlite_fts5_release_capability` -- [ ] `source .env.test && cargo test --test memory_episode_test fts_external_content_atomic` -- [ ] `source .env.test && cargo test --test memory_episode_test fts_query_parser_rejects_malformed` -- [ ] `source .env.test && cargo test --test db_migration_test memory_fts_old_reader_rejects_migrated_schema` -- [ ] `source .env.test && LIBRA_SKIP_WEB_BUILD=1 cargo test --release --test fts5_capability_test -- --nocapture` +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --test db_migration_test memory_episode_fts -- --nocapture`（3/3） +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --lib internal::ai::memory::fts_sql -- --nocapture`（6/6） +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --test fts5_capability_test -- --nocapture`（1/1） +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --lib internal::db::tests::memory_fts_old_reader_rejects_migrated_schema`（1/1） +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --lib internal::model::memory_model_tests::memory_entities_expose_expected_primary_keys`（1/1） +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --test db_migration_test gc_object_source_inventory_covers_every_oid_column`（1/1） +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --locked --release --features keyring --test fts5_capability_test -- --nocapture`（1/1） +- [x] `cargo +nightly fmt --all --check`；`cargo clippy --all-targets --all-features -- -D warnings` +- [ ] DEP-M2-CI-01 四平台 `memory-fts5-probe` 均为同一 pushed PR head SHA 且 `success` **Dependencies:** M2-02（核心 schema 与 migration registry）；DEP-M2-CI-01（本卡交付并消费的远端 D-01 执行面）。 **Deliverables:** N/A。 -**Implementation write set:** `Cargo.toml`、`Cargo.lock`、独立 FTS migration、`src/internal/db/migration.rs`、`src/internal/ai/memory/fts_sql.rs`、新增无发布副作用的 `.github/workflows/memory-portability.yml`、`tests/{db_migration_test.rs,memory_episode_test.rs,fts5_capability_test.rs,INDEX.md}`、migration README。 +**Implementation write set:** `sql/migrations/2026082402_memory_fts_search{,_down}.sql`、`src/internal/db/migration.rs`、`src/internal/db.rs`、`src/internal/model/{memory_episode_search_doc.rs,memory_model_tests.rs,mod.rs}`、`src/internal/ai/memory/{mod.rs,fts_sql.rs}`、`src/command/maintenance.rs` 的 GC source inventory、`src/internal/mutable_state_ownership.rs`、`.github/workflows/memory-portability.yml`、`tests/{db_migration_test.rs,agent_capture_migration_test.rs,command/worktree_isolation_test.rs,fts5_capability_test.rs,INDEX.md}`、migration README、`memory.md` 与本计划。`Cargo.toml` / `Cargo.lock` 不变，因为现有 bundled SQLite 已提供 FTS5。 **Release write set:** N/A。 -**Files likely touched:** build feature、一个 migration、FTS SQL owner、一个 workflow，≤12 个生产/配置文件。 +**Files likely touched:** 一个 migration pair、一个 SeaORM entity、FTS SQL owner、migration/model/GC registry、一个 workflow、受 tip 影响的既有测试与两份设计文档。 **Docs and compatibility impact:** `memory.md` 与 migration README 固定 tokenizer/weights/build capability；无公开命令。 **Rollback mode:** `forward-only` -**Migration and rollback:** 空搜索表可 down；写入后只以前滚 migration 修复。bundled SQLite 回退仅在确认所有受支持目标仍通过 FTS probe 后允许。 +**Migration and rollback:** 空搜索表且 FTS integrity check 通过时可 down；写入后 down fail-closed，只以前滚 migration 修复。`memory_episode_search_doc` 对 revision 使用 `ON DELETE RESTRICT`，调用方必须先按 FTS 删除协议移除 posting/content，防止级联删除留下孤立索引。 **Security and privacy:** query 只使用参数绑定；FTS 文本已在 Writer 内完成授权与脱敏。 diff --git a/docs/development/tracing/memory.md b/docs/development/tracing/memory.md index d52c1e287..f7367aa09 100644 --- a/docs/development/tracing/memory.md +++ b/docs/development/tracing/memory.md @@ -850,6 +850,52 @@ source-input fingerprint 拆列保存 `version/key_id/digest`；`purpose=source_ 由不可绕过的 `SourceInputFingerprint` 类型固定，不在表中重复存字符串。 迁移本身不得初始化、读取或轮换 repository keyed-digest seed。 +M2-02F 随后的版本化迁移 +`sql/migrations/2026082402_memory_fts_search.sql` 与对应 down 文件增加 +Episode 搜索投影。这里的「搜索正文表」（search document）指 SQLite 普通表 +`memory_episode_search_doc`；它保存唯一一份可搜索文本。「全文倒排索引」 +（FTS5 index）指 external-content 虚拟表 `memory_episode_fts`；它只保存由正文生成的 +posting，不再复制正文。二者共享稳定的 `INTEGER rowid`，并遵守以下固定合同： + +| 存储项 | 字段与约束 | 用途 | +|---|---|---| +| `memory_episode_search_doc` 身份 | `rowid` 主键；`(note_id, revision_oid)` 唯一并复合外键到 `memory_revision_index`；父项删除使用 `RESTRICT` | 一条 Episode revision 对应一个稳定搜索文档；先执行受控 FTS 删除，才能删除父 revision | +| root / 过滤元数据 | `root_kind=task\|intent`、有界 `root_id`、`completion_status`、`code_change_status`、可空 `ended_at` | 在 MATCH 前做 root/状态/时间过滤；尚未结束的来源允许 `ended_at=NULL` | +| 唯一正文 | `goal`、`summary`、`decisions`、`failed_attempts`、`unresolved` 均非空字符串，合计最多 64 KiB | 保留结局级结构，而非把整个 session/transcript 放进 SQLite | +| `memory_episode_fts` | 同样的五列与相同顺序；`content='memory_episode_search_doc'`、`content_rowid='rowid'` | 只承载可重建倒排索引 | +| tokenizer | `unicode61 remove_diacritics 2` | Unicode 分词；例如查询 `cafe` 可命中 `café` | +| BM25 权重 | `goal=8`、`summary=5`、`decisions=4`、`failed_attempts=3`、`unresolved=2`，分数按升序排列 | M2-11 reader 使用的首版固定排序合同；SQLite `bm25()` 分数越小越相关 | + +external-content FTS5 不会替调用方同步普通表。唯一 SQL owner 是 crate-private +`src/internal/ai/memory/fts_sql.rs`。调用方先取得不可从普通事务构造的 +`MemoryWriteTransaction`；该包装通过现有 `begin_write_transaction` 在任何读取前取得 +SQLite 写锁，再向同一事务暴露普通投影写入接口。这样既能与其它 Memory 投影原子提交， +也不会在并发写入时发生 deferred transaction 的 read→write upgrade `SQLITE_BUSY`。 +`fts_sql` 不自行提交： + +1. 新建时先写 `memory_episode_search_doc`，取得 rowid，再写对应 FTS posting。 +2. 更新时先读取旧五列，用 FTS5 的 `delete` 命令删除旧 posting；随后原位更新 + 同一 rowid 的正文，并写入新 posting。 +3. 删除时同样先用旧五列删除 posting，再删除普通表行。不能依赖 FK cascade， + 因为虚拟表无法参与普通外键级联。 +4. 任一步失败，调用方回滚整个事务；旧正文和旧 posting 必须同时恢复。 +5. 重建使用 FTS5 `rebuild` 命令，并立即执行 external-content `integrity-check`。 + +查询入口只接受普通文本。`normalize_plain_text_v1` 先提取连续的 Unicode 字母/数字 +词项，以 ASCII 大小写折叠键去重，再按 FTS literal 引号规则转义并以受控 `OR` +连接；其它 Unicode 词项按原文去重。空输入、控制字符、超过 4 KiB 的 query、超过 +256 bytes 的词项或超过 32 个去重后词项会明确报错。最终表达式仍通过 `MATCH ?` +参数绑定执行。这样 `authentication/cache` 会生成两个独立词项，自然语言查询只需 +命中部分有效词项；调用方仍不能直接传 `OR`、`NEAR`、列过滤器或其它 FTS 语法。 +首版不内置特定语言的停用词表；召回质量、多语言分词与多字段权重由 M2-14 +benchmark 校准。 + +该能力不新增数据库服务或第二套 SQLite。当前 SeaORM/sqlx-sqlite 依赖链已经通过 +`libsqlite3-sys` bundled build 启用 `SQLITE_ENABLE_FTS5`；发布门使用与正式二进制 +相同的 `--release --features keyring`，分别在 Linux amd64/arm64、macOS arm64 和 +Windows amd64 上运行 capability probe。任一平台不能 create/MATCH/`bm25()` 时， +reader 不得以 `LIKE` 全表扫描代替。 + 下面以普通 `CREATE TABLE` 形式给出表结构，落地时请置于上述迁移文件中并加上 `IF NOT EXISTS` 幂等保护： ```sql @@ -1761,7 +1807,7 @@ query -> fail-closed: 无 embedding 配置或 provider 失败 -> 仅 Channel 0 ``` -- **Channel 0（常开）**：路径前缀（§6.2）+ BM25/FTS5 关键词检索；无任何 embedding 配置即可用、可测（memweave 的「零外部服务 + 纯关键词降级」与 agentmemory 的「BM25 always on」为同向证据）。 +- **Channel 0（常开）**：路径前缀（§6.2）+ BM25/FTS5 关键词检索；无任何 embedding 配置即可用、可测（memweave 的「零外部服务 + 纯关键词降级」与 agentmemory 的「BM25 always on」为同向证据）。首版搜索文档、tokenizer、plain-text query normalization、五列权重与事务维护顺序已经在 §5.2 固定；M2-11 只实现结构化过滤、`MATCH ?`、`bm25(..., 8, 5, 4, 3, 2) ASC` 与稳定 tie-break，不能再定义另一套 FTS schema 或接受原始 MATCH 语法。 - **Channel 1（可选）**：向量相似度。embedding 必须可本地运行（llama.cpp / Ollama 等）；embedding 按内容哈希缓存（`memory_embedding_cache`，§5.2，可整体丢弃）。provider 缺失、失败或超时 → 自动回退 Channel 0；**不得**把不可验证的向量结果注入 prompt。 - **Channel 2（Phase E）**：§6.4 已定义的实体图有界一跳/两跳查询，只返回候选。 - **融合与去重**：RRF 融合使用固定 k（起点 k=60，agentmemory 口径）；MMR 或等价去重 + 会话多样化上限（起点 max 3/session），避免同一事实多 chunk 重复注入。 diff --git a/sql/migrations/2026082402_memory_fts_search.sql b/sql/migrations/2026082402_memory_fts_search.sql new file mode 100644 index 000000000..3bfc0b223 --- /dev/null +++ b/sql/migrations/2026082402_memory_fts_search.sql @@ -0,0 +1,73 @@ +-- M2-02F: rebuildable Episode search document plus external-content FTS5. +-- +-- The ordinary table owns the only copy of searchable text. The virtual table +-- stores postings only and is maintained by internal::ai::memory::fts_sql in +-- the same caller-owned transaction as the content row. + +CREATE TABLE IF NOT EXISTS `memory_episode_search_doc` ( + `rowid` INTEGER PRIMARY KEY, + `note_id` TEXT NOT NULL CHECK ( + length(`note_id`) = 36 + AND substr(`note_id`, 9, 1) = '-' + AND substr(`note_id`, 14, 1) = '-' + AND substr(`note_id`, 19, 1) = '-' + AND substr(`note_id`, 24, 1) = '-' + AND length(replace(`note_id`, '-', '')) = 32 + AND replace(`note_id`, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + `revision_oid` TEXT NOT NULL CHECK ( + length(`revision_oid`) IN (40,64) + AND `revision_oid` NOT GLOB '*[^0-9a-f]*' + ), + `root_kind` TEXT NOT NULL CHECK (`root_kind` IN ('task','intent')), + `root_id` TEXT NOT NULL CHECK ( + length(`root_id`) > 0 + AND length(CAST(`root_id` AS BLOB)) <= 120 + ), + `completion_status` TEXT NOT NULL CHECK ( + `completion_status` IN ('completed','failed','cancelled') + ), + `code_change_status` TEXT NOT NULL CHECK ( + `code_change_status` IN ('changed','unchanged','unknown') + ), + `ended_at` TEXT CHECK ( + `ended_at` IS NULL + OR length(CAST(`ended_at` AS BLOB)) BETWEEN 1 AND 64 + ), + `goal` TEXT NOT NULL, + `summary` TEXT NOT NULL, + `decisions` TEXT NOT NULL, + `failed_attempts` TEXT NOT NULL, + `unresolved` TEXT NOT NULL, + UNIQUE (`note_id`, `revision_oid`), + CHECK ( + length(CAST(`goal` AS BLOB)) + + length(CAST(`summary` AS BLOB)) + + length(CAST(`decisions` AS BLOB)) + + length(CAST(`failed_attempts` AS BLOB)) + + length(CAST(`unresolved` AS BLOB)) <= 65536 + ), + FOREIGN KEY (`note_id`, `revision_oid`) + REFERENCES `memory_revision_index`(`note_id`, `revision_oid`) + ON DELETE RESTRICT +); + +CREATE INDEX IF NOT EXISTS `idx_memory_episode_search_root` + ON `memory_episode_search_doc`( + `root_kind`, `root_id`, `ended_at`, `note_id`, `revision_oid` + ); +CREATE INDEX IF NOT EXISTS `idx_memory_episode_search_filters` + ON `memory_episode_search_doc`( + `completion_status`, `code_change_status`, `ended_at`, `note_id`, `revision_oid` + ); + +CREATE VIRTUAL TABLE IF NOT EXISTS `memory_episode_fts` USING fts5( + `goal`, + `summary`, + `decisions`, + `failed_attempts`, + `unresolved`, + content='memory_episode_search_doc', + content_rowid='rowid', + tokenize='unicode61 remove_diacritics 2' +); diff --git a/sql/migrations/2026082402_memory_fts_search_down.sql b/sql/migrations/2026082402_memory_fts_search_down.sql new file mode 100644 index 000000000..48d274ded --- /dev/null +++ b/sql/migrations/2026082402_memory_fts_search_down.sql @@ -0,0 +1,22 @@ +-- Rollback of 2026082402_memory_fts_search. +-- +-- An empty search projection can be removed. Once a document was indexed, +-- repair moves forward so rollback cannot silently discard search state. + +INSERT INTO `memory_episode_fts`(`memory_episode_fts`, `rank`) +VALUES ('integrity-check', 1); + +CREATE TABLE IF NOT EXISTS `memory_fts_search_down_guard` ( + `blocked` INTEGER NOT NULL, + CONSTRAINT `memory_fts_search_down_guard_empty` CHECK (`blocked` = 0) +); + +INSERT INTO `memory_fts_search_down_guard` (`blocked`) +SELECT COUNT(*) FROM `memory_episode_search_doc`; + +DROP TABLE `memory_fts_search_down_guard`; + +DROP TABLE IF EXISTS `memory_episode_fts`; +DROP INDEX IF EXISTS `idx_memory_episode_search_filters`; +DROP INDEX IF EXISTS `idx_memory_episode_search_root`; +DROP TABLE IF EXISTS `memory_episode_search_doc`; diff --git a/sql/migrations/README.md b/sql/migrations/README.md index f4976c716..414c62f3f 100644 --- a/sql/migrations/README.md +++ b/sql/migrations/README.md @@ -171,6 +171,7 @@ helpers in `db.rs`. Subsequent CEXes have populated this directory. | `2026080403` | `agent_usage_event_session_scope` | `2026080403_agent_usage_event_session_scope{,_down}.sql` (W2-12 follow-up: replay event IDs are unique within the durable session, permitting the same browser command ID in independent sessions.) | | `2026081301` | `approved_permission_provenance` | `2026081301_approved_permission_provenance{,_down}.sql` (plan-20260715 W4-07: Always-approval provenance columns; empty backfill; `project_id` not rewritten; down fail-closed with provenance or linked HEAD evidence.) | | `2026082401` | `memory_core` | `2026082401_memory_core{,_down}.sql` (M2-02: rebuildable Memory projections plus bounded per-root compiler job and source-observer state; FTS5 and receipt storage land in later migrations.) | +| `2026082402` | `memory_fts_search` | `2026082402_memory_fts_search{,_down}.sql` (M2-02F: rebuildable Episode search document plus external-content FTS5 using the fixed `unicode61 remove_diacritics 2` tokenizer; non-empty rollback is refused.) | All registered migrations are loaded via `include_str!`. New migrations must follow the same pattern — inline SQL strings in `builtin_migrations()` are no diff --git a/src/command/maintenance.rs b/src/command/maintenance.rs index 3fd6a18ac..ce19ac222 100644 --- a/src/command/maintenance.rs +++ b/src/command/maintenance.rs @@ -3335,6 +3335,17 @@ pub const GC_OBJECT_SOURCE_INVENTORY: &[GcObjectSource] = &[ "revision_oid", "Rebuildable Episode path lookup keyed by a Memory revision; it accelerates filtering and contributes no reachability authority", ), + GcObjectSource { + origin: GcSourceOrigin::Column, + location: "memory_episode_search_doc", + column: "revision_oid", + status: GcSourceStatus::IndexOnly, + kind: GcStorageKind::SqliteColumn, + schema: "2026082402_memory_fts_search rebuildable projection", + read_bound: "not read by GC; projection replay rebuilds the search document and FTS postings", + corruption: GcCorruptionPolicy::LenientSkip, + note: "Episode search lookup keyed by an authoritative Memory revision; SQLite search state never owns object reachability", + }, memory_runtime_oid( "memory_compile_job", "terminal_source_oid", diff --git a/src/internal/ai/memory/fts_sql.rs b/src/internal/ai/memory/fts_sql.rs new file mode 100644 index 000000000..8738f0310 --- /dev/null +++ b/src/internal/ai/memory/fts_sql.rs @@ -0,0 +1,792 @@ +//! Atomic owner for the Episode external-content FTS5 projection. +//! +//! Callers own the surrounding SQLite transaction. Keeping that boundary +//! outside this module lets a future projection writer update ordinary Memory +//! indexes, the search document, FTS postings, and its watermark atomically. + +use std::collections::HashSet; + +use chrono::{DateTime, Utc}; +use git_internal::hash::ObjectHash; +use sea_orm::{ConnectionTrait, DatabaseConnection, DatabaseTransaction, DbErr, Statement}; +use thiserror::Error; + +use super::domain::{CodeChangeStatus, CompletionStatus, EpisodeRoot, EpisodeRootKind}; +use crate::internal::db; + +const MAX_SEARCH_TEXT_BYTES: usize = 64 * 1024; +const MAX_MATCH_INPUT_BYTES: usize = 4 * 1024; +const MAX_MATCH_TERM_BYTES: usize = 256; +const MAX_MATCH_TERMS: usize = 32; +const MATCH_QUERY_VERSION: u32 = 1; + +#[derive(Debug, Error)] +pub(crate) enum MemoryFtsError { + #[error("invalid Memory search document field: {field}")] + InvalidDocument { field: &'static str }, + #[error("invalid plain-text Memory search query: {reason}")] + InvalidQuery { reason: &'static str }, + #[error("Memory search projection is internally inconsistent")] + CorruptProjection, + #[error("Memory search storage operation failed")] + Storage(#[from] DbErr), +} + +/// A caller-owned transaction that acquired SQLite's write lock before any +/// reads were allowed. +/// +/// External-content maintenance is read-then-write: updates and deletes must +/// fetch the old text before removing its posting. Accepting an arbitrary +/// deferred transaction would allow a read-lock-to-write-lock upgrade, where +/// SQLite returns `SQLITE_BUSY` immediately instead of honoring busy timeout. +/// The private field prevents callers from certifying an arbitrary transaction +/// as write-locked. +pub(crate) struct MemoryWriteTransaction(DatabaseTransaction); + +impl MemoryWriteTransaction { + pub(crate) async fn begin(database: &DatabaseConnection) -> Result { + Ok(Self(db::begin_write_transaction(database).await?)) + } + + /// Borrow the same transaction for the other Memory projection writes + /// that must commit atomically with the FTS posting. + pub(crate) const fn as_database_transaction(&self) -> &DatabaseTransaction { + &self.0 + } + + pub(crate) async fn commit(self) -> Result<(), MemoryFtsError> { + self.0.commit().await?; + Ok(()) + } + + pub(crate) async fn rollback(self) -> Result<(), MemoryFtsError> { + self.0.rollback().await?; + Ok(()) + } +} + +pub(crate) struct EpisodeSearchText { + goal: String, + summary: String, + decisions: String, + failed_attempts: String, + unresolved: String, +} + +impl EpisodeSearchText { + pub(crate) fn new( + goal: impl Into, + summary: impl Into, + decisions: impl Into, + failed_attempts: impl Into, + unresolved: impl Into, + ) -> Result { + let text = Self { + goal: goal.into(), + summary: summary.into(), + decisions: decisions.into(), + failed_attempts: failed_attempts.into(), + unresolved: unresolved.into(), + }; + let fields = [ + ("goal", text.goal.as_str()), + ("summary", text.summary.as_str()), + ("decisions", text.decisions.as_str()), + ("failed_attempts", text.failed_attempts.as_str()), + ("unresolved", text.unresolved.as_str()), + ]; + let mut total_bytes = 0usize; + for (field, value) in fields { + if value.contains('\0') { + return Err(MemoryFtsError::InvalidDocument { field }); + } + total_bytes = + total_bytes + .checked_add(value.len()) + .ok_or(MemoryFtsError::InvalidDocument { + field: "search_text", + })?; + } + if total_bytes > MAX_SEARCH_TEXT_BYTES { + return Err(MemoryFtsError::InvalidDocument { + field: "search_text", + }); + } + Ok(text) + } +} + +pub(crate) struct EpisodeSearchDocument { + root: EpisodeRoot, + revision_oid: ObjectHash, + completion_status: CompletionStatus, + code_change_status: CodeChangeStatus, + ended_at: Option>, + text: EpisodeSearchText, +} + +impl EpisodeSearchDocument { + pub(crate) const fn new( + root: EpisodeRoot, + revision_oid: ObjectHash, + completion_status: CompletionStatus, + code_change_status: CodeChangeStatus, + ended_at: Option>, + text: EpisodeSearchText, + ) -> Self { + Self { + root, + revision_oid, + completion_status, + code_change_status, + ended_at, + text, + } + } + + fn note_id(&self) -> String { + self.root.note_id().to_string() + } + + fn revision_oid(&self) -> String { + self.revision_oid.to_string() + } + + fn root_kind(&self) -> &'static str { + match self.root.kind() { + EpisodeRootKind::Task => "task", + EpisodeRootKind::Intent => "intent", + } + } + + fn completion_status(&self) -> &'static str { + match self.completion_status { + CompletionStatus::Completed => "completed", + CompletionStatus::Failed => "failed", + CompletionStatus::Cancelled => "cancelled", + } + } + + fn code_change_status(&self) -> &'static str { + match self.code_change_status { + CodeChangeStatus::Changed => "changed", + CodeChangeStatus::Unchanged => "unchanged", + CodeChangeStatus::Unknown => "unknown", + } + } + + fn ended_at(&self) -> Option { + self.ended_at.map(|value| value.to_rfc3339()) + } +} + +pub(crate) struct BoundMatchQuery(String); + +impl BoundMatchQuery { + pub(crate) const fn version(&self) -> u32 { + MATCH_QUERY_VERSION + } + + pub(crate) fn as_str(&self) -> &str { + &self.0 + } +} + +/// Convert ordinary text to a bounded FTS5 literal expression. +/// +/// Unicode letter/number runs are deduplicated with ASCII case folding, quoted, +/// and joined with a controlled `OR`. Strings such as `OR`, `NEAR`, `-foo`, or +/// `column:value` therefore remain literal terms rather than FTS syntax. The +/// result is still passed through `MATCH ?`; callers never provide SQL or FTS +/// fragments. +pub(crate) fn normalize_plain_text_v1(input: &str) -> Result { + if input.is_empty() || input.len() > MAX_MATCH_INPUT_BYTES { + return Err(MemoryFtsError::InvalidQuery { + reason: "input size", + }); + } + if input.chars().any(char::is_control) { + return Err(MemoryFtsError::InvalidQuery { + reason: "control character", + }); + } + + let mut literals = Vec::new(); + let mut seen_terms = HashSet::new(); + for term in input.split(|character: char| !character.is_alphanumeric()) { + if term.is_empty() { + continue; + } + if term.len() > MAX_MATCH_TERM_BYTES { + return Err(MemoryFtsError::InvalidQuery { + reason: "term size", + }); + } + if !seen_terms.insert(term.to_ascii_lowercase()) { + continue; + } + if literals.len() == MAX_MATCH_TERMS { + return Err(MemoryFtsError::InvalidQuery { + reason: "term count", + }); + } + literals.push(format!("\"{}\"", term.replace('"', "\"\""))); + } + if literals.is_empty() { + return Err(MemoryFtsError::InvalidQuery { + reason: "no searchable term", + }); + } + Ok(BoundMatchQuery(literals.join(" OR "))) +} + +struct StoredSearchDocument { + rowid: i64, + goal: String, + summary: String, + decisions: String, + failed_attempts: String, + unresolved: String, +} + +pub(crate) async fn upsert_document( + transaction: &MemoryWriteTransaction, + document: &EpisodeSearchDocument, +) -> Result { + let transaction = transaction.as_database_transaction(); + let note_id = document.note_id(); + let revision_oid = document.revision_oid(); + if let Some(stored) = read_document(transaction, ¬e_id, &revision_oid).await? { + delete_posting(transaction, &stored).await?; + let result = transaction + .execute_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "UPDATE memory_episode_search_doc SET + root_kind = ?, root_id = ?, completion_status = ?, + code_change_status = ?, ended_at = ?, goal = ?, summary = ?, + decisions = ?, failed_attempts = ?, unresolved = ? + WHERE rowid = ? AND note_id = ? AND revision_oid = ?", + [ + document.root_kind().into(), + document.root.id().into(), + document.completion_status().into(), + document.code_change_status().into(), + document.ended_at().into(), + document.text.goal.as_str().into(), + document.text.summary.as_str().into(), + document.text.decisions.as_str().into(), + document.text.failed_attempts.as_str().into(), + document.text.unresolved.as_str().into(), + stored.rowid.into(), + note_id.into(), + revision_oid.into(), + ], + )) + .await?; + if result.rows_affected() != 1 { + return Err(MemoryFtsError::CorruptProjection); + } + insert_posting(transaction, stored.rowid, &document.text).await?; + return Ok(stored.rowid); + } + + let result = transaction + .execute_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "INSERT INTO memory_episode_search_doc ( + note_id, revision_oid, root_kind, root_id, completion_status, + code_change_status, ended_at, goal, summary, decisions, + failed_attempts, unresolved + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + [ + note_id.into(), + revision_oid.into(), + document.root_kind().into(), + document.root.id().into(), + document.completion_status().into(), + document.code_change_status().into(), + document.ended_at().into(), + document.text.goal.as_str().into(), + document.text.summary.as_str().into(), + document.text.decisions.as_str().into(), + document.text.failed_attempts.as_str().into(), + document.text.unresolved.as_str().into(), + ], + )) + .await?; + let rowid = + i64::try_from(result.last_insert_id()).map_err(|_| MemoryFtsError::CorruptProjection)?; + insert_posting(transaction, rowid, &document.text).await?; + Ok(rowid) +} + +pub(crate) async fn delete_document( + transaction: &MemoryWriteTransaction, + note_id: uuid::Uuid, + revision_oid: ObjectHash, +) -> Result { + let transaction = transaction.as_database_transaction(); + let note_id = note_id.to_string(); + let revision_oid = revision_oid.to_string(); + let Some(stored) = read_document(transaction, ¬e_id, &revision_oid).await? else { + return Ok(false); + }; + delete_posting(transaction, &stored).await?; + let result = transaction + .execute_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "DELETE FROM memory_episode_search_doc + WHERE rowid = ? AND note_id = ? AND revision_oid = ?", + [stored.rowid.into(), note_id.into(), revision_oid.into()], + )) + .await?; + if result.rows_affected() != 1 { + return Err(MemoryFtsError::CorruptProjection); + } + Ok(true) +} + +pub(crate) async fn rebuild_index( + transaction: &MemoryWriteTransaction, +) -> Result<(), MemoryFtsError> { + let transaction = transaction.as_database_transaction(); + transaction + .execute_unprepared("INSERT INTO memory_episode_fts(memory_episode_fts) VALUES('rebuild')") + .await?; + verify_index(transaction).await +} + +async fn verify_index(transaction: &DatabaseTransaction) -> Result<(), MemoryFtsError> { + transaction + .execute_unprepared( + "INSERT INTO memory_episode_fts(memory_episode_fts, rank) + VALUES('integrity-check', 1)", + ) + .await?; + Ok(()) +} + +async fn read_document( + transaction: &DatabaseTransaction, + note_id: &str, + revision_oid: &str, +) -> Result, MemoryFtsError> { + let row = transaction + .query_one_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "SELECT rowid, goal, summary, decisions, failed_attempts, unresolved + FROM memory_episode_search_doc + WHERE note_id = ? AND revision_oid = ?", + [note_id.into(), revision_oid.into()], + )) + .await?; + row.map(|row| { + Ok(StoredSearchDocument { + rowid: row + .try_get("", "rowid") + .map_err(|_| MemoryFtsError::CorruptProjection)?, + goal: row + .try_get("", "goal") + .map_err(|_| MemoryFtsError::CorruptProjection)?, + summary: row + .try_get("", "summary") + .map_err(|_| MemoryFtsError::CorruptProjection)?, + decisions: row + .try_get("", "decisions") + .map_err(|_| MemoryFtsError::CorruptProjection)?, + failed_attempts: row + .try_get("", "failed_attempts") + .map_err(|_| MemoryFtsError::CorruptProjection)?, + unresolved: row + .try_get("", "unresolved") + .map_err(|_| MemoryFtsError::CorruptProjection)?, + }) + }) + .transpose() +} + +async fn insert_posting( + transaction: &DatabaseTransaction, + rowid: i64, + text: &EpisodeSearchText, +) -> Result<(), MemoryFtsError> { + transaction + .execute_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "INSERT INTO memory_episode_fts ( + rowid, goal, summary, decisions, failed_attempts, unresolved + ) VALUES (?, ?, ?, ?, ?, ?)", + [ + rowid.into(), + text.goal.as_str().into(), + text.summary.as_str().into(), + text.decisions.as_str().into(), + text.failed_attempts.as_str().into(), + text.unresolved.as_str().into(), + ], + )) + .await?; + Ok(()) +} + +async fn delete_posting( + transaction: &DatabaseTransaction, + stored: &StoredSearchDocument, +) -> Result<(), MemoryFtsError> { + transaction + .execute_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "INSERT INTO memory_episode_fts ( + memory_episode_fts, rowid, goal, summary, decisions, + failed_attempts, unresolved + ) VALUES ('delete', ?, ?, ?, ?, ?, ?)", + [ + stored.rowid.into(), + stored.goal.as_str().into(), + stored.summary.as_str().into(), + stored.decisions.as_str().into(), + stored.failed_attempts.as_str().into(), + stored.unresolved.as_str().into(), + ], + )) + .await?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use git_internal::internal::object::types::ObjectType; + use sea_orm::{Database, DatabaseConnection}; + + use super::*; + use crate::internal::db::migration::run_builtin_migrations; + + async fn test_database() -> DatabaseConnection { + let database = Database::connect("sqlite::memory:") + .await + .expect("connect test database"); + run_builtin_migrations(&database) + .await + .expect("apply built-in migrations"); + database + } + + fn revision_oid(seed: &[u8]) -> ObjectHash { + ObjectHash::from_type_and_data(ObjectType::Blob, seed) + } + + async fn seed_revision( + database: &DatabaseConnection, + root: &EpisodeRoot, + revision_oid: ObjectHash, + ) { + database + .execute_raw(Statement::from_sql_and_values( + database.get_database_backend(), + "INSERT INTO memory_note_index ( + note_id, scope_key, namespace, path, kind, lifecycle, + review_state, confidence, trust, sensitivity, visibility, + acl_policy_id, origin, idempotency_key, created_at + ) VALUES (?, 'repo', 'default', ?, 'episodic', 'accretive', + 'confirmed', 'high', 'repo_evidence', 'internal', + 'repo_local', 'default', 'episode_compiler', ?, + '2026-08-24T00:00:00Z')", + [ + root.note_id().to_string().into(), + root.path().into(), + format!("episode:{}", root.id()).into(), + ], + )) + .await + .expect("seed Memory note"); + database + .execute_raw(Statement::from_sql_and_values( + database.get_database_backend(), + "INSERT INTO memory_revision_index ( + revision_oid, note_id, scope_key, namespace, origin, + producer, rules_version, policy_version, + input_fingerprints_json, created_at + ) VALUES (?, ?, 'repo', 'default', 'episode_compiler', + 'test', 1, 'v1', '[]', '2026-08-24T00:00:00Z')", + [ + revision_oid.to_string().into(), + root.note_id().to_string().into(), + ], + )) + .await + .expect("seed Memory revision"); + } + + fn document( + root: EpisodeRoot, + revision_oid: ObjectHash, + goal: &str, + ended_at: Option>, + ) -> EpisodeSearchDocument { + EpisodeSearchDocument::new( + root, + revision_oid, + CompletionStatus::Completed, + CodeChangeStatus::Changed, + ended_at, + EpisodeSearchText::new(goal, "summary", "decision", "failure", "unresolved") + .expect("valid search text"), + ) + } + + async fn match_count(database: &DatabaseConnection, query: &str) -> i64 { + database + .query_one_raw(Statement::from_sql_and_values( + database.get_database_backend(), + "SELECT COUNT(*) AS count FROM memory_episode_fts + WHERE memory_episode_fts MATCH ?", + [query.into()], + )) + .await + .expect("query FTS") + .expect("count row") + .try_get("", "count") + .expect("count value") + } + + async fn assert_integrity(database: &DatabaseConnection) { + database + .execute_unprepared( + "INSERT INTO memory_episode_fts(memory_episode_fts, rank) + VALUES('integrity-check', 1)", + ) + .await + .expect("external-content index remains consistent"); + } + + #[test] + fn plain_text_normalizer_quotes_operators_and_rejects_unbounded_input() { + let query = normalize_plain_text_v1("Alpha alpha ALPHA OR title:beta") + .expect("ordinary text produces a bound query"); + assert_eq!(query.version(), 1); + assert_eq!( + query.as_str(), + "\"Alpha\" OR \"OR\" OR \"title\" OR \"beta\"" + ); + + assert!(normalize_plain_text_v1("").is_err()); + assert!(normalize_plain_text_v1("--- !!!").is_err()); + assert!(normalize_plain_text_v1("alpha\nbeta").is_err()); + assert!(normalize_plain_text_v1(&"x".repeat(MAX_MATCH_TERM_BYTES + 1)).is_err()); + let too_many_terms = (0..=MAX_MATCH_TERMS) + .map(|index| format!("term{index}")) + .collect::>() + .join(" "); + assert!(normalize_plain_text_v1(&too_many_terms).is_err()); + } + + #[tokio::test] + async fn natural_language_query_matches_when_only_some_terms_are_present() { + let database = test_database().await; + let root = EpisodeRoot::task("task-fts-natural-language").expect("valid root"); + let oid = revision_oid(b"fts-natural-language"); + seed_revision(&database, &root, oid).await; + + let transaction = MemoryWriteTransaction::begin(&database) + .await + .expect("begin insert"); + upsert_document( + &transaction, + &document(root, oid, "cache invalidation", None), + ) + .await + .expect("insert search document"); + transaction.commit().await.expect("commit insert"); + + let query = normalize_plain_text_v1("why did the agent change authentication/cache?") + .expect("normalize natural-language query"); + assert_eq!(match_count(&database, query.as_str()).await, 1); + } + + #[tokio::test] + async fn external_content_insert_update_delete_preserves_rowid_and_integrity() { + let database = test_database().await; + let root = EpisodeRoot::task("task-fts-lifecycle").expect("valid root"); + let oid = revision_oid(b"fts-lifecycle"); + seed_revision(&database, &root, oid).await; + + let transaction = MemoryWriteTransaction::begin(&database) + .await + .expect("begin insert"); + let rowid = upsert_document(&transaction, &document(root.clone(), oid, "oldtoken", None)) + .await + .expect("insert search document"); + transaction.commit().await.expect("commit insert"); + assert_eq!(match_count(&database, "oldtoken").await, 1); + + let ended_at = "2026-08-24T01:02:03Z" + .parse::>() + .expect("timestamp"); + let transaction = MemoryWriteTransaction::begin(&database) + .await + .expect("begin update"); + let updated_rowid = upsert_document( + &transaction, + &document(root.clone(), oid, "newtoken", Some(ended_at)), + ) + .await + .expect("update search document"); + transaction.commit().await.expect("commit update"); + assert_eq!(updated_rowid, rowid, "an update must preserve FTS rowid"); + assert_eq!(match_count(&database, "oldtoken").await, 0); + assert_eq!(match_count(&database, "newtoken").await, 1); + + let stored_ended_at: Option = database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + "SELECT ended_at FROM memory_episode_search_doc".to_string(), + )) + .await + .expect("read ended_at") + .expect("search row") + .try_get("", "ended_at") + .expect("nullable ended_at"); + assert!(stored_ended_at.is_some()); + assert_integrity(&database).await; + + let parent_delete = database + .execute_raw(Statement::from_sql_and_values( + database.get_database_backend(), + "DELETE FROM memory_revision_index WHERE revision_oid = ?", + [oid.to_string().into()], + )) + .await; + assert!( + parent_delete.is_err(), + "the content FK must restrict a parent delete that would orphan FTS postings" + ); + + let transaction = MemoryWriteTransaction::begin(&database) + .await + .expect("begin delete"); + assert!( + delete_document(&transaction, root.note_id(), oid) + .await + .expect("delete search document") + ); + transaction.commit().await.expect("commit delete"); + assert_eq!(match_count(&database, "newtoken").await, 0); + assert_integrity(&database).await; + } + + #[tokio::test] + async fn failed_content_update_rolls_back_the_prior_posting_delete() { + let database = test_database().await; + let root = EpisodeRoot::task("task-fts-rollback").expect("valid root"); + let oid = revision_oid(b"fts-rollback"); + seed_revision(&database, &root, oid).await; + + let transaction = MemoryWriteTransaction::begin(&database) + .await + .expect("begin insert"); + upsert_document( + &transaction, + &document(root.clone(), oid, "stabletoken", None), + ) + .await + .expect("insert search document"); + transaction.commit().await.expect("commit insert"); + + database + .execute_unprepared( + "CREATE TRIGGER memory_fts_test_abort + BEFORE UPDATE ON memory_episode_search_doc + BEGIN SELECT RAISE(ABORT, 'forced test failure'); END", + ) + .await + .expect("install deterministic failure"); + let transaction = MemoryWriteTransaction::begin(&database) + .await + .expect("begin failing update"); + let error = upsert_document(&transaction, &document(root, oid, "losttoken", None)) + .await + .expect_err("content update must fail after deleting the old posting"); + assert!(matches!(error, MemoryFtsError::Storage(_))); + transaction + .rollback() + .await + .expect("roll back failed update"); + + assert_eq!(match_count(&database, "stabletoken").await, 1); + assert_eq!(match_count(&database, "losttoken").await, 0); + assert_integrity(&database).await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 3)] + async fn write_locked_transaction_waits_for_a_competing_writer() { + let directory = tempfile::tempdir().expect("temporary repository database directory"); + let path = directory.path().join("memory-fts-write-lock.db"); + let path = path.to_str().expect("UTF-8 test database path"); + let holder = db::create_database(path) + .await + .expect("create repository database"); + let waiter = db::establish_connection(path) + .await + .expect("open an independent connection"); + let root = EpisodeRoot::task("task-fts-write-lock").expect("valid root"); + let oid = revision_oid(b"fts-write-lock"); + seed_revision(&holder, &root, oid).await; + + let held = db::begin_write_transaction(&holder) + .await + .expect("holder acquires SQLite write lock"); + let release = tokio::spawn(async move { + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + held.commit().await.expect("release competing writer"); + }); + + let started = std::time::Instant::now(); + let transaction = MemoryWriteTransaction::begin(&waiter) + .await + .expect("Memory writer waits for the lock instead of upgrading after a read"); + upsert_document(&transaction, &document(root, oid, "contendedtoken", None)) + .await + .expect("write after acquiring the lock"); + transaction.commit().await.expect("commit Memory write"); + release.await.expect("join competing writer"); + + assert!( + started.elapsed() >= std::time::Duration::from_millis(250), + "the Memory transaction must wait for the holder before reading" + ); + assert_eq!(match_count(&waiter, "contendedtoken").await, 1); + assert_integrity(&waiter).await; + } + + #[tokio::test] + async fn rebuild_restores_external_content_postings() { + let database = test_database().await; + let root = EpisodeRoot::intent("intent-fts-rebuild").expect("valid root"); + let oid = revision_oid(b"fts-rebuild"); + seed_revision(&database, &root, oid).await; + + let transaction = MemoryWriteTransaction::begin(&database) + .await + .expect("begin insert"); + upsert_document(&transaction, &document(root, oid, "rebuildtoken", None)) + .await + .expect("insert search document"); + transaction.commit().await.expect("commit insert"); + + database + .execute_unprepared( + "INSERT INTO memory_episode_fts(memory_episode_fts) VALUES('delete-all')", + ) + .await + .expect("erase postings without changing content"); + assert_eq!(match_count(&database, "rebuildtoken").await, 0); + + let transaction = MemoryWriteTransaction::begin(&database) + .await + .expect("begin rebuild"); + rebuild_index(&transaction).await.expect("rebuild FTS"); + transaction.commit().await.expect("commit rebuild"); + assert_eq!(match_count(&database, "rebuildtoken").await, 1); + assert_integrity(&database).await; + } +} diff --git a/src/internal/ai/memory/mod.rs b/src/internal/ai/memory/mod.rs index a051012cc..3eea4ef6e 100644 --- a/src/internal/ai/memory/mod.rs +++ b/src/internal/ai/memory/mod.rs @@ -11,5 +11,6 @@ mod canonical; mod domain; +mod fts_sql; mod job_sql; mod validation; diff --git a/src/internal/db.rs b/src/internal/db.rs index 57fe3c887..fde6680e4 100644 --- a/src/internal/db.rs +++ b/src/internal/db.rs @@ -888,6 +888,17 @@ mod tests { ); } + #[test] + fn memory_fts_old_reader_rejects_migrated_schema() { + assert_eq!( + classify_schema_compatibility(Some(2026082402), Some(2026082401)), + SchemaCompatibility::UnsupportedFuture { + current_version: 2026082402, + latest_version: Some(2026082401), + } + ); + } + /// TestDbPath is a helper struct create and delete test database file struct TestDbPath(String); impl Drop for TestDbPath { diff --git a/src/internal/db/migration.rs b/src/internal/db/migration.rs index 3d7490ce7..f513d0629 100644 --- a/src/internal/db/migration.rs +++ b/src/internal/db/migration.rs @@ -1389,6 +1389,15 @@ pub fn builtin_migrations() -> Vec { include_str!("../../../sql/migrations/2026082401_memory_core.sql"), include_str!("../../../sql/migrations/2026082401_memory_core_down.sql"), ), + // M2-02F: a single-copy Episode search document and its + // external-content FTS5 postings. Runtime synchronization is owned by + // internal::ai::memory::fts_sql; no triggers or fallback scan exist. + sql_migration( + 2026082402, + "memory_fts_search", + include_str!("../../../sql/migrations/2026082402_memory_fts_search.sql"), + include_str!("../../../sql/migrations/2026082402_memory_fts_search_down.sql"), + ), ] } @@ -1836,9 +1845,9 @@ mod tests { // `builtin_migrations()` so silent registry regressions surface // here in addition to `tests/db_migration_test.rs`. let runner = builtin_runner().expect("CEX-12.5 builtin registry must build clean"); - assert_eq!(runner.len(), 56); + assert_eq!(runner.len(), 57); assert!(!runner.is_empty()); - assert_eq!(runner.max_registered_version(), Some(2026082401)); + assert_eq!(runner.max_registered_version(), Some(2026082402)); } #[test] diff --git a/src/internal/model/memory_episode_search_doc.rs b/src/internal/model/memory_episode_search_doc.rs new file mode 100644 index 000000000..34b217ba8 --- /dev/null +++ b/src/internal/model/memory_episode_search_doc.rs @@ -0,0 +1,27 @@ +//! SeaORM entity for the rebuildable Episode external-content search source. + +use sea_orm::entity::prelude::*; + +#[derive(Clone, Debug, Eq, PartialEq, DeriveEntityModel)] +#[sea_orm(table_name = "memory_episode_search_doc")] +pub struct Model { + #[sea_orm(primary_key, auto_increment = true, column_name = "rowid")] + pub rowid: i64, + pub note_id: String, + pub revision_oid: String, + pub root_kind: String, + pub root_id: String, + pub completion_status: String, + pub code_change_status: String, + pub ended_at: Option, + pub goal: String, + pub summary: String, + pub decisions: String, + pub failed_attempts: String, + pub unresolved: String, +} + +#[derive(Copy, Clone, Debug, EnumIter, DeriveRelation)] +pub enum Relation {} + +impl ActiveModelBehavior for ActiveModel {} diff --git a/src/internal/model/memory_model_tests.rs b/src/internal/model/memory_model_tests.rs index 05e2b5120..4cbaf4ff1 100644 --- a/src/internal/model/memory_model_tests.rs +++ b/src/internal/model/memory_model_tests.rs @@ -1,8 +1,8 @@ use sea_orm::{Iterable, PrimaryKeyTrait}; use super::{ - memory_episode_path, memory_head, memory_link_index, memory_note_index, memory_path_summary, - memory_projection_state, memory_revision_index, + memory_episode_path, memory_episode_search_doc, memory_head, memory_link_index, + memory_note_index, memory_path_summary, memory_projection_state, memory_revision_index, }; fn key_names

() -> Vec @@ -39,6 +39,10 @@ fn memory_entities_expose_expected_primary_keys() { key_names::(), ["NoteId", "RevisionOid", "CodePath"] ); + assert_eq!( + key_names::(), + ["Rowid"] + ); assert!(!memory_head::PrimaryKey::auto_increment()); assert!(!memory_path_summary::PrimaryKey::auto_increment()); @@ -47,4 +51,5 @@ fn memory_entities_expose_expected_primary_keys() { assert!(!memory_link_index::PrimaryKey::auto_increment()); assert!(!memory_projection_state::PrimaryKey::auto_increment()); assert!(!memory_episode_path::PrimaryKey::auto_increment()); + assert!(memory_episode_search_doc::PrimaryKey::auto_increment()); } diff --git a/src/internal/model/mod.rs b/src/internal/model/mod.rs index 022df80d6..b3887144e 100644 --- a/src/internal/model/mod.rs +++ b/src/internal/model/mod.rs @@ -24,6 +24,7 @@ pub mod config_kv; pub mod layer; pub mod layer_path; pub mod memory_episode_path; +pub mod memory_episode_search_doc; pub mod memory_head; pub mod memory_link_index; pub mod memory_note_index; diff --git a/src/internal/mutable_state_ownership.rs b/src/internal/mutable_state_ownership.rs index 31ddf6f06..301c4fe4e 100644 --- a/src/internal/mutable_state_ownership.rs +++ b/src/internal/mutable_state_ownership.rs @@ -321,6 +321,82 @@ pub const MUTABLE_STATE_OWNERSHIP: &[MutableStateSurface] = &[ owner: StateOwner::Repository, rationale: "approvals and append-only logs (repo-wide by design, §C.4.1.1)", }, + // ── Memory projections and local compiler state (M2) ──────────────── + MutableStateSurface { + table: "memory_compile_job", + owner: StateOwner::Repository, + rationale: "bounded per-root Memory compiler jobs live in the shared repository database", + }, + MutableStateSurface { + table: "memory_compile_observer_state", + owner: StateOwner::Repository, + rationale: "Memory source-ref scan watermarks are shared repository runtime state", + }, + MutableStateSurface { + table: "memory_episode_path", + owner: StateOwner::Repository, + rationale: "rebuildable Episode-to-code-path projection; logical scope_key is not database ownership", + }, + MutableStateSurface { + table: "memory_head", + owner: StateOwner::Repository, + rationale: "rebuildable Memory head projection shared by repository worktrees", + }, + MutableStateSurface { + table: "memory_link_index", + owner: StateOwner::Repository, + rationale: "rebuildable Memory relationship projection shared by repository worktrees", + }, + MutableStateSurface { + table: "memory_note_index", + owner: StateOwner::Repository, + rationale: "rebuildable Memory note projection; semantic scope is stored in scope_key", + }, + MutableStateSurface { + table: "memory_path_summary", + owner: StateOwner::Repository, + rationale: "rebuildable Memory taxonomy summary shared by repository worktrees", + }, + MutableStateSurface { + table: "memory_projection_state", + owner: StateOwner::Repository, + rationale: "repository Memory projection watermark keyed by logical scope_key", + }, + MutableStateSurface { + table: "memory_revision_index", + owner: StateOwner::Repository, + rationale: "rebuildable Memory revision projection shared by repository worktrees", + }, + MutableStateSurface { + table: "memory_episode_search_doc", + owner: StateOwner::Repository, + rationale: "rebuildable single-copy Episode search documents shared by repository worktrees", + }, + MutableStateSurface { + table: "memory_episode_fts", + owner: StateOwner::Repository, + rationale: "rebuildable external-content FTS5 virtual table for Episode search", + }, + MutableStateSurface { + table: "memory_episode_fts_config", + owner: StateOwner::Repository, + rationale: "SQLite-owned persistent FTS5 configuration shadow table", + }, + MutableStateSurface { + table: "memory_episode_fts_data", + owner: StateOwner::Repository, + rationale: "SQLite-owned persistent FTS5 segment-data shadow table", + }, + MutableStateSurface { + table: "memory_episode_fts_docsize", + owner: StateOwner::Repository, + rationale: "SQLite-owned persistent FTS5 document-size shadow table", + }, + MutableStateSurface { + table: "memory_episode_fts_idx", + owner: StateOwner::Repository, + rationale: "SQLite-owned persistent FTS5 segment-index shadow table", + }, MutableStateSurface { table: "cherry_pick_state", owner: StateOwner::Repository, @@ -460,6 +536,8 @@ pub const MIGRATION_ONLY_TABLES: &[&str] = &[ "head_scope_unique_guard", "layer__down_guard_2026072303", "layer__legacy_rows_need_explicit_adopt_2026072303", + "memory_core_down_guard", + "memory_fts_search_down_guard", "operation__down_guard_2026073003", "operation__down_guard_2026073004", "operation_scope_provenance_down_guard", @@ -622,7 +700,12 @@ mod tests { created } - /// Every `CREATE TABLE` name in a lowercased SQL corpus. + /// Every persistent table name in a lowercased SQL corpus. + /// + /// FTS5 virtual tables materialize four persistent SQLite-owned shadow + /// tables for this schema shape. They are mutable repository state too, + /// so derive and classify them explicitly instead of hiding them behind + /// a name-pattern exemption. fn tables_in(sql: &str) -> BTreeSet { let mut created = BTreeSet::new(); for chunk in sql.split("create table").skip(1) { @@ -631,9 +714,43 @@ mod tests { created.insert(name); } } + for chunk in sql.split("create virtual table").skip(1) { + let name = table_name_after(chunk); + if name.is_empty() { + continue; + } + created.insert(name.clone()); + let statement = chunk.split(';').next().unwrap_or(""); + if statement.contains("using fts5") { + for suffix in ["config", "data", "docsize", "idx"] { + created.insert(format!("{name}_{suffix}")); + } + } + } created } + #[test] + fn fts5_virtual_table_scan_includes_persistent_shadow_tables() { + let tables = tables_in( + "create virtual table if not exists memory_episode_fts using fts5(\ + goal, content='memory_episode_search_doc', content_rowid='rowid');", + ); + assert_eq!( + tables, + [ + "memory_episode_fts", + "memory_episode_fts_config", + "memory_episode_fts_data", + "memory_episode_fts_docsize", + "memory_episode_fts_idx", + ] + .into_iter() + .map(str::to_owned) + .collect() + ); + } + /// PRODUCTION Rust DDL across the whole `src/` tree, minus the explicit /// non-repository sources. `#[cfg(test)]` items are dropped by the AST /// walk above; whole test-only FILES carry no marker of their own (their diff --git a/tests/INDEX.md b/tests/INDEX.md index a75f357c7..e466d341c 100644 --- a/tests/INDEX.md +++ b/tests/INDEX.md @@ -106,6 +106,7 @@ | `compat_help_no_impl_meta_leak` | 1 | No `libra --help` body leaks contributor-facing rustdoc into clap's long_about; forbids 6 phrase classes (e.g. `Codex pass-`, raw markdown headings, code fences) | `src/cli.rs`, `src/command/**` | | `verify_pack_multi_test` | 1 | Guards `verify-pack ...` multi-index verification, JSON wrapping, and `--pack` argument rejection | `src/command/verify_pack*.rs` | | `db_migration_test` | 1 | SQLite schema bootstrap + migration round-trip, including M4 import identity/tombstone and M5 subagent content empty up→down→up, non-empty recovery-state rollback refusal, the §C.8 workspace-lease down guard (non-terminal rows refuse, settled rows roll back, live leases block deeper W3 rollbacks transitively), and uniqueness/state failure matrix | `src/internal/db.rs`, `sql/` | +| `fts5_capability_test` | 1 | M2-02F release-linked SQLite capability proof: external-content FTS5 create/insert, parameter-bound MATCH, fixed weighted BM25 ASC semantics, and unicode61 diacritic behavior | `src/internal/ai/memory/fts_sql.rs`, `sql/migrations/2026082402_memory_fts_search.sql` | | `workspace_lease_test` | 1 | plan-20260714 §C.8 W4 workspace association/lease store: DB-arbitrated single winner per linked `(repo_id, worktree_id)` including a proven two-connection overlap (contender with a 50ms busy timeout must hit the uncommitted writer's lock, then lose to the index on retry) and failpoint-windowed takeovers (a parked doctor reports only the fence it wrote), canonical-path alias refusal (`.`/`..`, trailing separator, symlink, symlinked-parent `..` traversal, dangling-symlink parent) across kinds, repository identity resolved from `libra.repoid` (never the caller; padded/empty values refused as corrupt, write and read paths pinned to the same value, a rewrite with live rows fails closed) and cross-repository isolation, owner+monotonic-fence conditional renew/release/activate/abandon with stale-owner refusal (`LBR-AGENT-023`), no implicit steal of an expired lease, doctor reclaim only after expiry (`LBR-AGENT-022`) with concurrent reclaims handing out only the fence they wrote, provisioning→active publication (an `active` record requires an existing directory), released/orphaned identity release, foreign-identity recovery (bounded keyset listing plus adopt, with identity-drift refusals), bounded expired-lease sweep, keyset listing, and the association-IDs-only column pin | `src/internal/workspace.rs`, `sql/migrations/2026072501_workspace_record.sql` | ## Wave 2 — Code UI & local automation diff --git a/tests/agent_capture_migration_test.rs b/tests/agent_capture_migration_test.rs index 44dc5842c..d202ac988 100644 --- a/tests/agent_capture_migration_test.rs +++ b/tests/agent_capture_migration_test.rs @@ -205,7 +205,9 @@ async fn capture_workspace_scope_migration_preserves_legacy_unknown_and_fences_d // Rollback returns every rolled-back version, newest first; the two // agent-usage migrations (2026080402/2026080403) sit on top of the W4 // scope migration and ride along without touching agent_session. - vec![2026082401, 2026081301, 2026080403, 2026080402, 2026080401] + vec![ + 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401 + ] ); // This focused migration fixture intentionally does not install the @@ -237,7 +239,9 @@ async fn capture_workspace_scope_migration_preserves_legacy_unknown_and_fences_d .expect("upgrade legacy capture row to W4 scope schema"), // Ascending application order: W4 scope first, then the two // agent-usage migrations re-applied on top of it. - vec![2026080401, 2026080402, 2026080403, 2026081301, 2026082401] + vec![ + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 + ] ); let row = conn .query_one_raw(Statement::from_string( diff --git a/tests/command/worktree_isolation_test.rs b/tests/command/worktree_isolation_test.rs index 19940d950..c2a567687 100644 --- a/tests/command/worktree_isolation_test.rs +++ b/tests/command/worktree_isolation_test.rs @@ -5896,9 +5896,9 @@ async fn worktree_commands_apply_capability_marker_before_registry_io() { assert_eq!( rolled, vec![ - 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, - 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, - 2026072501, 2026072403, 2026072402, 2026072401 + 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, + 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, + 2026072502, 2026072501, 2026072403, 2026072402, 2026072401 ] ); conn.close().await.expect("close"); @@ -8556,7 +8556,7 @@ async fn worktree_doctor_does_not_upgrade_a_behind_schema_repository() { let db_url = format!("sqlite://{}?mode=rwc", db.display()); // Use the real down migration rather than deleting its ledger row. The - // Memory core migration creates physical tables, so undoing it represents + // Memory FTS migration creates physical tables, so undoing it represents // a repository that is genuinely one migration behind. let conn = Database::connect(&db_url) .await @@ -8564,15 +8564,15 @@ async fn worktree_doctor_does_not_upgrade_a_behind_schema_repository() { assert_eq!( builtin_runner() .expect("builtin runner") - .rollback_to(&conn, 2026081301) + .rollback_to(&conn, 2026082401) .await .expect("roll back newest migration"), - vec![2026082401] + vec![2026082402] ); conn.close().await.expect("close repository db"); assert!( - sqlite_max_schema_version(&db) < 2026082401, - "2026082401 must be the NEWEST migration for this test to leave one \ + sqlite_max_schema_version(&db) < 2026082402, + "2026082402 must be the NEWEST migration for this test to leave one \ pending — retarget it at the new newest migration" ); let before = std::fs::read(&db).expect("db before"); diff --git a/tests/db_migration_test.rs b/tests/db_migration_test.rs index bb0655321..08697b468 100644 --- a/tests/db_migration_test.rs +++ b/tests/db_migration_test.rs @@ -54,7 +54,8 @@ fn builtin_migrations_register_current_schema_migrations() { 2026071405, 2026071406, 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, - 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 + 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, + 2026082402 ] ); assert_eq!( @@ -116,13 +117,14 @@ fn builtin_migrations_register_current_schema_migrations() { "agent_usage_event_session_scope", "approved_permission_provenance", "memory_core", + "memory_fts_search", ] ); let runner = builtin_runner().expect("builtin registry must build clean"); assert!(!runner.is_empty()); - assert_eq!(runner.len(), 56); - assert_eq!(runner.max_registered_version(), Some(2026082401)); + assert_eq!(runner.len(), 57); + assert_eq!(runner.max_registered_version(), Some(2026082402)); } const MEMORY_CORE_TABLES: [&str; 9] = [ @@ -137,6 +139,21 @@ const MEMORY_CORE_TABLES: [&str; 9] = [ "memory_revision_index", ]; +const MEMORY_CORE_INDEXES: [&str; 12] = [ + "idx_memory_compile_job_runnable", + "idx_memory_compile_job_scope_generation", + "idx_memory_episode_path_code", + "idx_memory_head_lookup", + "idx_memory_head_path_prefix", + "idx_memory_link_source", + "idx_memory_link_target", + "idx_memory_note_idempotency_cell", + "idx_memory_note_idempotency_ns", + "idx_memory_path_summary_prefix", + "idx_memory_revision_note", + "idx_memory_revision_producer", +]; + async fn memory_core_schema_snapshot(conn: &DatabaseConnection) -> Vec<(String, String, String)> { conn.query_all_raw(Statement::from_string( conn.get_database_backend(), @@ -149,12 +166,21 @@ async fn memory_core_schema_snapshot(conn: &DatabaseConnection) -> Vec<(String, .await .expect("read Memory schema") .into_iter() - .map(|row| { - ( - row.try_get("", "type").expect("schema object type"), - row.try_get("", "name").expect("schema object name"), - row.try_get("", "sql").expect("schema object DDL"), - ) + .filter_map(|row| { + let kind: String = row.try_get("", "type").expect("schema object type"); + let name: String = row.try_get("", "name").expect("schema object name"); + let belongs_to_core = match kind.as_str() { + "table" => MEMORY_CORE_TABLES.contains(&name.as_str()), + "index" => MEMORY_CORE_INDEXES.contains(&name.as_str()), + _ => false, + }; + belongs_to_core.then(|| { + ( + kind, + name, + row.try_get("", "sql").expect("schema object DDL"), + ) + }) }) .collect() } @@ -181,21 +207,7 @@ async fn memory_episode_schema() { .map(|(_, name, _)| name.as_str()) .collect(); assert_eq!( - indexes, - vec![ - "idx_memory_compile_job_runnable", - "idx_memory_compile_job_scope_generation", - "idx_memory_episode_path_code", - "idx_memory_head_lookup", - "idx_memory_head_path_prefix", - "idx_memory_link_source", - "idx_memory_link_target", - "idx_memory_note_idempotency_cell", - "idx_memory_note_idempotency_ns", - "idx_memory_path_summary_prefix", - "idx_memory_revision_note", - "idx_memory_revision_producer", - ], + indexes, MEMORY_CORE_INDEXES, "the Memory migration must expose the complete reviewed hot-query index set" ); @@ -370,26 +382,37 @@ async fn memory_episode_schema() { async fn memory_episode_schema_idempotent() { let (_fresh_dir, fresh_url, _fresh_path) = fresh_db_url(); let fresh = connect(&fresh_url).await; - let fresh_applied = run_builtin_migrations(&fresh) - .await - .expect("apply full registry to fresh database"); + let migrations = builtin_migrations(); + let (memory_fts_search, through_memory_core) = migrations + .split_last() + .expect("built-in registry contains Memory FTS migration"); + assert_eq!(memory_fts_search.version, 2026082402); + let memory_core = through_memory_core + .last() + .expect("built-in registry contains Memory core migration"); + assert_eq!(memory_core.version, 2026082401); + let mut through_core_runner = MigrationRunner::new(); + through_core_runner + .extend(through_memory_core.iter().cloned()) + .expect("register through Memory core"); + let fresh_applied = through_core_runner + .run_pending(&fresh) + .await + .expect("apply through Memory core to fresh database"); assert_eq!(fresh_applied.last(), Some(&2026082401)); assert!( - run_builtin_migrations(&fresh) + through_core_runner + .run_pending(&fresh) .await - .expect("repeat full registry") + .expect("repeat through-core registry") .is_empty(), "second migration run must be a no-op" ); let fresh_snapshot = memory_core_schema_snapshot(&fresh).await; - let repeated_up = builtin_migrations() - .into_iter() - .last() - .expect("built-in registry contains Memory core migration"); fresh .execute_raw(Statement::from_string( fresh.get_database_backend(), - repeated_up.up, + memory_core.up, )) .await .expect("the Memory core SQL body itself must be idempotent"); @@ -401,8 +424,7 @@ async fn memory_episode_schema_idempotent() { let (_upgrade_dir, upgrade_url, _upgrade_path) = fresh_db_url(); let upgrade = connect(&upgrade_url).await; - let migrations = builtin_migrations(); - let (memory_core, previous) = migrations + let (memory_core, previous) = through_memory_core .split_last() .expect("built-in registry contains Memory core migration"); assert_eq!(memory_core.version, 2026082401); @@ -422,7 +444,7 @@ async fn memory_episode_schema_idempotent() { run_builtin_migrations(&upgrade) .await .expect("upgrade old database"), - vec![2026082401] + vec![2026082401, 2026082402] ); assert_eq!(memory_core_schema_snapshot(&upgrade).await, fresh_snapshot); } @@ -571,6 +593,341 @@ async fn memory_episode_down_guard() { } } +async fn seed_memory_search_fixture(conn: &DatabaseConnection) { + conn.execute_unprepared( + "INSERT INTO memory_note_index ( + note_id, scope_key, namespace, path, kind, lifecycle, review_state, + confidence, trust, sensitivity, visibility, acl_policy_id, origin, + idempotency_key, created_at + ) VALUES ( + '523e4567-e89b-42d3-a456-426614174000', 'repo', 'default', + 'episodic.tasks.fts', 'episodic', 'accretive', 'confirmed', + 'high', 'repo_evidence', 'internal', 'repo_local', 'default', + 'episode_compiler', 'fts-fixture', '2026-08-24T00:00:00Z' + ); + INSERT INTO memory_revision_index ( + revision_oid, note_id, scope_key, namespace, origin, producer, + rules_version, policy_version, input_fingerprints_json, created_at + ) VALUES ( + 'cccccccccccccccccccccccccccccccccccccccc', + '523e4567-e89b-42d3-a456-426614174000', 'repo', 'default', + 'episode_compiler', 'test', 1, 'v1', '[]', + '2026-08-24T00:00:00Z' + ); + INSERT INTO memory_episode_search_doc ( + note_id, revision_oid, root_kind, root_id, completion_status, + code_change_status, ended_at, goal, summary, decisions, + failed_attempts, unresolved + ) VALUES ( + '523e4567-e89b-42d3-a456-426614174000', + 'cccccccccccccccccccccccccccccccccccccccc', + 'task', 'task-fts', 'failed', 'unchanged', NULL, + 'repair café parser', 'summary', 'decision', 'attempt', 'unresolved' + ); + INSERT INTO memory_episode_fts ( + rowid, goal, summary, decisions, failed_attempts, unresolved + ) SELECT + rowid, goal, summary, decisions, failed_attempts, unresolved + FROM memory_episode_search_doc + WHERE note_id = '523e4567-e89b-42d3-a456-426614174000';", + ) + .await + .expect("seed Memory FTS fixture"); +} + +async fn memory_fts_schema_snapshot(conn: &DatabaseConnection) -> Vec<(String, String)> { + conn.query_all_raw(Statement::from_string( + conn.get_database_backend(), + "SELECT name, sql FROM sqlite_master + WHERE name IN ( + 'memory_episode_search_doc', 'memory_episode_fts', + 'idx_memory_episode_search_root', 'idx_memory_episode_search_filters' + ) + ORDER BY name" + .to_string(), + )) + .await + .expect("read Memory FTS schema") + .into_iter() + .map(|row| { + ( + row.try_get("", "name").expect("schema object name"), + row.try_get("", "sql").expect("schema object DDL"), + ) + }) + .collect() +} + +#[tokio::test] +async fn memory_episode_fts_schema() { + let (_dir, url, _path) = fresh_db_url(); + let conn = connect(&url).await; + let applied = run_builtin_migrations(&conn) + .await + .expect("apply Memory FTS migration"); + assert_eq!(applied.last(), Some(&2026082402)); + + let schema = memory_fts_schema_snapshot(&conn).await; + assert_eq!( + schema.len(), + 4, + "two tables and two query indexes are owned" + ); + let fts_ddl = schema + .iter() + .find(|(name, _)| name == "memory_episode_fts") + .map(|(_, ddl)| ddl.as_str()) + .expect("FTS virtual table DDL"); + for required in [ + "goal", + "summary", + "decisions", + "failed_attempts", + "unresolved", + "content='memory_episode_search_doc'", + "content_rowid='rowid'", + "tokenize='unicode61 remove_diacritics 2'", + ] { + assert!( + fts_ddl.contains(required), + "FTS DDL must retain {required}: {fts_ddl}" + ); + } + let column_positions: Vec = [ + "`goal`", + "`summary`", + "`decisions`", + "`failed_attempts`", + "`unresolved`", + ] + .into_iter() + .map(|column| fts_ddl.find(column).expect("FTS column")) + .collect(); + assert!( + column_positions.windows(2).all(|pair| pair[0] < pair[1]), + "FTS column order owns BM25 weights" + ); + + let ended_at = conn + .query_all_raw(Statement::from_string( + conn.get_database_backend(), + "PRAGMA table_info(memory_episode_search_doc)".to_string(), + )) + .await + .expect("inspect search document columns") + .into_iter() + .find(|row| row.try_get::("", "name").as_deref() == Ok("ended_at")) + .expect("ended_at column"); + let ended_at_not_null: i64 = ended_at + .try_get("", "notnull") + .expect("ended_at nullability"); + assert_eq!(ended_at_not_null, 0, "valid Episodes may omit ended_at"); + + let foreign_keys = conn + .query_all_raw(Statement::from_string( + conn.get_database_backend(), + "PRAGMA foreign_key_list(memory_episode_search_doc)".to_string(), + )) + .await + .expect("inspect search document foreign key"); + assert_eq!( + foreign_keys.len(), + 2, + "note/revision FK must stay composite" + ); + for row in &foreign_keys { + let target: String = row.try_get("", "table").expect("FK table"); + let on_delete: String = row.try_get("", "on_delete").expect("FK delete action"); + assert_eq!(target, "memory_revision_index"); + assert_eq!( + on_delete, "RESTRICT", + "parent deletes cannot orphan postings" + ); + } + let trigger_count: i64 = conn + .query_one_raw(Statement::from_string( + conn.get_database_backend(), + "SELECT COUNT(*) AS count FROM sqlite_master + WHERE type = 'trigger' + AND (tbl_name = 'memory_episode_search_doc' + OR tbl_name = 'memory_episode_fts')" + .to_string(), + )) + .await + .expect("inspect FTS triggers") + .expect("trigger count row") + .try_get("", "count") + .expect("trigger count"); + assert_eq!( + trigger_count, 0, + "fts_sql, not triggers, owns synchronization" + ); + + seed_memory_search_fixture(&conn).await; + let diacritic_hits: i64 = conn + .query_one_raw(Statement::from_sql_and_values( + conn.get_database_backend(), + "SELECT COUNT(*) AS count FROM memory_episode_fts + WHERE memory_episode_fts MATCH ?", + ["cafe".into()], + )) + .await + .expect("query migrated FTS table") + .expect("match count row") + .try_get("", "count") + .expect("match count"); + assert_eq!(diacritic_hits, 1); + let parent_delete = conn + .execute_unprepared( + "DELETE FROM memory_revision_index + WHERE revision_oid = 'cccccccccccccccccccccccccccccccccccccccc'", + ) + .await; + assert!( + parent_delete.is_err(), + "search content must restrict parent deletion" + ); +} + +#[tokio::test] +async fn memory_episode_fts_schema_idempotent_upgrade_and_empty_down() { + let (_fresh_dir, fresh_url, _fresh_path) = fresh_db_url(); + let fresh = connect(&fresh_url).await; + let fresh_applied = run_builtin_migrations(&fresh) + .await + .expect("apply full registry"); + assert_eq!(fresh_applied.last(), Some(&2026082402)); + assert!( + run_builtin_migrations(&fresh) + .await + .expect("repeat full registry") + .is_empty() + ); + let snapshot = memory_fts_schema_snapshot(&fresh).await; + let migration = builtin_migrations() + .into_iter() + .last() + .expect("Memory FTS migration"); + assert_eq!(migration.version, 2026082402); + fresh + .execute_raw(Statement::from_string( + fresh.get_database_backend(), + migration.up, + )) + .await + .expect("direct Memory FTS up is idempotent"); + assert_eq!(memory_fts_schema_snapshot(&fresh).await, snapshot); + + let (_upgrade_dir, upgrade_url, _upgrade_path) = fresh_db_url(); + let upgrade = connect(&upgrade_url).await; + let migrations = builtin_migrations(); + let (fts_migration, previous) = migrations.split_last().expect("Memory FTS migration"); + assert_eq!(fts_migration.version, 2026082402); + let mut old_runner = MigrationRunner::new(); + old_runner + .extend(previous.iter().cloned()) + .expect("register prior tip"); + old_runner + .run_pending(&upgrade) + .await + .expect("apply through Memory core"); + assert_eq!( + old_runner.current_version(&upgrade).await.unwrap(), + Some(2026082401) + ); + upgrade + .execute_unprepared( + "INSERT INTO memory_projection_state ( + scope_key, projected_ref_oid, last_event_seq, schema_version, + policy_version, rebuilt_at + ) VALUES ( + 'repo', 'dddddddddddddddddddddddddddddddddddddddd', 0, 1, 'v1', 1 + )", + ) + .await + .expect("seed a core projection row before upgrade"); + assert_eq!( + run_builtin_migrations(&upgrade) + .await + .expect("upgrade to Memory FTS"), + vec![2026082402] + ); + let core_rows: i64 = upgrade + .query_one_raw(Statement::from_string( + upgrade.get_database_backend(), + "SELECT COUNT(*) AS count FROM memory_projection_state".to_string(), + )) + .await + .expect("count preserved core rows") + .expect("core count row") + .try_get("", "count") + .expect("core count"); + assert_eq!( + core_rows, 1, + "FTS migration cannot rewrite core projections" + ); + + let runner = builtin_runner().expect("builtin runner"); + assert_eq!( + runner + .rollback_to(&upgrade, 2026082401) + .await + .expect("empty search projection can roll back"), + vec![2026082402] + ); + assert!(!table_exists(&upgrade, "memory_episode_search_doc").await); + assert!(!table_exists(&upgrade, "memory_episode_fts").await); + assert!(table_exists(&upgrade, "memory_projection_state").await); + assert_eq!( + runner + .run_pending(&upgrade) + .await + .expect("reapply search migration"), + vec![2026082402] + ); +} + +#[tokio::test] +async fn memory_episode_fts_nonempty_down_guard() { + let (_dir, url, _path) = fresh_db_url(); + let conn = connect(&url).await; + let runner = builtin_runner().expect("builtin runner"); + runner + .run_pending(&conn) + .await + .expect("apply full registry"); + seed_memory_search_fixture(&conn).await; + + let error = runner.rollback_to(&conn, 2026082401).await.unwrap_err(); + assert!( + format!("{error:#}").contains("memory_fts_search_down_guard_empty"), + "non-empty FTS rollback must fail with the stable guard: {error:#}" + ); + assert_eq!( + runner.current_version(&conn).await.unwrap(), + Some(2026082402) + ); + + conn.execute_unprepared( + "INSERT INTO memory_episode_fts ( + memory_episode_fts, rowid, goal, summary, decisions, + failed_attempts, unresolved + ) SELECT + 'delete', rowid, goal, summary, decisions, failed_attempts, unresolved + FROM memory_episode_search_doc; + DELETE FROM memory_episode_search_doc;", + ) + .await + .expect("clear search projection through the external-content protocol"); + assert_eq!( + runner + .rollback_to(&conn, 2026082401) + .await + .expect("empty search projection can roll back"), + vec![2026082402] + ); +} + // --------------------------------------------------------------------------- // run_pending on a fresh database: applies every registered migration // --------------------------------------------------------------------------- @@ -1651,7 +2008,8 @@ async fn run_builtin_migrations_applies_current_builtin_registry() { 2026071405, 2026071406, 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, - 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 + 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, + 2026082402 ] ); assert!(table_exists(&conn, "schema_versions").await); @@ -1845,7 +2203,7 @@ async fn agent_subagent_content_up_down_up_and_nonempty_guard() { 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, - 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 ] ); conn.execute_raw(Statement::from_string( @@ -1910,7 +2268,7 @@ async fn agent_subagent_content_up_down_up_and_nonempty_guard() { 2026071406, 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, - 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 + 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 ] ); assert!(table_exists(&conn, "agent_subagent_content_claim").await); @@ -2014,7 +2372,7 @@ async fn existing_agent_subagent_1406_schema_upgrades_to_replication() { 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, - 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 ] ); let claim = conn @@ -2148,7 +2506,7 @@ async fn evolved_agent_subagent_1406_columns_upgrade_idempotently() { 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, - 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 ] ); let cursor = conn @@ -2192,11 +2550,11 @@ async fn agent_import_identity_tombstone_up_down_up_round_trip() { assert_eq!( rolled, vec![ - 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, - 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, - 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303, 2026072302, - 2026072301, 2026072201, 2026072101, 2026071901, 2026071407, 2026071406, 2026071405, - 2026071404, 2026071403, 2026071402 + 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, + 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, + 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303, + 2026072302, 2026072301, 2026072201, 2026072101, 2026071901, 2026071407, 2026071406, + 2026071405, 2026071404, 2026071403, 2026071402 ] ); assert!(!table_exists(&conn, "agent_import_identity").await); @@ -2215,7 +2573,7 @@ async fn agent_import_identity_tombstone_up_down_up_round_trip() { 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, - 2026080403, 2026081301, 2026082401 + 2026080403, 2026081301, 2026082401, 2026082402 ] ); assert!(table_exists(&conn, "agent_import_identity").await); @@ -2246,11 +2604,11 @@ async fn existing_agent_tombstone_1403_schema_upgrades_to_compat_barrier() { assert_eq!( rolled, vec![ - 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, - 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, - 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303, 2026072302, - 2026072301, 2026072201, 2026072101, 2026071901, 2026071407, 2026071406, 2026071405, - 2026071404 + 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, + 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, + 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303, + 2026072302, 2026072301, 2026072201, 2026072101, 2026071901, 2026071407, 2026071406, + 2026071405, 2026071404 ] ); assert!(table_exists(&conn, "agent_import_tombstone").await); @@ -2269,7 +2627,7 @@ async fn existing_agent_tombstone_1403_schema_upgrades_to_compat_barrier() { 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, - 2026082401 + 2026082401, 2026082402 ] ); assert!(trigger_exists(&conn, "agent_tombstone_block_session_insert").await); @@ -2614,14 +2972,14 @@ async fn approved_permission_up_down_up_round_trip() { assert_eq!( rolled, vec![ - 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, - 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, - 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303, 2026072302, - 2026072301, 2026072201, 2026072101, 2026071901, 2026071407, 2026071406, 2026071405, - 2026071404, 2026071403, 2026071402, 2026071401, 2026071301, 2026070803, 2026070802, - 2026070801, 2026070701, 2026070601, 2026070501, 2026070401, 2026070301, 2026070202, - 2026070201, 2026062301, 2026061401, 2026060801, 2026060401, 2026060201, 2026053101, - 2026052301, 2026050801, 2026050601 + 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, + 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, + 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303, + 2026072302, 2026072301, 2026072201, 2026072101, 2026071901, 2026071407, 2026071406, + 2026071405, 2026071404, 2026071403, 2026071402, 2026071401, 2026071301, 2026070803, + 2026070802, 2026070801, 2026070701, 2026070601, 2026070501, 2026070401, 2026070301, + 2026070202, 2026070201, 2026062301, 2026061401, 2026060801, 2026060401, 2026060201, + 2026053101, 2026052301, 2026050801, 2026050601 ] ); assert!( @@ -2652,7 +3010,7 @@ async fn approved_permission_up_down_up_round_trip() { 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, - 2026080403, 2026081301, 2026082401 + 2026080403, 2026081301, 2026082401, 2026082402 ] ); assert!(table_exists(&conn, "approved_permission").await); @@ -3524,9 +3882,9 @@ async fn legacy_layer_rows_with_linked_fail_migration() { .await .expect("rollback layer scope"), vec![ - 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, - 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, - 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303 + 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, + 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, + 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303 ] ); conn.execute_raw(Statement::from_string( @@ -3579,7 +3937,7 @@ async fn legacy_layer_rows_with_linked_fail_migration() { vec![ 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, - 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 + 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 ] ); let row = conn @@ -3856,9 +4214,9 @@ async fn sparse_migration_projects_last_wins_toggle() { .await .expect("rollback sparse scope"), vec![ - 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, - 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, - 2026072501, 2026072403, 2026072402, 2026072401, 2026072304 + 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, + 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, + 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304 ] ); // Duplicate legacy rows: stale `true` (lower id) then effective `false` @@ -3894,7 +4252,7 @@ async fn sparse_migration_projects_last_wins_toggle() { vec![ 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, - 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 ] ); let row = conn @@ -3929,9 +4287,9 @@ async fn legacy_sparse_state_with_linked_requires_adopt_or_clear() { .await .expect("rollback sparse scope"), vec![ - 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, - 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, - 2026072501, 2026072403, 2026072402, 2026072401, 2026072304 + 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, + 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, + 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304 ] ); conn.execute_raw(Statement::from_string( @@ -3974,7 +4332,7 @@ async fn legacy_sparse_state_with_linked_requires_adopt_or_clear() { vec![ 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, - 2026080401, 2026080402, 2026080403, 2026081301, 2026082401 + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 ] ); assert!(column_exists(&conn, "sparse_view", "worktree_id").await); @@ -4195,9 +4553,9 @@ async fn worktree_registry_v2_capability_marker_round_trip() { .await .expect("rollback capability marker"), vec![ - 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005, - 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, 2026072502, - 2026072501, 2026072403, 2026072402, 2026072401 + 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, + 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, + 2026072502, 2026072501, 2026072403, 2026072402, 2026072401 ] ); assert!(!table_exists(&conn, "worktree_registry_capability").await); @@ -4208,7 +4566,7 @@ async fn worktree_registry_v2_capability_marker_round_trip() { vec![ 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, - 2026080402, 2026080403, 2026081301, 2026082401 + 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 ] ); assert!(table_exists(&conn, "worktree_registry_capability").await); @@ -4331,7 +4689,7 @@ async fn registry_v2_down_migration_rejects_nonterminal_state() { vec![ 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, - 2026080403, 2026081301, 2026082401 + 2026080403, 2026081301, 2026082401, 2026082402 ] ); } @@ -4414,7 +4772,7 @@ async fn workspace_record_down_migration_rejects_nonterminal_state() { vec![ 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, - 2026082401 + 2026082401, 2026082402 ] ); @@ -4906,7 +5264,7 @@ async fn registry_v3_rollback_refuses_live_generations() { .current_version(&conn) .await .expect("current version"), - Some(2026082401), + Some(2026082402), "and the schema is untouched" ); } @@ -4953,7 +5311,8 @@ async fn registry_v3_rollback_allows_absent_generations() { assert_eq!( rolled, vec![ - 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005 + 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, + 2026073005 ], "{label}: exactly v3 rolled back" ); @@ -4982,7 +5341,8 @@ async fn registry_v3_rollback_allows_unreadable_registry() { .await .expect("an unparseable registry does not block the rollback"), vec![ - 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, 2026073005 + 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, + 2026073005 ] ); } @@ -5278,8 +5638,8 @@ async fn stash_generation_fence_up_down_up_round_trip() { ); assert_eq!( runner.current_version(&conn).await.expect("version"), - Some(2026082401), - "the Memory core migration is the newest migration — retarget this test when a newer one lands" + Some(2026082402), + "the Memory FTS migration is the newest migration — retarget this test when a newer one lands" ); let rolled = runner @@ -5289,7 +5649,7 @@ async fn stash_generation_fence_up_down_up_round_trip() { assert_eq!( rolled, vec![ - 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101 + 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101 ], "the runtime attribution, scope, and fence migrations roll back in order" ); @@ -5372,7 +5732,7 @@ async fn approved_permission_provenance_migration() { // Up: empty provenance backfill; project_id unchanged. assert_eq!( runner.run_pending(&conn).await.expect("apply W4-07"), - vec![2026081301, 2026082401] + vec![2026081301, 2026082401, 2026082402] ); assert!(column_exists(&conn, "approved_permission", "source_worktree_id").await); assert!(column_exists(&conn, "approved_permission", "source_session_id").await); @@ -5464,7 +5824,7 @@ async fn approved_permission_provenance_migration() { assert_eq!( runner.run_pending(&conn).await.expect("re-up"), - vec![2026081301, 2026082401] + vec![2026081301, 2026082401, 2026082402] ); let kept_after: i64 = conn .query_one_raw(Statement::from_string( @@ -5499,7 +5859,7 @@ async fn approved_permission_old_reader_rejects_migrated_schema() { .await .expect("read tip") .expect("applied tip"); - assert_eq!(current, 2026082401); + assert_eq!(current, 2026082402); // An old binary whose registry tip is still 2026080403 would see this // repository as UnsupportedFuture. Prove the refuse path on repository // DBs (not global config.db) by planting a version above this binary. diff --git a/tests/fts5_capability_test.rs b/tests/fts5_capability_test.rs new file mode 100644 index 000000000..ffd54c64c --- /dev/null +++ b/tests/fts5_capability_test.rs @@ -0,0 +1,104 @@ +//! Release-profile capability proof for Libra's linked SQLite FTS5 build. + +use sea_orm::{ConnectionTrait, Database, Statement, TransactionTrait}; + +#[tokio::test] +async fn sqlite_fts5_release_capability() { + let database = Database::connect("sqlite::memory:") + .await + .expect("connect with Libra's linked SQLite"); + let enabled: i64 = database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + "SELECT sqlite_compileoption_used('ENABLE_FTS5') AS enabled".to_string(), + )) + .await + .expect("read SQLite compile options") + .expect("compile-option row") + .try_get("", "enabled") + .expect("compile-option value"); + assert_eq!(enabled, 1, "the release-linked SQLite must compile FTS5 in"); + + database + .execute_unprepared( + "CREATE TABLE capability_doc ( + rowid INTEGER PRIMARY KEY, + goal TEXT NOT NULL, + summary TEXT NOT NULL, + decisions TEXT NOT NULL, + failed_attempts TEXT NOT NULL, + unresolved TEXT NOT NULL + ); + CREATE VIRTUAL TABLE capability_fts USING fts5( + goal, summary, decisions, failed_attempts, unresolved, + content='capability_doc', content_rowid='rowid', + tokenize='unicode61 remove_diacritics 2' + );", + ) + .await + .expect("create the production-shape external-content FTS5 schema"); + + let transaction = database.begin().await.expect("begin capability insert"); + transaction + .execute_unprepared( + "INSERT INTO capability_doc VALUES + (1, 'needle', '', '', '', ''), + (2, '', '', '', '', 'needle'), + (3, 'café', '', '', '', ''); + INSERT INTO capability_fts ( + rowid, goal, summary, decisions, failed_attempts, unresolved + ) SELECT + rowid, goal, summary, decisions, failed_attempts, unresolved + FROM capability_doc;", + ) + .await + .expect("insert content and matching postings"); + transaction.commit().await.expect("commit capability rows"); + + let ranked = database + .query_all_raw(Statement::from_sql_and_values( + database.get_database_backend(), + "SELECT rowid, + bm25(capability_fts, 8.0, 5.0, 4.0, 3.0, 2.0) AS score + FROM capability_fts + WHERE capability_fts MATCH ? + ORDER BY score ASC, rowid ASC", + ["needle".into()], + )) + .await + .expect("execute parameter-bound MATCH and BM25"); + let ranked_ids: Vec = ranked + .iter() + .map(|row| row.try_get("", "rowid").expect("ranked rowid")) + .collect(); + assert_eq!( + ranked_ids, + vec![1, 2], + "a goal hit with weight 8 must sort before an unresolved hit with weight 2" + ); + let scores: Vec = ranked + .iter() + .map(|row| row.try_get("", "score").expect("BM25 score")) + .collect(); + assert!( + scores[0] < scores[1], + "SQLite BM25 ranks smaller scores first" + ); + + let diacritic_matches: i64 = database + .query_one_raw(Statement::from_sql_and_values( + database.get_database_backend(), + "SELECT COUNT(*) AS count FROM capability_fts + WHERE capability_fts MATCH ?", + ["cafe".into()], + )) + .await + .expect("query unicode61 diacritic behavior") + .expect("diacritic count row") + .try_get("", "count") + .expect("diacritic count"); + assert_eq!( + diacritic_matches, 1, + "remove_diacritics=2 must make cafe match café" + ); +} From 5a2a30edae181834caf6e155b2dc1bb79d4cc7b0 Mon Sep 17 00:00:00 2001 From: anduin9527 Date: Mon, 24 Aug 2026 21:21:46 +0800 Subject: [PATCH 05/18] feat(memory): add context selection receipt ledger Signed-off-by: anduin9527 --- docs/development/gap/mainline.md | 2 +- docs/development/plan/plan-20260819.md | 33 +- docs/development/tracing/memory.md | 14 +- .../2026082403_context_selection_receipt.sql | 141 +++ ...6082403_context_selection_receipt_down.sql | 20 + sql/migrations/README.md | 1 + src/command/maintenance.rs | 66 ++ src/internal/ai/context_budget/mod.rs | 7 + src/internal/ai/context_budget/receipt.rs | 994 ++++++++++++++++++ .../ai/context_budget/receipt_store.rs | 957 +++++++++++++++++ src/internal/ai/keyed_digest.rs | 214 +++- src/internal/db.rs | 11 + src/internal/db/migration.rs | 12 +- tests/db_migration_test.rs | 359 +++++-- 14 files changed, 2712 insertions(+), 119 deletions(-) create mode 100644 sql/migrations/2026082403_context_selection_receipt.sql create mode 100644 sql/migrations/2026082403_context_selection_receipt_down.sql create mode 100644 src/internal/ai/context_budget/receipt.rs create mode 100644 src/internal/ai/context_budget/receipt_store.rs diff --git a/docs/development/gap/mainline.md b/docs/development/gap/mainline.md index 5517add5c..95a92ec43 100644 --- a/docs/development/gap/mainline.md +++ b/docs/development/gap/mainline.md @@ -159,7 +159,7 @@ flowchart LR | IntentSpec 规范化/草稿/校验/评审/scope | `src/internal/ai/intentspec/{canonical,draft,validator,review,scope}.rs` | ML-02 seal、ML-04 指纹（scope 文件维） | | git-internal Intent/Decision 对象 + MCP create | `src/internal/ai/mcp/`、`intentspec/persistence.rs`、`workflow_objects.rs` | ML-02/ML-03 | | 外部 agent 捕获 + redaction | `src/internal/ai/hooks/runtime.rs` | ML-08 注入（复用 Redactor） | -| ContextFrame / ContextSnapshot + local reviewed MemoryAnchor | `context_budget/{frame,memory_anchor}.rs`、`runtime/phase0.rs` | ML-05 选择回执的本地关联与 review/revoke/expiry UX；当前仍无 branch-aware team recall 或可共享 receipt | +| ContextFrame / ContextSnapshot + local reviewed MemoryAnchor + shared local receipt ledger | `context_budget/{frame,memory_anchor,receipt,receipt_store}.rs`、`runtime/phase0.rs` | `ContextSelectionReceiptV1`、SQLite 账本与 retention 已落地；ML-05 仍需把 intent 检索结果接入该写入器。当前仍无 branch-aware team recall；receipt 保持 local-only | | 嵌入式 Next.js + 单向 publish 导出 + C4 observe-only API | `src/internal/publish/ai_export.rs`、`src/internal/ai/web/`（C4 /api/code/*） | ML-09 Hub | | 稳定错误码 + `--json/--machine` 输出 | `src/utils/{error,output}.rs` | 全部命令 | diff --git a/docs/development/plan/plan-20260819.md b/docs/development/plan/plan-20260819.md index d1cc260fe..992e00e9d 100644 --- a/docs/development/plan/plan-20260819.md +++ b/docs/development/plan/plan-20260819.md @@ -919,39 +919,38 @@ flowchart TB **Task type:** `migration` -**Lifecycle / Acceptance:** `pending` / 空 +**Lifecycle / Acceptance:** `in-progress` / `locally-accepted` **Description:** 在现有 SQLite 中建立共享 `ContextSelectionReceiptV1` append-only ledger、retention 元数据和单一 `ReceiptStore`，供 Memory 与后续 mainline 共用。 **Out of scope:** Memory 候选选择和 prompt 注入；由 M2-12 承接。 -**Current evidence:** `ContextFrame` 可含 raw content/attachment，不能扩成 receipt；`docs/development/gap/mainline.md:265-272`。Memory 草案已有 local-only receipt 方向但表名与 mainline 分叉；`memory.md:981-1005,1334-1356`。 +**Current evidence:** `ContextSelectionReceiptV1`、绑定规范仓库身份与 digest key generation 的单一 `ReceiptStore`、`context_selection_receipt` / retention migration 已落地；20,000 行 fixture 收敛到 10,000，29/30/31 天边界使用注入时钟固定验证。migration fresh/idempotent/upgrade/empty-down/nonempty-guard、schema-tip preflight，以及删除发生后的事务回滚均已覆盖。`ContextFrame` wire schema 未修改，Memory ref/投影不参与 receipt append/prune。 **Acceptance criteria:** -- [ ] migration 创建 `context_selection_receipt` 与 `context_selection_receipt_retention`，字段严格匹配 ADR-M2-10。 -- [ ] receipt ID 使用现有 `Uuid::now_v7()`；`(repository_id, recorded_at)` 和 `recorded_at` 索引支持时间判断与有界裁剪，不新增 ULID 依赖。 -- [ ] schema/validator 拒绝 raw query/body、可逆 principal、SecretLike、未知 source kind 与未知 `digest_key_id`；principal/query digest 只调用 M2-01K typed purpose API。 -- [ ] `ReceiptStore::append` 在一个短事务写回执并更新 retention 元数据，不修改 Memory ref/投影。 -- [ ] 默认每仓保留 30 天且最多 10,000 行；append 时按索引删除过期和超额最旧行。 -- [ ] 缺失 UUIDv7 的内嵌时间早于 `pruned_before` 时返回 `expired`，source/policy/index snapshot 或 digest key 缺失返回 `non_reproducible`，其它缺失返回 `not_found`。 -- [ ] migration up/idempotent/down/fresh-vs-upgrade 与 append/prune crash recovery 全绿；前一 registry tip 的 reader 对已迁移 DB 在 schema preflight 明确拒绝。 -- [ ] 20,000 行容量 fixture 收敛到硬上限，裁剪不改变任何权威 Memory 状态。 +- [x] migration 创建 `context_selection_receipt` 与 `context_selection_receipt_retention`，字段严格匹配 ADR-M2-10。 +- [x] receipt ID 使用现有 `Uuid::now_v7()`；`(repository_id, recorded_at, receipt_id)` 和 `(recorded_at, receipt_id)` 索引支持时间判断与有界裁剪，不新增 ULID 依赖。 +- [x] schema/validator 在事务前拒绝 raw query/body、可逆 principal、SecretLike、未知 source kind、跨仓库/未知 digest key generation、超出 SQLite JSON envelope 的结构化字段；principal/query digest 只调用 M2-01K typed purpose API。 +- [x] `ReceiptStore::append` 在一个短事务写回执并更新 retention 元数据，不修改 Memory ref/投影。 +- [x] 默认每仓保留 30 天且最多 10,000 行；append 时按索引删除过期和超额最旧行。 +- [x] 缺失 UUIDv7 的内嵌时间早于 `pruned_before` 时返回 `expired`，source/policy/index snapshot 或 digest key 缺失返回 `non_reproducible`，其它缺失返回 `not_found`。 +- [x] migration up/idempotent/down/fresh-vs-upgrade 与 append/prune crash recovery 全绿；前一 registry tip 的 reader 对已迁移 DB 在 schema preflight 明确拒绝。 +- [x] 20,000 行容量 fixture 收敛到硬上限，裁剪不改变任何权威 Memory 状态。 **Verification:** -- [ ] `source .env.test && cargo test --test db_migration_test context_selection_receipt_schema` -- [ ] `source .env.test && cargo test --lib internal::ai::context_budget::receipt` -- [ ] `source .env.test && cargo test --test memory_episode_test receipt_append_and_prune_atomic` -- [ ] `source .env.test && cargo test --test memory_episode_test receipt_expired_vs_non_reproducible` -- [ ] `source .env.test && cargo test --test memory_episode_test receipt_retention_20000_rows` -- [ ] `source .env.test && cargo test --test db_migration_test context_receipt_old_reader_rejects_migrated_schema` +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --lib internal::ai::context_budget::receipt -- --nocapture`（8/8） +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --test db_migration_test context_selection_receipt -- --nocapture`（2/2） +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --lib internal::db::tests::context_receipt_old_reader_rejects_migrated_schema`（1/1） +- [x] `LIBRA_SKIP_WEB_BUILD=1 cargo test --test db_migration_test -- --nocapture`（80/80） +- [x] `cargo +nightly fmt --all --check`；`LIBRA_SKIP_WEB_BUILD=1 cargo clippy --all-targets --all-features -- -D warnings` **Dependencies:** M2-02F（串行 migration registry/build window）。 **Deliverables:** N/A。 -**Implementation write set:** 独立 receipt migration、`src/internal/db/migration.rs`、`src/internal/ai/context_budget/{receipt.rs,receipt_store.rs}`、`src/internal/ai/context_budget/mod.rs`、现有 `uuid` v7 API、`tests/{db_migration_test.rs,memory_episode_test.rs,INDEX.md}`、migration README、`memory.md`、`mainline.md`。 +**Implementation write set:** 独立 receipt migration、`src/internal/db/{migration.rs}`、`src/internal/db.rs`、`src/internal/ai/context_budget/{receipt.rs,receipt_store.rs,mod.rs}`、`src/internal/ai/keyed_digest.rs`、`src/command/maintenance.rs` GC inventory、`tests/db_migration_test.rs`、migration README、`memory.md`、`mainline.md` 与本计划。没有新增 test target，因此 `tests/INDEX.md` 不变。 **Release write set:** N/A。 diff --git a/docs/development/tracing/memory.md b/docs/development/tracing/memory.md index f7367aa09..4a3aa0f59 100644 --- a/docs/development/tracing/memory.md +++ b/docs/development/tracing/memory.md @@ -1365,9 +1365,9 @@ CREATE TABLE context_selection_receipt ( recorded_at TEXT NOT NULL ); CREATE INDEX idx_context_selection_receipt_repository_time - ON context_selection_receipt(repository_id, recorded_at); + ON context_selection_receipt(repository_id, recorded_at, receipt_id); CREATE INDEX idx_context_selection_receipt_time - ON context_selection_receipt(recorded_at); + ON context_selection_receipt(recorded_at, receipt_id); -- 每仓回执保留水位。ReceiptStore 与回执追加/裁剪在同一短事务更新它； -- pruned_before 是已经删除的最晚 recorded_at，用来区分 expired 与 not_found。 @@ -1375,7 +1375,8 @@ CREATE TABLE context_selection_receipt_retention ( repository_id TEXT PRIMARY KEY, pruned_before TEXT, last_pruned_at TEXT, - retained_rows INTEGER NOT NULL DEFAULT 0 CHECK (retained_rows >= 0) + retained_rows INTEGER NOT NULL DEFAULT 0 + CHECK (retained_rows BETWEEN 0 AND 10000) ); ``` @@ -1794,6 +1795,13 @@ memory.get(scope, namespace, path) -> Vec - **单一 receipt 原语。** Rust 类型与 mainline ML-05 / ML-08 共用同一定义（§0.0.10）；本文与 mainline 各自的注入管线写同一张回执面，不得分叉出两种 schema。 - `memory inspect-injection` 从回执读取并重放展示，而非从当前投影反推。被预算丢弃的项直接记录在 receipt 的 `omissions` 中，不再为每次读取追加 `PromptTrimmed` 权威事件。 +M2-02R 已落地上述共享类型、`ReceiptStore`、SQLite migration 与 retention +水位。当前阶段只提供 crate-private 写入/查找原语；Memory 候选选择和 prompt +注入仍由 M2-12 接线。回执中的 `code_commit`、`source_heads`、 +`projection_watermarks`、`selected`、`policy_hash` 与 `bundle_hash` 已在 GC +source inventory 中声明为 non-root：回执负责审计选择，不取得对象保活权； +依赖被清理后，重放状态转为 `non_reproducible`。 + ### 8.7 混合检索通道（Hybrid Retrieval Channels） §8.1–§8.4 的召回原语之上，检索面由**确定性基线 + 可选增强通道**组成（实现 MEM-02 的检索面）。向量与图永远是可替换的候选层（§0.0.11），不改变 §8.6 的确定性承诺。 diff --git a/sql/migrations/2026082403_context_selection_receipt.sql b/sql/migrations/2026082403_context_selection_receipt.sql new file mode 100644 index 000000000..f59aa42f1 --- /dev/null +++ b/sql/migrations/2026082403_context_selection_receipt.sql @@ -0,0 +1,141 @@ +-- M2-02R: shared, local-only context selection receipt ledger. +-- +-- ReceiptStore is the only writer. The ledger is append-only except for its +-- bounded retention prune, which updates the per-repository watermark in the +-- same short transaction. + +CREATE TABLE IF NOT EXISTS `context_selection_receipt` ( + `receipt_id` TEXT PRIMARY KEY CHECK ( + length(`receipt_id`) = 36 + AND substr(`receipt_id`, 9, 1) = '-' + AND substr(`receipt_id`, 14, 1) = '-' + AND substr(`receipt_id`, 15, 1) = '7' + AND substr(`receipt_id`, 19, 1) = '-' + AND substr(`receipt_id`, 20, 1) IN ('8', '9', 'a', 'b') + AND substr(`receipt_id`, 24, 1) = '-' + AND length(replace(`receipt_id`, '-', '')) = 32 + AND replace(`receipt_id`, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + `schema_version` INTEGER NOT NULL CHECK (`schema_version` = 1), + `source_kind` TEXT NOT NULL CHECK (`source_kind` IN ('memory', 'intent', 'hook')), + `repository_id` TEXT NOT NULL CHECK ( + length(CAST(`repository_id` AS BLOB)) BETWEEN 1 AND 512 + AND trim(`repository_id`) = `repository_id` + ), + `digest_key_id` TEXT NOT NULL CHECK ( + length(`digest_key_id`) = 36 + AND substr(`digest_key_id`, 9, 1) = '-' + AND substr(`digest_key_id`, 14, 1) = '-' + AND substr(`digest_key_id`, 15, 1) = '4' + AND substr(`digest_key_id`, 19, 1) = '-' + AND substr(`digest_key_id`, 20, 1) IN ('8', '9', 'a', 'b') + AND substr(`digest_key_id`, 24, 1) = '-' + AND length(replace(`digest_key_id`, '-', '')) = 32 + AND replace(`digest_key_id`, '-', '') NOT GLOB '*[^0-9a-f]*' + ), + `principal_hmac` TEXT NOT NULL CHECK ( + length(`principal_hmac`) = 113 + AND substr(`principal_hmac`, 1, 12) = 'hmac-sha256:' + AND substr(`principal_hmac`, 13, 36) = `digest_key_id` + AND substr(`principal_hmac`, 49, 1) = ':' + AND substr(`principal_hmac`, 50) NOT GLOB '*[^0-9a-f]*' + ), + `query_hmac` TEXT NOT NULL CHECK ( + length(`query_hmac`) = 113 + AND substr(`query_hmac`, 1, 12) = 'hmac-sha256:' + AND substr(`query_hmac`, 13, 36) = `digest_key_id` + AND substr(`query_hmac`, 49, 1) = ':' + AND substr(`query_hmac`, 50) NOT GLOB '*[^0-9a-f]*' + ), + `effective_at` TEXT NOT NULL CHECK ( + length(CAST(`effective_at` AS BLOB)) BETWEEN 20 AND 40 + ), + `code_commit` TEXT CHECK ( + `code_commit` IS NULL OR ( + length(`code_commit`) IN (40, 64) + AND `code_commit` NOT GLOB '*[^0-9a-f]*' + ) + ), + `full_branch_ref` TEXT CHECK ( + `full_branch_ref` IS NULL OR ( + length(CAST(`full_branch_ref` AS BLOB)) BETWEEN 12 AND 4096 + AND substr(`full_branch_ref`, 1, 11) = 'refs/heads/' + ) + ), + `source_heads_json` TEXT NOT NULL CHECK ( + length(CAST(`source_heads_json` AS BLOB)) BETWEEN 2 AND 65536 + AND json_valid(`source_heads_json`) + AND json_type(`source_heads_json`) = 'object' + ), + `projection_watermarks_json` TEXT NOT NULL CHECK ( + length(CAST(`projection_watermarks_json` AS BLOB)) BETWEEN 2 AND 65536 + AND json_valid(`projection_watermarks_json`) + AND json_type(`projection_watermarks_json`) = 'object' + ), + `policy_hash` TEXT NOT NULL CHECK ( + length(`policy_hash`) = 71 + AND substr(`policy_hash`, 1, 7) = 'sha256:' + AND substr(`policy_hash`, 8) NOT GLOB '*[^0-9a-f]*' + ), + `selector_version` TEXT NOT NULL CHECK ( + length(CAST(`selector_version` AS BLOB)) BETWEEN 1 AND 128 + AND trim(`selector_version`) = `selector_version` + ), + `token_budget` INTEGER NOT NULL CHECK (`token_budget` >= 0), + `selected_json` TEXT NOT NULL CHECK ( + length(CAST(`selected_json` AS BLOB)) BETWEEN 2 AND 1048576 + AND json_valid(`selected_json`) + AND json_type(`selected_json`) = 'array' + ), + `omissions_json` TEXT NOT NULL CHECK ( + length(CAST(`omissions_json` AS BLOB)) BETWEEN 2 AND 65536 + AND json_valid(`omissions_json`) + AND json_type(`omissions_json`) = 'array' + ), + `bundle_hash` TEXT NOT NULL CHECK ( + length(`bundle_hash`) = 71 + AND substr(`bundle_hash`, 1, 7) = 'sha256:' + AND substr(`bundle_hash`, 8) NOT GLOB '*[^0-9a-f]*' + ), + `reproducibility_state` TEXT NOT NULL CHECK ( + `reproducibility_state` IN ('reproducible', 'stale', 'expired', 'non_reproducible') + ), + `frame_id` TEXT CHECK ( + `frame_id` IS NULL OR ( + length(`frame_id`) = 36 + AND substr(`frame_id`, 9, 1) = '-' + AND substr(`frame_id`, 14, 1) = '-' + AND substr(`frame_id`, 19, 1) = '-' + AND substr(`frame_id`, 24, 1) = '-' + AND length(replace(`frame_id`, '-', '')) = 32 + AND replace(`frame_id`, '-', '') NOT GLOB '*[^0-9a-f]*' + ) + ), + `recorded_at` TEXT NOT NULL CHECK ( + length(CAST(`recorded_at` AS BLOB)) BETWEEN 20 AND 40 + ) +); + +CREATE INDEX IF NOT EXISTS `idx_context_selection_receipt_repository_time` + ON `context_selection_receipt` (`repository_id`, `recorded_at`, `receipt_id`); + +CREATE INDEX IF NOT EXISTS `idx_context_selection_receipt_time` + ON `context_selection_receipt` (`recorded_at`, `receipt_id`); + +CREATE TABLE IF NOT EXISTS `context_selection_receipt_retention` ( + `repository_id` TEXT PRIMARY KEY CHECK ( + length(CAST(`repository_id` AS BLOB)) BETWEEN 1 AND 512 + AND trim(`repository_id`) = `repository_id` + ), + `pruned_before` TEXT CHECK ( + `pruned_before` IS NULL + OR length(CAST(`pruned_before` AS BLOB)) BETWEEN 20 AND 40 + ), + `last_pruned_at` TEXT CHECK ( + `last_pruned_at` IS NULL + OR length(CAST(`last_pruned_at` AS BLOB)) BETWEEN 20 AND 40 + ), + `retained_rows` INTEGER NOT NULL DEFAULT 0 CHECK ( + `retained_rows` BETWEEN 0 AND 10000 + ) +); diff --git a/sql/migrations/2026082403_context_selection_receipt_down.sql b/sql/migrations/2026082403_context_selection_receipt_down.sql new file mode 100644 index 000000000..bb205e670 --- /dev/null +++ b/sql/migrations/2026082403_context_selection_receipt_down.sql @@ -0,0 +1,20 @@ +-- Rollback of 2026082403_context_selection_receipt. +-- +-- A non-empty ledger or retention watermark is audit evidence. Refuse to drop +-- either and require a forward migration instead. + +CREATE TABLE IF NOT EXISTS `context_selection_receipt_down_guard` ( + `blocked` INTEGER NOT NULL, + CONSTRAINT `context_selection_receipt_down_guard_empty` CHECK (`blocked` = 0) +); + +INSERT INTO `context_selection_receipt_down_guard` (`blocked`) +SELECT 1 +WHERE EXISTS (SELECT 1 FROM `context_selection_receipt` LIMIT 1) + OR EXISTS (SELECT 1 FROM `context_selection_receipt_retention` LIMIT 1); + +DROP TABLE `context_selection_receipt_down_guard`; +DROP INDEX IF EXISTS `idx_context_selection_receipt_time`; +DROP INDEX IF EXISTS `idx_context_selection_receipt_repository_time`; +DROP TABLE IF EXISTS `context_selection_receipt_retention`; +DROP TABLE IF EXISTS `context_selection_receipt`; diff --git a/sql/migrations/README.md b/sql/migrations/README.md index 414c62f3f..08a695c4f 100644 --- a/sql/migrations/README.md +++ b/sql/migrations/README.md @@ -172,6 +172,7 @@ helpers in `db.rs`. Subsequent CEXes have populated this directory. | `2026081301` | `approved_permission_provenance` | `2026081301_approved_permission_provenance{,_down}.sql` (plan-20260715 W4-07: Always-approval provenance columns; empty backfill; `project_id` not rewritten; down fail-closed with provenance or linked HEAD evidence.) | | `2026082401` | `memory_core` | `2026082401_memory_core{,_down}.sql` (M2-02: rebuildable Memory projections plus bounded per-root compiler job and source-observer state; FTS5 and receipt storage land in later migrations.) | | `2026082402` | `memory_fts_search` | `2026082402_memory_fts_search{,_down}.sql` (M2-02F: rebuildable Episode search document plus external-content FTS5 using the fixed `unicode61 remove_diacritics 2` tokenizer; non-empty rollback is refused.) | +| `2026082403` | `context_selection_receipt` | `2026082403_context_selection_receipt{,_down}.sql` (M2-02R: shared local-only context selection receipt ledger with per-repository 30-day/10,000-row retention metadata; non-empty rollback is refused.) | All registered migrations are loaded via `include_str!`. New migrations must follow the same pattern — inline SQL strings in `builtin_migrations()` are no diff --git a/src/command/maintenance.rs b/src/command/maintenance.rs index ce19ac222..1b9c4da44 100644 --- a/src/command/maintenance.rs +++ b/src/command/maintenance.rs @@ -3356,6 +3356,72 @@ pub const GC_OBJECT_SOURCE_INVENTORY: &[GcObjectSource] = &[ "scanned_through_oid", "First-parent observer watermark into an authoritative source ref; rescanning repairs it and the cursor itself keeps no object alive", ), + GcObjectSource { + origin: GcSourceOrigin::Column, + location: "context_selection_receipt", + column: "code_commit", + status: GcSourceStatus::NonRoot, + kind: GcStorageKind::SqliteColumn, + schema: "2026082403 local-only bounded context selection receipt", + read_bound: "never read by GC", + corruption: GcCorruptionPolicy::NotApplicable, + note: "audit-time code anchor only; the receipt records why context was selected but does not own object reachability, so a collected commit makes replay non-reproducible", + }, + GcObjectSource { + origin: GcSourceOrigin::Column, + location: "context_selection_receipt", + column: "source_heads_json", + status: GcSourceStatus::NonRoot, + kind: GcStorageKind::SqliteColumn, + schema: "2026082403 bounded JSON map of receipt source names to observed OIDs", + read_bound: "never read by GC", + corruption: GcCorruptionPolicy::NotApplicable, + note: "audit-time source snapshot only; authoritative refs own reachability and a collected source object makes replay non-reproducible", + }, + GcObjectSource { + origin: GcSourceOrigin::Column, + location: "context_selection_receipt", + column: "projection_watermarks_json", + status: GcSourceStatus::NonRoot, + kind: GcStorageKind::SqliteColumn, + schema: "2026082403 bounded JSON map of projection names to observed OIDs", + read_bound: "never read by GC", + corruption: GcCorruptionPolicy::NotApplicable, + note: "audit-time projection watermark only; projections are rebuildable and the receipt cannot keep their source objects alive", + }, + GcObjectSource { + origin: GcSourceOrigin::Column, + location: "context_selection_receipt", + column: "selected_json", + status: GcSourceStatus::NonRoot, + kind: GcStorageKind::SqliteColumn, + schema: "2026082403 bounded JSON array containing selected Memory revision OIDs", + read_bound: "never read by GC", + corruption: GcCorruptionPolicy::NotApplicable, + note: "audit-time selection explanation only; Memory refs and events own reachability, so missing selected revisions make replay non-reproducible", + }, + GcObjectSource { + origin: GcSourceOrigin::Column, + location: "context_selection_receipt", + column: "policy_hash", + status: GcSourceStatus::NonRoot, + kind: GcStorageKind::SqliteColumn, + schema: "2026082403 SHA-256 policy content digest", + read_bound: "never read by GC", + corruption: GcCorruptionPolicy::NotApplicable, + note: "domain-separated policy digest used for receipt replay checks; it is not an object-store address and keeps no Libra object alive", + }, + GcObjectSource { + origin: GcSourceOrigin::Column, + location: "context_selection_receipt", + column: "bundle_hash", + status: GcSourceStatus::NonRoot, + kind: GcStorageKind::SqliteColumn, + schema: "2026082403 SHA-256 rendered bundle digest", + read_bound: "never read by GC", + corruption: GcCorruptionPolicy::NotApplicable, + note: "integrity digest of the selected context bundle; the bundle is not stored as an object under this hash, so the value contributes no reachability root", + }, GcObjectSource { origin: GcSourceOrigin::Column, location: "object_obliteration", diff --git a/src/internal/ai/context_budget/mod.rs b/src/internal/ai/context_budget/mod.rs index 7357326f8..27edf2d57 100644 --- a/src/internal/ai/context_budget/mod.rs +++ b/src/internal/ai/context_budget/mod.rs @@ -13,6 +13,13 @@ pub mod frame; pub mod handoff; pub mod memory_anchor; pub mod projection; +// M2-02R freezes the shared receipt seam before M2-12 wires retrieval into +// prompt assembly. Keep the staged crate-private implementation lint-clean +// without exposing it as a public API prematurely. +#[allow(dead_code)] +pub(crate) mod receipt; +#[allow(dead_code)] +pub(crate) mod receipt_store; pub use allocator::{ AllocationOmissionReason, ContextAllocation, ContextAllocationOmission, ContextBudgetAllocator, diff --git a/src/internal/ai/context_budget/receipt.rs b/src/internal/ai/context_budget/receipt.rs new file mode 100644 index 000000000..cc0840c89 --- /dev/null +++ b/src/internal/ai/context_budget/receipt.rs @@ -0,0 +1,994 @@ +//! Shared context selection receipt domain contract. + +use std::collections::{BTreeMap, HashSet}; + +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; +use uuid::Uuid; + +use crate::internal::ai::keyed_digest::{PrincipalDigest, QueryDigest}; + +const RECEIPT_SCHEMA_VERSION: u32 = 1; +const MAX_REPOSITORY_ID_BYTES: usize = 512; +const MAX_BRANCH_REF_BYTES: usize = 4 * 1024; +const MAX_SNAPSHOT_ENTRIES: usize = 64; +const MAX_SNAPSHOT_KEY_BYTES: usize = 128; +const MAX_SELECTOR_VERSION_BYTES: usize = 128; +const MAX_SELECTED: usize = 1_024; +const MAX_OMISSIONS: usize = 64; +const MAX_OBJECT_ID_BYTES: usize = 512; +const MAX_SUMMARY_KEY_BYTES: usize = 4 * 1024; +const MAX_REASON_CODES: usize = 16; +const MAX_REASON_CODE_BYTES: usize = 64; +const MAX_SCORE_COMPONENTS: usize = 32; +const MAX_SNAPSHOT_JSON_BYTES: usize = 64 * 1024; +const MAX_SELECTED_JSON_BYTES: usize = 1024 * 1024; +const MAX_OMISSIONS_JSON_BYTES: usize = 64 * 1024; + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum ReceiptSourceKind { + Memory, + Intent, + Hook, +} + +impl ReceiptSourceKind { + pub(super) const fn as_str(self) -> &'static str { + match self { + Self::Memory => "memory", + Self::Intent => "intent", + Self::Hook => "hook", + } + } + + pub(super) fn parse(value: &str) -> Option { + match value { + "memory" => Some(Self::Memory), + "intent" => Some(Self::Intent), + "hook" => Some(Self::Hook), + _ => None, + } + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub(crate) enum ReceiptReproducibilityState { + Reproducible, + Stale, + Expired, + NonReproducible, +} + +impl ReceiptReproducibilityState { + pub(super) const fn as_str(self) -> &'static str { + match self { + Self::Reproducible => "reproducible", + Self::Stale => "stale", + Self::Expired => "expired", + Self::NonReproducible => "non_reproducible", + } + } + + pub(super) fn parse(value: &str) -> Option { + match value { + "reproducible" => Some(Self::Reproducible), + "stale" => Some(Self::Stale), + "expired" => Some(Self::Expired), + "non_reproducible" => Some(Self::NonReproducible), + _ => None, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ReceiptSensitivity { + Allowed, + SecretLike, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ReceiptDependencyAvailability { + Exact, + Stale, + Missing, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ReceiptReplayAvailabilityV1 { + pub(crate) digest_key: ReceiptDependencyAvailability, + pub(crate) source_snapshot: ReceiptDependencyAvailability, + pub(crate) policy: ReceiptDependencyAvailability, + pub(crate) index_snapshot: ReceiptDependencyAvailability, +} + +impl ReceiptReplayAvailabilityV1 { + pub(crate) const fn all_exact() -> Self { + Self { + digest_key: ReceiptDependencyAvailability::Exact, + source_snapshot: ReceiptDependencyAvailability::Exact, + policy: ReceiptDependencyAvailability::Exact, + index_snapshot: ReceiptDependencyAvailability::Exact, + } + } +} + +pub(crate) struct ReceiptSelectionInputV1 { + pub(crate) object_id: String, + pub(crate) revision_oid: String, + pub(crate) summary_key: String, + pub(crate) order: u32, + pub(crate) reason_codes: Vec, + pub(crate) score_components: BTreeMap, + pub(crate) sensitivity: ReceiptSensitivity, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ReceiptSelectionV1 { + pub(crate) object_id: String, + pub(crate) revision_oid: String, + pub(crate) summary_key: String, + pub(crate) order: u32, + pub(crate) reason_codes: Vec, + pub(crate) score_components: BTreeMap, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ReceiptOmissionV1 { + pub(crate) reason_code: String, + pub(crate) count: u32, +} + +pub(crate) struct ReceiptDraftFieldsV1 { + pub(crate) source_kind: ReceiptSourceKind, + pub(crate) repository_id: String, + pub(crate) principal_digest: PrincipalDigest, + pub(crate) query_digest: QueryDigest, + pub(crate) effective_at: DateTime, + pub(crate) code_commit: Option, + pub(crate) full_branch_ref: Option, + pub(crate) source_heads: BTreeMap, + pub(crate) projection_watermarks: BTreeMap, + pub(crate) policy_hash: String, + pub(crate) selector_version: String, + pub(crate) selected: Vec, + pub(crate) omissions: Vec, + pub(crate) token_budget: u64, + pub(crate) bundle_hash: String, + pub(crate) reproducibility_state: ReceiptReproducibilityState, + pub(crate) frame_id: Option, +} + +pub(crate) struct ContextSelectionReceiptDraftV1 { + source_kind: ReceiptSourceKind, + repository_id: String, + digest_key_id: Uuid, + principal_hmac: String, + query_hmac: String, + effective_at: DateTime, + code_commit: Option, + full_branch_ref: Option, + source_heads: BTreeMap, + projection_watermarks: BTreeMap, + policy_hash: String, + selector_version: String, + selected: Vec, + omissions: Vec, + token_budget: u64, + bundle_hash: String, + reproducibility_state: ReceiptReproducibilityState, + frame_id: Option, +} + +impl ContextSelectionReceiptDraftV1 { + pub(crate) fn new(fields: ReceiptDraftFieldsV1) -> Result { + if fields.principal_digest.version() != RECEIPT_SCHEMA_VERSION as u8 + || fields.query_digest.version() != RECEIPT_SCHEMA_VERSION as u8 + { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::DigestVersion, + )); + } + if fields.principal_digest.key_id() != fields.query_digest.key_id() { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::DigestKeyMismatch, + )); + } + if !valid_bounded(&fields.repository_id, MAX_REPOSITORY_ID_BYTES) { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::RepositoryId, + )); + } + if fields + .code_commit + .as_deref() + .is_some_and(|value| !valid_object_id(value)) + { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::CodeCommit, + )); + } + if fields.full_branch_ref.as_deref().is_some_and(|value| { + value.len() < 12 + || !value.starts_with("refs/heads/") + || !valid_bounded(value, MAX_BRANCH_REF_BYTES) + }) { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::BranchRef, + )); + } + validate_snapshot(&fields.source_heads)?; + validate_snapshot(&fields.projection_watermarks)?; + if !valid_sha256(&fields.policy_hash) { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::PolicyHash, + )); + } + if !valid_bounded(&fields.selector_version, MAX_SELECTOR_VERSION_BYTES) { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::SelectorVersion, + )); + } + if fields.token_budget > i64::MAX as u64 { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::TokenBudget, + )); + } + if !valid_sha256(&fields.bundle_hash) { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::BundleHash, + )); + } + + let selected = validate_selected(fields.selected)?; + validate_omissions(&fields.omissions)?; + validate_json_envelope( + &fields.source_heads, + MAX_SNAPSHOT_JSON_BYTES, + ReceiptValidationErrorKind::Snapshot, + )?; + validate_json_envelope( + &fields.projection_watermarks, + MAX_SNAPSHOT_JSON_BYTES, + ReceiptValidationErrorKind::Snapshot, + )?; + validate_json_envelope( + &selected, + MAX_SELECTED_JSON_BYTES, + ReceiptValidationErrorKind::Selected, + )?; + validate_json_envelope( + &fields.omissions, + MAX_OMISSIONS_JSON_BYTES, + ReceiptValidationErrorKind::Omissions, + )?; + let digest_key_id = fields.principal_digest.key_id(); + Ok(Self { + source_kind: fields.source_kind, + repository_id: fields.repository_id, + digest_key_id, + principal_hmac: fields.principal_digest.encoded(), + query_hmac: fields.query_digest.encoded(), + effective_at: fields.effective_at, + code_commit: fields.code_commit, + full_branch_ref: fields.full_branch_ref, + source_heads: fields.source_heads, + projection_watermarks: fields.projection_watermarks, + policy_hash: fields.policy_hash, + selector_version: fields.selector_version, + selected, + omissions: fields.omissions, + token_budget: fields.token_budget, + bundle_hash: fields.bundle_hash, + reproducibility_state: fields.reproducibility_state, + frame_id: fields.frame_id, + }) + } + + pub(crate) const fn schema_version(&self) -> u32 { + RECEIPT_SCHEMA_VERSION + } + + pub(crate) const fn digest_key_id(&self) -> Uuid { + self.digest_key_id + } + + pub(crate) fn repository_id(&self) -> &str { + &self.repository_id + } + + pub(crate) const fn source_kind(&self) -> ReceiptSourceKind { + self.source_kind + } + + pub(crate) fn selected(&self) -> &[ReceiptSelectionV1] { + &self.selected + } + + pub(crate) fn omissions(&self) -> &[ReceiptOmissionV1] { + &self.omissions + } +} + +pub(super) struct PersistedReceiptFieldsV1 { + pub(super) receipt_id: Uuid, + pub(super) schema_version: u32, + pub(super) source_kind: ReceiptSourceKind, + pub(super) repository_id: String, + pub(super) digest_key_id: Uuid, + pub(super) principal_hmac: String, + pub(super) query_hmac: String, + pub(super) effective_at: DateTime, + pub(super) code_commit: Option, + pub(super) full_branch_ref: Option, + pub(super) source_heads: BTreeMap, + pub(super) projection_watermarks: BTreeMap, + pub(super) policy_hash: String, + pub(super) selector_version: String, + pub(super) token_budget: u64, + pub(super) selected: Vec, + pub(super) omissions: Vec, + pub(super) bundle_hash: String, + pub(super) reproducibility_state: ReceiptReproducibilityState, + pub(super) frame_id: Option, + pub(super) recorded_at: DateTime, +} + +#[derive(Clone, Eq, PartialEq)] +pub(crate) struct ContextSelectionReceiptV1 { + pub(super) receipt_id: Uuid, + pub(super) schema_version: u32, + pub(super) source_kind: ReceiptSourceKind, + pub(super) repository_id: String, + pub(super) digest_key_id: Uuid, + pub(super) principal_hmac: String, + pub(super) query_hmac: String, + pub(super) effective_at: DateTime, + pub(super) code_commit: Option, + pub(super) full_branch_ref: Option, + pub(super) source_heads: BTreeMap, + pub(super) projection_watermarks: BTreeMap, + pub(super) policy_hash: String, + pub(super) selector_version: String, + pub(super) token_budget: u64, + pub(super) selected: Vec, + pub(super) omissions: Vec, + pub(super) bundle_hash: String, + pub(super) reproducibility_state: ReceiptReproducibilityState, + pub(super) frame_id: Option, + pub(super) recorded_at: DateTime, +} + +impl ContextSelectionReceiptV1 { + pub(super) fn from_draft( + receipt_id: Uuid, + recorded_at: DateTime, + draft: ContextSelectionReceiptDraftV1, + ) -> Self { + Self { + receipt_id, + schema_version: RECEIPT_SCHEMA_VERSION, + source_kind: draft.source_kind, + repository_id: draft.repository_id, + digest_key_id: draft.digest_key_id, + principal_hmac: draft.principal_hmac, + query_hmac: draft.query_hmac, + effective_at: draft.effective_at, + code_commit: draft.code_commit, + full_branch_ref: draft.full_branch_ref, + source_heads: draft.source_heads, + projection_watermarks: draft.projection_watermarks, + policy_hash: draft.policy_hash, + selector_version: draft.selector_version, + token_budget: draft.token_budget, + selected: draft.selected, + omissions: draft.omissions, + bundle_hash: draft.bundle_hash, + reproducibility_state: draft.reproducibility_state, + frame_id: draft.frame_id, + recorded_at, + } + } + + pub(super) fn from_persisted( + fields: PersistedReceiptFieldsV1, + ) -> Result { + if fields.receipt_id.get_version_num() != 7 { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::ReceiptId, + )); + } + if fields.schema_version != RECEIPT_SCHEMA_VERSION { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::SchemaVersion, + )); + } + if fields.digest_key_id.get_version_num() != 4 + || !valid_receipt_hmac(&fields.principal_hmac, fields.digest_key_id) + || !valid_receipt_hmac(&fields.query_hmac, fields.digest_key_id) + { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::DigestEncoding, + )); + } + if !valid_bounded(&fields.repository_id, MAX_REPOSITORY_ID_BYTES) { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::RepositoryId, + )); + } + if fields + .code_commit + .as_deref() + .is_some_and(|value| !valid_object_id(value)) + { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::CodeCommit, + )); + } + if fields.full_branch_ref.as_deref().is_some_and(|value| { + value.len() < 12 + || !value.starts_with("refs/heads/") + || !valid_bounded(value, MAX_BRANCH_REF_BYTES) + }) { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::BranchRef, + )); + } + validate_snapshot(&fields.source_heads)?; + validate_snapshot(&fields.projection_watermarks)?; + if !valid_sha256(&fields.policy_hash) + || !valid_sha256(&fields.bundle_hash) + || !valid_bounded(&fields.selector_version, MAX_SELECTOR_VERSION_BYTES) + || fields.token_budget > i64::MAX as u64 + { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::PersistedEnvelope, + )); + } + validate_stored_selected(&fields.selected)?; + validate_omissions(&fields.omissions)?; + validate_json_envelope( + &fields.source_heads, + MAX_SNAPSHOT_JSON_BYTES, + ReceiptValidationErrorKind::PersistedEnvelope, + )?; + validate_json_envelope( + &fields.projection_watermarks, + MAX_SNAPSHOT_JSON_BYTES, + ReceiptValidationErrorKind::PersistedEnvelope, + )?; + validate_json_envelope( + &fields.selected, + MAX_SELECTED_JSON_BYTES, + ReceiptValidationErrorKind::PersistedEnvelope, + )?; + validate_json_envelope( + &fields.omissions, + MAX_OMISSIONS_JSON_BYTES, + ReceiptValidationErrorKind::PersistedEnvelope, + )?; + + Ok(Self { + receipt_id: fields.receipt_id, + schema_version: fields.schema_version, + source_kind: fields.source_kind, + repository_id: fields.repository_id, + digest_key_id: fields.digest_key_id, + principal_hmac: fields.principal_hmac, + query_hmac: fields.query_hmac, + effective_at: fields.effective_at, + code_commit: fields.code_commit, + full_branch_ref: fields.full_branch_ref, + source_heads: fields.source_heads, + projection_watermarks: fields.projection_watermarks, + policy_hash: fields.policy_hash, + selector_version: fields.selector_version, + token_budget: fields.token_budget, + selected: fields.selected, + omissions: fields.omissions, + bundle_hash: fields.bundle_hash, + reproducibility_state: fields.reproducibility_state, + frame_id: fields.frame_id, + recorded_at: fields.recorded_at, + }) + } + + pub(crate) const fn receipt_id(&self) -> Uuid { + self.receipt_id + } + + pub(crate) fn repository_id(&self) -> &str { + &self.repository_id + } + + pub(crate) fn replay_state( + &self, + availability: ReceiptReplayAvailabilityV1, + ) -> ReceiptReproducibilityState { + let dependencies = [ + availability.digest_key, + availability.source_snapshot, + availability.policy, + availability.index_snapshot, + ]; + let dependency_state = if dependencies.contains(&ReceiptDependencyAvailability::Missing) { + ReceiptReproducibilityState::NonReproducible + } else if dependencies.contains(&ReceiptDependencyAvailability::Stale) { + ReceiptReproducibilityState::Stale + } else { + ReceiptReproducibilityState::Reproducible + }; + more_severe_replay_state(self.reproducibility_state, dependency_state) + } +} + +const fn replay_state_severity(state: ReceiptReproducibilityState) -> u8 { + match state { + ReceiptReproducibilityState::Reproducible => 0, + ReceiptReproducibilityState::Stale => 1, + ReceiptReproducibilityState::Expired => 2, + ReceiptReproducibilityState::NonReproducible => 3, + } +} + +const fn more_severe_replay_state( + stored: ReceiptReproducibilityState, + current: ReceiptReproducibilityState, +) -> ReceiptReproducibilityState { + if replay_state_severity(stored) >= replay_state_severity(current) { + stored + } else { + current + } +} + +fn validate_stored_selected(selected: &[ReceiptSelectionV1]) -> Result<(), ReceiptValidationError> { + if selected.len() > MAX_SELECTED { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::Selected, + )); + } + let mut identities = HashSet::with_capacity(selected.len()); + for (index, item) in selected.iter().enumerate() { + if usize::try_from(item.order).ok() != Some(index) + || !valid_bounded(&item.object_id, MAX_OBJECT_ID_BYTES) + || !valid_object_id(&item.revision_oid) + || !valid_bounded(&item.summary_key, MAX_SUMMARY_KEY_BYTES) + || item.reason_codes.is_empty() + || item.reason_codes.len() > MAX_REASON_CODES + || item + .reason_codes + .iter() + .any(|reason| !valid_reason_code(reason, MAX_REASON_CODE_BYTES)) + || item.score_components.len() > MAX_SCORE_COMPONENTS + || item + .score_components + .keys() + .any(|key| !valid_reason_code(key, MAX_REASON_CODE_BYTES)) + || !identities.insert((item.object_id.as_str(), item.revision_oid.as_str())) + { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::Selected, + )); + } + } + Ok(()) +} + +fn valid_receipt_hmac(value: &str, key_id: Uuid) -> bool { + let prefix = format!("hmac-sha256:{key_id}:"); + value.len() == prefix.len() + 64 + && value.starts_with(&prefix) + && value[prefix.len()..] + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn validate_snapshot(snapshot: &BTreeMap) -> Result<(), ReceiptValidationError> { + if snapshot.len() > MAX_SNAPSHOT_ENTRIES + || snapshot.iter().any(|(key, value)| { + !valid_reason_code(key, MAX_SNAPSHOT_KEY_BYTES) || !valid_object_id(value) + }) + { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::Snapshot, + )); + } + Ok(()) +} + +fn validate_selected( + selected: Vec, +) -> Result, ReceiptValidationError> { + if selected.len() > MAX_SELECTED { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::Selected, + )); + } + let mut identities = HashSet::with_capacity(selected.len()); + let mut stored = Vec::with_capacity(selected.len()); + for (index, item) in selected.into_iter().enumerate() { + if item.sensitivity == ReceiptSensitivity::SecretLike { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::SecretLikeSelection, + )); + } + if usize::try_from(item.order).ok() != Some(index) + || !valid_bounded(&item.object_id, MAX_OBJECT_ID_BYTES) + || !valid_object_id(&item.revision_oid) + || !valid_bounded(&item.summary_key, MAX_SUMMARY_KEY_BYTES) + || item.reason_codes.is_empty() + || item.reason_codes.len() > MAX_REASON_CODES + || item + .reason_codes + .iter() + .any(|reason| !valid_reason_code(reason, MAX_REASON_CODE_BYTES)) + || item.score_components.len() > MAX_SCORE_COMPONENTS + || item + .score_components + .keys() + .any(|key| !valid_reason_code(key, MAX_REASON_CODE_BYTES)) + || !identities.insert((item.object_id.clone(), item.revision_oid.clone())) + { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::Selected, + )); + } + stored.push(ReceiptSelectionV1 { + object_id: item.object_id, + revision_oid: item.revision_oid, + summary_key: item.summary_key, + order: item.order, + reason_codes: item.reason_codes, + score_components: item.score_components, + }); + } + Ok(stored) +} + +fn validate_omissions(omissions: &[ReceiptOmissionV1]) -> Result<(), ReceiptValidationError> { + let mut reasons = HashSet::with_capacity(omissions.len()); + if omissions.len() > MAX_OMISSIONS + || omissions.iter().any(|omission| { + !valid_reason_code(&omission.reason_code, MAX_REASON_CODE_BYTES) + || !reasons.insert(omission.reason_code.as_str()) + }) + { + return Err(ReceiptValidationError::new( + ReceiptValidationErrorKind::Omissions, + )); + } + Ok(()) +} + +fn validate_json_envelope( + value: &T, + max_bytes: usize, + kind: ReceiptValidationErrorKind, +) -> Result<(), ReceiptValidationError> { + let encoded = serde_json::to_vec(value).map_err(|_| ReceiptValidationError::new(kind))?; + if !(2..=max_bytes).contains(&encoded.len()) { + return Err(ReceiptValidationError::new(kind)); + } + Ok(()) +} + +fn valid_bounded(value: &str, max_bytes: usize) -> bool { + !value.is_empty() + && value.len() <= max_bytes + && value.trim() == value + && !value.chars().any(char::is_control) +} + +fn valid_reason_code(value: &str, max_bytes: usize) -> bool { + valid_bounded(value, max_bytes) + && value.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || matches!(byte, b'_' | b'-' | b'.') + }) +} + +fn valid_object_id(value: &str) -> bool { + matches!(value.len(), 40 | 64) + && value + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +fn valid_sha256(value: &str) -> bool { + value.len() == 71 + && value.starts_with("sha256:") + && value[7..] + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ReceiptValidationErrorKind { + ReceiptId, + SchemaVersion, + DigestVersion, + DigestKeyMismatch, + DigestEncoding, + RepositoryId, + CodeCommit, + BranchRef, + Snapshot, + PolicyHash, + SelectorVersion, + TokenBudget, + Selected, + SecretLikeSelection, + Omissions, + BundleHash, + PersistedEnvelope, +} + +#[derive(Debug, Error)] +#[error("context selection receipt is invalid ({kind:?})")] +pub(crate) struct ReceiptValidationError { + kind: ReceiptValidationErrorKind, +} + +impl ReceiptValidationError { + const fn new(kind: ReceiptValidationErrorKind) -> Self { + Self { kind } + } + + pub(crate) const fn kind(&self) -> ReceiptValidationErrorKind { + self.kind + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use chrono::{DateTime, Utc}; + use uuid::Uuid; + + use super::*; + use crate::internal::ai::keyed_digest::RepositoryKeyedDigest; + + fn fields( + provider: &RepositoryKeyedDigest, + sensitivity: ReceiptSensitivity, + ) -> ReceiptDraftFieldsV1 { + let mut source_heads = BTreeMap::new(); + source_heads.insert("memory_repo".to_string(), "a".repeat(40)); + let mut projection_watermarks = BTreeMap::new(); + projection_watermarks.insert("memory_repo".to_string(), "a".repeat(40)); + let mut score_components = BTreeMap::new(); + score_components.insert("bm25".to_string(), -42); + + ReceiptDraftFieldsV1 { + source_kind: ReceiptSourceKind::Memory, + repository_id: "repo-42".to_string(), + principal_digest: provider + .principal_digest(b"agent:alice") + .expect("principal digest"), + query_digest: provider + .query_digest(b"normalized query") + .expect("query digest"), + effective_at: "2026-08-24T00:00:00Z" + .parse::>() + .expect("effective timestamp"), + code_commit: Some("b".repeat(40)), + full_branch_ref: Some("refs/heads/feature/memory".to_string()), + source_heads, + projection_watermarks, + policy_hash: format!("sha256:{}", "c".repeat(64)), + selector_version: "memory-v1".to_string(), + selected: vec![ReceiptSelectionInputV1 { + object_id: "episode:task-42".to_string(), + revision_oid: "d".repeat(40), + summary_key: "episodic/tasks/task-42".to_string(), + order: 0, + reason_codes: vec!["bm25_match".to_string()], + score_components, + sensitivity, + }], + omissions: vec![ReceiptOmissionV1 { + reason_code: "budget".to_string(), + count: 2, + }], + token_budget: 1_600, + bundle_hash: format!("sha256:{}", "e".repeat(64)), + reproducibility_state: ReceiptReproducibilityState::Reproducible, + frame_id: Some(Uuid::now_v7()), + } + } + + #[test] + fn draft_accepts_only_bounded_structured_fields_and_purpose_locked_digests() { + let key_id = Uuid::parse_str("123e4567-e89b-42d3-a456-426614174000").expect("fixed UUIDv4"); + let provider = RepositoryKeyedDigest::for_receipt_tests( + "repo-42", + key_id, + [0x31; 32], + "receipt-domain-test-key", + ); + let draft = + ContextSelectionReceiptDraftV1::new(fields(&provider, ReceiptSensitivity::Allowed)) + .expect("valid structured receipt draft"); + + assert_eq!(draft.schema_version(), 1); + assert_eq!(draft.digest_key_id(), key_id); + assert_eq!(draft.source_kind(), ReceiptSourceKind::Memory); + assert_eq!(draft.selected().len(), 1); + assert_eq!(draft.omissions().len(), 1); + } + + #[test] + fn draft_rejects_secret_like_selection_and_mixed_digest_keys() { + let first_key = + Uuid::parse_str("123e4567-e89b-42d3-a456-426614174000").expect("first UUIDv4"); + let second_key = + Uuid::parse_str("223e4567-e89b-42d3-a456-426614174000").expect("second UUIDv4"); + let first = RepositoryKeyedDigest::for_receipt_tests( + "repo-42", + first_key, + [0x32; 32], + "receipt-domain-first-key", + ); + let second = RepositoryKeyedDigest::for_receipt_tests( + "repo-42", + second_key, + [0x33; 32], + "receipt-domain-second-key", + ); + + let secret_error = + ContextSelectionReceiptDraftV1::new(fields(&first, ReceiptSensitivity::SecretLike)) + .err() + .expect("SecretLike content cannot enter a receipt"); + assert_eq!( + secret_error.kind(), + ReceiptValidationErrorKind::SecretLikeSelection + ); + + let mut mixed = fields(&first, ReceiptSensitivity::Allowed); + mixed.query_digest = second + .query_digest(b"normalized query") + .expect("second query digest"); + let mixed_error = ContextSelectionReceiptDraftV1::new(mixed) + .err() + .expect("receipt digests must use the same repository key"); + assert_eq!( + mixed_error.kind(), + ReceiptValidationErrorKind::DigestKeyMismatch + ); + } + + #[test] + fn draft_rejects_schema_incompatible_branch_and_json_envelope() { + let key_id = Uuid::parse_str("123e4567-e89b-42d3-a456-426614174000").expect("fixed UUIDv4"); + let provider = RepositoryKeyedDigest::for_receipt_tests( + "repo-42", + key_id, + [0x35; 32], + "receipt-domain-envelope-key", + ); + + let mut empty_branch = fields(&provider, ReceiptSensitivity::Allowed); + empty_branch.full_branch_ref = Some("refs/heads/".to_string()); + assert_eq!( + ContextSelectionReceiptDraftV1::new(empty_branch) + .err() + .expect("empty branch name must be rejected") + .kind(), + ReceiptValidationErrorKind::BranchRef + ); + + let mut oversized = fields(&provider, ReceiptSensitivity::Allowed); + let template = oversized.selected.pop().expect("selection template"); + oversized.selected = (0..MAX_SELECTED) + .map(|order| ReceiptSelectionInputV1 { + object_id: format!("episode:{order:04}"), + revision_oid: format!("{order:040x}"), + summary_key: "s".repeat(MAX_SUMMARY_KEY_BYTES), + order: u32::try_from(order).expect("bounded order"), + reason_codes: template.reason_codes.clone(), + score_components: template.score_components.clone(), + sensitivity: ReceiptSensitivity::Allowed, + }) + .collect(); + assert_eq!( + ContextSelectionReceiptDraftV1::new(oversized) + .err() + .expect("serialized selection envelope must fit the SQLite CHECK") + .kind(), + ReceiptValidationErrorKind::Selected + ); + + assert!( + validate_json_envelope( + &Vec::::new(), + 2, + ReceiptValidationErrorKind::Selected + ) + .is_ok() + ); + assert!( + validate_json_envelope(&vec!["x"], 4, ReceiptValidationErrorKind::Selected).is_err() + ); + assert!(validate_json_envelope(&"xxxx", 6, ReceiptValidationErrorKind::Selected).is_ok()); + assert!(validate_json_envelope(&"xxxxx", 6, ReceiptValidationErrorKind::Selected).is_err()); + } + + #[test] + fn replay_state_distinguishes_stale_from_missing_dependencies() { + let key_id = Uuid::parse_str("123e4567-e89b-42d3-a456-426614174000").expect("fixed UUIDv4"); + let provider = RepositoryKeyedDigest::for_receipt_tests( + "repo-42", + key_id, + [0x34; 32], + "receipt-domain-replay-key", + ); + let receipt = ContextSelectionReceiptV1::from_draft( + Uuid::now_v7(), + Utc::now(), + ContextSelectionReceiptDraftV1::new(fields(&provider, ReceiptSensitivity::Allowed)) + .expect("valid receipt draft"), + ); + + assert_eq!( + receipt.replay_state(ReceiptReplayAvailabilityV1::all_exact()), + ReceiptReproducibilityState::Reproducible + ); + assert_eq!( + receipt.replay_state(ReceiptReplayAvailabilityV1 { + policy: ReceiptDependencyAvailability::Stale, + ..ReceiptReplayAvailabilityV1::all_exact() + }), + ReceiptReproducibilityState::Stale + ); + for availability in [ + ReceiptReplayAvailabilityV1 { + digest_key: ReceiptDependencyAvailability::Missing, + ..ReceiptReplayAvailabilityV1::all_exact() + }, + ReceiptReplayAvailabilityV1 { + source_snapshot: ReceiptDependencyAvailability::Missing, + ..ReceiptReplayAvailabilityV1::all_exact() + }, + ReceiptReplayAvailabilityV1 { + policy: ReceiptDependencyAvailability::Missing, + ..ReceiptReplayAvailabilityV1::all_exact() + }, + ReceiptReplayAvailabilityV1 { + index_snapshot: ReceiptDependencyAvailability::Missing, + ..ReceiptReplayAvailabilityV1::all_exact() + }, + ] { + assert_eq!( + receipt.replay_state(availability), + ReceiptReproducibilityState::NonReproducible + ); + } + + for stored in [ + ReceiptReproducibilityState::Expired, + ReceiptReproducibilityState::NonReproducible, + ] { + let mut stored_fields = fields(&provider, ReceiptSensitivity::Allowed); + stored_fields.reproducibility_state = stored; + let stored_receipt = ContextSelectionReceiptV1::from_draft( + Uuid::now_v7(), + Utc::now(), + ContextSelectionReceiptDraftV1::new(stored_fields).expect("valid stored state"), + ); + assert_eq!( + stored_receipt.replay_state(ReceiptReplayAvailabilityV1 { + policy: ReceiptDependencyAvailability::Stale, + ..ReceiptReplayAvailabilityV1::all_exact() + }), + stored, + "replay classification cannot reduce an existing terminal severity" + ); + } + } +} diff --git a/src/internal/ai/context_budget/receipt_store.rs b/src/internal/ai/context_budget/receipt_store.rs new file mode 100644 index 000000000..eb4a060e5 --- /dev/null +++ b/src/internal/ai/context_budget/receipt_store.rs @@ -0,0 +1,957 @@ +//! SQLite owner for the shared context selection receipt ledger. + +use std::sync::Arc; + +use chrono::{DateTime, SecondsFormat, TimeDelta, Utc}; +use sea_orm::{ + ConnectionTrait, DatabaseConnection, DatabaseTransaction, DbErr, QueryResult, Statement, +}; +use thiserror::Error; +use uuid::Uuid; + +use super::receipt::{ + ContextSelectionReceiptDraftV1, ContextSelectionReceiptV1, PersistedReceiptFieldsV1, + ReceiptReproducibilityState, ReceiptSourceKind, +}; +use crate::internal::{ai::keyed_digest::RepositoryKeyedDigest, db, workspace::RepoIdentity}; + +const DEFAULT_RETENTION_DAYS: i64 = 30; +const MAX_RECEIPTS_PER_REPOSITORY: i64 = 10_000; + +pub(crate) struct ReceiptStore<'database> { + database: &'database DatabaseConnection, + repository_id: String, + digest_key_id: Uuid, + _digest_provider: Arc, + clock: fn() -> DateTime, +} + +impl<'database> ReceiptStore<'database> { + pub(crate) async fn new( + database: &'database DatabaseConnection, + digest_provider: Arc, + ) -> Result { + Self::with_clock(database, digest_provider, Utc::now).await + } + + async fn with_clock( + database: &'database DatabaseConnection, + digest_provider: Arc, + clock: fn() -> DateTime, + ) -> Result { + let repository_identity = RepoIdentity::resolve(database) + .await + .map_err(|_| ReceiptStoreError::identity())?; + if repository_identity.as_str() != digest_provider.repository_id() { + return Err(ReceiptStoreError::repository_mismatch()); + } + digest_provider + .validate_for_connection(database) + .await + .map_err(|_| ReceiptStoreError::digest_key_mismatch())?; + let repository_id = repository_identity.as_str().to_string(); + let digest_key_id = digest_provider.key_id(); + Ok(Self { + database, + repository_id, + digest_key_id, + _digest_provider: digest_provider, + clock, + }) + } + + pub(crate) async fn append( + &self, + draft: ContextSelectionReceiptDraftV1, + ) -> Result { + if draft.repository_id() != self.repository_id { + return Err(ReceiptStoreError::repository_mismatch()); + } + if draft.digest_key_id() != self.digest_key_id { + return Err(ReceiptStoreError::digest_key_mismatch()); + } + let now = (self.clock)(); + let receipt = ContextSelectionReceiptV1::from_draft(Uuid::now_v7(), now, draft); + let transaction = db::begin_write_transaction(self.database) + .await + .map_err(|_| ReceiptStoreError::storage())?; + let result = async { + self.validate_transaction_binding(&transaction).await?; + append_in_transaction(&transaction, &receipt).await + } + .await; + match result { + Ok(()) => { + transaction + .commit() + .await + .map_err(|_| ReceiptStoreError::storage())?; + Ok(receipt) + } + Err(error) => match transaction.rollback().await { + Ok(()) => Err(error), + Err(_) => Err(ReceiptStoreError::storage()), + }, + } + } + + async fn validate_transaction_binding( + &self, + transaction: &DatabaseTransaction, + ) -> Result<(), ReceiptStoreError> { + let identity = RepoIdentity::resolve(transaction) + .await + .map_err(|_| ReceiptStoreError::identity())?; + if identity.as_str() != self.repository_id + || identity.as_str() != self._digest_provider.repository_id() + { + return Err(ReceiptStoreError::repository_mismatch()); + } + self._digest_provider + .validate_for_connection(transaction) + .await + .map_err(|_| ReceiptStoreError::digest_key_mismatch()) + } + + pub(crate) async fn lookup( + &self, + receipt_id: Uuid, + ) -> Result { + let row = self + .database + .query_one_raw(Statement::from_sql_and_values( + self.database.get_database_backend(), + "SELECT receipt_id, schema_version, source_kind, repository_id, + digest_key_id, principal_hmac, query_hmac, effective_at, + code_commit, full_branch_ref, source_heads_json, + projection_watermarks_json, policy_hash, selector_version, + token_budget, selected_json, omissions_json, bundle_hash, + reproducibility_state, frame_id, recorded_at + FROM context_selection_receipt + WHERE repository_id = ? AND receipt_id = ?", + [ + self.repository_id.as_str().into(), + receipt_id.to_string().into(), + ], + )) + .await + .map_err(|_| ReceiptStoreError::storage())?; + if let Some(row) = row { + return decode_receipt(row).map(|receipt| ReceiptLookup::Found(Box::new(receipt))); + } + + let retention = self + .database + .query_one_raw(Statement::from_sql_and_values( + self.database.get_database_backend(), + "SELECT pruned_before + FROM context_selection_receipt_retention + WHERE repository_id = ?", + [self.repository_id.as_str().into()], + )) + .await + .map_err(|_| ReceiptStoreError::storage())?; + let pruned_before = retention + .map(|row| { + row.try_get::>("", "pruned_before") + .map_err(|_| ReceiptStoreError::corrupt()) + }) + .transpose()? + .flatten() + .map(|value| parse_timestamp(&value)) + .transpose()?; + if pruned_before.is_some_and(|watermark| { + uuid_v7_timestamp(receipt_id).is_some_and(|timestamp| timestamp < watermark) + }) { + Ok(ReceiptLookup::Expired) + } else { + Ok(ReceiptLookup::NotFound) + } + } +} + +async fn append_in_transaction( + transaction: &DatabaseTransaction, + receipt: &ContextSelectionReceiptV1, +) -> Result<(), ReceiptStoreError> { + let source_heads = serde_json::to_string(&receipt.source_heads) + .map_err(|_| ReceiptStoreError::serialization())?; + let projection_watermarks = serde_json::to_string(&receipt.projection_watermarks) + .map_err(|_| ReceiptStoreError::serialization())?; + let selected = + serde_json::to_string(&receipt.selected).map_err(|_| ReceiptStoreError::serialization())?; + let omissions = serde_json::to_string(&receipt.omissions) + .map_err(|_| ReceiptStoreError::serialization())?; + let recorded_at = format_timestamp(receipt.recorded_at); + let effective_at = format_timestamp(receipt.effective_at); + let token_budget = + i64::try_from(receipt.token_budget).map_err(|_| ReceiptStoreError::serialization())?; + + transaction + .execute_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "INSERT INTO context_selection_receipt ( + receipt_id, schema_version, source_kind, repository_id, + digest_key_id, principal_hmac, query_hmac, effective_at, + code_commit, full_branch_ref, source_heads_json, + projection_watermarks_json, policy_hash, selector_version, + token_budget, selected_json, omissions_json, bundle_hash, + reproducibility_state, frame_id, recorded_at + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + [ + receipt.receipt_id.to_string().into(), + i64::from(receipt.schema_version).into(), + receipt.source_kind.as_str().into(), + receipt.repository_id.as_str().into(), + receipt.digest_key_id.to_string().into(), + receipt.principal_hmac.as_str().into(), + receipt.query_hmac.as_str().into(), + effective_at.into(), + receipt.code_commit.as_deref().into(), + receipt.full_branch_ref.as_deref().into(), + source_heads.into(), + projection_watermarks.into(), + receipt.policy_hash.as_str().into(), + receipt.selector_version.as_str().into(), + token_budget.into(), + selected.into(), + omissions.into(), + receipt.bundle_hash.as_str().into(), + receipt.reproducibility_state.as_str().into(), + receipt.frame_id.map(|value| value.to_string()).into(), + recorded_at.clone().into(), + ], + )) + .await + .map_err(|_| ReceiptStoreError::storage())?; + + prune_and_update_retention( + transaction, + &receipt.repository_id, + receipt.recorded_at, + &recorded_at, + ) + .await +} + +async fn prune_and_update_retention( + transaction: &DatabaseTransaction, + repository_id: &str, + recorded_at: DateTime, + recorded_at_text: &str, +) -> Result<(), ReceiptStoreError> { + let cutoff = format_timestamp(recorded_at - TimeDelta::days(DEFAULT_RETENTION_DAYS)); + let candidate = transaction + .query_one_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "SELECT COUNT(*) AS count, MAX(recorded_at) AS pruned_before + FROM context_selection_receipt + WHERE repository_id = ? + AND receipt_id NOT IN ( + SELECT receipt_id + FROM context_selection_receipt + WHERE repository_id = ? AND recorded_at >= ? + ORDER BY recorded_at DESC, receipt_id DESC + LIMIT 10000 + )", + [ + repository_id.into(), + repository_id.into(), + cutoff.clone().into(), + ], + )) + .await + .map_err(|_| ReceiptStoreError::storage())? + .ok_or_else(ReceiptStoreError::corrupt)?; + let pruned_count: i64 = candidate + .try_get("", "count") + .map_err(|_| ReceiptStoreError::corrupt())?; + let pruned_before: Option = candidate + .try_get("", "pruned_before") + .map_err(|_| ReceiptStoreError::corrupt())?; + if pruned_count > 0 { + transaction + .execute_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "DELETE FROM context_selection_receipt + WHERE repository_id = ? + AND receipt_id NOT IN ( + SELECT receipt_id + FROM context_selection_receipt + WHERE repository_id = ? AND recorded_at >= ? + ORDER BY recorded_at DESC, receipt_id DESC + LIMIT 10000 + )", + [repository_id.into(), repository_id.into(), cutoff.into()], + )) + .await + .map_err(|_| ReceiptStoreError::storage())?; + } + + let retained_rows: i64 = transaction + .query_one_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "SELECT COUNT(*) AS count FROM context_selection_receipt + WHERE repository_id = ?", + [repository_id.into()], + )) + .await + .map_err(|_| ReceiptStoreError::storage())? + .ok_or_else(ReceiptStoreError::corrupt)? + .try_get("", "count") + .map_err(|_| ReceiptStoreError::corrupt())?; + if retained_rows > MAX_RECEIPTS_PER_REPOSITORY { + return Err(ReceiptStoreError::corrupt()); + } + let last_pruned_at = pruned_before.as_ref().map(|_| recorded_at_text); + transaction + .execute_raw(Statement::from_sql_and_values( + transaction.get_database_backend(), + "INSERT INTO context_selection_receipt_retention ( + repository_id, pruned_before, last_pruned_at, retained_rows + ) VALUES (?, ?, ?, ?) + ON CONFLICT(repository_id) DO UPDATE SET + pruned_before = CASE + WHEN excluded.pruned_before IS NULL + THEN context_selection_receipt_retention.pruned_before + WHEN context_selection_receipt_retention.pruned_before IS NULL + OR context_selection_receipt_retention.pruned_before < excluded.pruned_before + THEN excluded.pruned_before + ELSE context_selection_receipt_retention.pruned_before + END, + last_pruned_at = CASE + WHEN excluded.pruned_before IS NULL + THEN context_selection_receipt_retention.last_pruned_at + ELSE excluded.last_pruned_at + END, + retained_rows = excluded.retained_rows", + [ + repository_id.into(), + pruned_before.into(), + last_pruned_at.into(), + retained_rows.into(), + ], + )) + .await + .map_err(|_| ReceiptStoreError::storage())?; + Ok(()) +} + +fn decode_receipt(row: QueryResult) -> Result { + let receipt_id = parse_uuid(row.try_get("", "receipt_id")?)?; + let schema_version = u32::try_from(row.try_get::("", "schema_version")?) + .map_err(|_| ReceiptStoreError::corrupt())?; + let source_kind = ReceiptSourceKind::parse(&row.try_get::("", "source_kind")?) + .ok_or_else(ReceiptStoreError::corrupt)?; + let digest_key_id = parse_uuid(row.try_get("", "digest_key_id")?)?; + let token_budget = u64::try_from(row.try_get::("", "token_budget")?) + .map_err(|_| ReceiptStoreError::corrupt())?; + let frame_id = row + .try_get::>("", "frame_id")? + .map(parse_uuid) + .transpose()?; + let reproducibility_state = + ReceiptReproducibilityState::parse(&row.try_get::("", "reproducibility_state")?) + .ok_or_else(ReceiptStoreError::corrupt)?; + + ContextSelectionReceiptV1::from_persisted(PersistedReceiptFieldsV1 { + receipt_id, + schema_version, + source_kind, + repository_id: row.try_get("", "repository_id")?, + digest_key_id, + principal_hmac: row.try_get("", "principal_hmac")?, + query_hmac: row.try_get("", "query_hmac")?, + effective_at: parse_timestamp(&row.try_get::("", "effective_at")?)?, + code_commit: row.try_get("", "code_commit")?, + full_branch_ref: row.try_get("", "full_branch_ref")?, + source_heads: decode_json(&row.try_get::("", "source_heads_json")?)?, + projection_watermarks: decode_json( + &row.try_get::("", "projection_watermarks_json")?, + )?, + policy_hash: row.try_get("", "policy_hash")?, + selector_version: row.try_get("", "selector_version")?, + token_budget, + selected: decode_json(&row.try_get::("", "selected_json")?)?, + omissions: decode_json(&row.try_get::("", "omissions_json")?)?, + bundle_hash: row.try_get("", "bundle_hash")?, + reproducibility_state, + frame_id, + recorded_at: parse_timestamp(&row.try_get::("", "recorded_at")?)?, + }) + .map_err(|_| ReceiptStoreError::corrupt()) +} + +fn decode_json(value: &str) -> Result { + serde_json::from_str(value).map_err(|_| ReceiptStoreError::corrupt()) +} + +fn parse_uuid(value: String) -> Result { + Uuid::parse_str(&value).map_err(|_| ReceiptStoreError::corrupt()) +} + +fn parse_timestamp(value: &str) -> Result, ReceiptStoreError> { + DateTime::parse_from_rfc3339(value) + .map(|timestamp| timestamp.with_timezone(&Utc)) + .map_err(|_| ReceiptStoreError::corrupt()) +} + +fn format_timestamp(value: DateTime) -> String { + value.to_rfc3339_opts(SecondsFormat::Nanos, true) +} + +fn uuid_v7_timestamp(value: Uuid) -> Option> { + if value.get_version_num() != 7 { + return None; + } + let bytes = value.as_bytes(); + let milliseconds = bytes[..6].iter().fold(0_u64, |accumulator, byte| { + (accumulator << 8) | u64::from(*byte) + }); + DateTime::from_timestamp_millis(i64::try_from(milliseconds).ok()?) +} + +pub(crate) enum ReceiptLookup { + Found(Box), + Expired, + NotFound, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum ReceiptStoreErrorKind { + Identity, + RepositoryMismatch, + DigestKeyMismatch, + Serialization, + Storage, + Corrupt, +} + +#[derive(Debug, Error)] +#[error("context selection receipt storage failed ({kind:?})")] +pub(crate) struct ReceiptStoreError { + kind: ReceiptStoreErrorKind, +} + +impl ReceiptStoreError { + const fn new(kind: ReceiptStoreErrorKind) -> Self { + Self { kind } + } + + const fn identity() -> Self { + Self::new(ReceiptStoreErrorKind::Identity) + } + + const fn repository_mismatch() -> Self { + Self::new(ReceiptStoreErrorKind::RepositoryMismatch) + } + + const fn digest_key_mismatch() -> Self { + Self::new(ReceiptStoreErrorKind::DigestKeyMismatch) + } + + const fn serialization() -> Self { + Self::new(ReceiptStoreErrorKind::Serialization) + } + + const fn storage() -> Self { + Self::new(ReceiptStoreErrorKind::Storage) + } + + const fn corrupt() -> Self { + Self::new(ReceiptStoreErrorKind::Corrupt) + } + + pub(crate) const fn kind(&self) -> ReceiptStoreErrorKind { + self.kind + } +} + +impl From for ReceiptStoreError { + fn from(_: DbErr) -> Self { + Self::corrupt() + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use chrono::{DateTime, Utc}; + use sea_orm::{ConnectionTrait, Database, Statement}; + use uuid::Uuid; + + use super::*; + use crate::internal::{ + ai::{ + context_budget::receipt::{ + ContextSelectionReceiptDraftV1, ReceiptDraftFieldsV1, ReceiptOmissionV1, + ReceiptReproducibilityState, ReceiptSelectionInputV1, ReceiptSensitivity, + ReceiptSourceKind, + }, + keyed_digest::RepositoryKeyedDigest, + }, + config::{ConfigKv, MEMORY_KEYED_DIGEST_CONFIG_KEY}, + db::migration::run_builtin_migrations, + }; + + const TEST_CIPHERTEXT: &str = "receipt-store-test-ciphertext"; + + async fn database( + repository_id: &str, + ) -> (sea_orm::DatabaseConnection, Arc) { + let database = Database::connect("sqlite::memory:") + .await + .expect("connect receipt test database"); + run_builtin_migrations(&database) + .await + .expect("apply receipt migration"); + ConfigKv::set_with_conn(&database, "libra.repoid", repository_id, false) + .await + .expect("seed canonical repository identity"); + assert!( + ConfigKv::insert_vault_internal_if_absent_with_conn( + &database, + MEMORY_KEYED_DIGEST_CONFIG_KEY, + TEST_CIPHERTEXT, + ) + .await + .expect("seed repository digest config") + ); + let provider = Arc::new(RepositoryKeyedDigest::for_receipt_tests( + repository_id, + digest_key_id(), + [0x41; 32], + TEST_CIPHERTEXT, + )); + (database, provider) + } + + fn digest_key_id() -> Uuid { + Uuid::parse_str("123e4567-e89b-42d3-a456-426614174000").expect("fixed UUIDv4") + } + + fn fixed_now() -> DateTime { + "2026-08-24T12:00:00Z" + .parse::>() + .expect("fixed receipt clock") + } + + async fn store<'database>( + database: &'database sea_orm::DatabaseConnection, + provider: Arc, + ) -> ReceiptStore<'database> { + ReceiptStore::with_clock(database, provider, fixed_now) + .await + .expect("valid receipt store identity") + } + + fn draft( + repository_id: &str, + provider: &RepositoryKeyedDigest, + ) -> ContextSelectionReceiptDraftV1 { + let mut source_heads = BTreeMap::new(); + source_heads.insert("memory_repo".to_string(), "a".repeat(40)); + let mut projection_watermarks = BTreeMap::new(); + projection_watermarks.insert("memory_repo".to_string(), "a".repeat(40)); + + ContextSelectionReceiptDraftV1::new(ReceiptDraftFieldsV1 { + source_kind: ReceiptSourceKind::Memory, + repository_id: repository_id.to_string(), + principal_digest: provider + .principal_digest(b"agent:alice") + .expect("principal digest"), + query_digest: provider + .query_digest(b"normalized query") + .expect("query digest"), + effective_at: "2026-08-24T00:00:00Z" + .parse::>() + .expect("effective timestamp"), + code_commit: Some("b".repeat(40)), + full_branch_ref: Some("refs/heads/feature/memory".to_string()), + source_heads, + projection_watermarks, + policy_hash: format!("sha256:{}", "c".repeat(64)), + selector_version: "memory-v1".to_string(), + selected: vec![ReceiptSelectionInputV1 { + object_id: "episode:task-42".to_string(), + revision_oid: "d".repeat(40), + summary_key: "episodic/tasks/task-42".to_string(), + order: 0, + reason_codes: vec!["bm25_match".to_string()], + score_components: BTreeMap::new(), + sensitivity: ReceiptSensitivity::Allowed, + }], + omissions: vec![ReceiptOmissionV1 { + reason_code: "budget".to_string(), + count: 2, + }], + token_budget: 1_600, + bundle_hash: format!("sha256:{}", "e".repeat(64)), + reproducibility_state: ReceiptReproducibilityState::Reproducible, + frame_id: None, + }) + .expect("valid receipt draft") + } + + #[tokio::test] + async fn append_and_retention_metadata_commit_atomically() { + let (database, provider) = database("repo-42").await; + let store = store(&database, Arc::clone(&provider)).await; + let receipt = store + .append(draft("repo-42", &provider)) + .await + .expect("append receipt"); + assert_eq!(receipt.receipt_id().get_version_num(), 7); + assert_eq!(receipt.repository_id(), "repo-42"); + + match store + .lookup(receipt.receipt_id()) + .await + .expect("lookup stored receipt") + { + ReceiptLookup::Found(found) => assert_eq!(found.receipt_id(), receipt.receipt_id()), + ReceiptLookup::Expired | ReceiptLookup::NotFound => { + panic!("stored receipt disappeared") + } + } + + let retained_rows: i64 = database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + "SELECT retained_rows FROM context_selection_receipt_retention + WHERE repository_id = 'repo-42'" + .to_string(), + )) + .await + .expect("read retention metadata") + .expect("retention row") + .try_get("", "retained_rows") + .expect("retained row count"); + assert_eq!(retained_rows, 1); + + database + .execute_unprepared( + "UPDATE context_selection_receipt + SET recorded_at = '2026-07-24T11:59:59.000000000Z' + WHERE repository_id = 'repo-42'", + ) + .await + .expect("make the existing receipt old enough to prune"); + + database + .execute_unprepared( + "CREATE TRIGGER receipt_retention_test_abort + BEFORE INSERT ON context_selection_receipt_retention + BEGIN SELECT RAISE(ABORT, 'forced retention failure'); END", + ) + .await + .expect("install deterministic retention failure"); + let error = store + .append(draft("repo-42", &provider)) + .await + .err() + .expect("retention failure must roll back the receipt insert"); + assert_eq!(error.kind(), ReceiptStoreErrorKind::Storage); + let failed_rows: i64 = database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + "SELECT COUNT(*) AS count FROM context_selection_receipt + WHERE repository_id = 'repo-42'" + .to_string(), + )) + .await + .expect("count failed append rows") + .expect("count row") + .try_get("", "count") + .expect("count value"); + assert_eq!( + failed_rows, 1, + "rollback must restore the old row deleted by retention and remove the new row" + ); + assert!(matches!( + store + .lookup(receipt.receipt_id()) + .await + .expect("old receipt remains readable after rollback"), + ReceiptLookup::Found(_) + )); + let retention_after_failure = database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + "SELECT pruned_before, retained_rows + FROM context_selection_receipt_retention + WHERE repository_id = 'repo-42'" + .to_string(), + )) + .await + .expect("read retention metadata after rollback") + .expect("retention metadata row"); + let pruned_before: Option = retention_after_failure + .try_get("", "pruned_before") + .expect("pruned-before watermark"); + let retained_rows: i64 = retention_after_failure + .try_get("", "retained_rows") + .expect("retained row count"); + assert_eq!((pruned_before, retained_rows), (None, 1)); + } + + #[tokio::test] + async fn receipt_retention_20000_rows_and_lookup_outcomes() { + let (database, provider) = database("repo-capacity").await; + database + .execute_unprepared( + "WITH RECURSIVE sequence(value) AS ( + SELECT 0 + UNION ALL + SELECT value + 1 FROM sequence WHERE value < 19999 + ) + INSERT INTO context_selection_receipt ( + receipt_id, schema_version, source_kind, repository_id, + digest_key_id, principal_hmac, query_hmac, effective_at, + source_heads_json, projection_watermarks_json, policy_hash, + selector_version, token_budget, selected_json, omissions_json, + bundle_hash, reproducibility_state, recorded_at + ) + SELECT + printf('0198a7e0-%04x-7%03x-8000-%012x', + (value >> 12) & 65535, value & 4095, value), + 1, 'memory', 'repo-capacity', + '123e4567-e89b-42d3-a456-426614174000', + 'hmac-sha256:123e4567-e89b-42d3-a456-426614174000:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'hmac-sha256:123e4567-e89b-42d3-a456-426614174000:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + '2026-08-24T00:00:00.000000000Z', + '{\"memory_repo\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"}', + '{\"memory_repo\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"}', + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'memory-v1', 1600, '[]', '[]', + 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', + 'reproducible', '2026-08-24T00:00:00.000000000Z' + FROM sequence", + ) + .await + .expect("seed the bounded capacity fixture"); + + let store = store(&database, Arc::clone(&provider)).await; + store + .append(draft("repo-capacity", &provider)) + .await + .expect("append triggers indexed retention pruning"); + + let retained_rows: i64 = database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + "SELECT COUNT(*) AS count FROM context_selection_receipt + WHERE repository_id = 'repo-capacity'" + .to_string(), + )) + .await + .expect("count retained receipts") + .expect("retained count row") + .try_get("", "count") + .expect("retained count"); + assert_eq!(retained_rows, MAX_RECEIPTS_PER_REPOSITORY); + + let retention_rows: i64 = database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + "SELECT retained_rows FROM context_selection_receipt_retention + WHERE repository_id = 'repo-capacity'" + .to_string(), + )) + .await + .expect("read retention metadata") + .expect("retention metadata row") + .try_get("", "retained_rows") + .expect("retention count"); + assert_eq!(retention_rows, MAX_RECEIPTS_PER_REPOSITORY); + + let pruned_receipt = + Uuid::parse_str("0198a7e0-0000-7000-8000-000000000000").expect("fixed pruned UUIDv7"); + assert!(matches!( + store + .lookup(pruned_receipt) + .await + .expect("classify a pruned receipt"), + ReceiptLookup::Expired + )); + assert!(matches!( + store + .lookup(Uuid::now_v7()) + .await + .expect("classify an unknown current receipt"), + ReceiptLookup::NotFound + )); + + let memory_rows: i64 = database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + "SELECT + (SELECT COUNT(*) FROM memory_head) + + (SELECT COUNT(*) FROM memory_revision_index) + + (SELECT COUNT(*) FROM memory_projection_state) AS count" + .to_string(), + )) + .await + .expect("count authoritative Memory state") + .expect("Memory state count row") + .try_get("", "count") + .expect("Memory state count"); + assert_eq!(memory_rows, 0, "receipt pruning cannot mutate Memory state"); + } + + #[tokio::test] + async fn store_rejects_cross_repository_and_digest_key_before_writing() { + let (database, provider) = database("repo-42").await; + let store = store(&database, Arc::clone(&provider)).await; + + let provider_for_other_repository = Arc::new(RepositoryKeyedDigest::for_receipt_tests( + "repo-other", + digest_key_id(), + [0x41; 32], + TEST_CIPHERTEXT, + )); + let store_binding_error = ReceiptStore::new(&database, provider_for_other_repository) + .await + .err() + .expect("a provider resolved for another repository cannot bind this database"); + assert_eq!( + store_binding_error.kind(), + ReceiptStoreErrorKind::RepositoryMismatch + ); + + let repository_error = store + .append(draft("repo-other", &provider)) + .await + .err() + .expect("store cannot accept a draft from another repository"); + assert_eq!( + repository_error.kind(), + ReceiptStoreErrorKind::RepositoryMismatch + ); + + let other_key = + Uuid::parse_str("223e4567-e89b-42d3-a456-426614174000").expect("second UUIDv4"); + let other_provider = Arc::new(RepositoryKeyedDigest::for_receipt_tests( + "repo-42", + other_key, + [0x42; 32], + "different-repository-digest-config", + )); + let other_key_draft = draft("repo-42", &other_provider); + let key_binding_error = ReceiptStore::new(&database, Arc::clone(&other_provider)) + .await + .err() + .expect("an unknown digest generation cannot bind this database"); + assert_eq!( + key_binding_error.kind(), + ReceiptStoreErrorKind::DigestKeyMismatch + ); + let key_error = store + .append(other_key_draft) + .await + .err() + .expect("store cannot accept a draft using another repository key"); + assert_eq!(key_error.kind(), ReceiptStoreErrorKind::DigestKeyMismatch); + + let removed_generation_draft = draft("repo-42", &provider); + assert_eq!( + ConfigKv::unset_all_with_conn(&database, MEMORY_KEYED_DIGEST_CONFIG_KEY) + .await + .expect("remove the persisted digest generation after store construction"), + 1 + ); + let removed_generation_error = store + .append(removed_generation_draft) + .await + .err() + .expect("append must revalidate the persisted digest generation"); + assert_eq!( + removed_generation_error.kind(), + ReceiptStoreErrorKind::DigestKeyMismatch + ); + + let rows: i64 = database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + "SELECT COUNT(*) AS count FROM context_selection_receipt".to_string(), + )) + .await + .expect("count rejected writes") + .expect("count row") + .try_get("", "count") + .expect("count value"); + assert_eq!(rows, 0); + } + + #[tokio::test] + async fn retention_uses_injected_clock_at_29_30_and_31_day_boundaries() { + let (database, provider) = database("repo-age").await; + database + .execute_unprepared( + "INSERT INTO context_selection_receipt ( + receipt_id, schema_version, source_kind, repository_id, + digest_key_id, principal_hmac, query_hmac, effective_at, + source_heads_json, projection_watermarks_json, policy_hash, + selector_version, token_budget, selected_json, omissions_json, + bundle_hash, reproducibility_state, recorded_at + ) VALUES + ('0198a7e0-0000-7000-8000-000000000001', 1, 'memory', 'repo-age', + '123e4567-e89b-42d3-a456-426614174000', + 'hmac-sha256:123e4567-e89b-42d3-a456-426614174000:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'hmac-sha256:123e4567-e89b-42d3-a456-426614174000:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + '2026-07-24T12:00:00.000000000Z', '{}', '{}', + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'memory-v1', 1, '[]', '[]', + 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', + 'reproducible', '2026-07-24T12:00:00.000000000Z'), + ('0198a7e0-0000-7000-8000-000000000002', 1, 'memory', 'repo-age', + '123e4567-e89b-42d3-a456-426614174000', + 'hmac-sha256:123e4567-e89b-42d3-a456-426614174000:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'hmac-sha256:123e4567-e89b-42d3-a456-426614174000:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + '2026-07-25T12:00:00.000000000Z', '{}', '{}', + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'memory-v1', 1, '[]', '[]', + 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', + 'reproducible', '2026-07-25T12:00:00.000000000Z'), + ('0198a7e0-0000-7000-8000-000000000003', 1, 'memory', 'repo-age', + '123e4567-e89b-42d3-a456-426614174000', + 'hmac-sha256:123e4567-e89b-42d3-a456-426614174000:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'hmac-sha256:123e4567-e89b-42d3-a456-426614174000:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + '2026-07-26T12:00:00.000000000Z', '{}', '{}', + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'memory-v1', 1, '[]', '[]', + 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', + 'reproducible', '2026-07-26T12:00:00.000000000Z')", + ) + .await + .expect("seed age boundary receipts"); + + store(&database, Arc::clone(&provider)) + .await + .append(draft("repo-age", &provider)) + .await + .expect("apply deterministic age retention"); + let retained: Vec = database + .query_all_raw(Statement::from_string( + database.get_database_backend(), + "SELECT receipt_id FROM context_selection_receipt + WHERE repository_id = 'repo-age' + ORDER BY recorded_at" + .to_string(), + )) + .await + .expect("read age boundary survivors") + .into_iter() + .map(|row| row.try_get("", "receipt_id").expect("receipt id")) + .collect(); + assert_eq!( + retained.len(), + 3, + "31-day row pruned; 30-day and 29-day rows retained" + ); + assert!(!retained.iter().any(|id| id.ends_with("0001"))); + assert!(retained.iter().any(|id| id.ends_with("0002"))); + assert!(retained.iter().any(|id| id.ends_with("0003"))); + } +} diff --git a/src/internal/ai/keyed_digest.rs b/src/internal/ai/keyed_digest.rs index f50765db9..d16303122 100644 --- a/src/internal/ai/keyed_digest.rs +++ b/src/internal/ai/keyed_digest.rs @@ -111,6 +111,63 @@ impl KeyedDigestEnvelope { } } +/// Purpose-locked digest for an authenticated principal written to a context +/// selection receipt. +/// +/// This wrapper deliberately does not implement `Debug` or serialization so a +/// caller cannot accidentally log it or persist a generic digest envelope. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct PrincipalDigest(KeyedDigestEnvelope); + +impl PrincipalDigest { + pub(crate) const fn version(&self) -> u8 { + self.0.version() + } + + pub(crate) const fn key_id(&self) -> Uuid { + self.0.key_id() + } + + pub(crate) fn digest_hex(&self) -> &str { + self.0.digest_hex() + } + + pub(crate) fn encoded(&self) -> String { + encode_receipt_digest(&self.0) + } +} + +/// Purpose-locked digest for normalized retrieval inputs written to a context +/// selection receipt. +#[derive(Clone, PartialEq, Eq)] +pub(crate) struct QueryDigest(KeyedDigestEnvelope); + +impl QueryDigest { + pub(crate) const fn version(&self) -> u8 { + self.0.version() + } + + pub(crate) const fn key_id(&self) -> Uuid { + self.0.key_id() + } + + pub(crate) fn digest_hex(&self) -> &str { + self.0.digest_hex() + } + + pub(crate) fn encoded(&self) -> String { + encode_receipt_digest(&self.0) + } +} + +fn encode_receipt_digest(envelope: &KeyedDigestEnvelope) -> String { + format!( + "hmac-sha256:{}:{}", + envelope.key_id(), + envelope.digest_hex() + ) +} + /// Purpose-locked fingerprint for a compiler root's canonical source inputs. /// /// The wrapper deliberately has no `Debug` or serialization implementation: @@ -281,6 +338,7 @@ impl hkdf::KeyType for HmacSha256KeyLength { } pub(crate) struct RepositoryKeyedDigest { + repository_id: String, key_id: Uuid, keys: [hmac::Key; 4], persisted_config_fingerprint: [u8; 32], @@ -297,6 +355,22 @@ impl fmt::Debug for RepositoryKeyedDigest { } impl RepositoryKeyedDigest { + #[cfg(test)] + pub(crate) fn for_receipt_tests( + repository_id: &str, + key_id: Uuid, + seed: [u8; 32], + persisted_ciphertext: &str, + ) -> Self { + Self::from_seed( + repository_id.to_string(), + key_id, + seed, + config_fingerprint(persisted_ciphertext), + ) + .expect("fixed test seed must construct a keyed-digest provider") + } + pub(crate) async fn load_or_initialize( repository_db_path: &Path, ) -> Result, KeyedDigestError> { @@ -325,8 +399,14 @@ impl RepositoryKeyedDigest { return Err(KeyedDigestError::new(KeyedDigestErrorKind::CacheCapacity)); } - let provider = - Arc::new(Self::load_or_initialize_uncached(&database, &canonical_db_path).await?); + let provider = Arc::new( + Self::load_or_initialize_uncached( + &database, + &canonical_db_path, + &cache_key.repository_id, + ) + .await?, + ); cache.insert(cache_key, Arc::clone(&provider)); Ok(provider) } @@ -334,13 +414,18 @@ impl RepositoryKeyedDigest { async fn load_or_initialize_uncached( database: &DatabaseConnection, repository_db_path: &Path, + repository_id: &str, ) -> Result { let transaction = db::begin_write_transaction(database) .await .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::StateQueryFailed))?; - let result = - Self::load_or_initialize_in_transaction(&transaction, repository_db_path).await; + let result = Self::load_or_initialize_in_transaction( + &transaction, + repository_db_path, + repository_id, + ) + .await; match result { Ok(provider) => { transaction @@ -359,21 +444,25 @@ impl RepositoryKeyedDigest { async fn load_or_initialize_in_transaction( database: &C, repository_db_path: &Path, + repository_id: &str, ) -> Result { let rows = ConfigKv::get_all_with_conn(database, MEMORY_KEYED_DIGEST_CONFIG_KEY) .await .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::StateQueryFailed))?; match rows.as_slice() { - [row] => load_persisted_provider(database, repository_db_path, row).await, + [row] => { + load_persisted_provider(database, repository_db_path, repository_id, row).await + } [] => { ensure_initialization_is_eligible(database).await?; - initialize_provider(database, repository_db_path).await + initialize_provider(database, repository_db_path, repository_id).await } _ => Err(KeyedDigestError::new(KeyedDigestErrorKind::DuplicateConfig)), } } fn from_seed( + repository_id: String, key_id: Uuid, seed: [u8; 32], persisted_config_fingerprint: [u8; 32], @@ -403,6 +492,7 @@ impl RepositoryKeyedDigest { .try_into() .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::Derivation))?; Ok(Self { + repository_id, key_id, keys, persisted_config_fingerprint, @@ -414,6 +504,17 @@ impl RepositoryKeyedDigest { self.key_id } + pub(crate) fn repository_id(&self) -> &str { + &self.repository_id + } + + pub(crate) async fn validate_for_connection( + &self, + database: &C, + ) -> Result<(), KeyedDigestError> { + validate_cached_provider(database, self).await + } + fn invalidate(&self) { self.valid.store(false, Ordering::Release); } @@ -428,7 +529,7 @@ impl RepositoryKeyedDigest { } } - pub(crate) fn digest( + fn digest( &self, purpose: DigestPurpose, input: &[u8], @@ -454,6 +555,18 @@ impl RepositoryKeyedDigest { self.digest(DigestPurpose::SourceInput, input) .map(SourceInputFingerprint) } + + pub(crate) fn principal_digest( + &self, + input: &[u8], + ) -> Result { + self.digest(DigestPurpose::Principal, input) + .map(PrincipalDigest) + } + + pub(crate) fn query_digest(&self, input: &[u8]) -> Result { + self.digest(DigestPurpose::Query, input).map(QueryDigest) + } } fn config_fingerprint(ciphertext_hex: &str) -> [u8; 32] { @@ -569,6 +682,7 @@ async fn ensure_initialization_is_eligible( async fn initialize_provider( database: &C, repository_db_path: &Path, + repository_id: &str, ) -> Result { use ring::rand::{SecureRandom, SystemRandom}; @@ -601,12 +715,18 @@ async fn initialize_provider( if !inserted { return Err(KeyedDigestError::new(KeyedDigestErrorKind::PersistFailed)); } - RepositoryKeyedDigest::from_seed(key_id, seed, config_fingerprint(&ciphertext_hex)) + RepositoryKeyedDigest::from_seed( + repository_id.to_string(), + key_id, + seed, + config_fingerprint(&ciphertext_hex), + ) } async fn load_persisted_provider( database: &C, repository_db_path: &Path, + repository_id: &str, row: &crate::internal::config::ConfigKvEntry, ) -> Result { if !row.encrypted { @@ -645,7 +765,12 @@ async fn load_persisted_provider( let seed: [u8; 32] = seed_bytes .try_into() .map_err(|_| KeyedDigestError::new(KeyedDigestErrorKind::PayloadInvalid))?; - RepositoryKeyedDigest::from_seed(payload.key_id, seed, config_fingerprint(&row.value)) + RepositoryKeyedDigest::from_seed( + repository_id.to_string(), + payload.key_id, + seed, + config_fingerprint(&row.value), + ) } #[cfg(test)] @@ -673,7 +798,7 @@ mod tests { use super::{ DigestPurpose, KeyedDigestError, KeyedDigestErrorKind, PERSISTED_GENERATION, PERSISTED_SCHEMA_VERSION, PersistedDigestKeyV1, RepositoryKeyedDigest, - SourceInputFingerprint, SourceInputFingerprintErrorKind, config_fingerprint, + SourceInputFingerprint, SourceInputFingerprintErrorKind, config_fingerprint, repository_id, reset_digest_cache_for_tests, }; use crate::{ @@ -800,17 +925,25 @@ mod tests { async fn insert_receipt(&self) { let conn = self.connection().await; - let backend = conn.get_database_backend(); - conn.execute_raw(Statement::from_string( - backend, - "CREATE TABLE context_selection_receipt (id INTEGER PRIMARY KEY)", - )) - .await - .expect("receipt table must be created"); - conn.execute_raw(Statement::from_string( - backend, - "INSERT INTO context_selection_receipt (id) VALUES (1)", - )) + conn.execute_unprepared( + "INSERT INTO context_selection_receipt ( + receipt_id, schema_version, source_kind, repository_id, + digest_key_id, principal_hmac, query_hmac, effective_at, + source_heads_json, projection_watermarks_json, policy_hash, + selector_version, token_budget, selected_json, omissions_json, + bundle_hash, reproducibility_state, recorded_at + ) VALUES ( + '0198a7e0-7c00-7000-8000-000000000001', 1, 'memory', 'repo-test', + '123e4567-e89b-42d3-a456-426614174000', + 'hmac-sha256:123e4567-e89b-42d3-a456-426614174000:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'hmac-sha256:123e4567-e89b-42d3-a456-426614174000:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + '2026-08-24T00:00:00.000000000Z', '{}', '{}', + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'memory-v1', 1, '[]', '[]', + 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', + 'reproducible', '2026-08-24T00:00:00.000000000Z' + )", + ) .await .expect("receipt must be inserted"); } @@ -879,6 +1012,7 @@ mod tests { fn keyed_digest_domains_are_distinct_and_match_frozen_vectors() { let seed: [u8; 32] = std::array::from_fn(|index| index as u8); let provider = RepositoryKeyedDigest::from_seed( + "test-repository".to_string(), Uuid::parse_str("123e4567-e89b-42d3-a456-426614174000") .expect("fixed UUIDv4 must parse"), seed, @@ -944,6 +1078,7 @@ mod tests { let key_id = Uuid::parse_str("123e4567-e89b-42d3-a456-426614174000") .expect("fixed UUIDv4 must parse"); let provider = RepositoryKeyedDigest::from_seed( + "test-repository".to_string(), key_id, [0x24; 32], config_fingerprint("source-input-fingerprint"), @@ -985,6 +1120,40 @@ mod tests { ); } + #[test] + fn receipt_digests_are_purpose_locked_and_share_the_repository_key() { + let key_id = Uuid::parse_str("123e4567-e89b-42d3-a456-426614174000") + .expect("fixed UUIDv4 must parse"); + let provider = RepositoryKeyedDigest::from_seed( + "test-repository".to_string(), + key_id, + [0x25; 32], + config_fingerprint("receipt-digests"), + ) + .expect("fixed seed must construct a provider"); + + let principal = provider + .principal_digest(b"agent:alice") + .expect("valid provider produces a principal digest"); + let query = provider + .query_digest(b"normalized retrieval inputs") + .expect("valid provider produces a query digest"); + + assert_eq!(principal.key_id(), key_id); + assert_eq!(query.key_id(), key_id); + assert_eq!(principal.version(), 1); + assert_eq!(query.version(), 1); + assert_ne!(principal.digest_hex(), query.digest_hex()); + assert_eq!( + principal.encoded(), + format!("hmac-sha256:{key_id}:{}", principal.digest_hex()) + ); + assert_eq!( + query.encoded(), + format!("hmac-sha256:{key_id}:{}", query.digest_hex()) + ); + } + #[test] fn keyed_digest_error_contract_is_stable_and_actionable() { let error = KeyedDigestError::new(KeyedDigestErrorKind::UnsupportedGeneration); @@ -1136,7 +1305,8 @@ mod tests { ) .await .expect("independent repository connection must open"); - RepositoryKeyedDigest::load_or_initialize_uncached(&conn, &db_path) + let repository_id = repository_id(&conn).await?; + RepositoryKeyedDigest::load_or_initialize_uncached(&conn, &db_path, &repository_id) .await .map(|provider| provider.key_id()) })); diff --git a/src/internal/db.rs b/src/internal/db.rs index fde6680e4..3de21b549 100644 --- a/src/internal/db.rs +++ b/src/internal/db.rs @@ -899,6 +899,17 @@ mod tests { ); } + #[test] + fn context_receipt_old_reader_rejects_migrated_schema() { + assert_eq!( + classify_schema_compatibility(Some(2026082403), Some(2026082402)), + SchemaCompatibility::UnsupportedFuture { + current_version: 2026082403, + latest_version: Some(2026082402), + } + ); + } + /// TestDbPath is a helper struct create and delete test database file struct TestDbPath(String); impl Drop for TestDbPath { diff --git a/src/internal/db/migration.rs b/src/internal/db/migration.rs index f513d0629..38bc88c71 100644 --- a/src/internal/db/migration.rs +++ b/src/internal/db/migration.rs @@ -1398,6 +1398,14 @@ pub fn builtin_migrations() -> Vec { include_str!("../../../sql/migrations/2026082402_memory_fts_search.sql"), include_str!("../../../sql/migrations/2026082402_memory_fts_search_down.sql"), ), + // M2-02R: the single local-only context selection receipt ledger shared + // by Memory and mainline, plus its bounded retention watermark. + sql_migration( + 2026082403, + "context_selection_receipt", + include_str!("../../../sql/migrations/2026082403_context_selection_receipt.sql"), + include_str!("../../../sql/migrations/2026082403_context_selection_receipt_down.sql"), + ), ] } @@ -1845,9 +1853,9 @@ mod tests { // `builtin_migrations()` so silent registry regressions surface // here in addition to `tests/db_migration_test.rs`. let runner = builtin_runner().expect("CEX-12.5 builtin registry must build clean"); - assert_eq!(runner.len(), 57); + assert_eq!(runner.len(), 58); assert!(!runner.is_empty()); - assert_eq!(runner.max_registered_version(), Some(2026082402)); + assert_eq!(runner.max_registered_version(), Some(2026082403)); } #[test] diff --git a/tests/db_migration_test.rs b/tests/db_migration_test.rs index 08697b468..e4fc70359 100644 --- a/tests/db_migration_test.rs +++ b/tests/db_migration_test.rs @@ -55,7 +55,7 @@ fn builtin_migrations_register_current_schema_migrations() { 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, - 2026082402 + 2026082402, 2026082403 ] ); assert_eq!( @@ -118,13 +118,14 @@ fn builtin_migrations_register_current_schema_migrations() { "approved_permission_provenance", "memory_core", "memory_fts_search", + "context_selection_receipt", ] ); let runner = builtin_runner().expect("builtin registry must build clean"); assert!(!runner.is_empty()); - assert_eq!(runner.len(), 57); - assert_eq!(runner.max_registered_version(), Some(2026082402)); + assert_eq!(runner.len(), 58); + assert_eq!(runner.max_registered_version(), Some(2026082403)); } const MEMORY_CORE_TABLES: [&str; 9] = [ @@ -383,10 +384,11 @@ async fn memory_episode_schema_idempotent() { let (_fresh_dir, fresh_url, _fresh_path) = fresh_db_url(); let fresh = connect(&fresh_url).await; let migrations = builtin_migrations(); - let (memory_fts_search, through_memory_core) = migrations - .split_last() - .expect("built-in registry contains Memory FTS migration"); - assert_eq!(memory_fts_search.version, 2026082402); + let memory_core_index = migrations + .iter() + .position(|migration| migration.version == 2026082401) + .expect("built-in registry contains Memory core migration"); + let through_memory_core = &migrations[..=memory_core_index]; let memory_core = through_memory_core .last() .expect("built-in registry contains Memory core migration"); @@ -444,7 +446,7 @@ async fn memory_episode_schema_idempotent() { run_builtin_migrations(&upgrade) .await .expect("upgrade old database"), - vec![2026082401, 2026082402] + vec![2026082401, 2026082402, 2026082403] ); assert_eq!(memory_core_schema_snapshot(&upgrade).await, fresh_snapshot); } @@ -665,7 +667,7 @@ async fn memory_episode_fts_schema() { let applied = run_builtin_migrations(&conn) .await .expect("apply Memory FTS migration"); - assert_eq!(applied.last(), Some(&2026082402)); + assert!(applied.contains(&2026082402)); let schema = memory_fts_schema_snapshot(&conn).await; assert_eq!( @@ -796,7 +798,7 @@ async fn memory_episode_fts_schema_idempotent_upgrade_and_empty_down() { let fresh_applied = run_builtin_migrations(&fresh) .await .expect("apply full registry"); - assert_eq!(fresh_applied.last(), Some(&2026082402)); + assert_eq!(fresh_applied.last(), Some(&2026082403)); assert!( run_builtin_migrations(&fresh) .await @@ -806,7 +808,7 @@ async fn memory_episode_fts_schema_idempotent_upgrade_and_empty_down() { let snapshot = memory_fts_schema_snapshot(&fresh).await; let migration = builtin_migrations() .into_iter() - .last() + .find(|migration| migration.version == 2026082402) .expect("Memory FTS migration"); assert_eq!(migration.version, 2026082402); fresh @@ -821,8 +823,11 @@ async fn memory_episode_fts_schema_idempotent_upgrade_and_empty_down() { let (_upgrade_dir, upgrade_url, _upgrade_path) = fresh_db_url(); let upgrade = connect(&upgrade_url).await; let migrations = builtin_migrations(); - let (fts_migration, previous) = migrations.split_last().expect("Memory FTS migration"); - assert_eq!(fts_migration.version, 2026082402); + let fts_index = migrations + .iter() + .position(|migration| migration.version == 2026082402) + .expect("Memory FTS migration"); + let previous = &migrations[..fts_index]; let mut old_runner = MigrationRunner::new(); old_runner .extend(previous.iter().cloned()) @@ -850,7 +855,7 @@ async fn memory_episode_fts_schema_idempotent_upgrade_and_empty_down() { run_builtin_migrations(&upgrade) .await .expect("upgrade to Memory FTS"), - vec![2026082402] + vec![2026082402, 2026082403] ); let core_rows: i64 = upgrade .query_one_raw(Statement::from_string( @@ -873,7 +878,7 @@ async fn memory_episode_fts_schema_idempotent_upgrade_and_empty_down() { .rollback_to(&upgrade, 2026082401) .await .expect("empty search projection can roll back"), - vec![2026082402] + vec![2026082403, 2026082402] ); assert!(!table_exists(&upgrade, "memory_episode_search_doc").await); assert!(!table_exists(&upgrade, "memory_episode_fts").await); @@ -883,7 +888,7 @@ async fn memory_episode_fts_schema_idempotent_upgrade_and_empty_down() { .run_pending(&upgrade) .await .expect("reapply search migration"), - vec![2026082402] + vec![2026082402, 2026082403] ); } @@ -928,6 +933,205 @@ async fn memory_episode_fts_nonempty_down_guard() { ); } +#[tokio::test] +async fn context_selection_receipt_schema() { + let (_dir, url, _path) = fresh_db_url(); + let conn = connect(&url).await; + let applied = run_builtin_migrations(&conn) + .await + .expect("apply context receipt migration"); + assert_eq!(applied.last(), Some(&2026082403)); + + assert!(table_exists(&conn, "context_selection_receipt").await); + assert!(table_exists(&conn, "context_selection_receipt_retention").await); + assert!(index_exists(&conn, "idx_context_selection_receipt_repository_time").await); + assert!(index_exists(&conn, "idx_context_selection_receipt_time").await); + + let columns = conn + .query_all_raw(Statement::from_string( + conn.get_database_backend(), + "PRAGMA table_info(context_selection_receipt)".to_string(), + )) + .await + .expect("inspect context receipt columns") + .into_iter() + .map(|row| row.try_get::("", "name").expect("column name")) + .collect::>(); + assert_eq!( + columns, + [ + "receipt_id", + "schema_version", + "source_kind", + "repository_id", + "digest_key_id", + "principal_hmac", + "query_hmac", + "effective_at", + "code_commit", + "full_branch_ref", + "source_heads_json", + "projection_watermarks_json", + "policy_hash", + "selector_version", + "token_budget", + "selected_json", + "omissions_json", + "bundle_hash", + "reproducibility_state", + "frame_id", + "recorded_at", + ] + ); + for forbidden in ["raw_query", "query", "body", "content", "principal"] { + assert!( + !columns.iter().any(|column| column == forbidden), + "raw or reversible field {forbidden} must not enter the ledger" + ); + } + + let invalid_source = conn + .execute_unprepared( + "INSERT INTO context_selection_receipt ( + receipt_id, schema_version, source_kind, repository_id, + digest_key_id, principal_hmac, query_hmac, effective_at, + source_heads_json, projection_watermarks_json, policy_hash, + selector_version, token_budget, selected_json, omissions_json, + bundle_hash, reproducibility_state, recorded_at + ) VALUES ( + '0198a7e0-7c00-7000-8000-000000000001', 1, 'unknown', 'repo', + '123e4567-e89b-42d3-a456-426614174000', + 'hmac-sha256:123e4567-e89b-42d3-a456-426614174000:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'hmac-sha256:123e4567-e89b-42d3-a456-426614174000:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + '2026-08-24T00:00:00.000000000Z', '{}', '{}', + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'memory-v1', 100, '[]', '[]', + 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', + 'reproducible', '2026-08-24T00:00:00.000000000Z' + )", + ) + .await; + assert!( + invalid_source.is_err(), + "unknown source kinds must fail closed" + ); + + let receipt_migration = builtin_migrations() + .into_iter() + .find(|migration| migration.version == 2026082403) + .expect("context receipt migration"); + let snapshot = conn + .query_all_raw(Statement::from_string( + conn.get_database_backend(), + "SELECT type, name, sql FROM sqlite_master + WHERE name LIKE 'context_selection_receipt%' + OR name LIKE 'idx_context_selection_receipt%' + ORDER BY type, name" + .to_string(), + )) + .await + .expect("snapshot receipt schema") + .into_iter() + .map(|row| { + ( + row.try_get::("", "type").expect("schema type"), + row.try_get::("", "name").expect("schema name"), + row.try_get::("", "sql").expect("schema SQL"), + ) + }) + .collect::>(); + conn.execute_raw(Statement::from_string( + conn.get_database_backend(), + receipt_migration.up, + )) + .await + .expect("context receipt up SQL is idempotent"); + let repeated_snapshot = conn + .query_all_raw(Statement::from_string( + conn.get_database_backend(), + "SELECT type, name, sql FROM sqlite_master + WHERE name LIKE 'context_selection_receipt%' + OR name LIKE 'idx_context_selection_receipt%' + ORDER BY type, name" + .to_string(), + )) + .await + .expect("snapshot repeated receipt schema") + .into_iter() + .map(|row| { + ( + row.try_get::("", "type").expect("schema type"), + row.try_get::("", "name").expect("schema name"), + row.try_get::("", "sql").expect("schema SQL"), + ) + }) + .collect::>(); + assert_eq!(repeated_snapshot, snapshot); + + let runner = builtin_runner().expect("builtin runner"); + assert_eq!( + runner + .rollback_to(&conn, 2026082402) + .await + .expect("empty receipt ledger can roll back"), + vec![2026082403] + ); + assert!(!table_exists(&conn, "context_selection_receipt").await); + assert_eq!( + runner + .run_pending(&conn) + .await + .expect("upgrade from the previous registry tip"), + vec![2026082403] + ); + assert!(table_exists(&conn, "context_selection_receipt").await); +} + +#[tokio::test] +async fn context_selection_receipt_nonempty_down_guard() { + let (_dir, url, _path) = fresh_db_url(); + let conn = connect(&url).await; + let runner = builtin_runner().expect("builtin runner"); + runner + .run_pending(&conn) + .await + .expect("apply context receipt migration"); + conn.execute_unprepared( + "INSERT INTO context_selection_receipt ( + receipt_id, schema_version, source_kind, repository_id, + digest_key_id, principal_hmac, query_hmac, effective_at, + source_heads_json, projection_watermarks_json, policy_hash, + selector_version, token_budget, selected_json, omissions_json, + bundle_hash, reproducibility_state, recorded_at + ) VALUES ( + '0198a7e0-7c00-7000-8000-000000000001', 1, 'memory', 'repo', + '123e4567-e89b-42d3-a456-426614174000', + 'hmac-sha256:123e4567-e89b-42d3-a456-426614174000:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + 'hmac-sha256:123e4567-e89b-42d3-a456-426614174000:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + '2026-08-24T00:00:00.000000000Z', '{}', '{}', + 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', + 'memory-v1', 100, '[]', '[]', + 'sha256:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd', + 'reproducible', '2026-08-24T00:00:00.000000000Z' + )", + ) + .await + .expect("seed audit receipt"); + + let error = runner + .rollback_to(&conn, 2026082402) + .await + .expect_err("non-empty audit evidence must block rollback"); + assert!( + format!("{error:#}").contains("context_selection_receipt_down_guard_empty"), + "non-empty receipt rollback must fail with the stable guard: {error:#}" + ); + assert_eq!( + runner.current_version(&conn).await.expect("current tip"), + Some(2026082403) + ); +} + // --------------------------------------------------------------------------- // run_pending on a fresh database: applies every registered migration // --------------------------------------------------------------------------- @@ -1839,7 +2043,7 @@ async fn connect_with_busy_timeout(url: &str) -> DatabaseConnection { opts.sqlx_logging(false); // Match the production busy-timeout path so the test exercises the // realistic concurrency model. - opts.map_sqlx_sqlite_opts(move |sqlx_opts| sqlx_opts.busy_timeout(Duration::from_secs(5))); + opts.map_sqlx_sqlite_opts(move |sqlx_opts| sqlx_opts.busy_timeout(Duration::from_secs(30))); Database::connect(opts).await.expect("connect") } @@ -2009,7 +2213,7 @@ async fn run_builtin_migrations_applies_current_builtin_registry() { 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, - 2026082402 + 2026082402, 2026082403 ] ); assert!(table_exists(&conn, "schema_versions").await); @@ -2203,7 +2407,7 @@ async fn agent_subagent_content_up_down_up_and_nonempty_guard() { 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, - 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402, 2026082403 ] ); conn.execute_raw(Statement::from_string( @@ -2268,7 +2472,8 @@ async fn agent_subagent_content_up_down_up_and_nonempty_guard() { 2026071406, 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, - 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 + 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402, + 2026082403 ] ); assert!(table_exists(&conn, "agent_subagent_content_claim").await); @@ -2372,7 +2577,7 @@ async fn existing_agent_subagent_1406_schema_upgrades_to_replication() { 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, - 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402, 2026082403 ] ); let claim = conn @@ -2506,7 +2711,7 @@ async fn evolved_agent_subagent_1406_columns_upgrade_idempotently() { 2026071407, 2026071901, 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, - 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402, 2026082403 ] ); let cursor = conn @@ -2550,11 +2755,11 @@ async fn agent_import_identity_tombstone_up_down_up_round_trip() { assert_eq!( rolled, vec![ - 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, - 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, - 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303, - 2026072302, 2026072301, 2026072201, 2026072101, 2026071901, 2026071407, 2026071406, - 2026071405, 2026071404, 2026071403, 2026071402 + 2026082403, 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, + 2026073101, 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, + 2026072901, 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, + 2026072303, 2026072302, 2026072301, 2026072201, 2026072101, 2026071901, 2026071407, + 2026071406, 2026071405, 2026071404, 2026071403, 2026071402 ] ); assert!(!table_exists(&conn, "agent_import_identity").await); @@ -2573,7 +2778,7 @@ async fn agent_import_identity_tombstone_up_down_up_round_trip() { 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, - 2026080403, 2026081301, 2026082401, 2026082402 + 2026080403, 2026081301, 2026082401, 2026082402, 2026082403 ] ); assert!(table_exists(&conn, "agent_import_identity").await); @@ -2604,11 +2809,11 @@ async fn existing_agent_tombstone_1403_schema_upgrades_to_compat_barrier() { assert_eq!( rolled, vec![ - 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, - 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, - 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303, - 2026072302, 2026072301, 2026072201, 2026072101, 2026071901, 2026071407, 2026071406, - 2026071405, 2026071404 + 2026082403, 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, + 2026073101, 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, + 2026072901, 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, + 2026072303, 2026072302, 2026072301, 2026072201, 2026072101, 2026071901, 2026071407, + 2026071406, 2026071405, 2026071404 ] ); assert!(table_exists(&conn, "agent_import_tombstone").await); @@ -2627,7 +2832,7 @@ async fn existing_agent_tombstone_1403_schema_upgrades_to_compat_barrier() { 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, - 2026082401, 2026082402 + 2026082401, 2026082402, 2026082403 ] ); assert!(trigger_exists(&conn, "agent_tombstone_block_session_insert").await); @@ -2972,14 +3177,14 @@ async fn approved_permission_up_down_up_round_trip() { assert_eq!( rolled, vec![ - 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, - 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, - 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303, - 2026072302, 2026072301, 2026072201, 2026072101, 2026071901, 2026071407, 2026071406, - 2026071405, 2026071404, 2026071403, 2026071402, 2026071401, 2026071301, 2026070803, - 2026070802, 2026070801, 2026070701, 2026070601, 2026070501, 2026070401, 2026070301, - 2026070202, 2026070201, 2026062301, 2026061401, 2026060801, 2026060401, 2026060201, - 2026053101, 2026052301, 2026050801, 2026050601 + 2026082403, 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, + 2026073101, 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, + 2026072901, 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, + 2026072303, 2026072302, 2026072301, 2026072201, 2026072101, 2026071901, 2026071407, + 2026071406, 2026071405, 2026071404, 2026071403, 2026071402, 2026071401, 2026071301, + 2026070803, 2026070802, 2026070801, 2026070701, 2026070601, 2026070501, 2026070401, + 2026070301, 2026070202, 2026070201, 2026062301, 2026061401, 2026060801, 2026060401, + 2026060201, 2026053101, 2026052301, 2026050801, 2026050601 ] ); assert!( @@ -3010,7 +3215,7 @@ async fn approved_permission_up_down_up_round_trip() { 2026072101, 2026072201, 2026072301, 2026072302, 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, - 2026080403, 2026081301, 2026082401, 2026082402 + 2026080403, 2026081301, 2026082401, 2026082402, 2026082403 ] ); assert!(table_exists(&conn, "approved_permission").await); @@ -3446,8 +3651,8 @@ async fn bisect_state_migration_keeps_newest_row_per_scope() { async fn concurrent_run_pending_applies_each_migration_exactly_once() { for round in 0..3 { let (_dir, url, _path) = fresh_db_url(); - let conn_a = connect(&url).await; - let conn_b = connect(&url).await; + let conn_a = connect_with_busy_timeout(&url).await; + let conn_b = connect_with_busy_timeout(&url).await; let runner_a = builtin_runner().expect("builtin runner A"); let runner_b = builtin_runner().expect("builtin runner B"); @@ -3882,9 +4087,10 @@ async fn legacy_layer_rows_with_linked_fail_migration() { .await .expect("rollback layer scope"), vec![ - 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, - 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, - 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, 2026072303 + 2026082403, 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, + 2026073101, 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, + 2026072901, 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304, + 2026072303 ] ); conn.execute_raw(Statement::from_string( @@ -3937,7 +4143,8 @@ async fn legacy_layer_rows_with_linked_fail_migration() { vec![ 2026072303, 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, - 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 + 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402, + 2026082403 ] ); let row = conn @@ -4214,9 +4421,9 @@ async fn sparse_migration_projects_last_wins_toggle() { .await .expect("rollback sparse scope"), vec![ - 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, - 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, - 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304 + 2026082403, 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, + 2026073101, 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, + 2026072901, 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304 ] ); // Duplicate legacy rows: stale `true` (lower id) then effective `false` @@ -4252,7 +4459,7 @@ async fn sparse_migration_projects_last_wins_toggle() { vec![ 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, - 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402, 2026082403 ] ); let row = conn @@ -4287,9 +4494,9 @@ async fn legacy_sparse_state_with_linked_requires_adopt_or_clear() { .await .expect("rollback sparse scope"), vec![ - 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, - 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, - 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304 + 2026082403, 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, + 2026073101, 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, + 2026072901, 2026072502, 2026072501, 2026072403, 2026072402, 2026072401, 2026072304 ] ); conn.execute_raw(Statement::from_string( @@ -4332,7 +4539,7 @@ async fn legacy_sparse_state_with_linked_requires_adopt_or_clear() { vec![ 2026072304, 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, - 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 + 2026080401, 2026080402, 2026080403, 2026081301, 2026082401, 2026082402, 2026082403 ] ); assert!(column_exists(&conn, "sparse_view", "worktree_id").await); @@ -4498,6 +4705,9 @@ async fn gc_object_source_inventory_covers_every_oid_column() { ("operation_view_workspace", "pointer_value"), ("object_index", "o_id"), ("metadata_kv", "value"), + ("context_selection_receipt", "source_heads_json"), + ("context_selection_receipt", "projection_watermarks_json"), + ("context_selection_receipt", "selected_json"), ] { let inventoried = GC_OBJECT_SOURCE_INVENTORY .iter() @@ -4553,9 +4763,9 @@ async fn worktree_registry_v2_capability_marker_round_trip() { .await .expect("rollback capability marker"), vec![ - 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, - 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, 2026072901, - 2026072502, 2026072501, 2026072403, 2026072402, 2026072401 + 2026082403, 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, + 2026073101, 2026073005, 2026073004, 2026073003, 2026073002, 2026073001, 2026072902, + 2026072901, 2026072502, 2026072501, 2026072403, 2026072402, 2026072401 ] ); assert!(!table_exists(&conn, "worktree_registry_capability").await); @@ -4566,7 +4776,7 @@ async fn worktree_registry_v2_capability_marker_round_trip() { vec![ 2026072401, 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, - 2026080402, 2026080403, 2026081301, 2026082401, 2026082402 + 2026080402, 2026080403, 2026081301, 2026082401, 2026082402, 2026082403 ] ); assert!(table_exists(&conn, "worktree_registry_capability").await); @@ -4689,7 +4899,7 @@ async fn registry_v2_down_migration_rejects_nonterminal_state() { vec![ 2026072402, 2026072403, 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, - 2026080403, 2026081301, 2026082401, 2026082402 + 2026080403, 2026081301, 2026082401, 2026082402, 2026082403 ] ); } @@ -4772,7 +4982,7 @@ async fn workspace_record_down_migration_rejects_nonterminal_state() { vec![ 2026072501, 2026072502, 2026072901, 2026072902, 2026073001, 2026073002, 2026073003, 2026073004, 2026073005, 2026073101, 2026080401, 2026080402, 2026080403, 2026081301, - 2026082401, 2026082402 + 2026082401, 2026082402, 2026082403 ] ); @@ -5264,7 +5474,7 @@ async fn registry_v3_rollback_refuses_live_generations() { .current_version(&conn) .await .expect("current version"), - Some(2026082402), + Some(2026082403), "and the schema is untouched" ); } @@ -5311,8 +5521,8 @@ async fn registry_v3_rollback_allows_absent_generations() { assert_eq!( rolled, vec![ - 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, - 2026073005 + 2026082403, 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, + 2026073101, 2026073005 ], "{label}: exactly v3 rolled back" ); @@ -5341,8 +5551,8 @@ async fn registry_v3_rollback_allows_unreadable_registry() { .await .expect("an unparseable registry does not block the rollback"), vec![ - 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101, - 2026073005 + 2026082403, 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, + 2026073101, 2026073005 ] ); } @@ -5638,8 +5848,8 @@ async fn stash_generation_fence_up_down_up_round_trip() { ); assert_eq!( runner.current_version(&conn).await.expect("version"), - Some(2026082402), - "the Memory FTS migration is the newest migration — retarget this test when a newer one lands" + Some(2026082403), + "the context receipt migration is the newest migration — retarget this test when a newer one lands" ); let rolled = runner @@ -5649,7 +5859,8 @@ async fn stash_generation_fence_up_down_up_round_trip() { assert_eq!( rolled, vec![ - 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, 2026073101 + 2026082403, 2026082402, 2026082401, 2026081301, 2026080403, 2026080402, 2026080401, + 2026073101 ], "the runtime attribution, scope, and fence migrations roll back in order" ); @@ -5732,7 +5943,7 @@ async fn approved_permission_provenance_migration() { // Up: empty provenance backfill; project_id unchanged. assert_eq!( runner.run_pending(&conn).await.expect("apply W4-07"), - vec![2026081301, 2026082401, 2026082402] + vec![2026081301, 2026082401, 2026082402, 2026082403] ); assert!(column_exists(&conn, "approved_permission", "source_worktree_id").await); assert!(column_exists(&conn, "approved_permission", "source_session_id").await); @@ -5824,7 +6035,7 @@ async fn approved_permission_provenance_migration() { assert_eq!( runner.run_pending(&conn).await.expect("re-up"), - vec![2026081301, 2026082401, 2026082402] + vec![2026081301, 2026082401, 2026082402, 2026082403] ); let kept_after: i64 = conn .query_one_raw(Statement::from_string( @@ -5859,7 +6070,7 @@ async fn approved_permission_old_reader_rejects_migrated_schema() { .await .expect("read tip") .expect("applied tip"); - assert_eq!(current, 2026082402); + assert_eq!(current, 2026082403); // An old binary whose registry tip is still 2026080403 would see this // repository as UnsupportedFuture. Prove the refuse path on repository // DBs (not global config.db) by planting a version above this binary. From 0df535e2cacaae82e0f191c89d35f8d9218e664d Mon Sep 17 00:00:00 2001 From: anduin9527 Date: Mon, 24 Aug 2026 22:11:15 +0800 Subject: [PATCH 06/18] feat(memory): add typed linear ref transaction Signed-off-by: anduin9527 --- src/internal/ai/history.rs | 320 +++++++------------ src/internal/ai/linear_ref.rs | 565 ++++++++++++++++++++++++++++++++++ src/internal/ai/mod.rs | 2 + 3 files changed, 674 insertions(+), 213 deletions(-) create mode 100644 src/internal/ai/linear_ref.rs diff --git a/src/internal/ai/history.rs b/src/internal/ai/history.rs index 8a6e7d2eb..5077573ec 100644 --- a/src/internal/ai/history.rs +++ b/src/internal/ai/history.rs @@ -54,8 +54,7 @@ use git_internal::{ }; use sea_orm::{ ActiveModelTrait, ColumnTrait, ConnectionTrait, DatabaseConnection, DatabaseTransaction, DbErr, - EntityTrait, QueryFilter, QueryResult, Set, SqlErr, Statement, TransactionTrait, Value, - sea_query::Expr, + EntityTrait, QueryFilter, QueryResult, Set, Statement, TransactionTrait, Value, }; use serde::{Deserialize, Serialize}; use tokio::{ @@ -67,7 +66,13 @@ use tokio::{ use crate::utils::storage::tiered::verify_fetched_object; use crate::{ internal::{ - ai::observed_agents::RedactedBytes, + ai::{ + linear_ref::{ + LinearRefCompanion, LinearRefDeadlineExceeded, LinearRefTransactionOutcome, + OwnedRefSpec, linear_ref_transaction, + }, + observed_agents::RedactedBytes, + }, model::reference::{self, ConfigKind}, }, utils::{ @@ -339,20 +344,6 @@ struct TracesWriterFence { generation: String, } -/// Outcome of a compare-and-swap reference update. -/// -/// Used by [`HistoryManager::update_ref_if_matches`] to communicate whether -/// the ref moved successfully (`Updated`) or whether the expected head was -/// stale and the caller must restart the splice (`HeadChanged`). -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum RefUpdateOutcome { - /// The ref was atomically advanced to the new commit. - Updated, - /// Another writer advanced the ref before our CAS — caller should - /// re-read the head and rebuild the commit on top of it. - HeadChanged, -} - /// Detect transient SQLite contention that should trigger a retry. /// /// Functional scope: @@ -375,16 +366,6 @@ fn anyhow_is_sqlite_busy(err: &anyhow::Error) -> bool { .any(is_sqlite_busy) } -/// Detect unique-constraint violations on the `reference` table. -/// -/// Functional scope: -/// - Used by the optimistic CAS path: when two writers race to insert the -/// same ref name, one will see a unique-constraint violation; we treat -/// that as a `HeadChanged` outcome rather than a hard error. -fn is_sqlite_unique_violation(err: &DbErr) -> bool { - matches!(err.sql_err(), Some(SqlErr::UniqueConstraintViolation(_))) -} - fn read_cleanup_regular_file( path: &Path, per_file_limit: u64, @@ -994,17 +975,16 @@ impl HistoryManager { Self::new_with_ref(storage, repo_path, db_conn, AI_REF) } - /// Build a manager bound to an arbitrary ref name. + /// Build a manager bound to a named Libra-owned history ref. /// /// Functional scope: - /// - Used by tests and tooling that need to write a parallel AI history - /// under a custom ref (e.g. for staging, comparison, or namespace - /// isolation). + /// - Used by traces writers and repair tooling that operate on a parallel + /// Libra-owned history. /// /// Boundary conditions: - /// - The ref name is not validated here; callers must ensure it is a - /// legal Git ref. The CAS path will fail loudly if the database - /// constraint rejects it. + /// - Reads and legacy maintenance paths retain the supplied name. The + /// append/checkpoint conditional-CAS path is fail-closed unless the name + /// maps to an [`OwnedRefSpec`]. pub fn new_with_ref( storage: Arc, repo_path: PathBuf, @@ -1137,11 +1117,13 @@ impl HistoryManager { .update_ref_if_matches(&self.ref_name, parent_commit_id, commit_hash) .await? { - RefUpdateOutcome::Updated => return Ok(()), - RefUpdateOutcome::HeadChanged if attempt < HISTORY_HEAD_CONFLICT_MAX_RETRIES => { + LinearRefTransactionOutcome::Updated => return Ok(()), + LinearRefTransactionOutcome::HeadChanged + if attempt < HISTORY_HEAD_CONFLICT_MAX_RETRIES => + { continue; } - RefUpdateOutcome::HeadChanged => { + LinearRefTransactionOutcome::HeadChanged => { return Err(anyhow!( "history head changed repeatedly while appending {}/{}", object_type, @@ -2019,7 +2001,7 @@ impl HistoryManager { ref_name: &str, expected_head: Option, new_hash: ObjectHash, - ) -> Result { + ) -> Result { self.update_ref_if_matches_with_extra(ref_name, expected_head, new_hash, None, None, None) .await } @@ -2039,172 +2021,33 @@ impl HistoryManager { extra: Option<(&dyn TracesTxnExtra, &TracesCommitCtx)>, deadline: Option, marker_fence: Option<&TracesWriterFence>, - ) -> Result { - let expected_commit = expected_head.map(|hash| hash.to_string()); - let new_commit = new_hash.to_string(); - - for attempt in 0..=SQLITE_BUSY_MAX_RETRIES { - if deadline.is_some_and(|deadline| Instant::now() >= deadline) { - bail!("checkpoint append exceeded the historical import execution deadline"); - } - let txn: DatabaseTransaction = - match crate::internal::db::begin_write_transaction(self.db_conn.as_ref()).await { - Ok(txn) => txn, - Err(err) if is_sqlite_busy(&err) && attempt < SQLITE_BUSY_MAX_RETRIES => { - sleep(Duration::from_millis( - SQLITE_BUSY_RETRY_BASE_MS * (attempt as u64 + 1), - )) - .await; - continue; - } - Err(err) => return Err(err).context("Failed to begin transaction"), - }; - - // An expired ordinary marker may have been fenced and retired by - // crash recovery while this writer was stalled. The marker check - // rides the same SQLite writer transaction as the ref/catalog CAS: - // cleanup wins first => this writer cannot publish; this writer - // wins first => cleanup observes the committed root/catalog. - if let Some(marker_fence) = marker_fence { - let entry = crate::internal::metadata::MetadataKv::get_with_conn( - &txn, - crate::internal::metadata::MetadataScope::AgentTracesInflight, - &marker_fence.session_id, - &marker_fence.attempt_id, - ) - .await - .context("revalidate checkpoint writer marker before ref update")?; - let Some(entry) = entry else { - txn.rollback().await.ok(); - bail!( - "checkpoint writer marker was fenced before ref update; retry the operation" - ); - }; - let marker = decode_and_validate_traces_inflight_marker( - &entry.value, - &entry.target, - &entry.key, - )?; - if let Err(error) = Self::ensure_marker_matches_fence(&marker, marker_fence) { - txn.rollback().await.ok(); - return Err(error.context("revalidate marker generation before ref update")); - } - } - - let existing = match reference::Entity::find() - .filter(reference::Column::Name.eq(ref_name)) - .filter(reference::Column::Kind.eq(ConfigKind::Branch)) - .one(&txn) - .await - { - Ok(existing) => existing, - Err(err) if is_sqlite_busy(&err) && attempt < SQLITE_BUSY_MAX_RETRIES => { - let _ = txn.rollback().await; - sleep(Duration::from_millis( - SQLITE_BUSY_RETRY_BASE_MS * (attempt as u64 + 1), - )) - .await; - continue; - } - Err(err) => return Err(err).context("Failed to query reference"), - }; - - let write_result = match existing { - Some(model) if model.commit != expected_commit => { - let _ = txn.rollback().await; - return Ok(RefUpdateOutcome::HeadChanged); - } - Some(model) => { - let mut update = reference::Entity::update_many() - .filter(reference::Column::Id.eq(model.id)) - .filter(reference::Column::Name.eq(ref_name)) - .filter(reference::Column::Kind.eq(ConfigKind::Branch)); - update = match expected_commit.as_ref() { - Some(commit) => update.filter(reference::Column::Commit.eq(commit.clone())), - None => update.filter(reference::Column::Commit.is_null()), - }; - - update - .col_expr( - reference::Column::Commit, - Expr::value(Some(new_commit.clone())), - ) - .exec(&txn) - .await - .map(Some) - } - None if expected_commit.is_some() => { - let _ = txn.rollback().await; - return Ok(RefUpdateOutcome::HeadChanged); - } - None => { - let new_ref = reference::ActiveModel { - name: Set(Some(ref_name.to_string())), - kind: Set(ConfigKind::Branch), - commit: Set(Some(new_commit.clone())), - remote: Set(None), - ..Default::default() - }; - match new_ref.insert(&txn).await { - Ok(_) => Ok(None), - Err(err) if is_sqlite_unique_violation(&err) => { - let _ = txn.rollback().await; - return Ok(RefUpdateOutcome::HeadChanged); - } - Err(err) => Err(err), - } - } - }; - - let rows_affected = match write_result { - Ok(rows_affected) => rows_affected, - Err(err) if is_sqlite_busy(&err) && attempt < SQLITE_BUSY_MAX_RETRIES => { - let _ = txn.rollback().await; - sleep(Duration::from_millis( - SQLITE_BUSY_RETRY_BASE_MS * (attempt as u64 + 1), - )) - .await; - continue; - } - Err(err) => return Err(err).context("Failed to compare-and-swap history head"), - }; - - if rows_affected.is_some_and(|result| result.rows_affected != 1) { - let _ = txn.rollback().await; - return Ok(RefUpdateOutcome::HeadChanged); - } - - // ADR-DR-10: companion writes ride the ref transaction. A - // failure here must NOT move the ref — roll back and fail - // closed (no HeadChanged retry: the failure is a gate/fence - // violation or DB fault, not a CAS race). - if let Some((extra, ctx)) = extra - && let Err(err) = extra.apply(&txn, ctx).await - { - let _ = txn.rollback().await; - return Err( - err.context("transactional companion writes failed; ref update rolled back") - ); - } - - if deadline.is_some_and(|deadline| Instant::now() >= deadline) { - let _ = txn.rollback().await; - bail!("checkpoint append exceeded the historical import execution deadline"); - } - - match txn.commit().await { - Ok(()) => return Ok(RefUpdateOutcome::Updated), - Err(err) if is_sqlite_busy(&err) && attempt < SQLITE_BUSY_MAX_RETRIES => { - sleep(Duration::from_millis( - SQLITE_BUSY_RETRY_BASE_MS * (attempt as u64 + 1), - )) - .await; - } - Err(err) => return Err(err).context("Failed to commit transaction"), - } + ) -> Result { + let spec = OwnedRefSpec::for_history_storage_name(ref_name) + .with_context(|| format!("history ref '{ref_name}' is not a named Libra-owned ref"))?; + if deadline.is_some_and(|deadline| Instant::now() >= deadline) { + bail!("checkpoint append exceeded the historical import execution deadline"); } - unreachable!("sqlite busy retry loop must return on success or terminal error") + let companion = HistoryLinearRefCompanion { + extra, + deadline, + marker_fence, + }; + let result = linear_ref_transaction( + self.db_conn.as_ref(), + spec, + expected_head, + new_hash, + deadline, + Some(&companion), + ) + .await; + match result { + Err(error) if error.downcast_ref::().is_some() => Err( + anyhow!("checkpoint append exceeded the historical import execution deadline"), + ), + result => result, + } } /// Append a checkpoint commit to this manager's ref. @@ -2600,7 +2443,7 @@ impl HistoryManager { ) .await? { - RefUpdateOutcome::Updated => { + LinearRefTransactionOutcome::Updated => { if cfg!(debug_assertions) && let Ok(value) = std::env::var("LIBRA_TEST_CHECKPOINT_POST_COMMIT_DELAY_MS") @@ -2617,10 +2460,12 @@ impl HistoryManager { object_count, }); } - RefUpdateOutcome::HeadChanged if attempt < HISTORY_HEAD_CONFLICT_MAX_RETRIES => { + LinearRefTransactionOutcome::HeadChanged + if attempt < HISTORY_HEAD_CONFLICT_MAX_RETRIES => + { continue; } - RefUpdateOutcome::HeadChanged => { + LinearRefTransactionOutcome::HeadChanged => { return Err(anyhow!( "history head changed repeatedly while appending checkpoint {}", params.checkpoint_id @@ -3435,7 +3280,7 @@ impl HistoryManager { .await? { ( - RefUpdateOutcome::Updated, + LinearRefTransactionOutcome::Updated, removed_checkpoints, deleted_object_index_rows, deleted_import_identities, @@ -3450,12 +3295,12 @@ impl HistoryManager { deleted_import_identities, }); } - (RefUpdateOutcome::HeadChanged, _, _, _) + (LinearRefTransactionOutcome::HeadChanged, _, _, _) if attempt < HISTORY_HEAD_CONFLICT_MAX_RETRIES => { continue; } - (RefUpdateOutcome::HeadChanged, _, _, _) => { + (LinearRefTransactionOutcome::HeadChanged, _, _, _) => { return Err(anyhow!( "traces head changed repeatedly while pruning checkpoints" )); @@ -4014,7 +3859,7 @@ impl HistoryManager { remove_ids: &HashSet, unreachable_oids: &[String], record_cloud_tombstones: bool, - ) -> Result<(RefUpdateOutcome, u64, u64, u64)> { + ) -> Result<(LinearRefTransactionOutcome, u64, u64, u64)> { let expected_commit = expected_head.map(|hash| hash.to_string()); let new_commit = new_head.map(|hash| hash.to_string()); let _object_index_deletion_fence = @@ -4064,7 +3909,7 @@ impl HistoryManager { let write_ref = match existing { Some(model) if model.commit != expected_commit => { let _ = txn.rollback().await; - return Ok((RefUpdateOutcome::HeadChanged, 0, 0, 0)); + return Ok((LinearRefTransactionOutcome::HeadChanged, 0, 0, 0)); } Some(model) => { let mut active: reference::ActiveModel = model.into(); @@ -4073,7 +3918,7 @@ impl HistoryManager { } None if expected_commit.is_some() => { let _ = txn.rollback().await; - return Ok((RefUpdateOutcome::HeadChanged, 0, 0, 0)); + return Ok((LinearRefTransactionOutcome::HeadChanged, 0, 0, 0)); } None => { let new_ref = reference::ActiveModel { @@ -4378,7 +4223,7 @@ impl HistoryManager { match txn.commit().await { Ok(()) => { return Ok(( - RefUpdateOutcome::Updated, + LinearRefTransactionOutcome::Updated, removed, deleted_object_index_rows, deleted_import_identities, @@ -4752,6 +4597,55 @@ pub trait TracesTxnExtra: Send + Sync { async fn apply(&self, txn: &DatabaseTransaction, ctx: &TracesCommitCtx) -> Result<()>; } +struct HistoryLinearRefCompanion<'a> { + extra: Option<(&'a dyn TracesTxnExtra, &'a TracesCommitCtx)>, + deadline: Option, + marker_fence: Option<&'a TracesWriterFence>, +} + +#[async_trait::async_trait] +impl LinearRefCompanion for HistoryLinearRefCompanion<'_> { + async fn apply(&self, txn: &DatabaseTransaction) -> Result<()> { + // An expired ordinary marker may have been fenced and retired while + // this writer was stalled. Revalidation remains inside the winning + // ref transaction. + if let Some(marker_fence) = self.marker_fence { + let entry = crate::internal::metadata::MetadataKv::get_with_conn( + txn, + crate::internal::metadata::MetadataScope::AgentTracesInflight, + &marker_fence.session_id, + &marker_fence.attempt_id, + ) + .await + .context("revalidate checkpoint writer marker before ref update")?; + let Some(entry) = entry else { + bail!("checkpoint writer marker was fenced before ref update; retry the operation"); + }; + let marker = decode_and_validate_traces_inflight_marker( + &entry.value, + &entry.target, + &entry.key, + )?; + HistoryManager::ensure_marker_matches_fence(&marker, marker_fence) + .context("revalidate marker generation before ref update")?; + } + + if let Some((extra, ctx)) = self.extra { + extra + .apply(txn, ctx) + .await + .context("transactional companion writes failed; ref update rolled back")?; + } + if self + .deadline + .is_some_and(|deadline| Instant::now() >= deadline) + { + bail!("checkpoint append exceeded the historical import execution deadline"); + } + Ok(()) + } +} + impl std::fmt::Debug for dyn TracesTxnExtra + '_ { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str("TracesTxnExtra") @@ -7113,7 +7007,7 @@ mod tests { .update_ref_if_matches(AI_REF, stale_head, stale_commit) .await .expect("stale ref update should not error"); - assert_eq!(outcome, RefUpdateOutcome::HeadChanged); + assert_eq!(outcome, LinearRefTransactionOutcome::HeadChanged); manager.append("plan", "plan-1", plan_hash).await.unwrap(); diff --git a/src/internal/ai/linear_ref.rs b/src/internal/ai/linear_ref.rs new file mode 100644 index 000000000..7f983f1ee --- /dev/null +++ b/src/internal/ai/linear_ref.rs @@ -0,0 +1,565 @@ +//! Atomic compare-and-swap updates for Libra-owned linear refs. +//! +//! Object construction deliberately happens before this module is called. +//! This module owns the SQLite transaction that advances a named ref and +//! applies its companion projection/catalog writes. A stale expected head is +//! returned to the caller as data so the caller can rebuild its proposal; it +//! is never retried here. + +use std::time::{Duration, Instant}; + +use anyhow::{Context, Result}; +use git_internal::hash::ObjectHash; +use sea_orm::{ + ActiveModelTrait, ColumnTrait, DatabaseConnection, DatabaseTransaction, DbErr, EntityTrait, + QueryFilter, Set, SqlErr, sea_query::Expr, +}; +use tokio::time::sleep; + +use crate::internal::{ + ai::history::AI_REF, + branch::{LEGACY_TRACES_BRANCH, TRACES_BRANCH}, + model::reference::{self, ConfigKind}, +}; + +const SQLITE_BUSY_MAX_RETRIES: usize = 15; +const SQLITE_BUSY_RETRY_BASE_MS: u64 = 100; + +/// Transfer policy attached to a Libra-owned ref. +#[cfg_attr( + not(test), + expect( + dead_code, + reason = "M2-03 freezes the policy consumed by the M2-04 MemoryWriter" + ) +)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum OwnedRefTransportPolicy { + /// The ref participates in ordinary repository transport. + Ordinary, + /// The ref is transported only through its dedicated command path. + DedicatedOnly, + /// The ref must stay in the local repository. + LocalOnly, +} + +/// Closed set of refs whose mutation policy is owned by Libra. +/// +/// Callers select a variant instead of supplying a name, so a user-controlled +/// string cannot impersonate the local-only Memory ref. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum OwnedRefSpec { + AiHistory, + Traces, + LegacyTraces, + #[cfg_attr( + not(test), + expect( + dead_code, + reason = "M2-03 freezes the owned ref consumed by the M2-04 MemoryWriter" + ) + )] + MemoryRepo, +} + +impl OwnedRefSpec { + pub(crate) const fn kind(self) -> ConfigKind { + ConfigKind::Branch + } + + pub(crate) const fn storage_name(self) -> &'static str { + match self { + Self::AiHistory => AI_REF, + Self::Traces => TRACES_BRANCH, + Self::LegacyTraces => LEGACY_TRACES_BRANCH, + Self::MemoryRepo => "libra/memory/repo", + } + } + + #[cfg_attr( + not(test), + expect( + dead_code, + reason = "M2-03 freezes the full ref consumed by M2-04 and M2-05" + ) + )] + pub(crate) const fn full_ref(self) -> &'static str { + match self { + Self::AiHistory => "refs/heads/libra/intent", + Self::Traces => "refs/libra/traces", + Self::LegacyTraces => "refs/libra/agent-traces", + Self::MemoryRepo => "refs/heads/libra/memory/repo", + } + } + + #[cfg_attr( + not(test), + expect( + dead_code, + reason = "M2-03 freezes the transport policy consumed by M2-04 and M2-05" + ) + )] + pub(crate) const fn transport_policy(self) -> OwnedRefTransportPolicy { + match self { + Self::AiHistory => OwnedRefTransportPolicy::Ordinary, + Self::Traces | Self::LegacyTraces => OwnedRefTransportPolicy::DedicatedOnly, + Self::MemoryRepo => OwnedRefTransportPolicy::LocalOnly, + } + } + + /// Resolve the exact storage names accepted by `HistoryManager`. + /// + /// Full Memory ref classification belongs to M2-05. This conversion is + /// intentionally narrower: it only admits the two histories that already + /// use `HistoryManager` today. + pub(crate) fn for_history_storage_name(name: &str) -> Option { + match name { + AI_REF => Some(Self::AiHistory), + TRACES_BRANCH => Some(Self::Traces), + LEGACY_TRACES_BRANCH => Some(Self::LegacyTraces), + _ => None, + } + } +} + +/// Result of one conditional ref transaction. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum LinearRefTransactionOutcome { + Updated, + HeadChanged, +} + +#[derive(Debug, thiserror::Error)] +#[error("linear ref transaction exceeded its execution deadline")] +pub(crate) struct LinearRefDeadlineExceeded; + +/// Companion mutation applied after the ref CAS succeeds and before commit. +#[async_trait::async_trait] +pub(crate) trait LinearRefCompanion: Send + Sync { + async fn apply(&self, txn: &DatabaseTransaction) -> Result<()>; +} + +/// Advance an owned ref and apply `companion` in the same SQLite transaction. +/// +/// Transient SQLite lock failures are retried with a bounded delay. A stale +/// expected head is not retried: callers must rebuild any objects/proposal on +/// the winning head and explicitly invoke the primitive again. +pub(crate) async fn linear_ref_transaction( + db_conn: &DatabaseConnection, + spec: OwnedRefSpec, + expected_head: Option, + new_head: ObjectHash, + deadline: Option, + companion: Option<&dyn LinearRefCompanion>, +) -> Result { + let expected_commit = expected_head.map(|hash| hash.to_string()); + let new_commit = new_head.to_string(); + + for attempt in 0..=SQLITE_BUSY_MAX_RETRIES { + ensure_before_deadline(deadline)?; + let txn = match crate::internal::db::begin_write_transaction(db_conn).await { + Ok(txn) => txn, + Err(err) if is_sqlite_busy(&err) && attempt < SQLITE_BUSY_MAX_RETRIES => { + retry_after_busy(attempt).await; + continue; + } + Err(err) => return Err(err).context("failed to begin linear ref transaction"), + }; + + let existing = match reference::Entity::find() + .filter(reference::Column::Name.eq(spec.storage_name())) + .filter(reference::Column::Kind.eq(spec.kind())) + .one(&txn) + .await + { + Ok(existing) => existing, + Err(err) if is_sqlite_busy(&err) && attempt < SQLITE_BUSY_MAX_RETRIES => { + let _ = txn.rollback().await; + retry_after_busy(attempt).await; + continue; + } + Err(err) => return Err(err).context("failed to query owned reference"), + }; + + let write_result = match existing { + Some(model) if model.commit != expected_commit => { + let _ = txn.rollback().await; + return Ok(LinearRefTransactionOutcome::HeadChanged); + } + Some(model) => { + let mut update = reference::Entity::update_many() + .filter(reference::Column::Id.eq(model.id)) + .filter(reference::Column::Name.eq(spec.storage_name())) + .filter(reference::Column::Kind.eq(spec.kind())); + update = match expected_commit.as_ref() { + Some(commit) => update.filter(reference::Column::Commit.eq(commit.clone())), + None => update.filter(reference::Column::Commit.is_null()), + }; + update + .col_expr( + reference::Column::Commit, + Expr::value(Some(new_commit.clone())), + ) + .exec(&txn) + .await + .map(Some) + } + None if expected_commit.is_some() => { + let _ = txn.rollback().await; + return Ok(LinearRefTransactionOutcome::HeadChanged); + } + None => { + let new_ref = reference::ActiveModel { + name: Set(Some(spec.storage_name().to_string())), + kind: Set(spec.kind()), + commit: Set(Some(new_commit.clone())), + remote: Set(None), + ..Default::default() + }; + match new_ref.insert(&txn).await { + Ok(_) => Ok(None), + Err(err) if is_sqlite_unique_violation(&err) => { + let _ = txn.rollback().await; + return Ok(LinearRefTransactionOutcome::HeadChanged); + } + Err(err) => Err(err), + } + } + }; + + let rows_affected = match write_result { + Ok(rows_affected) => rows_affected, + Err(err) if is_sqlite_busy(&err) && attempt < SQLITE_BUSY_MAX_RETRIES => { + let _ = txn.rollback().await; + retry_after_busy(attempt).await; + continue; + } + Err(err) => return Err(err).context("failed to compare-and-swap owned reference"), + }; + + if rows_affected.is_some_and(|result| result.rows_affected != 1) { + let _ = txn.rollback().await; + return Ok(LinearRefTransactionOutcome::HeadChanged); + } + + if let Some(companion) = companion + && let Err(err) = companion.apply(&txn).await + { + let _ = txn.rollback().await; + return Err(err.context("companion mutation failed; owned ref update rolled back")); + } + + if let Err(err) = ensure_before_deadline(deadline) { + let _ = txn.rollback().await; + return Err(err); + } + + // COMMIT is deliberately not cancellable. Once it starts, wait for a + // definitive SQLite result so callers never observe an ambiguous + // "reported timeout but possibly committed" outcome. + match txn.commit().await { + Ok(()) => return Ok(LinearRefTransactionOutcome::Updated), + Err(err) if is_sqlite_busy(&err) && attempt < SQLITE_BUSY_MAX_RETRIES => { + retry_after_busy(attempt).await; + } + Err(err) => return Err(err).context("failed to commit linear ref transaction"), + } + } + + Err(anyhow::anyhow!( + "linear ref transaction exhausted its bounded SQLite retry budget" + )) +} + +fn ensure_before_deadline(deadline: Option) -> Result<()> { + if deadline.is_some_and(|deadline| Instant::now() >= deadline) { + return Err(LinearRefDeadlineExceeded.into()); + } + Ok(()) +} + +async fn retry_after_busy(attempt: usize) { + sleep(Duration::from_millis( + SQLITE_BUSY_RETRY_BASE_MS * (attempt as u64 + 1), + )) + .await; +} + +fn is_sqlite_busy(err: &DbErr) -> bool { + let message = err.to_string(); + message.contains("database is locked") || message.contains("database schema is locked") +} + +fn is_sqlite_unique_violation(err: &DbErr) -> bool { + matches!(err.sql_err(), Some(SqlErr::UniqueConstraintViolation(_))) +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use anyhow::bail; + use git_internal::hash::ObjectHash; + use sea_orm::{ConnectionTrait, Database, DatabaseConnection, Statement}; + + use super::*; + + struct InsertCompanion; + + #[async_trait::async_trait] + impl LinearRefCompanion for InsertCompanion { + async fn apply(&self, txn: &DatabaseTransaction) -> Result<()> { + txn.execute_raw(Statement::from_string( + txn.get_database_backend(), + "INSERT INTO config_kv(key, value, encrypted) VALUES ('projection', '1', 0)" + .to_string(), + )) + .await?; + Ok(()) + } + } + + struct FailingCompanion; + + #[async_trait::async_trait] + impl LinearRefCompanion for FailingCompanion { + async fn apply(&self, txn: &DatabaseTransaction) -> Result<()> { + txn.execute_raw(Statement::from_string( + txn.get_database_backend(), + "INSERT INTO config_kv(key, value, encrypted) VALUES ('rolled-back', '1', 0)" + .to_string(), + )) + .await?; + bail!("simulated companion failure") + } + } + + struct DeadlineExpiringCompanion; + + #[async_trait::async_trait] + impl LinearRefCompanion for DeadlineExpiringCompanion { + async fn apply(&self, txn: &DatabaseTransaction) -> Result<()> { + txn.execute_raw(Statement::from_string( + txn.get_database_backend(), + "INSERT INTO config_kv(key, value, encrypted) VALUES ('deadline', '1', 0)" + .to_string(), + )) + .await?; + tokio::time::sleep(Duration::from_millis(20)).await; + Ok(()) + } + } + + async fn test_database() -> DatabaseConnection { + let database = Database::connect("sqlite::memory:") + .await + .expect("connect in-memory database"); + database + .execute_raw(Statement::from_string( + database.get_database_backend(), + "CREATE TABLE reference(\ + id INTEGER PRIMARY KEY AUTOINCREMENT,\ + name TEXT, kind TEXT NOT NULL, \"commit\" TEXT, remote TEXT, worktree_id TEXT);\ + CREATE UNIQUE INDEX idx_name_kind ON reference(name, kind);\ + CREATE TABLE config_kv(\ + id INTEGER PRIMARY KEY AUTOINCREMENT, key TEXT NOT NULL,\ + value TEXT NOT NULL, encrypted INTEGER NOT NULL DEFAULT 0);" + .to_string(), + )) + .await + .expect("create ref transaction fixture"); + database + } + + fn oid(value: &str) -> ObjectHash { + ObjectHash::from_str(value).expect("valid test oid") + } + + #[test] + fn linear_ref_transaction_memory_spec_is_closed_and_local_only() { + let expected = [ + ( + OwnedRefSpec::AiHistory, + "libra/intent", + "refs/heads/libra/intent", + OwnedRefTransportPolicy::Ordinary, + ), + ( + OwnedRefSpec::Traces, + "traces", + "refs/libra/traces", + OwnedRefTransportPolicy::DedicatedOnly, + ), + ( + OwnedRefSpec::LegacyTraces, + "agent-traces", + "refs/libra/agent-traces", + OwnedRefTransportPolicy::DedicatedOnly, + ), + ( + OwnedRefSpec::MemoryRepo, + "libra/memory/repo", + "refs/heads/libra/memory/repo", + OwnedRefTransportPolicy::LocalOnly, + ), + ]; + for (spec, storage_name, full_ref, transport_policy) in expected { + assert_eq!(spec.kind(), ConfigKind::Branch); + assert_eq!(spec.storage_name(), storage_name); + assert_eq!(spec.full_ref(), full_ref); + assert_eq!(spec.transport_policy(), transport_policy); + } + assert_eq!( + OwnedRefSpec::for_history_storage_name("libra/memory/repo"), + None + ); + assert_eq!( + OwnedRefSpec::for_history_storage_name("libra/memory/repo-user"), + None + ); + } + + #[tokio::test] + async fn linear_ref_transaction_commits_ref_and_companion_atomically() { + let database = test_database().await; + let new_head = oid("e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"); + + let outcome = linear_ref_transaction( + &database, + OwnedRefSpec::MemoryRepo, + None, + new_head, + None, + Some(&InsertCompanion), + ) + .await + .expect("commit ref transaction"); + assert_eq!(outcome, LinearRefTransactionOutcome::Updated); + + let ref_count: i64 = database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + "SELECT COUNT(*) AS count FROM reference WHERE name = 'libra/memory/repo'" + .to_string(), + )) + .await + .expect("query ref") + .expect("ref row") + .try_get("", "count") + .expect("ref count"); + let companion_count: i64 = database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + "SELECT COUNT(*) AS count FROM config_kv WHERE key = 'projection'".to_string(), + )) + .await + .expect("query companion") + .expect("companion row") + .try_get("", "count") + .expect("companion count"); + assert_eq!((ref_count, companion_count), (1, 1)); + } + + #[tokio::test] + async fn linear_ref_transaction_rolls_back_ref_when_companion_fails() { + let database = test_database().await; + let new_head = oid("e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"); + + let error = linear_ref_transaction( + &database, + OwnedRefSpec::MemoryRepo, + None, + new_head, + None, + Some(&FailingCompanion), + ) + .await + .expect_err("companion failure must roll back ref"); + assert!(format!("{error:#}").contains("simulated companion failure")); + + let ref_count: i64 = database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + "SELECT COUNT(*) AS count FROM reference".to_string(), + )) + .await + .expect("query refs") + .expect("count row") + .try_get("", "count") + .expect("ref count"); + let companion_count: i64 = database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + "SELECT COUNT(*) AS count FROM config_kv WHERE key = 'rolled-back'".to_string(), + )) + .await + .expect("query rolled-back companion") + .expect("count row") + .try_get("", "count") + .expect("companion count"); + assert_eq!((ref_count, companion_count), (0, 0)); + } + + #[tokio::test] + async fn linear_ref_transaction_reports_stale_head_without_companion() { + let database = test_database().await; + let winner = oid("e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"); + let stale = oid("f4e6d0434b8b29ae775ad8c2e48c5391e69de29b"); + let proposed = oid("a4e6d0434b8b29ae775ad8c2e48c5391e69de29b"); + linear_ref_transaction( + &database, + OwnedRefSpec::MemoryRepo, + None, + winner, + None, + None, + ) + .await + .expect("seed winner"); + + let outcome = linear_ref_transaction( + &database, + OwnedRefSpec::MemoryRepo, + Some(stale), + proposed, + None, + Some(&FailingCompanion), + ) + .await + .expect("stale CAS is a typed outcome"); + assert_eq!(outcome, LinearRefTransactionOutcome::HeadChanged); + } + + #[tokio::test] + async fn linear_ref_transaction_deadline_rolls_back_before_commit() { + let database = test_database().await; + let new_head = oid("e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"); + let deadline = Instant::now() + Duration::from_millis(5); + + let error = linear_ref_transaction( + &database, + OwnedRefSpec::MemoryRepo, + None, + new_head, + Some(deadline), + Some(&DeadlineExpiringCompanion), + ) + .await + .expect_err("deadline reached before commit must abort the transaction"); + assert!(error.downcast_ref::().is_some()); + + for table in ["reference", "config_kv"] { + let count: i64 = database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + format!("SELECT COUNT(*) AS count FROM {table}"), + )) + .await + .expect("query table after deadline rollback") + .expect("count row") + .try_get("", "count") + .expect("row count"); + assert_eq!(count, 0, "{table} mutation must roll back"); + } + } +} diff --git a/src/internal/ai/mod.rs b/src/internal/ai/mod.rs index cd4c1f408..1fc083caf 100644 --- a/src/internal/ai/mod.rs +++ b/src/internal/ai/mod.rs @@ -110,6 +110,8 @@ pub mod intent; pub mod intentspec; // VCS-side helpers used by tools that touch the repository. pub mod libra_vcs; +// Atomic CAS + companion writes for Libra-owned linear refs. +pub(crate) mod linear_ref; // Model Context Protocol server exposing Libra to MCP-aware clients. pub mod mcp; // Versioned Agent Memory contracts. Storage and compiler implementations stay From d0193aa051019e5964e680651ba6a9434b2a417f Mon Sep 17 00:00:00 2001 From: anduin9527 Date: Mon, 24 Aug 2026 23:39:47 +0800 Subject: [PATCH 07/18] feat(memory): add authoritative memory writer Signed-off-by: anduin9527 --- docs/error-codes.md | 8 + src/internal/ai/linear_ref.rs | 21 - src/internal/ai/memory/error.rs | 57 ++ src/internal/ai/memory/mod.rs | 5 + src/internal/ai/memory/policy.rs | 233 ++++++ src/internal/ai/memory/store.rs | 534 +++++++++++++ src/internal/ai/memory/tree.rs | 722 ++++++++++++++++++ src/internal/ai/memory/writer.rs | 1215 ++++++++++++++++++++++++++++++ 8 files changed, 2774 insertions(+), 21 deletions(-) create mode 100644 src/internal/ai/memory/error.rs create mode 100644 src/internal/ai/memory/policy.rs create mode 100644 src/internal/ai/memory/store.rs create mode 100644 src/internal/ai/memory/tree.rs create mode 100644 src/internal/ai/memory/writer.rs diff --git a/docs/error-codes.md b/docs/error-codes.md index cc10e5763..30eb87de5 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -89,6 +89,10 @@ structured report is always present. | `128` | `LBR-REPO-002` | `repo` | Repository metadata is corrupt or incompatible | missing DB, corrupted metadata | | `128` | `LBR-REPO-003` | `repo` | Repository state blocks the operation | no commits yet, detached state mismatch, missing configured remote | | `128` | `LBR-MEMORY-001` | `repo` | Repository Memory digest key is missing, invalid, or cannot be decrypted | missing encrypted `memory.keyed_digest.v1`, duplicate/plaintext entry, unsupported generation, unavailable repository vault key, or cached/persisted key mismatch | +| `128` | `LBR-MEMORY-002` | `repo` | Memory proposal violates the persisted object contract | unsupported schema, malformed Episode envelope, or non-canonical note/event payload | +| `128` | `LBR-MEMORY-003` | `repo` | Memory writer policy rejected the proposal | authenticated target mismatch, non-local Memory scope, or unknown repository digest key ID | +| `128` | `LBR-MEMORY-004` | `repo` | Memory authority or its rebuildable projection is corrupt | invalid manifest, merge commit on the linear Memory ref, broken revision ancestry, or projection watermark mismatch | +| `128` | `LBR-MEMORY-005` | `repo` | Memory writer could not commit an atomic revision | local object write failure, SQLite companion failure, or exhausted bounded ref-conflict retries | | `128` | `LBR-WORKTREE-001` | `repo` | Pagination cursor is malformed, foreign, or expired | `libra worktree doctor --cursor ` | | `128` | `LBR-WORKTREE-002` | `repo` | A worktree/workspace scope is corrupt or unreadable, so the diagnosis would be incomplete | `libra worktree doctor` where a `workspace_record` row or the worktree registry cannot be read | | `128` | `LBR-CONFIG-001` | `config` | Global config DB schema is newer than this Libra binary supports | `pull`, `push`, `fetch`, `clone`, or `cloud` would otherwise silently ignore global storage config | @@ -165,6 +169,10 @@ structured report is always present. | `LBR-REPO-002` | Repository metadata is corrupt or incompatible | | `LBR-REPO-003` | Repository state blocks the operation | | `LBR-MEMORY-001` | Repository Memory digest key is missing, invalid, or cannot be decrypted; restore the original encrypted entry or repair the repository vault before writing new Memory data | +| `LBR-MEMORY-002` | The Memory proposal is incompatible with the persisted contract; regenerate it with the supported schema and canonical fields | +| `LBR-MEMORY-003` | The Memory proposal failed repository policy; use the authenticated target and the current repository digest key | +| `LBR-MEMORY-004` | Memory authority and projection disagree or contain invalid history; stop writes and rebuild or repair the projection before retrying | +| `LBR-MEMORY-005` | The Memory revision could not be committed atomically; fix local storage/SQLite health or retry after ref contention subsides | | `LBR-WORKTREE-001` | The pagination cursor is malformed or expired; drop it and re-read the first page | | `LBR-WORKTREE-002` | A worktree/workspace scope is corrupt or unreadable; repair it before trusting any diagnostic report | diff --git a/src/internal/ai/linear_ref.rs b/src/internal/ai/linear_ref.rs index 7f983f1ee..a23b97a0c 100644 --- a/src/internal/ai/linear_ref.rs +++ b/src/internal/ai/linear_ref.rs @@ -26,13 +26,6 @@ const SQLITE_BUSY_MAX_RETRIES: usize = 15; const SQLITE_BUSY_RETRY_BASE_MS: u64 = 100; /// Transfer policy attached to a Libra-owned ref. -#[cfg_attr( - not(test), - expect( - dead_code, - reason = "M2-03 freezes the policy consumed by the M2-04 MemoryWriter" - ) -)] #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) enum OwnedRefTransportPolicy { /// The ref participates in ordinary repository transport. @@ -52,13 +45,6 @@ pub(crate) enum OwnedRefSpec { AiHistory, Traces, LegacyTraces, - #[cfg_attr( - not(test), - expect( - dead_code, - reason = "M2-03 freezes the owned ref consumed by the M2-04 MemoryWriter" - ) - )] MemoryRepo, } @@ -92,13 +78,6 @@ impl OwnedRefSpec { } } - #[cfg_attr( - not(test), - expect( - dead_code, - reason = "M2-03 freezes the transport policy consumed by M2-04 and M2-05" - ) - )] pub(crate) const fn transport_policy(self) -> OwnedRefTransportPolicy { match self { Self::AiHistory => OwnedRefTransportPolicy::Ordinary, diff --git a/src/internal/ai/memory/error.rs b/src/internal/ai/memory/error.rs new file mode 100644 index 000000000..99f4100d4 --- /dev/null +++ b/src/internal/ai/memory/error.rs @@ -0,0 +1,57 @@ +use thiserror::Error; + +use super::domain::MemoryContractError; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum MemoryWriterErrorKind { + DigestKeyUnavailable, + InvalidProposal, + PolicyRejected, + UnknownDigestKey, + CorruptHistory, + CorruptProjection, + StorageFailure, + ConflictExhausted, +} + +impl MemoryWriterErrorKind { + pub(crate) const fn stable_code(self) -> &'static str { + match self { + Self::DigestKeyUnavailable => "LBR-MEMORY-001", + Self::InvalidProposal => "LBR-MEMORY-002", + Self::PolicyRejected | Self::UnknownDigestKey => "LBR-MEMORY-003", + Self::CorruptHistory | Self::CorruptProjection => "LBR-MEMORY-004", + Self::StorageFailure | Self::ConflictExhausted => "LBR-MEMORY-005", + } + } +} + +#[derive(Clone, Debug, Error)] +#[error("{code}: {summary}", code = .kind.stable_code())] +pub(crate) struct MemoryWriterError { + kind: MemoryWriterErrorKind, + summary: String, +} + +impl MemoryWriterError { + pub(crate) fn new(kind: MemoryWriterErrorKind, summary: impl Into) -> Self { + Self { + kind, + summary: summary.into(), + } + } + + pub(crate) const fn kind(&self) -> MemoryWriterErrorKind { + self.kind + } + + pub(crate) const fn stable_code(&self) -> &'static str { + self.kind.stable_code() + } +} + +impl From for MemoryWriterError { + fn from(error: MemoryContractError) -> Self { + Self::new(MemoryWriterErrorKind::InvalidProposal, error.to_string()) + } +} diff --git a/src/internal/ai/memory/mod.rs b/src/internal/ai/memory/mod.rs index 3eea4ef6e..bb698a284 100644 --- a/src/internal/ai/memory/mod.rs +++ b/src/internal/ai/memory/mod.rs @@ -11,6 +11,11 @@ mod canonical; mod domain; +mod error; mod fts_sql; mod job_sql; +mod policy; +mod store; +mod tree; mod validation; +mod writer; diff --git a/src/internal/ai/memory/policy.rs b/src/internal/ai/memory/policy.rs new file mode 100644 index 000000000..9eb895f17 --- /dev/null +++ b/src/internal/ai/memory/policy.rs @@ -0,0 +1,233 @@ +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +use super::{ + domain::{ + ActorRefV1, EpisodeClaimV1, EpisodeRoot, EvidenceRefV1, MemoryNoteV1, MemoryScopeV1, + MemorySensitivity, MemoryVisibility, + }, + error::{MemoryWriterError, MemoryWriterErrorKind}, +}; + +const REPO_EPISODE_POLICY_VERSION: &str = "repo-policy-v1"; +const REPO_EPISODE_ACL_POLICY_ID: &str = "repo-default-v1"; +const REPO_EPISODE_PRODUCER: &str = "libra-memory/1"; +const REPO_EPISODE_POLICY_SNAPSHOT: &[u8] = br#"{ + "acl_policy":"repo-default-v1", + "auto_confirm":true, + "producer":"libra-memory/1", + "scope":"repo", + "transport":"local_only", + "visibility":"repo_local", + "writer_version":1 +}"#; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct AuthenticatedMemoryContext { + repository_id: String, + actor: ActorRefV1, +} + +impl AuthenticatedMemoryContext { + pub(crate) fn new( + repository_id: impl Into, + actor: ActorRefV1, + ) -> Result { + let repository_id = repository_id.into(); + if repository_id.is_empty() || actor.principal_id.is_empty() { + return Err(MemoryWriterError::new( + MemoryWriterErrorKind::PolicyRejected, + "authenticated repository and actor identity are required", + )); + } + Ok(Self { + repository_id, + actor, + }) + } + + pub(crate) fn repository_id(&self) -> &str { + &self.repository_id + } + + pub(crate) fn actor(&self) -> &ActorRefV1 { + &self.actor + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct TrustedMemoryTarget { + root: EpisodeRoot, +} + +impl TrustedMemoryTarget { + pub(crate) fn episode(root: EpisodeRoot) -> Self { + Self { root } + } + + pub(crate) fn root(&self) -> &EpisodeRoot { + &self.root + } + + pub(crate) const fn scope_key(&self) -> &'static str { + "repo" + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct DeterministicMemoryProposal { + note: MemoryNoteV1, +} + +impl DeterministicMemoryProposal { + pub(crate) fn new(note: MemoryNoteV1) -> Self { + Self { note } + } + + pub(crate) fn note(&self) -> &MemoryNoteV1 { + &self.note + } + + #[cfg(test)] + pub(super) fn note_mut(&mut self) -> &mut MemoryNoteV1 { + &mut self.note + } +} + +pub(super) fn validate_writer_policy( + context: &AuthenticatedMemoryContext, + target: &TrustedMemoryTarget, + proposal: &DeterministicMemoryProposal, + repository_id: &str, + key_id: Uuid, +) -> Result<(), MemoryWriterError> { + let note = proposal.note(); + if context.repository_id() != repository_id { + return Err(policy_error( + "authenticated repository does not match writer repository", + )); + } + if note.compile_record.policy_version != REPO_EPISODE_POLICY_VERSION { + return Err(policy_error( + "proposal policy version is not supported by the repo Episode writer", + )); + } + if note.acl_policy_id != REPO_EPISODE_ACL_POLICY_ID { + return Err(policy_error( + "proposal ACL policy is not supported by the repo Episode writer", + )); + } + if note.sensitivity == MemorySensitivity::SecretLike { + return Err(policy_error( + "secret-like Memory cannot enter the unencrypted repository object store", + )); + } + if note.compile_record.origin != super::domain::CompileOriginV1::EpisodeCompiler + || note.compile_record.producer != REPO_EPISODE_PRODUCER + { + return Err(policy_error( + "proposal producer is not supported by the repo Episode writer", + )); + } + if ¬e.author != context.actor() { + return Err(policy_error( + "proposal author does not match authenticated actor", + )); + } + if note.note_id != target.root().note_id() + || note.namespace != target.root().namespace() + || note.path != target.root().path() + || note.scope != MemoryScopeV1::Repo + || note.visibility != MemoryVisibility::RepoLocal + || note.episode.as_ref().is_none_or(|episode| { + episode.root_kind != target.root().kind() || episode.root_id != target.root().id() + }) + { + return Err(policy_error( + "proposal does not match the trusted Episode target", + )); + } + + validate_digest_key(¬e.compile_record.idempotency_key, key_id)?; + for digest in ¬e.compile_record.input_hashes { + validate_optional_digest_key(digest, key_id)?; + } + for evidence in all_evidence(note) { + validate_optional_digest_key(&evidence.fragment_digest, key_id)?; + } + Ok(()) +} + +pub(super) fn policy_snapshot_digest(policy_version: &str) -> Result { + if policy_version != REPO_EPISODE_POLICY_VERSION { + return Err(policy_error( + "Memory history uses an unsupported repo Episode policy version", + )); + } + let mut hasher = Sha256::new(); + hasher.update(policy_version.as_bytes()); + hasher.update(b"\0"); + hasher.update(REPO_EPISODE_POLICY_SNAPSHOT); + Ok(format!("sha256:{}", hex::encode(hasher.finalize()))) +} + +fn all_evidence(note: &MemoryNoteV1) -> Vec<&EvidenceRefV1> { + let mut evidence = Vec::new(); + evidence.extend(note.evidence_refs.iter()); + for link in ¬e.links { + evidence.extend(link.evidence_refs.iter()); + } + for entity in ¬e.entities { + evidence.extend(entity.evidence_refs.iter()); + } + if let Some(episode) = ¬e.episode { + push_claim_evidence(&mut evidence, &episode.goal); + push_claim_evidence(&mut evidence, &episode.summary); + for claim in episode + .observations + .iter() + .chain(&episode.inferences) + .chain(&episode.decisions) + .chain(&episode.failed_attempts) + .chain(&episode.unresolved) + { + push_claim_evidence(&mut evidence, claim); + } + } + evidence +} + +fn push_claim_evidence<'a>(output: &mut Vec<&'a EvidenceRefV1>, claim: &'a EpisodeClaimV1) { + output.extend(claim.evidence_refs.iter()); +} + +fn validate_optional_digest_key(digest: &str, expected: Uuid) -> Result<(), MemoryWriterError> { + if digest.starts_with("hmac-sha256:") { + validate_digest_key(digest, expected)?; + } + Ok(()) +} + +fn validate_digest_key(digest: &str, expected: Uuid) -> Result<(), MemoryWriterError> { + let mut parts = digest.split(':'); + let algorithm = parts.next(); + let key = parts.next(); + let value = parts.next(); + if algorithm != Some("hmac-sha256") || value.is_none() || parts.next().is_some() { + return Err(policy_error("keyed digest envelope is malformed")); + } + let key_id = key + .and_then(|value| Uuid::parse_str(value).ok()) + .ok_or_else(|| policy_error("keyed digest key ID is malformed"))?; + if key_id != expected { + return Err(MemoryWriterError::new( + MemoryWriterErrorKind::UnknownDigestKey, + "proposal references an unknown repository digest key", + )); + } + Ok(()) +} + +fn policy_error(summary: &'static str) -> MemoryWriterError { + MemoryWriterError::new(MemoryWriterErrorKind::PolicyRejected, summary) +} diff --git a/src/internal/ai/memory/store.rs b/src/internal/ai/memory/store.rs new file mode 100644 index 000000000..1aa98b14f --- /dev/null +++ b/src/internal/ai/memory/store.rs @@ -0,0 +1,534 @@ +use std::sync::Arc; + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use git_internal::hash::ObjectHash; +use sea_orm::{ConnectionTrait, DatabaseConnection, DatabaseTransaction, QueryResult, Statement}; +use serde::Serialize; + +use super::{ + domain::{MemoryEventV1, MemoryNoteV1}, + error::{MemoryWriterError, MemoryWriterErrorKind}, + tree::parse_oid, +}; +use crate::internal::{ + ai::{keyed_digest::RepositoryKeyedDigest, linear_ref::LinearRefCompanion}, + workspace::RepoIdentity, +}; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct ProjectedCell { + pub(super) note_id: String, + pub(super) latest_revision_oid: ObjectHash, + pub(super) live_revision_oid: Option, +} + +pub(super) async fn read_memory_ref_head( + database: &DatabaseConnection, +) -> Result, MemoryWriterError> { + let rows = database + .query_all_raw(Statement::from_sql_and_values( + database.get_database_backend(), + "SELECT `commit` FROM reference + WHERE kind = 'Branch' AND remote IS NULL AND name = ? LIMIT 2", + ["libra/memory/repo".into()], + )) + .await + .map_err(|error| projection_error("query Memory ref", error))?; + if rows.len() > 1 { + return Err(corrupt_projection("duplicate repository Memory refs exist")); + } + rows.into_iter() + .next() + .map(|row| { + let value: Option = row + .try_get("", "commit") + .map_err(|error| projection_error("decode Memory ref", error))?; + value + .ok_or_else(|| corrupt_projection("Memory ref has no commit")) + .and_then(|value| parse_oid(&value)) + }) + .transpose() +} + +pub(super) async fn validate_projection_watermark( + database: &DatabaseConnection, + head: Option, + event_seq: u64, +) -> Result<(), MemoryWriterError> { + let row = database + .query_one_raw(Statement::from_sql_and_values( + database.get_database_backend(), + "SELECT projected_ref_oid, last_event_seq, schema_version + FROM memory_projection_state WHERE scope_key = 'repo'", + [], + )) + .await + .map_err(|error| projection_error("query Memory projection watermark", error))?; + match (head, row) { + (None, None) => Ok(()), + (Some(head), Some(row)) => { + let projected: String = row + .try_get("", "projected_ref_oid") + .map_err(|error| projection_error("decode Memory projection watermark", error))?; + let projected_seq: i64 = row + .try_get("", "last_event_seq") + .map_err(|error| projection_error("decode Memory projection sequence", error))?; + let schema_version: i64 = row + .try_get("", "schema_version") + .map_err(|error| projection_error("decode Memory projection schema", error))?; + let expected_seq = i64::try_from(event_seq).map_err(|_| { + corrupt_projection("Memory manifest event sequence exceeds SQLite range") + })?; + if projected == head.to_string() && projected_seq == expected_seq && schema_version == 1 + { + Ok(()) + } else { + Err(corrupt_projection( + "Memory projection watermark does not match the authoritative ref", + )) + } + } + _ => Err(corrupt_projection( + "Memory ref and projection watermark are not initialized together", + )), + } +} + +pub(super) async fn find_cell( + database: &DatabaseConnection, + namespace: &str, + path: &str, +) -> Result, MemoryWriterError> { + let rows = database + .query_all_raw(Statement::from_sql_and_values( + database.get_database_backend(), + "SELECT note_id, latest_revision_oid, live_revision_oid + FROM memory_head + WHERE scope_key = 'repo' AND namespace = ? AND path = ? LIMIT 2", + [namespace.into(), path.into()], + )) + .await + .map_err(|error| projection_error("query Memory Cell", error))?; + decode_unique_cell(rows) +} + +fn decode_unique_cell(rows: Vec) -> Result, MemoryWriterError> { + if rows.len() > 1 { + return Err(corrupt_projection( + "duplicate Memory heads exist for one Cell", + )); + } + rows.into_iter() + .next() + .map(projected_cell_from_row) + .transpose() +} + +fn projected_cell_from_row(row: QueryResult) -> Result { + let note_id = row + .try_get("", "note_id") + .map_err(|error| projection_error("decode Memory note ID", error))?; + let latest: String = row + .try_get("", "latest_revision_oid") + .map_err(|error| projection_error("decode Memory revision OID", error))?; + let live_revision_oid = row + .try_get("", "live_revision_oid") + .map_err(|error| projection_error("decode Memory live revision OID", error))?; + Ok(ProjectedCell { + note_id, + latest_revision_oid: parse_oid(&latest)?, + live_revision_oid, + }) +} + +#[derive(Clone)] +pub(super) struct ProjectionMutation { + pub(super) note: MemoryNoteV1, + pub(super) event: MemoryEventV1, + pub(super) revision_oid: ObjectHash, + pub(super) commit_oid: ObjectHash, + pub(super) is_create: bool, + pub(super) rebuilt_at_ms: i64, + pub(super) expected_head: Option, + pub(super) expected_event_seq: u64, + pub(super) expected_cell: Option, + pub(super) repository_id: String, + pub(super) digest_provider: Arc, +} + +#[async_trait] +impl LinearRefCompanion for ProjectionMutation { + async fn apply(&self, txn: &DatabaseTransaction) -> Result<()> { + revalidate_snapshot(txn, self).await?; + if self.is_create { + insert_note(txn, &self.note).await?; + } else { + update_note(txn, &self.note).await?; + } + insert_revision(txn, &self.note, self.revision_oid).await?; + replace_links(txn, &self.note, self.revision_oid).await?; + replace_episode_paths(txn, &self.note, self.revision_oid).await?; + upsert_head(txn, &self.note, &self.event, self.revision_oid).await?; + upsert_projection_state( + txn, + &self.note, + &self.event, + self.commit_oid, + self.rebuilt_at_ms, + ) + .await?; + Ok(()) + } +} + +async fn revalidate_snapshot( + txn: &DatabaseTransaction, + mutation: &ProjectionMutation, +) -> Result<(), MemoryWriterError> { + let repository = RepoIdentity::resolve(txn).await.map_err(|error| { + MemoryWriterError::new( + MemoryWriterErrorKind::CorruptProjection, + format!("repository identity is invalid during Memory commit: {error}"), + ) + })?; + if repository.as_str() != mutation.repository_id + || mutation.digest_provider.repository_id() != mutation.repository_id + { + return Err(corrupt_projection( + "repository identity changed during Memory commit", + )); + } + mutation + .digest_provider + .validate_for_connection(txn) + .await + .map_err(|error| { + MemoryWriterError::new( + MemoryWriterErrorKind::DigestKeyUnavailable, + error.to_string(), + ) + })?; + + let projection = txn + .query_one_raw(Statement::from_string( + txn.get_database_backend(), + "SELECT projected_ref_oid, last_event_seq FROM memory_projection_state + WHERE scope_key = 'repo'" + .to_string(), + )) + .await + .map_err(|error| projection_error("revalidate Memory projection watermark", error))?; + match (mutation.expected_head, projection) { + (None, None) => {} + (Some(expected), Some(row)) => { + let projected: String = row.try_get("", "projected_ref_oid").map_err(|error| { + projection_error("decode revalidated Memory projection watermark", error) + })?; + let event_seq: i64 = row.try_get("", "last_event_seq").map_err(|error| { + projection_error("decode revalidated Memory projection sequence", error) + })?; + let expected_event_seq = i64::try_from(mutation.expected_event_seq).map_err(|_| { + corrupt_projection("Memory snapshot event sequence exceeds SQLite range") + })?; + if projected != expected.to_string() || event_seq != expected_event_seq { + return Err(corrupt_projection( + "Memory projection changed after the writer snapshot", + )); + } + } + _ => { + return Err(corrupt_projection( + "Memory projection appeared or disappeared after the writer snapshot", + )); + } + } + + let current_cell = + find_cell_in_transaction(txn, &mutation.note.namespace, &mutation.note.path).await?; + if current_cell != mutation.expected_cell { + return Err(corrupt_projection( + "Memory Cell changed after the writer snapshot", + )); + } + Ok(()) +} + +async fn find_cell_in_transaction( + txn: &DatabaseTransaction, + namespace: &str, + path: &str, +) -> Result, MemoryWriterError> { + let rows = txn + .query_all_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "SELECT note_id, latest_revision_oid, live_revision_oid + FROM memory_head + WHERE scope_key = 'repo' AND namespace = ? AND path = ? LIMIT 2", + [namespace.into(), path.into()], + )) + .await + .map_err(|error| projection_error("revalidate Memory Cell", error))?; + decode_unique_cell(rows) +} + +async fn insert_note(txn: &DatabaseTransaction, note: &MemoryNoteV1) -> Result<()> { + execute( + txn, + "INSERT INTO memory_note_index ( + note_id, scope_key, namespace, path, kind, lifecycle, review_state, + confidence, trust, sensitivity, visibility, acl_policy_id, origin, + idempotency_key, idempotency_scope, created_at + ) VALUES (?, 'repo', ?, ?, ?, ?, 'confirmed', ?, ?, ?, ?, ?, ?, ?, ?, ?)", + vec![ + note.note_id.to_string().into(), + note.namespace.clone().into(), + note.path.clone().into(), + enum_label(¬e.kind)?.into(), + enum_label(¬e.lifecycle)?.into(), + enum_label(¬e.confidence)?.into(), + enum_label(¬e.trust)?.into(), + enum_label(¬e.sensitivity)?.into(), + enum_label(¬e.visibility)?.into(), + note.acl_policy_id.clone().into(), + enum_label(¬e.compile_record.origin)?.into(), + note.compile_record.idempotency_key.clone().into(), + enum_label(¬e.compile_record.idempotency_scope)?.into(), + note.created_at.to_rfc3339().into(), + ], + ) + .await +} + +async fn update_note(txn: &DatabaseTransaction, note: &MemoryNoteV1) -> Result<()> { + let result = txn + .execute_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "UPDATE memory_note_index SET + review_state = 'confirmed', confidence = ?, trust = ?, sensitivity = ?, + visibility = ?, acl_policy_id = ? + WHERE note_id = ? AND scope_key = 'repo' AND namespace = ? AND path = ?", + [ + enum_label(¬e.confidence)?.into(), + enum_label(¬e.trust)?.into(), + enum_label(¬e.sensitivity)?.into(), + enum_label(¬e.visibility)?.into(), + note.acl_policy_id.clone().into(), + note.note_id.to_string().into(), + note.namespace.clone().into(), + note.path.clone().into(), + ], + )) + .await + .context("update Memory note projection")?; + if result.rows_affected() != 1 { + anyhow::bail!("Memory note projection disappeared during writer transaction"); + } + Ok(()) +} + +async fn insert_revision( + txn: &DatabaseTransaction, + note: &MemoryNoteV1, + revision_oid: ObjectHash, +) -> Result<()> { + execute( + txn, + "INSERT INTO memory_revision_index ( + revision_oid, note_id, scope_key, namespace, origin, producer, + rules_version, prompt_version, model_id, policy_version, + input_fingerprints_json, created_at + ) VALUES (?, ?, 'repo', ?, ?, ?, ?, ?, ?, ?, ?, ?)", + vec![ + revision_oid.to_string().into(), + note.note_id.to_string().into(), + note.namespace.clone().into(), + enum_label(¬e.compile_record.origin)?.into(), + note.compile_record.producer.clone().into(), + i64::from(note.compile_record.rules_version).into(), + note.compile_record.prompt_version.clone().into(), + note.compile_record.model_id.clone().into(), + note.compile_record.policy_version.clone().into(), + serde_json::to_string(¬e.compile_record.input_hashes)?.into(), + note.created_at.to_rfc3339().into(), + ], + ) + .await +} + +async fn replace_links( + txn: &DatabaseTransaction, + note: &MemoryNoteV1, + revision_oid: ObjectHash, +) -> Result<()> { + for link in ¬e.links { + let target = txn + .query_one_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "SELECT path FROM memory_note_index WHERE note_id = ?", + [link.target_note_id.to_string().into()], + )) + .await? + .ok_or_else(|| anyhow::anyhow!("Memory link target does not exist"))?; + let target_path: String = target.try_get("", "path")?; + execute( + txn, + "INSERT INTO memory_link_index ( + source_scope_key, source_namespace, source_note_id, + source_revision_oid, target_note_id, target_revision_oid, + link_kind, source_path, target_path, evidence_refs_json, + valid_from, valid_until + ) VALUES ('repo', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + vec![ + note.namespace.clone().into(), + note.note_id.to_string().into(), + revision_oid.to_string().into(), + link.target_note_id.to_string().into(), + link.target_revision_oid.clone().into(), + enum_label(&link.kind)?.into(), + note.path.clone().into(), + target_path.into(), + serde_json::to_string(&link.evidence_refs)?.into(), + link.valid_from.map(|value| value.to_rfc3339()).into(), + link.valid_until.map(|value| value.to_rfc3339()).into(), + ], + ) + .await?; + } + Ok(()) +} + +async fn replace_episode_paths( + txn: &DatabaseTransaction, + note: &MemoryNoteV1, + revision_oid: ObjectHash, +) -> Result<()> { + if let Some(episode) = ¬e.episode { + for path in &episode.code.paths { + execute( + txn, + "INSERT INTO memory_episode_path(note_id, revision_oid, code_path) + VALUES (?, ?, ?)", + vec![ + note.note_id.to_string().into(), + revision_oid.to_string().into(), + path.clone().into(), + ], + ) + .await?; + } + } + Ok(()) +} + +async fn upsert_head( + txn: &DatabaseTransaction, + note: &MemoryNoteV1, + event: &MemoryEventV1, + revision_oid: ObjectHash, +) -> Result<()> { + execute( + txn, + "INSERT INTO memory_head ( + scope_key, namespace, path, note_id, latest_revision_oid, + live_revision_oid, latest_action, latest_review_state, kind, + lifecycle, confidence, trust, sensitivity, visibility, acl_policy_id, + valid_from, valid_until, effective_from_commit, effective_until_commit, + expires_at, rank_hint, last_event_seq, updated_at + ) VALUES ('repo', ?, ?, ?, ?, ?, ?, 'confirmed', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?) + ON CONFLICT(scope_key, namespace, path, note_id) DO UPDATE SET + latest_revision_oid = excluded.latest_revision_oid, + live_revision_oid = excluded.live_revision_oid, + latest_action = excluded.latest_action, + latest_review_state = excluded.latest_review_state, + confidence = excluded.confidence, + trust = excluded.trust, + sensitivity = excluded.sensitivity, + visibility = excluded.visibility, + acl_policy_id = excluded.acl_policy_id, + effective_from_commit = excluded.effective_from_commit, + last_event_seq = excluded.last_event_seq, + updated_at = excluded.updated_at", + vec![ + note.namespace.clone().into(), + note.path.clone().into(), + note.note_id.to_string().into(), + revision_oid.to_string().into(), + revision_oid.to_string().into(), + enum_label(&event.action)?.into(), + enum_label(¬e.kind)?.into(), + enum_label(¬e.lifecycle)?.into(), + enum_label(¬e.confidence)?.into(), + enum_label(¬e.trust)?.into(), + enum_label(¬e.sensitivity)?.into(), + enum_label(¬e.visibility)?.into(), + note.acl_policy_id.clone().into(), + note.valid_from.map(|value| value.to_rfc3339()).into(), + note.valid_until.map(|value| value.to_rfc3339()).into(), + note.effective_from_commit.clone().into(), + note.effective_until_commit.clone().into(), + note.expires_at.map(|value| value.to_rfc3339()).into(), + i64::try_from(event.event_seq)?.into(), + event.at.to_rfc3339().into(), + ], + ) + .await +} + +async fn upsert_projection_state( + txn: &DatabaseTransaction, + note: &MemoryNoteV1, + event: &MemoryEventV1, + commit_oid: ObjectHash, + rebuilt_at_ms: i64, +) -> Result<()> { + execute( + txn, + "INSERT INTO memory_projection_state ( + scope_key, projected_ref_oid, last_event_seq, schema_version, + policy_version, rebuilt_at + ) VALUES ('repo', ?, ?, 1, ?, ?) + ON CONFLICT(scope_key) DO UPDATE SET + projected_ref_oid = excluded.projected_ref_oid, + last_event_seq = excluded.last_event_seq, + schema_version = excluded.schema_version, + policy_version = excluded.policy_version, + rebuilt_at = excluded.rebuilt_at", + vec![ + commit_oid.to_string().into(), + i64::try_from(event.event_seq)?.into(), + note.compile_record.policy_version.clone().into(), + rebuilt_at_ms.into(), + ], + ) + .await +} + +async fn execute(txn: &DatabaseTransaction, sql: &str, values: Vec) -> Result<()> { + txn.execute_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + sql, + values, + )) + .await?; + Ok(()) +} + +fn enum_label(value: &T) -> Result { + let value = serde_json::to_value(value).context("serialize Memory enum")?; + value + .as_str() + .map(str::to_owned) + .ok_or_else(|| anyhow::anyhow!("Memory enum did not serialize as a string")) +} + +fn projection_error(action: &'static str, error: impl std::fmt::Display) -> MemoryWriterError { + MemoryWriterError::new( + MemoryWriterErrorKind::StorageFailure, + format!("{action} failed: {error}"), + ) +} + +fn corrupt_projection(summary: &'static str) -> MemoryWriterError { + MemoryWriterError::new(MemoryWriterErrorKind::CorruptProjection, summary) +} diff --git a/src/internal/ai/memory/tree.rs b/src/internal/ai/memory/tree.rs new file mode 100644 index 000000000..b4448cf41 --- /dev/null +++ b/src/internal/ai/memory/tree.rs @@ -0,0 +1,722 @@ +use std::{ + collections::{BTreeMap, HashSet}, + path::Path, + str::FromStr, +}; + +use git_internal::{ + hash::ObjectHash, + internal::object::{ + ObjectTrait, + commit::Commit, + signature::{Signature, SignatureType}, + tree::{Tree, TreeItem, TreeItemMode}, + }, +}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::{ + error::{MemoryWriterError, MemoryWriterErrorKind}, + policy::policy_snapshot_digest, + validation::{parse_memory_event_v1, parse_memory_note_v1}, +}; +use crate::utils::{ + object::{read_git_object_bounded_validated, write_git_object}, + tree::sort_tree_items_for_git, +}; + +const MAX_COMMIT_BYTES: u64 = 1024 * 1024; +const MAX_TREE_BYTES: u64 = 16 * 1024 * 1024; +const MAX_MANIFEST_BYTES: u64 = 32 * 1024; +const MAX_MEMORY_TREE_ENTRIES: usize = 65_536; +const MEMORY_MANIFEST_SCHEMA_V1: u32 = 1; + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(super) struct MemoryManifestV1 { + pub(super) schema_version: u32, + pub(super) scope_key: String, + pub(super) last_event_seq: u64, + pub(super) commit_count: u64, + pub(super) policy_version: String, + pub(super) policy_snapshot_digest: String, + pub(super) writer_version: u32, + pub(super) index_version: u32, +} + +impl MemoryManifestV1 { + pub(super) fn initial(policy_version: String) -> Result { + let policy_snapshot_digest = policy_snapshot_digest(&policy_version)?; + Ok(Self { + schema_version: MEMORY_MANIFEST_SCHEMA_V1, + scope_key: "repo".to_string(), + last_event_seq: 0, + commit_count: 0, + policy_snapshot_digest, + policy_version, + writer_version: 1, + index_version: 1, + }) + } + + pub(super) fn validate(&self) -> Result<(), MemoryWriterError> { + let expected_policy_digest = + policy_snapshot_digest(&self.policy_version).map_err(|_| { + MemoryWriterError::new( + MemoryWriterErrorKind::CorruptHistory, + "Memory manifest names an unsupported policy snapshot", + ) + })?; + if self.schema_version != MEMORY_MANIFEST_SCHEMA_V1 + || self.scope_key != "repo" + || self.policy_version.is_empty() + || self.policy_snapshot_digest != expected_policy_digest + || self.writer_version != 1 + || self.index_version == 0 + { + return Err(corrupt( + "Memory manifest has an unsupported or invalid shape", + )); + } + Ok(()) + } +} + +pub(super) struct MemoryTreeSnapshot { + pub(super) root_items: Vec, + pub(super) manifest: MemoryManifestV1, +} + +pub(super) struct MemoryCommitObjects { + pub(super) revision_oid: ObjectHash, + pub(super) commit_oid: ObjectHash, +} + +pub(super) struct MemoryCommitInput<'a> { + pub(super) note_id: &'a str, + pub(super) namespace: &'a str, + pub(super) note_bytes: &'a [u8], + pub(super) events: &'a [MemoryEventInput<'a>], +} + +pub(super) struct MemoryEventInput<'a> { + pub(super) event_seq: u64, + pub(super) event_id: &'a str, + pub(super) event_bytes: &'a [u8], +} + +pub(super) fn load_snapshot( + storage_path: &Path, + head: Option, + policy_version: &str, +) -> Result { + let Some(head) = head else { + return Ok(MemoryTreeSnapshot { + root_items: Vec::new(), + manifest: MemoryManifestV1::initial(policy_version.to_string())?, + }); + }; + + let commit = load_commit(storage_path, head)?; + if commit.parent_commit_ids.len() > 1 { + return Err(corrupt("Memory history contains a merge commit")); + } + let root_items = load_tree(storage_path, commit.tree_id)?; + let manifest_entry = root_items + .iter() + .find(|item| item.name == "manifest.json" && item.mode == TreeItemMode::Blob) + .ok_or_else(|| corrupt("Memory commit is missing manifest.json"))?; + let (object_type, manifest_bytes) = + read_git_object_bounded_validated(storage_path, &manifest_entry.id, MAX_MANIFEST_BYTES) + .map_err(|error| history_error("read Memory manifest", error))?; + if object_type != "blob" { + return Err(corrupt("Memory manifest entry is not a blob")); + } + let manifest: MemoryManifestV1 = serde_json::from_slice(&manifest_bytes) + .map_err(|_| corrupt("Memory manifest is not valid JSON"))?; + manifest.validate()?; + if manifest.commit_count == 0 { + return Err(corrupt("persisted Memory manifest has no commits")); + } + let expected_parent_count = usize::from(manifest.commit_count > 1); + if commit.parent_commit_ids.len() != expected_parent_count { + return Err(corrupt( + "Memory commit parent count does not match its event sequence", + )); + } + validate_append_edge(storage_path, &commit, &root_items, &manifest)?; + if manifest.policy_version != policy_version { + return Err(MemoryWriterError::new( + MemoryWriterErrorKind::PolicyRejected, + "proposal policy version does not match the Memory history policy", + )); + } + Ok(MemoryTreeSnapshot { + root_items, + manifest, + }) +} + +fn validate_append_edge( + storage_path: &Path, + commit: &Commit, + root_items: &[TreeItem], + manifest: &MemoryManifestV1, +) -> Result<(), MemoryWriterError> { + validate_root_shape(root_items)?; + let (parent_items, parent_manifest) = match commit.parent_commit_ids.as_slice() { + [] => { + if manifest.commit_count != 1 || manifest.last_event_seq != 2 { + return Err(corrupt( + "Memory root commit has an invalid manifest sequence", + )); + } + (None, None) + } + [parent_oid] => { + let parent = load_commit(storage_path, *parent_oid)?; + let parent_items = load_tree(storage_path, parent.tree_id)?; + validate_root_shape(&parent_items)?; + let parent_manifest = load_manifest(storage_path, &parent_items, "parent")?; + let expected_parent_count = usize::from(parent_manifest.commit_count > 1); + if parent.parent_commit_ids.len() != expected_parent_count + || parent_manifest.commit_count.checked_add(1) != Some(manifest.commit_count) + || parent_manifest.last_event_seq.checked_add(2) != Some(manifest.last_event_seq) + || parent_manifest.scope_key != manifest.scope_key + || parent_manifest.policy_version != manifest.policy_version + || parent_manifest.policy_snapshot_digest != manifest.policy_snapshot_digest + || parent_manifest.writer_version != manifest.writer_version + || parent_manifest.index_version != manifest.index_version + { + return Err(corrupt("Memory parent manifest edge is discontinuous")); + } + (Some(parent_items), Some(parent_manifest)) + } + _ => return Err(corrupt("Memory history contains a merge commit")), + }; + + let previous_seq = parent_manifest + .as_ref() + .map_or(0, |parent| parent.last_event_seq); + validate_event_edge( + storage_path, + parent_items.as_deref(), + root_items, + previous_seq, + manifest.last_event_seq, + ) +} + +fn validate_root_shape(items: &[TreeItem]) -> Result<(), MemoryWriterError> { + if items.len() != 3 + || items.iter().any(|item| { + !matches!(item.name.as_str(), "manifest.json" | "events" | "notes") + || (item.name == "manifest.json" && item.mode != TreeItemMode::Blob) + || (item.name != "manifest.json" && item.mode != TreeItemMode::Tree) + }) + { + return Err(corrupt("Memory root tree contains an unsupported entry")); + } + Ok(()) +} + +fn load_manifest( + storage_path: &Path, + items: &[TreeItem], + label: &'static str, +) -> Result { + let entry = items + .iter() + .find(|item| item.name == "manifest.json" && item.mode == TreeItemMode::Blob) + .ok_or_else(|| corrupt("Memory commit is missing manifest.json"))?; + let (object_type, bytes) = + read_git_object_bounded_validated(storage_path, &entry.id, MAX_MANIFEST_BYTES) + .map_err(|error| history_error("read Memory manifest", error))?; + if object_type != "blob" { + return Err(corrupt("Memory manifest entry is not a blob")); + } + let manifest: MemoryManifestV1 = serde_json::from_slice(&bytes).map_err(|_| { + MemoryWriterError::new( + MemoryWriterErrorKind::CorruptHistory, + format!("{label} Memory manifest is not valid JSON"), + ) + })?; + manifest.validate()?; + Ok(manifest) +} + +pub(super) fn write_revision_commit( + storage_path: &Path, + parent: Option, + mut snapshot: MemoryTreeSnapshot, + input: MemoryCommitInput<'_>, +) -> Result { + let note_entry_count = if snapshot.root_items.is_empty() { + 0 + } else { + note_blob_map(storage_path, &snapshot.root_items)?.entry_count + }; + ensure_append_capacity(snapshot.manifest.last_event_seq, note_entry_count)?; + let revision_oid = write_git_object(storage_path, "blob", input.note_bytes) + .map_err(|error| storage_error("write MemoryNote blob", error))?; + let revision_file = format!("{revision_oid}.json"); + let note_path = [ + "notes".to_string(), + encode_segment(input.namespace), + input.note_id.to_string(), + revision_file, + ]; + upsert_blob( + storage_path, + &mut snapshot.root_items, + ¬e_path, + revision_oid, + )?; + + for event in input.events { + let expected_seq = snapshot + .manifest + .last_event_seq + .checked_add(1) + .ok_or_else(|| corrupt("Memory event sequence overflowed"))?; + if event.event_seq != expected_seq { + return Err(corrupt( + "Memory commit input contains a non-contiguous event", + )); + } + let event_oid = write_git_object(storage_path, "blob", event.event_bytes) + .map_err(|error| storage_error("write MemoryEvent blob", error))?; + snapshot.manifest.last_event_seq = expected_seq; + let event_file = format!("{expected_seq:020}-{}.json", event.event_id); + let event_path = ["events".to_string(), event_file]; + upsert_blob( + storage_path, + &mut snapshot.root_items, + &event_path, + event_oid, + )?; + } + snapshot.manifest.commit_count = snapshot + .manifest + .commit_count + .checked_add(1) + .ok_or_else(|| corrupt("Memory commit count overflowed"))?; + + let manifest_bytes = serde_json::to_vec(&snapshot.manifest) + .map_err(|_| corrupt("Memory manifest could not be serialized"))?; + let manifest_oid = write_git_object(storage_path, "blob", &manifest_bytes) + .map_err(|error| storage_error("write Memory manifest", error))?; + upsert_blob( + storage_path, + &mut snapshot.root_items, + &["manifest.json".to_string()], + manifest_oid, + )?; + + let root_oid = write_tree(storage_path, snapshot.root_items)?; + let author = Signature::new( + SignatureType::Author, + "Libra Memory".to_string(), + "memory@libra".to_string(), + ); + let committer = Signature::new( + SignatureType::Committer, + "Libra Memory".to_string(), + "memory@libra".to_string(), + ); + let commit = Commit::new( + author, + committer, + root_oid, + parent.into_iter().collect(), + &format!("Record Memory event {}", snapshot.manifest.last_event_seq), + ); + let commit_bytes = commit + .to_data() + .map_err(|error| storage_error("serialize Memory commit", error))?; + let commit_oid = write_git_object(storage_path, "commit", &commit_bytes) + .map_err(|error| storage_error("write Memory commit", error))?; + Ok(MemoryCommitObjects { + revision_oid, + commit_oid, + }) +} + +fn ensure_append_capacity( + current_event_count: u64, + current_note_entry_count: usize, +) -> Result<(), MemoryWriterError> { + let max_events_before_append = u64::try_from(MAX_MEMORY_TREE_ENTRIES - 2) + .map_err(|_| corrupt("Memory validation budget is invalid"))?; + if current_event_count > max_events_before_append + || current_note_entry_count > MAX_MEMORY_TREE_ENTRIES - 3 + { + return Err(MemoryWriterError::new( + MemoryWriterErrorKind::StorageFailure, + "Memory history reached the writer validation capacity", + )); + } + Ok(()) +} + +pub(super) fn load_note_bytes( + storage_path: &Path, + revision_oid: ObjectHash, +) -> Result, MemoryWriterError> { + let (object_type, bytes) = + read_git_object_bounded_validated(storage_path, &revision_oid, 256 * 1024) + .map_err(|error| history_error("read MemoryNote revision", error))?; + if object_type != "blob" { + return Err(corrupt("Memory revision OID does not name a blob")); + } + Ok(bytes) +} + +fn upsert_blob( + storage_path: &Path, + tree_items: &mut Vec, + path: &[String], + blob_oid: ObjectHash, +) -> Result<(), MemoryWriterError> { + let (name, rest) = path + .split_first() + .ok_or_else(|| corrupt("empty Memory tree path"))?; + if rest.is_empty() { + tree_items.retain(|item| item.name != *name); + tree_items.push(TreeItem::new(TreeItemMode::Blob, blob_oid, name.clone())); + sort_tree_items_for_git(tree_items); + return Ok(()); + } + + let mut child_items = match tree_items.iter().find(|item| item.name == *name) { + Some(item) if item.mode == TreeItemMode::Tree => load_tree(storage_path, item.id)?, + Some(_) => return Err(corrupt("Memory tree path collides with a non-tree entry")), + None => Vec::new(), + }; + upsert_blob(storage_path, &mut child_items, rest, blob_oid)?; + let child_oid = write_tree(storage_path, child_items)?; + tree_items.retain(|item| item.name != *name); + tree_items.push(TreeItem::new(TreeItemMode::Tree, child_oid, name.clone())); + sort_tree_items_for_git(tree_items); + Ok(()) +} + +fn write_tree( + storage_path: &Path, + mut items: Vec, +) -> Result { + sort_tree_items_for_git(&mut items); + let tree = Tree::from_tree_items(items) + .map_err(|error| storage_error("construct Memory tree", error))?; + let bytes = tree + .to_data() + .map_err(|error| storage_error("serialize Memory tree", error))?; + write_git_object(storage_path, "tree", &bytes) + .map_err(|error| storage_error("write Memory tree", error)) +} + +fn load_commit(storage_path: &Path, oid: ObjectHash) -> Result { + let (object_type, bytes) = + read_git_object_bounded_validated(storage_path, &oid, MAX_COMMIT_BYTES) + .map_err(|error| history_error("read Memory commit", error))?; + if object_type != "commit" { + return Err(corrupt("Memory ref does not name a commit")); + } + Commit::from_bytes(&bytes, oid).map_err(|error| history_error("parse Memory commit", error)) +} + +fn load_tree(storage_path: &Path, oid: ObjectHash) -> Result, MemoryWriterError> { + let (object_type, bytes) = + read_git_object_bounded_validated(storage_path, &oid, MAX_TREE_BYTES) + .map_err(|error| history_error("read Memory tree", error))?; + if object_type != "tree" { + return Err(corrupt("Memory tree OID does not name a tree")); + } + Tree::from_bytes(&bytes, oid) + .map(|tree| tree.tree_items) + .map_err(|error| history_error("parse Memory tree", error)) +} + +fn validate_event_edge( + storage_path: &Path, + parent_root: Option<&[TreeItem]>, + root_items: &[TreeItem], + previous_seq: u64, + last_event_seq: u64, +) -> Result<(), MemoryWriterError> { + let events = tree_entries(storage_path, root_items, "events")?; + if events.len() > MAX_MEMORY_TREE_ENTRIES { + return Err(corrupt( + "Memory event tree exceeds the writer validation budget", + )); + } + if u64::try_from(events.len()).ok() != Some(last_event_seq) { + return Err(corrupt( + "Memory manifest sequence does not match the event count", + )); + } + let parent_events = match parent_root { + Some(parent) => tree_entries(storage_path, parent, "events")?, + None => Vec::new(), + }; + if u64::try_from(parent_events.len()).ok() != Some(previous_seq) + || events.len().checked_sub(parent_events.len()) != Some(2) + || !events.starts_with(&parent_events) + { + return Err(corrupt("Memory event tree did not append to its parent")); + } + + let mut event_ids = HashSet::with_capacity(events.len()); + for (index, item) in events.iter().enumerate() { + if item.mode != TreeItemMode::Blob { + return Err(corrupt("Memory event entry is not a blob")); + } + let sequence = u64::try_from(index) + .ok() + .and_then(|index| index.checked_add(1)) + .ok_or_else(|| corrupt("Memory event index overflowed"))?; + let event_id = event_id_from_filename(&item.name, sequence)?; + if !event_ids.insert(event_id) { + return Err(corrupt("Memory event ID is duplicated")); + } + } + + let parent_notes = match parent_root { + Some(parent) => note_blob_map(storage_path, parent)?, + None => NoteTreeIndex::default(), + }; + let notes = note_blob_map(storage_path, root_items)?; + if notes.blobs.len().checked_sub(parent_notes.blobs.len()) != Some(1) + || parent_notes + .blobs + .iter() + .any(|(path, oid)| notes.blobs.get(path) != Some(oid)) + { + return Err(corrupt("Memory note tree did not append to its parent")); + } + + let tail = &events[parent_events.len()..]; + let mut parsed = Vec::with_capacity(tail.len()); + for (offset, item) in tail.iter().enumerate() { + let sequence = previous_seq + .checked_add(u64::try_from(offset).map_err(|_| corrupt("Memory event overflowed"))?) + .and_then(|value| value.checked_add(1)) + .ok_or_else(|| corrupt("Memory event sequence overflowed"))?; + let event_id = event_id_from_filename(&item.name, sequence)?; + let (object_type, bytes) = + read_git_object_bounded_validated(storage_path, &item.id, 128 * 1024) + .map_err(|error| history_error("read MemoryEvent blob", error))?; + if object_type != "blob" { + return Err(corrupt("Memory event OID does not name a blob")); + } + let event = parse_memory_event_v1(&bytes).map_err(|error| { + MemoryWriterError::new( + MemoryWriterErrorKind::CorruptHistory, + format!("persisted MemoryEvent is invalid: {error}"), + ) + })?; + if event.event_seq != sequence || event.event_id != event_id { + return Err(corrupt( + "Memory event filename and payload identity disagree", + )); + } + let note_id = event + .note_id + .ok_or_else(|| corrupt("Memory revision event has no note ID"))?; + let revision_oid = event + .revision_oid + .as_deref() + .ok_or_else(|| corrupt("Memory revision event has no revision OID")) + .and_then(parse_oid)?; + let note_bytes = load_note_bytes(storage_path, revision_oid)?; + let note = parse_memory_note_v1(¬e_bytes).map_err(|error| { + MemoryWriterError::new( + MemoryWriterErrorKind::CorruptHistory, + format!("event references an invalid MemoryNote revision: {error}"), + ) + })?; + let expected_path = format!( + "{}/{}/{}.json", + encode_segment(¬e.namespace), + note.note_id, + revision_oid + ); + if note.note_id != note_id || notes.blobs.get(&expected_path) != Some(&revision_oid) { + return Err(corrupt( + "MemoryEvent revision is not reachable from its canonical note path", + )); + } + parsed.push(event); + } + let tail_note = parse_memory_note_v1(&load_note_bytes( + storage_path, + parse_oid( + parsed[0] + .revision_oid + .as_deref() + .ok_or_else(|| corrupt("Memory revision event has no revision OID"))?, + )?, + )?) + .map_err(|error| { + MemoryWriterError::new( + MemoryWriterErrorKind::CorruptHistory, + format!("event references an invalid MemoryNote revision: {error}"), + ) + })?; + let parent_note_prefix = format!( + "{}/{}/", + encode_segment(&tail_note.namespace), + tail_note.note_id + ); + let expected_first = if parent_notes + .blobs + .keys() + .any(|path| path.starts_with(&parent_note_prefix)) + { + super::domain::MemoryEventAction::Revised + } else { + super::domain::MemoryEventAction::Created + }; + if parsed[0].action != expected_first + || parsed[1].action != super::domain::MemoryEventAction::Confirmed + || parsed[0].note_id != parsed[1].note_id + || parsed[0].revision_oid != parsed[1].revision_oid + { + return Err(corrupt("Memory commit event pair is invalid")); + } + Ok(()) +} + +fn tree_entries( + storage_path: &Path, + root_items: &[TreeItem], + name: &'static str, +) -> Result, MemoryWriterError> { + let item = root_items + .iter() + .find(|item| item.name == name && item.mode == TreeItemMode::Tree) + .ok_or_else(|| corrupt("Memory commit is missing a required tree"))?; + load_tree(storage_path, item.id) +} + +fn event_id_from_filename(name: &str, sequence: u64) -> Result { + let prefix = format!("{sequence:020}-"); + name.strip_prefix(&prefix) + .and_then(|name| name.strip_suffix(".json")) + .and_then(|value| Uuid::parse_str(value).ok()) + .ok_or_else(|| corrupt("Memory event filename is not canonical")) +} + +#[derive(Default)] +struct NoteTreeIndex { + blobs: BTreeMap, + entry_count: usize, +} + +fn note_blob_map( + storage_path: &Path, + root_items: &[TreeItem], +) -> Result { + let notes = root_items + .iter() + .find(|item| item.name == "notes" && item.mode == TreeItemMode::Tree) + .ok_or_else(|| corrupt("Memory commit is missing the notes tree"))?; + let mut output = NoteTreeIndex::default(); + let mut remaining = MAX_MEMORY_TREE_ENTRIES; + collect_note_blobs( + storage_path, + notes.id, + 0, + String::new(), + &mut remaining, + &mut output.blobs, + )?; + output.entry_count = MAX_MEMORY_TREE_ENTRIES - remaining; + Ok(output) +} + +fn collect_note_blobs( + storage_path: &Path, + tree_oid: ObjectHash, + depth: usize, + prefix: String, + remaining: &mut usize, + output: &mut BTreeMap, +) -> Result<(), MemoryWriterError> { + if depth >= 3 { + return Err(corrupt("Memory note tree exceeds its canonical depth")); + } + for item in load_tree(storage_path, tree_oid)? { + if *remaining == 0 { + return Err(corrupt( + "Memory note tree exceeds the writer validation budget", + )); + } + *remaining -= 1; + let path = if prefix.is_empty() { + item.name.clone() + } else { + format!("{prefix}/{}", item.name) + }; + if depth == 2 { + if item.mode != TreeItemMode::Blob || output.insert(path, item.id).is_some() { + return Err(corrupt("Memory note revision entry is invalid")); + } + } else if item.mode == TreeItemMode::Tree { + collect_note_blobs(storage_path, item.id, depth + 1, path, remaining, output)?; + } else { + return Err(corrupt("Memory note tree has a non-canonical shape")); + } + } + Ok(()) +} + +fn encode_segment(value: &str) -> String { + format!("x{}", hex::encode(value.as_bytes())) +} + +fn corrupt(summary: &'static str) -> MemoryWriterError { + MemoryWriterError::new(MemoryWriterErrorKind::CorruptHistory, summary) +} + +fn storage_error(action: &'static str, error: impl std::fmt::Display) -> MemoryWriterError { + MemoryWriterError::new( + MemoryWriterErrorKind::StorageFailure, + format!("{action} failed: {error}"), + ) +} + +fn history_error(action: &'static str, error: impl std::fmt::Display) -> MemoryWriterError { + MemoryWriterError::new( + MemoryWriterErrorKind::CorruptHistory, + format!("{action} failed for authoritative history: {error}"), + ) +} + +pub(super) fn parse_oid(value: &str) -> Result { + ObjectHash::from_str(value).map_err(|_| corrupt("projection contains an invalid object ID")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn append_capacity_reserves_the_complete_writer_delta() { + let max = MAX_MEMORY_TREE_ENTRIES; + let max_events = u64::try_from(max).expect("test budget fits u64"); + assert!(ensure_append_capacity(max_events - 2, max - 3).is_ok()); + assert_eq!( + ensure_append_capacity(max_events - 1, 0) + .expect_err("two event slots must remain") + .kind(), + MemoryWriterErrorKind::StorageFailure + ); + assert_eq!( + ensure_append_capacity(0, max - 2) + .expect_err("three note-tree entry slots must remain") + .kind(), + MemoryWriterErrorKind::StorageFailure + ); + } +} diff --git a/src/internal/ai/memory/writer.rs b/src/internal/ai/memory/writer.rs new file mode 100644 index 000000000..1ab667c06 --- /dev/null +++ b/src/internal/ai/memory/writer.rs @@ -0,0 +1,1215 @@ +use std::{collections::HashSet, path::PathBuf, sync::Arc}; + +use chrono::Utc; +use git_internal::hash::ObjectHash; +use uuid::Uuid; + +use super::{ + canonical::memory_note_content_digest_v1, + domain::{MemoryEventAction, MemoryEventV1, MemoryNoteV1}, + error::{MemoryWriterError, MemoryWriterErrorKind}, + policy::{ + AuthenticatedMemoryContext, DeterministicMemoryProposal, TrustedMemoryTarget, + validate_writer_policy, + }, + store::{ + ProjectedCell, ProjectionMutation, find_cell, read_memory_ref_head, + validate_projection_watermark, + }, + tree::{ + MemoryCommitInput, MemoryEventInput, load_note_bytes, load_snapshot, write_revision_commit, + }, + validation::{parse_memory_event_v1, parse_memory_note_v1}, +}; +use crate::{ + internal::{ + ai::{ + keyed_digest::RepositoryKeyedDigest, + linear_ref::{ + LinearRefTransactionOutcome, OwnedRefSpec, OwnedRefTransportPolicy, + linear_ref_transaction, + }, + }, + db, + workspace::RepoIdentity, + }, + utils::{object::git_object_hash, util::DATABASE}, +}; + +const WRITER_HEAD_CONFLICT_MAX_RETRIES: usize = 3; +const MAX_REVISION_WALK: usize = 4096; +const MEMORY_EVENT_NAMESPACE_V1: Uuid = Uuid::from_bytes([ + 0x80, 0x3a, 0x58, 0x77, 0x40, 0x35, 0x4a, 0xf0, 0xa1, 0xd8, 0xf0, 0x61, 0x52, 0x4d, 0x0b, 0x52, +]); + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CommittedMemoryEnvelope { + note_id: Uuid, + revision_oid: ObjectHash, + commit_oid: ObjectHash, + event_seq: u64, + appended: bool, +} + +impl CommittedMemoryEnvelope { + pub(crate) const fn note_id(&self) -> Uuid { + self.note_id + } + + pub(crate) const fn revision_oid(&self) -> ObjectHash { + self.revision_oid + } + + pub(crate) const fn commit_oid(&self) -> ObjectHash { + self.commit_oid + } + + pub(crate) const fn event_seq(&self) -> u64 { + self.event_seq + } + + pub(crate) const fn appended(&self) -> bool { + self.appended + } +} + +pub(crate) struct MemoryWriter { + storage_path: PathBuf, + database: Arc, + digest_provider: Arc, + #[cfg(test)] + test_before_first_cas: std::sync::Mutex>>, +} + +impl MemoryWriter { + pub(crate) async fn open(storage_path: PathBuf) -> Result { + if OwnedRefSpec::MemoryRepo.transport_policy() != OwnedRefTransportPolicy::LocalOnly { + return Err(MemoryWriterError::new( + MemoryWriterErrorKind::PolicyRejected, + "Memory ref must use local-only object persistence", + )); + } + let storage_path = tokio::fs::canonicalize(storage_path) + .await + .map_err(|error| { + MemoryWriterError::new( + MemoryWriterErrorKind::StorageFailure, + format!("canonicalize repository storage for Memory writer: {error}"), + ) + })?; + let database_path = storage_path.join(DATABASE); + let database = Arc::new( + db::get_db_conn_instance_for_path(&database_path) + .await + .map_err(|error| { + MemoryWriterError::new( + MemoryWriterErrorKind::StorageFailure, + format!("open repository database for Memory writer: {error}"), + ) + })?, + ); + let digest_provider = RepositoryKeyedDigest::load_or_initialize(&database_path) + .await + .map_err(|error| { + MemoryWriterError::new( + MemoryWriterErrorKind::DigestKeyUnavailable, + error.to_string(), + ) + })?; + validate_repository_binding(database.as_ref(), &digest_provider).await?; + Ok(Self { + storage_path, + database, + digest_provider, + #[cfg(test)] + test_before_first_cas: std::sync::Mutex::new(None), + }) + } + + #[cfg(test)] + async fn for_tests( + storage_path: PathBuf, + database: Arc, + digest_provider: Arc, + ) -> Result { + validate_repository_binding(database.as_ref(), &digest_provider).await?; + Ok(Self { + storage_path, + database, + digest_provider, + test_before_first_cas: std::sync::Mutex::new(None), + }) + } + + #[cfg(test)] + fn set_test_before_first_cas(&self, barrier: Arc) { + let mut slot = self + .test_before_first_cas + .lock() + .unwrap_or_else(|error| error.into_inner()); + *slot = Some(barrier); + } + + pub(crate) async fn commit( + &self, + context: &AuthenticatedMemoryContext, + target: &TrustedMemoryTarget, + proposal: &DeterministicMemoryProposal, + expected_head: Option, + ) -> Result { + self.digest_provider + .validate_for_connection(self.database.as_ref()) + .await + .map_err(|error| { + MemoryWriterError::new( + MemoryWriterErrorKind::DigestKeyUnavailable, + error.to_string(), + ) + })?; + validate_repository_binding(self.database.as_ref(), &self.digest_provider).await?; + validate_writer_policy( + context, + target, + proposal, + self.digest_provider.repository_id(), + self.digest_provider.key_id(), + )?; + + let mut last_observed_head = expected_head; + for attempt in 0..=WRITER_HEAD_CONFLICT_MAX_RETRIES { + let current_head = read_memory_ref_head(self.database.as_ref()).await?; + // The caller's expected head is a snapshot hint. A mismatch means + // the proposal must be rebuilt on current authoritative state. + let _head_changed_since_request = last_observed_head != current_head; + + let policy_version = &proposal.note().compile_record.policy_version; + let snapshot = load_snapshot(&self.storage_path, current_head, policy_version)?; + validate_projection_watermark( + self.database.as_ref(), + current_head, + snapshot.manifest.last_event_seq, + ) + .await?; + + let cell = find_cell( + self.database.as_ref(), + target.root().namespace(), + target.root().path(), + ) + .await?; + if cell + .as_ref() + .is_some_and(|cell| cell.note_id != target.root().note_id().to_string()) + { + return Err(MemoryWriterError::new( + MemoryWriterErrorKind::CorruptProjection, + "Memory Cell points at a non-deterministic note ID", + )); + } + + if let Some(cell) = &cell + && let Some(revision_oid) = self.find_idempotent_revision( + cell, + &proposal.note().compile_record.idempotency_key, + )? + { + let commit_oid = current_head.ok_or_else(|| { + MemoryWriterError::new( + MemoryWriterErrorKind::CorruptProjection, + "Memory projection exists without an authoritative ref", + ) + })?; + return Ok(CommittedMemoryEnvelope { + note_id: target.root().note_id(), + revision_oid, + commit_oid, + event_seq: snapshot.manifest.last_event_seq, + appended: false, + }); + } + + let mut note = proposal.note().clone(); + note.parents = cell + .as_ref() + .map(|cell| vec![cell.latest_revision_oid.to_string()]) + .unwrap_or_default(); + note.content_digest = memory_note_content_digest_v1(¬e)?; + let note_bytes = serde_json::to_vec(¬e).map_err(|_| { + MemoryWriterError::new( + MemoryWriterErrorKind::InvalidProposal, + "MemoryNote could not be serialized", + ) + })?; + let note = parse_memory_note_v1(¬e_bytes)?; + let revision_oid = git_object_hash("blob", ¬e_bytes); + let base_event_seq = snapshot.manifest.last_event_seq; + let transition_seq = base_event_seq.checked_add(1).ok_or_else(|| { + MemoryWriterError::new( + MemoryWriterErrorKind::CorruptHistory, + "Memory event sequence overflowed", + ) + })?; + let action = if cell.is_some() { + MemoryEventAction::Revised + } else { + MemoryEventAction::Created + }; + let transition_id = event_id(¬e, revision_oid, action); + let transition = MemoryEventV1 { + schema_version: 1, + event_id: transition_id, + event_seq: transition_seq, + note_id: Some(note.note_id), + revision_oid: Some(revision_oid.to_string()), + namespace: None, + target_path: None, + action, + reason_code: Some("episode_compiled".to_string()), + actor: context.actor().clone(), + at: note.created_at, + evidence_refs: note.evidence_refs.clone(), + next_note_id: None, + }; + let transition_bytes = serde_json::to_vec(&transition).map_err(|_| { + MemoryWriterError::new( + MemoryWriterErrorKind::InvalidProposal, + "MemoryEvent could not be serialized", + ) + })?; + let transition = parse_memory_event_v1(&transition_bytes)?; + let event_seq = transition_seq.checked_add(1).ok_or_else(|| { + MemoryWriterError::new( + MemoryWriterErrorKind::CorruptHistory, + "Memory event sequence overflowed", + ) + })?; + let confirmed = MemoryEventV1 { + schema_version: 1, + event_id: event_id(¬e, revision_oid, MemoryEventAction::Confirmed), + event_seq, + note_id: Some(note.note_id), + revision_oid: Some(revision_oid.to_string()), + namespace: None, + target_path: None, + action: MemoryEventAction::Confirmed, + reason_code: Some("automatic_episode_policy".to_string()), + actor: context.actor().clone(), + at: note.created_at, + evidence_refs: note.evidence_refs.clone(), + next_note_id: None, + }; + let confirmed_bytes = serde_json::to_vec(&confirmed).map_err(|_| { + MemoryWriterError::new( + MemoryWriterErrorKind::InvalidProposal, + "Memory confirmation event could not be serialized", + ) + })?; + let confirmed = parse_memory_event_v1(&confirmed_bytes)?; + let transition_id = transition.event_id.to_string(); + let confirmed_id = confirmed.event_id.to_string(); + let event_inputs = [ + MemoryEventInput { + event_seq: transition.event_seq, + event_id: &transition_id, + event_bytes: &transition_bytes, + }, + MemoryEventInput { + event_seq: confirmed.event_seq, + event_id: &confirmed_id, + event_bytes: &confirmed_bytes, + }, + ]; + + let objects = write_revision_commit( + &self.storage_path, + current_head, + snapshot, + MemoryCommitInput { + note_id: ¬e.note_id.to_string(), + namespace: ¬e.namespace, + note_bytes: ¬e_bytes, + events: &event_inputs, + }, + )?; + if objects.revision_oid != revision_oid { + return Err(MemoryWriterError::new( + MemoryWriterErrorKind::StorageFailure, + "MemoryNote object identity changed while writing", + )); + } + + let mutation = ProjectionMutation { + note: note.clone(), + event: confirmed.clone(), + revision_oid, + commit_oid: objects.commit_oid, + is_create: cell.is_none(), + rebuilt_at_ms: Utc::now().timestamp_millis(), + expected_head: current_head, + expected_event_seq: base_event_seq, + expected_cell: cell.clone(), + repository_id: self.digest_provider.repository_id().to_string(), + digest_provider: Arc::clone(&self.digest_provider), + }; + #[cfg(test)] + if attempt == 0 { + let barrier = self + .test_before_first_cas + .lock() + .unwrap_or_else(|error| error.into_inner()) + .clone(); + if let Some(barrier) = barrier { + barrier.wait().await; + } + } + match linear_ref_transaction( + self.database.as_ref(), + OwnedRefSpec::MemoryRepo, + current_head, + objects.commit_oid, + None, + Some(&mutation), + ) + .await + .map_err(|error| { + error + .downcast_ref::() + .cloned() + .unwrap_or_else(|| { + MemoryWriterError::new( + MemoryWriterErrorKind::StorageFailure, + format!("commit Memory ref and projection transaction failed: {error}"), + ) + }) + })? { + LinearRefTransactionOutcome::Updated => { + return Ok(CommittedMemoryEnvelope { + note_id: note.note_id, + revision_oid, + commit_oid: objects.commit_oid, + event_seq, + appended: true, + }); + } + LinearRefTransactionOutcome::HeadChanged + if attempt < WRITER_HEAD_CONFLICT_MAX_RETRIES => + { + last_observed_head = current_head; + } + LinearRefTransactionOutcome::HeadChanged => { + return Err(MemoryWriterError::new( + MemoryWriterErrorKind::ConflictExhausted, + "Memory ref changed repeatedly while committing a revision", + )); + } + } + } + + Err(MemoryWriterError::new( + MemoryWriterErrorKind::ConflictExhausted, + "Memory writer exhausted its bounded retry budget", + )) + } + + fn find_idempotent_revision( + &self, + cell: &ProjectedCell, + idempotency_key: &str, + ) -> Result, MemoryWriterError> { + let mut next = Some(cell.latest_revision_oid); + let mut visited = HashSet::new(); + for _ in 0..MAX_REVISION_WALK { + let Some(revision_oid) = next else { + return Ok(None); + }; + if !visited.insert(revision_oid.to_string()) { + return Err(MemoryWriterError::new( + MemoryWriterErrorKind::CorruptHistory, + "Memory revision ancestry contains a cycle", + )); + } + let bytes = load_note_bytes(&self.storage_path, revision_oid)?; + let note = parse_memory_note_v1(&bytes).map_err(|error| { + MemoryWriterError::new( + MemoryWriterErrorKind::CorruptHistory, + format!("persisted Memory revision is invalid: {error}"), + ) + })?; + if note.note_id.to_string() != cell.note_id { + return Err(MemoryWriterError::new( + MemoryWriterErrorKind::CorruptHistory, + "Memory revision ancestry crosses note identities", + )); + } + if note.compile_record.idempotency_key == idempotency_key { + return Ok(Some(revision_oid)); + } + next = note + .parents + .first() + .map(|parent| parent.parse()) + .transpose() + .map_err(|_| { + MemoryWriterError::new( + MemoryWriterErrorKind::CorruptHistory, + "Memory revision contains an invalid parent OID", + ) + })?; + } + Err(MemoryWriterError::new( + MemoryWriterErrorKind::CorruptHistory, + "Memory revision ancestry exceeds the writer traversal bound", + )) + } +} + +async fn validate_repository_binding( + database: &sea_orm::DatabaseConnection, + digest_provider: &RepositoryKeyedDigest, +) -> Result<(), MemoryWriterError> { + let repository = RepoIdentity::resolve(database).await.map_err(|error| { + MemoryWriterError::new( + MemoryWriterErrorKind::CorruptProjection, + format!("repository identity is unavailable to Memory writer: {error}"), + ) + })?; + if repository.as_str() != digest_provider.repository_id() { + return Err(MemoryWriterError::new( + MemoryWriterErrorKind::CorruptProjection, + "repository database and digest provider identities do not match", + )); + } + Ok(()) +} + +fn event_id(note: &MemoryNoteV1, revision_oid: ObjectHash, action: MemoryEventAction) -> Uuid { + let action = match action { + MemoryEventAction::Created => "created", + MemoryEventAction::Revised => "revised", + MemoryEventAction::Confirmed => "confirmed", + _ => "unsupported", + }; + let identity = format!( + "{}\0{}\0{}\0{}", + note.note_id, note.compile_record.idempotency_key, revision_oid, action + ); + Uuid::new_v5(&MEMORY_EVENT_NAMESPACE_V1, identity.as_bytes()) +} + +#[cfg(test)] +mod tests { + use std::{fs, sync::Arc}; + + use chrono::{TimeZone, Utc}; + use sea_orm::{ConnectionTrait, Database, DatabaseConnection, Statement}; + + use super::*; + use crate::internal::{ + ai::{ + context_budget::MemoryAnchorConfidence, + keyed_digest::RepositoryKeyedDigest, + memory::{ + domain::{ + ActorKind, ActorRefV1, CodeChangeStatus, CompileOriginV1, CompileRecordV1, + CompletionStatus, EpisodeClaimV1, EpisodeCodeContextV1, EpisodeOmissionsV1, + EpisodePayloadV1, EpisodeRoot, EpisodeRootKind, EpistemicStatus, EvidenceKind, + EvidenceLocatorV1, EvidenceRefV1, EvidenceSourcePlane, EvidenceVisibility, + IdempotencyScopeV1, MemoryKind, MemoryLifecycle, MemoryNoteV1, MemoryScopeV1, + MemorySensitivity, MemoryTrust, MemoryVisibility, + }, + policy::{ + AuthenticatedMemoryContext, DeterministicMemoryProposal, TrustedMemoryTarget, + }, + }, + }, + db::migration::run_builtin_migrations, + }; + + const REPOSITORY_ID: &str = "memory-writer-test-repository"; + const TEST_CIPHERTEXT: &str = "memory-writer-test-ciphertext"; + const SOURCE_OID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + + struct Fixture { + _temp: tempfile::TempDir, + database: Arc, + writer: Arc, + context: AuthenticatedMemoryContext, + target: TrustedMemoryTarget, + key_id: Uuid, + } + + async fn fixture() -> Fixture { + let database = Database::connect("sqlite::memory:") + .await + .expect("connect test database"); + database + .execute_unprepared(include_str!("../../../../sql/sqlite_20260309_init.sql")) + .await + .expect("apply bootstrap schema"); + run_builtin_migrations(&database) + .await + .expect("apply built-in migrations"); + database + .execute_raw(Statement::from_sql_and_values( + database.get_database_backend(), + "INSERT INTO config_kv(key, value, encrypted) VALUES + ('libra.repoid', ?, 0), (?, ?, 1)", + [ + REPOSITORY_ID.into(), + "memory.keyed_digest.v1".into(), + TEST_CIPHERTEXT.into(), + ], + )) + .await + .expect("seed digest config"); + + let key_id = + Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000").expect("fixed UUID is valid"); + let provider = Arc::new(RepositoryKeyedDigest::for_receipt_tests( + REPOSITORY_ID, + key_id, + [7; 32], + TEST_CIPHERTEXT, + )); + let temp = tempfile::tempdir().expect("create object store"); + fs::create_dir_all(temp.path().join("objects")).expect("create objects directory"); + let writer = Arc::new( + MemoryWriter::for_tests( + temp.path().to_path_buf(), + Arc::new(database.clone()), + provider, + ) + .await + .expect("construct Memory writer"), + ); + let actor = ActorRefV1 { + kind: ActorKind::Agent, + principal_id: "agent:episode-compiler".to_string(), + }; + let context = AuthenticatedMemoryContext::new(REPOSITORY_ID, actor) + .expect("construct authenticated context"); + let target = TrustedMemoryTarget::episode( + EpisodeRoot::task("task-42").expect("construct trusted root"), + ); + Fixture { + _temp: temp, + database: Arc::new(database), + writer, + context, + target, + key_id, + } + } + + fn proposal( + target: &TrustedMemoryTarget, + key_id: Uuid, + generation: u8, + ) -> DeterministicMemoryProposal { + let evidence = EvidenceRefV1 { + schema_version: 1, + source_plane: EvidenceSourcePlane::Git, + kind: EvidenceKind::Code, + object_id: "src/lib.rs".to_string(), + source_ref_oid: SOURCE_OID.to_string(), + locator: EvidenceLocatorV1::CodeRange { + commit_oid: SOURCE_OID.to_string(), + path: "src/lib.rs".to_string(), + start_line: 1, + end_line: 4, + }, + fragment_digest: format!("sha256:{}", "b".repeat(64)), + visibility: EvidenceVisibility::RepoLocal, + captured_at: Utc.with_ymd_and_hms(2026, 8, 24, 8, 0, 0).single(), + code_commit: Some(SOURCE_OID.to_string()), + }; + let observation = EpisodeClaimV1 { + epistemic_status: EpistemicStatus::Observation, + claim: "the focused test failed before the retry fix".to_string(), + confidence: None, + evidence_refs: vec![evidence.clone()], + }; + let inference = EpisodeClaimV1 { + epistemic_status: EpistemicStatus::Inference, + claim: format!("generation {generation} attributes the failure to retry timing"), + confidence: Some(MemoryAnchorConfidence::High), + evidence_refs: vec![evidence.clone()], + }; + let episode = EpisodePayloadV1 { + schema_version: 1, + root_kind: EpisodeRootKind::Task, + root_id: target.root().id().to_string(), + related_intent_ids: Vec::new(), + related_task_ids: vec![target.root().id().to_string()], + related_run_ids: vec![format!("run-{generation}")], + started_at: Utc.with_ymd_and_hms(2026, 8, 24, 8, 0, 0).single(), + ended_at: Utc + .with_ymd_and_hms(2026, 8, 24, 9, u32::from(generation), 0) + .single(), + goal: observation.clone(), + completion_status: CompletionStatus::Completed, + code_change_status: CodeChangeStatus::Changed, + summary: inference.clone(), + observations: vec![observation], + inferences: vec![inference], + decisions: Vec::new(), + failed_attempts: Vec::new(), + unresolved: Vec::new(), + code: EpisodeCodeContextV1 { + base_oid: Some(SOURCE_OID.to_string()), + result_oid: Some(SOURCE_OID.to_string()), + branch_ref: Some("refs/heads/main".to_string()), + paths: vec!["src/lib.rs".to_string()], + }, + omissions: EpisodeOmissionsV1::default(), + }; + let keyed = |fill: char| format!("hmac-sha256:{key_id}:{}", fill.to_string().repeat(64)); + let note = MemoryNoteV1 { + schema_version: 1, + note_id: target.root().note_id(), + content_digest: format!("sha256:{}", "0".repeat(64)), + namespace: target.root().namespace().to_string(), + path: target.root().path().to_string(), + kind: MemoryKind::Episodic, + scope: MemoryScopeV1::Repo, + visibility: MemoryVisibility::RepoLocal, + acl_policy_id: "repo-default-v1".to_string(), + lifecycle: MemoryLifecycle::Accretive, + body: format!("Task episode generation {generation}"), + rationale: None, + episode: Some(episode), + evidence_refs: vec![evidence], + links: Vec::new(), + entities: Vec::new(), + parents: Vec::new(), + tags: vec!["episode".to_string()], + confidence: MemoryAnchorConfidence::High, + trust: MemoryTrust::RepoEvidence, + sensitivity: MemorySensitivity::Internal, + valid_from: None, + valid_until: None, + effective_from_commit: Some(SOURCE_OID.to_string()), + effective_until_commit: None, + expires_at: None, + author: ActorRefV1 { + kind: ActorKind::Agent, + principal_id: "agent:episode-compiler".to_string(), + }, + created_at: Utc + .with_ymd_and_hms(2026, 8, 24, 9, u32::from(generation), 0) + .single() + .expect("valid fixture timestamp"), + compile_record: CompileRecordV1 { + schema_version: 1, + origin: CompileOriginV1::EpisodeCompiler, + producer: "libra-memory/1".to_string(), + rules_version: 1, + prompt_version: Some("episode-v1".to_string()), + model_id: Some("deterministic-test-model".to_string()), + policy_version: "repo-policy-v1".to_string(), + input_hashes: vec![keyed(char::from(b'c' + generation))], + idempotency_key: keyed(char::from(b'd' + generation)), + idempotency_scope: IdempotencyScopeV1::Cell, + }, + }; + DeterministicMemoryProposal::new(note) + } + + async fn count(database: &DatabaseConnection, table: &str) -> i64 { + let sql = format!("SELECT COUNT(*) AS count FROM {table}"); + database + .query_one_raw(Statement::from_string(database.get_database_backend(), sql)) + .await + .expect("query projection count") + .expect("count row exists") + .try_get("", "count") + .expect("decode projection count") + } + + async fn memory_ref_count(database: &DatabaseConnection) -> i64 { + database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + "SELECT COUNT(*) AS count FROM reference + WHERE kind = 'Branch' AND remote IS NULL + AND name = 'libra/memory/repo'" + .to_string(), + )) + .await + .expect("query Memory ref count") + .expect("Memory ref count row exists") + .try_get("", "count") + .expect("decode Memory ref count") + } + + #[tokio::test] + async fn writer_round_trip() { + let fixture = fixture().await; + let committed = fixture + .writer + .commit( + &fixture.context, + &fixture.target, + &proposal(&fixture.target, fixture.key_id, 1), + None, + ) + .await + .expect("commit first Memory revision"); + assert!(committed.appended()); + assert_eq!(committed.event_seq(), 2); + assert_eq!(committed.note_id(), fixture.target.root().note_id()); + assert_eq!(count(&fixture.database, "memory_note_index").await, 1); + assert_eq!(count(&fixture.database, "memory_revision_index").await, 1); + assert_eq!(count(&fixture.database, "memory_head").await, 1); + assert_eq!(count(&fixture.database, "memory_projection_state").await, 1); + let head = fixture + .database + .query_one_raw(Statement::from_string( + fixture.database.get_database_backend(), + "SELECT latest_review_state, live_revision_oid FROM memory_head".to_string(), + )) + .await + .expect("query committed Memory head") + .expect("Memory head exists"); + let review_state: String = head + .try_get("", "latest_review_state") + .expect("decode review state"); + let live_revision: String = head + .try_get("", "live_revision_oid") + .expect("decode live revision"); + assert_eq!(review_state, "confirmed"); + assert_eq!(live_revision, committed.revision_oid().to_string()); + } + + #[tokio::test] + async fn writer_same_key_idempotent() { + let fixture = fixture().await; + let proposal = proposal(&fixture.target, fixture.key_id, 1); + let first = fixture + .writer + .commit(&fixture.context, &fixture.target, &proposal, None) + .await + .expect("commit first Memory revision"); + let second = fixture + .writer + .commit( + &fixture.context, + &fixture.target, + &proposal, + Some(first.commit_oid()), + ) + .await + .expect("deduplicate Memory revision"); + assert!(!second.appended()); + assert_eq!(second.revision_oid(), first.revision_oid()); + assert_eq!(count(&fixture.database, "memory_revision_index").await, 1); + } + + #[tokio::test] + async fn writer_rejects_untrusted_acl_and_producer() { + let fixture = fixture().await; + let mut untrusted_acl = proposal(&fixture.target, fixture.key_id, 1); + untrusted_acl.note_mut().acl_policy_id = "arbitrary-policy".to_string(); + let error = fixture + .writer + .commit(&fixture.context, &fixture.target, &untrusted_acl, None) + .await + .expect_err("unknown ACL policy is rejected"); + assert_eq!(error.kind(), MemoryWriterErrorKind::PolicyRejected); + assert_eq!(error.stable_code(), "LBR-MEMORY-003"); + + let mut untrusted_producer = proposal(&fixture.target, fixture.key_id, 1); + untrusted_producer.note_mut().compile_record.producer = "external-agent/1".to_string(); + let error = fixture + .writer + .commit(&fixture.context, &fixture.target, &untrusted_producer, None) + .await + .expect_err("unknown producer is rejected"); + assert_eq!(error.kind(), MemoryWriterErrorKind::PolicyRejected); + assert_eq!(error.stable_code(), "LBR-MEMORY-003"); + + let mut secret = proposal(&fixture.target, fixture.key_id, 1); + secret.note_mut().sensitivity = MemorySensitivity::SecretLike; + let error = fixture + .writer + .commit(&fixture.context, &fixture.target, &secret, None) + .await + .expect_err("secret-like Memory is rejected before object persistence"); + assert_eq!(error.kind(), MemoryWriterErrorKind::PolicyRejected); + assert_eq!(memory_ref_count(&fixture.database).await, 0); + } + + #[tokio::test] + async fn writer_creates_two_distinct_episode_cells() { + let fixture = fixture().await; + let first = fixture + .writer + .commit( + &fixture.context, + &fixture.target, + &proposal(&fixture.target, fixture.key_id, 1), + None, + ) + .await + .expect("commit first Episode cell"); + let second_target = TrustedMemoryTarget::episode( + EpisodeRoot::task("task-43").expect("construct second trusted root"), + ); + let second = fixture + .writer + .commit( + &fixture.context, + &second_target, + &proposal(&second_target, fixture.key_id, 1), + Some(first.commit_oid()), + ) + .await + .expect("commit second Episode cell"); + assert!(second.appended()); + assert_eq!(second.event_seq(), 4); + assert_eq!(count(&fixture.database, "memory_note_index").await, 2); + } + + #[tokio::test] + async fn writer_fails_closed_on_duplicate_cell_heads() { + let fixture = fixture().await; + let committed = fixture + .writer + .commit( + &fixture.context, + &fixture.target, + &proposal(&fixture.target, fixture.key_id, 1), + None, + ) + .await + .expect("commit first Memory revision"); + fixture + .database + .execute_unprepared( + "INSERT INTO memory_note_index + SELECT '00000000-0000-4000-8000-000000000002', scope_key, namespace, path, + kind, lifecycle, review_state, confidence, trust, sensitivity, visibility, + acl_policy_id, origin, 'duplicate-cell-key', idempotency_scope, created_at + FROM memory_note_index; + INSERT INTO memory_revision_index + SELECT 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + '00000000-0000-4000-8000-000000000002', scope_key, namespace, origin, + producer, rules_version, prompt_version, model_id, policy_version, + input_fingerprints_json, created_at + FROM memory_revision_index; + INSERT INTO memory_head + SELECT scope_key, namespace, path, + '00000000-0000-4000-8000-000000000002', + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', latest_action, + latest_review_state, kind, lifecycle, confidence, trust, sensitivity, + visibility, acl_policy_id, valid_from, valid_until, effective_from_commit, + effective_until_commit, expires_at, rank_hint, last_event_seq, updated_at + FROM memory_head;", + ) + .await + .expect("seed duplicate Cell head corruption"); + let error = fixture + .writer + .commit( + &fixture.context, + &fixture.target, + &proposal(&fixture.target, fixture.key_id, 2), + Some(committed.commit_oid()), + ) + .await + .expect_err("duplicate Cell heads fail closed"); + assert_eq!(error.kind(), MemoryWriterErrorKind::CorruptProjection); + assert_eq!(error.stable_code(), "LBR-MEMORY-004"); + } + + #[tokio::test] + async fn production_open_round_trip_revalidates_persisted_key() { + let temp = tempfile::tempdir().expect("create production writer repository"); + let storage = temp.path().join(".libra"); + fs::create_dir_all(storage.join("objects")).expect("create production object store"); + let database_path = storage.join(DATABASE); + let database = db::create_database(&database_path.to_string_lossy()) + .await + .expect("create production repository database"); + database + .execute_raw(Statement::from_sql_and_values( + database.get_database_backend(), + "INSERT INTO config_kv(key, value, encrypted) VALUES + ('libra.repoid', ?, 0), ('vault.unsealkey', ?, 0)", + [REPOSITORY_ID.into(), hex::encode([0x42_u8; 32]).into()], + )) + .await + .expect("seed production repository identity and vault key"); + database.close().await.expect("close setup connection"); + + let writer = MemoryWriter::open(storage.clone()) + .await + .expect("open production Memory writer"); + let context = AuthenticatedMemoryContext::new( + REPOSITORY_ID, + ActorRefV1 { + kind: ActorKind::Agent, + principal_id: "agent:episode-compiler".to_string(), + }, + ) + .expect("construct production context"); + let target = TrustedMemoryTarget::episode( + EpisodeRoot::task("task-production").expect("construct production target"), + ); + let committed = writer + .commit( + &context, + &target, + &proposal(&target, writer.digest_provider.key_id(), 1), + None, + ) + .await + .expect("commit through production writer"); + assert!(committed.appended()); + + writer + .database + .execute_unprepared( + "UPDATE config_kv SET value = 'changed-ciphertext' + WHERE key = 'memory.keyed_digest.v1'", + ) + .await + .expect("corrupt persisted digest binding"); + let error = writer + .commit( + &context, + &target, + &proposal(&target, writer.digest_provider.key_id(), 2), + Some(committed.commit_oid()), + ) + .await + .expect_err("changed persisted digest key fails closed"); + assert_eq!(error.kind(), MemoryWriterErrorKind::DigestKeyUnavailable); + assert_eq!(error.stable_code(), "LBR-MEMORY-001"); + } + + #[tokio::test] + async fn writer_concurrent_first_create() { + let fixture = fixture().await; + fixture + .writer + .set_test_before_first_cas(Arc::new(tokio::sync::Barrier::new(2))); + let proposal = proposal(&fixture.target, fixture.key_id, 1); + let (first, second) = tokio::join!( + fixture + .writer + .commit(&fixture.context, &fixture.target, &proposal, None), + fixture + .writer + .commit(&fixture.context, &fixture.target, &proposal, None), + ); + let first = first.expect("first concurrent writer succeeds"); + let second = second.expect("second concurrent writer converges"); + assert_eq!(first.note_id(), second.note_id()); + assert_eq!(first.revision_oid(), second.revision_oid()); + assert_eq!(count(&fixture.database, "memory_revision_index").await, 1); + } + + #[tokio::test] + async fn writer_cas_rebuilds_revision() { + let fixture = fixture().await; + let first = fixture + .writer + .commit( + &fixture.context, + &fixture.target, + &proposal(&fixture.target, fixture.key_id, 1), + None, + ) + .await + .expect("commit first revision"); + let second = fixture + .writer + .commit( + &fixture.context, + &fixture.target, + &proposal(&fixture.target, fixture.key_id, 2), + None, + ) + .await + .expect("rebuild proposal on current Memory head"); + assert!(second.appended()); + assert_eq!(second.event_seq(), 4); + assert_ne!(first.revision_oid(), second.revision_oid()); + assert_eq!(count(&fixture.database, "memory_revision_index").await, 2); + let note_row = fixture + .database + .query_one_raw(Statement::from_string( + fixture.database.get_database_backend(), + "SELECT idempotency_key, origin FROM memory_note_index".to_string(), + )) + .await + .expect("query note creation identity") + .expect("note projection exists"); + let first_key: String = note_row + .try_get("", "idempotency_key") + .expect("decode creation key"); + let origin: String = note_row.try_get("", "origin").expect("decode origin"); + assert_eq!( + first_key, + proposal(&fixture.target, fixture.key_id, 1) + .note() + .compile_record + .idempotency_key + ); + assert_eq!(origin, "episode_compiler"); + } + + #[tokio::test] + async fn writer_fault_windows() { + let object_fault = fixture().await; + fs::remove_dir_all(object_fault._temp.path().join("objects")) + .expect("remove object directory"); + fs::write(object_fault._temp.path().join("objects"), b"blocked") + .expect("block object directory creation"); + let error = object_fault + .writer + .commit( + &object_fault.context, + &object_fault.target, + &proposal(&object_fault.target, object_fault.key_id, 1), + None, + ) + .await + .expect_err("object fault rejects commit"); + assert_eq!(error.kind(), MemoryWriterErrorKind::StorageFailure); + assert_eq!(count(&object_fault.database, "memory_head").await, 0); + assert_eq!(memory_ref_count(&object_fault.database).await, 0); + + let projection_fault = fixture().await; + projection_fault + .database + .execute_unprepared( + "CREATE TRIGGER memory_writer_projection_fault + BEFORE INSERT ON memory_revision_index + BEGIN SELECT RAISE(ABORT, 'injected projection fault'); END;", + ) + .await + .expect("inject projection failure"); + let error = projection_fault + .writer + .commit( + &projection_fault.context, + &projection_fault.target, + &proposal(&projection_fault.target, projection_fault.key_id, 1), + None, + ) + .await + .expect_err("projection fault rejects commit"); + assert_eq!(error.kind(), MemoryWriterErrorKind::StorageFailure); + assert_eq!( + count(&projection_fault.database, "memory_note_index").await, + 0 + ); + assert_eq!( + count(&projection_fault.database, "memory_revision_index").await, + 0 + ); + assert_eq!( + count(&projection_fault.database, "memory_projection_state").await, + 0 + ); + assert_eq!(memory_ref_count(&projection_fault.database).await, 0); + } + + #[tokio::test] + async fn writer_rejects_unknown_digest_key_with_stable_code() { + let fixture = fixture().await; + let mut proposal = proposal(&fixture.target, fixture.key_id, 1); + proposal.note_mut().compile_record.idempotency_key = format!( + "hmac-sha256:550e8400-e29b-41d4-a716-446655440001:{}", + "e".repeat(64) + ); + let error = fixture + .writer + .commit(&fixture.context, &fixture.target, &proposal, None) + .await + .expect_err("unknown digest key is rejected"); + assert_eq!(error.kind(), MemoryWriterErrorKind::UnknownDigestKey); + assert_eq!(error.stable_code(), "LBR-MEMORY-003"); + } + + #[tokio::test] + async fn writer_repository_boundaries_fail_closed() { + let missing_key = fixture().await; + missing_key + .database + .execute_unprepared("DELETE FROM config_kv WHERE key = 'memory.keyed_digest.v1'") + .await + .expect("remove digest config"); + let error = missing_key + .writer + .commit( + &missing_key.context, + &missing_key.target, + &proposal(&missing_key.target, missing_key.key_id, 1), + None, + ) + .await + .expect_err("missing digest config blocks Memory writes"); + assert_eq!(error.kind(), MemoryWriterErrorKind::DigestKeyUnavailable); + assert_eq!(error.stable_code(), "LBR-MEMORY-001"); + + let missing_identity = fixture().await; + missing_identity + .database + .execute_unprepared("DELETE FROM config_kv WHERE key = 'libra.repoid'") + .await + .expect("remove repository identity"); + let error = missing_identity + .writer + .commit( + &missing_identity.context, + &missing_identity.target, + &proposal(&missing_identity.target, missing_identity.key_id, 1), + None, + ) + .await + .expect_err("missing repository identity blocks Memory writes"); + assert_eq!(error.kind(), MemoryWriterErrorKind::CorruptProjection); + assert_eq!(error.stable_code(), "LBR-MEMORY-004"); + } + + #[tokio::test] + async fn writer_corrupt_revision_fails_loud() { + let fixture = fixture().await; + let committed = fixture + .writer + .commit( + &fixture.context, + &fixture.target, + &proposal(&fixture.target, fixture.key_id, 2), + None, + ) + .await + .expect("commit first Memory revision"); + let oid = committed.revision_oid().to_string(); + fs::remove_file( + fixture + ._temp + .path() + .join("objects") + .join(&oid[..2]) + .join(&oid[2..]), + ) + .expect("remove authoritative revision object"); + let error = fixture + .writer + .commit( + &fixture.context, + &fixture.target, + &proposal(&fixture.target, fixture.key_id, 1), + Some(committed.commit_oid()), + ) + .await + .expect_err("missing authoritative revision fails loud"); + assert_eq!(error.kind(), MemoryWriterErrorKind::CorruptHistory); + assert_eq!(error.stable_code(), "LBR-MEMORY-004"); + } +} From 523bf55629996c44b12b6503f1dcb5083b8d0c8f Mon Sep 17 00:00:00 2001 From: anduin9527 Date: Tue, 25 Aug 2026 00:56:45 +0800 Subject: [PATCH 08/18] feat(memory): protect local-only memory ref Signed-off-by: anduin9527 --- COMPATIBILITY.md | 2 +- docs/commands/branch.md | 3 + docs/commands/clone.md | 3 +- docs/commands/fetch.md | 5 + docs/commands/op.md | 7 +- docs/commands/push.md | 13 +- src/command/branch.rs | 11 +- src/command/clone.rs | 22 ++- src/command/fetch.rs | 125 +++++++++++++- src/command/maintenance.rs | 34 +++- src/command/op.rs | 113 ++++++++++++- src/command/push.rs | 226 +++++++++++++++++++++++--- src/command/symbolic_ref.rs | 12 +- src/command/update_ref.rs | 11 +- src/internal/ai/linear_ref.rs | 122 +++++++++++++- src/internal/ai/memory/writer.rs | 99 +++++++++-- src/internal/branch.rs | 33 ++-- src/internal/operation_wrapper.rs | 20 +++ src/internal/protocol/local_client.rs | 49 +++++- tests/command/branch_test.rs | 58 +++++++ tests/command/op_test.rs | 83 +++++++++- tests/operation_wrapper_test.rs | 36 ++++ 22 files changed, 993 insertions(+), 94 deletions(-) diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index 1f34d28d7..cbed60961 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -155,7 +155,7 @@ compatibility guard is pinned by `compat_global_config_schema_future`. | describe | partial | basic describe, `--tags`, `--always`, `--abbrev`, `--exact-match`, `--long`, `--dirty[=]`, `--first-parent`, `--match`/`--exclude` (wax globs, ≤256 chars; exclude wins over match), `--candidates ` (n=0 ⇒ exact-match), `--all` (use any ref — branches/remotes/tags — with `heads/`/`remotes/`/`tags/` prefixes), and `--contains` (git name-rev: name a commit relative to the nearest descendant tag — ``, `~`, or `~^~`) supported | | notes | partial | `add` / `append` / `copy` / `edit` / `show` / `list` / `remove` / `merge` supported; `--ref` supported; `notes merge` is a 2-way merge of the flat note rows (Libra notes are SQLite-backed, not commit-backed trees) with `--strategy=manual` (default; aborts on a conflicting note — no NOTES_MERGE worktree)/`ours`/`theirs`/`union`/`cat_sort_uniq`; `prune` (remove notes whose annotated object no longer exists in the object store — `-n`/`--dry-run` and `-v`) and `get-ref` (print the active notes ref) supported; the interactive editor fallback for `add`/`edit`/`append` when no `-m`/`-F` is given is supported (`edit` pre-fills the existing note; notes preserve `#` lines — not stripped as comments) | | cherry-pick | partial | one-or-more commit replay, preserving the picked commit's author metadata while using current committer metadata (including `GIT_COMMITTER_*` date/identity overrides), de-signing source messages before deriving the replayed body/subject, `-n/--no-commit` (now also for multi-commit), `-x`, `-s/--signoff`, `-e/--edit`, `-m/--mainline`, `--ff`, `-S/--gpg-sign`, `--allow-empty`, `--allow-empty-message`, `--keep-redundant-commits`, `--empty=` (`stop` default / `drop` skips a redundant pick / `keep` records the empty commit — == `--keep-redundant-commits`), `--cleanup=` (`strip`/`whitespace`/`verbatim`/`scissors`/`default` message cleanup — cleans the body/edited buffer first, then appends `-x`/`Signed-off-by` trailers; `default`/`scissors` fall back to `whitespace` without an editor), repeatable last-wins `-X/--strategy-option ours|theirs` (favor only conflicting hunks while retaining clean changes; whole-path add/delete conflicts choose the requested side), and the SQLite conflict sequencer (`--continue`/`--skip`/`--abort`/`--quit` with line-level three-way conflict markers — diverging hunks only, like Git; delete/modify and binary fall back to whole-file; the `merge.conflictStyle` config is honored (`diff3` adds the `||||||| base` block) — and a merge/rebase mutex) supported; `--rerere-autoupdate` is honoured (when `rerere.enabled` it makes the rerere hook stage a replayed resolution; a no-op with rerere off); custom `--strategy` values remain explicitly rejected and unimplemented; cherry-pick's conflict state now lives in the unified `sequence_state` table (lore.md 2.6 — the lazy `cherry_pick_state` DDL and the `revert_sequence` orphan are retired by migration 2026070401; storage change is transparent). lore.md 2.6 (unified sequencer): any in-progress multi-step sequence (merge / revert / cherry-pick / rebase) now REJECTS starting a DIFFERENT one with `LBR-CONFLICT-002` naming the blocking op — a symmetric cross-op mutex (previously only merge/rebase blocked an in-progress cherry-pick); the in-progress op's own `--continue`/`--abort`/`--skip` stay available, and `libra status` surfaces a non-merge sequence in progress. | -| push | partial | branch/tag update, multi-refspec, delete (`-d`/`--delete` or a `:` refspec), `--tags`, and `--mirror` supported; `--force-with-lease[=[:]]` (validates the remote still matches the tracking-ref/expected OID before sending; conflicts with `--force`) and `--porcelain` (machine-readable per-ref lines; conflicts with `--json`/`--machine`) supported; `--atomic` supported (advertises the `atomic` capability so the remote applies all ref updates together; refused up-front if the remote does not advertise `atomic`); `--push-option`/`-o ` supported (sends a push-options section gated on the remote's `push-options` capability); `--follow-tags` supported (also pushes annotated tags reachable from a pushed ref and missing on the remote); `--signed` supported (builds a GPG-signed push certificate via the vault signer, gated on the remote's `push-cert` capability/nonce); `--no-progress` supported (suppresses the "Compressing objects" / "Writing objects" progress meter on stderr, like `git push --no-progress`); `--no-verify` accepted as a **no-op** (Libra runs no client-side `pre-push` hook; declined-decision D3); `--no-thin` spells the self-contained default explicitly. local file remote rejected — intentional (see [docs/development/commands/_compatibility.md#d2-本地-file-remote-的-push](docs/development/commands/_compatibility.md#d2-本地-file-remote-的-push)); lore.md 2.10 refinements: `--force-if-includes` is REAL (with the All/Ref lease forms it additionally requires the remote-tracking tip to be integrated locally — pushing the tip, a descendant, or a tip REACHABLE from the pushed branch's reflog; conservative rejections, whole-push error rather than a per-ref porcelain row — documented divergence; silent no-op with the exact lease form or no lease, Git parity); `--thin` is REAL (REF_DELTA entries against server-known bases — the advertised old tips; self-contained Libra delta encoder with git-convention 64KiB copy ops since git-internal's delta module is private; net-win + 8MiB caps, full fallback on any miss; verified round-trip against real `git receive-pack` on BOTH unpack paths); self-contained remains the DEFAULT (`push.thin` config unsupported — intentionally different from git's thin-by-default, revisit recorded); lease tracking lookup now consults both tracking-ref naming conventions (clone=short, fetch=full — a pre-existing fetch-then-lease miss); real-git-server interop matrix in L1 (capability degrades refuse cleanly with nothing sent; push-options round-trip via pre-receive hook) | +| push | partial | branch/tag update, multi-refspec, delete (`-d`/`--delete` or a `:` refspec), `--all`, `--tags`, and `--mirror` supported; the repository-local Memory authority is excluded from ordinary, `--all`, and `--mirror` plans, and exact explicit refspecs are rejected before transfer; `--force-with-lease[=[:]]` (validates the remote still matches the tracking-ref/expected OID before sending; conflicts with `--force`) and `--porcelain` (machine-readable per-ref lines; conflicts with `--json`/`--machine`) supported; `--atomic` supported (advertises the `atomic` capability so the remote applies all ref updates together; refused up-front if the remote does not advertise `atomic`); `--push-option`/`-o ` supported (sends a push-options section gated on the remote's `push-options` capability); `--follow-tags` supported (also pushes annotated tags reachable from a pushed ref and missing on the remote); `--signed` supported (builds a GPG-signed push certificate via the vault signer, gated on the remote's `push-cert` capability/nonce); `--no-progress` supported (suppresses the "Compressing objects" / "Writing objects" progress meter on stderr, like `git push --no-progress`); `--no-verify` accepted as a **no-op** (Libra runs no client-side `pre-push` hook; declined-decision D3); `--no-thin` spells the self-contained default explicitly. local file remote rejected — intentional (see [docs/development/commands/_compatibility.md#d2-本地-file-remote-的-push](docs/development/commands/_compatibility.md#d2-本地-file-remote-的-push)); lore.md 2.10 refinements: `--force-if-includes` is REAL (with the All/Ref lease forms it additionally requires the remote-tracking tip to be integrated locally — pushing the tip, a descendant, or a tip REACHABLE from the pushed branch's reflog; conservative rejections, whole-push error rather than a per-ref porcelain row — documented divergence; silent no-op with the exact lease form or no lease, Git parity); `--thin` is REAL (REF_DELTA entries against server-known bases — the advertised old tips; self-contained Libra delta encoder with git-convention 64KiB copy ops since git-internal's delta module is private; net-win + 8MiB caps, full fallback on any miss; verified round-trip against real `git receive-pack` on BOTH unpack paths); self-contained remains the DEFAULT (`push.thin` config unsupported — intentionally different from git's thin-by-default, revisit recorded); lease tracking lookup now consults both tracking-ref naming conventions (clone=short, fetch=full — a pre-existing fetch-then-lease miss); real-git-server interop matrix in L1 (capability degrades refuse cleanly with nothing sent; push-options round-trip via pre-receive hook) | | fetch | partial | repository/refspec supports short sources, exact `:`, one wildcard per side, case-insensitive `remote..fetch`, and exact destinations limited to `refs/heads/*` / `refs/remotes//*` (reserved `HEAD` is refused). Destination/reflog/remote-HEAD writes are transactional; a checked-out destination in any worktree and non-fast-forward mappings without `+`/`--force` fail closed. `--all`, `--depth`, `--dry-run` (no download/writes; unknown pre-download ancestry is not reported forced), `-v`/`--verbose`, `--porcelain` (rejects `--json`), tags, `-f`/`--force`, `--no-auto-gc`, `--no-progress`, `--notes`, and `FETCH_HEAD`/`--append` are supported. `--prune`/`-p` deletes tracking refs not live under the effective configured destination mapping; one-off explicit refspecs preserve configured mappings plus ordinary advertised tracking destinations; empty advertisements skip prune. Deletes/reflogs are transactional and `--dry-run` only reports. `--no-prune` overrides strict `remote..prune` then `fetch.prune` defaults. `--refmap`, `--atomic`, and shallow-expansion flags remain deferred. Network fetches have bounded connect, idle/read, and first-byte timeouts; local remotes are exempt. | | format-patch | partial | `A..B`/single-revision ranges, exact `-1 [rev]`, `--root [rev]`, and `--ignore-if-in-upstream` stable patch-id suppression are supported; merge commits are skipped. Output controls include `-o`/`--output-directory`, `--stdout`, `-n`/`--numbered`, `--start-number`, `--numbered-files`, `--suffix`, `--subject-prefix`, `--cover-letter`, `--thread`/`--no-thread`, `--in-reply-to`, `-v`/`--reroll-count`, `-s`/`--signoff` and `--no-signoff`, `--full-index`, `--minimal` (equivalent to Libra's already-shortest default Myers), `--histogram`, `--src-prefix`/`--dst-prefix`, `--no-stat`, `--keep-subject`, `--zero-commit`, signature controls, and RFC 2047 header encoding. `format.subjectPrefix`, `format.signOff`, `format.outputDirectory`, and `format.suffix` use the strict local→global→system cascade with explicit CLI precedence (`--stdout` bypasses outputDirectory). Mail controls include repeatable `--to`/`--cc`, `--no-to`/`--no-cc`, `--from`, `--notes[=]`, and mutually exclusive `--attach`/`--inline`; generated plain and MIME output is consumed by real Git `am`, and Git plain format-patch output is consumed by Libra `am`. `--base ` emits a `base-commit:` plus oldest-first stable `prerequisite-patch-id:` lines on the last patch/cover letter; the base must be an ancestor, `--base=auto` is rejected, and binary prerequisite ids are not guaranteed to match Git. `--interdiff` and `--range-diff` remain unexposed (`--force` is not a Git format-patch flag). | | pull | partial | fetch + fast-forward/three-way merge supported; `--ff-only`, `--rebase`, `--no-rebase` (countermands `--rebase`, last wins), `--ff`, `--no-ff` (forces a merge commit), and `pull.rebase` / `branch..rebase` / `pull.ff=true|false|only` defaults when CLI flags are absent (config cascade local→global→system, case-insensitive variable matching, local/global encrypted values decrypted, legacy config rows honored; empty/invalid values fail before fetch with `LBR-CLI-002`, local/global config reads with `LBR-IO-001`, unsupported `merges|interactive` modes produce an explicit `LBR-CLI-002`, system-scope failures skipped), fetch `--depth` (shallow pull), `--squash`, `--no-commit`, `--commit` (commit the merge result; may be combined with `--ff`/`--no-ff`/`--ff-only`, does not override fast-forward policy, and is last-one-wins with `--no-commit`), `--autostash` (stash tracked changes before integrating and re-apply after), `--no-progress` (forward `--no-progress` to the fetch, suppressing its progress meter), and `--notes` (forward to the fetch — import the `refs/notes/deps` dependency graph from a local Libra upstream; default OFF, D17) exposed; `pull --autostash` on the MERGE path now rides the merge-owned autostash (held on conflict, applied by merge --continue/--abort; `merge.autostash` config honored) — the rebase path keeps the legacy push/pop wrap; no-upstream pull renders Git-style tracking advice (`libra pull ` / `libra branch --set-upstream-to=...`); runs in a linked worktree in ALL modes since W2 (merge/ff v0.19.35, rebase v0.19.42, and `--rebase --autostash` v0.19.54 — the autostash wrap uses the stash stack-lock + by-id CAS protocol and pops exactly its own entry) | diff --git a/docs/commands/branch.md b/docs/commands/branch.md index 931501e0f..85219cc82 100644 --- a/docs/commands/branch.md +++ b/docs/commands/branch.md @@ -57,6 +57,9 @@ The `--contains` and `--no-contains` filters (aliased as `--with` and `--without | | `--no-column` | | Do not lay the branch list out in columns (equivalent to `--column=never`), countermanding an earlier `--column` (last one wins). Branches list one-per-line by default, so on its own this is a no-op. | | `-v` | `--verbose` | | List each branch with its tip's short sha and commit subject. Repeat (`-vv`) to also show the upstream-tracking segment `[: ahead N, behind M]` (counts omitted when the remote-tracking ref has not been fetched; nothing shown for a branch with no configured upstream). Takes precedence over `--column`. | +Libra-owned local-only Memory state is hidden from local, remote, and `--all` +branch listings. Ordinary user branches with similar names remain visible. + ### Flag examples ```bash diff --git a/docs/commands/clone.md b/docs/commands/clone.md index 4df8f4df1..75cfdd2e0 100644 --- a/docs/commands/clone.md +++ b/docs/commands/clone.md @@ -149,7 +149,8 @@ repository. Not supported for `libra+cloud://` sources (rejected with Narrowings vs Git: (1) Git mirrors `refs/*:refs/*` verbatim; Libra mirrors only what its fetch transfers — every fetched branch is promoted to `refs/heads/*` and tags are kept, but ref namespaces Libra does not fetch (e.g. `refs/notes/*`) are -not mirrored. (2) Because Libra's fetch collapses `refs/heads/mr/*` and +not mirrored. The repository-local Memory authority is never fetched or +promoted. (2) Because Libra's fetch collapses `refs/heads/mr/*` and `refs/mr/*` into one tracking namespace, any such refs are mirrored as `refs/heads/mr/*` (provenance is not preserved). (3) The `mirror=true` marker is informational — no `+refs/*:refs/*` refspec is recorded and `libra fetch` is not diff --git a/docs/commands/fetch.md b/docs/commands/fetch.md index e0046db4e..5f64563fa 100644 --- a/docs/commands/fetch.md +++ b/docs/commands/fetch.md @@ -22,6 +22,11 @@ one source ref and may map it to an exact local destination (`:`). Whe explicit refspec is given, `remote..fetch` entries are honored; if none exist, all advertised branches use the default `refs/remotes//*` mapping. +The repository-local Memory authority (`refs/heads/libra/memory/repo`) is +excluded from ordinary discovery plans and wildcard expansion. Explicit +refspecs cannot name it as a source or local destination, and a remote HEAD +that points at it is ignored when choosing the default working branch. + Fetch supports SSH, HTTPS, local file, and `git://` transports. Vault-backed SSH keys are loaded automatically when configured via `vault.ssh..privkey`. diff --git a/docs/commands/op.md b/docs/commands/op.md index ba3402be8..82d9b5465 100644 --- a/docs/commands/op.md +++ b/docs/commands/op.md @@ -100,6 +100,11 @@ refs and Libra-owned internal refs (the locked `main`/`intent`/`traces` branches and the reserved `libra/` namespace, e.g. the AI history branch `libra/intent`) are never pruned. +New operation snapshots omit the repository-local Memory authority. When an +older snapshot already contains `libra/memory/repo`, restore skips it, reports +the skipped name, and leaves both its current object ID and projection +watermark unchanged. + ```bash libra op restore [--force] [--dry-run] ``` @@ -147,4 +152,4 @@ libra op restore @{1} --dry-run - `op restore --dry-run` does not write a new operation. - Restore resets HEAD and the branch refs captured in the target view, and prunes local branches that are absent from that view (the restored HEAD branch - is always kept; remote-tracking refs are left untouched). \ No newline at end of file + is always kept; remote-tracking refs are left untouched). diff --git a/docs/commands/push.md b/docs/commands/push.md index ee041145d..76f900bf0 100644 --- a/docs/commands/push.md +++ b/docs/commands/push.md @@ -17,9 +17,14 @@ libra push [OPTIONS] [ [...]] remote. When invoked without arguments it pushes the current branch to its configured upstream remote. When a `repository` and one or more `refspec` values are given, all refspecs are validated before any network write and then sent in one receive-pack -request. `--tags` pushes all local tags, and `--mirror` mirrors local branch/tag refs +request. `--all` pushes all ordinary local branches, `--tags` pushes all local tags, +and `--mirror` mirrors ordinary local branch/tag refs to the remote, including deletion of remote-only refs. +The repository-local Memory authority (`refs/heads/libra/memory/repo`) never +participates in ordinary push, `--all`, or `--mirror`. An explicit source, +destination, or deletion refspec naming it is rejected before object transfer. + The command negotiates with the remote to determine which objects are missing, packs them into a single pack file, and sends the pack along with a ref-update request. If the remote ref has diverged (non-fast-forward), the push is rejected unless `--force` is used. @@ -54,11 +59,11 @@ global storage config for that run. | Flag / Argument | Description | Example | |-----------------|-------------|---------| -| `` | Remote name (e.g. `origin`). Required when ``, `--tags`, or `--mirror` is used. | `libra push origin main` | +| `` | Remote name (e.g. `origin`). Required when ``, `--all`, `--tags`, or `--mirror` is used. | `libra push origin main` | | `...` | Local ref, `:` mapping, or `:` deletion. Multiple values are sent as one update set. | `libra push origin main feature:release` | | `-u`, `--set-upstream` | Set the upstream tracking branch after a successful single branch push. | `libra push -u origin feature-x` | | `-f`, `--force` | Allow non-fast-forward updates that overwrite remote history. | `libra push --force origin main` | -| `-d`, `--delete` | Delete the named remote refs (each `` is rewritten to a `:` deletion). Requires at least one ref; conflicts with `--set-upstream`/`--tags`/`--mirror`. | `libra push -d origin feature-x` | +| `-d`, `--delete` | Delete the named remote refs (each `` is rewritten to a `:` deletion). Requires at least one ref; conflicts with `--set-upstream`/`--tags`/`--all`/`--mirror`. | `libra push -d origin feature-x` | | `--force-with-lease[=[:]]` | Allow a non-fast-forward update only if the remote ref still matches the expected OID (the tracking-ref OID by default, or an explicit ``). Conflicts with `--force`. | `libra push --force-with-lease origin main` | | `--force-if-includes` | With `--force-with-lease` (All/Ref forms): additionally require the remote-tracking tip to be integrated locally (reachable from the pushed branch's reflog). Silent no-op with the exact lease form or without a lease (Git parity). | | `--thin` | Send REF_DELTA entries against server-known bases (the advertised old tips) — smaller packs on large-blob edits; the server completes them (`index-pack --fix-thin`). Self-contained packs remain the default (unlike git). | @@ -67,6 +72,7 @@ global storage config for that run. | `--porcelain` | Machine-readable output: a `To ` header then `\t:\t

` per ref. Conflicts with `--json`/`--machine`. | `libra push --porcelain origin main` | | `-n`, `--dry-run` | Perform negotiation and object collection but skip the actual upload. Reports what would be pushed. | `libra push --dry-run` | | `--tags` | Push all local `refs/tags/*` refs. Existing identical remote tags are skipped. | `libra push --tags origin` | +| `--all` | Push every ordinary local `refs/heads/*` branch. Libra-owned local-only Memory is excluded. | `libra push --all origin` | | `--mirror` | Mirror local `refs/heads/*` and `refs/tags/*` to the remote, deleting remote-only branch/tag refs. Use with `--dry-run` to preview. | `libra push --mirror --dry-run origin` | | `--json` | Emit structured JSON envelope to stdout (global flag). | `libra push --json` | | `--machine` | Compact single-line JSON; suppresses progress (global flag). | `libra push --machine` | @@ -88,6 +94,7 @@ libra push origin main feature:release libra push origin :stale-branch libra push origin refs/tags/v1.0:refs/tags/v1.0 libra push --tags origin +libra push --all origin libra push --mirror --dry-run origin libra push --json ``` diff --git a/src/command/branch.rs b/src/command/branch.rs index 2e3763495..c39c9cadd 100644 --- a/src/command/branch.rs +++ b/src/command/branch.rs @@ -32,7 +32,10 @@ use crate::{ command::{get_target_commit, load_object, log::get_reachable_commits}, info_println, internal::{ - ai::automation::{VCS_EVENT_POST_BRANCH, dispatch_current_repo_vcs_event_to_history}, + ai::{ + automation::{VCS_EVENT_POST_BRANCH, dispatch_current_repo_vcs_event_to_history}, + linear_ref::OwnedRefSpec, + }, branch::{self, Branch}, config::ConfigKv, db::get_db_conn_instance, @@ -1839,6 +1842,12 @@ async fn collect_branch_output(args: &BranchArgs) -> Result Some( diff --git a/src/command/clone.rs b/src/command/clone.rs index 8a62470a3..a9d008dd8 100644 --- a/src/command/clone.rs +++ b/src/command/clone.rs @@ -32,7 +32,7 @@ use crate::{ restore::{RestoreArgs, RestoreError}, }, internal::{ - ai::history::HistoryManager, + ai::{history::HistoryManager, linear_ref::OwnedRefSpec}, branch::{self, Branch}, config::{ ConfigKv, LocalIdentityTarget, RemoteConfig, read_cascaded_config_value_decrypted, @@ -3610,6 +3610,14 @@ async fn normalize_mirror_refs(remote_name: &str) -> Result<(), CloneError> { .name .strip_prefix(&tracking_prefix) .unwrap_or(&branch.name); + if OwnedRefSpec::for_storage_name(short_name) + .is_some_and(|spec| !spec.policy().operation_snapshot) + { + Branch::delete_branch_result_with_conn(&db, &branch.name, Some(remote_name)) + .await + .map_err(|source| CloneError::LocalBranchState { source })?; + continue; + } // Promote every fetched tracking ref to a verbatim local branch // (idempotent: the default branch already exists from setup_repository // and is simply re-affirmed). We promote ALL tracking rows rather than @@ -4427,6 +4435,14 @@ mod tests { Branch::update_branch_with_conn(&db, "refs/remotes/origin/feature", &hash, Some("origin")) .await .expect("seed tracking feature"); + Branch::update_branch_with_conn( + &db, + "refs/remotes/origin/libra/memory/repo", + &hash, + Some("origin"), + ) + .await + .expect("seed legacy tracking Memory ref"); Head::update_with_conn(&db, Head::Branch("main".to_string()), Some("origin")).await; assert!( Head::remote_current_with_conn(&db, "origin") @@ -4454,6 +4470,10 @@ mod tests { local.iter().any(|n| n == "feature"), "feature promoted: {local:?}" ); + assert!( + !local.iter().any(|n| n == "libra/memory/repo"), + "local-only Memory ref must not be promoted: {local:?}" + ); // No remote-tracking branches and no cached remote HEAD remain. let tracking = Branch::list_branches_result_with_conn(&db, Some("origin")) diff --git a/src/command/fetch.rs b/src/command/fetch.rs index 862c4b681..4d4f38891 100644 --- a/src/command/fetch.rs +++ b/src/command/fetch.rs @@ -35,6 +35,7 @@ use crate::{ }, git_protocol::ServiceType::{self, UploadPack}, internal::{ + ai::linear_ref::{OwnedRefSpec, OwnedRefTransportPolicy}, branch::Branch, config::{ConfigKv, ConfigKvEntry, RemoteConfig}, head::Head, @@ -1471,6 +1472,11 @@ fn validate_fetch_destination(destination: &str, refspec: &str) -> Result<(), Fe } } +fn fetch_ref_is_local_only(name: &str) -> bool { + OwnedRefSpec::for_transport_ref(name) + .is_some_and(|spec| spec.transport_policy() == OwnedRefTransportPolicy::LocalOnly) +} + fn parse_fetch_refspec(raw: &str, remote: &str) -> Result { if raw.is_empty() || raw.matches(':').count() > 1 { return Err(FetchError::InvalidRefspec { @@ -1522,6 +1528,18 @@ fn parse_fetch_refspec(raw: &str, remote: &str) -> Result, ) -> Option { + let ordinary_heads: Vec<&DiscRef> = ref_heads + .iter() + .filter(|reference| !fetch_ref_is_local_only(&reference._ref)) + .collect(); // 1. `symref=HEAD:refs/heads/` capability. for cap in capabilities { if let Some(rest) = cap.strip_prefix("symref=HEAD:") && let Some(branch) = rest.strip_prefix("refs/heads/") && !branch.is_empty() + && !fetch_ref_is_local_only(rest) { return Some(branch.to_string()); } } // 2. Match HEAD's advertised OID against a branch tip. if let Some(remote_head) = remote_head - && let Some(branch) = ref_heads + && let Some(branch) = ordinary_heads .iter() .find(|r| r._hash == remote_head._hash) .and_then(|r| r._ref.strip_prefix("refs/heads/")) @@ -3013,14 +3046,18 @@ pub(crate) fn resolve_remote_default_branch( return Some(branch.to_string()); } // 3. Heuristic fallback: main, then master, then the first branch. - if ref_heads.is_empty() { + if ordinary_heads.is_empty() { return None; } - ref_heads + ordinary_heads .iter() .find(|r| r._ref == "refs/heads/main") - .or_else(|| ref_heads.iter().find(|r| r._ref == "refs/heads/master")) - .or(ref_heads.first()) + .or_else(|| { + ordinary_heads + .iter() + .find(|r| r._ref == "refs/heads/master") + }) + .or(ordinary_heads.first()) .and_then(|r| r._ref.strip_prefix("refs/heads/")) .map(str::to_owned) } @@ -3140,6 +3177,25 @@ async fn update_references( crate::internal::db::write_transaction(&db, |txn| { Box::pin(async move { let mut updates = Vec::new(); + for storage_name in [ + "libra/memory/repo".to_string(), + format!( + "refs/remotes/{}/libra/memory/repo", + remote_config.name + ), + ] { + ref_model::Entity::delete_many() + .filter(ref_model::Column::Kind.eq(ref_model::ConfigKind::Branch)) + .filter(ref_model::Column::Remote.eq(&remote_config.name)) + .filter(ref_model::Column::Name.eq(storage_name)) + .exec(txn) + .await + .map_err(|error| FetchError::UpdateRefs { + message: format!( + "failed to remove prohibited Memory tracking ref: {error}" + ), + })?; + } let checked_out_branches = checked_out_local_branches_with_conn(txn).await?; for plan in &plans { let (storage_name, remote_scope) = @@ -3809,6 +3865,63 @@ mod tests { assert_eq!(resolve_remote_default_branch(&[], &[], None), None); } + #[test] + fn fetch_refspec_rejects_exact_memory_and_wildcard_omits_it() { + use super::{DiscRef, expand_refspec, parse_fetch_refspec}; + + assert!(parse_fetch_refspec("libra/memory/repo", "origin").is_err()); + assert!( + parse_fetch_refspec("refs/heads/main:refs/heads/libra/memory/repo", "origin").is_err() + ); + assert!( + parse_fetch_refspec( + "refs/heads/main:refs/remotes/origin/libra/memory/repo", + "origin" + ) + .is_err() + ); + assert!(parse_fetch_refspec("libra/memory/repo-user", "origin").is_ok()); + + let wildcard = parse_fetch_refspec("+refs/heads/*:refs/remotes/origin/*", "origin") + .expect("ordinary wildcard refspec"); + let refs = vec![ + DiscRef { + _hash: "aaa".to_string(), + _ref: "refs/heads/main".to_string(), + }, + DiscRef { + _hash: "bbb".to_string(), + _ref: "refs/heads/libra/memory/repo".to_string(), + }, + ]; + let plans = expand_refspec(&wildcard, &refs, "origin").expect("expand wildcard"); + assert_eq!(plans.len(), 1); + assert_eq!(plans[0].reference._ref, "refs/heads/main"); + } + + #[test] + fn remote_default_branch_ignores_memory_symref_and_oid() { + use super::{DiscRef, resolve_remote_default_branch}; + + let memory = DiscRef { + _hash: "same".to_string(), + _ref: "refs/heads/libra/memory/repo".to_string(), + }; + let main = DiscRef { + _hash: "main".to_string(), + _ref: "refs/heads/main".to_string(), + }; + let head = DiscRef { + _hash: "same".to_string(), + _ref: "HEAD".to_string(), + }; + let capabilities = vec!["symref=HEAD:refs/heads/libra/memory/repo".to_string()]; + assert_eq!( + resolve_remote_default_branch(&capabilities, &[memory, main], Some(&head)).as_deref(), + Some("main") + ); + } + #[test] fn format_fetch_porcelain_layout_is_space_separated() { use super::{FetchOutput, FetchRefUpdate, FetchRepositoryResult, format_fetch_porcelain}; diff --git a/src/command/maintenance.rs b/src/command/maintenance.rs index 1b9c4da44..0fc909799 100644 --- a/src/command/maintenance.rs +++ b/src/command/maintenance.rs @@ -29,8 +29,11 @@ use std::{ use clap::{Parser, Subcommand, ValueEnum}; use git_internal::{ + errors::GitError, hash::{HashKind, ObjectHash, get_hash_kind}, - internal::object::{commit::Commit, tag::Tag as GitTag, tree::Tree, types::ObjectType}, + internal::object::{ + ObjectTrait, commit::Commit, tag::Tag as GitTag, tree::Tree, types::ObjectType, + }, }; use sea_orm::EntityTrait; use serde::Serialize; @@ -40,7 +43,7 @@ use sha1::Digest; use sha2::Digest as _; use crate::{ - command::{fetch::fetch_repository_safe, load_object_raw, log::get_reachable_commits}, + command::{fetch::fetch_repository_safe, log::get_reachable_commits}, internal::{ branch::Branch, config::ConfigKv, @@ -4341,7 +4344,7 @@ fn walk_reachable( match obj_type { ObjectType::Commit => { - let commit = load_object_raw::(hash).map_err(|error| { + let commit = load_reachable_object::(storage, hash).map_err(|error| { CliError::fatal(format!( "reachable commit {hash} is corrupt while computing GC roots: {error}" )) @@ -4358,7 +4361,7 @@ fn walk_reachable( } } ObjectType::Tree => { - let tree = load_object_raw::(hash).map_err(|error| { + let tree = load_reachable_object::(storage, hash).map_err(|error| { CliError::fatal(format!( "reachable tree {hash} is corrupt while computing GC roots: {error}" )) @@ -4369,7 +4372,7 @@ fn walk_reachable( } } ObjectType::Tag => { - let tag = load_object_raw::(hash).map_err(|error| { + let tag = load_reachable_object::(storage, hash).map_err(|error| { CliError::fatal(format!( "reachable tag {hash} is corrupt while computing GC roots: {error}" )) @@ -4389,6 +4392,14 @@ fn walk_reachable( Ok(()) } +fn load_reachable_object( + storage: &ClientStorage, + hash: &ObjectHash, +) -> Result { + let data = storage.get(hash)?; + T::from_bytes(&data.to_vec(), *hash) +} + /// List all loose objects in the repository, returning (hash, path) pairs. pub(crate) fn list_loose_objects(repo_path: &Path) -> io::Result> { let objects_dir = repo_path.join("objects"); @@ -4511,6 +4522,19 @@ fn info_println(output: &OutputConfig, message: &str) { mod tests { use super::*; + #[test] + fn memory_ref_policy_remains_a_gc_root() { + use crate::internal::ai::linear_ref::OwnedRefSpec; + + let spec = OwnedRefSpec::for_storage_name("libra/memory/repo") + .expect("canonical Memory ref classifies"); + assert!(spec.policy().gc_root); + assert_eq!( + OwnedRefSpec::for_storage_name("libra/memory/repo-user"), + None + ); + } + /// The 4 MiB ledger cap is enforced on the way OUT, not only on the way /// in. Checking it only on read lets THIS run write a ledger every later /// run then refuses to read — the quarantine clock stops for a file this diff --git a/src/command/op.rs b/src/command/op.rs index a3fd95093..1b0592e27 100644 --- a/src/command/op.rs +++ b/src/command/op.rs @@ -10,6 +10,7 @@ use serde::Serialize; use crate::{ command::status, internal::{ + ai::linear_ref::OwnedRefSpec, branch::{Branch, is_locked_branch}, config::ConfigKv, db::get_db_conn_instance, @@ -127,6 +128,17 @@ pub enum OpOutput { new_op_id: String, /// Human-readable restore confirmation. message: String, + /// Canonical local-only refs omitted from an old operation snapshot. + skipped_owned_refs: Vec, + }, + #[serde(rename = "restore-preview")] + RestorePreview { + target_op_id: String, + head_kind: String, + head_target: String, + restored_refs: Vec, + pruned_refs: Vec, + skipped_owned_refs: Vec, }, } @@ -338,7 +350,11 @@ fn restore_keep_set(graph: &OperationGraphRecord) -> HashSet { let mut keep: HashSet = graph .refs .iter() - .filter(|r| r.ref_kind == "branch" && r.ref_remote.is_none()) + .filter(|r| { + r.ref_kind == "branch" + && r.ref_remote.is_none() + && !operation_restore_excludes_ref(&r.ref_name) + }) .map(|r| r.ref_name.clone()) .collect(); if graph.view.head_kind == "branch" { @@ -347,6 +363,29 @@ fn restore_keep_set(graph: &OperationGraphRecord) -> HashSet { keep } +fn operation_restore_excludes_ref(name: &str) -> bool { + OwnedRefSpec::for_storage_name(name).is_some_and(|spec| !spec.policy().operation_snapshot) +} + +fn skipped_owned_refs(graph: &OperationGraphRecord) -> Vec { + let mut skipped: Vec = graph + .refs + .iter() + .filter(|record| { + record.ref_kind == "branch" + && record.ref_remote.is_none() + && operation_restore_excludes_ref(&record.ref_name) + }) + .map(|record| record.ref_name.clone()) + .collect(); + if graph.view.head_kind == "branch" && operation_restore_excludes_ref(&graph.view.head_target) { + skipped.push(graph.view.head_target.clone()); + } + skipped.sort(); + skipped.dedup(); + skipped +} + /// List the local branches that an `op restore` would prune for the given /// `keep` set: present now, absent from the target view, and not a Libra-owned /// locked branch (`main`/`intent`/`traces`/`agent-traces`), which are never @@ -395,6 +434,7 @@ async fn handle_op_restore( let target_op_id = resolve_op_ref(&db, &repo_id, &op_ref).await?; let target_graph = load_operation_graph(&db, &target_op_id).await?; let target_op = target_graph.operation.clone(); + let skipped_owned_refs = skipped_owned_refs(&target_graph); // plan-20260714 W0 (§C.11, ADR-0714-08): restoring rewrites THIS // worktree's HEAD, index and working tree from a snapshot. Doing that @@ -509,6 +549,34 @@ async fn handle_op_restore( } if dry_run { + let restored_refs: Vec = target_graph + .refs + .iter() + .filter(|record| { + !(record.ref_kind == "branch" + && record.ref_remote.is_none() + && operation_restore_excludes_ref(&record.ref_name)) + }) + .map(|record| record.ref_name.clone()) + .collect(); + let keep = restore_keep_set(&target_graph); + let pruned = local_branches_to_prune(&db, &keep) + .await + .map_err(|e| CliError::fatal(format!("failed to inspect branches: {e}")))?; + if output.is_json() { + return emit_json_data( + "op", + &OpOutput::RestorePreview { + target_op_id, + head_kind: target_graph.view.head_kind.clone(), + head_target: target_graph.view.head_target.clone(), + restored_refs, + pruned_refs: pruned, + skipped_owned_refs, + }, + output, + ); + } let short_id = &target_op_id[..8.min(target_op_id.len())]; println!( "Would restore to operation {} ({})", @@ -519,17 +587,20 @@ async fn handle_op_restore( target_graph.view.head_target, target_graph.view.head_kind ); println!("Refs that would be restored:"); - for ref_rec in &target_graph.refs { + for ref_rec in target_graph + .refs + .iter() + .filter(|record| restored_refs.contains(&record.ref_name)) + { println!( " {}: {}", ref_rec.ref_name, &ref_rec.target_oid[..7.min(ref_rec.target_oid.len())] ); } - let keep = restore_keep_set(&target_graph); - let pruned = local_branches_to_prune(&db, &keep) - .await - .map_err(|e| CliError::fatal(format!("failed to inspect branches: {e}")))?; + for name in &skipped_owned_refs { + println!(" Skipping Libra-owned local ref: {name}"); + } if pruned.is_empty() { println!("No branches would be pruned."); } else { @@ -554,7 +625,10 @@ async fn handle_op_restore( let mut affected: Vec = target_graph .refs .iter() - .filter(|r| r.ref_kind == "branch") + .filter(|r| { + r.ref_kind == "branch" + && !(r.ref_remote.is_none() && operation_restore_excludes_ref(&r.ref_name)) + }) .map(|r| r.ref_name.clone()) .collect(); affected.extend(pruned); @@ -595,7 +669,13 @@ async fn handle_op_restore( let result = with_operation_log(restore_meta, OperationScope::default(), move |txn| { Box::pin(async move { - let new_head = if restore_graph.view.head_kind == "branch" { + let new_head = if restore_graph.view.head_kind == "branch" + && operation_restore_excludes_ref(&restore_graph.view.head_target) + { + Head::current_result_with_conn(txn) + .await + .map_err(|e| DbErr::Custom(e.to_string()))? + } else if restore_graph.view.head_kind == "branch" { Head::Branch(restore_graph.view.head_target.clone()) } else { Head::Detached( @@ -608,7 +688,10 @@ async fn handle_op_restore( .map_err(|e| DbErr::Custom(e.to_string()))?; for ref_rec in &restore_graph.refs { - if ref_rec.ref_kind == "branch" { + if ref_rec.ref_kind == "branch" + && !(ref_rec.ref_remote.is_none() + && operation_restore_excludes_ref(&ref_rec.ref_name)) + { Branch::update_branch_with_conn( txn, &ref_rec.ref_name, @@ -647,6 +730,7 @@ async fn handle_op_restore( &target_op_id[..8.min(target_op_id.len())], target_op.description ), + skipped_owned_refs: skipped_owned_refs.clone(), }; if output.is_json() { @@ -660,6 +744,9 @@ async fn handle_op_restore( _ => unreachable!(), } ); + for name in &skipped_owned_refs { + println!("Skipped Libra-owned local ref: {name}"); + } println!( "New operation recorded: {}", &result.op_id[..8.min(result.op_id.len())] @@ -797,6 +884,7 @@ mod tests { "libra/intent".to_string(), "libra/src".to_string(), "libra/target".to_string(), + "libra/memory/repo".to_string(), "keep".to_string(), "ephemeral".to_string(), ]; @@ -806,6 +894,13 @@ mod tests { assert_eq!(pruned, vec!["ephemeral".to_string()]); } + #[test] + fn restore_skips_only_exact_memory_ref_from_legacy_views() { + assert!(operation_restore_excludes_ref("libra/memory/repo")); + assert!(!operation_restore_excludes_ref("libra/memory/repo-user")); + assert!(!operation_restore_excludes_ref("feature/memory")); + } + /// A branch in the keep set is never a prune candidate even if it shares a /// name shape with user branches; an empty keep set still protects locked /// branches. diff --git a/src/command/push.rs b/src/command/push.rs index aace829fc..2cc98cd7d 100644 --- a/src/command/push.rs +++ b/src/command/push.rs @@ -34,7 +34,10 @@ use crate::{ git_protocol::{ServiceType::ReceivePack, add_pkt_line_string, read_pkt_line}, info_println, internal::{ - ai::automation::{VCS_EVENT_POST_PUSH, dispatch_current_repo_vcs_event_to_history}, + ai::{ + automation::{VCS_EVENT_POST_PUSH, dispatch_current_repo_vcs_event_to_history}, + linear_ref::{OwnedRefSpec, OwnedRefTransportPolicy}, + }, branch::{Branch, BranchStoreError}, config::ConfigKv, db::get_db_conn_instance, @@ -80,6 +83,7 @@ EXAMPLES: libra push origin :feature Delete the remote feature branch libra push -d origin feature Delete the remote feature branch (short form) libra push --tags origin Push local tags + libra push --all origin Push all ordinary local branches libra push --mirror --dry-run origin Preview a mirror sync without writing libra push -u origin feature-x Push and set upstream tracking @@ -166,6 +170,10 @@ pub struct PushArgs { #[clap(long, requires("repository"))] pub tags: bool, + /// Push all ordinary local branch refs under refs/heads/* + #[clap(long, requires("repository"))] + pub all: bool, + /// Mirror all local refs/heads/* and refs/tags/* to the remote, deleting remote-only refs #[clap(long, requires("repository"))] pub mirror: bool, @@ -203,6 +211,7 @@ impl PushArgs { porcelain: false, dry_run: false, tags: false, + all: false, mirror: false, no_verify: false, no_progress: false, @@ -254,6 +263,9 @@ pub enum PushError { #[error("source ref '{0}' not found")] SourceRefNotFound(String), + #[error("Memory ref '{0}' is local-only and cannot use ordinary push")] + LocalOnlyRef(String), + #[error("pushing to local file repositories is not supported")] UnsupportedLocalFileRemote, @@ -359,6 +371,9 @@ impl From for CliError { PushError::SourceRefNotFound(..) => CliError::fatal(error.to_string()) .with_stable_code(StableErrorCode::CliInvalidTarget) .with_hint("verify the local branch/ref exists before pushing"), + PushError::LocalOnlyRef(..) => CliError::fatal(error.to_string()) + .with_stable_code(StableErrorCode::ConflictOperationBlocked) + .with_hint("Memory publication will use a dedicated command in a later milestone"), PushError::UnsupportedLocalFileRemote => CliError::fatal(error.to_string()) .with_stable_code(StableErrorCode::CliInvalidTarget) .with_hint( @@ -696,6 +711,7 @@ pub async fn execute_safe(args: PushArgs, output: &OutputConfig) -> CliResult<() args.delete, args.set_upstream, args.tags, + args.all, args.mirror, ) .map_err(CliError::from)?; @@ -731,14 +747,16 @@ fn apply_delete_flag( delete: bool, set_upstream: bool, tags: bool, + all: bool, mirror: bool, ) -> Result, PushError> { if !delete { return Ok(refspecs); } - if set_upstream || tags || mirror { + if set_upstream || tags || all || mirror { return Err(PushError::InvalidArguments( - "--delete cannot be combined with --set-upstream, --tags, or --mirror".to_string(), + "--delete cannot be combined with --set-upstream, --tags, --all, or --mirror" + .to_string(), )); } if refspecs.is_empty() { @@ -761,17 +779,25 @@ fn apply_delete_flag( } fn validate_push_args(args: &PushArgs) -> Result<(), PushError> { - if args.repository.is_none() && (!args.refspecs.is_empty() || args.tags || args.mirror) { + if args.repository.is_none() + && (!args.refspecs.is_empty() || args.tags || args.all || args.mirror) + { return Err(PushError::InvalidArguments( - "repository is required when specifying refspecs, --tags, or --mirror".to_string(), + "repository is required when specifying refspecs, --tags, --all, or --mirror" + .to_string(), )); } - if args.repository.is_some() && args.refspecs.is_empty() && !args.tags && !args.mirror { + if args.repository.is_some() + && args.refspecs.is_empty() + && !args.tags + && !args.all + && !args.mirror + { return Err(PushError::InvalidArguments( - "repository-only push requires a refspec, --tags, or --mirror".to_string(), + "repository-only push requires a refspec, --tags, --all, or --mirror".to_string(), )); } - if args.set_upstream && (args.refspecs.len() != 1 || args.tags) { + if args.set_upstream && (args.refspecs.len() != 1 || args.tags || args.all) { return Err(PushError::InvalidArguments( "--set-upstream requires exactly one branch refspec".to_string(), )); @@ -781,6 +807,12 @@ fn validate_push_args(args: &PushArgs) -> Result<(), PushError> { "--mirror cannot be combined with refspecs, --tags, or --set-upstream".to_string(), )); } + if args.all && (!args.refspecs.is_empty() || args.tags || args.mirror || args.set_upstream) { + return Err(PushError::InvalidArguments( + "--all cannot be combined with refspecs, --tags, --mirror, or --set-upstream" + .to_string(), + )); + } Ok(()) } @@ -790,7 +822,7 @@ async fn validate_local_refspecs(args: &PushArgs, current_branch: &str) -> Resul return Ok(()); } - if args.refspecs.is_empty() && !args.tags { + if args.refspecs.is_empty() && !args.tags && !args.all { resolve_local_ref(current_branch).await?; } @@ -916,6 +948,8 @@ pub async fn run_push(args: PushArgs, output: &OutputConfig) -> Result Result { } fn normalize_destination_ref(input: &str, source_kind: LocalRefKind) -> Result { - if input.starts_with("refs/") { - return ensure_valid_ref(input.to_string(), input); - } - match source_kind { - LocalRefKind::Branch => normalize_branch_ref(input), - LocalRefKind::Tag => normalize_tag_ref(input), - } + let normalized = if input.starts_with("refs/") { + ensure_valid_ref(input.to_string(), input)? + } else { + match source_kind { + LocalRefKind::Branch => normalize_branch_ref(input), + LocalRefKind::Tag => normalize_tag_ref(input), + }? + }; + ensure_not_local_only_ref(&normalized)?; + Ok(normalized) } fn normalize_delete_ref(input: &str) -> Result { - if input.starts_with("refs/") { - ensure_valid_ref(input.to_string(), input) + let normalized = if input.starts_with("refs/") { + ensure_valid_ref(input.to_string(), input)? } else { - normalize_branch_ref(input) + normalize_branch_ref(input)? + }; + ensure_not_local_only_ref(&normalized)?; + Ok(normalized) +} + +fn ensure_not_local_only_ref(name: &str) -> Result<(), PushError> { + let spec = OwnedRefSpec::for_transport_ref(name); + if spec.is_some_and(|spec| spec.transport_policy() == OwnedRefTransportPolicy::LocalOnly) { + return Err(PushError::LocalOnlyRef(name.to_string())); } + Ok(()) } async fn resolve_local_ref(input: &str) -> Result { + ensure_not_local_only_ref(input)?; if input.starts_with("refs/heads/") { let short_name = input .strip_prefix("refs/heads/") @@ -1943,6 +1991,9 @@ async fn build_mirror_update_plan( .map_err(|error| PushError::RepoState(error.to_string()))?; for branch in branches { let full_ref = normalize_branch_ref(&branch.name)?; + if ensure_not_local_only_ref(&full_ref).is_err() { + continue; + } local_refs.insert(full_ref.clone()); add_update_ref_plan( ResolvedLocalRef { @@ -1988,6 +2039,9 @@ async fn build_mirror_update_plan( if local_refs.contains(remote_ref) { continue; } + if ensure_not_local_only_ref(remote_ref).is_err() { + continue; + } add_delete_ref_plan( remote_ref.clone(), remote_refs, @@ -2009,6 +2063,38 @@ async fn build_mirror_update_plan( Ok(plans) } +async fn build_all_branch_update_plan( + remote_refs: &HashMap, + force: bool, + warnings: &mut Vec, +) -> Result, PushError> { + let branches = Branch::list_branches_result(None) + .await + .map_err(|error| PushError::RepoState(error.to_string()))?; + let mut plans = Vec::new(); + let mut seen_remote_refs = HashSet::new(); + for branch in branches { + let full_ref = normalize_branch_ref(&branch.name)?; + if ensure_not_local_only_ref(&full_ref).is_err() { + continue; + } + add_update_ref_plan( + ResolvedLocalRef { + full_ref: full_ref.clone(), + oid: branch.commit, + kind: LocalRefKind::Branch, + }, + full_ref, + remote_refs, + force, + warnings, + &mut seen_remote_refs, + &mut plans, + )?; + } + Ok(plans) +} + fn tag_object_hash(object: &tag::TagObject) -> ObjectHash { match object { tag::TagObject::Commit(commit) => commit.id, @@ -2932,8 +3018,11 @@ mod test { tree::{Tree, TreeItem, TreeItemMode}, }, }; + use serial_test::serial; + use tempfile::tempdir; use super::*; + use crate::utils::test::{ChangeDirGuard, setup_with_new_libra_in}; fn save_test_blob(content: &str) -> Blob { let blob = Blob::from_content(content); @@ -2941,6 +3030,46 @@ mod test { blob } + #[tokio::test(flavor = "current_thread")] + #[serial] + async fn all_and_mirror_plans_exclude_memory_ref() { + let repo = tempdir().unwrap(); + setup_with_new_libra_in(repo.path()).await; + let _guard = ChangeDirGuard::new(repo.path()); + let oid = "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"; + Branch::update_branch("main", oid, None).await.unwrap(); + Branch::update_branch("libra/memory/repo", oid, None) + .await + .unwrap(); + Branch::update_branch("libra/memory/repo-user", oid, None) + .await + .unwrap(); + + let mut warnings = Vec::new(); + let all = build_all_branch_update_plan(&HashMap::new(), false, &mut warnings) + .await + .unwrap(); + let destinations: HashSet<&str> = all + .iter() + .map(|plan| plan.update.remote_ref.as_str()) + .collect(); + assert!(destinations.contains("refs/heads/main")); + assert!(destinations.contains("refs/heads/libra/memory/repo-user")); + assert!(!destinations.contains("refs/heads/libra/memory/repo")); + + let remote_refs = HashMap::from([ + ("refs/heads/main".to_string(), oid.to_string()), + ("refs/heads/libra/memory/repo".to_string(), oid.to_string()), + ]); + let mirror = build_mirror_update_plan(&remote_refs, &mut warnings) + .await + .unwrap(); + assert!(mirror.iter().all(|plan| { + plan.update.remote_ref != "refs/heads/libra/memory/repo" + && plan.update.local_ref != "refs/heads/libra/memory/repo" + })); + } + fn save_test_tree(items: Vec) -> Tree { let tree = Tree::from_tree_items(items).expect("test tree should be valid"); crate::command::save_object(&tree, &tree.id).expect("test tree should save"); @@ -3312,6 +3441,10 @@ mod test { PushError::SourceRefNotFound("topic/x".to_string()).to_string(), "source ref 'topic/x' not found", ); + assert_eq!( + PushError::LocalOnlyRef("refs/heads/libra/memory/repo".to_string()).to_string(), + "Memory ref 'refs/heads/libra/memory/repo' is local-only and cannot use ordinary push", + ); assert_eq!( PushError::UnsupportedLocalFileRemote.to_string(), "pushing to local file repositories is not supported", @@ -3501,6 +3634,11 @@ mod test { assert!(args.refspecs.is_empty()); assert!(args.tags); + let args = PushArgs::parse_from(["push", "--all", "origin"]); + assert_eq!(args.repository.as_deref(), Some("origin")); + assert!(args.all); + assert!(args.refspecs.is_empty()); + let args = vec!["push", "--mirror", "--dry-run", "origin"]; let args = PushArgs::parse_from(args); assert_eq!(args.repository, Some("origin".to_string())); @@ -3551,7 +3689,7 @@ mod test { fn apply_delete_flag_rewrites_and_validates() { // Without --delete, the refspecs pass through unchanged. assert_eq!( - apply_delete_flag(vec!["main".to_string()], false, false, false, false).unwrap(), + apply_delete_flag(vec!["main".to_string()], false, false, false, false, false).unwrap(), vec!["main".to_string()] ); // --delete rewrites each plain ref name to a `:` deletion request. @@ -3561,19 +3699,31 @@ mod test { true, false, false, + false, false ) .unwrap(), vec![":main".to_string(), ":feature".to_string()] ); // --delete requires at least one ref. - assert!(apply_delete_flag(vec![], true, false, false, false).is_err()); + assert!(apply_delete_flag(vec![], true, false, false, false, false).is_err()); // --delete rejects a refspec that already carries a ':'. - assert!(apply_delete_flag(vec!["a:b".to_string()], true, false, false, false).is_err()); - // --delete cannot combine with --set-upstream / --tags / --mirror. - assert!(apply_delete_flag(vec!["main".to_string()], true, true, false, false).is_err()); - assert!(apply_delete_flag(vec!["main".to_string()], true, false, true, false).is_err()); - assert!(apply_delete_flag(vec!["main".to_string()], true, false, false, true).is_err()); + assert!( + apply_delete_flag(vec!["a:b".to_string()], true, false, false, false, false).is_err() + ); + // --delete cannot combine with --set-upstream / --tags / --all / --mirror. + assert!( + apply_delete_flag(vec!["main".to_string()], true, true, false, false, false).is_err() + ); + assert!( + apply_delete_flag(vec!["main".to_string()], true, false, true, false, false).is_err() + ); + assert!( + apply_delete_flag(vec!["main".to_string()], true, false, false, true, false).is_err() + ); + assert!( + apply_delete_flag(vec!["main".to_string()], true, false, false, false, true).is_err() + ); } #[test] @@ -3623,6 +3773,7 @@ mod test { porcelain: false, dry_run: false, tags: false, + all: false, mirror: false, no_verify: false, no_progress: false, @@ -3630,7 +3781,7 @@ mod test { assert!(matches!( validate_push_args(&args), Err(PushError::InvalidArguments(message)) - if message == "repository is required when specifying refspecs, --tags, or --mirror" + if message == "repository is required when specifying refspecs, --tags, --all, or --mirror" )); } @@ -3862,6 +4013,19 @@ old1 new1 refs/heads/main\n" assert_eq!(remote_ref, "refs/libra/traces"); } + #[test] + fn ordinary_push_rejects_exact_memory_ref_and_allows_lookalikes() { + for name in ["libra/memory/repo", "refs/heads/libra/memory/repo"] { + assert!(matches!( + ensure_not_local_only_ref(name), + Err(PushError::LocalOnlyRef(blocked)) if blocked == name + )); + } + assert!(ensure_not_local_only_ref("libra/memory/repo-user").is_ok()); + assert!(normalize_destination_ref("libra/memory/repo", LocalRefKind::Branch).is_err()); + assert!(normalize_delete_ref("refs/heads/libra/memory/repo").is_err()); + } + #[test] fn test_normalize_branch_ref_still_rejects_private_refs_source() { assert!(matches!( @@ -3942,6 +4106,14 @@ old1 new1 refs/heads/main\n" assert_eq!(err.exit_code(), 129); } + #[test] + fn local_only_memory_push_has_stable_conflict_error() { + let err: CliError = + PushError::LocalOnlyRef("refs/heads/libra/memory/repo".to_string()).into(); + assert_eq!(err.stable_code(), StableErrorCode::ConflictOperationBlocked); + assert!(!err.hints().is_empty()); + } + #[test] fn test_push_error_to_cli_error_unsupported_local_remote() { let err: CliError = PushError::UnsupportedLocalFileRemote.into(); diff --git a/src/command/symbolic_ref.rs b/src/command/symbolic_ref.rs index 24ab9173d..5feb19b3f 100644 --- a/src/command/symbolic_ref.rs +++ b/src/command/symbolic_ref.rs @@ -7,7 +7,10 @@ use serde::Serialize; use crate::{ command::branch::is_valid_git_branch_name, - internal::{branch::BranchStoreError, head::Head}, + internal::{ + branch::{BranchStoreError, is_ai_managed_branch}, + head::Head, + }, utils::{ error::{CliError, CliResult, StableErrorCode}, output::{OutputConfig, emit_json_data}, @@ -141,6 +144,13 @@ fn validate_name(name: &str) -> CliResult<()> { async fn set_head_target(target: &str) -> CliResult<()> { let branch_name = branch_name_from_full_ref(target)?; + if is_ai_managed_branch(branch_name) { + return Err(CliError::failure(format!( + "cannot point HEAD at Libra-managed branch '{branch_name}'" + )) + .with_stable_code(StableErrorCode::ConflictOperationBlocked) + .with_hint("choose an ordinary working branch")); + } // Part C W0 (§C.11): pointing this worktree's HEAD at a branch already // checked out in ANOTHER worktree would create a forbidden duplicate // checkout. `branch_checked_out_elsewhere` excludes the current worktree, so diff --git a/src/command/update_ref.rs b/src/command/update_ref.rs index 959c7b010..9c742d771 100644 --- a/src/command/update_ref.rs +++ b/src/command/update_ref.rs @@ -20,7 +20,7 @@ use serde::Serialize; use crate::{ internal::{ - branch::Branch, + branch::{Branch, is_ai_managed_branch}, db::get_db_conn_instance, reflog::{Reflog, ReflogAction, ReflogContext}, }, @@ -121,6 +121,15 @@ pub async fn execute_safe(args: UpdateRefArgs, output: &OutputConfig) -> CliResu if !util::is_valid_refname(&args.ref_name) { return Err(fatal(format!("invalid ref name '{}'", args.ref_name))); } + if is_ai_managed_branch(branch) { + return Err(CliError::fatal(format!( + "cannot update '{}': branch '{branch}' is managed by Libra", + args.ref_name + )) + .with_exit_code(128) + .with_stable_code(StableErrorCode::ConflictOperationBlocked) + .with_hint("use the dedicated Libra command that owns this internal ref")); + } // Part C W0 (§C.11): refuse to move or delete a branch checked out in // ANOTHER worktree — its HEAD would be left dangling or its working tree // would silently diverge. `branch_checked_out_elsewhere` excludes the diff --git a/src/internal/ai/linear_ref.rs b/src/internal/ai/linear_ref.rs index a23b97a0c..aeeda3146 100644 --- a/src/internal/ai/linear_ref.rs +++ b/src/internal/ai/linear_ref.rs @@ -36,6 +36,19 @@ pub(crate) enum OwnedRefTransportPolicy { LocalOnly, } +/// Behaviour attached to one canonical Libra-owned ref. +/// +/// This record is deliberately value-only: command adapters can enforce the +/// same decision without querying SQLite or duplicating name checks. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct OwnedRefPolicy { + pub(crate) visible_to_branch: bool, + pub(crate) mutable_by_user: bool, + pub(crate) operation_snapshot: bool, + pub(crate) gc_root: bool, + pub(crate) transport: OwnedRefTransportPolicy, +} + /// Closed set of refs whose mutation policy is owned by Libra. /// /// Callers select a variant instead of supplying a name, so a user-controlled @@ -86,19 +99,77 @@ impl OwnedRefSpec { } } - /// Resolve the exact storage names accepted by `HistoryManager`. - /// - /// Full Memory ref classification belongs to M2-05. This conversion is - /// intentionally narrower: it only admits the two histories that already - /// use `HistoryManager` today. - pub(crate) fn for_history_storage_name(name: &str) -> Option { + pub(crate) const fn policy(self) -> OwnedRefPolicy { + match self { + Self::MemoryRepo => OwnedRefPolicy { + visible_to_branch: false, + mutable_by_user: false, + operation_snapshot: false, + gc_root: true, + transport: OwnedRefTransportPolicy::LocalOnly, + }, + Self::AiHistory => OwnedRefPolicy { + visible_to_branch: true, + mutable_by_user: false, + operation_snapshot: true, + gc_root: true, + transport: OwnedRefTransportPolicy::Ordinary, + }, + Self::Traces | Self::LegacyTraces => OwnedRefPolicy { + visible_to_branch: true, + mutable_by_user: false, + operation_snapshot: true, + gc_root: true, + transport: OwnedRefTransportPolicy::DedicatedOnly, + }, + } + } + + /// Classify an exact name as stored in the local `reference` table. + pub(crate) fn for_storage_name(name: &str) -> Option { match name { AI_REF => Some(Self::AiHistory), TRACES_BRANCH => Some(Self::Traces), LEGACY_TRACES_BRANCH => Some(Self::LegacyTraces), + "libra/memory/repo" => Some(Self::MemoryRepo), + _ => None, + } + } + + /// Classify an exact fully-qualified ref name. + pub(crate) fn for_full_ref(name: &str) -> Option { + match name { + "refs/heads/libra/intent" => Some(Self::AiHistory), + "refs/libra/traces" => Some(Self::Traces), + "refs/libra/agent-traces" => Some(Self::LegacyTraces), + "refs/heads/libra/memory/repo" => Some(Self::MemoryRepo), _ => None, } } + + /// Classify a transport-visible ref name without broad prefix matching. + /// + /// Fetch stores remote-tracking rows in fully-qualified form, so transport + /// boundaries must also recognize the exact branch suffix after the remote + /// component. Lookalike branches remain ordinary. + pub(crate) fn for_transport_ref(name: &str) -> Option { + Self::for_storage_name(name) + .or_else(|| Self::for_full_ref(name)) + .or_else(|| { + name.strip_prefix("refs/remotes/") + .and_then(|rest| rest.split_once('/')) + .and_then(|(_, branch)| Self::for_storage_name(branch)) + }) + } + + /// Resolve the exact storage names accepted by `HistoryManager`. + /// + /// Full Memory ref classification belongs to M2-05. This conversion is + /// intentionally narrower: it only admits the two histories that already + /// use `HistoryManager` today. + pub(crate) fn for_history_storage_name(name: &str) -> Option { + Self::for_storage_name(name).filter(|spec| !matches!(spec, Self::MemoryRepo)) + } } /// Result of one conditional ref transaction. @@ -398,6 +469,45 @@ mod tests { ); } + #[test] + fn owned_ref_policy_classifies_only_exact_memory_names() { + for name in ["libra/memory/repo", "refs/heads/libra/memory/repo"] { + let spec = OwnedRefSpec::for_storage_name(name) + .or_else(|| OwnedRefSpec::for_full_ref(name)) + .expect("canonical Memory ref must classify"); + assert_eq!(spec, OwnedRefSpec::MemoryRepo); + assert_eq!( + spec.policy(), + OwnedRefPolicy { + visible_to_branch: false, + mutable_by_user: false, + operation_snapshot: false, + gc_root: true, + transport: OwnedRefTransportPolicy::LocalOnly, + } + ); + } + + for lookalike in [ + "libra/memory/repo-user", + "libra/memory/repo/child", + "refs/heads/libra/memory/repo-user", + "refs/heads/libra/memory/repo/child", + "refs/remotes/origin/libra/memory/repo", + ] { + assert_eq!(OwnedRefSpec::for_storage_name(lookalike), None); + assert_eq!(OwnedRefSpec::for_full_ref(lookalike), None); + } + assert_eq!( + OwnedRefSpec::for_transport_ref("refs/remotes/origin/libra/memory/repo"), + Some(OwnedRefSpec::MemoryRepo) + ); + assert_eq!( + OwnedRefSpec::for_transport_ref("refs/remotes/origin/libra/memory/repo-user"), + None + ); + } + #[tokio::test] async fn linear_ref_transaction_commits_ref_and_companion_atomically() { let database = test_database().await; diff --git a/src/internal/ai/memory/writer.rs b/src/internal/ai/memory/writer.rs index 1ab667c06..4408da13d 100644 --- a/src/internal/ai/memory/writer.rs +++ b/src/internal/ai/memory/writer.rs @@ -502,27 +502,33 @@ mod tests { use chrono::{TimeZone, Utc}; use sea_orm::{ConnectionTrait, Database, DatabaseConnection, Statement}; + use serial_test::serial; use super::*; - use crate::internal::{ - ai::{ - context_budget::MemoryAnchorConfidence, - keyed_digest::RepositoryKeyedDigest, - memory::{ - domain::{ - ActorKind, ActorRefV1, CodeChangeStatus, CompileOriginV1, CompileRecordV1, - CompletionStatus, EpisodeClaimV1, EpisodeCodeContextV1, EpisodeOmissionsV1, - EpisodePayloadV1, EpisodeRoot, EpisodeRootKind, EpistemicStatus, EvidenceKind, - EvidenceLocatorV1, EvidenceRefV1, EvidenceSourcePlane, EvidenceVisibility, - IdempotencyScopeV1, MemoryKind, MemoryLifecycle, MemoryNoteV1, MemoryScopeV1, - MemorySensitivity, MemoryTrust, MemoryVisibility, - }, - policy::{ - AuthenticatedMemoryContext, DeterministicMemoryProposal, TrustedMemoryTarget, + use crate::{ + internal::{ + ai::{ + context_budget::MemoryAnchorConfidence, + keyed_digest::RepositoryKeyedDigest, + memory::{ + domain::{ + ActorKind, ActorRefV1, CodeChangeStatus, CompileOriginV1, CompileRecordV1, + CompletionStatus, EpisodeClaimV1, EpisodeCodeContextV1, EpisodeOmissionsV1, + EpisodePayloadV1, EpisodeRoot, EpisodeRootKind, EpistemicStatus, + EvidenceKind, EvidenceLocatorV1, EvidenceRefV1, EvidenceSourcePlane, + EvidenceVisibility, IdempotencyScopeV1, MemoryKind, MemoryLifecycle, + MemoryNoteV1, MemoryScopeV1, MemorySensitivity, MemoryTrust, + MemoryVisibility, + }, + policy::{ + AuthenticatedMemoryContext, DeterministicMemoryProposal, + TrustedMemoryTarget, + }, }, }, + db::migration::run_builtin_migrations, }, - db::migration::run_builtin_migrations, + utils::{client_storage::ClientStorage, test::ChangeDirGuard}, }; const REPOSITORY_ID: &str = "memory-writer-test-repository"; @@ -989,6 +995,67 @@ mod tests { assert_eq!(error.stable_code(), "LBR-MEMORY-001"); } + #[tokio::test(flavor = "current_thread")] + #[serial] + async fn memory_authority_is_a_reachable_gc_root() { + let temp = tempfile::tempdir().expect("create repository"); + let storage_path = temp.path().join(".libra"); + fs::create_dir_all(storage_path.join("objects")).expect("create object store"); + let database_path = storage_path.join(DATABASE); + let database = db::create_database(&database_path.to_string_lossy()) + .await + .expect("create repository database"); + database + .execute_raw(Statement::from_sql_and_values( + database.get_database_backend(), + "INSERT INTO config_kv(key, value, encrypted) VALUES + ('libra.repoid', ?, 0), ('vault.unsealkey', ?, 0)", + [REPOSITORY_ID.into(), hex::encode([0x24_u8; 32]).into()], + )) + .await + .expect("seed repository identity and vault key"); + database.close().await.expect("close setup connection"); + + let writer = MemoryWriter::open(storage_path.clone()) + .await + .expect("open Memory writer"); + let context = AuthenticatedMemoryContext::new( + REPOSITORY_ID, + ActorRefV1 { + kind: ActorKind::Agent, + principal_id: "agent:episode-compiler".to_string(), + }, + ) + .expect("construct context"); + let target = TrustedMemoryTarget::episode( + EpisodeRoot::task("task-gc-root").expect("construct target"), + ); + let committed = writer + .commit( + &context, + &target, + &proposal(&target, writer.digest_provider.key_id(), 1), + None, + ) + .await + .expect("commit Memory authority"); + + let _cwd = ChangeDirGuard::new(temp.path()); + let storage = ClientStorage::init_local_existing(storage_path.join("objects")); + let reachable = crate::command::maintenance::collect_reachable_objects_with_conn( + &storage, + writer.database.as_ref(), + ) + .await + .expect("collect GC roots"); + assert!(reachable.contains(&committed.commit_oid())); + assert!(reachable.contains(&committed.revision_oid())); + assert!( + reachable.len() >= 6, + "Memory commit closure must include commit, tree, event and note objects" + ); + } + #[tokio::test] async fn writer_concurrent_first_create() { let fixture = fixture().await; diff --git a/src/internal/branch.rs b/src/internal/branch.rs index 383f50da8..94e9b8730 100644 --- a/src/internal/branch.rs +++ b/src/internal/branch.rs @@ -27,7 +27,7 @@ use sea_orm::{ }; use tokio::time::sleep; -use crate::internal::{db::get_db_conn_instance, model::reference}; +use crate::internal::{ai::linear_ref::OwnedRefSpec, db::get_db_conn_instance, model::reference}; /// The default trunk branch. Created on `libra init` and treated as a locked /// branch (cannot be deleted while it is HEAD). @@ -52,16 +52,13 @@ pub const LEGACY_TRACES_BRANCH: &str = "agent-traces"; /// Return `true` for branches that the CLI refuses to delete, rename, or /// otherwise route user-facing destructive ops at. /// -/// Functional scope: covers [`DEFAULT_BRANCH`], [`INTENT_BRANCH`], -/// [`TRACES_BRANCH`], and its legacy alias [`LEGACY_TRACES_BRANCH`]. The check -/// is purely syntactic — it does not consult the storage layer. Callers that -/// need a richer policy (e.g. branch protection rules) must layer additional -/// checks on top. +/// Functional scope: covers [`DEFAULT_BRANCH`] plus exact Libra-owned branch +/// names whose policy forbids ordinary user mutation. The check is purely +/// syntactic and does not consult the storage layer. pub fn is_locked_branch(name: &str) -> bool { name == DEFAULT_BRANCH - || name == INTENT_BRANCH - || name == TRACES_BRANCH - || name == LEGACY_TRACES_BRANCH + || name == "refs/heads/main" + || owned_ref_spec(name).is_some_and(|spec| !spec.policy().mutable_by_user) } /// Return `true` for Libra-owned AI branches whose checked-out worktree must @@ -71,7 +68,11 @@ pub fn is_locked_branch(name: &str) -> bool { /// branch-management operations such as delete/rename, but it remains the /// normal user worktree branch. pub fn is_ai_managed_branch(name: &str) -> bool { - name == INTENT_BRANCH || name == TRACES_BRANCH || name == LEGACY_TRACES_BRANCH + owned_ref_spec(name).is_some_and(|spec| !spec.policy().mutable_by_user) +} + +fn owned_ref_spec(name: &str) -> Option { + OwnedRefSpec::for_storage_name(name).or_else(|| OwnedRefSpec::for_full_ref(name)) } /// Return `true` if the user-supplied revision string targets a locked @@ -86,6 +87,7 @@ pub fn is_ai_managed_branch(name: &str) -> bool { /// the bare ref name. pub fn is_locked_revision(rev: &str) -> bool { let head = rev.split(['~', '^', '@']).next().unwrap_or(rev); + let head = head.strip_prefix("refs/heads/").unwrap_or(head); is_locked_branch(head) } @@ -1167,6 +1169,9 @@ mod tests { assert!(is_locked_branch(INTENT_BRANCH)); assert!(is_locked_branch(TRACES_BRANCH)); assert!(is_locked_branch(LEGACY_TRACES_BRANCH)); + assert!(is_locked_branch("libra/memory/repo")); + assert!(is_locked_branch("refs/heads/libra/memory/repo")); + assert!(!is_locked_branch("libra/memory/repo-user")); assert!(!is_locked_branch("traces-feature")); assert!(!is_locked_branch("agent-traces-feature")); assert!(!is_locked_branch("not-locked")); @@ -1179,6 +1184,9 @@ mod tests { assert!(is_ai_managed_branch(INTENT_BRANCH)); assert!(is_ai_managed_branch(TRACES_BRANCH)); assert!(is_ai_managed_branch(LEGACY_TRACES_BRANCH)); + assert!(is_ai_managed_branch("libra/memory/repo")); + assert!(is_ai_managed_branch("refs/heads/libra/memory/repo")); + assert!(!is_ai_managed_branch("libra/memory/repo-user")); assert!(!is_ai_managed_branch("traces-feature")); assert!(!is_ai_managed_branch("agent-traces-feature")); assert!(!is_ai_managed_branch("")); @@ -1195,6 +1203,10 @@ mod tests { assert!(is_locked_revision("agent-traces")); assert!(is_locked_revision("intent")); assert!(is_locked_revision(DEFAULT_BRANCH)); + assert!(is_locked_revision("libra/memory/repo")); + assert!(is_locked_revision("libra/memory/repo~1")); + assert!(is_locked_revision("refs/heads/libra/memory/repo")); + assert!(is_locked_revision("refs/heads/libra/memory/repo^")); // Single-suffix variants. assert!(is_locked_revision("traces~1")); @@ -1224,6 +1236,7 @@ mod tests { assert!(!is_locked_revision("agent-traces-feature")); assert!(!is_locked_revision("agent-traces-feature^")); assert!(!is_locked_revision("not-locked@{0}")); + assert!(!is_locked_revision("libra/memory/repo-user")); assert!(!is_locked_revision("")); } } diff --git a/src/internal/operation_wrapper.rs b/src/internal/operation_wrapper.rs index 26253e26a..a94fbfc2a 100644 --- a/src/internal/operation_wrapper.rs +++ b/src/internal/operation_wrapper.rs @@ -22,6 +22,7 @@ use tokio::time::sleep; use uuid::Uuid; use crate::internal::{ + ai::linear_ref::OwnedRefSpec, branch::Branch, head::Head, model::reference, @@ -1465,6 +1466,9 @@ async fn collect_final_view_with_conn( .await .map_err(|err| DbErr::Custom(format!("failed to list local branches: {err}")))?; for branch in local_branches { + if !operation_snapshot_includes_branch(&branch.name) { + continue; + } records.push(OperationViewRefRecord { view_id: view_id.to_string(), ref_kind: "branch".to_string(), @@ -1522,6 +1526,10 @@ async fn collect_final_view_with_conn( }) } +fn operation_snapshot_includes_branch(name: &str) -> bool { + OwnedRefSpec::for_storage_name(name).is_none_or(|spec| spec.policy().operation_snapshot) +} + /* #[cfg(test)] mod tests { @@ -2130,3 +2138,15 @@ mod tests { } } */ + +#[cfg(test)] +mod owned_ref_policy_tests { + use super::operation_snapshot_includes_branch; + + #[test] + fn operation_snapshot_omits_only_exact_memory_ref() { + assert!(!operation_snapshot_includes_branch("libra/memory/repo")); + assert!(operation_snapshot_includes_branch("libra/memory/repo-user")); + assert!(operation_snapshot_includes_branch("feature/memory")); + } +} diff --git a/src/internal/protocol/local_client.rs b/src/internal/protocol/local_client.rs index d796f9a80..f79b39e51 100644 --- a/src/internal/protocol/local_client.rs +++ b/src/internal/protocol/local_client.rs @@ -36,8 +36,13 @@ use crate::{ command::{load_object, log::get_reachable_commits}, git_protocol::ServiceType, internal::{ - branch::Branch, config::ConfigKv, db::get_db_conn_instance_for_path, head::Head, - protocol::DiscRef, reflog, tag, + ai::linear_ref::{OwnedRefSpec, OwnedRefTransportPolicy}, + branch::Branch, + config::ConfigKv, + db::get_db_conn_instance_for_path, + head::Head, + protocol::DiscRef, + reflog, tag, }, utils::{ client_storage::ClientStorage, @@ -419,7 +424,10 @@ impl LocalClient { let local_branches = Branch::list_branches_result(None) .await - .map_err(|error| GitError::CustomError(error.to_string()))?; + .map_err(|error| GitError::CustomError(error.to_string()))? + .into_iter() + .filter(|branch| ordinary_transport_branch(&branch.name)) + .collect::>(); let remote_configs = ConfigKv::all_remote_configs() .await @@ -429,7 +437,9 @@ impl LocalClient { remote_branches.extend( Branch::list_branches_result(Some(&remote.name)) .await - .map_err(|error| GitError::CustomError(error.to_string()))?, + .map_err(|error| GitError::CustomError(error.to_string()))? + .into_iter() + .filter(|branch| ordinary_transport_branch(&branch.name)), ); } let head_commit = Head::current_commit_result() @@ -619,6 +629,11 @@ impl LocalClient { } } +fn ordinary_transport_branch(name: &str) -> bool { + OwnedRefSpec::for_transport_ref(name) + .is_none_or(|spec| spec.transport_policy() != OwnedRefTransportPolicy::LocalOnly) +} + /// Read `objectformat` from a foreign Git repository's `config`, defaulting to /// SHA-1 (the overwhelmingly common case for local Git remotes). fn git_repo_hash_kind(repo_path: &Path) -> HashKind { @@ -1411,6 +1426,32 @@ mod tests { ); } + #[tokio::test(flavor = "current_thread")] + #[serial] + async fn libra_discovery_hides_exact_local_only_memory_ref() { + let repo_dir = tempdir().unwrap(); + setup_with_new_libra_in(repo_dir.path()).await; + let _guard = ChangeDirGuard::new(repo_dir.path()); + let oid = "e69de29bb2d1d6434b8b29ae775ad8c2e48c5391"; + Branch::update_branch("main", oid, None).await.unwrap(); + Branch::update_branch("libra/memory/repo", oid, None) + .await + .unwrap(); + Branch::update_branch("libra/memory/repo-user", oid, None) + .await + .unwrap(); + + let client = LocalClient::from_path(repo_dir.path()).unwrap(); + let result = client + .discovery_reference(ServiceType::UploadPack) + .await + .unwrap(); + let names: HashSet<&str> = result.refs.iter().map(|item| item._ref.as_str()).collect(); + assert!(names.contains("refs/heads/main")); + assert!(names.contains("refs/heads/libra/memory/repo-user")); + assert!(!names.contains("refs/heads/libra/memory/repo")); + } + #[tokio::test(flavor = "current_thread")] #[serial] async fn with_repo_current_dir_restores_current_dir_when_task_is_cancelled() { diff --git a/tests/command/branch_test.rs b/tests/command/branch_test.rs index 320966238..c9d10d005 100644 --- a/tests/command/branch_test.rs +++ b/tests/command/branch_test.rs @@ -35,6 +35,64 @@ use tempfile::tempdir; use super::*; +#[tokio::test] +#[serial] +async fn memory_branch_is_hidden_and_locked_across_user_mutation_commands() { + let repo = create_committed_repo_via_cli(); + let _guard = ChangeDirGuard::new(repo.path()); + let oid = Head::current_commit() + .await + .expect("current commit") + .to_string(); + Branch::update_branch("libra/memory/repo", &oid, None) + .await + .expect("seed Memory ref"); + Branch::update_branch("libra/memory/repo-user", &oid, None) + .await + .expect("seed lookalike user ref"); + + let listing = run_libra_command(&["branch"], repo.path()); + assert_cli_success(&listing, "branch list"); + let listing_text = String::from_utf8_lossy(&listing.stdout); + let lines: Vec<&str> = listing_text + .lines() + .map(|line| line.trim_start_matches(['*', ' '])) + .collect(); + assert!(lines.contains(&"libra/memory/repo-user")); + assert!(!lines.contains(&"libra/memory/repo")); + + for (label, args) in [ + ("delete", vec!["branch", "-D", "libra/memory/repo"]), + ( + "rename", + vec!["branch", "-m", "libra/memory/repo", "renamed"], + ), + ("switch", vec!["switch", "libra/memory/repo"]), + ("checkout", vec!["checkout", "libra/memory/repo"]), + ("reset", vec!["reset", "libra/memory/repo"]), + ( + "update-ref", + vec!["update-ref", "refs/heads/libra/memory/repo", oid.as_str()], + ), + ( + "symbolic-ref", + vec!["symbolic-ref", "HEAD", "refs/heads/libra/memory/repo"], + ), + ] { + let output = run_libra_command(&args, repo.path()); + assert!( + !output.status.success(), + "{label} must reject the Memory ref" + ); + } + assert!( + Branch::find_branch_result("libra/memory/repo", None) + .await + .expect("query Memory ref") + .is_some() + ); +} + /// Scenario: `libra branch ` must reject the invalid start /// point with exit 129 and a structured `LBR-CLI-003` error. Pins the CLI /// usage error envelope. diff --git a/tests/command/op_test.rs b/tests/command/op_test.rs index 4f44a5906..4c90b0916 100644 --- a/tests/command/op_test.rs +++ b/tests/command/op_test.rs @@ -5,9 +5,15 @@ use std::path::Path; use libra::{ - internal::{branch::Branch, head::Head}, + internal::{ + branch::Branch, + db::get_db_conn_instance, + head::Head, + operation::{OperationService, OperationViewRefRecord}, + }, utils::test::ChangeDirGuard, }; +use sea_orm::{ConnectionTrait, DbBackend, Statement}; use serde_json::Value; use super::*; @@ -816,3 +822,78 @@ async fn test_op_restore_json_records_new_operation_and_restores_head_and_branch "op restore" ); } + +#[tokio::test] +#[serial] +/// Legacy operation views may contain Memory, but restore must preserve its authority and watermark. +async fn op_restore_preserves_memory_ref_and_projection_watermark() { + let repo = create_committed_repo_via_cli(); + let branch_output = run_libra_command(&["branch", "feature"], repo.path()); + assert_cli_success(&branch_output, "branch feature"); + let target_op_id = latest_operation_id(repo.path()); + + let _guard = ChangeDirGuard::new(repo.path()); + let current_oid = Head::current_commit() + .await + .expect("current commit") + .to_string(); + Branch::update_branch("libra/memory/repo", ¤t_oid, None) + .await + .expect("seed Memory authority"); + + let db = get_db_conn_instance().await; + let graph = OperationService::load_restore_view_by_operation_with_conn(&db, &target_op_id) + .await + .expect("load target operation") + .expect("target operation graph"); + let mut refs = graph.refs.clone(); + refs.push(OperationViewRefRecord { + view_id: graph.view.view_id.clone(), + ref_kind: "branch".to_string(), + ref_name: "libra/memory/repo".to_string(), + ref_remote: None, + target_oid: "2222222222222222222222222222222222222222".to_string(), + }); + OperationService::replace_view_refs_with_conn(&db, &graph.view.view_id, &refs) + .await + .expect("seed legacy operation view"); + db.execute_raw(Statement::from_sql_and_values( + DbBackend::Sqlite, + "INSERT INTO memory_projection_state(scope_key, projected_ref_oid, last_event_seq, schema_version, policy_version, rebuilt_at) VALUES (?, ?, 42, 1, 'repo-default-v1', 1)", + ["test-scope".into(), current_oid.clone().into()], + )) + .await + .expect("seed projection watermark"); + + let preview = run_json_op(repo.path(), &["restore", &target_op_id, "--dry-run"]); + assert_eq!(preview["data"]["action"], "restore-preview"); + assert_eq!( + preview["data"]["skipped_owned_refs"], + serde_json::json!(["libra/memory/repo"]) + ); + + let restored = run_json_op(repo.path(), &["restore", &target_op_id]); + assert_eq!( + restored["data"]["skipped_owned_refs"], + serde_json::json!(["libra/memory/repo"]) + ); + let memory = Branch::find_branch_result("libra/memory/repo", None) + .await + .expect("query Memory authority") + .expect("Memory authority remains"); + assert_eq!(memory.commit.to_string(), current_oid); + + let row = db + .query_one_raw(Statement::from_string( + DbBackend::Sqlite, + "SELECT projected_ref_oid, last_event_seq FROM memory_projection_state WHERE scope_key = 'test-scope'" + .to_string(), + )) + .await + .expect("query watermark") + .expect("watermark row"); + let projected: String = row.try_get("", "projected_ref_oid").unwrap(); + let sequence: i64 = row.try_get("", "last_event_seq").unwrap(); + assert_eq!(projected, current_oid); + assert_eq!(sequence, 42); +} diff --git a/tests/operation_wrapper_test.rs b/tests/operation_wrapper_test.rs index 391f348ea..84f033e2e 100644 --- a/tests/operation_wrapper_test.rs +++ b/tests/operation_wrapper_test.rs @@ -213,6 +213,42 @@ async fn success_path_exposes_parent_selection_metrics() { assert_eq!(result.parent_metrics.selected_parent_count, 1); } +#[tokio::test] +/// Memory authority is local-only state and must not be captured by an operation view. +async fn operation_snapshot_omits_memory_ref_but_keeps_lookalike_user_branch() { + let db = Database::connect("sqlite::memory:").await.unwrap(); + create_operation_schema(&db).await; + create_reference_table_with_head(&db).await; + db.execute_raw(Statement::from_string( + DbBackend::Sqlite, + "INSERT INTO reference(name, kind, \"commit\", remote) VALUES + ('libra/memory/repo', 'Branch', '2222222222222222222222222222222222222222', NULL), + ('libra/memory/repo-user', 'Branch', '3333333333333333333333333333333333333333', NULL)" + .to_string(), + )) + .await + .unwrap(); + + let result = + with_operation_log_with_conn(&db, valid_meta(), OperationScope::default(), |_txn| { + Box::pin(async move { Ok::<_, DbErr>(()) }) + }) + .await + .unwrap(); + let graph = OperationService::load_restore_view_by_operation_with_conn(&db, &result.op_id) + .await + .unwrap() + .expect("operation graph"); + let names: HashSet<&str> = graph + .refs + .iter() + .map(|record| record.ref_name.as_str()) + .collect(); + assert!(names.contains("main")); + assert!(names.contains("libra/memory/repo-user")); + assert!(!names.contains("libra/memory/repo")); +} + #[tokio::test] /// Verifies that invalid parent-policy combinations are rejected before execution. async fn invalid_parent_policy_is_rejected() { From 79d8039d02f6ff705c46bdfa8bbf1c91bb4239b6 Mon Sep 17 00:00:00 2001 From: anduin9527 Date: Tue, 25 Aug 2026 02:14:26 +0800 Subject: [PATCH 09/18] feat(memory): add projection replay and rebuild --- docs/error-codes.md | 2 + src/internal/ai/history.rs | 6 +- src/internal/ai/linear_ref.rs | 30 +- src/internal/ai/memory/error.rs | 2 + src/internal/ai/memory/fts_sql.rs | 17 +- src/internal/ai/memory/mod.rs | 2 + src/internal/ai/memory/projection.rs | 1261 ++++++++++++++++++++++++++ src/internal/ai/memory/replay.rs | 680 ++++++++++++++ src/internal/ai/memory/store.rs | 159 ++-- src/internal/ai/memory/tree.rs | 249 +++-- src/internal/ai/memory/writer.rs | 22 +- 11 files changed, 2236 insertions(+), 194 deletions(-) create mode 100644 src/internal/ai/memory/projection.rs create mode 100644 src/internal/ai/memory/replay.rs diff --git a/docs/error-codes.md b/docs/error-codes.md index 30eb87de5..4b1fbbf88 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -93,6 +93,7 @@ structured report is always present. | `128` | `LBR-MEMORY-003` | `repo` | Memory writer policy rejected the proposal | authenticated target mismatch, non-local Memory scope, or unknown repository digest key ID | | `128` | `LBR-MEMORY-004` | `repo` | Memory authority or its rebuildable projection is corrupt | invalid manifest, merge commit on the linear Memory ref, broken revision ancestry, or projection watermark mismatch | | `128` | `LBR-MEMORY-005` | `repo` | Memory writer could not commit an atomic revision | local object write failure, SQLite companion failure, or exhausted bounded ref-conflict retries | +| `128` | `LBR-MEMORY-PROJECTION-STALE` | `repo` | Memory projection does not match the pinned repository Memory ref | the ref advanced after a frozen read, the projection is missing, or another replay won the transaction | | `128` | `LBR-WORKTREE-001` | `repo` | Pagination cursor is malformed, foreign, or expired | `libra worktree doctor --cursor ` | | `128` | `LBR-WORKTREE-002` | `repo` | A worktree/workspace scope is corrupt or unreadable, so the diagnosis would be incomplete | `libra worktree doctor` where a `workspace_record` row or the worktree registry cannot be read | | `128` | `LBR-CONFIG-001` | `config` | Global config DB schema is newer than this Libra binary supports | `pull`, `push`, `fetch`, `clone`, or `cloud` would otherwise silently ignore global storage config | @@ -173,6 +174,7 @@ structured report is always present. | `LBR-MEMORY-003` | The Memory proposal failed repository policy; use the authenticated target and the current repository digest key | | `LBR-MEMORY-004` | Memory authority and projection disagree or contain invalid history; stop writes and rebuild or repair the projection before retrying | | `LBR-MEMORY-005` | The Memory revision could not be committed atomically; fix local storage/SQLite health or retry after ref contention subsides | +| `LBR-MEMORY-PROJECTION-STALE` | The Memory projection is not current for the pinned ref; freeze a new view or rebuild/advance the projection before reading it | | `LBR-WORKTREE-001` | The pagination cursor is malformed or expired; drop it and re-read the first page | | `LBR-WORKTREE-002` | A worktree/workspace scope is corrupt or unreadable; repair it before trusting any diagnostic report | diff --git a/src/internal/ai/history.rs b/src/internal/ai/history.rs index 5077573ec..e8ab07caf 100644 --- a/src/internal/ai/history.rs +++ b/src/internal/ai/history.rs @@ -4605,7 +4605,11 @@ struct HistoryLinearRefCompanion<'a> { #[async_trait::async_trait] impl LinearRefCompanion for HistoryLinearRefCompanion<'_> { - async fn apply(&self, txn: &DatabaseTransaction) -> Result<()> { + async fn apply( + &self, + txn: &crate::internal::ai::linear_ref::LinearRefWriteTransaction<'_>, + ) -> Result<()> { + let txn = txn.as_database_transaction(); // An expired ordinary marker may have been fenced and retired while // this writer was stalled. Revalidation remains inside the winning // ref transaction. diff --git a/src/internal/ai/linear_ref.rs b/src/internal/ai/linear_ref.rs index aeeda3146..fbb41380e 100644 --- a/src/internal/ai/linear_ref.rs +++ b/src/internal/ai/linear_ref.rs @@ -183,10 +183,25 @@ pub(crate) enum LinearRefTransactionOutcome { #[error("linear ref transaction exceeded its execution deadline")] pub(crate) struct LinearRefDeadlineExceeded; +/// Borrowed proof that the enclosing owned-ref transaction acquired SQLite's +/// write lock before any companion reads or writes. +/// +/// The constructor stays inside this module so companion implementations +/// cannot certify an arbitrary deferred transaction as write-locked. +pub(crate) struct LinearRefWriteTransaction<'a> { + transaction: &'a DatabaseTransaction, +} + +impl<'a> LinearRefWriteTransaction<'a> { + pub(crate) const fn as_database_transaction(&self) -> &'a DatabaseTransaction { + self.transaction + } +} + /// Companion mutation applied after the ref CAS succeeds and before commit. #[async_trait::async_trait] pub(crate) trait LinearRefCompanion: Send + Sync { - async fn apply(&self, txn: &DatabaseTransaction) -> Result<()>; + async fn apply(&self, txn: &LinearRefWriteTransaction<'_>) -> Result<()>; } /// Advance an owned ref and apply `companion` in the same SQLite transaction. @@ -293,7 +308,9 @@ pub(crate) async fn linear_ref_transaction( } if let Some(companion) = companion - && let Err(err) = companion.apply(&txn).await + && let Err(err) = companion + .apply(&LinearRefWriteTransaction { transaction: &txn }) + .await { let _ = txn.rollback().await; return Err(err.context("companion mutation failed; owned ref update rolled back")); @@ -358,7 +375,8 @@ mod tests { #[async_trait::async_trait] impl LinearRefCompanion for InsertCompanion { - async fn apply(&self, txn: &DatabaseTransaction) -> Result<()> { + async fn apply(&self, txn: &LinearRefWriteTransaction<'_>) -> Result<()> { + let txn = txn.as_database_transaction(); txn.execute_raw(Statement::from_string( txn.get_database_backend(), "INSERT INTO config_kv(key, value, encrypted) VALUES ('projection', '1', 0)" @@ -373,7 +391,8 @@ mod tests { #[async_trait::async_trait] impl LinearRefCompanion for FailingCompanion { - async fn apply(&self, txn: &DatabaseTransaction) -> Result<()> { + async fn apply(&self, txn: &LinearRefWriteTransaction<'_>) -> Result<()> { + let txn = txn.as_database_transaction(); txn.execute_raw(Statement::from_string( txn.get_database_backend(), "INSERT INTO config_kv(key, value, encrypted) VALUES ('rolled-back', '1', 0)" @@ -388,7 +407,8 @@ mod tests { #[async_trait::async_trait] impl LinearRefCompanion for DeadlineExpiringCompanion { - async fn apply(&self, txn: &DatabaseTransaction) -> Result<()> { + async fn apply(&self, txn: &LinearRefWriteTransaction<'_>) -> Result<()> { + let txn = txn.as_database_transaction(); txn.execute_raw(Statement::from_string( txn.get_database_backend(), "INSERT INTO config_kv(key, value, encrypted) VALUES ('deadline', '1', 0)" diff --git a/src/internal/ai/memory/error.rs b/src/internal/ai/memory/error.rs index 99f4100d4..3d3b22406 100644 --- a/src/internal/ai/memory/error.rs +++ b/src/internal/ai/memory/error.rs @@ -10,6 +10,7 @@ pub(crate) enum MemoryWriterErrorKind { UnknownDigestKey, CorruptHistory, CorruptProjection, + ProjectionStale, StorageFailure, ConflictExhausted, } @@ -21,6 +22,7 @@ impl MemoryWriterErrorKind { Self::InvalidProposal => "LBR-MEMORY-002", Self::PolicyRejected | Self::UnknownDigestKey => "LBR-MEMORY-003", Self::CorruptHistory | Self::CorruptProjection => "LBR-MEMORY-004", + Self::ProjectionStale => "LBR-MEMORY-PROJECTION-STALE", Self::StorageFailure | Self::ConflictExhausted => "LBR-MEMORY-005", } } diff --git a/src/internal/ai/memory/fts_sql.rs b/src/internal/ai/memory/fts_sql.rs index 8738f0310..7faeb2ae5 100644 --- a/src/internal/ai/memory/fts_sql.rs +++ b/src/internal/ai/memory/fts_sql.rs @@ -12,7 +12,7 @@ use sea_orm::{ConnectionTrait, DatabaseConnection, DatabaseTransaction, DbErr, S use thiserror::Error; use super::domain::{CodeChangeStatus, CompletionStatus, EpisodeRoot, EpisodeRootKind}; -use crate::internal::db; +use crate::internal::{ai::linear_ref::LinearRefWriteTransaction, db}; const MAX_SEARCH_TEXT_BYTES: usize = 64 * 1024; const MAX_MATCH_INPUT_BYTES: usize = 4 * 1024; @@ -253,7 +253,20 @@ pub(crate) async fn upsert_document( transaction: &MemoryWriteTransaction, document: &EpisodeSearchDocument, ) -> Result { - let transaction = transaction.as_database_transaction(); + upsert_document_on(transaction.as_database_transaction(), document).await +} + +pub(super) async fn upsert_document_in_linear_transaction( + transaction: &LinearRefWriteTransaction<'_>, + document: &EpisodeSearchDocument, +) -> Result { + upsert_document_on(transaction.as_database_transaction(), document).await +} + +async fn upsert_document_on( + transaction: &DatabaseTransaction, + document: &EpisodeSearchDocument, +) -> Result { let note_id = document.note_id(); let revision_oid = document.revision_oid(); if let Some(stored) = read_document(transaction, ¬e_id, &revision_oid).await? { diff --git a/src/internal/ai/memory/mod.rs b/src/internal/ai/memory/mod.rs index bb698a284..dc2bb82ef 100644 --- a/src/internal/ai/memory/mod.rs +++ b/src/internal/ai/memory/mod.rs @@ -15,6 +15,8 @@ mod error; mod fts_sql; mod job_sql; mod policy; +mod projection; +mod replay; mod store; mod tree; mod validation; diff --git a/src/internal/ai/memory/projection.rs b/src/internal/ai/memory/projection.rs new file mode 100644 index 000000000..7b1dea98a --- /dev/null +++ b/src/internal/ai/memory/projection.rs @@ -0,0 +1,1261 @@ +//! Deterministic projection replay and rebuild for repository Memory history. + +use std::{collections::BTreeSet, path::PathBuf, sync::Arc}; + +use anyhow::Result; +use git_internal::hash::ObjectHash; +use sea_orm::{ConnectionTrait, DatabaseConnection, DatabaseTransaction, Statement}; +use uuid::Uuid; + +use super::{ + domain::{EpisodeRoot, MemoryEventAction, MemoryNoteV1}, + error::{MemoryWriterError, MemoryWriterErrorKind}, + fts_sql::{ + EpisodeSearchDocument, EpisodeSearchText, MemoryWriteTransaction, rebuild_index, + upsert_document, upsert_document_in_linear_transaction, + }, + replay::{ProjectedNote, ProjectedReviewState, ReducedProjection, ReplayRecord}, + store::{ + enum_label, execute, insert_note, insert_revision, read_memory_ref_head, + replace_episode_paths, replace_links, update_note, + }, + tree::{MemoryHistoryDelta, load_history_delta, parse_oid}, +}; +use crate::internal::ai::linear_ref::LinearRefWriteTransaction; + +const PROJECTION_SCHEMA_VERSION: i64 = 1; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) enum MemoryProjectionStatus { + Empty, + Current { + head: ObjectHash, + last_event_seq: u64, + }, + Stale { + head: ObjectHash, + projected: Option, + last_event_seq: u64, + }, + Corrupt { + head: Option, + projected: Option, + last_event_seq: Option, + }, +} + +pub(crate) struct MemoryProjection { + database: Arc, + storage_path: PathBuf, + policy_version: String, +} + +impl MemoryProjection { + pub(crate) fn new( + database: Arc, + storage_path: PathBuf, + policy_version: impl Into, + ) -> Self { + Self { + database, + storage_path, + policy_version: policy_version.into(), + } + } + + pub(crate) async fn status( + &self, + pinned_head: Option, + ) -> Result { + let row = projection_watermark(self.database.as_ref()).await?; + match (pinned_head, row) { + (None, None) => Ok(MemoryProjectionStatus::Empty), + (Some(head), Some(row)) => { + let projected = parse_oid(&row.projected_ref_oid).ok(); + let last_event_seq = u64::try_from(row.last_event_seq).ok(); + if row.schema_version != PROJECTION_SCHEMA_VERSION + || projected.is_none() + || last_event_seq.is_none() + { + return Ok(MemoryProjectionStatus::Corrupt { + head: Some(head), + projected: Some(row.projected_ref_oid), + last_event_seq: Some(row.last_event_seq), + }); + } + if projected == Some(head) { + let last_event_seq = last_event_seq.unwrap_or_default(); + match load_history_delta( + &self.storage_path, + head, + Some(head), + &self.policy_version, + ) { + Ok(history) if history.manifest.last_event_seq == last_event_seq => { + Ok(MemoryProjectionStatus::Current { + head, + last_event_seq, + }) + } + _ => Ok(MemoryProjectionStatus::Corrupt { + head: Some(head), + projected: Some(head.to_string()), + last_event_seq: i64::try_from(last_event_seq).ok(), + }), + } + } else { + Ok(MemoryProjectionStatus::Stale { + head, + projected, + last_event_seq: last_event_seq.unwrap_or_default(), + }) + } + } + (Some(head), None) => Ok(MemoryProjectionStatus::Stale { + head, + projected: None, + last_event_seq: 0, + }), + (None, Some(row)) => Ok(MemoryProjectionStatus::Corrupt { + head: None, + projected: Some(row.projected_ref_oid), + last_event_seq: Some(row.last_event_seq), + }), + } + } + + pub(crate) async fn advance( + &self, + pinned_head: ObjectHash, + rebuilt_at_ms: i64, + ) -> Result<(), MemoryWriterError> { + ensure_pinned_ref(self.database.as_ref(), pinned_head).await?; + let watermark = projection_watermark(self.database.as_ref()).await?; + let (after, last_event_seq) = match watermark { + Some(row) => { + if row.schema_version != PROJECTION_SCHEMA_VERSION || row.last_event_seq < 0 { + return Err(corrupt_projection("Memory projection watermark is invalid")); + } + ( + Some(parse_oid(&row.projected_ref_oid)?), + u64::try_from(row.last_event_seq) + .map_err(|_| corrupt_projection("Memory projection sequence is invalid"))?, + ) + } + None => (None, 0), + }; + let history = + load_history_delta(&self.storage_path, pinned_head, after, &self.policy_version)?; + if history.records.is_empty() { + if history.manifest.last_event_seq != last_event_seq { + return Err(corrupt_projection( + "Memory projection sequence does not match the pinned manifest", + )); + } + return Ok(()); + } + let note_ids = history + .records + .iter() + .filter_map(|record| record.event.note_id) + .collect::>(); + let mut reduced = load_projection_seed(self.database.as_ref(), ¬e_ids).await?; + reduced.last_event_seq = last_event_seq; + apply_history(&mut reduced, history)?; + + let transaction = MemoryWriteTransaction::begin(self.database.as_ref()) + .await + .map_err(fts_error)?; + ensure_transaction_snapshot( + transaction.as_database_transaction(), + pinned_head, + after, + last_event_seq, + ) + .await?; + materialize( + ProjectionTransaction::Standalone(&transaction), + &reduced, + pinned_head, + &self.policy_version, + rebuilt_at_ms, + ) + .await?; + transaction.commit().await.map_err(fts_error) + } + + pub(crate) async fn rebuild( + &self, + pinned_head: ObjectHash, + rebuilt_at_ms: i64, + ) -> Result<(), MemoryWriterError> { + ensure_pinned_ref(self.database.as_ref(), pinned_head).await?; + let history = + load_history_delta(&self.storage_path, pinned_head, None, &self.policy_version)?; + let mut reduced = ReducedProjection::default(); + apply_history(&mut reduced, history)?; + + let transaction = MemoryWriteTransaction::begin(self.database.as_ref()) + .await + .map_err(fts_error)?; + ensure_pinned_ref_in_transaction(transaction.as_database_transaction(), pinned_head) + .await?; + clear_rebuildable_projection(transaction.as_database_transaction()).await?; + materialize( + ProjectionTransaction::Standalone(&transaction), + &reduced, + pinned_head, + &self.policy_version, + rebuilt_at_ms, + ) + .await?; + rebuild_index(&transaction).await.map_err(fts_error)?; + transaction.commit().await.map_err(fts_error) + } +} + +fn apply_history( + reduced: &mut ReducedProjection, + history: MemoryHistoryDelta, +) -> Result<(), MemoryWriterError> { + for record in history.records { + reduced.apply(ReplayRecord { + event: record.event, + revision_oid: record.revision_oid, + note: record.note, + })?; + } + if reduced.last_event_seq != history.manifest.last_event_seq { + return Err(MemoryWriterError::new( + MemoryWriterErrorKind::CorruptHistory, + "Memory replay sequence does not match the pinned manifest", + )); + } + Ok(()) +} + +enum ProjectionTransaction<'a> { + Standalone(&'a MemoryWriteTransaction), + Linear(&'a LinearRefWriteTransaction<'a>), +} + +impl ProjectionTransaction<'_> { + fn as_database_transaction(&self) -> &DatabaseTransaction { + match self { + Self::Standalone(transaction) => transaction.as_database_transaction(), + Self::Linear(transaction) => transaction.as_database_transaction(), + } + } + + async fn upsert_search( + &self, + document: &EpisodeSearchDocument, + ) -> Result<(), MemoryWriterError> { + match self { + Self::Standalone(transaction) => { + upsert_document(transaction, document) + .await + .map_err(fts_error)?; + } + Self::Linear(transaction) => { + upsert_document_in_linear_transaction(transaction, document) + .await + .map_err(fts_error)?; + } + } + Ok(()) + } +} + +pub(super) async fn materialize_linear( + transaction: &LinearRefWriteTransaction<'_>, + reduced: &ReducedProjection, + pinned_head: ObjectHash, + policy_version: &str, + rebuilt_at_ms: i64, +) -> Result<(), MemoryWriterError> { + materialize( + ProjectionTransaction::Linear(transaction), + reduced, + pinned_head, + policy_version, + rebuilt_at_ms, + ) + .await +} + +async fn materialize( + write: ProjectionTransaction<'_>, + reduced: &ReducedProjection, + pinned_head: ObjectHash, + policy_version: &str, + rebuilt_at_ms: i64, +) -> Result<(), MemoryWriterError> { + let txn = write.as_database_transaction(); + let mut inserted_notes = BTreeSet::new(); + for revision_oid in &reduced.new_revision_order { + let note = reduced.new_revisions.get(revision_oid).ok_or_else(|| { + corrupt_projection("ordered Memory revision is absent from reducer state") + })?; + if reduced.created_notes.contains(¬e.note_id) && inserted_notes.insert(note.note_id) { + insert_note(txn, note).await.map_err(storage_error)?; + } else { + update_note(txn, note).await.map_err(storage_error)?; + } + insert_revision(txn, note, parse_oid(revision_oid)?) + .await + .map_err(storage_error)?; + } + for revision_oid in &reduced.new_revision_order { + let note = reduced.new_revisions.get(revision_oid).ok_or_else(|| { + corrupt_projection("ordered Memory revision is absent from reducer state") + })?; + let revision_oid = parse_oid(revision_oid)?; + replace_links(txn, note, revision_oid) + .await + .map_err(storage_error)?; + replace_episode_paths(txn, note, revision_oid) + .await + .map_err(storage_error)?; + if let Some(document) = episode_search_document(note, revision_oid)? { + write.upsert_search(&document).await?; + } + } + for note_id in &reduced.changed_notes { + let projected = reduced.notes.get(note_id).ok_or_else(|| { + corrupt_projection("changed Memory note is absent from reducer state") + })?; + if let Some(note) = reduced + .new_revisions + .get(&projected.latest_revision_oid.to_string()) + { + upsert_head(txn, note, projected).await?; + } else { + update_head_state(txn, *note_id, projected).await?; + } + update_note_review_state(txn, *note_id, projected.review_state).await?; + } + refresh_path_summaries(txn, &reduced.changed_notes).await?; + upsert_watermark( + txn, + pinned_head, + reduced.last_event_seq, + policy_version, + rebuilt_at_ms, + ) + .await +} + +async fn update_note_review_state( + txn: &DatabaseTransaction, + note_id: Uuid, + review_state: ProjectedReviewState, +) -> Result<(), MemoryWriterError> { + let result = txn + .execute_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "UPDATE memory_note_index SET review_state = ? + WHERE scope_key = 'repo' AND note_id = ?", + [ + review_state_label(review_state).into(), + note_id.to_string().into(), + ], + )) + .await + .map_err(|error| projection_storage("update Memory note review state", error))?; + if result.rows_affected() != 1 { + return Err(corrupt_projection( + "Memory note disappeared while updating review state", + )); + } + Ok(()) +} + +async fn load_projection_seed( + database: &DatabaseConnection, + note_ids: &BTreeSet, +) -> Result { + let mut reduced = ReducedProjection::default(); + for note_id in note_ids { + let row = database + .query_one_raw(Statement::from_sql_and_values( + database.get_database_backend(), + "SELECT latest_revision_oid, live_revision_oid, latest_action, + latest_review_state, last_event_seq, updated_at + FROM memory_head WHERE scope_key = 'repo' AND note_id = ?", + [note_id.to_string().into()], + )) + .await + .map_err(|error| projection_storage("read Memory projection seed", error))?; + let Some(row) = row else { + continue; + }; + let latest: String = row + .try_get("", "latest_revision_oid") + .map_err(storage_error)?; + let live: Option = row + .try_get("", "live_revision_oid") + .map_err(storage_error)?; + let latest_action: String = row.try_get("", "latest_action").map_err(storage_error)?; + let review_state: String = row + .try_get("", "latest_review_state") + .map_err(storage_error)?; + let last_event_seq: i64 = row.try_get("", "last_event_seq").map_err(storage_error)?; + let updated_at: String = row.try_get("", "updated_at").map_err(storage_error)?; + let revisions = database + .query_all_raw(Statement::from_sql_and_values( + database.get_database_backend(), + "SELECT revision_oid FROM memory_revision_index + WHERE scope_key = 'repo' AND note_id = ? ORDER BY revision_oid", + [note_id.to_string().into()], + )) + .await + .map_err(|error| projection_storage("read Memory revision seed", error))? + .into_iter() + .map(|row| row.try_get("", "revision_oid").map_err(storage_error)) + .collect::, _>>()?; + reduced.notes.insert( + *note_id, + ProjectedNote { + latest_revision_oid: parse_oid(&latest)?, + live_revision_oid: live.as_deref().map(parse_oid).transpose()?, + latest_action: parse_action(&latest_action)?, + review_state: parse_review_state(&review_state)?, + last_event_seq: u64::try_from(last_event_seq) + .map_err(|_| corrupt_projection("Memory head sequence is invalid"))?, + updated_at: chrono::DateTime::parse_from_rfc3339(&updated_at) + .map_err(|_| corrupt_projection("Memory head timestamp is invalid"))? + .with_timezone(&chrono::Utc), + revisions, + }, + ); + } + Ok(reduced) +} + +struct ProjectionWatermark { + projected_ref_oid: String, + last_event_seq: i64, + schema_version: i64, +} + +async fn projection_watermark( + database: &impl ConnectionTrait, +) -> Result, MemoryWriterError> { + database + .query_one_raw(Statement::from_string( + database.get_database_backend(), + "SELECT projected_ref_oid, last_event_seq, schema_version + FROM memory_projection_state WHERE scope_key = 'repo'" + .to_string(), + )) + .await + .map_err(|error| projection_storage("read Memory projection watermark", error))? + .map(|row| { + Ok(ProjectionWatermark { + projected_ref_oid: row + .try_get("", "projected_ref_oid") + .map_err(storage_error)?, + last_event_seq: row.try_get("", "last_event_seq").map_err(storage_error)?, + schema_version: row.try_get("", "schema_version").map_err(storage_error)?, + }) + }) + .transpose() +} + +async fn ensure_pinned_ref( + database: &DatabaseConnection, + pinned_head: ObjectHash, +) -> Result<(), MemoryWriterError> { + if read_memory_ref_head(database).await? != Some(pinned_head) { + return Err(MemoryWriterError::new( + MemoryWriterErrorKind::ProjectionStale, + "pinned Memory ref no longer matches the repository ref", + )); + } + Ok(()) +} + +async fn ensure_pinned_ref_in_transaction( + txn: &DatabaseTransaction, + pinned_head: ObjectHash, +) -> Result<(), MemoryWriterError> { + let row = txn + .query_one_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "SELECT `commit` FROM reference + WHERE kind = 'Branch' AND remote IS NULL AND name = 'libra/memory/repo'", + [], + )) + .await + .map_err(|error| projection_storage("revalidate pinned Memory ref", error))?; + let row = row.ok_or_else(|| corrupt_projection("repository Memory ref disappeared"))?; + let value: String = row + .try_get("", "commit") + .map_err(|error| projection_storage("decode pinned Memory ref", error))?; + if parse_oid(&value)? != pinned_head { + return Err(MemoryWriterError::new( + MemoryWriterErrorKind::ProjectionStale, + "pinned Memory ref changed before projection commit", + )); + } + Ok(()) +} + +async fn ensure_transaction_snapshot( + txn: &DatabaseTransaction, + pinned_head: ObjectHash, + expected_projected: Option, + expected_seq: u64, +) -> Result<(), MemoryWriterError> { + ensure_pinned_ref_in_transaction(txn, pinned_head).await?; + let current = projection_watermark(txn).await?; + let expected_seq = i64::try_from(expected_seq) + .map_err(|_| corrupt_projection("Memory projection sequence exceeds SQLite range"))?; + match (expected_projected, current) { + (None, None) => Ok(()), + (Some(expected), Some(row)) + if row.projected_ref_oid == expected.to_string() + && row.last_event_seq == expected_seq + && row.schema_version == PROJECTION_SCHEMA_VERSION => + { + Ok(()) + } + _ => Err(MemoryWriterError::new( + MemoryWriterErrorKind::ProjectionStale, + "Memory projection changed before incremental replay committed", + )), + } +} + +async fn clear_rebuildable_projection(txn: &DatabaseTransaction) -> Result<(), MemoryWriterError> { + txn.execute_unprepared( + "INSERT INTO memory_episode_fts(memory_episode_fts) VALUES('delete-all')", + ) + .await + .map_err(|error| projection_storage("clear Memory FTS postings", error))?; + for table in [ + "memory_link_index", + "memory_episode_path", + "memory_head", + "memory_path_summary", + "memory_episode_search_doc", + "memory_revision_index", + "memory_note_index", + "memory_projection_state", + ] { + txn.execute_unprepared(&format!("DELETE FROM {table}")) + .await + .map_err(|error| projection_storage("clear rebuildable Memory projection", error))?; + } + Ok(()) +} + +async fn upsert_head( + txn: &DatabaseTransaction, + note: &MemoryNoteV1, + projected: &ProjectedNote, +) -> Result<(), MemoryWriterError> { + execute( + txn, + "INSERT INTO memory_head ( + scope_key, namespace, path, note_id, latest_revision_oid, + live_revision_oid, latest_action, latest_review_state, kind, + lifecycle, confidence, trust, sensitivity, visibility, acl_policy_id, + valid_from, valid_until, effective_from_commit, effective_until_commit, + expires_at, rank_hint, last_event_seq, updated_at + ) VALUES ('repo', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?) + ON CONFLICT(scope_key, namespace, path, note_id) DO UPDATE SET + latest_revision_oid = excluded.latest_revision_oid, + live_revision_oid = excluded.live_revision_oid, + latest_action = excluded.latest_action, + latest_review_state = excluded.latest_review_state, + confidence = excluded.confidence, + trust = excluded.trust, + sensitivity = excluded.sensitivity, + visibility = excluded.visibility, + acl_policy_id = excluded.acl_policy_id, + valid_from = excluded.valid_from, + valid_until = excluded.valid_until, + effective_from_commit = excluded.effective_from_commit, + effective_until_commit = excluded.effective_until_commit, + expires_at = excluded.expires_at, + last_event_seq = excluded.last_event_seq, + updated_at = excluded.updated_at", + vec![ + note.namespace.clone().into(), + note.path.clone().into(), + note.note_id.to_string().into(), + projected.latest_revision_oid.to_string().into(), + projected + .live_revision_oid + .map(|oid| oid.to_string()) + .into(), + enum_label(&projected.latest_action) + .map_err(storage_error)? + .into(), + review_state_label(projected.review_state).into(), + enum_label(¬e.kind).map_err(storage_error)?.into(), + enum_label(¬e.lifecycle).map_err(storage_error)?.into(), + enum_label(¬e.confidence).map_err(storage_error)?.into(), + enum_label(¬e.trust).map_err(storage_error)?.into(), + enum_label(¬e.sensitivity).map_err(storage_error)?.into(), + enum_label(¬e.visibility).map_err(storage_error)?.into(), + note.acl_policy_id.clone().into(), + note.valid_from.map(|value| value.to_rfc3339()).into(), + note.valid_until.map(|value| value.to_rfc3339()).into(), + note.effective_from_commit.clone().into(), + note.effective_until_commit.clone().into(), + note.expires_at.map(|value| value.to_rfc3339()).into(), + i64::try_from(projected.last_event_seq) + .map_err(|_| corrupt_projection("Memory head sequence exceeds SQLite range"))? + .into(), + projected.updated_at.to_rfc3339().into(), + ], + ) + .await + .map_err(storage_error) +} + +async fn update_head_state( + txn: &DatabaseTransaction, + note_id: Uuid, + projected: &ProjectedNote, +) -> Result<(), MemoryWriterError> { + let result = txn + .execute_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "UPDATE memory_head SET live_revision_oid = ?, latest_action = ?, + latest_review_state = ?, last_event_seq = ?, updated_at = ? + WHERE scope_key = 'repo' AND note_id = ?", + [ + projected + .live_revision_oid + .map(|oid| oid.to_string()) + .into(), + enum_label(&projected.latest_action) + .map_err(storage_error)? + .into(), + review_state_label(projected.review_state).into(), + i64::try_from(projected.last_event_seq) + .map_err(|_| corrupt_projection("Memory head sequence exceeds SQLite range"))? + .into(), + projected.updated_at.to_rfc3339().into(), + note_id.to_string().into(), + ], + )) + .await + .map_err(|error| projection_storage("update Memory head state", error))?; + if result.rows_affected() != 1 { + return Err(corrupt_projection("Memory head disappeared during replay")); + } + Ok(()) +} + +async fn refresh_path_summaries( + txn: &DatabaseTransaction, + changed_notes: &BTreeSet, +) -> Result<(), MemoryWriterError> { + let mut affected = BTreeSet::new(); + for note_id in changed_notes { + let row = txn + .query_one_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "SELECT namespace, path FROM memory_head + WHERE scope_key = 'repo' AND note_id = ?", + [note_id.to_string().into()], + )) + .await + .map_err(|error| projection_storage("read changed Memory path", error))? + .ok_or_else(|| corrupt_projection("changed Memory head has no path"))?; + let namespace: String = row.try_get("", "namespace").map_err(storage_error)?; + let path: String = row.try_get("", "path").map_err(storage_error)?; + let mut prefix = String::new(); + for segment in path.split('.') { + if !prefix.is_empty() { + prefix.push('.'); + } + prefix.push_str(segment); + affected.insert((namespace.clone(), prefix.clone())); + } + } + + for (namespace, path) in affected { + let descendant_pattern = format!("{path}.%"); + let row = txn + .query_one_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "SELECT + SUM(CASE WHEN path = ? AND live_revision_oid IS NOT NULL THEN 1 ELSE 0 END) + AS confirmed_count, + SUM(CASE WHEN path = ? AND latest_review_state = 'quarantined' + THEN 1 ELSE 0 END) AS quarantined_count, + SUM(CASE WHEN live_revision_oid IS NOT NULL THEN 1 ELSE 0 END) + AS prefix_count, + MAX(updated_at) AS last_changed_at + FROM memory_head + WHERE scope_key = 'repo' AND namespace = ? + AND (path = ? OR path LIKE ? ESCAPE '\\')", + [ + path.clone().into(), + path.clone().into(), + namespace.clone().into(), + path.clone().into(), + descendant_pattern.clone().into(), + ], + )) + .await + .map_err(|error| projection_storage("aggregate Memory path summary", error))? + .ok_or_else(|| corrupt_projection("Memory path aggregate disappeared"))?; + let confirmed_count: i64 = row + .try_get::>("", "confirmed_count") + .map_err(storage_error)? + .unwrap_or_default(); + let quarantined_count: i64 = row + .try_get::>("", "quarantined_count") + .map_err(storage_error)? + .unwrap_or_default(); + let prefix_count: i64 = row + .try_get::>("", "prefix_count") + .map_err(storage_error)? + .unwrap_or_default(); + let last_changed_at: String = row + .try_get::>("", "last_changed_at") + .map_err(storage_error)? + .ok_or_else(|| corrupt_projection("Memory path aggregate has no timestamp"))?; + + let paths = txn + .query_all_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "SELECT DISTINCT path FROM memory_head + WHERE scope_key = 'repo' AND namespace = ? AND path LIKE ? ESCAPE '\\'", + [namespace.clone().into(), descendant_pattern.into()], + )) + .await + .map_err(|error| projection_storage("read Memory child paths", error))?; + let child_prefix = format!("{path}."); + let mut children = BTreeSet::new(); + for row in paths { + let candidate: String = row.try_get("", "path").map_err(storage_error)?; + if let Some(child) = candidate + .strip_prefix(&child_prefix) + .and_then(|rest| rest.split('.').next()) + { + children.insert(child.to_owned()); + } + } + let child_count = children.len(); + let preview_row = txn + .query_one_raw(Statement::from_sql_and_values( + txn.get_database_backend(), + "SELECT document.summary FROM memory_head AS head + JOIN memory_episode_search_doc AS document + ON document.note_id = head.note_id + AND document.revision_oid = head.live_revision_oid + WHERE head.scope_key = 'repo' AND head.namespace = ? AND head.path = ? + ORDER BY head.note_id LIMIT 1", + [namespace.clone().into(), path.clone().into()], + )) + .await + .map_err(|error| projection_storage("read Memory path preview", error))?; + let preview = match preview_row { + Some(row) => row + .try_get::("", "summary") + .map_err(storage_error)? + .chars() + .take(240) + .collect(), + None => String::new(), + }; + + execute( + txn, + "INSERT INTO memory_path_summary ( + scope_key, namespace, path, confirmed_count, quarantined_count, + child_count, prefix_count, preview, last_changed_at + ) VALUES ('repo', ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(scope_key, namespace, path) DO UPDATE SET + confirmed_count = excluded.confirmed_count, + quarantined_count = excluded.quarantined_count, + child_count = excluded.child_count, + prefix_count = excluded.prefix_count, + preview = excluded.preview, + last_changed_at = excluded.last_changed_at", + vec![ + namespace.into(), + path.into(), + confirmed_count.into(), + quarantined_count.into(), + i64::try_from(child_count) + .map_err(|_| corrupt_projection("Memory child count exceeds SQLite range"))? + .into(), + prefix_count.into(), + preview.into(), + last_changed_at.into(), + ], + ) + .await + .map_err(storage_error)?; + } + Ok(()) +} + +async fn upsert_watermark( + txn: &DatabaseTransaction, + pinned_head: ObjectHash, + last_event_seq: u64, + policy_version: &str, + rebuilt_at_ms: i64, +) -> Result<(), MemoryWriterError> { + execute( + txn, + "INSERT INTO memory_projection_state ( + scope_key, projected_ref_oid, last_event_seq, schema_version, + policy_version, rebuilt_at + ) VALUES ('repo', ?, ?, 1, ?, ?) + ON CONFLICT(scope_key) DO UPDATE SET + projected_ref_oid = excluded.projected_ref_oid, + last_event_seq = excluded.last_event_seq, + schema_version = excluded.schema_version, + policy_version = excluded.policy_version, + rebuilt_at = excluded.rebuilt_at", + vec![ + pinned_head.to_string().into(), + i64::try_from(last_event_seq) + .map_err(|_| corrupt_projection("Memory watermark exceeds SQLite range"))? + .into(), + policy_version.into(), + rebuilt_at_ms.into(), + ], + ) + .await + .map_err(storage_error) +} + +fn episode_search_document( + note: &MemoryNoteV1, + revision_oid: ObjectHash, +) -> Result, MemoryWriterError> { + let Some(episode) = ¬e.episode else { + return Ok(None); + }; + let root = match episode.root_kind { + super::domain::EpisodeRootKind::Task => EpisodeRoot::task(episode.root_id.clone()), + super::domain::EpisodeRootKind::Intent => EpisodeRoot::intent(episode.root_id.clone()), + } + .map_err(MemoryWriterError::from)?; + let join_claims = |claims: &[super::domain::EpisodeClaimV1]| { + claims + .iter() + .map(|claim| claim.claim.as_str()) + .collect::>() + .join("\n") + }; + let text = EpisodeSearchText::new( + &episode.goal.claim, + &episode.summary.claim, + join_claims(&episode.decisions), + join_claims(&episode.failed_attempts), + join_claims(&episode.unresolved), + ) + .map_err(fts_error)?; + Ok(Some(EpisodeSearchDocument::new( + root, + revision_oid, + episode.completion_status, + episode.code_change_status, + episode.ended_at, + text, + ))) +} + +fn parse_action(value: &str) -> Result { + serde_json::from_str(&format!("\"{value}\"")) + .map_err(|_| corrupt_projection("Memory head action is invalid")) +} + +fn parse_review_state(value: &str) -> Result { + match value { + "draft" => Ok(ProjectedReviewState::Draft), + "confirmed" => Ok(ProjectedReviewState::Confirmed), + "quarantined" => Ok(ProjectedReviewState::Quarantined), + "revoked" => Ok(ProjectedReviewState::Revoked), + "superseded" => Ok(ProjectedReviewState::Superseded), + "forgotten" => Ok(ProjectedReviewState::Forgotten), + _ => Err(corrupt_projection("Memory head review state is invalid")), + } +} + +const fn review_state_label(value: ProjectedReviewState) -> &'static str { + match value { + ProjectedReviewState::Draft => "draft", + ProjectedReviewState::Confirmed => "confirmed", + ProjectedReviewState::Quarantined => "quarantined", + ProjectedReviewState::Revoked => "revoked", + ProjectedReviewState::Superseded => "superseded", + ProjectedReviewState::Forgotten => "forgotten", + } +} + +fn projection_error(summary: &'static str) -> MemoryWriterError { + MemoryWriterError::new(MemoryWriterErrorKind::CorruptProjection, summary) +} + +fn corrupt_projection(summary: &'static str) -> MemoryWriterError { + projection_error(summary) +} + +fn projection_storage(action: &'static str, error: impl std::fmt::Display) -> MemoryWriterError { + MemoryWriterError::new( + MemoryWriterErrorKind::StorageFailure, + format!("{action} failed: {error}"), + ) +} + +fn storage_error(error: impl std::fmt::Display) -> MemoryWriterError { + projection_storage("materialize Memory projection", error) +} + +fn fts_error(error: impl std::fmt::Display) -> MemoryWriterError { + projection_storage("maintain Memory FTS projection", error) +} + +#[cfg(test)] +mod tests { + use std::fs; + + use sea_orm::{ConnectionTrait, QueryResult}; + + use super::*; + use crate::internal::ai::memory::{ + domain::{MemoryLinkKind, MemoryLinkV1}, + policy::TrustedMemoryTarget, + writer::tests::{fixture, proposal}, + }; + + fn projection_for(fixture: &super::super::writer::tests::Fixture) -> MemoryProjection { + MemoryProjection::new( + Arc::clone(&fixture.database), + fixture._temp.path().to_path_buf(), + "repo-policy-v1", + ) + } + + async fn commit_generation( + fixture: &super::super::writer::tests::Fixture, + generation: u8, + expected_head: Option, + ) -> super::super::writer::CommittedMemoryEnvelope { + fixture + .writer + .commit( + &fixture.context, + &fixture.target, + &proposal(&fixture.target, fixture.key_id, generation), + expected_head, + ) + .await + .expect("commit Memory fixture generation") + } + + async fn selected_lines(database: &DatabaseConnection, sql: &str) -> Vec { + database + .query_all_raw(Statement::from_string( + database.get_database_backend(), + sql.to_string(), + )) + .await + .expect("query semantic projection rows") + .into_iter() + .map(|row: QueryResult| row.try_get("", "value").expect("decode semantic row")) + .collect() + } + + async fn semantic_snapshot(database: &DatabaseConnection) -> Vec { + let queries = [ + "SELECT 'note|' || json_object( + 'note_id', note_id, 'scope_key', scope_key, 'namespace', namespace, + 'path', path, 'kind', kind, 'lifecycle', lifecycle, + 'review_state', review_state, 'confidence', confidence, 'trust', trust, + 'sensitivity', sensitivity, 'visibility', visibility, + 'acl_policy_id', acl_policy_id, 'origin', origin, + 'idempotency_key', idempotency_key, + 'idempotency_scope', idempotency_scope, 'created_at', created_at) AS value + FROM memory_note_index ORDER BY note_id", + "SELECT 'revision|' || json_object( + 'revision_oid', revision_oid, 'note_id', note_id, 'scope_key', scope_key, + 'namespace', namespace, 'origin', origin, 'producer', producer, + 'rules_version', rules_version, 'prompt_version', prompt_version, + 'model_id', model_id, 'policy_version', policy_version, + 'input_fingerprints_json', input_fingerprints_json, 'created_at', created_at) + AS value + FROM memory_revision_index ORDER BY revision_oid", + "SELECT 'head|' || json_object( + 'scope_key', scope_key, 'namespace', namespace, 'path', path, + 'note_id', note_id, 'latest_revision_oid', latest_revision_oid, + 'live_revision_oid', live_revision_oid, 'latest_action', latest_action, + 'latest_review_state', latest_review_state, 'kind', kind, + 'lifecycle', lifecycle, 'confidence', confidence, 'trust', trust, + 'sensitivity', sensitivity, 'visibility', visibility, + 'acl_policy_id', acl_policy_id, 'valid_from', valid_from, + 'valid_until', valid_until, 'effective_from_commit', effective_from_commit, + 'effective_until_commit', effective_until_commit, 'expires_at', expires_at, + 'rank_hint', rank_hint, 'last_event_seq', last_event_seq, + 'updated_at', updated_at) AS value + FROM memory_head ORDER BY note_id", + "SELECT 'link|' || json_object( + 'source_scope_key', source_scope_key, + 'source_namespace', source_namespace, 'source_note_id', source_note_id, + 'source_revision_oid', source_revision_oid, + 'target_note_id', target_note_id, + 'target_revision_oid', target_revision_oid, 'link_kind', link_kind, + 'source_path', source_path, 'target_path', target_path, + 'evidence_refs_json', evidence_refs_json, + 'valid_from', valid_from, 'valid_until', valid_until) AS value + FROM memory_link_index + ORDER BY source_revision_oid, target_note_id, link_kind", + "SELECT 'path|' || json_object( + 'scope_key', scope_key, 'namespace', namespace, 'path', path, + 'confirmed_count', confirmed_count, + 'quarantined_count', quarantined_count, 'child_count', child_count, + 'prefix_count', prefix_count, 'preview', preview, + 'last_changed_at', last_changed_at) AS value + FROM memory_path_summary ORDER BY namespace, path", + "SELECT 'episode-path|' || json_object( + 'note_id', note_id, 'revision_oid', revision_oid, 'code_path', code_path) + AS value + FROM memory_episode_path ORDER BY note_id, revision_oid, code_path", + "SELECT 'search|' || json_object( + 'rowid', rowid, 'note_id', note_id, 'revision_oid', revision_oid, + 'root_kind', root_kind, 'root_id', root_id, + 'completion_status', completion_status, + 'code_change_status', code_change_status, 'ended_at', ended_at, + 'goal', goal, 'summary', summary, 'decisions', decisions, + 'failed_attempts', failed_attempts, 'unresolved', unresolved) AS value + FROM memory_episode_search_doc ORDER BY note_id, revision_oid", + "SELECT 'watermark|' || json_object( + 'scope_key', scope_key, 'projected_ref_oid', projected_ref_oid, + 'last_event_seq', last_event_seq, 'schema_version', schema_version, + 'policy_version', policy_version) AS value + FROM memory_projection_state ORDER BY scope_key", + "SELECT 'fts-retry|' || COUNT(*) AS value FROM memory_episode_fts + WHERE memory_episode_fts MATCH 'retry'", + "SELECT 'fts-generation|' || COUNT(*) AS value FROM memory_episode_fts + WHERE memory_episode_fts MATCH 'generation'", + "SELECT 'fts-timing|' || COUNT(*) AS value FROM memory_episode_fts + WHERE memory_episode_fts MATCH 'timing'", + ]; + let mut snapshot = Vec::new(); + for query in queries { + snapshot.extend(selected_lines(database, query).await); + } + snapshot + } + + async fn set_memory_ref(database: &DatabaseConnection, head: ObjectHash) { + let result = database + .execute_raw(Statement::from_sql_and_values( + database.get_database_backend(), + "UPDATE reference SET `commit` = ? + WHERE kind = 'Branch' AND remote IS NULL AND name = 'libra/memory/repo'", + [head.to_string().into()], + )) + .await + .expect("move test Memory ref"); + assert_eq!(result.rows_affected(), 1); + } + + #[tokio::test] + async fn projection_rebuild_equivalence() { + let fixture = fixture().await; + let first = commit_generation(&fixture, 1, None).await; + let second = commit_generation(&fixture, 2, Some(first.commit_oid())).await; + let linked_target = TrustedMemoryTarget::episode( + EpisodeRoot::task("task-43").expect("construct linked task root"), + ); + let mut linked_proposal = proposal(&linked_target, fixture.key_id, 1); + linked_proposal.note_mut().links.push(MemoryLinkV1 { + kind: MemoryLinkKind::Supports, + target_note_id: fixture.target.root().note_id(), + target_revision_oid: Some(second.revision_oid().to_string()), + evidence_refs: Vec::new(), + valid_from: None, + valid_until: None, + }); + let linked = fixture + .writer + .commit( + &fixture.context, + &linked_target, + &linked_proposal, + Some(second.commit_oid()), + ) + .await + .expect("commit linked Memory fixture"); + fixture + .database + .execute_raw(Statement::from_sql_and_values( + fixture.database.get_database_backend(), + "INSERT INTO memory_compile_observer_state( + scope_key, source_ref_name, scanned_through_oid, updated_at + ) VALUES ('repo', 'libra/memory/repo', ?, 77)", + [linked.commit_oid().to_string().into()], + )) + .await + .expect("seed non-projection observer state"); + let before = semantic_snapshot(&fixture.database).await; + let projection = projection_for(&fixture); + + projection + .rebuild(linked.commit_oid(), 1234) + .await + .expect("rebuild Memory projection"); + + assert_eq!(semantic_snapshot(&fixture.database).await, before); + assert_eq!( + selected_lines( + &fixture.database, + "SELECT source_ref_name || '|' || scanned_through_oid || '|' || updated_at + AS value FROM memory_compile_observer_state ORDER BY source_ref_name", + ) + .await, + vec![format!("libra/memory/repo|{}|77", linked.commit_oid())], + ); + assert_eq!( + read_memory_ref_head(&fixture.database) + .await + .expect("read Memory ref after rebuild"), + Some(linked.commit_oid()), + ); + assert_eq!( + projection + .status(Some(linked.commit_oid())) + .await + .expect("read projection status"), + MemoryProjectionStatus::Current { + head: linked.commit_oid(), + last_event_seq: 6, + }, + ); + } + + #[tokio::test] + async fn projection_incremental_idempotent() { + let fixture = fixture().await; + let first = commit_generation(&fixture, 1, None).await; + let second = commit_generation(&fixture, 2, Some(first.commit_oid())).await; + let expected = semantic_snapshot(&fixture.database).await; + let projection = projection_for(&fixture); + + set_memory_ref(&fixture.database, first.commit_oid()).await; + projection + .rebuild(first.commit_oid(), 100) + .await + .expect("rebuild first projection generation"); + set_memory_ref(&fixture.database, second.commit_oid()).await; + projection + .advance(second.commit_oid(), 200) + .await + .expect("advance one projection generation"); + assert_eq!(semantic_snapshot(&fixture.database).await, expected); + + projection + .advance(second.commit_oid(), 300) + .await + .expect("repeated advance is a no-op"); + assert_eq!(semantic_snapshot(&fixture.database).await, expected); + } + + #[tokio::test] + async fn projection_corruption_stops_watermark() { + let fixture = fixture().await; + let committed = commit_generation(&fixture, 1, None).await; + let before = semantic_snapshot(&fixture.database).await; + let hash = committed.commit_oid().to_string(); + let object_path = fixture + ._temp + .path() + .join("objects") + .join(&hash[..2]) + .join(&hash[2..]); + fs::remove_file(object_path).expect("remove temporary commit object"); + + let error = projection_for(&fixture) + .rebuild(committed.commit_oid(), 500) + .await + .expect_err("missing authority object stops rebuild"); + assert_eq!(error.kind(), MemoryWriterErrorKind::CorruptHistory); + assert_eq!(semantic_snapshot(&fixture.database).await, before); + } + + #[tokio::test] + async fn projection_stale_fails_closed() { + let fixture = fixture().await; + let first = commit_generation(&fixture, 1, None).await; + let _second = commit_generation(&fixture, 2, Some(first.commit_oid())).await; + + let error = projection_for(&fixture) + .advance(first.commit_oid(), 600) + .await + .expect_err("stale pinned head fails closed"); + assert_eq!(error.kind(), MemoryWriterErrorKind::ProjectionStale); + assert_eq!(error.stable_code(), "LBR-MEMORY-PROJECTION-STALE"); + assert_eq!( + error.to_string(), + "LBR-MEMORY-PROJECTION-STALE: pinned Memory ref no longer matches the repository ref", + ); + } + + #[tokio::test] + async fn projection_rejects_same_head_with_wrong_sequence() { + let fixture = fixture().await; + let committed = commit_generation(&fixture, 1, None).await; + fixture + .database + .execute_unprepared( + "UPDATE memory_projection_state SET last_event_seq = 1 + WHERE scope_key = 'repo'", + ) + .await + .expect("corrupt projection sequence"); + + let error = projection_for(&fixture) + .advance(committed.commit_oid(), 650) + .await + .expect_err("same-head sequence mismatch fails closed"); + assert_eq!(error.kind(), MemoryWriterErrorKind::CorruptProjection); + assert_eq!( + projection_for(&fixture) + .status(Some(committed.commit_oid())) + .await + .expect("diagnose corrupt projection"), + MemoryProjectionStatus::Corrupt { + head: Some(committed.commit_oid()), + projected: Some(committed.commit_oid().to_string()), + last_event_seq: Some(1), + }, + ); + } + + #[tokio::test] + async fn projection_rebuild_transaction_failure_rolls_back() { + let fixture = fixture().await; + let committed = commit_generation(&fixture, 1, None).await; + let before = semantic_snapshot(&fixture.database).await; + fixture + .database + .execute_unprepared( + "CREATE TRIGGER fail_memory_path_summary + BEFORE INSERT ON memory_path_summary + BEGIN SELECT RAISE(ABORT, 'simulated projection failure'); END;", + ) + .await + .expect("install projection failure trigger"); + + projection_for(&fixture) + .rebuild(committed.commit_oid(), 700) + .await + .expect_err("transaction failure rolls back rebuild"); + assert_eq!(semantic_snapshot(&fixture.database).await, before); + } +} diff --git a/src/internal/ai/memory/replay.rs b/src/internal/ai/memory/replay.rs new file mode 100644 index 000000000..8de31b03f --- /dev/null +++ b/src/internal/ai/memory/replay.rs @@ -0,0 +1,680 @@ +//! I/O-free reduction of validated Memory events into projection state. +//! +//! Object traversal and SQLite materialization deliberately live outside this +//! module. Rebuild, incremental replay, and the Writer companion all cross the +//! same reducer interface so lifecycle semantics cannot drift between paths. + +use std::collections::{BTreeMap, BTreeSet, HashSet}; + +use chrono::{DateTime, Utc}; +use git_internal::hash::ObjectHash; +use uuid::Uuid; + +use super::{ + domain::{MemoryEventAction, MemoryEventV1, MemoryNoteV1}, + error::{MemoryWriterError, MemoryWriterErrorKind}, + tree::parse_oid, +}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum ProjectedReviewState { + Draft, + Confirmed, + Quarantined, + Revoked, + Superseded, + Forgotten, +} + +#[derive(Clone)] +pub(super) struct ProjectedNote { + pub(super) latest_revision_oid: ObjectHash, + pub(super) live_revision_oid: Option, + pub(super) latest_action: MemoryEventAction, + pub(super) review_state: ProjectedReviewState, + pub(super) last_event_seq: u64, + pub(super) updated_at: DateTime, + pub(super) revisions: BTreeSet, +} + +#[derive(Clone, Default)] +pub(super) struct ReducedProjection { + pub(super) last_event_seq: u64, + pub(super) event_ids: HashSet, + pub(super) notes: BTreeMap, + pub(super) new_revisions: BTreeMap, + /// Revision OIDs in authoritative event order. The map above provides + /// lookup; this sequence prevents object-hash ordering from changing which + /// revision supplies the latest note-level projection values on rebuild. + pub(super) new_revision_order: Vec, + pub(super) created_notes: BTreeSet, + pub(super) changed_notes: BTreeSet, +} + +pub(super) struct ReplayRecord { + pub(super) event: MemoryEventV1, + /// The validated revision addressed by the event. Taxonomy events have no + /// revision; all note lifecycle events must carry one. + pub(super) revision_oid: Option, + pub(super) note: Option, +} + +impl ReducedProjection { + pub(super) fn apply(&mut self, record: ReplayRecord) -> Result<(), MemoryWriterError> { + let event_at = record.event.at; + let expected_seq = self + .last_event_seq + .checked_add(1) + .ok_or_else(|| corrupt("Memory projection event sequence overflowed"))?; + if record.event.event_seq != expected_seq { + return Err(corrupt( + "Memory projection event sequence is not contiguous", + )); + } + if !self.event_ids.insert(record.event.event_id) { + return Err(corrupt("Memory projection event ID is duplicated")); + } + + if record.event.action == MemoryEventAction::TaxonomyExpanded { + if record.revision_oid.is_some() || record.note.is_some() { + return Err(corrupt("taxonomy event unexpectedly addresses a revision")); + } + self.last_event_seq = expected_seq; + return Ok(()); + } + + let note_id = record + .event + .note_id + .ok_or_else(|| corrupt("Memory lifecycle event has no note ID"))?; + let revision_oid = record + .revision_oid + .ok_or_else(|| corrupt("Memory lifecycle event has no reachable revision"))?; + let note = record + .note + .ok_or_else(|| corrupt("Memory lifecycle event has no reachable note"))?; + let event_revision = record + .event + .revision_oid + .as_deref() + .ok_or_else(|| corrupt("Memory lifecycle event has no revision OID")) + .and_then(parse_oid)?; + if note_id != note.note_id || event_revision != revision_oid { + return Err(corrupt( + "Memory lifecycle event target disagrees with its revision", + )); + } + + match record.event.action { + MemoryEventAction::Created => { + if self.notes.contains_key(¬e_id) || !note.parents.is_empty() { + return Err(corrupt( + "Memory Created transition targets an existing note", + )); + } + let mut revisions = BTreeSet::new(); + revisions.insert(revision_oid.to_string()); + self.notes.insert( + note_id, + ProjectedNote { + latest_revision_oid: revision_oid, + live_revision_oid: None, + latest_action: MemoryEventAction::Created, + review_state: ProjectedReviewState::Draft, + last_event_seq: expected_seq, + updated_at: event_at, + revisions, + }, + ); + let revision_oid = revision_oid.to_string(); + self.new_revisions.insert(revision_oid.clone(), note); + self.new_revision_order.push(revision_oid); + self.created_notes.insert(note_id); + self.changed_notes.insert(note_id); + } + MemoryEventAction::Revised => { + let projected = self + .notes + .get_mut(¬e_id) + .ok_or_else(|| corrupt("Memory Revised transition targets an unknown note"))?; + if projected.revisions.contains(&revision_oid.to_string()) + || !note + .parents + .iter() + .any(|parent| parent == &projected.latest_revision_oid.to_string()) + { + return Err(corrupt( + "Memory Revised transition has invalid revision ancestry", + )); + } + projected.revisions.insert(revision_oid.to_string()); + projected.latest_revision_oid = revision_oid; + projected.latest_action = MemoryEventAction::Revised; + projected.review_state = ProjectedReviewState::Draft; + projected.last_event_seq = expected_seq; + projected.updated_at = event_at; + let revision_oid = revision_oid.to_string(); + self.new_revisions.insert(revision_oid.clone(), note); + self.new_revision_order.push(revision_oid); + self.changed_notes.insert(note_id); + } + MemoryEventAction::Confirmed => { + let projected = known_revision_mut(&mut self.notes, note_id, revision_oid)?; + if projected.latest_revision_oid != revision_oid + || !matches!( + projected.review_state, + ProjectedReviewState::Draft | ProjectedReviewState::Quarantined + ) + { + return Err(corrupt( + "Memory Confirmed transition targets a stale revision", + )); + } + projected.live_revision_oid = Some(revision_oid); + projected.latest_action = MemoryEventAction::Confirmed; + projected.review_state = ProjectedReviewState::Confirmed; + projected.last_event_seq = expected_seq; + projected.updated_at = event_at; + self.changed_notes.insert(note_id); + } + MemoryEventAction::Quarantined => { + let projected = known_revision_mut(&mut self.notes, note_id, revision_oid)?; + if projected.latest_revision_oid != revision_oid + || !matches!( + projected.review_state, + ProjectedReviewState::Draft | ProjectedReviewState::Confirmed + ) + { + return Err(corrupt("Memory Quarantined transition is invalid")); + } + if projected.live_revision_oid == Some(revision_oid) { + projected.live_revision_oid = None; + } + projected.latest_action = MemoryEventAction::Quarantined; + projected.review_state = ProjectedReviewState::Quarantined; + projected.last_event_seq = expected_seq; + projected.updated_at = event_at; + self.changed_notes.insert(note_id); + } + MemoryEventAction::Superseded => { + terminal_transition( + &mut self.notes, + note_id, + revision_oid, + TerminalRule::SUPERSEDED, + expected_seq, + event_at, + )?; + self.changed_notes.insert(note_id); + } + MemoryEventAction::Revoked => { + terminal_transition( + &mut self.notes, + note_id, + revision_oid, + TerminalRule::REVOKED, + expected_seq, + event_at, + )?; + self.changed_notes.insert(note_id); + } + MemoryEventAction::Forgotten => { + terminal_transition( + &mut self.notes, + note_id, + revision_oid, + TerminalRule::FORGOTTEN, + expected_seq, + event_at, + )?; + self.changed_notes.insert(note_id); + } + MemoryEventAction::Consolidated => { + let projected = known_revision_mut(&mut self.notes, note_id, revision_oid)?; + if projected.latest_revision_oid != revision_oid { + return Err(corrupt( + "Memory Consolidated event targets a stale revision", + )); + } + projected.latest_action = MemoryEventAction::Consolidated; + projected.last_event_seq = expected_seq; + projected.updated_at = event_at; + self.changed_notes.insert(note_id); + } + MemoryEventAction::TaxonomyExpanded => { + return Err(corrupt("taxonomy event reached note lifecycle reduction")); + } + } + self.last_event_seq = expected_seq; + Ok(()) + } +} + +fn known_revision_mut( + notes: &mut BTreeMap, + note_id: Uuid, + revision_oid: ObjectHash, +) -> Result<&mut ProjectedNote, MemoryWriterError> { + let projected = notes + .get_mut(¬e_id) + .ok_or_else(|| corrupt("Memory transition targets an unknown note"))?; + if !projected.revisions.contains(&revision_oid.to_string()) { + return Err(corrupt("Memory transition targets an unknown revision")); + } + Ok(projected) +} + +#[derive(Clone, Copy)] +struct TerminalRule { + action: MemoryEventAction, + state: ProjectedReviewState, + allowed_from: &'static [ProjectedReviewState], +} + +impl TerminalRule { + const SUPERSEDED: Self = Self { + action: MemoryEventAction::Superseded, + state: ProjectedReviewState::Superseded, + allowed_from: &[ + ProjectedReviewState::Confirmed, + ProjectedReviewState::Quarantined, + ], + }; + const REVOKED: Self = Self { + action: MemoryEventAction::Revoked, + state: ProjectedReviewState::Revoked, + allowed_from: &[ + ProjectedReviewState::Draft, + ProjectedReviewState::Confirmed, + ProjectedReviewState::Quarantined, + ], + }; + const FORGOTTEN: Self = Self { + action: MemoryEventAction::Forgotten, + state: ProjectedReviewState::Forgotten, + allowed_from: &[ + ProjectedReviewState::Draft, + ProjectedReviewState::Confirmed, + ProjectedReviewState::Quarantined, + ], + }; +} + +fn terminal_transition( + notes: &mut BTreeMap, + note_id: Uuid, + revision_oid: ObjectHash, + rule: TerminalRule, + event_seq: u64, + event_at: DateTime, +) -> Result<(), MemoryWriterError> { + let projected = known_revision_mut(notes, note_id, revision_oid)?; + if projected.latest_revision_oid != revision_oid + || !rule.allowed_from.contains(&projected.review_state) + { + return Err(corrupt("Memory terminal transition is invalid")); + } + if projected.live_revision_oid == Some(revision_oid) { + projected.live_revision_oid = None; + } + projected.latest_action = rule.action; + projected.review_state = rule.state; + projected.last_event_seq = event_seq; + projected.updated_at = event_at; + Ok(()) +} + +fn corrupt(summary: &'static str) -> MemoryWriterError { + MemoryWriterError::new(MemoryWriterErrorKind::CorruptHistory, summary) +} + +#[cfg(test)] +mod tests { + use chrono::{TimeZone, Utc}; + use git_internal::internal::object::types::ObjectType; + + use super::*; + use crate::internal::ai::{ + context_budget::MemoryAnchorConfidence, + memory::domain::{ + ActorKind, ActorRefV1, CompileOriginV1, CompileRecordV1, IdempotencyScopeV1, + MemoryKind, MemoryLifecycle, MemoryScopeV1, MemorySensitivity, MemoryTrust, + MemoryVisibility, + }, + }; + + fn oid(seed: &[u8]) -> ObjectHash { + ObjectHash::from_type_and_data(ObjectType::Blob, seed) + } + + fn note(note_id: Uuid, parents: Vec, label: &str) -> MemoryNoteV1 { + MemoryNoteV1 { + schema_version: 1, + note_id, + content_digest: format!("sha256:{}", "0".repeat(64)), + namespace: "default".to_string(), + path: "episodic.tasks.r-test".to_string(), + kind: MemoryKind::Episodic, + scope: MemoryScopeV1::Repo, + visibility: MemoryVisibility::RepoLocal, + acl_policy_id: "repo-default-v1".to_string(), + lifecycle: MemoryLifecycle::Accretive, + body: label.to_string(), + rationale: None, + episode: None, + evidence_refs: Vec::new(), + links: Vec::new(), + entities: Vec::new(), + parents, + tags: Vec::new(), + confidence: MemoryAnchorConfidence::High, + trust: MemoryTrust::RepoEvidence, + sensitivity: MemorySensitivity::Internal, + valid_from: None, + valid_until: None, + effective_from_commit: None, + effective_until_commit: None, + expires_at: None, + author: ActorRefV1 { + kind: ActorKind::Agent, + principal_id: "agent:test".to_string(), + }, + created_at: Utc + .with_ymd_and_hms(2026, 8, 25, 0, 0, 0) + .single() + .expect("valid fixture time"), + compile_record: CompileRecordV1 { + schema_version: 1, + origin: CompileOriginV1::EpisodeCompiler, + producer: "test".to_string(), + rules_version: 1, + prompt_version: None, + model_id: None, + policy_version: "repo-policy-v1".to_string(), + input_hashes: Vec::new(), + idempotency_key: label.to_string(), + idempotency_scope: IdempotencyScopeV1::Cell, + }, + } + } + + fn record( + event_id: Uuid, + event_seq: u64, + action: MemoryEventAction, + note: MemoryNoteV1, + revision_oid: ObjectHash, + ) -> ReplayRecord { + ReplayRecord { + event: MemoryEventV1 { + schema_version: 1, + event_id, + event_seq, + note_id: Some(note.note_id), + revision_oid: Some(revision_oid.to_string()), + namespace: None, + target_path: None, + action, + reason_code: Some("test".to_string()), + actor: note.author.clone(), + at: note.created_at, + evidence_refs: Vec::new(), + next_note_id: None, + }, + revision_oid: Some(revision_oid), + note: Some(note), + } + } + + fn event_id(seq: u128) -> Uuid { + Uuid::from_u128(seq) + } + + fn confirmed_projection() -> (ReducedProjection, Uuid, ObjectHash, MemoryNoteV1) { + let note_id = Uuid::from_u128(100); + let revision_oid = oid(b"revision-1"); + let note = note(note_id, Vec::new(), "revision-1"); + let mut projection = ReducedProjection::default(); + projection + .apply(record( + event_id(1), + 1, + MemoryEventAction::Created, + note.clone(), + revision_oid, + )) + .expect("create note"); + projection + .apply(record( + event_id(2), + 2, + MemoryEventAction::Confirmed, + note.clone(), + revision_oid, + )) + .expect("confirm note"); + (projection, note_id, revision_oid, note) + } + + #[test] + fn reducer_covers_revision_review_and_terminal_transitions() { + let (mut projection, note_id, first_oid, first_note) = confirmed_projection(); + let second_oid = oid(b"revision-2"); + let second_note = note(note_id, vec![first_oid.to_string()], "revision-2"); + projection + .apply(record( + event_id(3), + 3, + MemoryEventAction::Revised, + second_note.clone(), + second_oid, + )) + .expect("revise note"); + let revised = projection.notes.get(¬e_id).expect("projected note"); + assert_eq!(revised.review_state, ProjectedReviewState::Draft); + assert_eq!(revised.live_revision_oid, Some(first_oid)); + + projection + .apply(record( + event_id(4), + 4, + MemoryEventAction::Quarantined, + second_note.clone(), + second_oid, + )) + .expect("quarantine draft"); + assert_eq!( + projection + .notes + .get(¬e_id) + .expect("projected note") + .live_revision_oid, + Some(first_oid), + ); + + projection + .apply(record( + event_id(5), + 5, + MemoryEventAction::Confirmed, + second_note.clone(), + second_oid, + )) + .expect("confirm second revision"); + projection + .apply(record( + event_id(6), + 6, + MemoryEventAction::Consolidated, + second_note.clone(), + second_oid, + )) + .expect("annotate consolidation"); + assert_eq!( + projection + .notes + .get(¬e_id) + .expect("projected note") + .review_state, + ProjectedReviewState::Confirmed, + ); + + projection + .apply(record( + event_id(7), + 7, + MemoryEventAction::Revoked, + second_note, + second_oid, + )) + .expect("revoke live revision"); + let revoked = projection.notes.get(¬e_id).expect("projected note"); + assert_eq!(revoked.review_state, ProjectedReviewState::Revoked); + assert_eq!(revoked.live_revision_oid, None); + + projection + .apply(ReplayRecord { + event: MemoryEventV1 { + schema_version: 1, + event_id: event_id(8), + event_seq: 8, + note_id: None, + revision_oid: None, + namespace: Some("default".to_string()), + target_path: Some("episodic".to_string()), + action: MemoryEventAction::TaxonomyExpanded, + reason_code: Some("test".to_string()), + actor: first_note.author, + at: first_note.created_at, + evidence_refs: Vec::new(), + next_note_id: None, + }, + revision_oid: None, + note: None, + }) + .expect("taxonomy annotation"); + assert_eq!(projection.last_event_seq, 8); + } + + #[test] + fn reducer_covers_superseded_and_forgotten_terminal_states() { + for (action, expected) in [ + ( + MemoryEventAction::Superseded, + ProjectedReviewState::Superseded, + ), + ( + MemoryEventAction::Forgotten, + ProjectedReviewState::Forgotten, + ), + ] { + let (mut projection, note_id, revision_oid, note) = confirmed_projection(); + projection + .apply(record(event_id(3), 3, action, note, revision_oid)) + .expect("apply terminal transition"); + let projected = projection.notes.get(¬e_id).expect("projected note"); + assert_eq!(projected.review_state, expected); + assert_eq!(projected.live_revision_oid, None); + } + } + + #[test] + fn reducer_rejects_gap_duplicate_and_unknown_revision() { + let note_id = Uuid::from_u128(200); + let revision_oid = oid(b"invalid-revision"); + let note = note(note_id, Vec::new(), "invalid"); + let mut projection = ReducedProjection::default(); + assert!( + projection + .apply(record( + event_id(1), + 2, + MemoryEventAction::Created, + note.clone(), + revision_oid, + )) + .is_err(), + ); + assert!( + projection + .apply(record( + event_id(2), + 1, + MemoryEventAction::Confirmed, + note.clone(), + revision_oid, + )) + .is_err(), + ); + projection + .apply(record( + event_id(3), + 1, + MemoryEventAction::Created, + note.clone(), + revision_oid, + )) + .expect("valid create"); + assert!( + projection + .apply(record( + event_id(3), + 2, + MemoryEventAction::Confirmed, + note, + revision_oid, + )) + .is_err(), + ); + } + + #[test] + fn reducer_rejects_terminal_resurrection_and_invalid_supersede() { + let (mut revoked, _note_id, revision_oid, confirmed_note) = confirmed_projection(); + revoked + .apply(record( + event_id(3), + 3, + MemoryEventAction::Revoked, + confirmed_note.clone(), + revision_oid, + )) + .expect("revoke confirmed revision"); + assert!( + revoked + .apply(record( + event_id(4), + 4, + MemoryEventAction::Confirmed, + confirmed_note, + revision_oid, + )) + .is_err(), + ); + + let note_id = Uuid::from_u128(300); + let draft_oid = oid(b"draft-supersede"); + let draft = note(note_id, Vec::new(), "draft-supersede"); + let mut projection = ReducedProjection::default(); + projection + .apply(record( + event_id(10), + 1, + MemoryEventAction::Created, + draft.clone(), + draft_oid, + )) + .expect("create draft"); + assert!( + projection + .apply(record( + event_id(11), + 2, + MemoryEventAction::Superseded, + draft, + draft_oid, + )) + .is_err(), + ); + } +} diff --git a/src/internal/ai/memory/store.rs b/src/internal/ai/memory/store.rs index 1aa98b14f..6c442bdbe 100644 --- a/src/internal/ai/memory/store.rs +++ b/src/internal/ai/memory/store.rs @@ -1,4 +1,4 @@ -use std::sync::Arc; +use std::{collections::BTreeSet, sync::Arc}; use anyhow::{Context, Result}; use async_trait::async_trait; @@ -9,10 +9,15 @@ use serde::Serialize; use super::{ domain::{MemoryEventV1, MemoryNoteV1}, error::{MemoryWriterError, MemoryWriterErrorKind}, + projection::materialize_linear, + replay::{ProjectedNote, ProjectedReviewState, ReducedProjection, ReplayRecord}, tree::parse_oid, }; use crate::internal::{ - ai::{keyed_digest::RepositoryKeyedDigest, linear_ref::LinearRefCompanion}, + ai::{ + keyed_digest::RepositoryKeyedDigest, + linear_ref::{LinearRefCompanion, LinearRefWriteTransaction}, + }, workspace::RepoIdentity, }; @@ -145,10 +150,10 @@ fn projected_cell_from_row(row: QueryResult) -> Result, pub(super) expected_event_seq: u64, @@ -159,22 +164,47 @@ pub(super) struct ProjectionMutation { #[async_trait] impl LinearRefCompanion for ProjectionMutation { - async fn apply(&self, txn: &DatabaseTransaction) -> Result<()> { - revalidate_snapshot(txn, self).await?; - if self.is_create { - insert_note(txn, &self.note).await?; - } else { - update_note(txn, &self.note).await?; + async fn apply(&self, txn: &LinearRefWriteTransaction<'_>) -> Result<()> { + revalidate_snapshot(txn.as_database_transaction(), self).await?; + let mut reduced = ReducedProjection { + last_event_seq: self.expected_event_seq, + ..ReducedProjection::default() + }; + if let Some(cell) = &self.expected_cell { + let mut revisions = BTreeSet::new(); + revisions.insert(cell.latest_revision_oid.to_string()); + reduced.notes.insert( + self.note.note_id, + ProjectedNote { + latest_revision_oid: cell.latest_revision_oid, + live_revision_oid: cell + .live_revision_oid + .as_deref() + .map(parse_oid) + .transpose()?, + latest_action: super::domain::MemoryEventAction::Confirmed, + review_state: ProjectedReviewState::Confirmed, + last_event_seq: self.expected_event_seq, + updated_at: self.transition.at, + revisions, + }, + ); } - insert_revision(txn, &self.note, self.revision_oid).await?; - replace_links(txn, &self.note, self.revision_oid).await?; - replace_episode_paths(txn, &self.note, self.revision_oid).await?; - upsert_head(txn, &self.note, &self.event, self.revision_oid).await?; - upsert_projection_state( + reduced.apply(ReplayRecord { + event: self.transition.clone(), + revision_oid: Some(self.revision_oid), + note: Some(self.note.clone()), + })?; + reduced.apply(ReplayRecord { + event: self.event.clone(), + revision_oid: Some(self.revision_oid), + note: Some(self.note.clone()), + })?; + materialize_linear( txn, - &self.note, - &self.event, + &reduced, self.commit_oid, + &self.note.compile_record.policy_version, self.rebuilt_at_ms, ) .await?; @@ -272,7 +302,7 @@ async fn find_cell_in_transaction( decode_unique_cell(rows) } -async fn insert_note(txn: &DatabaseTransaction, note: &MemoryNoteV1) -> Result<()> { +pub(super) async fn insert_note(txn: &DatabaseTransaction, note: &MemoryNoteV1) -> Result<()> { execute( txn, "INSERT INTO memory_note_index ( @@ -300,7 +330,7 @@ async fn insert_note(txn: &DatabaseTransaction, note: &MemoryNoteV1) -> Result<( .await } -async fn update_note(txn: &DatabaseTransaction, note: &MemoryNoteV1) -> Result<()> { +pub(super) async fn update_note(txn: &DatabaseTransaction, note: &MemoryNoteV1) -> Result<()> { let result = txn .execute_raw(Statement::from_sql_and_values( txn.get_database_backend(), @@ -327,7 +357,7 @@ async fn update_note(txn: &DatabaseTransaction, note: &MemoryNoteV1) -> Result<( Ok(()) } -async fn insert_revision( +pub(super) async fn insert_revision( txn: &DatabaseTransaction, note: &MemoryNoteV1, revision_oid: ObjectHash, @@ -356,7 +386,7 @@ async fn insert_revision( .await } -async fn replace_links( +pub(super) async fn replace_links( txn: &DatabaseTransaction, note: &MemoryNoteV1, revision_oid: ObjectHash, @@ -398,7 +428,7 @@ async fn replace_links( Ok(()) } -async fn replace_episode_paths( +pub(super) async fn replace_episode_paths( txn: &DatabaseTransaction, note: &MemoryNoteV1, revision_oid: ObjectHash, @@ -421,90 +451,11 @@ async fn replace_episode_paths( Ok(()) } -async fn upsert_head( - txn: &DatabaseTransaction, - note: &MemoryNoteV1, - event: &MemoryEventV1, - revision_oid: ObjectHash, -) -> Result<()> { - execute( - txn, - "INSERT INTO memory_head ( - scope_key, namespace, path, note_id, latest_revision_oid, - live_revision_oid, latest_action, latest_review_state, kind, - lifecycle, confidence, trust, sensitivity, visibility, acl_policy_id, - valid_from, valid_until, effective_from_commit, effective_until_commit, - expires_at, rank_hint, last_event_seq, updated_at - ) VALUES ('repo', ?, ?, ?, ?, ?, ?, 'confirmed', ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 0, ?, ?) - ON CONFLICT(scope_key, namespace, path, note_id) DO UPDATE SET - latest_revision_oid = excluded.latest_revision_oid, - live_revision_oid = excluded.live_revision_oid, - latest_action = excluded.latest_action, - latest_review_state = excluded.latest_review_state, - confidence = excluded.confidence, - trust = excluded.trust, - sensitivity = excluded.sensitivity, - visibility = excluded.visibility, - acl_policy_id = excluded.acl_policy_id, - effective_from_commit = excluded.effective_from_commit, - last_event_seq = excluded.last_event_seq, - updated_at = excluded.updated_at", - vec![ - note.namespace.clone().into(), - note.path.clone().into(), - note.note_id.to_string().into(), - revision_oid.to_string().into(), - revision_oid.to_string().into(), - enum_label(&event.action)?.into(), - enum_label(¬e.kind)?.into(), - enum_label(¬e.lifecycle)?.into(), - enum_label(¬e.confidence)?.into(), - enum_label(¬e.trust)?.into(), - enum_label(¬e.sensitivity)?.into(), - enum_label(¬e.visibility)?.into(), - note.acl_policy_id.clone().into(), - note.valid_from.map(|value| value.to_rfc3339()).into(), - note.valid_until.map(|value| value.to_rfc3339()).into(), - note.effective_from_commit.clone().into(), - note.effective_until_commit.clone().into(), - note.expires_at.map(|value| value.to_rfc3339()).into(), - i64::try_from(event.event_seq)?.into(), - event.at.to_rfc3339().into(), - ], - ) - .await -} - -async fn upsert_projection_state( +pub(super) async fn execute( txn: &DatabaseTransaction, - note: &MemoryNoteV1, - event: &MemoryEventV1, - commit_oid: ObjectHash, - rebuilt_at_ms: i64, + sql: &str, + values: Vec, ) -> Result<()> { - execute( - txn, - "INSERT INTO memory_projection_state ( - scope_key, projected_ref_oid, last_event_seq, schema_version, - policy_version, rebuilt_at - ) VALUES ('repo', ?, ?, 1, ?, ?) - ON CONFLICT(scope_key) DO UPDATE SET - projected_ref_oid = excluded.projected_ref_oid, - last_event_seq = excluded.last_event_seq, - schema_version = excluded.schema_version, - policy_version = excluded.policy_version, - rebuilt_at = excluded.rebuilt_at", - vec![ - commit_oid.to_string().into(), - i64::try_from(event.event_seq)?.into(), - note.compile_record.policy_version.clone().into(), - rebuilt_at_ms.into(), - ], - ) - .await -} - -async fn execute(txn: &DatabaseTransaction, sql: &str, values: Vec) -> Result<()> { txn.execute_raw(Statement::from_sql_and_values( txn.get_database_backend(), sql, @@ -514,7 +465,7 @@ async fn execute(txn: &DatabaseTransaction, sql: &str, values: Vec(value: &T) -> Result { +pub(super) fn enum_label(value: &T) -> Result { let value = serde_json::to_value(value).context("serialize Memory enum")?; value .as_str() diff --git a/src/internal/ai/memory/tree.rs b/src/internal/ai/memory/tree.rs index b4448cf41..d19721276 100644 --- a/src/internal/ai/memory/tree.rs +++ b/src/internal/ai/memory/tree.rs @@ -92,6 +92,18 @@ pub(super) struct MemoryCommitObjects { pub(super) commit_oid: ObjectHash, } +#[derive(Clone)] +pub(super) struct MemoryHistoryRecord { + pub(super) event: super::domain::MemoryEventV1, + pub(super) revision_oid: Option, + pub(super) note: Option, +} + +pub(super) struct MemoryHistoryDelta { + pub(super) manifest: MemoryManifestV1, + pub(super) records: Vec, +} + pub(super) struct MemoryCommitInput<'a> { pub(super) note_id: &'a str, pub(super) namespace: &'a str, @@ -144,7 +156,7 @@ pub(super) fn load_snapshot( "Memory commit parent count does not match its event sequence", )); } - validate_append_edge(storage_path, &commit, &root_items, &manifest)?; + let _ = validate_append_edge(storage_path, &commit, &root_items, &manifest)?; if manifest.policy_version != policy_version { return Err(MemoryWriterError::new( MemoryWriterErrorKind::PolicyRejected, @@ -162,11 +174,11 @@ fn validate_append_edge( commit: &Commit, root_items: &[TreeItem], manifest: &MemoryManifestV1, -) -> Result<(), MemoryWriterError> { +) -> Result, MemoryWriterError> { validate_root_shape(root_items)?; let (parent_items, parent_manifest) = match commit.parent_commit_ids.as_slice() { [] => { - if manifest.commit_count != 1 || manifest.last_event_seq != 2 { + if manifest.commit_count != 1 || manifest.last_event_seq == 0 { return Err(corrupt( "Memory root commit has an invalid manifest sequence", )); @@ -181,7 +193,7 @@ fn validate_append_edge( let expected_parent_count = usize::from(parent_manifest.commit_count > 1); if parent.parent_commit_ids.len() != expected_parent_count || parent_manifest.commit_count.checked_add(1) != Some(manifest.commit_count) - || parent_manifest.last_event_seq.checked_add(2) != Some(manifest.last_event_seq) + || parent_manifest.last_event_seq >= manifest.last_event_seq || parent_manifest.scope_key != manifest.scope_key || parent_manifest.policy_version != manifest.policy_version || parent_manifest.policy_snapshot_digest != manifest.policy_snapshot_digest @@ -207,6 +219,77 @@ fn validate_append_edge( ) } +/// Read and validate the first-parent suffix ending at `head`. +/// +/// `after` is the already-projected ancestor and is excluded from the result. +/// Supplying a non-ancestor fails closed rather than silently rebuilding from +/// an unrelated history. +pub(super) fn load_history_delta( + storage_path: &Path, + head: ObjectHash, + after: Option, + policy_version: &str, +) -> Result { + const MAX_REPLAY_COMMITS: usize = 4096; + + if Some(head) == after { + let snapshot = load_snapshot(storage_path, Some(head), policy_version)?; + return Ok(MemoryHistoryDelta { + manifest: snapshot.manifest, + records: Vec::new(), + }); + } + + let mut suffix = Vec::new(); + let mut cursor = head; + loop { + if Some(cursor) == after { + break; + } + if suffix.len() == MAX_REPLAY_COMMITS { + return Err(corrupt("Memory replay exceeds the commit budget")); + } + let commit = load_commit(storage_path, cursor)?; + if commit.parent_commit_ids.len() > 1 { + return Err(corrupt("Memory history contains a merge commit")); + } + let root_items = load_tree(storage_path, commit.tree_id)?; + let manifest = load_manifest(storage_path, &root_items, "replay")?; + if manifest.policy_version != policy_version { + return Err(MemoryWriterError::new( + MemoryWriterErrorKind::PolicyRejected, + "Memory replay policy does not match the authoritative history", + )); + } + let parent = commit.parent_commit_ids.first().copied(); + suffix.push((cursor, commit, root_items, manifest)); + match parent { + Some(parent) => cursor = parent, + None if after.is_none() => break, + None => { + return Err(corrupt( + "Memory projection watermark is not an ancestor of the pinned head", + )); + } + } + } + suffix.reverse(); + let manifest = suffix + .last() + .map(|(_, _, _, manifest)| manifest.clone()) + .ok_or_else(|| corrupt("Memory replay suffix is empty"))?; + let mut records = Vec::new(); + for (_, commit, root_items, commit_manifest) in &suffix { + records.extend(validate_append_edge( + storage_path, + commit, + root_items, + commit_manifest, + )?); + } + Ok(MemoryHistoryDelta { manifest, records }) +} + fn validate_root_shape(items: &[TreeItem]) -> Result<(), MemoryWriterError> { if items.len() != 3 || items.iter().any(|item| { @@ -443,7 +526,7 @@ fn validate_event_edge( root_items: &[TreeItem], previous_seq: u64, last_event_seq: u64, -) -> Result<(), MemoryWriterError> { +) -> Result, MemoryWriterError> { let events = tree_entries(storage_path, root_items, "events")?; if events.len() > MAX_MEMORY_TREE_ENTRIES { return Err(corrupt( @@ -460,7 +543,7 @@ fn validate_event_edge( None => Vec::new(), }; if u64::try_from(parent_events.len()).ok() != Some(previous_seq) - || events.len().checked_sub(parent_events.len()) != Some(2) + || events.len() <= parent_events.len() || !events.starts_with(&parent_events) { return Err(corrupt("Memory event tree did not append to its parent")); @@ -486,17 +569,17 @@ fn validate_event_edge( None => NoteTreeIndex::default(), }; let notes = note_blob_map(storage_path, root_items)?; - if notes.blobs.len().checked_sub(parent_notes.blobs.len()) != Some(1) - || parent_notes - .blobs - .iter() - .any(|(path, oid)| notes.blobs.get(path) != Some(oid)) + if parent_notes + .blobs + .iter() + .any(|(path, oid)| notes.blobs.get(path) != Some(oid)) { return Err(corrupt("Memory note tree did not append to its parent")); } let tail = &events[parent_events.len()..]; - let mut parsed = Vec::with_capacity(tail.len()); + let mut revision_events = BTreeMap::new(); + let mut records = Vec::with_capacity(tail.len()); for (offset, item) in tail.iter().enumerate() { let sequence = previous_seq .checked_add(u64::try_from(offset).map_err(|_| corrupt("Memory event overflowed"))?) @@ -520,71 +603,95 @@ fn validate_event_edge( "Memory event filename and payload identity disagree", )); } - let note_id = event - .note_id - .ok_or_else(|| corrupt("Memory revision event has no note ID"))?; - let revision_oid = event - .revision_oid - .as_deref() - .ok_or_else(|| corrupt("Memory revision event has no revision OID")) - .and_then(parse_oid)?; - let note_bytes = load_note_bytes(storage_path, revision_oid)?; - let note = parse_memory_note_v1(¬e_bytes).map_err(|error| { - MemoryWriterError::new( - MemoryWriterErrorKind::CorruptHistory, - format!("event references an invalid MemoryNote revision: {error}"), - ) - })?; - let expected_path = format!( - "{}/{}/{}.json", - encode_segment(¬e.namespace), - note.note_id, - revision_oid - ); - if note.note_id != note_id || notes.blobs.get(&expected_path) != Some(&revision_oid) { - return Err(corrupt( - "MemoryEvent revision is not reachable from its canonical note path", - )); - } - parsed.push(event); - } - let tail_note = parse_memory_note_v1(&load_note_bytes( - storage_path, - parse_oid( - parsed[0] + if event.action != super::domain::MemoryEventAction::TaxonomyExpanded { + let note_id = event + .note_id + .ok_or_else(|| corrupt("Memory revision event has no note ID"))?; + let revision_oid = event .revision_oid .as_deref() - .ok_or_else(|| corrupt("Memory revision event has no revision OID"))?, - )?, - )?) - .map_err(|error| { - MemoryWriterError::new( - MemoryWriterErrorKind::CorruptHistory, - format!("event references an invalid MemoryNote revision: {error}"), - ) - })?; - let parent_note_prefix = format!( - "{}/{}/", - encode_segment(&tail_note.namespace), - tail_note.note_id - ); - let expected_first = if parent_notes + .ok_or_else(|| corrupt("Memory revision event has no revision OID")) + .and_then(parse_oid)?; + let note_bytes = load_note_bytes(storage_path, revision_oid)?; + let note = parse_memory_note_v1(¬e_bytes).map_err(|error| { + MemoryWriterError::new( + MemoryWriterErrorKind::CorruptHistory, + format!("event references an invalid MemoryNote revision: {error}"), + ) + })?; + let expected_path = format!( + "{}/{}/{}.json", + encode_segment(¬e.namespace), + note.note_id, + revision_oid + ); + if note.note_id != note_id || notes.blobs.get(&expected_path) != Some(&revision_oid) { + return Err(corrupt( + "MemoryEvent revision is not reachable from its canonical note path", + )); + } + if matches!( + event.action, + super::domain::MemoryEventAction::Created + | super::domain::MemoryEventAction::Revised + ) && revision_events + .insert(expected_path, event.action) + .is_some() + { + return Err(corrupt( + "Memory revision is introduced by more than one transition", + )); + } + records.push(MemoryHistoryRecord { + event, + revision_oid: Some(revision_oid), + note: Some(note), + }); + } else { + records.push(MemoryHistoryRecord { + event, + revision_oid: None, + note: None, + }); + } + } + + let added_notes = notes .blobs .keys() - .any(|path| path.starts_with(&parent_note_prefix)) - { - super::domain::MemoryEventAction::Revised - } else { - super::domain::MemoryEventAction::Created - }; - if parsed[0].action != expected_first - || parsed[1].action != super::domain::MemoryEventAction::Confirmed - || parsed[0].note_id != parsed[1].note_id - || parsed[0].revision_oid != parsed[1].revision_oid + .filter(|path| !parent_notes.blobs.contains_key(*path)) + .cloned() + .collect::>(); + if added_notes.len() != revision_events.len() + || added_notes + .iter() + .any(|path| !revision_events.contains_key(path)) { - return Err(corrupt("Memory commit event pair is invalid")); + return Err(corrupt( + "Memory note tree additions do not match revision transitions", + )); } - Ok(()) + for (path, action) in revision_events { + let note_prefix = path + .rsplit_once('/') + .map(|(prefix, _)| format!("{prefix}/")) + .ok_or_else(|| corrupt("Memory note path is not canonical"))?; + let existed = parent_notes + .blobs + .keys() + .any(|candidate| candidate.starts_with(¬e_prefix)); + let expected = if existed { + super::domain::MemoryEventAction::Revised + } else { + super::domain::MemoryEventAction::Created + }; + if action != expected { + return Err(corrupt( + "Memory revision transition disagrees with note ancestry", + )); + } + } + Ok(records) } fn tree_entries( diff --git a/src/internal/ai/memory/writer.rs b/src/internal/ai/memory/writer.rs index 4408da13d..7d3a50094 100644 --- a/src/internal/ai/memory/writer.rs +++ b/src/internal/ai/memory/writer.rs @@ -340,10 +340,10 @@ impl MemoryWriter { let mutation = ProjectionMutation { note: note.clone(), + transition: transition.clone(), event: confirmed.clone(), revision_oid, commit_oid: objects.commit_oid, - is_create: cell.is_none(), rebuilt_at_ms: Utc::now().timestamp_millis(), expected_head: current_head, expected_event_seq: base_event_seq, @@ -497,7 +497,7 @@ fn event_id(note: &MemoryNoteV1, revision_oid: ObjectHash, action: MemoryEventAc } #[cfg(test)] -mod tests { +pub(in crate::internal::ai::memory) mod tests { use std::{fs, sync::Arc}; use chrono::{TimeZone, Utc}; @@ -535,16 +535,16 @@ mod tests { const TEST_CIPHERTEXT: &str = "memory-writer-test-ciphertext"; const SOURCE_OID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; - struct Fixture { - _temp: tempfile::TempDir, - database: Arc, - writer: Arc, - context: AuthenticatedMemoryContext, - target: TrustedMemoryTarget, - key_id: Uuid, + pub(in crate::internal::ai::memory) struct Fixture { + pub(in crate::internal::ai::memory) _temp: tempfile::TempDir, + pub(in crate::internal::ai::memory) database: Arc, + pub(in crate::internal::ai::memory) writer: Arc, + pub(in crate::internal::ai::memory) context: AuthenticatedMemoryContext, + pub(in crate::internal::ai::memory) target: TrustedMemoryTarget, + pub(in crate::internal::ai::memory) key_id: Uuid, } - async fn fixture() -> Fixture { + pub(in crate::internal::ai::memory) async fn fixture() -> Fixture { let database = Database::connect("sqlite::memory:") .await .expect("connect test database"); @@ -607,7 +607,7 @@ mod tests { } } - fn proposal( + pub(in crate::internal::ai::memory) fn proposal( target: &TrustedMemoryTarget, key_id: Uuid, generation: u8, From 460db36d79f0a3223487494e4eae38ca4b3427cb Mon Sep 17 00:00:00 2001 From: anduin9527 Date: Tue, 25 Aug 2026 09:52:34 +0800 Subject: [PATCH 10/18] feat(memory): add bounded episode source admission Signed-off-by: anduin9527 --- docs/development/tracing/memory.md | 23 + docs/error-codes.md | 12 +- src/internal/ai/history.rs | 293 +++++ src/internal/ai/keyed_digest.rs | 4 + src/internal/ai/memory/admission.rs | 313 +++++ src/internal/ai/memory/compiler/mod.rs | 115 ++ src/internal/ai/memory/error.rs | 34 +- src/internal/ai/memory/limits.rs | 48 + src/internal/ai/memory/mod.rs | 4 + src/internal/ai/memory/policy.rs | 6 +- src/internal/ai/memory/source.rs | 1572 ++++++++++++++++++++++++ src/internal/ai/memory/writer.rs | 71 +- 12 files changed, 2482 insertions(+), 13 deletions(-) create mode 100644 src/internal/ai/memory/admission.rs create mode 100644 src/internal/ai/memory/compiler/mod.rs create mode 100644 src/internal/ai/memory/limits.rs create mode 100644 src/internal/ai/memory/source.rs diff --git a/docs/development/tracing/memory.md b/docs/development/tracing/memory.md index 4a3aa0f59..cd1b9ae6c 100644 --- a/docs/development/tracing/memory.md +++ b/docs/development/tracing/memory.md @@ -626,6 +626,29 @@ M2 从现有 Intent / Task / Run / Evidence / Decision / PatchSet / Session 与 | `decisions` / `failed_attempts` / `unresolved` | compiler proposal | 每项显式标记 observation 或 inference，并带 EvidenceRef | | `omissions` | Writer | 记录每个有界集合被裁掉的条目数 | +Episode 编译前先由 `EpisodeSourceResolver` 固定一份「研发历程来源窗口」 +（Episode source window）。它只接收已认证的仓库与 Agent 身份、受信任的 +Task/Intent 根、AI 历史固定版本 OID 和仓库冻结限制，不接收请求体自报的用户名、 +目录或读取范围。解析器从该固定版本验证根与关系，读取相关 Run、终态事件、Evidence、 +Decision、PatchSet、ToolInvocation 与有明确关联的 ContextFrame。SQLite 反向索引只能 +提示候选 ID，任何候选仍须在固定版本树中重新验证；无法建立明确关系的整段 Session +不会被拼入模型输入。 + +来源窗口按以下顺序处理：读取原始对象 → secret 规则 → `` 标记规则 → +高置信度邮箱与用户主目录规则 → 构造不可由外部代码创建的 +`RedactedEpisodeSource`。默认限制为 256 个入选对象、4096 个候选对象、单对象 +128 KiB、树对象 4 MiB、脱敏正文合计 2 MiB、64 个 ContextFrame 片段、约 +512K tokens 和 2048 个 AI 历史祖先版本。可省略的超限项以稳定 reason code 写入 +omissions；根对象和终态事实若无法在限制内完整取得则整次解析失败，不生成不完整记忆。 + +每次解析都会产生一份规范来源清单（source manifest），保存根类型/ID、仓库 ID、 +主体 HMAC、固定来源 OID、上述限制、策略/脱敏版本、计数、omissions，以及每个实际输入 +片段的对象类型、对象 ID/OID、类型化 locator、脱敏片段 SHA-256、可选代码版本。 +清单不保存原始正文、主体明文或 secret。模型只返回引用 `fragment_id` 的自然语言提案； +准入层把它机械映射为 `EvidenceRefV1`，拒绝模型虚构的定位器。Writer 写入前使用同一身份、 +固定版本和限制重新解析并逐项比较清单、事实和脱敏片段；任何漂移、越权或摘要不一致均不 +推进 Memory ref。 + `EpisodeClaimV1` 固定为 `epistemic_status + claim + confidence? + evidence_refs`。Observation 禁止携带 confidence；Inference 必须携带 confidence。校验只能证明引用存在、可见且定位/digest 匹配，不能把自然语言蕴含声明为形式化证明。 首个仓库切片固定 `kind=Episodic`、`scope=Repo`、`visibility=RepoLocal`、`lifecycle=Accretive`、`namespace=default`，并使用： diff --git a/docs/error-codes.md b/docs/error-codes.md index 4b1fbbf88..f533f1805 100644 --- a/docs/error-codes.md +++ b/docs/error-codes.md @@ -89,9 +89,9 @@ structured report is always present. | `128` | `LBR-REPO-002` | `repo` | Repository metadata is corrupt or incompatible | missing DB, corrupted metadata | | `128` | `LBR-REPO-003` | `repo` | Repository state blocks the operation | no commits yet, detached state mismatch, missing configured remote | | `128` | `LBR-MEMORY-001` | `repo` | Repository Memory digest key is missing, invalid, or cannot be decrypted | missing encrypted `memory.keyed_digest.v1`, duplicate/plaintext entry, unsupported generation, unavailable repository vault key, or cached/persisted key mismatch | -| `128` | `LBR-MEMORY-002` | `repo` | Memory proposal violates the persisted object contract | unsupported schema, malformed Episode envelope, or non-canonical note/event payload | -| `128` | `LBR-MEMORY-003` | `repo` | Memory writer policy rejected the proposal | authenticated target mismatch, non-local Memory scope, or unknown repository digest key ID | -| `128` | `LBR-MEMORY-004` | `repo` | Memory authority or its rebuildable projection is corrupt | invalid manifest, merge commit on the linear Memory ref, broken revision ancestry, or projection watermark mismatch | +| `128` | `LBR-MEMORY-002` | `repo` | Memory proposal or bounded source exceeds the persisted contract | unsupported schema, malformed Episode envelope, non-canonical payload, or a required source fact exceeding its hard limit | +| `128` | `LBR-MEMORY-003` | `repo` | Memory source or writer policy rejected the operation | authenticated repository/target mismatch, unreachable pinned source, non-local scope, or unknown repository digest key ID | +| `128` | `LBR-MEMORY-004` | `repo` | Memory evidence, authority, or its rebuildable projection is corrupt | source manifest/fragment mismatch, invalid manifest, merge commit on the linear Memory ref, broken revision ancestry, or projection watermark mismatch | | `128` | `LBR-MEMORY-005` | `repo` | Memory writer could not commit an atomic revision | local object write failure, SQLite companion failure, or exhausted bounded ref-conflict retries | | `128` | `LBR-MEMORY-PROJECTION-STALE` | `repo` | Memory projection does not match the pinned repository Memory ref | the ref advanced after a frozen read, the projection is missing, or another replay won the transaction | | `128` | `LBR-WORKTREE-001` | `repo` | Pagination cursor is malformed, foreign, or expired | `libra worktree doctor --cursor ` | @@ -170,9 +170,9 @@ structured report is always present. | `LBR-REPO-002` | Repository metadata is corrupt or incompatible | | `LBR-REPO-003` | Repository state blocks the operation | | `LBR-MEMORY-001` | Repository Memory digest key is missing, invalid, or cannot be decrypted; restore the original encrypted entry or repair the repository vault before writing new Memory data | -| `LBR-MEMORY-002` | The Memory proposal is incompatible with the persisted contract; regenerate it with the supported schema and canonical fields | -| `LBR-MEMORY-003` | The Memory proposal failed repository policy; use the authenticated target and the current repository digest key | -| `LBR-MEMORY-004` | Memory authority and projection disagree or contain invalid history; stop writes and rebuild or repair the projection before retrying | +| `LBR-MEMORY-002` | The Memory proposal or required source exceeds the persisted contract; regenerate it with supported schemas, canonical fields, and the configured source limits | +| `LBR-MEMORY-003` | The Memory source or proposal failed repository policy; use the authenticated repository, trusted target, reachable pinned source, and current digest key | +| `LBR-MEMORY-004` | Memory evidence, authority, and projection disagree or contain invalid history; stop writes and rebuild or repair the source/projection before retrying | | `LBR-MEMORY-005` | The Memory revision could not be committed atomically; fix local storage/SQLite health or retry after ref contention subsides | | `LBR-MEMORY-PROJECTION-STALE` | The Memory projection is not current for the pinned ref; freeze a new view or rebuild/advance the projection before reading it | | `LBR-WORKTREE-001` | The pagination cursor is malformed or expired; drop it and re-read the first page | diff --git a/src/internal/ai/history.rs b/src/internal/ai/history.rs index e8ab07caf..4f39135db 100644 --- a/src/internal/ai/history.rs +++ b/src/internal/ai/history.rs @@ -120,6 +120,215 @@ pub const CHECKPOINT_OBJECT_IO_HELPER_ARG: &str = "--libra-internal-checkpoint-o pub const CHECKPOINT_OBJECT_IO_HELPER_INPUT_CAP: u64 = 32 * 1024 * 1024; pub const CHECKPOINT_OBJECT_IO_HELPER_OUTPUT_CAP: u64 = 32 * 1024 * 1024; const CHECKPOINT_OBJECT_READ_MAX_INFLATED_BYTES: u64 = 16 * 1024 * 1024; +const PINNED_HISTORY_COMMIT_MAX_BYTES: u64 = 256 * 1024; + +/// One object proven reachable from a caller-pinned AI history commit. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PinnedHistoryBlob { + object_id: String, + oid: ObjectHash, + bytes: Vec, +} + +impl PinnedHistoryBlob { + pub(crate) fn object_id(&self) -> &str { + &self.object_id + } + + pub(crate) const fn oid(&self) -> ObjectHash { + self.oid + } + + pub(crate) fn bytes(&self) -> &[u8] { + &self.bytes + } +} + +/// A bounded listing from one type subtree in a pinned AI history view. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PinnedHistoryListing { + entries: Vec, + omitted: usize, +} + +impl PinnedHistoryListing { + pub(crate) fn entries(&self) -> &[PinnedHistoryEntry] { + &self.entries + } + + pub(crate) const fn omitted(&self) -> usize { + self.omitted + } +} + +/// Opaque proof that one path resolved inside a specific pinned view. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PinnedHistoryEntry { + source_head: ObjectHash, + object_type: String, + object_id: String, + oid: ObjectHash, +} + +/// Read-only view whose head was proven to belong to the current first-parent +/// history. Callers cannot accidentally fall through to the moving ref. +pub(crate) struct PinnedHistoryView<'a> { + manager: &'a HistoryManager, + head: ObjectHash, + max_tree_bytes: u64, + root_items: Vec, +} + +impl PinnedHistoryView<'_> { + pub(crate) const fn head(&self) -> ObjectHash { + self.head + } + + pub(crate) fn list( + &self, + object_type: &str, + max_entries: usize, + ) -> Result { + validate_history_path_part("object type", object_type)?; + if max_entries == 0 { + bail!("pinned history listing limit must be greater than zero"); + } + let Some(type_entry) = self.root_items.iter().find(|item| item.name == object_type) else { + return Ok(PinnedHistoryListing { + entries: Vec::new(), + omitted: 0, + }); + }; + let mut entries = self + .manager + .load_tree_bounded(&type_entry.id, self.max_tree_bytes)?; + entries.sort_by(|left, right| left.name.cmp(&right.name)); + let omitted = entries.len().saturating_sub(max_entries); + entries.truncate(max_entries); + Ok(PinnedHistoryListing { + entries: entries + .into_iter() + .map(|item| PinnedHistoryEntry { + source_head: self.head, + object_type: object_type.to_string(), + object_id: item.name, + oid: item.id, + }) + .collect(), + omitted, + }) + } + + pub(crate) fn get_blob( + &self, + object_type: &str, + object_id: &str, + max_bytes: u64, + ) -> Result> { + validate_history_path_part("object type", object_type)?; + validate_history_path_part("object ID", object_id)?; + if max_bytes == 0 { + bail!("pinned history object byte limit must be greater than zero"); + } + let Some(type_entry) = self.root_items.iter().find(|item| item.name == object_type) else { + return Ok(None); + }; + let type_items = self + .manager + .load_tree_bounded(&type_entry.id, self.max_tree_bytes)?; + let Some(item) = type_items.iter().find(|item| item.name == object_id) else { + return Ok(None); + }; + let entry = PinnedHistoryEntry { + source_head: self.head, + object_type: object_type.to_string(), + object_id: object_id.to_string(), + oid: item.id, + }; + self.read_blob(&entry, max_bytes).map(Some) + } + + pub(crate) fn read_blob( + &self, + entry: &PinnedHistoryEntry, + max_bytes: u64, + ) -> Result { + if entry.source_head != self.head { + bail!("pinned history entry belongs to a different source view"); + } + validate_history_path_part("object type", &entry.object_type)?; + validate_history_path_part("object ID", &entry.object_id)?; + if max_bytes == 0 { + bail!("pinned history object byte limit must be greater than zero"); + } + let (kind, bytes) = + read_git_object_bounded_validated(&self.manager.repo_path, &entry.oid, max_bytes) + .with_context(|| { + format!( + "failed to read pinned {}/{}", + entry.object_type, entry.object_id + ) + })?; + if kind != "blob" { + bail!( + "pinned history entry {}/{} is not a blob", + entry.object_type, + entry.object_id + ); + } + Ok(PinnedHistoryBlob { + object_id: entry.object_id.clone(), + oid: entry.oid, + bytes, + }) + } +} + +fn validate_history_path_part(label: &str, value: &str) -> Result<()> { + if value.is_empty() + || value == "." + || value == ".." + || value.contains('/') + || value.contains('\\') + || value.chars().any(char::is_control) + { + bail!("invalid pinned history {label}"); + } + Ok(()) +} + +fn read_single_parent_commit(repo_path: &Path, oid: ObjectHash) -> Result> { + let (kind, bytes) = + read_git_object_bounded_validated(repo_path, &oid, PINNED_HISTORY_COMMIT_MAX_BYTES) + .with_context(|| format!("failed to read AI history commit {oid}"))?; + if kind != "commit" { + bail!("AI history OID {oid} does not identify a commit"); + } + let text = std::str::from_utf8(&bytes) + .with_context(|| format!("AI history commit {oid} is not UTF-8"))?; + let mut parents = Vec::new(); + let mut saw_tree = false; + for line in text.lines() { + if line.is_empty() || line.starts_with("author ") { + break; + } + if line.starts_with("tree ") { + saw_tree = true; + } else if let Some(parent) = line.strip_prefix("parent ") { + parents.push( + ObjectHash::from_str(parent) + .map_err(|_| anyhow!("AI history commit {oid} has invalid parent OID"))?, + ); + } + } + if !saw_tree { + bail!("AI history commit {oid} has no tree header"); + } + if parents.len() > 1 { + bail!("AI history commit {oid} is a merge commit"); + } + Ok(parents.into_iter().next()) +} #[cfg(test)] tokio::task_local! { @@ -1234,6 +1443,56 @@ impl HistoryManager { Ok(Vec::new()) } + /// Pin a read-only view to an exact commit on the current AI history's + /// first-parent chain. The caller-provided OID is treated as untrusted: + /// arbitrary repository commits and stale commits beyond the scan budget + /// are rejected before any typed object is returned. + pub(crate) async fn pin_history( + &self, + pinned_head: ObjectHash, + max_ancestry_commits: usize, + max_tree_bytes: u64, + ) -> Result> { + if max_ancestry_commits == 0 { + bail!("pinned history ancestry limit must be greater than zero"); + } + if max_tree_bytes == 0 { + bail!("pinned history tree byte limit must be greater than zero"); + } + let current_head = self + .resolve_history_head() + .await? + .ok_or_else(|| anyhow!("AI history is not initialized"))?; + let mut cursor = Some(current_head); + let mut visited = HashSet::new(); + let mut found = false; + for _ in 0..max_ancestry_commits { + let Some(commit_oid) = cursor else { + break; + }; + if !visited.insert(commit_oid) { + bail!("AI history contains a first-parent cycle"); + } + if commit_oid == pinned_head { + found = true; + break; + } + cursor = read_single_parent_commit(&self.repo_path, commit_oid)?; + } + if !found { + bail!( + "pinned source commit is not reachable from the current AI history within the configured limit" + ); + } + let root_items = self.load_commit_tree_bounded(&pinned_head, max_tree_bytes)?; + Ok(PinnedHistoryView { + manager: self, + head: pinned_head, + max_tree_bytes, + root_items, + }) + } + /// List all object types present at the current history head. /// /// Functional scope: @@ -1330,6 +1589,40 @@ impl HistoryManager { Err(anyhow!("Commit has no tree")) } + fn load_commit_tree_bounded( + &self, + commit_id: &ObjectHash, + max_tree_bytes: u64, + ) -> Result> { + let (kind, data) = read_git_object_bounded_validated( + &self.repo_path, + commit_id, + PINNED_HISTORY_COMMIT_MAX_BYTES, + )?; + if kind != "commit" { + bail!("pinned AI history OID {commit_id} is not a commit"); + } + let content = std::str::from_utf8(&data) + .with_context(|| format!("AI history commit {commit_id} is not UTF-8"))?; + for line in content.lines() { + if let Some(hash_str) = line.strip_prefix("tree ") { + let tree_hash = ObjectHash::from_str(hash_str) + .map_err(|error| anyhow!("invalid tree hash in pinned commit: {error}"))?; + return self.load_tree_bounded(&tree_hash, max_tree_bytes); + } + } + bail!("pinned AI history commit has no tree") + } + + fn load_tree_bounded(&self, tree_id: &ObjectHash, max_bytes: u64) -> Result> { + let (kind, data) = read_git_object_bounded_validated(&self.repo_path, tree_id, max_bytes)?; + if kind != "tree" { + bail!("pinned AI history tree OID {tree_id} is not a tree"); + } + let tree = Tree::from_bytes(&data, *tree_id)?; + Ok(tree.tree_items) + } + /// Load and parse a tree object's items. /// /// Functional scope: diff --git a/src/internal/ai/keyed_digest.rs b/src/internal/ai/keyed_digest.rs index d16303122..0784d44d2 100644 --- a/src/internal/ai/keyed_digest.rs +++ b/src/internal/ai/keyed_digest.rs @@ -244,6 +244,10 @@ impl SourceInputFingerprint { pub(crate) fn digest_hex(&self) -> &str { self.0.digest_hex() } + + pub(crate) fn encoded(&self) -> String { + encode_receipt_digest(&self.0) + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] diff --git a/src/internal/ai/memory/admission.rs b/src/internal/ai/memory/admission.rs new file mode 100644 index 000000000..884ecb400 --- /dev/null +++ b/src/internal/ai/memory/admission.rs @@ -0,0 +1,313 @@ +use std::collections::BTreeSet; + +use serde::Serialize; +use thiserror::Error; + +use super::{ + compiler::{ + EpisodeClaimProposalV1, EpisodeCompileConfig, EpisodeCompiler, EpisodeCompilerErrorKind, + EpisodeCompilerProposalV1, + }, + domain::{ + CompileOriginV1, CompileRecordV1, EpisodeClaimV1, EpisodeOmissionsV1, EpisodePayloadV1, + EpistemicStatus, IdempotencyScopeV1, MemoryKind, MemoryLifecycle, MemoryNoteV1, + MemoryScopeV1, MemorySensitivity, MemoryTrust, MemoryVisibility, + }, + policy::{ + AuthenticatedMemoryContext, DeterministicMemoryProposal, REPO_EPISODE_ACL_POLICY_ID, + REPO_EPISODE_POLICY_VERSION, REPO_EPISODE_PRODUCER, TrustedMemoryTarget, + }, + source::RedactedEpisodeSource, +}; +use crate::internal::ai::{ + context_budget::MemoryAnchorConfidence, keyed_digest::RepositoryKeyedDigest, +}; + +const MAX_CLAIM_BYTES: usize = 4096; +const MAX_CLAIMS_PER_SECTION: usize = 64; +const MAX_EVIDENCE_PER_CLAIM: usize = 32; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum EpisodeAdmissionErrorKind { + CompilerFailed, + InvalidProposal, + SourceMismatch, + DigestUnavailable, +} + +#[derive(Debug, Error)] +#[error("Episode admission failed ({kind:?})")] +pub(crate) struct EpisodeAdmissionError { + kind: EpisodeAdmissionErrorKind, +} + +impl EpisodeAdmissionError { + const fn new(kind: EpisodeAdmissionErrorKind) -> Self { + Self { kind } + } + + pub(crate) const fn kind(&self) -> EpisodeAdmissionErrorKind { + self.kind + } +} + +/// A proposal that crossed the compiler seam and deterministic admission. +/// Only this module can construct it, and it retains the exact redacted source +/// so the writer can re-resolve every EvidenceRef immediately before commit. +pub(crate) struct AdmittedEpisodeProposal { + proposal: DeterministicMemoryProposal, + source: RedactedEpisodeSource, +} + +impl AdmittedEpisodeProposal { + pub(super) fn proposal(&self) -> &DeterministicMemoryProposal { + &self.proposal + } + + pub(super) fn source(&self) -> &RedactedEpisodeSource { + &self.source + } +} + +pub(crate) struct EpisodeAdmission<'a> { + digest: &'a RepositoryKeyedDigest, +} + +impl<'a> EpisodeAdmission<'a> { + pub(crate) const fn new(digest: &'a RepositoryKeyedDigest) -> Self { + Self { digest } + } + + pub(crate) async fn compile( + &self, + compiler: &C, + config: &EpisodeCompileConfig, + context: &AuthenticatedMemoryContext, + target: &TrustedMemoryTarget, + source: RedactedEpisodeSource, + ) -> Result { + if context.repository_id() != self.digest.repository_id() + || source.manifest().root_kind != target.root().kind() + || source.manifest().root_id != target.root().id() + || config.producer() != REPO_EPISODE_PRODUCER + { + return Err(EpisodeAdmissionError::new( + EpisodeAdmissionErrorKind::SourceMismatch, + )); + } + let compiler_proposal = + compiler + .compile(&source, config) + .await + .map_err(|error| match error.kind() { + EpisodeCompilerErrorKind::InvalidConfig + | EpisodeCompilerErrorKind::MalformedOutput + | EpisodeCompilerErrorKind::OutputLimitExceeded + | EpisodeCompilerErrorKind::ProviderFailed => { + EpisodeAdmissionError::new(EpisodeAdmissionErrorKind::CompilerFailed) + } + })?; + let proposal = self.admit(config, context, target, &source, compiler_proposal)?; + Ok(AdmittedEpisodeProposal { proposal, source }) + } + + fn admit( + &self, + config: &EpisodeCompileConfig, + context: &AuthenticatedMemoryContext, + target: &TrustedMemoryTarget, + source: &RedactedEpisodeSource, + proposal: EpisodeCompilerProposalV1, + ) -> Result { + for section in [ + &proposal.observations, + &proposal.inferences, + &proposal.decisions, + &proposal.failed_attempts, + &proposal.unresolved, + ] { + if section.len() > MAX_CLAIMS_PER_SECTION { + return Err(invalid_proposal()); + } + } + let idempotency_input = serde_json::to_vec(&IdempotencyInput { + manifest: source.manifest(), + config, + proposal: &proposal, + }) + .map_err(|_| invalid_proposal())?; + let summary = claim(source, proposal.summary, Some(EpistemicStatus::Inference))?; + let observations = claims( + source, + proposal.observations, + Some(EpistemicStatus::Observation), + )?; + let inferences = claims( + source, + proposal.inferences, + Some(EpistemicStatus::Inference), + )?; + let decisions = claims(source, proposal.decisions, None)?; + let failed_attempts = claims(source, proposal.failed_attempts, None)?; + let unresolved = claims(source, proposal.unresolved, None)?; + let root_fragment = source.fragments().first().ok_or_else(invalid_proposal)?; + let goal = EpisodeClaimV1 { + epistemic_status: EpistemicStatus::Observation, + claim: source.facts().root_goal.clone(), + confidence: None, + evidence_refs: vec![root_fragment.evidence().clone()], + }; + let manifest_bytes = + serde_json::to_vec(source.manifest()).map_err(|_| invalid_proposal())?; + let manifest_fingerprint = self + .digest + .source_input_fingerprint(&manifest_bytes) + .map_err(|_| EpisodeAdmissionError::new(EpisodeAdmissionErrorKind::DigestUnavailable))? + .encoded(); + let idempotency_key = self + .digest + .source_input_fingerprint(&idempotency_input) + .map_err(|_| EpisodeAdmissionError::new(EpisodeAdmissionErrorKind::DigestUnavailable))? + .encoded(); + let source_evidence = source + .fragments() + .iter() + .map(|fragment| fragment.evidence().clone()) + .collect::>(); + let related_run_omissions = source + .manifest() + .omissions + .iter() + .filter(|omission| omission.object_type == "run") + .map(|omission| omission.count) + .sum::() + .try_into() + .unwrap_or(u32::MAX); + let episode = EpisodePayloadV1 { + schema_version: 1, + root_kind: target.root().kind(), + root_id: target.root().id().to_string(), + related_intent_ids: source.facts().related_intent_ids.clone(), + related_task_ids: source.facts().related_task_ids.clone(), + related_run_ids: source.facts().related_run_ids.clone(), + started_at: Some(source.facts().started_at), + ended_at: Some(source.facts().ended_at), + goal, + completion_status: source.facts().completion_status, + code_change_status: source.facts().code_change_status, + summary: summary.clone(), + observations, + inferences, + decisions, + failed_attempts, + unresolved, + code: source.facts().code.clone(), + omissions: EpisodeOmissionsV1 { + related_run_ids: related_run_omissions, + ..EpisodeOmissionsV1::default() + }, + }; + let note = MemoryNoteV1 { + schema_version: 1, + note_id: target.root().note_id(), + content_digest: String::new(), + namespace: target.root().namespace().to_string(), + path: target.root().path().to_string(), + kind: MemoryKind::Episodic, + scope: MemoryScopeV1::Repo, + visibility: MemoryVisibility::RepoLocal, + acl_policy_id: REPO_EPISODE_ACL_POLICY_ID.to_string(), + lifecycle: MemoryLifecycle::Accretive, + body: summary.claim, + rationale: None, + episode: Some(episode), + evidence_refs: source_evidence, + links: Vec::new(), + entities: Vec::new(), + parents: Vec::new(), + tags: vec!["episode".to_string()], + confidence: MemoryAnchorConfidence::Medium, + trust: MemoryTrust::RepoEvidence, + sensitivity: MemorySensitivity::Internal, + valid_from: None, + valid_until: None, + effective_from_commit: source + .facts() + .code + .result_oid + .clone() + .or_else(|| source.facts().code.base_oid.clone()), + effective_until_commit: None, + expires_at: None, + author: context.actor().clone(), + created_at: source.facts().ended_at, + compile_record: CompileRecordV1 { + schema_version: 1, + origin: CompileOriginV1::EpisodeCompiler, + producer: REPO_EPISODE_PRODUCER.to_string(), + rules_version: config.rules_version(), + prompt_version: Some(config.prompt_version().to_string()), + model_id: Some(config.model_id().to_string()), + policy_version: REPO_EPISODE_POLICY_VERSION.to_string(), + input_hashes: vec![manifest_fingerprint], + idempotency_key, + idempotency_scope: IdempotencyScopeV1::Cell, + }, + }; + Ok(DeterministicMemoryProposal::new(note)) + } +} + +#[derive(Serialize)] +struct IdempotencyInput<'a> { + manifest: &'a super::source::EpisodeSourceManifestV1, + config: &'a EpisodeCompileConfig, + proposal: &'a EpisodeCompilerProposalV1, +} + +fn claims( + source: &RedactedEpisodeSource, + proposals: Vec, + required_status: Option, +) -> Result, EpisodeAdmissionError> { + proposals + .into_iter() + .map(|proposal| claim(source, proposal, required_status)) + .collect() +} + +fn claim( + source: &RedactedEpisodeSource, + proposal: EpisodeClaimProposalV1, + required_status: Option, +) -> Result { + if proposal.claim.is_empty() + || proposal.claim.len() > MAX_CLAIM_BYTES + || proposal.evidence_fragment_ids.is_empty() + || proposal.evidence_fragment_ids.len() > MAX_EVIDENCE_PER_CLAIM + || required_status.is_some_and(|status| status != proposal.epistemic_status) + || (proposal.epistemic_status == EpistemicStatus::Inference + && proposal.confidence.is_none()) + { + return Err(invalid_proposal()); + } + let mut seen = BTreeSet::new(); + let mut evidence_refs = Vec::new(); + for fragment_id in proposal.evidence_fragment_ids { + if !seen.insert(fragment_id.clone()) { + continue; + } + let evidence = source.evidence(&fragment_id).ok_or_else(invalid_proposal)?; + evidence_refs.push(evidence.clone()); + } + Ok(EpisodeClaimV1 { + epistemic_status: proposal.epistemic_status, + claim: proposal.claim, + confidence: proposal.confidence, + evidence_refs, + }) +} + +fn invalid_proposal() -> EpisodeAdmissionError { + EpisodeAdmissionError::new(EpisodeAdmissionErrorKind::InvalidProposal) +} diff --git a/src/internal/ai/memory/compiler/mod.rs b/src/internal/ai/memory/compiler/mod.rs new file mode 100644 index 000000000..e2152af7d --- /dev/null +++ b/src/internal/ai/memory/compiler/mod.rs @@ -0,0 +1,115 @@ +use async_trait::async_trait; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use super::{domain::EpistemicStatus, source::RedactedEpisodeSource}; +use crate::internal::ai::context_budget::MemoryAnchorConfidence; + +const MAX_PRODUCER_BYTES: usize = 120; +const MAX_VERSION_BYTES: usize = 80; + +#[derive(Clone, Debug, Eq, PartialEq, Serialize)] +pub(crate) struct EpisodeCompileConfig { + producer: String, + rules_version: u32, + prompt_version: String, + model_id: String, +} + +impl EpisodeCompileConfig { + pub(crate) fn new( + producer: impl Into, + rules_version: u32, + prompt_version: impl Into, + model_id: impl Into, + ) -> Result { + let config = Self { + producer: producer.into(), + rules_version, + prompt_version: prompt_version.into(), + model_id: model_id.into(), + }; + if config.producer.is_empty() + || config.producer.len() > MAX_PRODUCER_BYTES + || config.rules_version == 0 + || config.prompt_version.is_empty() + || config.prompt_version.len() > MAX_VERSION_BYTES + || config.model_id.is_empty() + || config.model_id.len() > MAX_VERSION_BYTES + { + return Err(EpisodeCompilerError::new( + EpisodeCompilerErrorKind::InvalidConfig, + )); + } + Ok(config) + } + + pub(crate) fn producer(&self) -> &str { + &self.producer + } + + pub(crate) const fn rules_version(&self) -> u32 { + self.rules_version + } + + pub(crate) fn prompt_version(&self) -> &str { + &self.prompt_version + } + + pub(crate) fn model_id(&self) -> &str { + &self.model_id + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct EpisodeClaimProposalV1 { + pub(crate) epistemic_status: EpistemicStatus, + pub(crate) claim: String, + pub(crate) confidence: Option, + pub(crate) evidence_fragment_ids: Vec, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct EpisodeCompilerProposalV1 { + pub(crate) summary: EpisodeClaimProposalV1, + pub(crate) observations: Vec, + pub(crate) inferences: Vec, + pub(crate) decisions: Vec, + pub(crate) failed_attempts: Vec, + pub(crate) unresolved: Vec, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum EpisodeCompilerErrorKind { + InvalidConfig, + ProviderFailed, + MalformedOutput, + OutputLimitExceeded, +} + +#[derive(Debug, Error)] +#[error("Episode compiler failed ({kind:?})")] +pub(crate) struct EpisodeCompilerError { + kind: EpisodeCompilerErrorKind, +} + +impl EpisodeCompilerError { + pub(crate) const fn new(kind: EpisodeCompilerErrorKind) -> Self { + Self { kind } + } + + pub(crate) const fn kind(&self) -> EpisodeCompilerErrorKind { + self.kind + } +} + +/// Crate-private compiler seam. Adapters can inspect only redacted source and +/// return claim drafts keyed to resolver-issued fragment IDs. +#[async_trait] +pub(crate) trait EpisodeCompiler: Send + Sync { + async fn compile( + &self, + source: &RedactedEpisodeSource, + config: &EpisodeCompileConfig, + ) -> Result; +} diff --git a/src/internal/ai/memory/error.rs b/src/internal/ai/memory/error.rs index 3d3b22406..bfcc276bc 100644 --- a/src/internal/ai/memory/error.rs +++ b/src/internal/ai/memory/error.rs @@ -7,6 +7,9 @@ pub(crate) enum MemoryWriterErrorKind { DigestKeyUnavailable, InvalidProposal, PolicyRejected, + SourceRejected, + SourceLimitExceeded, + EvidenceMismatch, UnknownDigestKey, CorruptHistory, CorruptProjection, @@ -19,9 +22,13 @@ impl MemoryWriterErrorKind { pub(crate) const fn stable_code(self) -> &'static str { match self { Self::DigestKeyUnavailable => "LBR-MEMORY-001", - Self::InvalidProposal => "LBR-MEMORY-002", - Self::PolicyRejected | Self::UnknownDigestKey => "LBR-MEMORY-003", - Self::CorruptHistory | Self::CorruptProjection => "LBR-MEMORY-004", + Self::InvalidProposal | Self::SourceLimitExceeded => "LBR-MEMORY-002", + Self::PolicyRejected | Self::SourceRejected | Self::UnknownDigestKey => { + "LBR-MEMORY-003" + } + Self::CorruptHistory | Self::CorruptProjection | Self::EvidenceMismatch => { + "LBR-MEMORY-004" + } Self::ProjectionStale => "LBR-MEMORY-PROJECTION-STALE", Self::StorageFailure | Self::ConflictExhausted => "LBR-MEMORY-005", } @@ -57,3 +64,24 @@ impl From for MemoryWriterError { Self::new(MemoryWriterErrorKind::InvalidProposal, error.to_string()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn source_error_categories_have_stable_redacted_messages() { + for (kind, code) in [ + (MemoryWriterErrorKind::SourceLimitExceeded, "LBR-MEMORY-002"), + (MemoryWriterErrorKind::SourceRejected, "LBR-MEMORY-003"), + (MemoryWriterErrorKind::EvidenceMismatch, "LBR-MEMORY-004"), + ] { + let error = MemoryWriterError::new(kind, "source validation failed"); + assert_eq!(error.stable_code(), code); + assert_eq!( + error.to_string(), + format!("{code}: source validation failed") + ); + } + } +} diff --git a/src/internal/ai/memory/limits.rs b/src/internal/ai/memory/limits.rs new file mode 100644 index 000000000..e645dceff --- /dev/null +++ b/src/internal/ai/memory/limits.rs @@ -0,0 +1,48 @@ +/// Frozen safety limits for one Episode source resolution. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct EpisodeSourceLimits { + pub(crate) max_objects: usize, + pub(crate) max_candidate_objects: usize, + pub(crate) max_tree_bytes: u64, + pub(crate) max_object_bytes: u64, + pub(crate) max_total_bytes: usize, + pub(crate) max_context_fragments: usize, + pub(crate) max_token_estimate: usize, + pub(crate) max_ancestry_commits: usize, +} + +impl EpisodeSourceLimits { + pub(crate) const fn repo_v1() -> Self { + Self { + max_objects: 256, + max_candidate_objects: 4096, + max_tree_bytes: 4 * 1024 * 1024, + max_object_bytes: 128 * 1024, + max_total_bytes: 2 * 1024 * 1024, + max_context_fragments: 64, + max_token_estimate: 512 * 1024, + max_ancestry_commits: 2048, + } + } + + pub(crate) fn validate(self) -> Result { + if self.max_objects == 0 + || self.max_candidate_objects < self.max_objects + || self.max_tree_bytes == 0 + || self.max_object_bytes == 0 + || self.max_total_bytes == 0 + || self.max_context_fragments == 0 + || self.max_token_estimate == 0 + || self.max_ancestry_commits == 0 + { + return Err("Episode source limits are invalid"); + } + Ok(self) + } +} + +impl Default for EpisodeSourceLimits { + fn default() -> Self { + Self::repo_v1() + } +} diff --git a/src/internal/ai/memory/mod.rs b/src/internal/ai/memory/mod.rs index dc2bb82ef..4c98832e9 100644 --- a/src/internal/ai/memory/mod.rs +++ b/src/internal/ai/memory/mod.rs @@ -9,14 +9,18 @@ //! Storage, projection, compilation, and command adapters are implemented by //! later plan slices and must not become alternate write seams. +mod admission; mod canonical; +mod compiler; mod domain; mod error; mod fts_sql; mod job_sql; +mod limits; mod policy; mod projection; mod replay; +mod source; mod store; mod tree; mod validation; diff --git a/src/internal/ai/memory/policy.rs b/src/internal/ai/memory/policy.rs index 9eb895f17..2182ce1fe 100644 --- a/src/internal/ai/memory/policy.rs +++ b/src/internal/ai/memory/policy.rs @@ -9,9 +9,9 @@ use super::{ error::{MemoryWriterError, MemoryWriterErrorKind}, }; -const REPO_EPISODE_POLICY_VERSION: &str = "repo-policy-v1"; -const REPO_EPISODE_ACL_POLICY_ID: &str = "repo-default-v1"; -const REPO_EPISODE_PRODUCER: &str = "libra-memory/1"; +pub(super) const REPO_EPISODE_POLICY_VERSION: &str = "repo-policy-v1"; +pub(super) const REPO_EPISODE_ACL_POLICY_ID: &str = "repo-default-v1"; +pub(super) const REPO_EPISODE_PRODUCER: &str = "libra-memory/1"; const REPO_EPISODE_POLICY_SNAPSHOT: &[u8] = br#"{ "acl_policy":"repo-default-v1", "auto_confirm":true, diff --git a/src/internal/ai/memory/source.rs b/src/internal/ai/memory/source.rs new file mode 100644 index 000000000..55d9481d6 --- /dev/null +++ b/src/internal/ai/memory/source.rs @@ -0,0 +1,1572 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use chrono::{DateTime, Utc}; +use git_internal::hash::ObjectHash; +use regex::bytes::Regex; +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use sha2::{Digest, Sha256}; +use thiserror::Error; + +use super::{ + domain::{ + CodeChangeStatus, CompletionStatus, EpisodeCodeContextV1, EpisodeRootKind, EvidenceKind, + EvidenceLocatorV1, EvidenceRefV1, EvidenceSourcePlane, EvidenceVisibility, ToolCallPart, + }, + limits::EpisodeSourceLimits, + policy::{AuthenticatedMemoryContext, TrustedMemoryTarget}, +}; +use crate::internal::ai::{ + history::{HistoryManager, PinnedHistoryBlob, PinnedHistoryView}, + keyed_digest::RepositoryKeyedDigest, + observed_agents::Redactor, +}; + +const SOURCE_SCHEMA_VERSION: u32 = 1; +const SOURCE_POLICY_VERSION: &str = "repo-episode-source-v1"; +const REDACTION_POLICY_VERSION: &str = "memory-redaction-v1"; + +const TASK: &str = "task"; +const INTENT: &str = "intent"; +const RUN: &str = "run"; +const TASK_EVENT: &str = "task_event"; +const INTENT_EVENT: &str = "intent_event"; +const RUN_EVENT: &str = "run_event"; +const EVIDENCE: &str = "evidence"; +const DECISION: &str = "decision"; +const PATCHSET: &str = "patchset"; +const CONTEXT_FRAME: &str = "context_frame"; +const INVOCATION: &str = "invocation"; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum EpisodeSourceErrorKind { + Unauthorized, + InvalidRequest, + SourceNotReachable, + SourceCorrupt, + LimitExceeded, + RedactionFailed, + DigestUnavailable, +} + +#[derive(Debug, Error)] +#[error("Episode source resolution failed ({kind:?})")] +pub(crate) struct EpisodeSourceError { + kind: EpisodeSourceErrorKind, +} + +impl EpisodeSourceError { + const fn new(kind: EpisodeSourceErrorKind) -> Self { + Self { kind } + } + + pub(crate) const fn kind(&self) -> EpisodeSourceErrorKind { + self.kind + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct SourceOmissionV1 { + pub(crate) code: String, + pub(crate) object_type: String, + pub(crate) count: usize, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct EpisodeSourceLimitSnapshotV1 { + pub(crate) max_objects: usize, + pub(crate) max_candidate_objects: usize, + pub(crate) max_tree_bytes: u64, + pub(crate) max_object_bytes: u64, + pub(crate) max_total_bytes: usize, + pub(crate) max_context_fragments: usize, + pub(crate) max_token_estimate: usize, + pub(crate) max_ancestry_commits: usize, +} + +impl From for EpisodeSourceLimitSnapshotV1 { + fn from(limits: EpisodeSourceLimits) -> Self { + Self { + max_objects: limits.max_objects, + max_candidate_objects: limits.max_candidate_objects, + max_tree_bytes: limits.max_tree_bytes, + max_object_bytes: limits.max_object_bytes, + max_total_bytes: limits.max_total_bytes, + max_context_fragments: limits.max_context_fragments, + max_token_estimate: limits.max_token_estimate, + max_ancestry_commits: limits.max_ancestry_commits, + } + } +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct SourceManifestFragmentV1 { + pub(crate) fragment_id: String, + pub(crate) object_type: String, + pub(crate) object_id: String, + pub(crate) object_oid: String, + pub(crate) locator: EvidenceLocatorV1, + pub(crate) fragment_digest: String, + pub(crate) code_commit: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub(crate) struct EpisodeSourceManifestV1 { + pub(crate) schema_version: u32, + pub(crate) policy_version: String, + pub(crate) redaction_policy_version: String, + pub(crate) root_kind: EpisodeRootKind, + pub(crate) root_id: String, + pub(crate) repository_id: String, + pub(crate) principal_digest: String, + pub(crate) source_ref_oid: String, + pub(crate) limits: EpisodeSourceLimitSnapshotV1, + pub(crate) object_count: usize, + pub(crate) redacted_bytes: usize, + pub(crate) token_estimate: usize, + pub(crate) fragments: Vec, + pub(crate) omissions: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct EpisodeSourceFacts { + pub(crate) related_intent_ids: Vec, + pub(crate) related_task_ids: Vec, + pub(crate) related_run_ids: Vec, + pub(crate) root_goal: String, + pub(crate) started_at: DateTime, + pub(crate) ended_at: DateTime, + pub(crate) completion_status: CompletionStatus, + pub(crate) code_change_status: CodeChangeStatus, + pub(crate) code: EpisodeCodeContextV1, +} + +/// Redacted source fragment. Its content intentionally has no serde or Debug +/// implementation so ordinary manifests and diagnostics cannot print it. +#[derive(Clone, Eq, PartialEq)] +pub(crate) struct RedactedEpisodeFragment { + fragment_id: String, + object_type: String, + object_id: String, + object_oid: ObjectHash, + text: String, + evidence: EvidenceRefV1, +} + +impl RedactedEpisodeFragment { + pub(crate) fn fragment_id(&self) -> &str { + &self.fragment_id + } + + pub(crate) fn object_type(&self) -> &str { + &self.object_type + } + + pub(crate) fn object_id(&self) -> &str { + &self.object_id + } + + pub(crate) const fn object_oid(&self) -> ObjectHash { + self.object_oid + } + + pub(crate) fn text(&self) -> &str { + &self.text + } + + pub(crate) fn evidence(&self) -> &EvidenceRefV1 { + &self.evidence + } +} + +/// Compiler input that can only be created by [`EpisodeSourceResolver`]. +pub(crate) struct RedactedEpisodeSource { + manifest: EpisodeSourceManifestV1, + facts: EpisodeSourceFacts, + fragments: Vec, +} + +impl RedactedEpisodeSource { + pub(crate) fn manifest(&self) -> &EpisodeSourceManifestV1 { + &self.manifest + } + + pub(crate) fn facts(&self) -> &EpisodeSourceFacts { + &self.facts + } + + pub(crate) fn fragments(&self) -> &[RedactedEpisodeFragment] { + &self.fragments + } + + pub(crate) fn evidence(&self, fragment_id: &str) -> Option<&EvidenceRefV1> { + self.fragments + .iter() + .find(|fragment| fragment.fragment_id == fragment_id) + .map(RedactedEpisodeFragment::evidence) + } +} + +struct MemorySourceRedactor { + secrets: Redactor, + email: Regex, + home_path: Regex, +} + +impl MemorySourceRedactor { + fn new() -> Result { + Ok(Self { + secrets: Redactor::new_default(), + email: Regex::new(r"(?i)\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b") + .map_err(|_| EpisodeSourceError::new(EpisodeSourceErrorKind::RedactionFailed))?, + home_path: Regex::new(r#"/(?:Users|home)/[^/\s\"']+"#) + .map_err(|_| EpisodeSourceError::new(EpisodeSourceErrorKind::RedactionFailed))?, + }) + } + + fn redact(&self, raw: &[u8]) -> Result, EpisodeSourceError> { + let (secret_redacted, _) = self.secrets.redact(raw); + let private_redacted = redact_private_markers(secret_redacted.bytes())?; + let email_redacted = self + .email + .replace_all(&private_redacted, b"".as_slice()); + Ok(self + .home_path + .replace_all(&email_redacted, b"/".as_slice()) + .into_owned()) + } +} + +pub(crate) struct EpisodeSourceResolver<'a> { + history: &'a HistoryManager, + digest: &'a RepositoryKeyedDigest, + limits: EpisodeSourceLimits, + redactor: MemorySourceRedactor, +} + +impl<'a> EpisodeSourceResolver<'a> { + pub(crate) fn new( + history: &'a HistoryManager, + digest: &'a RepositoryKeyedDigest, + limits: EpisodeSourceLimits, + ) -> Result { + let limits = limits + .validate() + .map_err(|_| EpisodeSourceError::new(EpisodeSourceErrorKind::InvalidRequest))?; + Ok(Self { + history, + digest, + limits, + redactor: MemorySourceRedactor::new()?, + }) + } + + pub(crate) async fn resolve( + &self, + context: &AuthenticatedMemoryContext, + target: &TrustedMemoryTarget, + source_ref_oid: ObjectHash, + ) -> Result { + if context.repository_id() != self.digest.repository_id() { + return Err(EpisodeSourceError::new( + EpisodeSourceErrorKind::Unauthorized, + )); + } + let view = self + .history + .pin_history( + source_ref_oid, + self.limits.max_ancestry_commits, + self.limits.max_tree_bytes, + ) + .await + .map_err(|_| EpisodeSourceError::new(EpisodeSourceErrorKind::SourceNotReachable))?; + let principal_digest = self + .digest + .principal_digest(context.actor().principal_id.as_bytes()) + .map_err(|_| EpisodeSourceError::new(EpisodeSourceErrorKind::DigestUnavailable))? + .encoded(); + let mut collector = SourceCollector::new( + self, + view, + target, + context.repository_id(), + principal_digest, + ); + collector.collect()?; + collector.finish() + } + + pub(crate) async fn revalidate( + &self, + context: &AuthenticatedMemoryContext, + target: &TrustedMemoryTarget, + source: &RedactedEpisodeSource, + ) -> Result<(), EpisodeSourceError> { + let source_oid = source + .manifest + .source_ref_oid + .parse::() + .map_err(|_| EpisodeSourceError::new(EpisodeSourceErrorKind::SourceCorrupt))?; + let rebuilt = self.resolve(context, target, source_oid).await?; + if rebuilt.manifest != source.manifest + || rebuilt.facts != source.facts + || rebuilt.fragments != source.fragments + { + return Err(EpisodeSourceError::new( + EpisodeSourceErrorKind::SourceCorrupt, + )); + } + Ok(()) + } +} + +struct SourceCollector<'resolver, 'history> { + resolver: &'resolver EpisodeSourceResolver<'history>, + view: PinnedHistoryView<'history>, + target: &'resolver TrustedMemoryTarget, + repository_id: &'resolver str, + principal_digest: String, + fragments: Vec, + values: BTreeMap<(String, String), Value>, + omissions: BTreeMap<(String, String), usize>, + candidate_count: usize, + redacted_bytes: usize, + context_fragments: usize, + intent_ids: BTreeSet, + task_ids: BTreeSet, + run_ids: BTreeSet, +} + +impl<'resolver, 'history> SourceCollector<'resolver, 'history> { + fn new( + resolver: &'resolver EpisodeSourceResolver<'history>, + view: PinnedHistoryView<'history>, + target: &'resolver TrustedMemoryTarget, + repository_id: &'resolver str, + principal_digest: String, + ) -> Self { + let mut intent_ids = BTreeSet::new(); + let mut task_ids = BTreeSet::new(); + match target.root().kind() { + EpisodeRootKind::Task => { + task_ids.insert(target.root().id().to_string()); + } + EpisodeRootKind::Intent => { + intent_ids.insert(target.root().id().to_string()); + } + } + Self { + resolver, + view, + target, + repository_id, + principal_digest, + fragments: Vec::new(), + values: BTreeMap::new(), + omissions: BTreeMap::new(), + candidate_count: 0, + redacted_bytes: 0, + context_fragments: 0, + intent_ids, + task_ids, + run_ids: BTreeSet::new(), + } + } + + fn collect(&mut self) -> Result<(), EpisodeSourceError> { + let root_type = match self.target.root().kind() { + EpisodeRootKind::Task => TASK, + EpisodeRootKind::Intent => INTENT, + }; + let root_id = self.target.root().id().to_string(); + let root = self + .load_exact(root_type, &root_id)? + .ok_or_else(|| EpisodeSourceError::new(EpisodeSourceErrorKind::SourceCorrupt))?; + self.include(root_type, root, true)?; + + match self.target.root().kind() { + EpisodeRootKind::Task => { + let root_id = root_id.clone(); + self.scan_with_requirement( + TASK_EVENT, + move |value| field_id(value, "task_id") == Some(root_id.as_str()), + |value| { + matches!( + field_id(value, "kind"), + Some("done" | "failed" | "cancelled") + ) + }, + )?; + } + EpisodeRootKind::Intent => { + let root_id = root_id.clone(); + self.scan_with_requirement( + INTENT_EVENT, + move |value| field_id(value, "intent_id") == Some(root_id.as_str()), + |value| matches!(field_id(value, "kind"), Some("completed" | "cancelled")), + )?; + } + } + + if root_type == TASK { + let linked_intent = self + .value(TASK, &root_id)? + .get("intent") + .and_then(Value::as_str) + .map(str::to_string); + if let Some(intent_id) = linked_intent { + self.intent_ids.insert(intent_id.clone()); + if let Some(intent) = self.load_exact(INTENT, &intent_id)? { + self.include(INTENT, intent, false)?; + } + } + } + if root_type == INTENT { + let intent_ids = self.intent_ids.clone(); + self.scan(TASK, move |value| { + field_id(value, "intent").is_some_and(|id| intent_ids.contains(id)) + })?; + self.task_ids.extend( + self.values + .keys() + .filter(|(kind, _)| kind == TASK) + .map(|(_, id)| id.clone()), + ); + } + + let task_ids = self.task_ids.clone(); + self.scan(RUN, move |value| { + field_id(value, "task").is_some_and(|id| task_ids.contains(id)) + })?; + self.run_ids.extend( + self.values + .keys() + .filter(|(kind, _)| kind == RUN) + .map(|(_, id)| id.clone()), + ); + + match self.target.root().kind() { + EpisodeRootKind::Task => { + let intent_ids = self.intent_ids.clone(); + self.scan(INTENT_EVENT, move |value| { + field_id(value, "intent_id").is_some_and(|id| intent_ids.contains(id)) + })?; + } + EpisodeRootKind::Intent => { + let task_ids = self.task_ids.clone(); + self.scan(TASK_EVENT, move |value| { + field_id(value, "task_id").is_some_and(|id| task_ids.contains(id)) + })?; + } + } + for object_type in [RUN_EVENT, EVIDENCE, DECISION, PATCHSET, INVOCATION] { + let run_ids = self.run_ids.clone(); + self.scan(object_type, move |value| { + relation_run_id(value).is_some_and(|id| run_ids.contains(id)) + })?; + } + let run_ids = self.run_ids.clone(); + let intent_ids = self.intent_ids.clone(); + self.scan(CONTEXT_FRAME, move |value| { + field_id(value, "run_id").is_some_and(|id| run_ids.contains(id)) + || field_id(value, "intent_id").is_some_and(|id| intent_ids.contains(id)) + })?; + Ok(()) + } + + fn finish(self) -> Result { + let root_type = match self.target.root().kind() { + EpisodeRootKind::Task => TASK, + EpisodeRootKind::Intent => INTENT, + }; + let redacted_root = self + .fragments + .iter() + .find(|fragment| { + fragment.object_type == root_type && fragment.object_id == self.target.root().id() + }) + .ok_or_else(|| EpisodeSourceError::new(EpisodeSourceErrorKind::SourceCorrupt))?; + let redacted_root_value: Value = serde_json::from_str(redacted_root.text()) + .map_err(|_| EpisodeSourceError::new(EpisodeSourceErrorKind::SourceCorrupt))?; + let root_goal = match self.target.root().kind() { + EpisodeRootKind::Task => redacted_root_value + .get("title") + .and_then(Value::as_str) + .or_else(|| { + redacted_root_value + .get("description") + .and_then(Value::as_str) + }) + .unwrap_or("task"), + EpisodeRootKind::Intent => redacted_root_value + .get("prompt") + .and_then(Value::as_str) + .unwrap_or("intent"), + } + .to_string(); + let root_value = self.value(root_type, self.target.root().id())?; + let started_at = parse_timestamp(root_value, "created_at")?; + let (completion_status, ended_at) = terminal_fact( + &self.values, + self.target.root().kind(), + self.target.root().id(), + )?; + let code = derive_code_context(&self.values, &self.run_ids); + let code_change_status = match (&code.base_oid, &code.result_oid) { + (_, None) => CodeChangeStatus::Unknown, + (Some(base), Some(result)) if base == result => CodeChangeStatus::Unchanged, + (_, Some(_)) => CodeChangeStatus::Changed, + }; + let omissions = self + .omissions + .into_iter() + .map(|((code, object_type), count)| SourceOmissionV1 { + code, + object_type, + count, + }) + .collect::>(); + let token_estimate = estimate_tokens(self.redacted_bytes); + let manifest = EpisodeSourceManifestV1 { + schema_version: SOURCE_SCHEMA_VERSION, + policy_version: SOURCE_POLICY_VERSION.to_string(), + redaction_policy_version: REDACTION_POLICY_VERSION.to_string(), + root_kind: self.target.root().kind(), + root_id: self.target.root().id().to_string(), + repository_id: self.repository_id.to_string(), + principal_digest: self.principal_digest, + source_ref_oid: self.view.head().to_string(), + limits: self.resolver.limits.into(), + object_count: self.fragments.len(), + redacted_bytes: self.redacted_bytes, + token_estimate, + fragments: self + .fragments + .iter() + .map(|fragment| SourceManifestFragmentV1 { + fragment_id: fragment.fragment_id.clone(), + object_type: fragment.object_type.clone(), + object_id: fragment.object_id.clone(), + object_oid: fragment.object_oid.to_string(), + locator: fragment.evidence.locator.clone(), + fragment_digest: fragment.evidence.fragment_digest.clone(), + code_commit: fragment.evidence.code_commit.clone(), + }) + .collect(), + omissions, + }; + Ok(RedactedEpisodeSource { + manifest, + facts: EpisodeSourceFacts { + related_intent_ids: self.intent_ids.into_iter().collect(), + related_task_ids: self.task_ids.into_iter().collect(), + related_run_ids: self.run_ids.into_iter().collect(), + root_goal, + started_at, + ended_at, + completion_status, + code_change_status, + code, + }, + fragments: self.fragments, + }) + } + + fn value(&self, object_type: &str, object_id: &str) -> Result<&Value, EpisodeSourceError> { + self.values + .get(&(object_type.to_string(), object_id.to_string())) + .ok_or_else(|| EpisodeSourceError::new(EpisodeSourceErrorKind::SourceCorrupt)) + } + + fn load_exact( + &mut self, + object_type: &str, + object_id: &str, + ) -> Result, EpisodeSourceError> { + self.consume_candidate()?; + self.view + .get_blob( + object_type, + object_id, + self.resolver.limits.max_object_bytes, + ) + .map_err(|_| EpisodeSourceError::new(EpisodeSourceErrorKind::SourceCorrupt)) + } + + fn scan(&mut self, object_type: &str, predicate: F) -> Result<(), EpisodeSourceError> + where + F: Fn(&Value) -> bool, + { + self.scan_with_requirement(object_type, predicate, |_| false) + } + + fn scan_with_requirement( + &mut self, + object_type: &str, + predicate: F, + required: R, + ) -> Result<(), EpisodeSourceError> + where + F: Fn(&Value) -> bool, + R: Fn(&Value) -> bool, + { + let remaining = self + .resolver + .limits + .max_candidate_objects + .saturating_sub(self.candidate_count); + if remaining == 0 { + self.omit("candidate_limit", object_type, 1); + return Ok(()); + } + let listing = self + .view + .list(object_type, remaining) + .map_err(|_| EpisodeSourceError::new(EpisodeSourceErrorKind::SourceCorrupt))?; + if listing.omitted() > 0 { + self.omit("candidate_limit", object_type, listing.omitted()); + } + let entries = listing.entries().to_vec(); + for entry in entries { + self.consume_candidate()?; + let blob = self + .view + .read_blob(&entry, self.resolver.limits.max_object_bytes) + .map_err(|_| EpisodeSourceError::new(EpisodeSourceErrorKind::SourceCorrupt))?; + let value = parse_object(object_type, &blob)?; + if predicate(&value) { + let is_required = required(&value); + self.include_parsed(object_type, blob, value, is_required)?; + } + } + Ok(()) + } + + fn consume_candidate(&mut self) -> Result<(), EpisodeSourceError> { + self.candidate_count = self.candidate_count.saturating_add(1); + if self.candidate_count > self.resolver.limits.max_candidate_objects { + return Err(EpisodeSourceError::new( + EpisodeSourceErrorKind::LimitExceeded, + )); + } + Ok(()) + } + + fn include( + &mut self, + object_type: &str, + blob: PinnedHistoryBlob, + required: bool, + ) -> Result<(), EpisodeSourceError> { + let value = parse_object(object_type, &blob)?; + self.include_parsed(object_type, blob, value, required) + } + + fn include_parsed( + &mut self, + object_type: &str, + blob: PinnedHistoryBlob, + value: Value, + required: bool, + ) -> Result<(), EpisodeSourceError> { + let key = (object_type.to_string(), blob.object_id().to_string()); + if self.values.contains_key(&key) { + return Ok(()); + } + if self.fragments.len() == self.resolver.limits.max_objects { + if required { + return Err(EpisodeSourceError::new( + EpisodeSourceErrorKind::LimitExceeded, + )); + } + self.omit("object_limit", object_type, 1); + return Ok(()); + } + if object_type == CONTEXT_FRAME { + if self.context_fragments == self.resolver.limits.max_context_fragments { + self.omit("context_fragment_limit", object_type, 1); + return Ok(()); + } + self.context_fragments += 1; + } + let redacted = self.resolver.redactor.redact(blob.bytes())?; + let next_bytes = self.redacted_bytes.saturating_add(redacted.len()); + if next_bytes > self.resolver.limits.max_total_bytes + || estimate_tokens(next_bytes) > self.resolver.limits.max_token_estimate + { + if required { + return Err(EpisodeSourceError::new( + EpisodeSourceErrorKind::LimitExceeded, + )); + } + self.omit("source_budget", object_type, 1); + return Ok(()); + } + let text = String::from_utf8(redacted) + .map_err(|_| EpisodeSourceError::new(EpisodeSourceErrorKind::RedactionFailed))?; + let digest = format!("sha256:{}", hex::encode(Sha256::digest(text.as_bytes()))); + let fragment_id = format!("{object_type}:{}", blob.object_id()); + let evidence = EvidenceRefV1 { + schema_version: 1, + source_plane: EvidenceSourcePlane::AgentRuntime, + kind: evidence_kind(object_type), + object_id: blob.object_id().to_string(), + source_ref_oid: self.view.head().to_string(), + locator: evidence_locator(object_type, blob.object_id()), + fragment_digest: digest, + visibility: EvidenceVisibility::RepoLocal, + captured_at: value + .get("created_at") + .and_then(Value::as_str) + .and_then(|timestamp| timestamp.parse().ok()), + code_commit: code_commit(&value), + }; + self.redacted_bytes = next_bytes; + self.values.insert(key, value); + self.fragments.push(RedactedEpisodeFragment { + fragment_id, + object_type: object_type.to_string(), + object_id: blob.object_id().to_string(), + object_oid: blob.oid(), + text, + evidence, + }); + Ok(()) + } + + fn omit(&mut self, code: &str, object_type: &str, count: usize) { + *self + .omissions + .entry((code.to_string(), object_type.to_string())) + .or_default() += count; + } +} + +fn parse_object(object_type: &str, blob: &PinnedHistoryBlob) -> Result { + let value: Value = serde_json::from_slice(blob.bytes()) + .map_err(|_| EpisodeSourceError::new(EpisodeSourceErrorKind::SourceCorrupt))?; + if field_id(&value, "object_id") != Some(blob.object_id()) + || value.get("object_type").and_then(Value::as_str) != Some(object_type) + { + return Err(EpisodeSourceError::new( + EpisodeSourceErrorKind::SourceCorrupt, + )); + } + Ok(value) +} + +fn field_id<'a>(value: &'a Value, field: &str) -> Option<&'a str> { + value.get(field).and_then(Value::as_str) +} + +fn relation_run_id(value: &Value) -> Option<&str> { + field_id(value, "run_id").or_else(|| field_id(value, "run")) +} + +fn code_commit(value: &Value) -> Option { + value + .get("commit") + .and_then(|commit| { + commit + .as_str() + .or_else(|| commit.get("value").and_then(Value::as_str)) + }) + .map(str::to_string) + .or_else(|| field_id(value, "result_commit_sha").map(str::to_string)) +} + +fn parse_timestamp(value: &Value, field: &str) -> Result, EpisodeSourceError> { + field_id(value, field) + .and_then(|timestamp| timestamp.parse().ok()) + .ok_or_else(|| EpisodeSourceError::new(EpisodeSourceErrorKind::SourceCorrupt)) +} + +fn terminal_fact( + values: &BTreeMap<(String, String), Value>, + root_kind: EpisodeRootKind, + root_id: &str, +) -> Result<(CompletionStatus, DateTime), EpisodeSourceError> { + let (event_type, root_field) = match root_kind { + EpisodeRootKind::Task => (TASK_EVENT, "task_id"), + EpisodeRootKind::Intent => (INTENT_EVENT, "intent_id"), + }; + let mut terminal = values + .iter() + .filter(|((object_type, _), value)| { + object_type == event_type && field_id(value, root_field) == Some(root_id) + }) + .filter_map(|((_, object_id), value)| { + let status = match (root_kind, field_id(value, "kind")?) { + (EpisodeRootKind::Task, "done") | (EpisodeRootKind::Intent, "completed") => { + CompletionStatus::Completed + } + (EpisodeRootKind::Task, "failed") => CompletionStatus::Failed, + (EpisodeRootKind::Task, "cancelled") | (EpisodeRootKind::Intent, "cancelled") => { + CompletionStatus::Cancelled + } + _ => return None, + }; + let at = parse_timestamp(value, "created_at").ok()?; + Some((at, object_id.clone(), status)) + }) + .collect::>(); + terminal.sort_by(|left, right| left.0.cmp(&right.0).then_with(|| left.1.cmp(&right.1))); + terminal + .pop() + .map(|(at, _, status)| (status, at)) + .ok_or_else(|| EpisodeSourceError::new(EpisodeSourceErrorKind::InvalidRequest)) +} + +fn derive_code_context( + values: &BTreeMap<(String, String), Value>, + run_ids: &BTreeSet, +) -> EpisodeCodeContextV1 { + let mut run_commits = values + .iter() + .filter(|((object_type, object_id), _)| object_type == RUN && run_ids.contains(object_id)) + .filter_map(|((_, object_id), value)| { + Some(( + parse_timestamp(value, "created_at").ok()?, + object_id, + code_commit(value)?, + )) + }) + .collect::>(); + run_commits.sort_by(|left, right| left.0.cmp(&right.0).then_with(|| left.1.cmp(right.1))); + let base_oid = run_commits.first().map(|(_, _, commit)| commit.clone()); + + let mut result_commits = values + .iter() + .filter_map(|((object_type, object_id), value)| { + if object_type != DECISION && object_type != INTENT_EVENT { + return None; + } + let result = field_id(value, "result_commit_sha") + .or_else(|| field_id(value, "result_commit"))?; + Some(( + parse_timestamp(value, "created_at").ok()?, + object_id, + result.to_string(), + )) + }) + .collect::>(); + result_commits.sort_by(|left, right| left.0.cmp(&right.0).then_with(|| left.1.cmp(right.1))); + let result_oid = result_commits.pop().map(|(_, _, commit)| commit); + let paths = values + .iter() + .filter(|((object_type, _), _)| object_type == PATCHSET) + .flat_map(|(_, value)| { + value + .get("touched") + .and_then(Value::as_array) + .into_iter() + .flatten() + }) + .filter_map(|entry| field_id(entry, "path").map(str::to_string)) + .collect::>() + .into_iter() + .collect(); + EpisodeCodeContextV1 { + base_oid, + result_oid, + branch_ref: None, + paths, + } +} + +fn evidence_kind(object_type: &str) -> EvidenceKind { + match object_type { + INTENT | INTENT_EVENT => EvidenceKind::Intent, + TASK | TASK_EVENT => EvidenceKind::Task, + RUN | RUN_EVENT => EvidenceKind::Run, + EVIDENCE => EvidenceKind::Evidence, + DECISION => EvidenceKind::Decision, + PATCHSET => EvidenceKind::PatchSet, + CONTEXT_FRAME => EvidenceKind::Evidence, + INVOCATION => EvidenceKind::ToolCall, + _ => EvidenceKind::Evidence, + } +} + +fn evidence_locator(object_type: &str, object_id: &str) -> EvidenceLocatorV1 { + if object_type == INVOCATION { + EvidenceLocatorV1::ToolCall { + invocation_id: object_id.to_string(), + part: ToolCallPart::Invocation, + } + } else { + EvidenceLocatorV1::Object + } +} + +fn estimate_tokens(bytes: usize) -> usize { + bytes.saturating_add(3) / 4 +} + +fn redact_private_markers(input: &[u8]) -> Result, EpisodeSourceError> { + const OPEN: &[u8] = b""; + const CLOSE: &[u8] = b""; + const REPLACEMENT: &[u8] = b""; + + let mut output = Vec::with_capacity(input.len()); + let mut cursor = 0; + while let Some(relative_open) = find_bytes(&input[cursor..], OPEN) { + let open = cursor + relative_open; + output.extend_from_slice(&input[cursor..open]); + let body_start = open + OPEN.len(); + let Some(relative_close) = find_bytes(&input[body_start..], CLOSE) else { + return Err(EpisodeSourceError::new( + EpisodeSourceErrorKind::RedactionFailed, + )); + }; + output.extend_from_slice(REPLACEMENT); + cursor = body_start + relative_close + CLOSE.len(); + } + if find_bytes(&input[cursor..], CLOSE).is_some() { + return Err(EpisodeSourceError::new( + EpisodeSourceErrorKind::RedactionFailed, + )); + } + output.extend_from_slice(&input[cursor..]); + Ok(output) +} + +fn find_bytes(haystack: &[u8], needle: &[u8]) -> Option { + haystack + .windows(needle.len()) + .position(|window| window == needle) +} + +#[cfg(test)] +mod tests { + use std::sync::Arc; + + use git_internal::internal::object::{ + context_frame::{ContextFrame, FrameKind}, + intent::Intent, + run::Run, + task::Task, + task_event::{TaskEvent, TaskEventKind}, + types::{ActorRef, ObjectType}, + }; + use sea_orm::DatabaseConnection; + use tempfile::TempDir; + use uuid::Uuid; + + use super::*; + use crate::{ + internal::{ + ai::{ + context_budget::MemoryAnchorConfidence, + history::HistoryManager, + keyed_digest::RepositoryKeyedDigest, + memory::{ + admission::{EpisodeAdmission, EpisodeAdmissionErrorKind}, + compiler::{ + EpisodeClaimProposalV1, EpisodeCompileConfig, EpisodeCompiler, + EpisodeCompilerError, EpisodeCompilerProposalV1, + }, + domain::{ActorKind, ActorRefV1, EpisodeRoot, EpistemicStatus}, + writer::MemoryWriter, + }, + }, + config::ConfigKv, + db, + }, + utils::{storage::local::LocalStorage, storage_ext::StorageExt}, + }; + + const REPOSITORY_ID: &str = "source-test-repository"; + const SECRET: &str = "github_pat_abcdefghijklmnopqrstuvwxyz1234567890"; + + struct Fixture { + _temp: TempDir, + history: HistoryManager, + database: Arc, + digest: Arc, + context: AuthenticatedMemoryContext, + target: TrustedMemoryTarget, + source_head: ObjectHash, + before_task_head: ObjectHash, + unrelated_task_id: String, + } + + struct FakeCompiler { + evidence_fragment_id: String, + } + + #[async_trait::async_trait] + impl EpisodeCompiler for FakeCompiler { + async fn compile( + &self, + _source: &RedactedEpisodeSource, + _config: &EpisodeCompileConfig, + ) -> Result { + let observation = EpisodeClaimProposalV1 { + epistemic_status: EpistemicStatus::Observation, + claim: "the focused test failed before retry".to_string(), + confidence: None, + evidence_fragment_ids: vec![self.evidence_fragment_id.clone()], + }; + let inference = EpisodeClaimProposalV1 { + epistemic_status: EpistemicStatus::Inference, + claim: "retry timing caused the failure".to_string(), + confidence: Some(MemoryAnchorConfidence::Low), + evidence_fragment_ids: vec![self.evidence_fragment_id.clone()], + }; + Ok(EpisodeCompilerProposalV1 { + summary: inference.clone(), + observations: vec![observation], + inferences: vec![inference], + decisions: Vec::new(), + failed_attempts: Vec::new(), + unresolved: Vec::new(), + }) + } + } + + async fn fixture() -> Fixture { + fixture_with_terminal(TaskEventKind::Done).await + } + + async fn fixture_with_terminal(terminal_kind: TaskEventKind) -> Fixture { + let temp = tempfile::tempdir().expect("create source repository"); + let database: DatabaseConnection = db::create_database( + temp.path() + .join("libra.db") + .to_str() + .expect("temporary path must be UTF-8"), + ) + .await + .expect("create source database"); + ConfigKv::set_with_conn(&database, "libra.repoid", REPOSITORY_ID, false) + .await + .expect("persist repository identity"); + ConfigKv::set_with_conn( + &database, + "memory.keyed_digest.v1", + "source-test-ciphertext", + true, + ) + .await + .expect("persist digest configuration fingerprint"); + let database = Arc::new(database); + let storage = Arc::new(LocalStorage::new(temp.path().join("objects"))); + let history = HistoryManager::new( + storage.clone(), + temp.path().to_path_buf(), + Arc::clone(&database), + ); + let actor = ActorRef::agent("source-test-agent").expect("construct actor"); + + let intent = Intent::new(actor.clone(), "Implement bounded memory source") + .expect("construct intent"); + let intent_id = intent.header().object_id(); + storage + .put_tracked(&intent, &history) + .await + .expect("persist intent"); + let before_task_head = history + .resolve_history_head() + .await + .expect("read history head") + .expect("intent commit must exist"); + + let mut task = Task::new( + actor.clone(), + format!( + "Fix retry {SECRET} alice@example.com /Users/alice/project hidden note" + ), + None, + ) + .expect("construct task"); + task.set_intent(Some(intent_id)); + let task_id = task.header().object_id(); + storage + .put_tracked(&task, &history) + .await + .expect("persist task"); + + let run = Run::new(actor.clone(), task_id, "a".repeat(64)).expect("construct run"); + let run_id = run.header().object_id(); + storage + .put_tracked(&run, &history) + .await + .expect("persist run"); + + let mut frame = ContextFrame::new( + actor.clone(), + FrameKind::ErrorRecovery, + "Focused test failed before retry", + ) + .expect("construct context frame"); + frame.set_run_id(Some(run_id)); + storage + .put_tracked(&frame, &history) + .await + .expect("persist context frame"); + + let terminal = + TaskEvent::new(actor.clone(), task_id, terminal_kind).expect("construct terminal"); + storage + .put_tracked(&terminal, &history) + .await + .expect("persist task terminal event"); + + let unrelated = Task::new(actor, "Unrelated task", None).expect("construct unrelated task"); + let unrelated_task_id = unrelated.header().object_id().to_string(); + storage + .put_tracked(&unrelated, &history) + .await + .expect("persist unrelated task"); + let source_head = history + .resolve_history_head() + .await + .expect("read source head") + .expect("source head must exist"); + + let digest = Arc::new(RepositoryKeyedDigest::for_receipt_tests( + REPOSITORY_ID, + Uuid::parse_str("550e8400-e29b-41d4-a716-446655440000") + .expect("fixed key ID must be valid"), + [7; 32], + "source-test-ciphertext", + )); + let context = AuthenticatedMemoryContext::new( + REPOSITORY_ID, + ActorRefV1 { + kind: ActorKind::Agent, + principal_id: "agent:episode-compiler".to_string(), + }, + ) + .expect("construct authenticated context"); + let target = TrustedMemoryTarget::episode( + EpisodeRoot::task(task_id.to_string()).expect("construct task root"), + ); + Fixture { + _temp: temp, + history, + database, + digest, + context, + target, + source_head, + before_task_head, + unrelated_task_id, + } + } + + #[test] + fn private_markers_are_removed_and_malformed_markers_fail_closed() { + assert_eq!( + redact_private_markers(b"before secret after") + .expect("balanced marker must redact"), + b"before after" + ); + assert!(redact_private_markers(b"secret").is_err()); + assert!(redact_private_markers(b"secret").is_err()); + } + + #[test] + fn frozen_limits_reject_zero_and_inverted_candidate_budget() { + assert!(EpisodeSourceLimits::repo_v1().validate().is_ok()); + assert!( + EpisodeSourceLimits { + max_objects: 2, + max_candidate_objects: 1, + ..EpisodeSourceLimits::repo_v1() + } + .validate() + .is_err() + ); + } + + #[tokio::test] + async fn task_source_is_pinned_bounded_related_and_redacted() { + let fixture = fixture().await; + let resolver = EpisodeSourceResolver::new( + &fixture.history, + &fixture.digest, + EpisodeSourceLimits::repo_v1(), + ) + .expect("construct source resolver"); + let source = resolver + .resolve(&fixture.context, &fixture.target, fixture.source_head) + .await + .expect("resolve task source"); + + assert_eq!(source.manifest().root_kind, EpisodeRootKind::Task); + assert_eq!( + source.manifest().limits, + EpisodeSourceLimitSnapshotV1::from(EpisodeSourceLimits::repo_v1()) + ); + assert_eq!(source.manifest().fragments.len(), source.fragments().len()); + assert!( + source + .manifest() + .fragments + .iter() + .zip(source.fragments()) + .all(|(manifest, fragment)| { + manifest.fragment_id == fragment.fragment_id() + && manifest.object_type == fragment.object_type() + && manifest.object_id == fragment.object_id() + && manifest.object_oid == fragment.object_oid().to_string() + && manifest.locator == fragment.evidence().locator + && manifest.fragment_digest == fragment.evidence().fragment_digest + && manifest.code_commit == fragment.evidence().code_commit + }) + ); + let serialized_manifest = + serde_json::to_string(source.manifest()).expect("serialize source manifest"); + assert!(!serialized_manifest.contains(fixture.context.actor().principal_id.as_str())); + assert!( + source + .manifest() + .principal_digest + .starts_with("hmac-sha256:") + ); + assert!( + source + .fragments() + .iter() + .any(|fragment| fragment.object_type() == TASK) + ); + assert!( + source + .fragments() + .iter() + .any(|fragment| fragment.object_type() == INTENT) + ); + assert!( + source + .fragments() + .iter() + .any(|fragment| fragment.object_type() == RUN) + ); + assert!( + source + .fragments() + .iter() + .any(|fragment| fragment.object_type() == CONTEXT_FRAME) + ); + assert!( + source + .fragments() + .iter() + .all(|fragment| fragment.object_id() != fixture.unrelated_task_id) + ); + let compiler_input = source + .fragments() + .iter() + .map(RedactedEpisodeFragment::text) + .collect::>() + .join("\n"); + for forbidden in [SECRET, "alice@example.com", "/Users/alice", "hidden note"] { + assert!(!compiler_input.contains(forbidden)); + } + assert!(compiler_input.contains("")); + assert!(compiler_input.contains("")); + assert!(compiler_input.contains("")); + assert!(source.fragments().iter().all(|fragment| { + fragment.evidence().source_ref_oid == fixture.source_head.to_string() + && fragment.evidence().fragment_digest.starts_with("sha256:") + })); + resolver + .revalidate(&fixture.context, &fixture.target, &source) + .await + .expect("revalidate exact source fragments"); + + let other_context = AuthenticatedMemoryContext::new( + REPOSITORY_ID, + ActorRefV1 { + kind: ActorKind::Agent, + principal_id: "agent:other-compiler".to_string(), + }, + ) + .expect("construct other authenticated context"); + let Err(other_principal) = resolver + .revalidate(&other_context, &fixture.target, &source) + .await + else { + panic!("source manifest must remain bound to the resolving principal"); + }; + assert_eq!( + other_principal.kind(), + EpisodeSourceErrorKind::SourceCorrupt + ); + } + + #[tokio::test] + async fn source_rejects_foreign_identity_and_root_missing_at_pinned_head() { + let fixture = fixture().await; + let resolver = EpisodeSourceResolver::new( + &fixture.history, + &fixture.digest, + EpisodeSourceLimits::repo_v1(), + ) + .expect("construct source resolver"); + let foreign_context = + AuthenticatedMemoryContext::new("foreign-repository", fixture.context.actor().clone()) + .expect("construct foreign context"); + let Err(foreign) = resolver + .resolve(&foreign_context, &fixture.target, fixture.source_head) + .await + else { + panic!("foreign repository must be rejected"); + }; + assert_eq!(foreign.kind(), EpisodeSourceErrorKind::Unauthorized); + + let Err(missing) = resolver + .resolve(&fixture.context, &fixture.target, fixture.before_task_head) + .await + else { + panic!("future root must not be visible from an older pinned head"); + }; + assert_eq!(missing.kind(), EpisodeSourceErrorKind::SourceCorrupt); + + let task_blob = fixture + .history + .get_object_hash(TASK, fixture.target.root().id()) + .await + .expect("resolve task blob") + .expect("task blob must exist"); + let Err(not_a_history_commit) = resolver + .resolve(&fixture.context, &fixture.target, task_blob) + .await + else { + panic!("an arbitrary repository object must not be accepted as source head"); + }; + assert_eq!( + not_a_history_commit.kind(), + EpisodeSourceErrorKind::SourceNotReachable + ); + + let bounded_tree_resolver = EpisodeSourceResolver::new( + &fixture.history, + &fixture.digest, + EpisodeSourceLimits { + max_tree_bytes: 1, + ..EpisodeSourceLimits::repo_v1() + }, + ) + .expect("construct tree-bounded resolver"); + let Err(oversized_tree) = bounded_tree_resolver + .resolve(&fixture.context, &fixture.target, fixture.source_head) + .await + else { + panic!("tree reads must obey the configured byte limit"); + }; + assert_eq!( + oversized_tree.kind(), + EpisodeSourceErrorKind::SourceNotReachable + ); + } + + #[tokio::test] + async fn source_records_stable_omissions_at_object_limit() { + let fixture = fixture().await; + let resolver = EpisodeSourceResolver::new( + &fixture.history, + &fixture.digest, + EpisodeSourceLimits { + max_objects: 2, + ..EpisodeSourceLimits::repo_v1() + }, + ) + .expect("construct bounded source resolver"); + let source = resolver + .resolve(&fixture.context, &fixture.target, fixture.source_head) + .await + .expect("resolve root-only source"); + assert_eq!(source.fragments().len(), 2); + assert!( + source + .manifest() + .omissions + .iter() + .any(|omission| omission.code == "object_limit") + ); + } + + #[tokio::test] + async fn source_preserves_every_task_terminal_outcome() { + for (kind, expected) in [ + (TaskEventKind::Done, CompletionStatus::Completed), + (TaskEventKind::Failed, CompletionStatus::Failed), + (TaskEventKind::Cancelled, CompletionStatus::Cancelled), + ] { + let fixture = fixture_with_terminal(kind).await; + let resolver = EpisodeSourceResolver::new( + &fixture.history, + &fixture.digest, + EpisodeSourceLimits::repo_v1(), + ) + .expect("construct source resolver"); + let source = resolver + .resolve(&fixture.context, &fixture.target, fixture.source_head) + .await + .expect("resolve terminal task source"); + assert_eq!(source.facts().completion_status, expected); + } + } + + #[tokio::test] + async fn admission_maps_only_resolver_evidence_and_preserves_low_confidence() { + let fixture = fixture().await; + let resolver = EpisodeSourceResolver::new( + &fixture.history, + &fixture.digest, + EpisodeSourceLimits::repo_v1(), + ) + .expect("construct source resolver"); + let source = resolver + .resolve(&fixture.context, &fixture.target, fixture.source_head) + .await + .expect("resolve task source"); + let fragment_id = source.fragments()[0].fragment_id().to_string(); + let compiler = FakeCompiler { + evidence_fragment_id: fragment_id, + }; + let config = EpisodeCompileConfig::new( + "libra-memory/1", + 1, + "task-episode-v1", + "deterministic-test-provider", + ) + .expect("construct compiler config"); + let admitted = EpisodeAdmission::new(&fixture.digest) + .compile( + &compiler, + &config, + &fixture.context, + &fixture.target, + source, + ) + .await + .expect("admit deterministic compiler proposal"); + let note = admitted.proposal().note(); + let episode = note.episode.as_ref().expect("Episode payload must exist"); + assert_eq!(episode.completion_status, CompletionStatus::Completed); + for forbidden in [SECRET, "alice@example.com", "/Users/alice", "hidden note"] { + assert!(!episode.goal.claim.contains(forbidden)); + } + assert!(episode.goal.claim.contains("")); + assert_eq!( + episode.inferences[0].confidence, + Some(MemoryAnchorConfidence::Low) + ); + assert_eq!(note.compile_record.producer, "libra-memory/1"); + assert!( + note.compile_record + .idempotency_key + .starts_with("hmac-sha256:") + ); + assert!( + note.evidence_refs + .iter() + .all(|evidence| evidence.source_ref_oid == fixture.source_head.to_string()) + ); + } + + #[tokio::test] + async fn admission_rejects_compiler_invented_evidence_fragment() { + let fixture = fixture().await; + let resolver = EpisodeSourceResolver::new( + &fixture.history, + &fixture.digest, + EpisodeSourceLimits::repo_v1(), + ) + .expect("construct source resolver"); + let source = resolver + .resolve(&fixture.context, &fixture.target, fixture.source_head) + .await + .expect("resolve task source"); + let config = EpisodeCompileConfig::new( + "libra-memory/1", + 1, + "task-episode-v1", + "deterministic-test-provider", + ) + .expect("construct compiler config"); + let Err(error) = EpisodeAdmission::new(&fixture.digest) + .compile( + &FakeCompiler { + evidence_fragment_id: "task:invented".to_string(), + }, + &config, + &fixture.context, + &fixture.target, + source, + ) + .await + else { + panic!("invented evidence fragment must be rejected"); + }; + assert_eq!(error.kind(), EpisodeAdmissionErrorKind::InvalidProposal); + } + + #[tokio::test] + async fn writer_commits_only_after_exact_source_revalidation() { + let fixture = fixture().await; + let resolver = EpisodeSourceResolver::new( + &fixture.history, + &fixture.digest, + EpisodeSourceLimits::repo_v1(), + ) + .expect("construct source resolver"); + let source = resolver + .resolve(&fixture.context, &fixture.target, fixture.source_head) + .await + .expect("resolve task source"); + let compiler = FakeCompiler { + evidence_fragment_id: source.fragments()[0].fragment_id().to_string(), + }; + let config = EpisodeCompileConfig::new( + "libra-memory/1", + 1, + "task-episode-v1", + "deterministic-test-provider", + ) + .expect("construct compiler config"); + let admitted = EpisodeAdmission::new(&fixture.digest) + .compile( + &compiler, + &config, + &fixture.context, + &fixture.target, + source, + ) + .await + .expect("admit compiler proposal"); + let writer = MemoryWriter::for_tests( + fixture._temp.path().to_path_buf(), + Arc::clone(&fixture.database), + Arc::clone(&fixture.digest), + ) + .await + .expect("construct Memory writer"); + let committed = writer + .commit_admitted( + &resolver, + &fixture.context, + &fixture.target, + &admitted, + None, + ) + .await + .expect("commit revalidated admitted Episode"); + assert!(committed.appended()); + assert_eq!(committed.note_id(), fixture.target.root().note_id()); + assert_eq!(committed.event_seq(), 2); + } + + #[test] + fn object_type_constants_match_persisted_git_internal_names() { + assert_eq!(ObjectType::Task.to_string(), TASK); + assert_eq!(ObjectType::Intent.to_string(), INTENT); + assert_eq!(ObjectType::Run.to_string(), RUN); + assert_eq!(ObjectType::ContextFrame.to_string(), CONTEXT_FRAME); + } +} diff --git a/src/internal/ai/memory/writer.rs b/src/internal/ai/memory/writer.rs index 7d3a50094..a34d63873 100644 --- a/src/internal/ai/memory/writer.rs +++ b/src/internal/ai/memory/writer.rs @@ -5,6 +5,7 @@ use git_internal::hash::ObjectHash; use uuid::Uuid; use super::{ + admission::AdmittedEpisodeProposal, canonical::memory_note_content_digest_v1, domain::{MemoryEventAction, MemoryEventV1, MemoryNoteV1}, error::{MemoryWriterError, MemoryWriterErrorKind}, @@ -12,6 +13,7 @@ use super::{ AuthenticatedMemoryContext, DeterministicMemoryProposal, TrustedMemoryTarget, validate_writer_policy, }, + source::{EpisodeSourceErrorKind, EpisodeSourceResolver}, store::{ ProjectedCell, ProjectionMutation, find_cell, read_memory_ref_head, validate_projection_watermark, @@ -127,7 +129,7 @@ impl MemoryWriter { } #[cfg(test)] - async fn for_tests( + pub(in crate::internal::ai::memory) async fn for_tests( storage_path: PathBuf, database: Arc, digest_provider: Arc, @@ -150,12 +152,43 @@ impl MemoryWriter { *slot = Some(barrier); } + pub(crate) async fn commit_admitted( + &self, + resolver: &EpisodeSourceResolver<'_>, + context: &AuthenticatedMemoryContext, + target: &TrustedMemoryTarget, + admitted: &AdmittedEpisodeProposal, + expected_head: Option, + ) -> Result { + resolver + .revalidate(context, target, admitted.source()) + .await + .map_err(|error| { + let kind = writer_error_kind_for_source(error.kind()); + MemoryWriterError::new(kind, "Episode source evidence could not be revalidated") + })?; + self.commit_validated(context, target, admitted.proposal(), expected_head) + .await + } + + #[cfg(test)] pub(crate) async fn commit( &self, context: &AuthenticatedMemoryContext, target: &TrustedMemoryTarget, proposal: &DeterministicMemoryProposal, expected_head: Option, + ) -> Result { + self.commit_validated(context, target, proposal, expected_head) + .await + } + + async fn commit_validated( + &self, + context: &AuthenticatedMemoryContext, + target: &TrustedMemoryTarget, + proposal: &DeterministicMemoryProposal, + expected_head: Option, ) -> Result { self.digest_provider .validate_for_connection(self.database.as_ref()) @@ -463,6 +496,18 @@ impl MemoryWriter { } } +const fn writer_error_kind_for_source(kind: EpisodeSourceErrorKind) -> MemoryWriterErrorKind { + match kind { + EpisodeSourceErrorKind::DigestUnavailable => MemoryWriterErrorKind::DigestKeyUnavailable, + EpisodeSourceErrorKind::LimitExceeded => MemoryWriterErrorKind::SourceLimitExceeded, + EpisodeSourceErrorKind::Unauthorized + | EpisodeSourceErrorKind::InvalidRequest + | EpisodeSourceErrorKind::SourceNotReachable + | EpisodeSourceErrorKind::RedactionFailed => MemoryWriterErrorKind::SourceRejected, + EpisodeSourceErrorKind::SourceCorrupt => MemoryWriterErrorKind::EvidenceMismatch, + } +} + async fn validate_repository_binding( database: &sea_orm::DatabaseConnection, digest_provider: &RepositoryKeyedDigest, @@ -531,6 +576,30 @@ pub(in crate::internal::ai::memory) mod tests { utils::{client_storage::ClientStorage, test::ChangeDirGuard}, }; + #[test] + fn source_failures_map_to_stable_writer_categories() { + for (source, expected) in [ + ( + EpisodeSourceErrorKind::DigestUnavailable, + MemoryWriterErrorKind::DigestKeyUnavailable, + ), + ( + EpisodeSourceErrorKind::LimitExceeded, + MemoryWriterErrorKind::SourceLimitExceeded, + ), + ( + EpisodeSourceErrorKind::SourceNotReachable, + MemoryWriterErrorKind::SourceRejected, + ), + ( + EpisodeSourceErrorKind::SourceCorrupt, + MemoryWriterErrorKind::EvidenceMismatch, + ), + ] { + assert_eq!(writer_error_kind_for_source(source), expected); + } + } + const REPOSITORY_ID: &str = "memory-writer-test-repository"; const TEST_CIPHERTEXT: &str = "memory-writer-test-ciphertext"; const SOURCE_OID: &str = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; From 71236588ea2b65db036796c24f103ad0ddb9cff2 Mon Sep 17 00:00:00 2001 From: anduin9527 Date: Tue, 25 Aug 2026 11:18:30 +0800 Subject: [PATCH 11/18] feat(memory): add recoverable generation jobs Signed-off-by: anduin9527 --- docs/development/tracing/memory.md | 43 +- src/internal/ai/history.rs | 284 ++++++- src/internal/ai/mcp/server.rs | 14 +- src/internal/ai/memory/admission.rs | 11 +- src/internal/ai/memory/job.rs | 80 ++ src/internal/ai/memory/job_sql.rs | 749 ++++++++++++++++++- src/internal/ai/memory/job_state.rs | 270 +++++++ src/internal/ai/memory/mod.rs | 6 + src/internal/ai/memory/observer.rs | 786 ++++++++++++++++++++ src/internal/ai/memory/runner.rs | 548 ++++++++++++++ src/internal/ai/memory/source.rs | 1 + src/internal/ai/memory/store.rs | 16 + src/internal/ai/memory/tree.rs | 21 +- src/internal/ai/memory/writer.rs | 81 +- src/internal/ai/orchestrator/persistence.rs | 15 +- 15 files changed, 2889 insertions(+), 36 deletions(-) create mode 100644 src/internal/ai/memory/job.rs create mode 100644 src/internal/ai/memory/job_state.rs create mode 100644 src/internal/ai/memory/observer.rs create mode 100644 src/internal/ai/memory/runner.rs diff --git a/docs/development/tracing/memory.md b/docs/development/tracing/memory.md index cd1b9ae6c..5db0da986 100644 --- a/docs/development/tracing/memory.md +++ b/docs/development/tracing/memory.md @@ -1449,12 +1449,49 @@ expected 时正常推进；当前值已经等于 `scanned_through_oid` 时返回 batch。first-parent 后代关系与扫描预算由 M2-08 observer 在调用前验证，SQL Module 不读取 Git 对象。 -本卡只实现上述 observation transaction。ref 扫描、terminal 判断、canonical -input 构造、lease acquire/renew/takeover、runner、compiler、retry scheduling 与 -processed-generation completion 均属于 M2-08。down migration 仅在九张 M2-02 +M2-02 只实现上述 observation transaction；M2-08 在该 Interface 上补齐 ref 扫描、 +terminal 判断、canonical input 构造、lease acquire/takeover、runner、retry scheduling +与 processed-generation completion。down migration 仅在九张 M2-02 表全部为空时按 `observer/job → episode_path/link/head → revision → note → projection_state/path_summary` 的依赖顺序删除；任何表有行都以稳定错误拒绝，不能级联丢状态。 +#### 5.2.2 M2-08 terminal observer 与 generation runner + +自动编译由两条持久 observer 驱动。AI 历史 observer 只把 +`TaskEventKind::{Done, Failed, Cancelled}` 和 +`IntentEventKind::{Completed, Cancelled}` 视为外部触发；其它 append 仍被扫描， +但只推进 `libra/intent` 水位。Memory observer 扫描 +`libra/memory/repo`，只接受已经 `Confirmed` 的 Task Episode revision，并重新唤醒 +已经有终态 job 的父 Intent。Intent Episode revision 被忽略，因此编译输出不会反向 +触发自身。 + +两条 observer 都先固定 ref head，再沿 first-parent 读取不可变 append。单次扫描上限为 +2,048 个 commit，tree 与 blob 也有独立字节上限；head 非 cursor 后代、merge、损坏对象 +或预算超限都会报错，SQLite cursor 保持原值。完整扫描后，root job upsert 与 cursor +推进复用 §5.2.1 的同一短事务，进程在事务前后退出都可以通过下次启动修复重放。 + +规范输入指纹使用仓库 keyed digest： + +- Task：`root_id + terminal_source_commit_oid`； +- Intent：`root_id + terminal_source_commit_oid + 按 task_id 排序的 + (task_id, live_revision_oid | missing)`。 + +指纹和 terminal source 都未变化时不增加 generation。Task revision 变化只会改变父 +Intent 的指纹；缺失 revision 使用稳定的 `missing` marker，后续补齐时自然产生下一代。 + +runner 每次只领取一行可运行 job。领取会写入 owner、30 秒 deadline 和单调递增 fence； +过期 lease 可被新 owner 接管。编译完成后，Memory ref CAS、投影更新和 lease 有效性校验 +处于同一个 SQLite write transaction：owner/fence 已失效或 deadline 已过的 runner 不能 +提交权威 revision。完成、失败和释放也都带 owner/fence 条件，旧 runner 无法覆盖新 owner +的状态。编译期间出现新 generation 时，旧 generation 可以先完成，job 随后回到 `dirty` +继续处理最新输入。 + +transient failure 最多重试五次，使用从 500 ms 开始、上限 30 s 的指数退避；稳定的 +schema/auth/policy failure 进入 `failed`，直到新 generation 到来。job 只保存 +`LBR-MEMORY-201..206` 和经默认 Redactor 脱敏、UTF-8 安全截断到 1,024 bytes 的摘要， +不保存 provider 原始响应、prompt 或 transcript。终态写入只调度后台 observer，不等待 +扫描和模型；MCP runtime 构造时复用同一 repair seam 补齐崩溃窗口。 + `memory_head`、`memory_path_summary`、`memory_note_index`、`memory_revision_index`、`memory_link_index`、`memory_projection_state`、`memory_episode_path` 是 M2-02 的可重建投影。`memory_compile_job` 与 `memory_compile_observer_state` 是有界本地运行状态。后续的 `memory_entity_index`、`memory_taxonomy_node` 仍是可重建投影；`memory_classifier_cache` 与 `memory_embedding_cache`（§8.7）是可丢弃 cache；`memory_access_stats`、`context_selection_receipt` 与 `context_selection_receipt_retention` 是本地有界账本及其保留水位，不能从 Git 历史重建，`rebuild` 不触碰它们。删除账本会降低本地可观测性，但不能改变 live memory 语义。 查询实现必须始终带上 `scope_key` 与 `namespace`，禁止只按 `path` 做全局查询后在内存中过滤。跨 scope / namespace 的检索只能由显式 `--all-namespaces` 或策略允许的 scope fallback 触发，并且必须在结果中保留原始 `scope` 与 `namespace`，防止 prompt 注入时发生来源混淆。 diff --git a/src/internal/ai/history.rs b/src/internal/ai/history.rs index 4f39135db..483f56f47 100644 --- a/src/internal/ai/history.rs +++ b/src/internal/ai/history.rs @@ -170,6 +170,56 @@ pub(crate) struct PinnedHistoryEntry { oid: ObjectHash, } +/// One append commit decoded from a bounded first-parent history interval. +/// The source commit is retained separately from the leaf blob OID because +/// compiler generation identity is anchored to the immutable history point. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PinnedHistoryAppend { + source_commit_oid: ObjectHash, + object_type: String, + object_id: String, + object_oid: ObjectHash, + bytes: Vec, +} + +impl PinnedHistoryAppend { + pub(crate) const fn source_commit_oid(&self) -> ObjectHash { + self.source_commit_oid + } + + pub(crate) fn object_type(&self) -> &str { + &self.object_type + } + + pub(crate) fn object_id(&self) -> &str { + &self.object_id + } + + pub(crate) const fn object_oid(&self) -> ObjectHash { + self.object_oid + } + + pub(crate) fn bytes(&self) -> &[u8] { + &self.bytes + } +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct PinnedHistoryDelta { + head: ObjectHash, + appends: Vec, +} + +impl PinnedHistoryDelta { + pub(crate) const fn head(&self) -> ObjectHash { + self.head + } + + pub(crate) fn appends(&self) -> &[PinnedHistoryAppend] { + &self.appends + } +} + /// Read-only view whose head was proven to belong to the current first-parent /// history. Callers cannot accidentally fall through to the moving ref. pub(crate) struct PinnedHistoryView<'a> { @@ -297,7 +347,13 @@ fn validate_history_path_part(label: &str, value: &str) -> Result<()> { Ok(()) } -fn read_single_parent_commit(repo_path: &Path, oid: ObjectHash) -> Result> { +struct LinearHistoryCommit { + tree_oid: ObjectHash, + parent_oid: Option, + message: String, +} + +fn read_linear_history_commit(repo_path: &Path, oid: ObjectHash) -> Result { let (kind, bytes) = read_git_object_bounded_validated(repo_path, &oid, PINNED_HISTORY_COMMIT_MAX_BYTES) .with_context(|| format!("failed to read AI history commit {oid}"))?; @@ -306,28 +362,60 @@ fn read_single_parent_commit(repo_path: &Path, oid: ObjectHash) -> Result