diff --git a/.gitignore b/.gitignore index b3781e7..33bb0fb 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,8 @@ research/ *.db-journal *.db-wal *.db-shm + +# Generated by `make -C doc` from engram.texi. +doc/engram.info +doc/engram.html +doc/engram.pdf diff --git a/AGENTS.md b/AGENTS.md index b8ed1ef..308c62f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,6 +42,17 @@ cargo doc -p rmcp --open # verify rmcp 0.16 macro shape if build SQLite + FTS5, WAL journal mode, 5s busy timeout. The `memories_fts` virtual table is kept in sync via `AFTER INSERT`/`AFTER DELETE` triggers plus a **content-narrowed** `AFTER UPDATE OF content` trigger (M5 — access-tracking bumps and supersession updates must not churn the FTS index; `migrate()` drop+recreates it on every open since SQLite has no `CREATE OR REPLACE TRIGGER`). FTS5 query sanitization in `sanitize_fts_query` wraps every token as escaped quoted phrase — free-text queries must not hit raw FTS5 syntax. +## Project-root resolution + +`managed_file::find_git_root` walks up looking for a **working tree**, not for +the mere existence of `.git`. A directory must contain `.git/HEAD`; a `.git` +*file* (worktree or submodule pointer) also counts. Existence alone is not +enough: an empty `/tmp/.git` on the author's machine made `save-chat` resolve +its project root to `/tmp`, create `/tmp/chat/`, and add `chat/` to +`/tmp/.gitignore`. Any test that resolves a project root must plant its own +marker (`pinned_project` in `tests/cli.rs`) rather than depending on whether an +ancestor of the tempdir happens to look like a repository. + ## Conventions (Spacecraft Software Standard) - SPDX `SPDX-FileCopyrightText` + `SPDX-License-Identifier: GPL-3.0-or-later` on every `.rs` file. @@ -117,3 +128,163 @@ engram rule sync [--scope S] [--file PATH]... [--dry-run] - Call `search` before asserting something was already decided. - CLI, MCP, and HTTP hit the same `Store`; behavior is identical for remember/recall/search. Rules are on all three surfaces too. + +## Building and running + +**Build:** +```sh +cargo build --release # LTO, 1 codegen unit, panic=abort per Standard §3 +``` + +Verify the `rmcp` 0.16 macro shape (if build fails due to macro mismatch): +```sh +cargo doc -p rmcp --open +``` + +**Run (CLI):** +```sh +./target/release/engram remember --agent claude-code --scope my-task "Decided: X stays synchronous." +./target/release/engram recall --scope my-task +./target/release/engram search "synchronous" +``` + +**Run as MCP server** (wire into Claude Code / other clients): +```sh +./target/release/engram mcp --db ./engram.db +``` + +**Run as HTTP server** (local-only, `127.0.0.1:8420`, no auth): +```sh +./target/release/engram serve --db ./engram.db --addr 127.0.0.1:8420 +curl -s -XPOST localhost:8420/v1/memory -d '{"agent":"kimi","scope":"x","content":"..."}' +curl -s "localhost:8420/v1/memory/recall?scope=x&limit=10" +``` + +`cargo test` covers the rules subsystem (unit tests in `rules.rs`/`store.rs`) and the memory surfaces (integration tests in `tests/cli.rs`). CI runs rustfmt, clippy, and the test suite via `.github/workflows/ci.yml`. + +## Command reference + +**CLI:** +- `remember --agent --scope [--role ] [--dry-run] []` — store a message (or read from stdin); `--dry-run` validates and shows what would be stored without writing +- `recall --scope [--limit ] [--budget-tokens ]` — fetch last N messages for a scope (default 50). With `--budget-tokens`, results are packed to the budget newest-first (the oldest drop), output stays chronological, and the envelope carries `metadata.budget` +- `search [--scope ] [--limit ] [--budget-tokens ]` — full-text search (default limit 20). With `--budget-tokens`, results are packed in rank order and the envelope carries `metadata.budget` +- `context [--scope ] [--query ] [--budget-tokens ] [--limit ]` — assemble a budget-packed context block for session start: active rules first (**always all included**, even over budget — policy is never silently dropped), then memories selected newest-first, or by reciprocal-rank fusion of recency+FTS relevance+extracted-fact channels when `--query` is given; included memories are presented chronologically. Defaults: budget 3000, limit 50 per channel; scope resolves via the same cascade as the `rule` commands +- `consolidate [--extract] [--dedup [--yes]] [--report] [--scope ] [--dry-run]` — idle-time maintenance, three combinable phases (at least one required, else InvalidArgument exit 2): `--extract` (M4) runs the deterministic fact extractor and upserts into the facts index (`--dry-run` applies here only); `--dedup` (M5) finds near-duplicate groups of current non-rule memories per scope — normalized-exact text always, stored-vector cosine ≥ 0.92 when the hybrid gate passes, edges unioned — reporting winner (newest) + losers, and **only with `--yes`** supersedes each loser via `mark_superseded_by` (M2 semantics, no new row, never a delete; idempotent); `--report` (M5) is always report-only: contradiction pairs (word-set Jaccard ≥ 0.5 + negation marker on exactly one side — a heuristic; resolve via `remember --supersedes`, never auto-resolved) and the top-20 decay candidates (`staleness = age_days + 30/(1+access_count)`). Data shape is one optional section per phase: `{extract?, dedup?, report?}`. `--scope` omitted means **every** scope — no cascade, unlike the rule commands. CLI-only by design +- `save-chat [--scope ] [--file ] [--model ] [--dry-run]` — archive a scope's history as a complete Texinfo document. Paths resolve against the **project root** (`rules::resolve_scope`), not the process cwd, so the command targets the same file from any subdirectory; `--file` defaults to `chat/.texi`, and `chat/` is added to `.gitignore` when absent (reported as the `gitignore` object — `action` is `added`, `already-ignored` or `would-add` — not done silently). `--model` names the archiving model (falls back to `MODEL`/`LLM_MODEL`/`AI_AGENT`/`AGENT`, then `unknown-model`). **The document is a pure function of the scope**, exactly as the rules block is: an existing archive is rewritten *whole* (never appended to), re-running over an unchanged scope is byte-identical and reports `outcome: "unchanged"`, and the provenance header therefore carries the *last message's* timestamp rather than the export's wall clock. Rules are excluded (an archive is a transcript) and the read is untracked via `Store::export_history` (archiving is not retrieval — counting it would corrupt the M5 decay signal); superseded rows *are* included +- `ingest [--harness ] [--session ] [--scope ] [--cwd ] [--include-thinking] [--include-tools] [--include-sidechains] [--max-bytes ] [--max-chars-per-turn ] [--list] [--dry-run]` — **capture a harness's own session transcript** into a scope as ordinary memories, with roles `user`/`assistant` (values the schema always declared and nothing ever wrote until now). This is what makes "verbatim chat memory" literally true; before it, `save-chat` could only export what an agent chose to `remember`. Harness resolution: `--harness` → environment marker (`CLAUDECODE` etc.) → the single installed harness with a reader — **two candidates is an error, never a guess**. Scope maps by cwd through the same `rules::resolve_scope` cascade. `--list` reports sessions *plus* the whole harness table, so "no sessions here" is distinguishable from "cannot read this harness". CLI-only +- `rule add --id [--scope ] [--agent ] []` — record or revise a rule (stdin if text omitted) +- `rule list [--scope ] [--include-retired]` — rules in effect, ordered by id +- `rule retire --id [--scope ]` — withdraw a rule (tombstone; re-adding reinstates) +- `rule purge --id [--scope ] --yes [--dry-run]` — permanently delete a **retired** rule's row (the one true delete; CLI-only — destructive ops are not agent-invocable) +- `rule sync [--scope ] [--file ]... [--dry-run]` — render rules into `AGENTS.md`/`CLAUDE.md` +- `install [--harness ]... [--db-path ] [--list] [--dry-run] [--force]` — write engram's slash commands (`/engram-save-chat`, `/engram-ingest`, `/engram-context`) into each detected harness's own command directory. **Start with `--list`.** Only *detected* harnesses are written to (engram never `mkdir`s a home for software you don't have), only files carrying engram's banner are overwritten (`--force` overrides, and a hand-written file is reported `skipped` with a reason), and nothing is deleted. Idempotent by byte comparison — a second run reports every file `unchanged` and does not touch mtimes. **CLI-only, never an HTTP route** (see below) +- `mcp` — run as MCP server (stdio) +- `serve [--addr ]` — run HTTP server (default `127.0.0.1:8420`) +- `schema` — print JSON Schema, as `{"Memory": ..., "Rule": ...}` +- `describe` — print CLI Standard capability manifest (JSON) + +**Global flags:** +- `--db ` — database file (env: `ENGRAM_DB`, default: `engram.db`) +- `--json` — machine output; alias for `--format json` +- `--format ` — machine output format, overrides mode auto-detection. `jsonl`: first line is `{"metadata":...,"data":null}`, then one line per record (arrays) or one line with the object. `csv`: RFC 4180 rows on stdout (header from the first record's keys), metadata as one JSON line on stderr. `yaml`/`explore` are deferred +- `--no-color` — disable colors (respects `NO_COLOR` env var) +- `--accessible` — accessible output per Standard §18: plain linear text, no color, status tags. Also enabled by `SPACECRAFT_A11Y=1`; the flag wins over `SPACECRAFT_A11Y=0` +- `--no-track` — read-only auditing: do not update access counts on reads. CLI-only; MCP/HTTP have no opt-out (agent reads are exactly what the tracking measures) + +## Environment + +- `ENGRAM_DB` — override database path +- `ENGRAM_SCOPE` — default scope for `rule` commands +- `ENGRAM_AGENT` — default `--agent` for `rule add` +- `AI_AGENT`, `AGENT` (set non-empty), `CI` (truthy) — trigger machine output mode (detected for structured logging in CI/agent contexts) +- `SPACECRAFT_A11Y` — `1` enables accessible output, `0` disables auto-detection (`--accessible` still wins) +- `NO_COLOR` — disable colors + +## Semantic search (the `vector` feature, M3) + +Opt-in at build time: `cargo build --release --features vector`. The default build stays FTS5-only with zero ML dependencies. Facts: + +- **Engine:** Model2Vec static embeddings via `model2vec-rs` compiled with `default-features = false` + `local-only` — the hf-hub network fetch path is compiled OUT; engram never downloads a model (§9 PFA). Install one by hand (e.g. `minishlab/potion-base-8M`: `model.safetensors` + `tokenizer.json` + `config.json`). +- **Model cascade:** `--model-path` → `ENGRAM_MODEL` → `$XDG_DATA_HOME/engram/model` (default `~/.local/share/engram/model`). The directory basename is the model name in `memory_vectors.model`; vectors from different models are never compared. +- **Storage:** `memory_vectors` side table (memory_id PK/FK, model, dim, embedding BLOB f32-LE) — deliberately NOT sqlite-vec (alpha C extension vs §5.5 packaging) and NOT columns on `memories` (FTS triggers untouched). Similarity is brute-force cosine: sub-millisecond under 100k rows. +- **Indexing:** `engram remember` embeds live on the CLI when a model resolves; `engram index [--scope] [--batch] [--dry-run]` backfills everything else (MCP/HTTP writes, pre-model history). Rule rows are never embedded. +- **Retrieval:** `search --mode fts|hybrid`; omitted, hybrid engages automatically when (feature ∧ model resolves ∧ vectors indexed), else fts. Explicit `--mode hybrid` with a missing prerequisite is a structured exit-2/HTTP-400 error, never a silent fallback. Hybrid = FTS top-50 + cosine top-50 → `rrf_fuse(k=60)`; `context` gains the vector as a third channel the same way. +- **The gate:** measured 2026-08-02 on the held-out `bench/queries.jsonl` (frozen before implementation): hybrid 0.918 vs fts 0.856 recall@5 = +6.2 points ≥ +5 → PASS; the margin is entirely conceptual/synonym queries. See `bench/RESULTS.md` — including why the first (+77.9) measurement was rejected as a baseline defect. + +## Extracted-fact index (M4) + +The TencentDB L0↔L1 pattern: L0 is the verbatim memory, L1 is a *derived index* of the decision/constraint sentences inside it. Facts never replace verbatim — each `facts` row is a verbatim substring of its parent's content plus a drill-down pointer (`memory_id` → `engram get` / the MCP `get` tool). + +- **Extractor: `deterministic-v1` only — no LLM on the write path, ever.** `facts::extract` splits content into lines (plus sentence-splits of multi-sentence lines), trims bullet markers, and keeps units that start (case-insensitively) with one of 19 markers (`Decided:`, `Decision:`, `TODO`, `FIXME`, `NOTE:`, `Rule:`, `Fix:`, `Fixed:`, `Chose:`, `Chosen:`, `Rejected:`, `Constraint:`, `Gotcha:`, `Warning:`, `Never `, `Always `, `Must `, `Do not `, `Don't `). Floor 12 chars, cap 8 facts per memory (first eight distinct in document order), exact-dedupe. Facts are stored verbatim — rewriting would be the lossy-extraction trap. +- **Liveness derives from the parent.** Extraction is append-only (`INSERT OR REPLACE` on deterministic v5 ids — idempotent, re-runs don't grow the table); nothing deletes facts when a memory is superseded. Instead `fact_candidates` JOINs `memories` and applies the validity filter to the parent, so stale facts stop surfacing the moment their parent does. The fact columns `valid_to`/`superseded_by` are reserved and stay NULL. +- **Channel wiring.** With a `--query`, `context` fuses recency + FTS + **facts** (parents of matching facts, deduped, rank order) — plus vector when the hybrid gate passes — and reports `channels.facts`. Hybrid search is now fts + vector + facts. Plain FTS `search` is unchanged (memories only; facts are substrings of content, so the channel can only *boost* the memory that states a decision above ones that merely mention its words — it can never be a sole finder). +- **CLI-only, on purpose.** `engram consolidate --extract` exists on neither MCP nor HTTP: extraction is an operator's idle-time batch job; agents get facts through `context`/hybrid ranking automatically. CLI-only (see the MCP tool ledger below for the one canonical count). +- Rule rows are never extracted from — policy travels through the rules section, not retrieval. + +## Idle consolidation + decay (M5) + +`engram consolidate` grew two phases beyond `--extract` (all combinable; at least one required; still CLI-only): + +- **`--dedup [--yes]`** — near-duplicate detection over the CURRENT, non-rule memories of each scope. Two detectors run and their edges are **unioned** into connected components: *exact* (normalized text: trim, lowercase, collapse internal whitespace — always on) and *vector* (cosine ≥ 0.92 between **stored** embeddings, same-scope pairs only — runs exactly when the auto-hybrid gate would pass: feature ∧ model resolves ∧ vectors indexed). Each group's NEWEST row (max `created_at`, id tie-break) wins. Without `--yes` it is report-only; with `--yes` every loser goes through `Store::mark_superseded_by(loser, winner, now)` — **M2 supersession semantics reused** (`valid_to` + `superseded_by` set, `WHERE valid_to IS NULL`), *not* `remember_superseding`: no new row is inserted because the winner already exists. Dedup NEVER deletes, and it is idempotent — superseded losers are no longer Current, so a second run finds nothing. +- **`--report`** — always report-only, two sections. (a) *Contradictions*: pairs of CURRENT same-scope non-rule memories with word-set Jaccard ≥ 0.5 AND a negation marker (`not `, `never `, `no longer `, `don't `, `do not `, `isn't `, `wasn't `, `stopped `) on exactly one side. A documented heuristic — a human or agent resolves via `remember --supersedes`; the tool never auto-resolves. (b) *Decay*: every CURRENT non-rule memory scored `staleness = age_days * 1.0 + 30.0/(1+access_count)` (crude, but monotone in age and un-accessedness), top 20 returned with age, access_count, last_accessed_at. + +**Access tracking** feeds the decay signal: `recall`/`search`/`search_hybrid`/`context`/`get` bump `access_count`/`last_accessed_at` at the end of the read, inside the same lock, for the memories actually **returned** (never dropped candidates, never the rules section — `rules()` is untracked, and dry-runs write nothing). The columns are internal: `Memory` serialization is byte-identical with or without them. Opt-out is the global `--no-track` CLI flag (read-only auditing), wired right after `Store::open`; MCP and HTTP have no opt-out — agent reads are exactly what the tracking measures. + +## MCP tool ledger — the canonical count + +**Ten tools, and the ceiling is now reached.** This is the single place the count lives; it used to be restated in three sections and drifted. `src/mcp.rs`'s module doc carries the same statement for readers who are in the code. + +`remember`, `recall`, `search`, `get`, `context`, `rule_add`, `rule_list`, `rule_retire`, `rule_sync`, `save_chat`. + +Every tool's schema costs context on every turn of every conversation, which is why the cap exists (`doc/engram.texi`). **An eleventh tool must displace an existing one, and the displacement must be argued in the manual.** + +- `save_chat` (M4) earned the last slot only because it carries *both halves* of the capture story: `from_transcript: true` captures the session, then archives. Spending the slot on archiving alone would have left MCP able to export a conversation but never record one, with no slot left to fix it. +- **No `file` argument, ever.** The destination derives from the server's resolved project root. A caller-chosen path is a traversal primitive handed to a model whose input includes attacker-influenceable text — the same reasoning that keeps `--file` off `rule_sync`'s MCP surface. +- Deliberately CLI-only and **not** candidates for the slot: `install` (writes into `$HOME`), `ingest` (agents reach it through `save_chat --from-transcript`), `consolidate`/`index` (operator batch jobs whose results arrive through ranking anyway), `rule purge` (destructive ops are not agent-invocable). +- Both surfaces share one implementation: `archive::save_chat` and `transcript::capture` are called by the CLI *and* the MCP tool, so they cannot drift in what they write, filter, redact, or count. + +## Transcript capture (`engram ingest`) + +`src/harness.rs` + `src/transcript/{mod,claude_code,redact}.rs`. Reads the session file a harness already writes for itself and stores each message as an ordinary memory, so `recall`/`search`/`context`/`consolidate` see the real conversation. + +Two readers exist: `claude_code` and `codex`. Adding a third means adding a `ReaderKind` variant, which the two `match`es in `transcript/mod.rs` then force you to handle. + +- **Codex layout:** `~/.codex/sessions/YYYY/MM/DD/rollout--.jsonl`. The tree encodes the **date, not the cwd**, so there is nothing to mangle — each rollout's first record is a `session_meta` carrying `cwd` verbatim, and listing reads exactly that one line per file. +- **Codex has two channels, and `event_msg` wins.** `event_msg` is what the UI displayed (flat strings); `response_item` is the raw API traffic. `event_msg` is primary not merely because it parses more easily but because it is *less* noisy: on a real rollout it held 2 user messages where `response_item` held 3, and the extra one was an `` block the harness injects. `response_item` is a fallback used only when a rollout has no `event_msg` conversation at all, so retiring the display channel would degrade rather than silently yield nothing. When the display channel wins, the raw duplicates are counted as `non_message`. +- **Codex session ids are per-rollout, NOT `session_meta.session_id`.** That field is *not unique* — resuming a session writes a new file reusing the same id, and three files sharing one id exist on this machine. Since `turn_id` derives from the session id, reusing it would collide turns at the same line index across rollouts and `INSERT OR IGNORE` would silently drop them. Engram therefore keys on the file name minus `rollout-` (unique, sortable, still contains the uuid). There is a test for exactly this. +- **Codex records carry no per-record id**, so `source_uuid` is `{line_index}:{v5 digest of the text}`. The index alone would suffice for an append-only log; folding in the content means an inserted line does not renumber every later turn into a new identity. +- **`--max-bytes` is not theoretical.** A 114 MB rollout exists on this machine; the 64 MiB default refuses it with a structured error naming the override. Both readers stream line by line. +- **Claude Code layout:** `~/.claude/projects//.jsonl`. `mangle_cwd` replaces every `/` with `-` (so the leading slash becomes a leading dash) and **preserves case** — `-spacecraft-software-Majestic` and `…-majestic` are different directories. **Forward-only by construction**: a literal `-` in a path is indistinguishable from a separator in the result, so no inverse is exported. Sibling `/subagents/` transcripts are deliberately not read — a subagent is a different conversation and folding it in would interleave two narratives by timestamp. +- **Filtering is the feature, not a detail.** Measured on a real 1.7 MB session: 935 records in, **46 turns out** — 140 `tool_use`, 139 `tool_result`, 52 `thinking`, 226 non-message, 331 empty. Tool payloads and thinking are excluded **by default**; even with `--include-tools` a tool result is summarized to its byte size and the payload is *never* stored, because payloads are where file contents, command output, and credentials live. Every drop is counted in `filtered` and reported. +- **Never guess, two rules.** (a) Anything a read cannot turn into a turn is counted rather than skipped silently, in **three separate counters**, because the three mean different things and call for different responses. `unknown_record` is an unrecognized record `type` — a format change in a file engram does not own, fixed by extending an allowlist; it earned its keep by surfacing three Codex tool types (`web_search_call`, `tool_search_call`, `tool_search_output`) that the first implementation miscategorized. `torn_line` is an interrupted write, which lands mid-file and not only at EOF; nothing in engram is wrong, and it is *transient* when a transcript is read while its harness is still appending. `missing_uuid` is a conversation record with no `uuid` — the only one of the three where a real turn was lost. These shared one counter until 2026-08-08, which made every torn line read as a format change and sent a reader chasing a harness that had not moved: one session reported 56 "unknown records" that were all complete lines minutes later. A signal that cries wolf two times in three stops being read, which costs exactly the early warning the counter exists to give. (b) An unparseable timestamp is an **error**, never a substitution of now: `recall_inner` orders by `created_at`, so a wall-clock fallback would collapse a whole conversation into one instant and destroy reading order invisibly. +- **`created_at` is the transcript's timestamp**, and `valid_from` is set to match. This bends the documented "`created_at` is transaction time" reading, and has to, for the ordering reason above. +- **Idempotence comes from the id, not from bookkeeping.** `turn_id = uuid_v5(NAMESPACE_OID, "engram-turn:{harness}:{session}:{record}")` — the same discipline as `facts::fact_id` — plus `Store::ingest_turns`'s `INSERT OR IGNORE` in one transaction. Re-ingesting inserts 0; resuming a live session inserts only the new tail. `OR IGNORE` never deletes, so the external-content FTS trigger fires only for rows that really landed and the index cannot drift (contrast `extract_facts`, which uses `INSERT OR REPLACE` and therefore depends on `recursive_triggers`). +- **No reader is a typed variant, not a `bool`.** `TranscriptSupport::{Reader, NotImplemented{detail}, Unsupported{detail}}` makes "0 turns captured" structurally unreachable for a harness engram cannot read: the caller must match, and the reason is already written down. Antigravity (protobuf + SQLite summaries) and Copilot CLI (`session-store.db`) are `Unsupported`; Codex/Opencode/Goose/Qwen are `NotImplemented`. All of them exit 2 with a hint naming the `remember`-then-`save-chat` fallback, and **stdout stays empty** — an empty success is exactly the failure mode this design prevents. +- **Redaction** (`redact.rs`) replaces credential-shaped substrings before storage and counts them per kind in the envelope. Best-effort, not a guarantee — it catches machine-issued token shapes, not a password typed in prose. The real defense is the default filtering above. `harness::home_dir()` reads `$HOME` directly rather than via the `dirs` crate: a **testability decision**, since every harness path derives from it and a test that sets `HOME` to a tempdir is then hermetic by construction. Do not turn it into a dependency. +- **Fixtures are synthetic**, never copied sessions (`tests/fixtures/transcripts/README.md` explains why): a real transcript holds whatever the user pasted. + +## Harness command delivery (`engram install`) + +`src/install.rs` + `plugins/engram/`. Engram was already an MCP server in every harness on a typical machine; what was missing was a *command surface*. + +- **`plugins/engram/` is the single source of truth.** `install.rs` embeds the command bodies with `include_str!`, so the plugin directory and the installed files cannot drift and the compiler enforces the files exist. Exactly two substitutions, via `str::replace`, no template engine: `{{DB}}` and `{{HARNESS}}`. +- **`{{DB}}` is load-bearing.** The path is discovered from the harness's *own* MCP registration (`harness::registered_db`) — on a typical host all writable harnesses point at one shared store (here `~/.local/share/engram/engram.db`) — but see the drift note below: what they registered *yesterday* is not necessarily what a previously-generated command still pins. A generated command that omitted `--db` would fall back to clap's relative `engram.db` default and quietly write to a different store than the agents read. Config formats are scanned narrowly rather than deserialized: JSON (`mcpServers`, or Opencode's `mcp`), **JSONC** (comment-stripped by a string-aware pass — a `//` inside `"https://…"` must survive), and TOML (line-scanned, so engram needs no TOML dependency). Engram **reads** JSONC and never rewrites it; a serde round-trip would delete the user's comments. +- **5 of 8 harnesses can host something; only 4 host a *command*.** Claude Code, **OpenClaude**, Codex, and Opencode have writable command dirs. **Antigravity has no slash-command directory at all** — its extension surface is skills, packaged in plugins, and `agy plugin validate` reports a plugin's `commands/` as "2 processed (converted to skills)", so a command there is a skill either way. Engram writes it a plugin (`~/.gemini/config/plugins/engram/`: `plugin.json` + one `skills/engram-/SKILL.md` per command). Goose, Copilot CLI, and Qwen have nothing engram can write and each says so **in its own words** — one shared sentence described none of them accurately. +- **OpenClaude is a Claude Code fork** (`@gitlawb/openclaude`) with its own config root. Its MCP registration lives in `~/.openclaude.json` — the `~/.claude.json` analogue — **not** `~/.openclaude/settings.json`, which holds env/model/hooks and no servers block. Its transcripts are Claude Code's format down to the record keys, so `ReaderKind::ClaudeCode` serves both; the fork-only record types (`mode`, `file-history-snapshot`, `last-prompt`) are already in the non-message allowlist and must stay there, since a fork tripping `unknown_record` every run would train the reader to ignore its own drift alarm. +- **`CommandSurface` is an enum, not a bool.** `Markdown { dir, file, frontmatter }`, `Plugin { dir }`, `None { detail }`. Antigravity broke the old `command_frontmatter: bool` because the *shape* of the artifact differs, not just its header — and `None` carries a per-harness reason. +- **Frontmatter is per-surface.** `Markdown { frontmatter: false }` for Codex, whose prompts are plain markdown and would otherwise render the YAML block as literal text at the top of every prompt. A `Plugin` skill has a *different* contract again — `name` + `description`, no `argument-hint`, no `allowed-tools` — and lifts its description from the shared template so the two surfaces cannot describe the same command differently. +- **The banner carries no version** (``). Putting one there would make every release rewrite every installed file, turning `install` from idempotent into perpetually-updating. +- **Nix, and the corrected skill rule.** Engram installs into whatever surface a harness makes *writable*, and never into the Nix store. The older rule — "engram never ships a skill" — was written for `~/.claude/skills`, a read-only symlink into the store; it does not generalise. Antigravity's `~/.gemini/config/skills` is store-managed too, but its sibling `~/.gemini/config/plugins` is writable, and a plugin may contain skills — so that is where engram writes. `is_nix_managed` warns when a target resolves into the store, since the next `home-manager switch` would clobber the write; those users reference `plugins/` declaratively instead. +- **The pinned database is checked against the registered one.** `install` reads the `--db` already baked into a generated command and, when it differs from what the harness now registers, reports the drift on the file *and* the harness before correcting it. This is not hypothetical: on the author's machine every harness moved to `~/.local/share/engram/engram.db` after `install` had pinned `~/.gemini/engram.db`, so the slash commands and the MCP tools read different stores for weeks with nothing to say so. Every response also carries `db_origin` (`override` / `registered` / `env` / `default`), because `default` is a *relative* `engram.db` that resolves against whatever directory the command runs in. +- **`install` copies, never symlinks.** A symlink breaks when the repo moves and would hand `${CLAUDE_PLUGIN_ROOT}` semantics to a non-plugin context where it is undefined. +- **`--hooks` is opt-in twice over.** It merges a `SessionEnd` entry into `~/.claude/settings.json` (Claude Code is the only harness here with a hook system engram can write). The hook runs **`ingest`, never `save-chat`** — capturing into the database is invisible and reversible; writing a `.texi` into someone's repo at every session end, unasked, is not. Three properties matter: a **timestamped backup** is written before any change; **other people's `SessionEnd` hooks are left alone** (the field is an array, and several hooks on one event is legitimate, not a conflict); and a settings file that does not parse is **refused, never overwritten**. `serde_json` is compiled with `preserve_order` specifically so the merge does not alphabetize a config engram does not own — there is a test asserting key order survives. +- **CLI-only, and there must never be an HTTP route.** `POST /v1/rules/sync` already lets any local process rewrite a project's `AGENTS.md`; an HTTP `install` would extend that to `$HOME` — and, once hooks land, to code executed at every session end, on an unauthenticated port. + +## What's not yet implemented + +Landed in 0.2.0 (no longer gaps): `--format jsonl|csv`, `remember --dry-run`, real status codes on **all** HTTP routes (a breaking change — see the HTTP notes above), packaging manifests (`packaging/`), the Texinfo manual skeleton (`doc/engram.texi`), `CREDITS.md`, CI, and tests over the memory surfaces. + +Still missing: + +- `--format yaml` (deferred — `serde_yaml` is archived) and `--format explore` (no TUI yet). +- Authentication on the HTTP surface (currently `127.0.0.1`-only, no bearer check). diff --git a/CHANGELOG.md b/CHANGELOG.md index 51aefe0..5e744af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,45 @@ follows [Keep a Changelog](https://keepachangelog.com/); versions follow ### Added +- **OpenClaude** is a supported harness (the eighth). It is a Claude Code fork + with its own config root: commands go to `~/.openclaude/commands/`, the MCP + registration is read from `~/.openclaude.json`, and its transcripts are read + by the existing Claude Code reader — the record types the fork adds are + recognised rather than counted as format drift. +- **Antigravity** now gets a plugin at `~/.gemini/config/plugins/engram/` + (`plugin.json` plus one `skills/engram-/SKILL.md` per command). It has + no slash-command directory at all; `agy plugin validate` reports a plugin's + own `commands/` as "converted to skills", so engram writes skills directly. +- `install` reports `db_origin` (`override` / `registered` / `env` / `default`) + alongside the database it pins, so the relative-`engram.db` fallback is + visible rather than silent. + +### Fixed + +- **`install` now detects a stale database pin.** When a generated command + points at a different database than the harness currently registers, the + drift is reported on both the file and the harness before being corrected. + Previously the two could diverge indefinitely: if a harness's registration + moved after `install` ran, its `/engram-*` commands and its engram MCP tools + read different stores with nothing to say so. +- **`find_git_root` requires a working tree, not merely a `.git` entry.** A + directory must contain `.git/HEAD`; a `.git` file (worktree or submodule + pointer) also counts. An empty `.git` directory in a shared location — e.g. + `/tmp/.git` — previously captured every path beneath it, so `save-chat` would + resolve its project root there, create `chat/`, and edit that directory's + `.gitignore`. +- Harnesses with no writable surface each state their own reason instead of + sharing one sentence that described none of them precisely. + +### Changed + +- `HarnessSpec` models its command surface as an enum — + `CommandSurface::{Markdown, Plugin, None}` — replacing `commands_dir`, + `command_file`, and the `command_frontmatter` bool. Antigravity's surface + differs in artifact *shape*, not just in whether a header is read. + +### Added + - **`flake.nix`** — Engram is now consumable as a Nix flake input (`github:Spacecraft-Software/Engram`), exposing `packages.default`, `packages.engram`, `apps.default`, `checks.default`, and `default`/`docs` diff --git a/CLAUDE.md b/CLAUDE.md index a68125b..2e3a3b9 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,260 +1,6 @@ # CLAUDE.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +@AGENTS.md -## What this project is - -Engram is a shared verbatim chat memory store for multi-model LLM pipelines. It's a single SQLite file (with FTS5 full-text search) that multiple models/agents can read from and write to, enabling memory across different stages of a pipeline without requiring LLM calls to encode/retrieve context. - -## Building and running - -**Build:** -```sh -cargo build --release # LTO, 1 codegen unit, panic=abort per Standard §3 -``` - -Verify the `rmcp` 0.16 macro shape (if build fails due to macro mismatch): -```sh -cargo doc -p rmcp --open -``` - -**Run (CLI):** -```sh -./target/release/engram remember --agent claude-code --scope my-task "Decided: X stays synchronous." -./target/release/engram recall --scope my-task -./target/release/engram search "synchronous" -``` - -**Run as MCP server** (wire into Claude Code / other clients): -```sh -./target/release/engram mcp --db ./engram.db -``` - -**Run as HTTP server** (local-only, `127.0.0.1:8420`, no auth): -```sh -./target/release/engram serve --db ./engram.db --addr 127.0.0.1:8420 -curl -s -XPOST localhost:8420/v1/memory -d '{"agent":"kimi","scope":"x","content":"..."}' -curl -s "localhost:8420/v1/memory/recall?scope=x&limit=10" -``` - -`cargo test` covers the rules subsystem (unit tests in `rules.rs`/`store.rs`) and the memory surfaces (integration tests in `tests/cli.rs`). CI runs rustfmt, clippy, and the test suite via `.github/workflows/ci.yml`. - -## Architecture - -**Single source of truth:** The `Store` (`src/store.rs`) wraps a single `rusqlite::Connection` in `Arc>`. All three surfaces (CLI, MCP, HTTP) dispatch to the same `Store` methods. - -| Surface | Entrypoint | Caller | Notes | -|---|---|---|---| -| **CLI** | `engram remember/recall/search/context/save-chat/mcp/serve/schema/describe` | Command-line tools, shell scripts | Clap derive; stdin fallback for content | -| **MCP** | `engram mcp` | Claude Code, Codex, other MCP clients | rmcp 0.16 stdio; `#[tool_router]`/`#[tool_handler]` macros | -| **HTTP** | `engram serve` | Any HTTP client (curl, Kimi, Ollama Cloud, etc.) | Axum; `127.0.0.1:8420` only; no auth | - -**Module structure:** -- `main.rs` — entry point; parses CLI, instantiates `Store`, dispatches to surface handlers -- `store.rs` — `Store` struct; SQLite schema, migration, CRUD (remember/recall/search, rule_add/rules, context) -- `retrieval.rs` — token budgeting + retrieval assembly, pure functions (no DB handle): `estimate_tokens` (ceil(chars/4), estimator `"chars-div-4"`), `rrf_fuse` (reciprocal rank fusion, `RRF_K = 60.0`), `budget_recall`/`budget_search` (greedy drop-and-continue packing), `BudgetReport` -- `embed.rs` — local Model2Vec embeddings (cfg-gated `vector`): model-path cascade, process-wide embedder cache, cosine -- `rules.rs` — durable project rules: scope resolution, markdown rendering, sentinel-block sync -- `cli.rs` — clap command/argument definitions (`Command` enum, `Cli` struct) -- `mcp.rs` — MCP server (`#[tool_router]` registration, `#[tool_handler]` impls) -- `http.rs` — Axum HTTP server (routes: POST `/v1/memory`, GET `/v1/memory/recall`, GET `/v1/memory/search`, GET `/v1/context`, GET `/v1/health`, the `/v1/rules*` family) -- `error.rs` — `AppError` enum; error codes (InvalidArgument, DbError, etc.), exit codes, structured error emission -- `output/` — Output formatting and envelope - - `mode.rs` — `OutputMode` and `Format` (json, jsonl, csv); detection logic (`--format`/`--json`, env vars, TTY) - - `envelope.rs` — `Response` struct for all command outputs -- `time.rs` — ISO 8601 UTC timestamp generation via `jiff` (never local time) - -## Storage: SQLite + FTS5 - -**Schema:** -- `memories` table: id (PK), agent, scope, role, content, created_at, rule_id (nullable), updated_at (nullable), status (nullable — `active`/`retired`; NULL means active), the bi-temporal trio `valid_from`/`valid_to`/`superseded_by` (all nullable; NULL `valid_to` means currently valid, so every pre-supersession row stays valid by construction; `created_at` is transaction time, `valid_*` is validity time), plus the M5 access-tracking pair `access_count`/`last_accessed_at` (nullable; NULL `access_count` reads as 0; **internal** — `row_to_memory` never reads them, so `Memory` output is unchanged). `status` is exclusively the rules axis; supersession never touches it. -- `memories_fts` virtual table (FTS5): full-text index on `content`, kept in sync via `AFTER INSERT`/`AFTER DELETE` triggers plus a **content-narrowed** update trigger — `memories_au` is `AFTER UPDATE OF content` (M5): the original full-row trigger fired the FTS delete+reinsert on EVERY update, so each access-tracking bump (i.e. every read) and each supersession would churn the index. `migrate()` unconditionally `DROP TRIGGER IF EXISTS` + recreates it narrowed on every open (SQLite has no `CREATE OR REPLACE TRIGGER`; the drop+create is idempotent and converts pre-M5 databases in place) -- `facts` table (M4): id (PK — deterministic UUID v5 over `(memory_id, fact)`, see `facts::fact_id`), memory_id (FK → memories), scope, fact (verbatim extracted line/sentence), extractor (`"deterministic-v1"`), created_at, plus **reserved** `valid_to`/`superseded_by` (always NULL today — a fact's liveness derives from its PARENT's validity: every fact-channel query JOINs `memories` and applies the validity clause to the parent columns). Indices `idx_facts_scope`, `idx_facts_memory`; `facts_fts` (FTS5, content='facts') kept in sync by `facts_ai/ad/au` triggers. `PRAGMA recursive_triggers=ON` is set at open so `INSERT OR REPLACE` fires the delete trigger (otherwise the external-content FTS index drifts) -- Indices: `idx_memories_scope`, `idx_memories_created_at` (for recall queries); `idx_memories_rule` — **partial** unique index on `(scope, rule_id) WHERE rule_id IS NOT NULL`, enforcing one rule per id per scope without constraining ordinary messages -- Migration: `migrate()` in `store.rs` probes `pragma_table_info` before each `ALTER TABLE` (SQLite has no `ADD COLUMN IF NOT EXISTS`), so opening a pre-rules database upgrades it in place. Both new columns are nullable — every pre-existing row is a message, which has neither. -- Pragmas: `journal_mode=WAL`, `busy_timeout=5000ms` (allows concurrent readers while one writer is active) - -**Query safety:** FTS5 queries are sanitized in `sanitize_fts_query` — every token is wrapped as an escaped quoted phrase to prevent syntax injection, and tokens are joined with `OR` (not FTS5's implicit `AND`): natural-language queries almost always contain a filler word the stored text lacks, and one missing token zeroes an AND match. Measured on the M3 bench: AND-joined recall@5 0.108 → OR-joined 0.856 (`bench/RESULTS.md`); BM25 still ranks multi-token matches first. - -## Conventions (Spacecraft Software Standard §3, §4, §14) - -- **Memory safety:** Rust only. No unsafe blocks without explicit justification. -- **Licensing:** Every `.rs` file carries SPDX headers: - ```rust - // SPDX-FileCopyrightText: 2026 Mohamed Hammad & Spacecraft Software - // SPDX-License-Identifier: GPL-3.0-or-later - ``` -- **Timestamps:** ISO 8601 UTC only (via `jiff`, never local time). Suffix with `Z` if needed. -- **CLI shape:** Per the Spacecraft Software Dual-Mode Self-Documenting CLI Standard (v1.0.0) — all commands emit structured output. Mode cascade: explicit `--format`/`--json` > agent env vars (`AI_AGENT`/`AGENT` set non-empty, `CI` truthy) > non-TTY stdout ⇒ machine mode (JSON to stdout, structured errors to stderr). -- **Envelopes:** Every command/route returns `Response` (operation name, data, optional error). When token budgeting is requested, `metadata.budget` carries a `BudgetReport`: `requested_tokens`, `estimator` (`"chars-div-4"` — ceil(Unicode chars / 4), min 1; multi-model pipelines have no single correct tokenizer), `estimated_tokens` (included items only), `included`, `dropped`, `dropped_ids`, and `channels` (candidate count per channel: `recency`/`fts`/`rules`). -- **Role defaults:** `"note"` on all surfaces. Alternatives: `"user"`, `"assistant"`, `"system"`. - -## Command reference - -**CLI:** -- `remember --agent --scope [--role ] [--dry-run] []` — store a message (or read from stdin); `--dry-run` validates and shows what would be stored without writing -- `recall --scope [--limit ] [--budget-tokens ]` — fetch last N messages for a scope (default 50). With `--budget-tokens`, results are packed to the budget newest-first (the oldest drop), output stays chronological, and the envelope carries `metadata.budget` -- `search [--scope ] [--limit ] [--budget-tokens ]` — full-text search (default limit 20). With `--budget-tokens`, results are packed in rank order and the envelope carries `metadata.budget` -- `context [--scope ] [--query ] [--budget-tokens ] [--limit ]` — assemble a budget-packed context block for session start: active rules first (**always all included**, even over budget — policy is never silently dropped), then memories selected newest-first, or by reciprocal-rank fusion of recency+FTS relevance+extracted-fact channels when `--query` is given; included memories are presented chronologically. Defaults: budget 3000, limit 50 per channel; scope resolves via the same cascade as the `rule` commands -- `consolidate [--extract] [--dedup [--yes]] [--report] [--scope ] [--dry-run]` — idle-time maintenance, three combinable phases (at least one required, else InvalidArgument exit 2): `--extract` (M4) runs the deterministic fact extractor and upserts into the facts index (`--dry-run` applies here only); `--dedup` (M5) finds near-duplicate groups of current non-rule memories per scope — normalized-exact text always, stored-vector cosine ≥ 0.92 when the hybrid gate passes, edges unioned — reporting winner (newest) + losers, and **only with `--yes`** supersedes each loser via `mark_superseded_by` (M2 semantics, no new row, never a delete; idempotent); `--report` (M5) is always report-only: contradiction pairs (word-set Jaccard ≥ 0.5 + negation marker on exactly one side — a heuristic; resolve via `remember --supersedes`, never auto-resolved) and the top-20 decay candidates (`staleness = age_days + 30/(1+access_count)`). Data shape is one optional section per phase: `{extract?, dedup?, report?}`. `--scope` omitted means **every** scope — no cascade, unlike the rule commands. CLI-only by design -- `save-chat [--scope ] [--file ] [--model ] [--dry-run]` — archive a scope's history as a complete Texinfo document. Paths resolve against the **project root** (`rules::resolve_scope`), not the process cwd, so the command targets the same file from any subdirectory; `--file` defaults to `chat/.texi`, and `chat/` is added to `.gitignore` when absent (reported as the `gitignore` object — `action` is `added`, `already-ignored` or `would-add` — not done silently). `--model` names the archiving model (falls back to `MODEL`/`LLM_MODEL`/`AI_AGENT`/`AGENT`, then `unknown-model`). **The document is a pure function of the scope**, exactly as the rules block is: an existing archive is rewritten *whole* (never appended to), re-running over an unchanged scope is byte-identical and reports `outcome: "unchanged"`, and the provenance header therefore carries the *last message's* timestamp rather than the export's wall clock. Rules are excluded (an archive is a transcript) and the read is untracked via `Store::export_history` (archiving is not retrieval — counting it would corrupt the M5 decay signal); superseded rows *are* included -- `ingest [--harness ] [--session ] [--scope ] [--cwd ] [--include-thinking] [--include-tools] [--include-sidechains] [--max-bytes ] [--max-chars-per-turn ] [--list] [--dry-run]` — **capture a harness's own session transcript** into a scope as ordinary memories, with roles `user`/`assistant` (values the schema always declared and nothing ever wrote until now). This is what makes "verbatim chat memory" literally true; before it, `save-chat` could only export what an agent chose to `remember`. Harness resolution: `--harness` → environment marker (`CLAUDECODE` etc.) → the single installed harness with a reader — **two candidates is an error, never a guess**. Scope maps by cwd through the same `rules::resolve_scope` cascade. `--list` reports sessions *plus* the whole harness table, so "no sessions here" is distinguishable from "cannot read this harness". CLI-only -- `rule add --id [--scope ] [--agent ] []` — record or revise a rule (stdin if text omitted) -- `rule list [--scope ] [--include-retired]` — rules in effect, ordered by id -- `rule retire --id [--scope ]` — withdraw a rule (tombstone; re-adding reinstates) -- `rule purge --id [--scope ] --yes [--dry-run]` — permanently delete a **retired** rule's row (the one true delete; CLI-only — destructive ops are not agent-invocable) -- `rule sync [--scope ] [--file ]... [--dry-run]` — render rules into `AGENTS.md`/`CLAUDE.md` -- `install [--harness ]... [--db-path ] [--list] [--dry-run] [--force]` — write engram's slash commands (`/engram-save-chat`, `/engram-ingest`, `/engram-context`) into each detected harness's own command directory. **Start with `--list`.** Only *detected* harnesses are written to (engram never `mkdir`s a home for software you don't have), only files carrying engram's banner are overwritten (`--force` overrides, and a hand-written file is reported `skipped` with a reason), and nothing is deleted. Idempotent by byte comparison — a second run reports every file `unchanged` and does not touch mtimes. **CLI-only, never an HTTP route** (see below) -- `mcp` — run as MCP server (stdio) -- `serve [--addr ]` — run HTTP server (default `127.0.0.1:8420`) -- `schema` — print JSON Schema, as `{"Memory": ..., "Rule": ...}` -- `describe` — print CLI Standard capability manifest (JSON) - -**Global flags:** -- `--db ` — database file (env: `ENGRAM_DB`, default: `engram.db`) -- `--json` — machine output; alias for `--format json` -- `--format ` — machine output format, overrides mode auto-detection. `jsonl`: first line is `{"metadata":...,"data":null}`, then one line per record (arrays) or one line with the object. `csv`: RFC 4180 rows on stdout (header from the first record's keys), metadata as one JSON line on stderr. `yaml`/`explore` are deferred -- `--no-color` — disable colors (respects `NO_COLOR` env var) -- `--accessible` — accessible output per Standard §18: plain linear text, no color, status tags. Also enabled by `SPACECRAFT_A11Y=1`; the flag wins over `SPACECRAFT_A11Y=0` -- `--no-track` — read-only auditing: do not update access counts on reads. CLI-only; MCP/HTTP have no opt-out (agent reads are exactly what the tracking measures) - -## Rules (`src/rules.rs`) - -Durable policy, distinct from memories: a memory records what happened, a rule states what must keep being true. Implemented as `memories` rows with `role = "rule"` plus a stable `rule_id` — reusing the table keeps one write path, one FTS index, and identical behavior across surfaces. - -Three invariants worth not breaking: - -1. **`sync` is the delivery mechanism, not an export.** A row in SQLite never reaches a model's context. Rendering into `AGENTS.md`/`CLAUDE.md` — files harnesses auto-load — is what makes a rule take effect. Both surfaces return `next_step` reminding the caller. -2. **The rendered block is a pure function of the rules.** No generation timestamp, rules ordered by `rule_id`. That is what makes `sync` idempotent (`unchanged` ⇒ no write) and therefore safe in a hook or commit gate. Do not add a timestamp to the block. -3. **`add` upserts.** Re-using a `rule_id` revises in place; `created_at` survives, `updated_at` moves. Two competing copies of a rule are worse than none. -4. **`retire` tombstones, never deletes.** `status='retired'` hides a rule from `rules()` and from synced files, but the row survives and stays searchable — erasing the record of a policy that once applied would defeat the point of a memory store. `rule_add` on a retired id reinstates it (sets `status='active'`), which also avoids colliding with the unique index. Retiring is idempotent; an unknown id is an error (exit 3 / HTTP 404), not a silent success. - -`status IS NULL` means active — that is how rules written before the status column keep working. Any new query filtering on status must preserve that. - -Sentinels are `` / ``; only that region is rewritten. Rule text containing `engram:rules:` is rejected at write time (it would terminate its own block). An opening sentinel with no closing one is treated as a mangled block and replaced wholesale rather than appended after. - -Scope cascade: `--scope` → `ENGRAM_SCOPE` → git working-tree basename → cwd basename, reported as `scope_origin`. Under MCP this resolves against the **server process's** cwd, so a shared server needs an explicit `scope`. - -Rules are on all three surfaces. HTTP routes: `POST /v1/rules`, `GET /v1/rules` (`?scope=`, `?include_retired=`), `DELETE /v1/rules/:rule_id` (retires — soft), `POST /v1/rules/sync`. - -Two HTTP notes worth carrying forward: - -- **Status codes.** As of 0.2.0, **all** routes return real status codes via the `ApiResult`/`ok`/`err` helpers in `http.rs` — 400 on a malformed request (e.g. empty/whitespace `content` on `POST /v1/memory`), 404 on an unknown rule, 500 on storage failure. This was a deliberate breaking change: the 0.1.x `/v1/memory*` handlers answered `200 OK` with an `{"error":...}` body, and callers that only parsed the body must now check the HTTP status. Keep following this pattern when adding endpoints. -- **Path-param syntax.** Routes use `:rule_id`, not `{rule_id}`. axum is pinned at 0.7 (matchit 0.7), where the brace form compiles but matches only the literal string, so the route silently never fires. Change to braces when upgrading to axum 0.8+. - -`POST /v1/rules/sync` is the only route that writes outside the database. Targets derive from the server process's cwd, never from caller input (no traversal surface), and the CLI's `--file` override is deliberately not exposed. With the no-auth posture this means any local process can rewrite that project's `AGENTS.md`/`CLAUDE.md`. - -## Environment - -- `ENGRAM_DB` — override database path -- `ENGRAM_SCOPE` — default scope for `rule` commands -- `ENGRAM_AGENT` — default `--agent` for `rule add` -- `AI_AGENT`, `AGENT` (set non-empty), `CI` (truthy) — trigger machine output mode (detected for structured logging in CI/agent contexts) -- `SPACECRAFT_A11Y` — `1` enables accessible output, `0` disables auto-detection (`--accessible` still wins) -- `NO_COLOR` — disable colors - -## Agent usage - -In Claude Code or other multi-model pipelines: - -1. **Call `remember`** after any decision, fact, or design rationale worth persisting — scope it to your project/task/run ID so related sessions can recall it. -2. **Call `recall`** at the start of a session for that scope to load prior context (or search for specific topics) — or call `context` to get rules + budget-packed memories in one shot. -3. **Call `search`** before asserting something was already decided — verify rather than guess. - -4. **Call `rule_list`** at session start to load standing policy, and `rule_add` + `rule_sync` when the user states a requirement that must hold in future sessions (as opposed to a fact about this one). - -All three surfaces (CLI, MCP, HTTP) hit the same `Store`, so memories are shared across deployment modes. Rules are on all three surfaces too. - -## Semantic search (the `vector` feature, M3) - -Opt-in at build time: `cargo build --release --features vector`. The default build stays FTS5-only with zero ML dependencies. Facts: - -- **Engine:** Model2Vec static embeddings via `model2vec-rs` compiled with `default-features = false` + `local-only` — the hf-hub network fetch path is compiled OUT; engram never downloads a model (§9 PFA). Install one by hand (e.g. `minishlab/potion-base-8M`: `model.safetensors` + `tokenizer.json` + `config.json`). -- **Model cascade:** `--model-path` → `ENGRAM_MODEL` → `$XDG_DATA_HOME/engram/model` (default `~/.local/share/engram/model`). The directory basename is the model name in `memory_vectors.model`; vectors from different models are never compared. -- **Storage:** `memory_vectors` side table (memory_id PK/FK, model, dim, embedding BLOB f32-LE) — deliberately NOT sqlite-vec (alpha C extension vs §5.5 packaging) and NOT columns on `memories` (FTS triggers untouched). Similarity is brute-force cosine: sub-millisecond under 100k rows. -- **Indexing:** `engram remember` embeds live on the CLI when a model resolves; `engram index [--scope] [--batch] [--dry-run]` backfills everything else (MCP/HTTP writes, pre-model history). Rule rows are never embedded. -- **Retrieval:** `search --mode fts|hybrid`; omitted, hybrid engages automatically when (feature ∧ model resolves ∧ vectors indexed), else fts. Explicit `--mode hybrid` with a missing prerequisite is a structured exit-2/HTTP-400 error, never a silent fallback. Hybrid = FTS top-50 + cosine top-50 → `rrf_fuse(k=60)`; `context` gains the vector as a third channel the same way. -- **The gate:** measured 2026-08-02 on the held-out `bench/queries.jsonl` (frozen before implementation): hybrid 0.918 vs fts 0.856 recall@5 = +6.2 points ≥ +5 → PASS; the margin is entirely conceptual/synonym queries. See `bench/RESULTS.md` — including why the first (+77.9) measurement was rejected as a baseline defect. - -## Extracted-fact index (M4) - -The TencentDB L0↔L1 pattern: L0 is the verbatim memory, L1 is a *derived index* of the decision/constraint sentences inside it. Facts never replace verbatim — each `facts` row is a verbatim substring of its parent's content plus a drill-down pointer (`memory_id` → `engram get` / the MCP `get` tool). - -- **Extractor: `deterministic-v1` only — no LLM on the write path, ever.** `facts::extract` splits content into lines (plus sentence-splits of multi-sentence lines), trims bullet markers, and keeps units that start (case-insensitively) with one of 19 markers (`Decided:`, `Decision:`, `TODO`, `FIXME`, `NOTE:`, `Rule:`, `Fix:`, `Fixed:`, `Chose:`, `Chosen:`, `Rejected:`, `Constraint:`, `Gotcha:`, `Warning:`, `Never `, `Always `, `Must `, `Do not `, `Don't `). Floor 12 chars, cap 8 facts per memory (first eight distinct in document order), exact-dedupe. Facts are stored verbatim — rewriting would be the lossy-extraction trap. -- **Liveness derives from the parent.** Extraction is append-only (`INSERT OR REPLACE` on deterministic v5 ids — idempotent, re-runs don't grow the table); nothing deletes facts when a memory is superseded. Instead `fact_candidates` JOINs `memories` and applies the validity filter to the parent, so stale facts stop surfacing the moment their parent does. The fact columns `valid_to`/`superseded_by` are reserved and stay NULL. -- **Channel wiring.** With a `--query`, `context` fuses recency + FTS + **facts** (parents of matching facts, deduped, rank order) — plus vector when the hybrid gate passes — and reports `channels.facts`. Hybrid search is now fts + vector + facts. Plain FTS `search` is unchanged (memories only; facts are substrings of content, so the channel can only *boost* the memory that states a decision above ones that merely mention its words — it can never be a sole finder). -- **CLI-only, on purpose.** `engram consolidate --extract` exists on neither MCP nor HTTP: extraction is an operator's idle-time batch job; agents get facts through `context`/hybrid ranking automatically. CLI-only (see the MCP tool ledger below for the one canonical count). -- Rule rows are never extracted from — policy travels through the rules section, not retrieval. - -## Idle consolidation + decay (M5) - -`engram consolidate` grew two phases beyond `--extract` (all combinable; at least one required; still CLI-only): - -- **`--dedup [--yes]`** — near-duplicate detection over the CURRENT, non-rule memories of each scope. Two detectors run and their edges are **unioned** into connected components: *exact* (normalized text: trim, lowercase, collapse internal whitespace — always on) and *vector* (cosine ≥ 0.92 between **stored** embeddings, same-scope pairs only — runs exactly when the auto-hybrid gate would pass: feature ∧ model resolves ∧ vectors indexed). Each group's NEWEST row (max `created_at`, id tie-break) wins. Without `--yes` it is report-only; with `--yes` every loser goes through `Store::mark_superseded_by(loser, winner, now)` — **M2 supersession semantics reused** (`valid_to` + `superseded_by` set, `WHERE valid_to IS NULL`), *not* `remember_superseding`: no new row is inserted because the winner already exists. Dedup NEVER deletes, and it is idempotent — superseded losers are no longer Current, so a second run finds nothing. -- **`--report`** — always report-only, two sections. (a) *Contradictions*: pairs of CURRENT same-scope non-rule memories with word-set Jaccard ≥ 0.5 AND a negation marker (`not `, `never `, `no longer `, `don't `, `do not `, `isn't `, `wasn't `, `stopped `) on exactly one side. A documented heuristic — a human or agent resolves via `remember --supersedes`; the tool never auto-resolves. (b) *Decay*: every CURRENT non-rule memory scored `staleness = age_days * 1.0 + 30.0/(1+access_count)` (crude, but monotone in age and un-accessedness), top 20 returned with age, access_count, last_accessed_at. - -**Access tracking** feeds the decay signal: `recall`/`search`/`search_hybrid`/`context`/`get` bump `access_count`/`last_accessed_at` at the end of the read, inside the same lock, for the memories actually **returned** (never dropped candidates, never the rules section — `rules()` is untracked, and dry-runs write nothing). The columns are internal: `Memory` serialization is byte-identical with or without them. Opt-out is the global `--no-track` CLI flag (read-only auditing), wired right after `Store::open`; MCP and HTTP have no opt-out — agent reads are exactly what the tracking measures. - -## MCP tool ledger — the canonical count - -**Ten tools, and the ceiling is now reached.** This is the single place the count lives; it used to be restated in three sections and drifted. `src/mcp.rs`'s module doc carries the same statement for readers who are in the code. - -`remember`, `recall`, `search`, `get`, `context`, `rule_add`, `rule_list`, `rule_retire`, `rule_sync`, `save_chat`. - -Every tool's schema costs context on every turn of every conversation, which is why the cap exists (`doc/engram.texi`). **An eleventh tool must displace an existing one, and the displacement must be argued in the manual.** - -- `save_chat` (M4) earned the last slot only because it carries *both halves* of the capture story: `from_transcript: true` captures the session, then archives. Spending the slot on archiving alone would have left MCP able to export a conversation but never record one, with no slot left to fix it. -- **No `file` argument, ever.** The destination derives from the server's resolved project root. A caller-chosen path is a traversal primitive handed to a model whose input includes attacker-influenceable text — the same reasoning that keeps `--file` off `rule_sync`'s MCP surface. -- Deliberately CLI-only and **not** candidates for the slot: `install` (writes into `$HOME`), `ingest` (agents reach it through `save_chat --from-transcript`), `consolidate`/`index` (operator batch jobs whose results arrive through ranking anyway), `rule purge` (destructive ops are not agent-invocable). -- Both surfaces share one implementation: `archive::save_chat` and `transcript::capture` are called by the CLI *and* the MCP tool, so they cannot drift in what they write, filter, redact, or count. - -## Transcript capture (`engram ingest`) - -`src/harness.rs` + `src/transcript/{mod,claude_code,redact}.rs`. Reads the session file a harness already writes for itself and stores each message as an ordinary memory, so `recall`/`search`/`context`/`consolidate` see the real conversation. - -Two readers exist: `claude_code` and `codex`. Adding a third means adding a `ReaderKind` variant, which the two `match`es in `transcript/mod.rs` then force you to handle. - -- **Codex layout:** `~/.codex/sessions/YYYY/MM/DD/rollout--.jsonl`. The tree encodes the **date, not the cwd**, so there is nothing to mangle — each rollout's first record is a `session_meta` carrying `cwd` verbatim, and listing reads exactly that one line per file. -- **Codex has two channels, and `event_msg` wins.** `event_msg` is what the UI displayed (flat strings); `response_item` is the raw API traffic. `event_msg` is primary not merely because it parses more easily but because it is *less* noisy: on a real rollout it held 2 user messages where `response_item` held 3, and the extra one was an `` block the harness injects. `response_item` is a fallback used only when a rollout has no `event_msg` conversation at all, so retiring the display channel would degrade rather than silently yield nothing. When the display channel wins, the raw duplicates are counted as `non_message`. -- **Codex session ids are per-rollout, NOT `session_meta.session_id`.** That field is *not unique* — resuming a session writes a new file reusing the same id, and three files sharing one id exist on this machine. Since `turn_id` derives from the session id, reusing it would collide turns at the same line index across rollouts and `INSERT OR IGNORE` would silently drop them. Engram therefore keys on the file name minus `rollout-` (unique, sortable, still contains the uuid). There is a test for exactly this. -- **Codex records carry no per-record id**, so `source_uuid` is `{line_index}:{v5 digest of the text}`. The index alone would suffice for an append-only log; folding in the content means an inserted line does not renumber every later turn into a new identity. -- **`--max-bytes` is not theoretical.** A 114 MB rollout exists on this machine; the 64 MiB default refuses it with a structured error naming the override. Both readers stream line by line. -- **Claude Code layout:** `~/.claude/projects//.jsonl`. `mangle_cwd` replaces every `/` with `-` (so the leading slash becomes a leading dash) and **preserves case** — `-spacecraft-software-Majestic` and `…-majestic` are different directories. **Forward-only by construction**: a literal `-` in a path is indistinguishable from a separator in the result, so no inverse is exported. Sibling `/subagents/` transcripts are deliberately not read — a subagent is a different conversation and folding it in would interleave two narratives by timestamp. -- **Filtering is the feature, not a detail.** Measured on a real 1.7 MB session: 935 records in, **46 turns out** — 140 `tool_use`, 139 `tool_result`, 52 `thinking`, 226 non-message, 331 empty. Tool payloads and thinking are excluded **by default**; even with `--include-tools` a tool result is summarized to its byte size and the payload is *never* stored, because payloads are where file contents, command output, and credentials live. Every drop is counted in `filtered` and reported. -- **Never guess, two rules.** (a) Anything a read cannot turn into a turn is counted rather than skipped silently, in **three separate counters**, because the three mean different things and call for different responses. `unknown_record` is an unrecognized record `type` — a format change in a file engram does not own, fixed by extending an allowlist; it earned its keep by surfacing three Codex tool types (`web_search_call`, `tool_search_call`, `tool_search_output`) that the first implementation miscategorized. `torn_line` is an interrupted write, which lands mid-file and not only at EOF; nothing in engram is wrong, and it is *transient* when a transcript is read while its harness is still appending. `missing_uuid` is a conversation record with no `uuid` — the only one of the three where a real turn was lost. These shared one counter until 2026-08-08, which made every torn line read as a format change and sent a reader chasing a harness that had not moved: one session reported 56 "unknown records" that were all complete lines minutes later. A signal that cries wolf two times in three stops being read, which costs exactly the early warning the counter exists to give. (b) An unparseable timestamp is an **error**, never a substitution of now: `recall_inner` orders by `created_at`, so a wall-clock fallback would collapse a whole conversation into one instant and destroy reading order invisibly. -- **`created_at` is the transcript's timestamp**, and `valid_from` is set to match. This bends the documented "`created_at` is transaction time" reading, and has to, for the ordering reason above. -- **Idempotence comes from the id, not from bookkeeping.** `turn_id = uuid_v5(NAMESPACE_OID, "engram-turn:{harness}:{session}:{record}")` — the same discipline as `facts::fact_id` — plus `Store::ingest_turns`'s `INSERT OR IGNORE` in one transaction. Re-ingesting inserts 0; resuming a live session inserts only the new tail. `OR IGNORE` never deletes, so the external-content FTS trigger fires only for rows that really landed and the index cannot drift (contrast `extract_facts`, which uses `INSERT OR REPLACE` and therefore depends on `recursive_triggers`). -- **No reader is a typed variant, not a `bool`.** `TranscriptSupport::{Reader, NotImplemented{detail}, Unsupported{detail}}` makes "0 turns captured" structurally unreachable for a harness engram cannot read: the caller must match, and the reason is already written down. Antigravity (protobuf + SQLite summaries) and Copilot CLI (`session-store.db`) are `Unsupported`; Codex/Opencode/Goose/Qwen are `NotImplemented`. All of them exit 2 with a hint naming the `remember`-then-`save-chat` fallback, and **stdout stays empty** — an empty success is exactly the failure mode this design prevents. -- **Redaction** (`redact.rs`) replaces credential-shaped substrings before storage and counts them per kind in the envelope. Best-effort, not a guarantee — it catches machine-issued token shapes, not a password typed in prose. The real defense is the default filtering above. `harness::home_dir()` reads `$HOME` directly rather than via the `dirs` crate: a **testability decision**, since every harness path derives from it and a test that sets `HOME` to a tempdir is then hermetic by construction. Do not turn it into a dependency. -- **Fixtures are synthetic**, never copied sessions (`tests/fixtures/transcripts/README.md` explains why): a real transcript holds whatever the user pasted. - -## Harness command delivery (`engram install`) - -`src/install.rs` + `plugins/engram/`. Engram was already an MCP server in every harness on a typical machine; what was missing was a *command surface*. - -- **`plugins/engram/` is the single source of truth.** `install.rs` embeds the command bodies with `include_str!`, so the plugin directory and the installed files cannot drift and the compiler enforces the files exist. Exactly two substitutions, via `str::replace`, no template engine: `{{DB}}` and `{{HARNESS}}`. -- **`{{DB}}` is load-bearing.** The path is discovered from the harness's *own* MCP registration (`harness::registered_db`) — all three writable harnesses on this machine point at `/home/mj/.gemini/engram.db`. A generated command that omitted `--db` would fall back to clap's relative `engram.db` default and quietly write to a different store than the agents read. Config formats are scanned narrowly rather than deserialized: JSON (`mcpServers`, or Opencode's `mcp`), **JSONC** (comment-stripped by a string-aware pass — a `//` inside `"https://…"` must survive), and TOML (line-scanned, so engram needs no TOML dependency). Engram **reads** JSONC and never rewrites it; a serde round-trip would delete the user's comments. -- **Only 3 of 7 harnesses can host a command.** Claude Code, Codex, and Opencode have writable command dirs; Antigravity, Goose, Copilot CLI, and Qwen are reported `note: "no command surface engram can write"` rather than silently omitted. Saying "works in all seven" would make the feature read as broken on four of them. -- **Frontmatter is per-harness.** `command_frontmatter: false` for Codex, whose prompts are plain markdown and would otherwise render the YAML block as literal text at the top of every prompt. -- **The banner carries no version** (``). Putting one there would make every release rewrite every installed file, turning `install` from idempotent into perpetually-updating. -- **Nix:** `~/.claude/skills` is a read-only symlink into the Nix store, so runtime *skill* installation is impossible. Engram therefore never ships a skill — it ships commands, and command dirs are writable. `is_nix_managed` warns when a target resolves into the store, since the next `home-manager switch` would clobber the write; those users reference `plugins/engram/` declaratively instead. -- **`install` copies, never symlinks.** A symlink breaks when the repo moves and would hand `${CLAUDE_PLUGIN_ROOT}` semantics to a non-plugin context where it is undefined. -- **`--hooks` is opt-in twice over.** It merges a `SessionEnd` entry into `~/.claude/settings.json` (Claude Code is the only harness here with a hook system engram can write). The hook runs **`ingest`, never `save-chat`** — capturing into the database is invisible and reversible; writing a `.texi` into someone's repo at every session end, unasked, is not. Three properties matter: a **timestamped backup** is written before any change; **other people's `SessionEnd` hooks are left alone** (the field is an array, and several hooks on one event is legitimate, not a conflict); and a settings file that does not parse is **refused, never overwritten**. `serde_json` is compiled with `preserve_order` specifically so the merge does not alphabetize a config engram does not own — there is a test asserting key order survives. -- **CLI-only, and there must never be an HTTP route.** `POST /v1/rules/sync` already lets any local process rewrite a project's `AGENTS.md`; an HTTP `install` would extend that to `$HOME` — and, once hooks land, to code executed at every session end, on an unauthenticated port. - -## What's not yet implemented - -Landed in 0.2.0 (no longer gaps): `--format jsonl|csv`, `remember --dry-run`, real status codes on **all** HTTP routes (a breaking change — see the HTTP notes above), packaging manifests (`packaging/`), the Texinfo manual skeleton (`doc/engram.texi`), `CREDITS.md`, CI, and tests over the memory surfaces. - -Still missing: - -- `--format yaml` (deferred — `serde_yaml` is archived) and `--format explore` (no TUI yet). -- Authentication on the HTTP surface (currently `127.0.0.1`-only, no bearer check). - -## See also - -- `README.md` — project description, status, quick-start examples -- `AGENTS.md` — agent-oriented guidance (covers same content as this file but in a different form) -- `CONTRIBUTING.md` — licensing and contribution guidelines -- [The Steelbore Standard](https://Construct.SpacecraftSoftware.org/) — umbrella conventions on memory safety, CLI shape, SPDX licensing, timestamps, etc. -- [rmcp 0.16 documentation](https://docs.rs/rmcp/0.16/rmcp/) — if macros need debugging +> Record project knowledge in `AGENTS.md`, not here. This file holds only +> Claude-Code-only context (Standard §5.7). diff --git a/README.md b/README.md index 73eccca..7520a58 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ engram rule add --id skill-description-1000 \ # Read what's in effect. engram rule list -# Render into AGENTS.md and CLAUDE.md at the project root. +# Render into AGENTS.md at the project root. engram rule sync # Withdraw one when it stops applying, then re-sync. @@ -134,7 +134,7 @@ engram rule purge --id old-policy --yes # delete a retired ``` Everything outside them is preserved verbatim, so the block can sit inside a -hand-written `CLAUDE.md` indefinitely. The rendered block is a pure function of +hand-written `AGENTS.md` indefinitely. The rendered block is a pure function of the rules — no generation timestamp — so re-running `sync` with unchanged rules writes nothing at all. That makes it safe in a `SessionStart` hook, a pre-commit gate, or a CI check (`engram rule sync --dry-run` reports `updated` if someone @@ -214,7 +214,7 @@ storage failure. New routes should keep following that pattern. > touches anything outside the database. Target paths come from the server > process's own working directory and never from caller input, so there is no > path-traversal surface — but combined with the no-auth posture it means any -> local process can rewrite that project's `AGENTS.md` and `CLAUDE.md`. The +> local process can rewrite that project's `AGENTS.md`. The > CLI's `--file` override is deliberately not exposed over HTTP. Weigh this > before binding the server anywhere but `127.0.0.1`. diff --git a/doc/engram.texi b/doc/engram.texi index bc93276..45e7227 100644 --- a/doc/engram.texi +++ b/doc/engram.texi @@ -601,7 +601,7 @@ explicit). @code{--dry-run} previews the deletion. engram rule sync [--scope @var{id}] [--file @var{path}]@dots{} [--dry-run] @end example -Render the scope's rules into @file{AGENTS.md} and @file{CLAUDE.md} +Render the scope's rules into @file{AGENTS.md} (or the given @code{--file} targets, repeatable). Only the region between the engram sentinels is rewritten. With @code{--dry-run}, reports what would be written without touching any file. @@ -992,7 +992,7 @@ deletes. @item rule_sync @cindex rule_sync tool (MCP) -Render a scope's rules into @file{AGENTS.md} and @file{CLAUDE.md} at +Render a scope's rules into @file{AGENTS.md} at the project root. @item save_chat @@ -1099,7 +1099,7 @@ unknown rule, @code{200} with the outcome on success. @item POST /v1/rules/sync @cindex POST /v1/rules/sync -Render the scope's rules into @file{AGENTS.md} and @file{CLAUDE.md}. +Render the scope's rules into @file{AGENTS.md}. This is the only route that writes outside the database; targets derive from the server process's working directory, never from caller input. @@ -1175,9 +1175,10 @@ Four invariants worth not breaking: @item @cindex sync, as delivery @emph{Sync is the delivery mechanism, not an export.} A row in SQLite -never reaches a model's context. Rendering into @file{AGENTS.md} and -@file{CLAUDE.md} --- files harnesses auto-load --- is what makes a rule -take effect. +never reaches a model's context. Rendering into @file{AGENTS.md} --- +the file harnesses auto-load, and that Claude Code reaches through the +@code{@@AGENTS.md} import in its @file{CLAUDE.md} (Steelbore Standard +§5.7) --- is what makes a rule take effect. @item @emph{The rendered block is a pure function of the rules.} No @@ -1264,24 +1265,48 @@ installed, and never creates a directory for one that is not. Support is uneven, and the manual would rather say so than imply otherwise: -@multitable @columnfractions .22 .26 .22 .30 +@multitable @columnfractions .22 .26 .26 .26 @headitem Harness @tab Transcripts @tab Commands @tab Hooks @item claude-code @tab read @tab written @tab written (opt-in) +@item openclaude @tab read @tab written @tab --- @item codex @tab read @tab written @tab --- @item opencode @tab --- @tab written @tab --- +@item antigravity @tab --- @tab plugin (skills) @tab --- @item qwen @tab --- @tab --- @tab --- @item goose @tab --- @tab --- @tab --- -@item antigravity @tab --- @tab --- @tab --- @item copilot-cli @tab --- @tab --- @tab --- @end multitable +@cindex openclaude +@cindex forks, harness +OpenClaude is a fork of Claude Code with its own configuration root. Its +transcripts match Claude Code's format down to the record keys, so one +reader serves both; the records the fork adds are recognized rather than +counted as drift. Its MCP registration lives in @file{~/.openclaude.json}, +the analogue of @file{~/.claude.json} --- not in +@file{~/.openclaude/settings.json}, which holds unrelated settings. + +@cindex antigravity, plugin +@cindex skills +Antigravity is the one harness with no slash-command directory at all. +Its extension surface is @dfn{skills}, packaged in @dfn{plugins}, and a +plugin's own @file{commands/} directory is reported by +@command{agy plugin validate} as ``converted to skills'' --- so a command +there would become a skill regardless. Engram therefore writes a plugin +directly: a manifest plus one skill per command. The skill frontmatter is +a different contract from a command file's, but its description is lifted +from the same template, so the two surfaces cannot disagree about what a +command does. + Every gap in that table is reported at runtime with its reason attached, rather than omitted. @command{engram install --list} names the harnesses with no command surface; @command{engram ingest} fails with exit status@tie{}2 and an explanation when asked to read a harness it cannot. -A feature that quietly does nothing on four of seven harnesses would read -as broken; one that says which four, and why, is merely honest about its -reach. +A feature that quietly does nothing on three of eight harnesses would read +as broken; one that says which three, and why, is merely honest about its +reach. Each of those three states its own reason: unsurveyed session +storage and an absent command directory are different problems, and one +sentence shared between them described neither. @section Formats Engram does not control diff --git a/src/cli.rs b/src/cli.rs index 73798ec..e2965a2 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -460,14 +460,14 @@ EXAMPLES: #[arg(long)] scope: Option, }, - /// Render the scope's rules into AGENTS.md and CLAUDE.md. + /// Render the scope's rules into AGENTS.md. /// /// Rewrites only the region between the engram sentinels, leaving the rest /// of each file untouched. Running it twice with unchanged rules is a /// no-op, so it is safe to wire into a hook or a commit gate. #[command(after_help = "\ EXAMPLES: - # Render into AGENTS.md and CLAUDE.md at the project root. + # Render into AGENTS.md at the project root. engram rule sync # Read-only check: reports 'updated' if the block is stale or hand-edited. @@ -481,7 +481,7 @@ EXAMPLES: #[arg(long)] scope: Option, /// Target file, repeatable. Relative paths resolve against the project - /// root. Defaults to AGENTS.md and CLAUDE.md. + /// root. Defaults to AGENTS.md. #[arg(long = "file")] files: Vec, /// Report what would be written without touching any file. diff --git a/src/harness.rs b/src/harness.rs index 92ab068..b8bb606 100644 --- a/src/harness.rs +++ b/src/harness.rs @@ -26,6 +26,15 @@ use std::path::PathBuf; #[clap(rename_all = "kebab-case")] pub enum Harness { ClaudeCode, + /// A Claude Code fork with its own config root. + /// + /// Renamed explicitly on both derives: kebab-casing the variant would give + /// `open-claude`, but the fork calls itself `openclaude` and that is what + /// [`HarnessSpec::name`] carries. The two must agree — the name is what + /// `--harness` accepts and what every response serializes. + #[serde(rename = "openclaude")] + #[clap(name = "openclaude")] + OpenClaude, Codex, Opencode, Antigravity, @@ -55,6 +64,38 @@ pub enum TranscriptSupport { /// Everything engram knows about one harness. #[derive(Debug, Clone, Copy)] +/// How a harness lets engram install something the user can invoke by name. +/// +/// A bool was enough while every target was a markdown command file that either +/// did or did not carry frontmatter. Antigravity broke that: it has **no +/// slash-command directory at all** — its extension surface is skills, packaged +/// in plugins — so the shape of the artifact differs, not just its header. An +/// enum makes each surface carry exactly the fields it needs, and makes +/// "engram cannot install here" carry its reason instead of a shared sentence +/// that fits none of the harnesses it was applied to. +pub enum CommandSurface { + /// Markdown command files in a home-relative directory. + Markdown { + /// Home-relative directory the harness loads user commands from. + dir: &'static str, + /// File-name pattern; `{name}` is the command's short name. + file: &'static str, + /// Whether the harness reads YAML frontmatter. Codex prompts are plain + /// markdown and would render the block as literal text. + frontmatter: bool, + }, + /// A Gemini-family plugin directory. Skills inside it are discovered as + /// `plugins//skills//SKILL.md`, and expand the way a slash + /// command does (`agy --disable-slash-commands` disables both). + Plugin { + /// Home-relative directory holding plugin subdirectories. + dir: &'static str, + }, + /// Nothing engram can write, and why not. The reason is per-harness + /// because the reasons genuinely differ. + None { detail: &'static str }, +} + pub struct HarnessSpec { pub id: Harness, /// Stable kebab-case identifier, as accepted by `--harness` and emitted @@ -67,20 +108,11 @@ pub struct HarnessSpec { /// has one. `None` when transcripts live somewhere unstructured. pub sessions_dir: Option<&'static str>, pub transcript: TranscriptSupport, - /// Home-relative directory the harness loads user commands from. - /// - /// `None` means the harness has **no command surface engram can write**, - /// which is reported plainly rather than papered over. Of the seven - /// harnesses here, only some can host a slash command; claiming otherwise - /// would make `install` look broken on the rest. - pub commands_dir: Option<&'static str>, - /// File-name pattern for a command in `commands_dir`. `{name}` is - /// replaced by the command's short name. - pub command_file: &'static str, - /// Whether this harness reads YAML frontmatter at the top of a command - /// file. Codex prompts are plain markdown and would render the - /// frontmatter as literal text. - pub command_frontmatter: bool, + /// How, if at all, engram can put a command in front of this harness's + /// user. Reported plainly rather than papered over: only some harnesses + /// can host one, and claiming otherwise would make `install` look broken + /// on the rest. + pub command_surface: CommandSurface, /// Where this harness registers MCP servers, when engram knows. Read to /// discover which database the user already shares between harnesses. pub mcp_config: Option, @@ -119,21 +151,48 @@ pub const ALL: &[HarnessSpec] = &[ probe: &[".claude", ".claude.json"], sessions_dir: Some(".claude/projects"), transcript: TranscriptSupport::Reader(ReaderKind::ClaudeCode), - commands_dir: Some(".claude/commands"), - command_file: "engram-{name}.md", - command_frontmatter: true, + command_surface: CommandSurface::Markdown { + dir: ".claude/commands", + file: "engram-{name}.md", + frontmatter: true, + }, mcp_config: Some(McpConfigSource::Json(".claude.json")), hooks_config: Some(".claude/settings.json"), }, + HarnessSpec { + id: Harness::OpenClaude, + name: "openclaude", + // A Claude Code fork: same config layout under its own root, so the + // MCP registration lives in `~/.openclaude.json` (the `~/.claude.json` + // analogue) and NOT in `~/.openclaude/settings.json`, which holds + // env/model/hooks and no mcpServers block. + probe: &[".openclaude", ".openclaude.json"], + sessions_dir: Some(".openclaude/projects"), + // The transcripts are Claude Code's format down to the record keys, so + // the same reader serves both. Fork-specific record types are handled + // in the reader's allowlist rather than by a second reader. + transcript: TranscriptSupport::Reader(ReaderKind::ClaudeCode), + command_surface: CommandSurface::Markdown { + dir: ".openclaude/commands", + file: "engram-{name}.md", + frontmatter: true, + }, + mcp_config: Some(McpConfigSource::Json(".openclaude.json")), + // The fork has a `hooks` key, but its shape is unverified against a + // real run; `install --hooks` stays Claude-Code-only until it is. + hooks_config: None, + }, HarnessSpec { id: Harness::Codex, name: "codex", probe: &[".codex/config.toml", ".codex"], sessions_dir: Some(".codex/sessions"), transcript: TranscriptSupport::Reader(ReaderKind::Codex), - commands_dir: Some(".codex/prompts"), - command_file: "engram-{name}.md", - command_frontmatter: false, + command_surface: CommandSurface::Markdown { + dir: ".codex/prompts", + file: "engram-{name}.md", + frontmatter: false, + }, mcp_config: Some(McpConfigSource::Toml(".codex/config.toml")), hooks_config: None, }, @@ -145,9 +204,11 @@ pub const ALL: &[HarnessSpec] = &[ transcript: TranscriptSupport::NotImplemented { detail: "opencode's session storage has not been surveyed", }, - commands_dir: Some(".config/opencode/command"), - command_file: "engram-{name}.md", - command_frontmatter: true, + command_surface: CommandSurface::Markdown { + dir: ".config/opencode/command", + file: "engram-{name}.md", + frontmatter: true, + }, mcp_config: Some(McpConfigSource::Jsonc(".config/opencode/opencode.jsonc")), hooks_config: None, }, @@ -159,9 +220,9 @@ pub const ALL: &[HarnessSpec] = &[ transcript: TranscriptSupport::Unsupported { detail: "antigravity stores conversations as protocol buffers plus a SQLite summaries database; there is no line-oriented transcript to read", }, - commands_dir: None, - command_file: "engram-{name}.md", - command_frontmatter: true, + command_surface: CommandSurface::Plugin { + dir: ".gemini/config/plugins", + }, mcp_config: Some(McpConfigSource::Json(".gemini/antigravity/mcp_config.json")), hooks_config: None, }, @@ -173,9 +234,9 @@ pub const ALL: &[HarnessSpec] = &[ transcript: TranscriptSupport::NotImplemented { detail: "goose's session storage has not been surveyed", }, - commands_dir: None, - command_file: "engram-{name}.md", - command_frontmatter: true, + command_surface: CommandSurface::None { + detail: "goose has no user command directory engram has surveyed", + }, mcp_config: None, hooks_config: None, }, @@ -187,9 +248,9 @@ pub const ALL: &[HarnessSpec] = &[ transcript: TranscriptSupport::Unsupported { detail: "copilot cli stores sessions in session-store.db, a SQLite database with an undocumented schema", }, - commands_dir: None, - command_file: "engram-{name}.md", - command_frontmatter: true, + command_surface: CommandSurface::None { + detail: "copilot cli has no user-writable command or prompt directory", + }, mcp_config: Some(McpConfigSource::Json(".copilot/mcp-config.json")), hooks_config: None, }, @@ -201,9 +262,9 @@ pub const ALL: &[HarnessSpec] = &[ transcript: TranscriptSupport::NotImplemented { detail: "qwen's session storage has not been surveyed", }, - commands_dir: None, - command_file: "engram-{name}.md", - command_frontmatter: true, + command_surface: CommandSurface::None { + detail: "qwen's command format is unverified; engram will not guess at it", + }, mcp_config: Some(McpConfigSource::Json(".qwen/settings.json")), hooks_config: None, }, @@ -288,7 +349,18 @@ pub fn sessions_dir(spec: &HarnessSpec) -> Option { /// Absolute path to a harness's command directory, when it has one. pub fn commands_dir(spec: &HarnessSpec) -> Option { - spec.commands_dir.and_then(in_home) + match spec.command_surface { + CommandSurface::Markdown { dir, .. } | CommandSurface::Plugin { dir } => in_home(dir), + CommandSurface::None { .. } => None, + } +} + +/// Why engram cannot install a command here, when it cannot. +pub fn no_command_detail(spec: &HarnessSpec) -> Option<&'static str> { + match spec.command_surface { + CommandSurface::None { detail } => Some(detail), + _ => None, + } } /// The database path this harness already registered engram against. @@ -469,6 +541,43 @@ pub fn from_env() -> Option<&'static HarnessSpec> { mod tests { use super::*; + /// `--harness ` and the `harness` field in every response must be the + /// same string. They come from different places — clap derives one from the + /// enum variant, `HarnessSpec::name` hardcodes the other — so nothing but a + /// test keeps them aligned. `OpenClaude` broke this on arrival: clap + /// derived `open-claude` while the spec said `openclaude`. + #[test] + fn value_enum_names_match_spec_names() { + use clap::ValueEnum; + for spec in ALL { + let variant = spec + .id + .to_possible_value() + .expect("every harness is selectable"); + assert_eq!( + variant.get_name(), + spec.name, + "--harness value and HarnessSpec::name disagree for {:?}", + spec.id + ); + } + } + + /// Serialization must agree too: a response's `harness` field is read back + /// by scripts and fed to `--harness`. + #[test] + fn serialized_harness_names_match_spec_names() { + for spec in ALL { + let json = serde_json::to_string(&spec.id).expect("serialize"); + assert_eq!( + json.trim_matches('"'), + spec.name, + "serde name and HarnessSpec::name disagree for {:?}", + spec.id + ); + } + } + #[test] fn every_variant_has_exactly_one_spec() { for entry in ALL { diff --git a/src/http.rs b/src/http.rs index ef4de51..20a6451 100644 --- a/src/http.rs +++ b/src/http.rs @@ -445,7 +445,7 @@ async fn rule_add(State(store): State, Json(body): Json err( @@ -541,7 +541,7 @@ async fn rule_retire( "outcome": retire.outcome, "rule": retire.rule, "scope_origin": resolved.origin, - "next_step": "POST /v1/rules/sync to drop this rule from AGENTS.md and CLAUDE.md", + "next_step": "POST /v1/rules/sync to drop this rule from AGENTS.md", }), ), Err(e) => err( @@ -559,8 +559,8 @@ struct RuleSyncBody { dry_run: bool, } -/// `POST /v1/rules/sync` — render a scope's rules into `AGENTS.md` and -/// `CLAUDE.md` at the project root. +/// `POST /v1/rules/sync` — render a scope's rules into `AGENTS.md` at the +/// project root. /// /// This is the only route that writes outside the database. The target paths /// are derived from the server process's own working directory, never from diff --git a/src/install.rs b/src/install.rs index 48320d3..ae3b633 100644 --- a/src/install.rs +++ b/src/install.rs @@ -28,7 +28,7 @@ //! home directory and, once hooks land, to code executed at every session //! end. There is no route, and there should not be one. -use crate::harness::{self, HarnessSpec}; +use crate::harness::{self, CommandSurface, HarnessSpec}; use crate::managed_file::{self, ManagedFile, WritePolicy}; use serde::Serialize; use std::path::PathBuf; @@ -64,6 +64,59 @@ const COMMANDS: &[CommandTemplate] = &[ /// perpetually-updating. const BANNER: &str = ""; +/// Where the database path baked into a generated command came from. +/// +/// Reported for the same reason `scope_origin` is reported on every other +/// response: the value alone does not say whether engram read it from the +/// harness's own registration or fell back to a relative default that writes +/// somewhere else entirely. +#[derive(Debug, Serialize, PartialEq, Eq, Clone, Copy)] +#[serde(rename_all = "kebab-case")] +pub enum DbOrigin { + /// `--db-path` on the command line. + Override, + /// Read from the harness's own MCP registration. The good case: agents and + /// commands then share one store by construction. + Registered, + /// The `ENGRAM_DB` environment variable. + Env, + /// clap's relative `engram.db`. Dangerous, because it resolves against + /// whatever directory the command happens to run in. + Default, +} + +/// Resolves the database a harness's commands should pin, and says where the +/// value came from. +fn resolve_db(spec: &HarnessSpec, db_override: Option<&str>) -> (String, DbOrigin) { + if let Some(db) = db_override { + return (db.to_string(), DbOrigin::Override); + } + // The database the harness itself registered wins: it is the one the + // user's agents actually share. + if let Some(db) = harness::registered_db(spec) { + return (db.to_string_lossy().into_owned(), DbOrigin::Registered); + } + if let Ok(db) = std::env::var("ENGRAM_DB") { + if !db.trim().is_empty() { + return (db, DbOrigin::Env); + } + } + ("engram.db".to_string(), DbOrigin::Default) +} + +/// The `--db` value already pinned in an installed command file, if any. +/// +/// Read back so a stale pin can be reported rather than silently corrected. +/// A command file written before the user re-registered engram keeps pointing +/// at the old store, which means the slash commands and the MCP tools read +/// different databases and neither side says so. +fn pinned_db(path: &std::path::Path) -> Option { + let text = std::fs::read_to_string(path).ok()?; + let (_, rest) = text.split_once("--db ")?; + let value = rest.split_whitespace().next()?; + Some(value.to_string()) +} + /// What one harness got. #[derive(Debug, Serialize)] pub struct HarnessInstall { @@ -72,6 +125,9 @@ pub struct HarnessInstall { /// The database baked into this harness's generated commands. #[serde(skip_serializing_if = "Option::is_none")] pub db: Option, + /// Where that database path came from. + #[serde(skip_serializing_if = "Option::is_none")] + pub db_origin: Option, pub files: Vec, /// Why nothing was written, when nothing was. #[serde(skip_serializing_if = "Option::is_none")] @@ -108,7 +164,14 @@ pub struct InstallResult { /// the first line as the command's description — so every engram command listed /// as "Generated by `engram install`" instead of what it does. pub fn render_command(template_body: &str, spec: &HarnessSpec, db: &str) -> String { - let body = if spec.command_frontmatter { + let keeps_frontmatter = matches!( + spec.command_surface, + harness::CommandSurface::Markdown { + frontmatter: true, + .. + } + ); + let body = if keeps_frontmatter { template_body.to_string() } else { strip_frontmatter(template_body) @@ -120,6 +183,62 @@ pub fn render_command(template_body: &str, spec: &HarnessSpec, db: &str) -> Stri } } +/// The plugin directory engram owns inside a harness's plugins root. +const PLUGIN_NAME: &str = "engram"; + +/// A plugin manifest is just its marker; `name` defaults to the directory, but +/// stating it keeps the displayed name stable if the directory is ever moved. +const PLUGIN_MANIFEST: &str = "{\n \"name\": \"engram\"\n}\n"; + +/// Where one command lands for one harness. +/// +/// A markdown harness gets a flat file; a plugin harness gets a skill, which is +/// a *directory* holding `SKILL.md` — the vendor's loader discovers skills by +/// that shape, so a flat file would simply never be seen. +fn command_path(spec: &HarnessSpec, dir: &std::path::Path, name: &str) -> PathBuf { + match spec.command_surface { + CommandSurface::Markdown { file, .. } => dir.join(file.replace("{name}", name)), + CommandSurface::Plugin { .. } => dir + .join(PLUGIN_NAME) + .join("skills") + .join(format!("engram-{name}")) + .join("SKILL.md"), + // Unreachable: the caller returns early for a harness with no surface. + CommandSurface::None { .. } => dir.join(name), + } +} + +/// Renders one command as a Gemini-family skill. +/// +/// The frontmatter contract differs from a command file's: a skill is keyed by +/// `name` (which the loader matches against the directory) and `description`, +/// and carries neither `argument-hint` nor `allowed-tools`. Rather than ship a +/// second copy of every body, the shared template's description is lifted and +/// the rest of its frontmatter dropped — so the two surfaces cannot drift in +/// what they tell the user the command does. +/// +/// Standard §5.6 caps a shipped description at 1000 characters; these are two +/// orders of magnitude below that, and the assertion in the tests keeps it so. +fn render_skill(name: &str, template_body: &str, spec: &HarnessSpec, db: &str) -> String { + let description = + frontmatter_field(template_body, "description").unwrap_or_else(|| format!("engram {name}")); + let body = strip_frontmatter(template_body) + .replace("{{DB}}", db) + .replace("{{HARNESS}}", spec.name); + format!("---\nname: engram-{name}\ndescription: {description}\n---\n{BANNER}\n{body}") +} + +/// Reads one scalar field out of a leading YAML block. Deliberately not a YAML +/// parser: the templates are engram's own, the shape is fixed, and a dependency +/// here would buy nothing. +fn frontmatter_field(body: &str, key: &str) -> Option { + let (front, _) = split_frontmatter(body)?; + front + .lines() + .find_map(|l| l.strip_prefix(&format!("{key}: "))) + .map(|v| v.trim().trim_matches('"').to_string()) +} + /// Splits a leading `---`-delimited YAML block from the body, keeping the /// delimiters with the first half. `None` when there is no well-formed block. fn split_frontmatter(body: &str) -> Option<(&str, &str)> { @@ -186,8 +305,9 @@ pub fn install( harness: spec.name, present: detected.present, db: None, + db_origin: None, files: Vec::new(), - note: Some("no command surface engram can write"), + note: harness::no_command_detail(spec), warning: None, }); continue; @@ -197,6 +317,7 @@ pub fn install( harness: spec.name, present: false, db: None, + db_origin: None, files: Vec::new(), note: Some("not installed; engram will not create a home for it"), warning: None, @@ -204,15 +325,9 @@ pub fn install( continue; } - // The database the harness itself registered wins: it is the one the - // user's agents actually share. - let db = db_override - .map(str::to_string) - .or_else(|| harness::registered_db(spec).map(|p| p.to_string_lossy().into_owned())) - .or_else(|| std::env::var("ENGRAM_DB").ok()) - .unwrap_or_else(|| "engram.db".to_string()); + let (db, db_origin) = resolve_db(spec, db_override); - let warning = is_nix_managed(&dir).then(|| { + let nix_warning = is_nix_managed(&dir).then(|| { format!( "{} resolves into the Nix store; anything written here will be replaced by the \ next home-manager switch. Reference plugins/engram/ from your configuration \ @@ -222,8 +337,9 @@ pub fn install( }); let mut files = Vec::new(); + let mut drifted: Vec = Vec::new(); for command in COMMANDS { - let path = dir.join(spec.command_file.replace("{name}", command.name)); + let path = command_path(spec, &dir, command.name); if !force && !is_ours(&path) { skipped += 1; files.push(ManagedFile { @@ -234,14 +350,67 @@ pub fn install( }); continue; } - let body = render_command(command.body, spec, &db); - let written = managed_file::write_managed(&path, &body, WritePolicy::Owned, dry_run)?; + // Read the old pin BEFORE overwriting it. A file that pointed at a + // different database was sending this harness's slash commands to a + // different store than its MCP tools, and correcting that silently + // would destroy the only evidence the user had. + let stale = pinned_db(&path).filter(|old| *old != db); + + let body = match spec.command_surface { + CommandSurface::Plugin { .. } => { + render_skill(command.name, command.body, spec, &db) + } + _ => render_command(command.body, spec, &db), + }; + let mut written = + managed_file::write_managed(&path, &body, WritePolicy::Owned, dry_run)?; + if let Some(old) = &stale { + drifted.push(old.clone()); + written.reason = Some(format!( + "was pinned to {old}; re-pinned to {db}. Commands and MCP tools were reading \ + different databases." + )); + } if written.outcome != managed_file::FileOutcome::Unchanged { installed += 1; } files.push(written); } + // A plugin directory is only a plugin once its manifest exists. + if let CommandSurface::Plugin { .. } = spec.command_surface { + let manifest = dir.join(PLUGIN_NAME).join("plugin.json"); + let written = managed_file::write_managed( + &manifest, + PLUGIN_MANIFEST, + WritePolicy::Owned, + dry_run, + )?; + if written.outcome != managed_file::FileOutcome::Unchanged { + installed += 1; + } + files.push(written); + } + + // One `warning` field, two possible causes. Drift is stated first: a + // command pointing at the wrong database is a correctness problem now, + // where the Nix note is about a future rebuild. + let warning = match (drifted.first(), nix_warning) { + (Some(old), nix) => { + let mut w = format!( + "generated commands were pinned to {old} but this harness registers {db}; \ + re-pinned. Until now, /engram-* commands and engram's MCP tools were reading \ + different databases." + ); + if let Some(n) = nix { + w.push(' '); + w.push_str(&n); + } + Some(w) + } + (None, nix) => nix, + }; + // Hooks are opt-in twice over: this flag, and the fact that a hook // only ever runs `ingest`. if hooks { @@ -263,6 +432,7 @@ pub fn install( harness: spec.name, present: true, db: Some(db), + db_origin: Some(db_origin), files, note: None, warning, @@ -459,7 +629,7 @@ pub fn install_hooks( pub fn default_targets() -> Vec<&'static HarnessSpec> { harness::ALL .iter() - .filter(|s| s.commands_dir.is_some()) + .filter(|s| !matches!(s.command_surface, CommandSurface::None { .. })) .collect() } @@ -633,13 +803,75 @@ mod tests { let targets = default_targets(); assert!(!targets.is_empty()); for spec in &targets { - assert!(spec.commands_dir.is_some()); + assert!( + !matches!(spec.command_surface, CommandSurface::None { .. }), + "{} has no surface and must not be a default target", + spec.name + ); + } + // Harnesses with no surface are excluded, not silently pretended to + // work. Antigravity used to be one of these; it now has a plugin + // surface, so the exclusion moved to the harnesses that really have + // none. + for id in [ + harness::Harness::Goose, + harness::Harness::CopilotCli, + harness::Harness::Qwen, + ] { + assert!( + !targets.iter().any(|s| s.id == id), + "{id:?} has no command surface and must not be a default target" + ); + } + } + + /// Every harness without a surface explains itself. The reasons differ — + /// unsurveyed storage is not the same as no writable directory — and one + /// shared sentence fit none of them precisely. + #[test] + fn every_surfaceless_harness_states_its_own_reason() { + let mut seen: Vec<&str> = Vec::new(); + for spec in harness::ALL { + if let CommandSurface::None { detail } = spec.command_surface { + assert!(!detail.is_empty(), "{} has an empty reason", spec.name); + assert!( + !seen.contains(&detail), + "{} reuses another harness's reason verbatim", + spec.name + ); + seen.push(detail); + } + } + assert!(!seen.is_empty()); + } + + /// A skill's frontmatter is a different contract from a command file's: + /// `name` + `description`, no `argument-hint`, no `allowed-tools`. The + /// description is lifted from the shared template so the two surfaces + /// cannot describe the same command differently. + #[test] + fn skills_carry_gemini_frontmatter_and_the_shared_description() { + let spec = harness::spec(harness::Harness::Antigravity); + for command in COMMANDS { + let out = render_skill(command.name, command.body, spec, "/shared/engram.db"); + assert!(out.starts_with("---\n"), "{}: {out}", command.name); + assert!(out.contains(&format!("name: engram-{}", command.name))); + assert!(!out.contains("argument-hint:"), "{}", command.name); + assert!(!out.contains("allowed-tools:"), "{}", command.name); + assert!(out.contains("--db /shared/engram.db")); + assert!(!out.contains("{{DB}}")); + + let description = frontmatter_field(command.body, "description").expect("description"); + assert!(out.contains(&description), "{}", command.name); + // Standard §5.6: a shipped SKILL.md description is capped at 1000 + // characters. Assert the cap here, where it would first be broken. + assert!( + description.chars().count() <= 1000, + "{}: description is {} chars", + command.name, + description.chars().count() + ); } - // Harnesses with no command surface are excluded, not silently - // pretended to work. - assert!(!targets - .iter() - .any(|s| s.id == harness::Harness::Antigravity)); } } diff --git a/src/main.rs b/src/main.rs index dc996a6..ae46a55 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1148,7 +1148,7 @@ fn run_rule(action: RuleAction, store: &Arc>, mode: OutputMode) -> rule: upsert.rule, created: upsert.created, scope_origin: resolved.origin, - next_step: "run `engram rule sync` to render this rule into AGENTS.md and CLAUDE.md; \ + next_step: "run `engram rule sync` to render this rule into AGENTS.md; \ until then no agent will read it", }; emit_ok(Response::new("engram rule add", result), mode); @@ -1211,8 +1211,8 @@ fn run_rule(action: RuleAction, store: &Arc>, mode: OutputMode) -> let payload = RuleRetireResult { retire, scope_origin: resolved.origin, - next_step: "run `engram rule sync` to drop this rule from AGENTS.md and \ - CLAUDE.md; until then the synced files still assert it", + next_step: "run `engram rule sync` to drop this rule from AGENTS.md; \ + until then the synced file still asserts it", }; emit_ok(Response::new("engram rule retire", payload), mode); 0 diff --git a/src/managed_file.rs b/src/managed_file.rs index fff4e1a..c39f162 100644 --- a/src/managed_file.rs +++ b/src/managed_file.rs @@ -6,9 +6,9 @@ //! single implementation of both: //! //! * [`WritePolicy::Spliced`] — the file belongs to someone else and engram -//! owns only the region between a pair of sentinels. `AGENTS.md` and -//! `CLAUDE.md` are the canonical case: everything outside the sentinels is -//! preserved byte-for-byte (see [`crate::rules`]). +//! owns only the region between a pair of sentinels. `AGENTS.md` is the +//! canonical case: everything outside the sentinels is preserved +//! byte-for-byte (see [`crate::rules`]). //! * [`WritePolicy::Owned`] — engram authored the whole file and rewrites it //! wholesale. A chat archive is the canonical case. //! @@ -182,14 +182,69 @@ pub fn splice_block(existing: &str, block: &str, sentinel: &Sentinel) -> String pub fn find_git_root(start: &Path) -> Option { start .ancestors() - .find(|dir| dir.join(".git").exists()) + .find(|dir| is_git_root(dir)) .map(Path::to_path_buf) } +/// True when `dir` is the root of a real working tree. +/// +/// Existence of `.git` alone is not enough. An empty `.git` directory is not a +/// repository, and one sitting somewhere shared — `/tmp/.git` is the case that +/// prompted this — silently captures every path beneath it: `save-chat` would +/// resolve its project root to `/tmp`, write `/tmp/chat/`, and add `chat/` to +/// `/tmp/.gitignore`. Both forms git actually produces are accepted: a +/// directory containing `HEAD`, and the `gitdir:` *file* a worktree or submodule +/// checkout uses. +fn is_git_root(dir: &Path) -> bool { + let git = dir.join(".git"); + match std::fs::metadata(&git) { + Ok(m) if m.is_dir() => git.join("HEAD").exists(), + Ok(m) if m.is_file() => true, + _ => false, + } +} + #[cfg(test)] mod tests { use super::*; + /// An empty `.git` directory is not a repository, and must not capture + /// every path beneath it. + /// + /// This is not hypothetical: an empty `/tmp/.git` on the author's machine + /// made `save-chat` resolve its project root to `/tmp`, where it created + /// `/tmp/chat/` and added `chat/` to `/tmp/.gitignore`. Existence of `.git` + /// is not the test; being a working tree is. + #[test] + fn an_empty_git_directory_is_not_a_working_tree() { + let tmp = std::env::temp_dir().join(format!("engram-gitroot-{}", std::process::id())); + let shallow = tmp.join("empty-git"); + let nested = shallow.join("a").join("b"); + std::fs::create_dir_all(&nested).expect("create tree"); + std::fs::create_dir_all(shallow.join(".git")).expect("empty .git dir"); + assert_eq!( + find_git_root(&nested), + None, + "an empty .git directory must not be treated as a repository root" + ); + + // A real repository has HEAD. + std::fs::write(shallow.join(".git/HEAD"), "ref: refs/heads/main\n").expect("write HEAD"); + assert_eq!(find_git_root(&nested).as_deref(), Some(shallow.as_path())); + + // A worktree or submodule uses a `.git` file instead. + let linked = tmp.join("worktree"); + let linked_nested = linked.join("x"); + std::fs::create_dir_all(&linked_nested).expect("create worktree tree"); + std::fs::write(linked.join(".git"), "gitdir: /elsewhere\n").expect("write gitdir file"); + assert_eq!( + find_git_root(&linked_nested).as_deref(), + Some(linked.as_path()) + ); + + std::fs::remove_dir_all(&tmp).ok(); + } + fn block(body: &str) -> String { format!( "{} count=\"1\" -->\n{body}\n{}", diff --git a/src/mcp.rs b/src/mcp.rs index 2acb8fe..cf41255 100644 --- a/src/mcp.rs +++ b/src/mcp.rs @@ -386,7 +386,7 @@ impl EngramMcp { "created": upsert.created, "scope": resolved.name, "scope_origin": resolved.origin, - "next_step": "call rule_sync to render this rule into AGENTS.md and CLAUDE.md; \ + "next_step": "call rule_sync to render this rule into AGENTS.md; \ until then no agent will read it", }); Ok(CallToolResult::success(vec![Content::text( @@ -453,7 +453,7 @@ impl EngramMcp { "outcome": retire.outcome, "rule": retire.rule, "scope_origin": resolved.origin, - "next_step": "call rule_sync to drop this rule from AGENTS.md and CLAUDE.md; \ + "next_step": "call rule_sync to drop this rule from AGENTS.md; \ until then the synced files still assert it", }); Ok(CallToolResult::success(vec![Content::text( @@ -462,7 +462,7 @@ impl EngramMcp { } #[tool( - description = "Render a scope's rules into AGENTS.md and CLAUDE.md at the project root, \ + description = "Render a scope's rules into AGENTS.md at the project root, \ rewriting only the region between the engram sentinels and leaving the rest \ of each file untouched. Idempotent: re-running with unchanged rules writes \ nothing. This is the step that actually puts rules in front of a model." @@ -701,7 +701,7 @@ impl ServerHandler for EngramMcp { Rules: call `rule_list` at session start to load the policy governing this \ project. Call `rule_add` when the user states a standing requirement — \ something that must keep applying in future sessions, not a one-off fact — then \ - `rule_sync` to render it into AGENTS.md and CLAUDE.md. A stored rule that is \ + `rule_sync` to render it into AGENTS.md. A stored rule that is \ never synced is read by nobody. Call `rule_retire` when a rule no longer \ applies, then `rule_sync` again." .to_string(), diff --git a/src/rules.rs b/src/rules.rs index d6ca42d..9d4852f 100644 --- a/src/rules.rs +++ b/src/rules.rs @@ -10,8 +10,9 @@ //! # Why `sync` is not merely an export //! //! A row in SQLite never reaches a model's context on its own. Rendering rules -//! into `AGENTS.md` / `CLAUDE.md` — files that agent harnesses load -//! automatically — is what actually makes a rule take effect. The database is +//! into `AGENTS.md` — the file that agent harnesses load automatically, and +//! that Claude Code reaches through the `@AGENTS.md` import its `CLAUDE.md` +//! carries (Standard §5.7) — is what makes a rule take effect. The database is //! the durable source of truth; the managed markdown block is the delivery //! mechanism. Storing a rule without syncing it stores a rule nobody reads. @@ -38,9 +39,18 @@ pub const STATUS_ACTIVE: &str = "active"; /// session needs to be able to find, and `search` still reaches retired rules. pub const STATUS_RETIRED: &str = "retired"; -/// Files `engram rule sync` writes when no `--file` is supplied. Both are -/// auto-loaded by agent harnesses, which is the entire point of syncing. -pub const DEFAULT_TARGETS: [&str; 2] = ["AGENTS.md", "CLAUDE.md"]; +/// File `engram rule sync` writes when no `--file` is supplied. +/// +/// `AGENTS.md` only, deliberately. Steelbore Standard §5.7 makes `AGENTS.md` +/// the single harness-neutral source of truth and reduces `CLAUDE.md` to an +/// `@AGENTS.md` import plus Claude-only content — so a block written to both +/// arrives twice in Claude's context and, worse, becomes two copies that can +/// disagree once anything edits one of them. §5.7 puts the obligation on the +/// tooling: rendered blocks target `AGENTS.md`, and every harness that reads +/// `CLAUDE.md` picks them up through the import. +/// +/// Callers that genuinely need another destination still pass `--file`. +pub const DEFAULT_TARGETS: [&str; 1] = ["AGENTS.md"]; /// Sentinel pair delimiting the managed block. Defined in /// [`crate::managed_file`] so the splice machinery and the renderer cannot diff --git a/src/store.rs b/src/store.rs index 8a0d066..1b11d20 100644 --- a/src/store.rs +++ b/src/store.rs @@ -352,6 +352,22 @@ pub struct ContextResult { pub budget: crate::retrieval::BudgetReport, } +/// Decodes a stored embedding from its f32 little-endian blob. +/// +/// `as_chunks::<4>()` yields `[u8; 4]` arrays directly, so there is no fallible +/// conversion to justify with an `expect`. Any trailing bytes that do not fill +/// a whole `f32` are dropped, exactly as the previous `chunks_exact` did: a +/// truncated blob is a corrupt row, and a short vector cosines to 0.0 rather +/// than panicking mid-query. +fn decode_vector(blob: &[u8]) -> Vec { + blob.as_chunks::<4>() + .0 + .iter() + .copied() + .map(f32::from_le_bytes) + .collect() +} + impl Store { /// Opens (creating if needed) the shared database file. Point every /// agent at the same path — that's the entire "shared memory" story. @@ -979,10 +995,7 @@ impl Store { let mut scored: Vec<(String, f32)> = Vec::new(); for row in rows { let (id, blob) = row?; - let vec: Vec = blob - .chunks_exact(4) - .map(|b| f32::from_le_bytes(b.try_into().expect("chunks_exact yields 4 bytes"))) - .collect(); + let vec = decode_vector(&blob); // A dimension mismatch (different-width model under the same // name) cosines to 0.0 rather than erroring — see `cosine`. scored.push((id, crate::embed::cosine(query_vec, &vec))); @@ -1205,14 +1218,7 @@ impl Store { let mut vectors: HashMap> = HashMap::new(); for row in fetched { let (id, blob) = row?; - vectors.insert( - id, - blob.chunks_exact(4) - .map(|b| { - f32::from_le_bytes(b.try_into().expect("chunks_exact yields 4 bytes")) - }) - .collect(), - ); + vectors.insert(id, decode_vector(&blob)); } for i in 0..rows.len() { let Some(vec_i) = vectors.get(&rows[i].0) else { diff --git a/tests/cli.rs b/tests/cli.rs index eba1ab0..9b58149 100644 --- a/tests/cli.rs +++ b/tests/cli.rs @@ -468,7 +468,7 @@ fn rule_sync_dry_run_writes_no_files() { .success(); // Run from the temp dir: it has no .git, so the project root — and the - // default AGENTS.md/CLAUDE.md targets — resolve there. + // default AGENTS.md target — resolves there. let assert = engram(&db) .current_dir(tmp.path()) .args(["rule", "sync", "--scope", scope, "--dry-run"]) @@ -486,8 +486,15 @@ fn rule_sync_dry_run_writes_no_files() { .expect("data.files is an array"); assert_eq!( files.len(), - 2, - "default targets are AGENTS.md and CLAUDE.md" + 1, + "AGENTS.md is the sole default target (Standard §5.7); CLAUDE.md \ + receives the block through its @AGENTS.md import, so writing both \ + would deliver it twice and create two copies that can disagree" + ); + assert_eq!( + files[0]["path"].as_str().expect("file path is a string"), + tmp.path().join("AGENTS.md").to_string_lossy(), + "the default target is AGENTS.md, not CLAUDE.md" ); for file in files { assert_eq!(file["dry_run"], true); @@ -500,7 +507,7 @@ fn rule_sync_dry_run_writes_no_files() { ); assert!( missing.eval(&tmp.path().join("CLAUDE.md")), - "dry run must not create CLAUDE.md" + "CLAUDE.md is not a sync target at all (§5.7), dry run or not" ); } @@ -1358,6 +1365,21 @@ fn describe_advertises_consolidate_phases_and_access_tracking() { // ---------------------------------------------------------------- save-chat +/// Creates a project directory that is its own git root. +/// +/// Pinning the root is not decoration. `find_git_root` walks *up* from the +/// project, so without a marker a test's "project root" is whatever ancestor of +/// the tempdir happens to contain a `.git`. On a machine with an empty +/// `/tmp/.git`, that root is `/tmp` — and the suite then writes `chat/` and +/// edits `.gitignore` there instead of inside its own tempdir. A test must not +/// depend on whether the developer's `/tmp` looks like a repository. +fn pinned_project(tmp: &TempDir) -> std::path::PathBuf { + let project = tmp.path().join("project"); + std::fs::create_dir_all(&project).expect("create project dir"); + std::fs::write(project.join(".git"), "gitdir: elsewhere\n").expect("pin the git root"); + project +} + /// A `save-chat` invocation rooted at `project`, so scope resolution walks up /// from a scratch directory instead of the crate's own git tree — the command /// writes `chat/` and `.gitignore` at whatever root it resolves. @@ -1376,8 +1398,7 @@ fn save_chat(db: &Path, project: &Path, args: &[&str]) -> Command { fn save_chat_twice_is_byte_identical() { let tmp = TempDir::new().expect("tempdir"); let db = tmp.path().join("test.db"); - let project = tmp.path().join("project"); - std::fs::create_dir_all(&project).expect("create project dir"); + let project = pinned_project(&tmp); let scope = "archive-idempotent"; remember(&db, "a", scope, "the first message"); @@ -1447,8 +1468,7 @@ fn save_chat_twice_is_byte_identical() { fn save_chat_excludes_rule_rows() { let tmp = TempDir::new().expect("tempdir"); let db = tmp.path().join("test.db"); - let project = tmp.path().join("project"); - std::fs::create_dir_all(&project).expect("create project dir"); + let project = pinned_project(&tmp); let scope = "archive-rules"; remember(&db, "a", scope, "an ordinary message"); @@ -1490,8 +1510,7 @@ fn save_chat_excludes_rule_rows() { fn save_chat_does_not_bump_access_counts() { let tmp = TempDir::new().expect("tempdir"); let db = tmp.path().join("test.db"); - let project = tmp.path().join("project"); - std::fs::create_dir_all(&project).expect("create project dir"); + let project = pinned_project(&tmp); let scope = "archive-untracked"; remember(&db, "a", scope, "the archived memory"); @@ -1523,8 +1542,7 @@ fn save_chat_does_not_bump_access_counts() { fn save_chat_dry_run_writes_no_files() { let tmp = TempDir::new().expect("tempdir"); let db = tmp.path().join("test.db"); - let project = tmp.path().join("project"); - std::fs::create_dir_all(&project).expect("create project dir"); + let project = pinned_project(&tmp); let scope = "archive-dry-run"; remember(&db, "a", scope, "a message"); @@ -1600,8 +1618,7 @@ fn save_chat_resolves_relative_file_against_the_project_root() { fn save_chat_reports_each_gitignore_outcome_distinctly() { let tmp = TempDir::new().expect("tempdir"); let db = tmp.path().join("test.db"); - let project = tmp.path().join("project"); - std::fs::create_dir_all(&project).expect("create project"); + let project = pinned_project(&tmp); std::fs::write(project.join(".git"), "gitdir: elsewhere\n").expect("write .git marker"); let scope = "gitignore-outcomes"; remember(&db, "a", scope, "a message"); @@ -1647,8 +1664,7 @@ fn save_chat_output_compiles_under_makeinfo() { let tmp = TempDir::new().expect("tempdir"); let db = tmp.path().join("test.db"); - let project = tmp.path().join("project"); - std::fs::create_dir_all(&project).expect("create project dir"); + let project = pinned_project(&tmp); let scope = "archive-makeinfo"; // Exercise the escaping and the encoding declaration together: markup @@ -1719,10 +1735,8 @@ fn ingest(db: &Path, home: &Path, project: &Path, args: &[&str]) -> Command { /// Builds a fake home + project pair inside one tempdir. fn ingest_fixture(tmp: &TempDir) -> (std::path::PathBuf, std::path::PathBuf) { let home = tmp.path().join("home"); - let project = tmp.path().join("project"); std::fs::create_dir_all(&home).expect("create fake home"); - std::fs::create_dir_all(&project).expect("create project"); - (home, project) + (home, pinned_project(tmp)) } #[test] @@ -2081,7 +2095,7 @@ fn install_list_reports_every_harness_and_writes_nothing() { let data = parse_single_line_json(&assert.get_output().stdout)["data"].clone(); let harnesses = data["harnesses"].as_array().expect("harnesses"); - assert_eq!(harnesses.len(), 7, "every known harness must be reported"); + assert_eq!(harnesses.len(), 8, "every known harness must be reported"); let claude = harnesses .iter() @@ -2093,13 +2107,34 @@ fn install_list_reports_every_harness_and_writes_nothing() { .expect("commands_dir") .ends_with(".claude/commands")); - // Harnesses without a command surface say so with a null rather than - // being quietly omitted. + // The fork is listed with its own config root, not Claude Code's. + let openclaude = harnesses + .iter() + .find(|h| h["name"] == "openclaude") + .expect("openclaude listed"); + assert!(openclaude["commands_dir"] + .as_str() + .expect("commands_dir") + .ends_with(".openclaude/commands")); + + // Antigravity has no *command* directory, but it does have a plugin root, + // so it reports a writable location rather than a null. let antigravity = harnesses .iter() .find(|h| h["name"] == "antigravity") .expect("antigravity listed"); - assert!(antigravity["commands_dir"].is_null()); + assert!(antigravity["commands_dir"] + .as_str() + .expect("plugin root") + .ends_with(".gemini/config/plugins")); + + // Harnesses with nothing engram can write still say so with a null rather + // than being quietly omitted. + let goose = harnesses + .iter() + .find(|h| h["name"] == "goose") + .expect("goose listed"); + assert!(goose["commands_dir"].is_null()); // Nothing was created, not even for the harness that is "installed". assert!(predicate::path::missing().eval(&home.join(".claude/commands"))); @@ -2327,22 +2362,70 @@ fn install_discovers_the_database_from_the_harness_mcp_config() { ); } +/// Antigravity has no slash-command directory, but it does have a writable +/// plugin root, and a skill there expands the way a command does. Engram writes +/// a plugin: a manifest plus one skill directory per command. #[test] -fn install_reports_harnesses_with_no_command_surface() { +fn install_writes_an_antigravity_plugin_rather_than_commands() { let tmp = TempDir::new().expect("tempdir"); let db = tmp.path().join("test.db"); let home = tmp.path().join("home"); std::fs::create_dir_all(&home).expect("create fake home"); pretend_installed(&home, ".gemini/antigravity"); + install(&db, &home, &["--harness", "antigravity"]) + .assert() + .success(); + + let plugin = home.join(".gemini/config/plugins/engram"); + let manifest = std::fs::read_to_string(plugin.join("plugin.json")).expect("manifest written"); + let value: Value = serde_json::from_str(&manifest).expect("manifest is JSON"); + assert_eq!(value["name"], "engram"); + + for name in ["save-chat", "ingest", "context"] { + // A skill is a *directory* holding SKILL.md; the loader discovers it by + // that shape, so a flat file would never be seen. + let skill = plugin.join(format!("skills/engram-{name}/SKILL.md")); + let text = std::fs::read_to_string(&skill).unwrap_or_else(|e| panic!("{name}: {e}")); + assert!(text.starts_with("---\n"), "{name}: {text}"); + assert!(text.contains(&format!("name: engram-{name}")), "{name}"); + assert!(text.contains("description: "), "{name}"); + // Gemini skill frontmatter, not Claude command frontmatter. + assert!(!text.contains("argument-hint:"), "{name}"); + assert!(!text.contains("allowed-tools:"), "{name}"); + assert!(!text.contains("{{DB}}"), "{name}: placeholder left"); + } + + // Idempotent, like every other surface. let assert = install(&db, &home, &["--harness", "antigravity"]) .assert() .success(); let json = parse_single_line_json(&assert.get_output().stdout); + assert_eq!(json["data"]["installed"], 0); +} + +/// A harness with genuinely no surface says which one it is and why, in its own +/// words. One sentence shared by four harnesses described none of them. +#[test] +fn install_reports_harnesses_with_no_command_surface() { + let tmp = TempDir::new().expect("tempdir"); + let db = tmp.path().join("test.db"); + let home = tmp.path().join("home"); + std::fs::create_dir_all(&home).expect("create fake home"); + pretend_installed(&home, ".config/goose"); + + let assert = install(&db, &home, &["--harness", "goose"]) + .assert() + .success(); + let json = parse_single_line_json(&assert.get_output().stdout); let entry = &json["data"]["harnesses"][0]; - assert_eq!(entry["harness"], "antigravity"); + assert_eq!(entry["harness"], "goose"); assert!(entry["files"].as_array().expect("files").is_empty()); - assert_eq!(entry["note"], "no command surface engram can write"); + let note = entry["note"].as_str().expect("a reason, not a null"); + assert!( + note.contains("goose"), + "the reason names the harness: {note}" + ); } #[test] @@ -2979,3 +3062,229 @@ fn shell_words(line: &str) -> Vec { } out } + +/// A command file pinned to a database the harness no longer registers must be +/// reported, not silently corrected. +/// +/// This is what happened on the author's machine: `install` ran, then every +/// harness's MCP registration moved to a different store. The generated +/// commands kept pointing at the old one, so `/engram-*` and engram's MCP tools +/// read different databases for two weeks and nothing said so. Correcting the +/// pin without a word would have erased the only evidence. +#[test] +fn install_reports_a_database_pin_that_drifted() { + let tmp = TempDir::new().expect("tempdir"); + let db = tmp.path().join("test.db"); + let home = tmp.path().join("home"); + std::fs::create_dir_all(home.join(".claude")).expect("create fake home"); + + // The harness registers one database... + let registered = tmp.path().join("registered.db"); + std::fs::write( + home.join(".claude.json"), + serde_json::json!({ + "mcpServers": { + "engram": { + "type": "stdio", + "command": "engram", + "args": ["--db", registered.to_string_lossy(), "mcp"], + } + } + }) + .to_string(), + ) + .expect("write registration"); + + // ...but an earlier install pinned another. + install(&db, &home, &["--db-path", "/stale/engram.db"]) + .assert() + .success(); + let cmd = home.join(".claude/commands/engram-context.md"); + assert!( + std::fs::read_to_string(&cmd) + .expect("read") + .contains("--db /stale/engram.db"), + "precondition: the stale pin is in place" + ); + + // A plain re-install notices, says so, and corrects it. + let assert = install(&db, &home, &[]).assert().success(); + let json = parse_single_line_json(&assert.get_output().stdout); + let harness = json["data"]["harnesses"] + .as_array() + .expect("harnesses") + .iter() + .find(|h| h["harness"] == "claude-code") + .expect("claude-code") + .clone(); + + assert_eq!(harness["db_origin"], "registered"); + assert_eq!( + harness["db"].as_str().expect("db"), + registered.to_string_lossy() + ); + let warning = harness["warning"].as_str().expect("drift warning"); + assert!(warning.contains("/stale/engram.db"), "{warning}"); + assert!(warning.contains("different databases"), "{warning}"); + assert!( + harness["files"][0]["reason"] + .as_str() + .expect("per-file reason") + .contains("re-pinned"), + "the file itself carries the reason too" + ); + + let after = std::fs::read_to_string(&cmd).expect("read"); + assert!(after.contains(&format!("--db {}", registered.to_string_lossy()))); + assert!(!after.contains("/stale/engram.db")); + + // Settled: a third run is quiet. + let assert = install(&db, &home, &[]).assert().success(); + let json = parse_single_line_json(&assert.get_output().stdout); + let harness = json["data"]["harnesses"] + .as_array() + .expect("harnesses") + .iter() + .find(|h| h["harness"] == "claude-code") + .expect("claude-code") + .clone(); + assert!( + harness["warning"].is_null(), + "a settled pin must not keep warning: {harness:?}" + ); +} + +/// `db_origin` names the fallback that silently writes to a relative path. +#[test] +fn install_reports_the_dangerous_default_database_origin() { + let tmp = TempDir::new().expect("tempdir"); + let db = tmp.path().join("test.db"); + let home = tmp.path().join("home"); + std::fs::create_dir_all(home.join(".claude")).expect("create fake home"); + // No registration, and ENGRAM_DB is stripped by the hermetic env. + let assert = install(&db, &home, &[]).assert().success(); + let json = parse_single_line_json(&assert.get_output().stdout); + let harness = json["data"]["harnesses"] + .as_array() + .expect("harnesses") + .iter() + .find(|h| h["harness"] == "claude-code") + .expect("claude-code") + .clone(); + assert_eq!(harness["db_origin"], "default"); + assert_eq!(harness["db"], "engram.db"); +} + +/// Plants an OpenClaude transcript in the fork's own config root. +fn plant_openclaude_transcript( + home: &Path, + project: &Path, + session_id: &str, +) -> std::path::PathBuf { + let mangled = project.to_string_lossy().replace('/', "-"); + let dir = home.join(".openclaude").join("projects").join(mangled); + std::fs::create_dir_all(&dir).expect("create fake projects dir"); + let src = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/transcripts/openclaude/session-basic.jsonl"); + let dest = dir.join(format!("{session_id}.jsonl")); + std::fs::copy(&src, &dest).expect("copy fixture into the fake home"); + dest +} + +/// OpenClaude is a Claude Code fork, so the existing reader serves it. The +/// records that differ (`mode`, `file-history-snapshot`, `last-prompt`) are +/// already in the non-message allowlist and must be counted there rather than +/// as `unknown_record` — an unknown-record count is the signal that a harness +/// changed its format, and a fork tripping it every run would train the reader +/// to ignore it. +#[test] +fn ingest_reads_an_openclaude_transcript_with_the_claude_code_reader() { + let tmp = TempDir::new().expect("tempdir"); + let db = tmp.path().join("test.db"); + let (home, project) = ingest_fixture(&tmp); + plant_openclaude_transcript(&home, &project, "11111111-2222-3333-4444-555555555555"); + + let assert = ingest( + &db, + &home, + &project, + &["--harness", "openclaude", "--scope", "oc"], + ) + .assert() + .success(); + let data = parse_single_line_json(&assert.get_output().stdout)["data"].clone(); + + assert_eq!(data["harness"], "openclaude"); + assert_eq!(data["inserted"], 2, "one user turn and one assistant turn"); + assert_eq!( + data["filtered"]["unknown_record"], 0, + "fork-specific record types must be recognised, not reported as drift" + ); + assert_eq!(data["filtered"]["non_message"], 4); + + // Idempotent, exactly as for Claude Code. + let assert = ingest( + &db, + &home, + &project, + &["--harness", "openclaude", "--scope", "oc"], + ) + .assert() + .success(); + let data = parse_single_line_json(&assert.get_output().stdout)["data"].clone(); + assert_eq!(data["inserted"], 0); + assert_eq!(data["skipped_existing"], 2); + + // The turns are ordinary memories. + let mems = recall_data(&db, "oc"); + assert_eq!(mems.len(), 2); + assert_eq!(mems[0]["role"], "user"); + assert_eq!(mems[1]["role"], "assistant"); +} + +/// OpenClaude has its own writable command directory, and its MCP registration +/// lives in `~/.openclaude.json` — not in `~/.openclaude/settings.json`, which +/// holds env/model/hooks and no servers block. +#[test] +fn install_writes_openclaude_commands_and_reads_its_registration() { + let tmp = TempDir::new().expect("tempdir"); + let db = tmp.path().join("test.db"); + let home = tmp.path().join("home"); + std::fs::create_dir_all(home.join(".openclaude")).expect("create fake home"); + // A settings.json with no mcpServers must not be mistaken for the config. + std::fs::write( + home.join(".openclaude/settings.json"), + r#"{"model":"x","hooks":{}}"#, + ) + .expect("write settings"); + std::fs::write( + home.join(".openclaude.json"), + r#"{"mcpServers":{"engram":{"command":"engram","args":["--db","/shared/oc.db","mcp"]}}}"#, + ) + .expect("write registration"); + + let assert = install(&db, &home, &[]).assert().success(); + let json = parse_single_line_json(&assert.get_output().stdout); + let harness = json["data"]["harnesses"] + .as_array() + .expect("harnesses") + .iter() + .find(|h| h["harness"] == "openclaude") + .expect("openclaude is a known harness") + .clone(); + + assert_eq!(harness["present"], true); + assert_eq!(harness["db"], "/shared/oc.db"); + assert_eq!(harness["db_origin"], "registered"); + + for name in ["save-chat", "ingest", "context"] { + let path = home.join(format!(".openclaude/commands/engram-{name}.md")); + assert!( + path.exists(), + "{name} not written to the fork's command dir" + ); + let text = std::fs::read_to_string(&path).expect("read"); + assert!(text.starts_with("---\n"), "frontmatter must open the file"); + assert!(text.contains("--db /shared/oc.db")); + } +} diff --git a/tests/fixtures/transcripts/openclaude/session-basic.jsonl b/tests/fixtures/transcripts/openclaude/session-basic.jsonl new file mode 100644 index 0000000..dea2925 --- /dev/null +++ b/tests/fixtures/transcripts/openclaude/session-basic.jsonl @@ -0,0 +1,6 @@ +{"type":"mode","mode":"normal","sessionId":"11111111-2222-3333-4444-555555555555"} +{"parentUuid":null,"isSidechain":false,"type":"user","uuid":"aaaaaaa1-0000-4000-8000-000000000001","timestamp":"2026-08-20T10:00:00.000Z","message":{"role":"user","content":"Why is the pin stale?"}} +{"parentUuid":"aaaaaaa1-0000-4000-8000-000000000001","isSidechain":false,"type":"assistant","uuid":"aaaaaaa1-0000-4000-8000-000000000002","timestamp":"2026-08-20T10:00:05.000Z","message":{"role":"assistant","content":[{"type":"text","text":"Because the registration moved after install ran."}]}} +{"type":"file-history-snapshot","uuid":"aaaaaaa1-0000-4000-8000-000000000003","timestamp":"2026-08-20T10:00:06.000Z"} +{"type":"last-prompt","prompt":"Why is the pin stale?"} +{"type":"system","subtype":"local_command","content":"/status","uuid":"aaaaaaa1-0000-4000-8000-000000000004","timestamp":"2026-08-20T10:00:07.000Z"}