fix(proof): preserve sparse reads and relation integrity

.agents/skills/proof/reference.md

@@ -142,7 +142,11 @@ flatbread proof cache prune
   `--since`/`--until` bound `created_at` (gte/lte, ISO strings).
 - `--relations` values: `derives_from`, `supersedes`, `superseded_by`,
   `invalidates`, `invalidated_by`, `rejected_by`, `mitigated_by`,
-  `resolved_by`, `evidence`, `cites` (one hop, explicit only).
+  `resolved_by`, `evidence`, `cites` (one hop, explicit only). Missing targets
+  never produce a partial success. For `derives_from`, `invalidates`,
+  `invalidated_by`, `resolved_by`, and `evidence`, the CLI returns
+  `PROOF_DANGLING_RELATION`. Flatbread's core reference check rejects missing
+  targets for the other relations.
 - `--resolve head`: follow `superseded_by` to the current tip; ancestors
   render as checkpoint lines (max 5, then a count).
 - `blocking-decisions` membership (frozen): Decision in the effort with
@@ -157,7 +161,8 @@ flatbread proof cache prune
   generation, or fail. `--timeout-ms <ms>` bounds the wait (default 3000).
 - Errors (stderr JSON, exit 1): `PROOF_GENERATION_WAIT_TIMEOUT`,
   `PROOF_INVALID_CURSOR` (cursor reused across a different query or
-  generation).
+  generation), and `PROOF_DANGLING_RELATION` (a stored relation target is
+  missing).
 
 ## Configuration surface
 

CHANGELOG.md

@@ -3,6 +3,26 @@
 ## Unreleased
 
 - The DAG runner is now `@flatbread/oven` (`pnpm exec oven`); the memory package is now `@flatbread/proof` with the `flatbread proof` CLI.
+- `@flatbread/source-filesystem` reads a content directory that does not exist
+  as an empty collection instead of throwing `ENOENT`. Git cannot store an
+  empty directory, and a Proof write creates only the directory it writes, so
+  a sparse graph was unreadable until every collection directory existed.
+  Permission and other I/O faults still fail the read.
+  A failing read now also reports through the normal error path: `fetch` no
+  longer builds its results in an `async` promise executor, which used to
+  leave the caller waiting forever while the failure escaped as an unhandled
+  rejection and a raw stack trace.
+- **Breaking for writes:** a Proof create mutation now rejects a
+  `derives_from` id that no record answers to, with
+  `Unknown artifact <id>`. This matches the documented contract for forward
+  edges and the existing behavior of `cites`, `supersedes`, and `invalidates`.
+  Create the target record first, then link to it.
+- `flatbread proof relations` fails with `PROOF_DANGLING_RELATION` (stderr
+  JSON, exit 1) when a record stores a `derives_from` id that no record answers
+  to. It used to drop the edge and report `page.returned: 0`, which presented
+  incomplete provenance as complete. The error names the record, the relation,
+  and the missing id. Records written before this release keep any dangling
+  edge until you repair the file.
 
 Notes for the Flatbread release train. Some packages also keep their own
 changelog; this file covers the repository as a whole.

packages/flatbread/src/cli/proof.test.ts

@@ -15,7 +15,11 @@ import { join, relative } from 'node:path';
 import { tmpdir } from 'node:os';
 import { proofContent } from '@flatbread/proof';
 import { proofContent as publicProofContent } from '../index.js';
-import { ProofValidationError } from '@flatbread/proof';
+import {
+  ProofDanglingRelationError,
+  ProofValidationError,
+  serializeDocument,
+} from '@flatbread/proof';
 import {
   handleEffortBlockingDecisions,
   handleEffortBootstrap,
@@ -653,6 +657,177 @@ export default {
   }
 );
 
+test.serial(
+  'a strict read of a sparse Proof succeeds without pre-created collection directories',
+  async (t) => {
+    const cwd = await createTempProject('flatbread-effort-sparse-', t);
+    await writeFile(
+      join(cwd, 'flatbread.config.js'),
+      `import { source } from '@flatbread/source-filesystem';
+import { transformer } from '@flatbread/transformer-markdown';
+import { proofContent } from '@flatbread/proof';
+export default {
+  source: source(),
+  transformer: transformer(),
+  content: proofContent('.flatbread-proof'),
+};`
+    );
+    const effort = await handleEffortWrite(
+      JSON.stringify({
+        type: 'CreateEffort',
+        title: 'Critical path',
+        body: 'Exercise storage and consistency.',
+      }),
+      { cwd }
+    );
+    const effortId = effort.artifacts[0].id;
+    const issue = await handleEffortWrite(
+      JSON.stringify({
+        type: 'WriteIssue',
+        effort: effortId,
+        title: 'Need a decision',
+        body: 'A blocker for the strict-read path.',
+        kind: 'blocker',
+      }),
+      { cwd }
+    );
+
+    // Writes create only the directories they touch, and Git cannot store an
+    // empty directory, so a fresh clone always reads a sparse graph.
+    t.is(
+      await lstat(join(cwd, '.flatbread-proof', 'findings')).catch(() => null),
+      null
+    );
+
+    const result = await runCli(
+      cwd,
+      'proof',
+      'records',
+      effortId,
+      '--kinds',
+      'issue',
+      '--strict-min-generation',
+      issue.generation,
+      '--timeout-ms',
+      '3000'
+    );
+    t.is(result.code, 0);
+    t.is(result.stderr, '');
+    t.is(JSON.parse(result.stdout).page.returned, 1);
+  }
+);
+
+test.serial(
+  'a dangling derives_from is rejected on write and reported on read',
+  async (t) => {
+    const cwd = await createTempProject('flatbread-effort-dangling-', t);
+    // Create every collection directory so this case turns only on relation
+    // integrity, not on how a sparse graph reads.
+    for (const directory of [
+      'efforts',
+      'issues',
+      'findings',
+      'decisions',
+      'constraints',
+      'risks',
+      'citations',
+      'blobs',
+    ])
+      await mkdir(join(cwd, '.flatbread-proof', directory), {
+        recursive: true,
+      });
+    await writeFile(
+      join(cwd, 'flatbread.config.js'),
+      `import { source } from '@flatbread/source-filesystem';
+import { transformer } from '@flatbread/transformer-markdown';
+import { proofContent } from '@flatbread/proof';
+export default {
+  source: source(),
+  transformer: transformer(),
+  content: proofContent('.flatbread-proof'),
+};`
+    );
+    const effort = await handleEffortWrite(
+      JSON.stringify({ type: 'CreateEffort', title: 'Provenance', body: '' }),
+      { cwd }
+    );
+    const effortId = effort.artifacts[0].id;
+    const missingId = 'fnd-does-not-exist--0000000000000000';
+
+    await t.throwsAsync(
+      () =>
+        handleEffortWrite(
+          JSON.stringify({
+            type: 'WriteDecision',
+            effort: effortId,
+            title: 'Dangling derives-from',
+            body: 'This must have failed closed.',
+            derives_from: [missingId],
+          }),
+          { cwd }
+        ),
+      {
+        instanceOf: ProofValidationError,
+        message: new RegExp(`Unknown artifact ${missingId}`),
+      }
+    );
+    // The rejected write leaves the durable generation where the Effort left it.
+    t.is(
+      JSON.parse(
+        await readFile(
+          join(cwd, '.flatbread-proof', '.journal', 'generation.json'),
+          'utf8'
+        )
+      ).generation,
+      Number(effort.generation)
+    );
+
+    // Legacy or hand-edited data can still hold a dangling edge. Recall must say
+    // so rather than return provenance that dropped the edge in silence.
+    const decisionId = 'dec-legacy--0000000000000000';
+    await writeFile(
+      join(cwd, '.flatbread-proof', 'decisions', `${decisionId}.md`),
+      serializeDocument('Written before its evidence existed.', {
+        id: decisionId,
+        effort: effortId,
+        title: 'Legacy decision',
+        created_at: '2025-01-01T00:00:00.000Z',
+        state: 'proposed',
+        derives_from: [missingId],
+      })
+    );
+    const error = await t.throwsAsync<ProofDanglingRelationError>(
+      () =>
+        handleEffortRelations(effortId, decisionId, {
+          cwd,
+          relations: ['derives_from'],
+        }),
+      { instanceOf: ProofDanglingRelationError }
+    );
+    t.deepEqual(error?.shape, {
+      error: {
+        code: 'PROOF_DANGLING_RELATION',
+        message: `Record ${decisionId} stores relation targets that do not exist: derives_from -> ${missingId}`,
+        from_id: decisionId,
+        edges: [{ relation: 'derives_from', to_id: missingId }],
+      },
+    });
+    const result = await runCli(
+      cwd,
+      'proof',
+      'relations',
+      effortId,
+      decisionId,
+      '--relations',
+      'derives_from'
+    );
+    t.is(result.code, 1);
+    t.is(result.stdout, '');
+    t.deepEqual(JSON.parse(result.stderr), error?.shape);
+    t.false(result.stderr.includes(' at '));
+  }
+);
+
 test.serial(
   'bootstrap detects a ready custom root and verify returns action-required JSON state',
   async (t) => {

packages/flatbread/src/proof/read.ts

@@ -2,6 +2,7 @@ import { FlatbreadProvider, type LoadedFlatbreadConfig } from '@flatbread/core';
 import {
   canonicalizeReadQuery,
   ProofConsistencyError,
+  ProofDanglingRelationError,
   ProofInvalidCursorError,
   ProofReadValidationError,
   READ_RELATIONS,
@@ -663,17 +664,26 @@ export async function relations(
   if (!source || !sourceInEffort)
     throw new Error(`Record ${fromId} does not exist in effort ${effortId}`);
   const selected = new Map<string, ReadRecord>();
+  const dangling: { relation: ReadRelation; to_id: string }[] = [];
   for (const relation of relationNames) {
     for (const targetId of source.relations[relation] ?? []) {
       const targetCollection = collectionForId(targetId);
-      if (!targetCollection) continue;
-      const target = await projection.one(targetCollection, targetId);
-      // Only return targets that belong to this Effort. The writer should
-      // prevent foreign links, but this keeps hand-authored files contained.
-      if (target?.frontmatter.effort === effortId)
+      const target = targetCollection
+        ? await projection.one(targetCollection, targetId)
+        : undefined;
+      // A stored id that no record answers to means the graph is corrupt. Report
+      // it instead of returning provenance that dropped the edge in silence.
+      if (!target) {
+        dangling.push({ relation, to_id: targetId });
+        continue;
+      }
+      // Keep the existing effort-scoped read behavior for legacy or
+      // hand-authored cross-effort links.
+      if (target.frontmatter.effort === effortId)
         selected.set(target.id, target);
     }
   }
+  if (dangling.length) throw new ProofDanglingRelationError(fromId, dangling);
   const records = sortRecords([...selected.values()]);
   const edges = records.flatMap((record) =>
     relationNames

packages/proof/skills/proof/reference.md

@@ -142,7 +142,11 @@ flatbread proof cache prune
   `--since`/`--until` bound `created_at` (gte/lte, ISO strings).
 - `--relations` values: `derives_from`, `supersedes`, `superseded_by`,
   `invalidates`, `invalidated_by`, `rejected_by`, `mitigated_by`,
-  `resolved_by`, `evidence`, `cites` (one hop, explicit only).
+  `resolved_by`, `evidence`, `cites` (one hop, explicit only). Missing targets
+  never produce a partial success. For `derives_from`, `invalidates`,
+  `invalidated_by`, `resolved_by`, and `evidence`, the CLI returns
+  `PROOF_DANGLING_RELATION`. Flatbread's core reference check rejects missing
+  targets for the other relations.
 - `--resolve head`: follow `superseded_by` to the current tip; ancestors
   render as checkpoint lines (max 5, then a count).
 - `blocking-decisions` membership (frozen): Decision in the effort with
@@ -157,7 +161,8 @@ flatbread proof cache prune
   generation, or fail. `--timeout-ms <ms>` bounds the wait (default 3000).
 - Errors (stderr JSON, exit 1): `PROOF_GENERATION_WAIT_TIMEOUT`,
   `PROOF_INVALID_CURSOR` (cursor reused across a different query or
-  generation).
+  generation), and `PROOF_DANGLING_RELATION` (a stored relation target is
+  missing).
 
 ## Configuration surface
 

packages/proof/src/__tests__/planner.test.ts

@@ -768,3 +768,54 @@ test('29 same-effort cites and blob happy path', (t) => {
     citId,
   ]);
 });
+test('derives_from must target an existing record', (t) => {
+  const missing = 'fnd-does-not-exist--0000000000000000';
+  t.throws(
+    () =>
+      planMutation(
+        {
+          type: 'WriteDecision',
+          id: ids.decision,
+          effort: E,
+          title: 'D',
+          body: '',
+          derives_from: [missing],
+        },
+        snap(),
+        '/root',
+        now
+      ),
+    { message: new RegExp(`Unknown artifact ${missing}`) }
+  );
+});
+test('derives_from accepts an existing record', (t) => {
+  const issue = record(ids.issue, 'issue', {
+    id: ids.issue,
+    effort: E,
+    title: 'I',
+    kind: 'blocker',
+    created_at: '2025-01-01T00:00:00.000Z',
+    status: 'open',
+  });
+  const w = planMutation(
+    {
+      type: 'WriteDecision',
+      id: ids.decision,
+      effort: E,
+      title: 'D',
+      body: '',
+      derives_from: [ids.issue],
+    },
+    snap([issue]),
+    '/root',
+    now
+  );
+  one(t, w, ids.decision, `decisions/${ids.decision}.md`, 'create', {
+    id: ids.decision,
+    effort: E,
+    title: 'D',
+    derives_from: [ids.issue],
+    created_at: now.toISOString(),
+    state: 'proposed',
+  });
+});

packages/proof/src/index.ts

@@ -16,6 +16,7 @@ export * from './digest.js';
 export {
   READ_RELATIONS,
   ProofConsistencyError,
+  ProofDanglingRelationError,
   ProofInvalidCursorError,
   ProofReadValidationError,
   canonicalizeReadQuery,