From d12228011877fbd802129b59fa70a1fefa04fa7c Mon Sep 17 00:00:00 2001 From: Tony Ketcham Date: Sat, 15 Aug 2026 23:45:02 +0000 Subject: [PATCH] fix(proof): expose read completeness Add `complete` and `cap_reasons` to every Proof read envelope. Derive the JSON fields, digest header, and summary from the same state so pagination and hard caps cannot disagree. Keep pagination out of `cap_reasons`; `page.has_more` and its cursor describe it. Report primary record, displayed edge, and byte caps as stable, duplicate-free values. Tests: - `pnpm verify` Fixes #252 Change-Id: I4cbd9019dd5ef042482e5e5824211dc95d1e1592 --- .agents/skills/proof/reference.md | 8 +++ CHANGELOG.md | 3 + packages/flatbread/src/cli/proof.test.ts | 66 +++++++++++++++++++++ packages/proof/skills/proof/reference.md | 8 +++ packages/proof/src/__tests__/digest.test.ts | 58 ++++++++++++++++++ packages/proof/src/digest.ts | 46 ++++++++++---- 6 files changed, 176 insertions(+), 13 deletions(-) diff --git a/.agents/skills/proof/reference.md b/.agents/skills/proof/reference.md index f1842a44..5e4cfbea 100644 --- a/.agents/skills/proof/reference.md +++ b/.agents/skills/proof/reference.md @@ -99,6 +99,8 @@ the generated schema) and return a `ReadEnvelope`: "artifact_sha256": "...", "served_generation": "55", "consistency": { "mode": "eventual|strict", "min_generation": null }, + "complete": true, + "cap_reasons": [], "page": { "returned": 2, "has_more": false, "next_cursor": null }, "hints": ["getRecord(\"dec-...\")"] } @@ -121,6 +123,12 @@ than expecting more. If a `get` body alone exceeds the 64 KiB digest byte cap, the digest fails closed with a byte-cap banner (it does **not** fake a full body via the 600/12 excerpt). +Every read envelope carries `complete` and `cap_reasons`. A page with more +records has `complete: false`, an empty `cap_reasons`, and +`page.has_more: true`. Hard caps use the stable reasons `primary_records`, +`displayed_edges`, and `bytes`. Programs must read these fields from the JSON +envelope; do not parse the digest or `summary` as a data feed. + ### Commands ```bash diff --git a/CHANGELOG.md b/CHANGELOG.md index 705d05ae..6f8acc8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,9 @@ ## Unreleased - The DAG runner is now `@flatbread/oven` (`pnpm exec oven`); the memory package is now `@flatbread/proof` with the `flatbread proof` CLI. +- Proof read envelopes now expose `complete` and `cap_reasons`. Callers can + tell paging from the `primary_records`, `displayed_edges`, and `bytes` caps + without parsing the digest Markdown or `summary` text. - `@flatbread/source-filesystem` reads a content directory that does not exist as an empty collection instead of throwing `ENOENT`. Git cannot store an empty directory, and a Proof write creates only the directory it writes, so diff --git a/packages/flatbread/src/cli/proof.test.ts b/packages/flatbread/src/cli/proof.test.ts index e7795792..1405718d 100644 --- a/packages/flatbread/src/cli/proof.test.ts +++ b/packages/flatbread/src/cli/proof.test.ts @@ -332,6 +332,72 @@ export default { } ); +test.serial( + 'spawned CLI exposes complete, paged, and byte-capped reads', + async (t) => { + const cwd = await createTempProject('flatbread-effort-completeness-', t); + await writeFile( + join(cwd, 'flatbread.config.js'), + `import { source } from '@flatbread/source-filesystem'; +import { transformer } from '@flatbread/transformer-markdown'; +import { proofContent } from '@flatbread/proof'; +export default { + source: source(), + transformer: transformer(), + content: proofContent('.flatbread-proof'), +};` + ); + const firstEffort = await handleEffortWrite( + JSON.stringify({ type: 'CreateEffort', title: 'First', body: '' }), + { cwd } + ); + await handleEffortWrite( + JSON.stringify({ type: 'CreateEffort', title: 'Second', body: '' }), + { cwd } + ); + const blob = await handleEffortWrite( + JSON.stringify({ + type: 'WriteBlob', + effort: firstEffort.artifacts[0].id, + title: 'Large payload', + body: 'x'.repeat(70 * 1024), + kind: 'markdown', + }), + { cwd } + ); + + const completeResult = await runCli( + cwd, + 'proof', + 'get', + firstEffort.artifacts[0].id + ); + t.is(completeResult.code, 0); + const complete = JSON.parse(completeResult.stdout); + t.true(complete.complete); + t.deepEqual(complete.cap_reasons, []); + + const pagedResult = await runCli(cwd, 'proof', 'list', '--limit', '1'); + t.is(pagedResult.code, 0); + const paged = JSON.parse(pagedResult.stdout); + t.false(paged.complete); + t.deepEqual(paged.cap_reasons, []); + t.true(paged.page.has_more); + + const cappedResult = await runCli( + cwd, + 'proof', + 'get', + blob.artifacts[0].id + ); + t.is(cappedResult.code, 0); + const capped = JSON.parse(cappedResult.stdout); + t.false(capped.complete); + t.deepEqual(capped.cap_reasons, ['bytes']); + t.false(capped.page.has_more); + } +); + test.serial( 'effort list defaults to active and supports explicit statuses and cursors', async (t) => { diff --git a/packages/proof/skills/proof/reference.md b/packages/proof/skills/proof/reference.md index f1842a44..5e4cfbea 100644 --- a/packages/proof/skills/proof/reference.md +++ b/packages/proof/skills/proof/reference.md @@ -99,6 +99,8 @@ the generated schema) and return a `ReadEnvelope`: "artifact_sha256": "...", "served_generation": "55", "consistency": { "mode": "eventual|strict", "min_generation": null }, + "complete": true, + "cap_reasons": [], "page": { "returned": 2, "has_more": false, "next_cursor": null }, "hints": ["getRecord(\"dec-...\")"] } @@ -121,6 +123,12 @@ than expecting more. If a `get` body alone exceeds the 64 KiB digest byte cap, the digest fails closed with a byte-cap banner (it does **not** fake a full body via the 600/12 excerpt). +Every read envelope carries `complete` and `cap_reasons`. A page with more +records has `complete: false`, an empty `cap_reasons`, and +`page.has_more: true`. Hard caps use the stable reasons `primary_records`, +`displayed_edges`, and `bytes`. Programs must read these fields from the JSON +envelope; do not parse the digest or `summary` as a data feed. + ### Commands ```bash diff --git a/packages/proof/src/__tests__/digest.test.ts b/packages/proof/src/__tests__/digest.test.ts index c84ae46e..ec31cfbe 100644 --- a/packages/proof/src/__tests__/digest.test.ts +++ b/packages/proof/src/__tests__/digest.test.ts @@ -40,6 +40,8 @@ test('renderDigest is deterministic and reuses the atomic cache artifact', async const bytes = await readFile(first.artifact_path); const second = await renderDigest(input); t.deepEqual(first, second); + t.true(first.complete); + t.deepEqual(first.cap_reasons, []); t.is(await stat(first.artifact_path).then((x) => x.isFile()), true); const digest = bytes.toString(); t.true(digest.includes(longBody)); @@ -99,6 +101,8 @@ test('renderDigest fullBody byte-cap miss does not fake-full with excerpt', asyn edges: [], }); const digest = await readFile(result.artifact_path, 'utf8'); + t.false(result.complete); + t.deepEqual(result.cap_reasons, ['bytes']); t.true(digest.includes('complete: false')); t.true(digest.includes('cap_reasons')); t.true(digest.includes('body exceeded digest byte cap')); @@ -162,6 +166,8 @@ test('pagination is incomplete without adding a cap reason', async (t) => { nextCursor: 'next', }); const digest = await readFile(result.artifact_path, 'utf8'); + t.false(result.complete); + t.deepEqual(result.cap_reasons, []); t.true(digest.includes('complete: false')); t.true(digest.includes('"total_known":2')); t.false(digest.includes('cap_reasons')); @@ -170,6 +176,58 @@ test('pagination is incomplete without adding a cap reason', async (t) => { t.is(result.page.next_cursor, 'next'); }); +test('primary-record caps are machine readable', async (t) => { + const cacheRoot = await mkdtemp(join(tmpdir(), 'eg-digest-record-cap-')); + const result = await renderDigest({ + query: { type: 'listRecords', effort: 'eff-one--0123456789abcdef' }, + queryHash: 'record-cap', + generation: '4', + consistency: { mode: 'eventual' as const, min_generation: null }, + cacheRoot, + records: Array.from({ length: 26 }, (_, index) => ({ + id: `fnd-record-${index}--0123456789abcdef`, + kind: 'finding' as const, + path: `findings/record-${index}.md`, + frontmatter: { title: `Record ${index}` }, + body_excerpt: '', + relations: {}, + })), + edges: [], + }); + t.false(result.complete); + t.deepEqual(result.cap_reasons, ['primary_records']); + t.true(result.page.has_more); +}); + +test('displayed-edge caps are machine readable', async (t) => { + const cacheRoot = await mkdtemp(join(tmpdir(), 'eg-digest-edge-cap-')); + const result = await renderDigest({ + query: { type: 'relations', effort_id: 'eff-one--0123456789abcdef' }, + queryHash: 'edge-cap', + generation: '4', + consistency: { mode: 'eventual' as const, min_generation: null }, + cacheRoot, + records: [ + { + id: 'dec-one--0123456789abcdef', + kind: 'decision' as const, + path: 'decisions/one.md', + frontmatter: { title: 'One' }, + body_excerpt: '', + relations: {}, + }, + ], + edges: Array.from({ length: 51 }, (_, index) => ({ + from_id: 'dec-one--0123456789abcdef', + relation: 'derives_from' as const, + to_id: `fnd-edge-${index}--0123456789abcdef`, + })), + }); + t.false(result.complete); + t.deepEqual(result.cap_reasons, ['displayed_edges']); + t.is(new Set(result.cap_reasons).size, result.cap_reasons.length); +}); + test('renderDigest omits Blob bodies from bounded digests', async (t) => { const cacheRoot = await mkdtemp(join(tmpdir(), 'eg-digest-blob-')); const secret = 'SECRET_BLOB_PAYLOAD_SHOULD_NOT_APPEAR'; diff --git a/packages/proof/src/digest.ts b/packages/proof/src/digest.ts index 5495ed29..de371c15 100644 --- a/packages/proof/src/digest.ts +++ b/packages/proof/src/digest.ts @@ -30,12 +30,16 @@ export interface ReadEdge { to_id: string; } +export type ReadCapReason = 'primary_records' | 'displayed_edges' | 'bytes'; + export interface ReadEnvelope { summary: string; artifact_path: string; artifact_sha256: string; served_generation: string; consistency: { mode: 'eventual' | 'strict'; min_generation: string | null }; + complete: boolean; + cap_reasons: ReadCapReason[]; page: { returned: number; has_more: boolean; next_cursor: string | null }; hints: string[]; } @@ -93,7 +97,20 @@ function scalar(value: unknown): string { return JSON.stringify(value); } -function yamlHeader(input: DigestInput, complete: boolean, reasons: string[]) { +interface DigestCompleteness { + complete: boolean; + capReasons: ReadCapReason[]; +} + +function digestCompleteness( + hasMore: boolean, + reasons: readonly ReadCapReason[] +): DigestCompleteness { + const capReasons = [...new Set(reasons)].sort(); + return { complete: !hasMore && capReasons.length === 0, capReasons }; +} + +function yamlHeader(input: DigestInput, state: DigestCompleteness) { const query = JSON.stringify(input.query); return [ '---', @@ -107,15 +124,15 @@ function yamlHeader(input: DigestInput, complete: boolean, reasons: string[]) { total_known: input.totalKnown ?? input.records.length, has_more: Boolean(input.hasMore), })}`, - `complete: ${complete}`, + `complete: ${state.complete}`, `caps: ${JSON.stringify({ primary_records: CAP_RECORDS, relation_hops: 1, displayed_edges: CAP_EDGES, bytes: CAP_BYTES, })}`, - ...(reasons.length - ? [`cap_reasons: ${JSON.stringify(reasons.sort())}`] + ...(state.capReasons.length + ? [`cap_reasons: ${JSON.stringify(state.capReasons)}`] : []), '---', ].join('\n'); @@ -200,7 +217,7 @@ function renderRecord( function summary( records: readonly ReadRecord[], complete: boolean, - reasons: string[], + reasons: readonly string[], hasMore: boolean ): string { const states = new Map(); @@ -255,10 +272,10 @@ export async function renderDigest(input: DigestInput): Promise { ) ) .slice(0, CAP_EDGES); - const reasons = [ - ...(records.length > CAP_RECORDS ? ['primary_records'] : []), - ...(input.edges.length > CAP_EDGES ? ['displayed_edges'] : []), - ]; + const reasons: ReadCapReason[] = []; + if (records.length > CAP_RECORDS) reasons.push('primary_records'); + if (input.edges.length > CAP_EDGES) reasons.push('displayed_edges'); + let completeness = digestCompleteness(Boolean(input.hasMore), reasons); const checkpoints = input.checkpointLines?.length ? ['## Lineage checkpoints', ...input.checkpointLines, ''] : []; @@ -270,7 +287,7 @@ export async function renderDigest(input: DigestInput): Promise { const renderRelated = (record: ReadRecord) => renderRecord(record, { bodyMode: 'excerpt' }); let markdown = [ - yamlHeader(input, reasons.length === 0 && !input.hasMore, reasons), + yamlHeader(input, completeness), ...(input.anomaly ? [`> anomaly: ${input.anomaly}`, ''] : []), '# Proof read', '## Index', @@ -291,6 +308,7 @@ export async function renderDigest(input: DigestInput): Promise { ].join('\n'); if (Buffer.byteLength(markdown) > CAP_BYTES) { reasons.push('bytes'); + completeness = digestCompleteness(Boolean(input.hasMore), reasons); // Full-body digests must not silently fall back to the 600/12 excerpt. // Prefer a visible byte-cap miss banner over a fake "full" body. const overflowBodyMode: RecordBodyMode = input.fullBody @@ -303,7 +321,7 @@ export async function renderDigest(input: DigestInput): Promise { ? 'body exceeded digest byte cap' : input.anomaly; const header = [ - yamlHeader(input, false, reasons), + yamlHeader(input, completeness), ...(anomaly ? [`> anomaly: ${anomaly}`, ''] : []), '# Proof read', '## Index', @@ -363,14 +381,16 @@ export async function renderDigest(input: DigestInput): Promise { const envelope: ReadEnvelope = { summary: summary( visible, - reasons.length === 0 && !input.hasMore, - reasons, + completeness.complete, + completeness.capReasons, Boolean(input.hasMore) ), artifact_path: path, artifact_sha256: createHash('sha256').update(bytes).digest('hex'), served_generation: input.generation, consistency: input.consistency, + complete: completeness.complete, + cap_reasons: completeness.capReasons, page: { returned: visible.length, has_more: Boolean(input.hasMore || records.length > CAP_RECORDS),