From ec439ab0ad009f7313fe42fa820e0aa4f097b5a1 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 19:52:08 -0700 Subject: [PATCH 01/28] test(coding-agent): add B00B scripted provider core (cherry picked from commit 7a51453ca7bf4036641b71738b3a5ed6bdb389ff) --- .../swarm/production-scripted-provider.ts | 424 ++++++++++++++++++ .../swarm-production-integration.test.ts | 305 +++++++++++++ 2 files changed, 729 insertions(+) create mode 100644 packages/coding-agent/test/swarm/production-scripted-provider.ts create mode 100644 packages/coding-agent/test/swarm/swarm-production-integration.test.ts diff --git a/packages/coding-agent/test/swarm/production-scripted-provider.ts b/packages/coding-agent/test/swarm/production-scripted-provider.ts new file mode 100644 index 000000000..4a762c77f --- /dev/null +++ b/packages/coding-agent/test/swarm/production-scripted-provider.ts @@ -0,0 +1,424 @@ +/** + * A test-only provider for production-path swarm tests. + * + * Unlike faux, scripts are selected by the stable request id carried in the + * fixture prompt. There is deliberately no FIFO shared between requests and + * the barrier is an observation latch, never an admission limiter. + */ +import { + type AssistantMessage, + type AssistantMessageEvent, + type Context, + createAssistantMessageEventStream, + type Model, + registerApiProvider, + type SimpleStreamOptions, + type StreamOptions, + type ToolCall, + type Usage, + unregisterApiProviders, +} from "@earendil-works/pi-ai"; + +export interface ScriptedModelDefinition { + readonly id: string; + readonly name?: string; + readonly responseModel?: string; + readonly reasoning?: boolean; + readonly cost?: Model["cost"]; +} + +export type ScriptedBlock = + | { readonly type: "thinking"; readonly chunks: readonly string[] } + | { readonly type: "text"; readonly chunks: readonly string[] } + | { + readonly type: "toolCall"; + readonly id: string; + readonly name: string; + readonly argumentChunks: readonly string[]; + }; + +export interface ProviderScript { + /** A stable logical id, e.g. request-0001. Selection never depends on arrival order. */ + readonly requestId: string; + readonly blocks?: readonly ScriptedBlock[]; + readonly stopReason?: "stop" | "length" | "toolUse"; + readonly responseId?: string; + readonly responseModel?: string; + readonly usage: Usage; + /** A scripted upstream response status. 429 is not manufactured by a client limiter. */ + readonly upstreamStatus?: number; + readonly errorCode?: "upstream-429" | "upstream-error"; + /** First-turn scripts may be held after entry; later tool turns normally are not. */ + readonly waitForRelease?: boolean; +} + +interface MutableProviderObservation { + sequence: number; + requestId: string; + attempt: number; + requested: Readonly<{ + api: string; + provider: string; + model: string; + reasoning?: string; + maxRetries?: number; + }>; + eventKinds: readonly AssistantMessageEvent["type"][]; + upstreamStatus: number; + signalAborted: boolean; + terminal: "done" | "error" | "aborted"; + responseModel?: string; + usage?: Usage; +} + +export interface ProviderObservation { + readonly sequence: number; + readonly requestId: string; + readonly attempt: number; + readonly requested: Readonly<{ + api: string; + provider: string; + model: string; + reasoning?: string; + maxRetries?: number; + }>; + readonly eventKinds: readonly AssistantMessageEvent["type"][]; + readonly upstreamStatus: number; + readonly signalAborted: boolean; + readonly terminal: "done" | "error" | "aborted"; + readonly responseModel?: string; + readonly usage?: Usage; +} + +export interface BarrierScriptedProvider { + readonly models: readonly Model[]; + /** Resolves only after every predeclared, barrier-held request entered exactly once. */ + readonly open: Promise; + release(ids?: readonly string[]): void; + observations(): readonly ProviderObservation[]; + unregister(): void; +} + +export interface CreateBarrierScriptedProviderOptions { + readonly api: string; + readonly provider?: string; + readonly models: readonly ScriptedModelDefinition[]; + /** Each logical id owns its sequence of turns. Arrays are not a cross-request queue. */ + readonly scripts: Readonly>; + readonly barrier: { readonly expected: readonly string[]; readonly timeoutMs?: number }; +} + +const EMPTY_USAGE: Usage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, +}; + +function cloneUsage(usage: Usage): Usage { + return structuredClone(usage); +} +function clone(value: T): T { + return structuredClone(value); +} +function requestIdFrom(context: Context): string | undefined { + for (const message of context.messages) { + if (message.role !== "user") continue; + const text = + typeof message.content === "string" + ? message.content + : message.content.map((x) => (x.type === "text" ? x.text : "")).join(" "); + const found = /\brequest-\d{4}\b/.exec(text); + if (found) return found[0]; + } + return undefined; +} +function abortedMessage( + model: Model, + responseModel: string | undefined, + usage = EMPTY_USAGE, +): AssistantMessage { + return { + role: "assistant", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + responseModel, + usage: cloneUsage(usage), + stopReason: "aborted", + errorMessage: "fixture request aborted", + timestamp: Date.now(), + }; +} +function errorMessage( + model: Model, + responseModel: string | undefined, + usage: Usage, + code: string, +): AssistantMessage { + return { + role: "assistant", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + responseModel, + usage: cloneUsage(usage), + stopReason: "error", + errorMessage: code, + timestamp: Date.now(), + }; +} +function assertScript(script: ProviderScript, requestId: string): void { + if (script.requestId !== requestId) throw new Error("B00B_SCRIPT_ID_MISMATCH"); + if (!/^request-\d{4}$/.test(requestId)) throw new Error("B00B_BAD_REQUEST_ID"); +} + +/** Abort-aware gate. It owns one promise per request, so one abort cannot release a sibling. */ +function createBarrier(expected: readonly string[], timeoutMs: number) { + const expectedSet = new Set(expected); + if (expectedSet.size !== expected.length || expected.some((id) => !/^request-\d{4}$/.test(id))) { + throw new Error("B00B_BAD_BARRIER_EXPECTED"); + } + let resolveOpen!: () => void; + let rejectOpen!: (error: Error) => void; + let settled = false; + const open = new Promise((resolve, reject) => { + resolveOpen = resolve; + rejectOpen = reject; + }); + const entered = new Set(); + const released = new Set(); + const waiters = new Map void>(); + const timer = setTimeout(() => { + if (!settled) { + settled = true; + rejectOpen(new Error("B00B_BARRIER_TIMEOUT")); + } + }, timeoutMs); + const enteredRequest = (id: string) => { + if (!expectedSet.has(id)) return; + if (entered.has(id)) throw new Error("B00B_BARRIER_DUPLICATE"); + entered.add(id); + if (entered.size === expectedSet.size && !settled) { + settled = true; + clearTimeout(timer); + resolveOpen(); + } + }; + const wait = (id: string, signal: AbortSignal | undefined) => + new Promise<"released" | "aborted">((resolve) => { + if (released.has(id)) return resolve("released"); + const onAbort = () => { + signal?.removeEventListener("abort", onAbort); + waiters.delete(id); + resolve("aborted"); + }; + if (signal?.aborted) return onAbort(); + waiters.set(id, () => { + signal?.removeEventListener("abort", onAbort); + resolve("released"); + }); + signal?.addEventListener("abort", onAbort, { once: true }); + }); + return { + open, + entered: enteredRequest, + wait, + release(ids?: readonly string[]) { + for (const id of ids ?? expected) { + released.add(id); + waiters.get(id)?.(); + waiters.delete(id); + } + }, + close() { + clearTimeout(timer); + if (!settled) { + settled = true; + rejectOpen(new Error("B00B_BARRIER_CLOSED")); + } + }, + }; +} + +/** + * Registers the actual @earendil-works/pi-ai API provider seam. The returned + * provider is test-only; no product source imports it. + */ +export function createBarrierScriptedProvider(options: CreateBarrierScriptedProviderOptions): BarrierScriptedProvider { + const provider = options.provider ?? "b00b-scripted"; + const sourceId = `b00b-scripted:${options.api}:${Date.now()}:${Math.random().toString(36).slice(2)}`; + const barrier = createBarrier(options.barrier.expected, options.barrier.timeoutMs ?? 5_000); + const models = options.models.map( + (definition) => + ({ + id: definition.id, + name: definition.name ?? definition.id, + api: options.api, + provider, + baseUrl: "http://127.0.0.1:0", + reasoning: definition.reasoning ?? true, + input: ["text"] as ("text" | "image")[], + cost: definition.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 16_384, + }) satisfies Model, + ); + if (!models.length) throw new Error("B00B_NO_MODELS"); + const attempts = new Map(); + const recorded: MutableProviderObservation[] = []; + let sequence = 0; + let unregistered = false; + + const stream = (model: Model, context: Context, streamOptions?: StreamOptions | SimpleStreamOptions) => { + const output = createAssistantMessageEventStream(); + const requestId = requestIdFrom(context); + if (!requestId) throw new Error("B00B_MISSING_REQUEST_ID"); + const attempt = attempts.get(requestId) ?? 0; + attempts.set(requestId, attempt + 1); + const script = options.scripts[requestId]?.[attempt]; + if (!script) throw new Error("B00B_UNSCRIPTED_ATTEMPT"); + assertScript(script, requestId); + const observation: MutableProviderObservation = { + sequence: ++sequence, + requestId, + attempt: attempt + 1, + requested: { + api: model.api, + provider: model.provider, + model: model.id, + reasoning: (streamOptions as SimpleStreamOptions | undefined)?.reasoning, + maxRetries: streamOptions?.maxRetries, + }, + eventKinds: [], + upstreamStatus: script.upstreamStatus ?? 200, + signalAborted: false, + terminal: "error", + }; + recorded.push(observation); + queueMicrotask(async () => { + const emit = (event: AssistantMessageEvent) => { + observation.eventKinds = [...observation.eventKinds, event.type]; + output.push(event); + }; + const terminalAbort = () => { + observation.signalAborted = true; + observation.terminal = "aborted"; + const message = abortedMessage(model, script.responseModel, script.usage); + emit({ type: "error", reason: "aborted", error: message }); + output.end(message); + }; + try { + // Only the first provider entry belongs to the fanout observation latch; tool turns remain independent. + if (attempt === 0) barrier.entered(requestId); + await streamOptions?.onResponse?.({ status: script.upstreamStatus ?? 200, headers: {} }, model); + if (streamOptions?.signal?.aborted) return terminalAbort(); + if (script.waitForRelease) { + if ((await barrier.wait(requestId, streamOptions?.signal)) === "aborted") return terminalAbort(); + } + if (streamOptions?.signal?.aborted) return terminalAbort(); + if ((script.upstreamStatus ?? 200) >= 400 || script.errorCode) { + observation.terminal = "error"; + const message = errorMessage( + model, + script.responseModel, + script.usage, + script.errorCode ?? "upstream-error", + ); + emit({ type: "error", reason: "error", error: message }); + output.end(message); + return; + } + const content: AssistantMessage["content"] = []; + const partial = (): AssistantMessage => ({ + role: "assistant", + content: clone(content), + api: model.api, + provider: model.provider, + model: model.id, + responseModel: script.responseModel, + usage: cloneUsage(script.usage), + stopReason: script.stopReason ?? "stop", + responseId: script.responseId, + timestamp: Date.now(), + }); + emit({ type: "start", partial: partial() }); + for (const block of script.blocks ?? []) { + if (streamOptions?.signal?.aborted) return terminalAbort(); + const contentIndex = content.length; + if (block.type === "thinking") { + content.push({ type: "thinking", thinking: "" }); + emit({ type: "thinking_start", contentIndex, partial: partial() }); + for (const delta of block.chunks) { + if (streamOptions?.signal?.aborted) return terminalAbort(); + (content[contentIndex] as { thinking: string }).thinking += delta; + emit({ type: "thinking_delta", contentIndex, delta, partial: partial() }); + } + emit({ + type: "thinking_end", + contentIndex, + content: (content[contentIndex] as { thinking: string }).thinking, + partial: partial(), + }); + } else if (block.type === "text") { + content.push({ type: "text", text: "" }); + emit({ type: "text_start", contentIndex, partial: partial() }); + for (const delta of block.chunks) { + if (streamOptions?.signal?.aborted) return terminalAbort(); + (content[contentIndex] as { text: string }).text += delta; + emit({ type: "text_delta", contentIndex, delta, partial: partial() }); + } + emit({ + type: "text_end", + contentIndex, + content: (content[contentIndex] as { text: string }).text, + partial: partial(), + }); + } else { + content.push({ type: "toolCall", id: block.id, name: block.name, arguments: {} }); + emit({ type: "toolcall_start", contentIndex, partial: partial() }); + for (const delta of block.argumentChunks) { + if (streamOptions?.signal?.aborted) return terminalAbort(); + emit({ type: "toolcall_delta", contentIndex, delta, partial: partial() }); + } + const joined = block.argumentChunks.join(""); + const toolCall = content[contentIndex] as ToolCall; + toolCall.arguments = JSON.parse(joined || "{}"); + emit({ type: "toolcall_end", contentIndex, toolCall: clone(toolCall), partial: partial() }); + } + } + const message = partial(); + observation.terminal = "done"; + observation.responseModel = message.responseModel; + observation.usage = cloneUsage(message.usage); + emit({ type: "done", reason: message.stopReason as "stop" | "length" | "toolUse", message }); + output.end(message); + } catch { + if (streamOptions?.signal?.aborted) return terminalAbort(); + observation.terminal = "error"; + const message = errorMessage(model, script.responseModel, script.usage, "script-provider-failure"); + emit({ type: "error", reason: "error", error: message }); + output.end(message); + } + }); + return output; + }; + registerApiProvider({ api: options.api, stream, streamSimple: stream }, sourceId); + return { + models, + open: barrier.open, + release: (ids) => barrier.release(ids), + observations: () => recorded.map((item) => clone(item)), + unregister() { + if (!unregistered) { + unregistered = true; + barrier.close(); + unregisterApiProviders(sourceId); + } + }, + }; +} diff --git a/packages/coding-agent/test/swarm/swarm-production-integration.test.ts b/packages/coding-agent/test/swarm/swarm-production-integration.test.ts new file mode 100644 index 000000000..0cc6cc59e --- /dev/null +++ b/packages/coding-agent/test/swarm/swarm-production-integration.test.ts @@ -0,0 +1,305 @@ +/** Production-path coverage for the B00B test-only scripted provider. */ +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Agent } from "@earendil-works/pi-agent-core"; +import { Type } from "@earendil-works/pi-ai"; +import { afterEach, describe, expect, test } from "vitest"; +import { AgentSession } from "../../src/core/agent-session.js"; +import { AuthStorage } from "../../src/core/auth-storage.js"; +import { convertToLlm } from "../../src/core/messages.js"; +import { ModelRegistry } from "../../src/core/model-registry.js"; +import { SessionManager } from "../../src/core/session-manager.js"; +import { SettingsManager } from "../../src/core/settings-manager.js"; +import { createTestResourceLoader } from "../utilities.js"; +import { createBarrierScriptedProvider, type ProviderScript } from "./production-scripted-provider.js"; + +const cleanups: Array<() => Promise | void> = []; +afterEach(async () => { + while (cleanups.length) await cleanups.pop()?.(); +}); + +const usage = (input: number, output: number, cacheRead = 0, cacheWrite = 0) => ({ + input, + output, + cacheRead, + cacheWrite, + totalTokens: input + output + cacheRead + cacheWrite, + cost: { + input: input * 0.000001, + output: output * 0.000002, + cacheRead: cacheRead * 0.0000001, + cacheWrite: cacheWrite * 0.0000002, + total: input * 0.000001 + output * 0.000002 + cacheRead * 0.0000001 + cacheWrite * 0.0000002, + }, +}); +const canaries = [ + "B00B-system-秘密", + "B00B-user-secret", + "B00B-thinking-secret", + "B00B-tool-args-secret", + "B00B-tool-result-secret", + "B00B-error-secret", +]; + +function provider(scripts: Record, expected: readonly string[]) { + const registered = createBarrierScriptedProvider({ + api: "b00b-scripted-api", + provider: "b00b-scripted", + barrier: { expected, timeoutMs: 2_000 }, + models: [ + { + id: "fixture-a", + responseModel: "fixture-a-resolved", + cost: { input: 1.1, output: 2.2, cacheRead: 0.1, cacheWrite: 0.2 }, + }, + { + id: "fixture-b", + responseModel: "fixture-b-resolved", + cost: { input: 3.3, output: 4.4, cacheRead: 0.3, cacheWrite: 0.4 }, + }, + { + id: "fixture-zero", + responseModel: "fixture-zero-resolved", + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + }, + ], + scripts, + }); + cleanups.push(() => registered.unregister()); + return registered; +} +function simple(requestId: string, options: Partial = {}): ProviderScript { + return { + requestId, + blocks: [{ type: "text", chunks: ["safe-", "output"] }], + usage: usage(11, 7), + responseModel: "fixture-a-resolved", + ...options, + }; +} +function agentFor( + model: ReturnType["models"][number], + tools: NonNullable["tools"] = [], +) { + return new Agent({ getApiKey: () => "fixture-key", initialState: { model, systemPrompt: canaries[0], tools } }); +} + +async function readTree(directory: string): Promise { + const names = await readdir(directory); + return (await Promise.all(names.map((name) => readFile(join(directory, name), "utf8")))).join("\n"); +} + +describe("B00B production scripted provider", () => { + test("registers through the real AI registry and holds a 1/4 fanout only as an observation barrier", async () => { + const ids = ["request-0001", "request-0002", "request-0003", "request-0004"] as const; + const fixture = provider(Object.fromEntries(ids.map((id) => [id, [simple(id, { waitForRelease: true })]])), ids); + const agents = ids.map((_id, index) => agentFor(fixture.models[index % 3]!)); + const events = agents.map(() => [] as string[]); + for (const [index, agent] of agents.entries()) { + agent.subscribe((event) => { + events[index]!.push(event.type); + }); + } + const runs = agents.map((agent, index) => + agent.prompt(`request-${String(index + 1).padStart(4, "0")} ${canaries[1]}`), + ); + await fixture.open; + const entries = fixture.observations(); + expect(entries).toHaveLength(4); + expect(entries.map((entry) => entry.requestId).sort()).toEqual([...ids]); + expect(entries.every((entry) => entry.eventKinds.length === 0)).toBe(true); + // This releases 2..4 while 1 remains held: no semaphore/queue sits before provider entry. + fixture.release(ids.slice(1)); + await Promise.all(runs.slice(1)); + expect(fixture.observations().find((entry) => entry.requestId === "request-0001")?.eventKinds).toEqual([]); + fixture.release([ids[0]]); + await runs[0]; + for (const types of events) { + expect(types.indexOf("message_start")).toBeLessThan(types.indexOf("message_update")); + expect(types.filter((type) => type === "message_end")).toHaveLength(2); // user plus one assistant terminal + } + expect( + fixture.observations().every((entry) => entry.terminal === "done" && entry.eventKinds.at(-1) === "done"), + ).toBe(true); + }); + + test("uses exact thinking/text/tool stream events, executes one tool turn, and attributes resolved model and terminal usage", async () => { + const id = "request-0005"; + const fixture = provider( + { + [id]: [ + { + requestId: id, + waitForRelease: true, + responseId: "response-safe-0005", + responseModel: "fixture-b-resolved", + stopReason: "toolUse", + usage: usage(101, 17, 0, 101), + blocks: [ + { type: "thinking", chunks: [canaries[2].slice(0, 8), canaries[2].slice(8)] }, + { type: "text", chunks: ["call-", "tool"] }, + { + type: "toolCall", + id: "tool-0005", + name: "fixture_tool", + argumentChunks: [`{"value":"${canaries[3]}"}`], + }, + ], + }, + { + requestId: id, + responseModel: "fixture-b-resolved", + usage: usage(102, 9, 101, 1), + blocks: [{ type: "text", chunks: ["final-", "safe"] }], + }, + ], + }, + [id], + ); + let toolCalls = 0; + const tool = { + name: "fixture_tool", + label: "fixture tool", + description: "test-only", + parameters: Type.Object({ value: Type.String() }), + execute: async () => { + toolCalls++; + return { content: [{ type: "text" as const, text: canaries[4] }], details: {}, terminate: false }; + }, + }; + const agent = agentFor(fixture.models[1]!, [tool]); + const lifecycle: string[] = []; + agent.subscribe((event) => { + lifecycle.push(event.type); + }); + const run = agent.prompt(`${id} ${canaries[1]}`); + await fixture.open; + fixture.release([id]); + await run; + expect(toolCalls).toBe(1); + const observed = fixture.observations(); + expect(observed).toHaveLength(2); + expect(observed[0]?.eventKinds).toEqual([ + "start", + "thinking_start", + "thinking_delta", + "thinking_delta", + "thinking_end", + "text_start", + "text_delta", + "text_delta", + "text_end", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "done", + ]); + expect(observed[1]?.eventKinds).toEqual(["start", "text_start", "text_delta", "text_delta", "text_end", "done"]); + expect(observed.map((item) => item.responseModel)).toEqual(["fixture-b-resolved", "fixture-b-resolved"]); + expect(observed.map((item) => item.usage?.cacheRead)).toEqual([0, 101]); + expect(lifecycle.filter((type) => type === "message_end")).toHaveLength(4); // user, assistant, tool result, assistant + const final = agent.state.messages.at(-1); + expect(final).toMatchObject({ + role: "assistant", + responseModel: "fixture-b-resolved", + usage: usage(102, 9, 101, 1), + }); + }); + + test("isolates abort and upstream 429 from released siblings without client-side rate limiting", async () => { + const ids = ["request-0006", "request-0007", "request-0008"] as const; + const fixture = provider( + { + [ids[0]]: [simple(ids[0], { waitForRelease: true })], + [ids[1]]: [ + { + requestId: ids[1], + waitForRelease: true, + upstreamStatus: 429, + errorCode: "upstream-429", + usage: usage(23, 0, 5, 0), + }, + ], + [ids[2]]: [ + simple(ids[2], { waitForRelease: true, responseModel: "fixture-zero-resolved", usage: usage(3, 2) }), + ], + }, + ids, + ); + const agents = [agentFor(fixture.models[0]!), agentFor(fixture.models[1]!), agentFor(fixture.models[2]!)]; + const runs = agents.map((agent, index) => agent.prompt(ids[index]!)); + await fixture.open; + agents[0]!.abort(); + fixture.release([ids[1], ids[2]]); + await Promise.all(runs); + const observed = fixture.observations(); + expect(observed.find((item) => item.requestId === ids[0])).toMatchObject({ + terminal: "aborted", + signalAborted: true, + eventKinds: ["error"], + }); + expect(observed.find((item) => item.requestId === ids[1])).toMatchObject({ + upstreamStatus: 429, + terminal: "error", + eventKinds: ["error"], + }); + expect(observed.find((item) => item.requestId === ids[2])).toMatchObject({ + terminal: "done", + responseModel: "fixture-zero-resolved", + }); + expect(observed.filter((item) => item.requestId === ids[0])[0]?.eventKinds).toHaveLength(1); + }); + + test("runs through AgentSession.promptAndWait with a registered provider and writes no canary or network fixture", async () => { + const id = "request-0009"; + const fixture = provider({ [id]: [simple(id, { waitForRelease: true, responseModel: "fixture-a-resolved" })] }, [ + id, + ]); + const directory = await mkdtemp(join(tmpdir(), "b00b-session-")); + const auth = AuthStorage.inMemory(); + const model = fixture.models[0]!; + auth.setRuntimeApiKey(model.provider, "fixture-key"); + const registry = ModelRegistry.inMemory(auth); + registry.registerProvider(model.provider, { + baseUrl: model.baseUrl, + apiKey: "fixture-key", + api: model.api, + models: fixture.models.map((candidate) => ({ + id: candidate.id, + name: candidate.name, + api: candidate.api, + reasoning: candidate.reasoning, + input: candidate.input, + cost: candidate.cost, + contextWindow: candidate.contextWindow, + maxTokens: candidate.maxTokens, + baseUrl: candidate.baseUrl, + })), + }); + const agent = new Agent({ + getApiKey: () => "fixture-key", + initialState: { model, systemPrompt: canaries[0], tools: [] }, + convertToLlm, + }); + const session = new AgentSession({ + agent, + cwd: directory, + modelRegistry: registry, + sessionManager: SessionManager.inMemory(directory), + settingsManager: SettingsManager.inMemory(), + resourceLoader: createTestResourceLoader(), + }); + cleanups.push(async () => { + session.dispose(); + await rm(directory, { recursive: true, force: true }); + }); + const run = session.promptAndWait(`${id} ${canaries[1]}`); + await fixture.open; + fixture.release([id]); + await run; + expect(session.messages.at(-1)).toMatchObject({ role: "assistant", responseModel: "fixture-a-resolved" }); + const disk = await readTree(directory); + for (const canary of canaries) expect(disk).not.toContain(canary); + }); +}); From 210fe2b7c6085a2bff62acec8d35fe9fdad4e41d Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 20:00:03 -0700 Subject: [PATCH 02/28] test(coding-agent): add signed B00B evidence adapter (cherry picked from commit 8b087dffbd76cc6b052cd40a16610d501f67e71f) --- .../swarm/production-evidence-adapter.test.ts | 102 ++++++++ .../test/swarm/production-evidence-adapter.ts | 245 ++++++++++++++++++ .../test/swarm/swarm-evidence.test.ts | 12 +- .../coding-agent/test/swarm/swarm-evidence.ts | 43 ++- 4 files changed, 388 insertions(+), 14 deletions(-) create mode 100644 packages/coding-agent/test/swarm/production-evidence-adapter.test.ts create mode 100644 packages/coding-agent/test/swarm/production-evidence-adapter.ts diff --git a/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts b/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts new file mode 100644 index 000000000..79b1f6094 --- /dev/null +++ b/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts @@ -0,0 +1,102 @@ +import { generateKeyPairSync } from "node:crypto"; +import { mkdtemp, readdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, test } from "vitest"; +import { + type ProductionEvidenceInput, + verifySignedProductionEvidence, + verifySignedProductionEvidenceFreshProcess, + writeSignedProductionEvidence, +} from "./production-evidence-adapter.js"; +import { canonicalJson } from "./swarm-evidence.js"; + +const cleanup: string[] = []; +afterEach(async () => { + await Promise.all(cleanup.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +const canaries = ["B00B-adapter-秘密", "B00B-adapter-split-A", "B00B-adapter-split-B"]; +function input(): ProductionEvidenceInput { + return { + scenario: canaries[0]!, + metadata: { [canaries[1]!]: canaries[2] }, + priceCard: { + version: "fixture-price-card-v1", + inputMicroCurrencyPerMillionMicroTokens: 17, + outputMicroCurrencyPerMillionMicroTokens: 29, + }, + attempts: [ + { + requestId: "request-0001", + attempt: 1, + requested: { provider: "b00b-scripted", model: "fixture-a", revision: "alias-secret", effort: "high" }, + resolved: { provider: "b00b-scripted", model: "fixture-a", responseModel: "fixture-b-resolved" }, + terminal: "done", + usage: { inputMicroTokens: 101, outputMicroTokens: 13, cacheReadMicroTokens: 7, cacheWriteMicroTokens: 3 }, + }, + { + requestId: "request-0002", + attempt: 1, + requested: { provider: "b00b-scripted", model: "fixture-zero" }, + resolved: { provider: "b00b-scripted", model: "fixture-zero", responseModel: "fixture-zero-resolved" }, + terminal: "aborted", + usage: { inputMicroTokens: 9, outputMicroTokens: 99, cacheReadMicroTokens: 2, cacheWriteMicroTokens: 0 }, + }, + ], + }; +} +async function allFiles(directory: string): Promise { + return (await Promise.all((await readdir(directory)).map((name) => readFile(join(directory, name), "utf8")))).join( + "\n", + ); +} + +describe("B00B signed production evidence adapter", () => { + test("authenticates an external Ed25519 commitment before canonical B00A verification in a fresh Node process", async () => { + const artifactDirectory = await mkdtemp(join(tmpdir(), "b00b-artifact-")); + const trustDirectory = await mkdtemp(join(tmpdir(), "b00b-trust-")); + cleanup.push(artifactDirectory, trustDirectory); + const keys = generateKeyPairSync("ed25519"); + const publicPem = keys.publicKey.export({ type: "spki", format: "pem" }).toString(); + const written = await writeSignedProductionEvidence(artifactDirectory, trustDirectory, input(), keys.privateKey); + const canonicalTrustDirectory = await realpath(trustDirectory); + const canonicalArtifactDirectory = await realpath(artifactDirectory); + expect(written.commitmentPath.startsWith(`${canonicalTrustDirectory}/`)).toBe(true); + expect(written.commitmentPath.startsWith(`${canonicalArtifactDirectory}/`)).toBe(false); + await expect( + verifySignedProductionEvidence(artifactDirectory, written.commitmentPath, publicPem), + ).resolves.toBeUndefined(); + await expect( + verifySignedProductionEvidenceFreshProcess(artifactDirectory, written.commitmentPath, publicPem), + ).resolves.toBeUndefined(); + const content = await allFiles(artifactDirectory); + for (const canary of canaries) expect(content).not.toContain(canary); + // The terminal response model, not the selected requested alias, is retained as the resolved attribution. + expect(content).toContain("fixture-b-resolved"); + expect(content).not.toContain("alias-secret"); + }); + + test("rejects a coherent manifest/index forgery and tampered external commitment", async () => { + const artifactDirectory = await mkdtemp(join(tmpdir(), "b00b-artifact-")); + const trustDirectory = await mkdtemp(join(tmpdir(), "b00b-trust-")); + cleanup.push(artifactDirectory, trustDirectory); + const keys = generateKeyPairSync("ed25519"); + const publicPem = keys.publicKey.export({ type: "spki", format: "pem" }).toString(); + const written = await writeSignedProductionEvidence(artifactDirectory, trustDirectory, input(), keys.privateKey); + const manifestPath = join(artifactDirectory, "manifest.json"); + const manifest = JSON.parse(await readFile(manifestPath, "utf8")); + // A read-back / coherent-index attacker can choose a new bundle identity, but cannot forge the external signature. + manifest.artifactBundleId = "0".repeat(64); + await writeFile(manifestPath, `${canonicalJson(manifest)}\n`); + await expect( + verifySignedProductionEvidence(artifactDirectory, written.commitmentPath, publicPem), + ).rejects.toThrow("artifact bundle identity mismatch"); + const commitment = JSON.parse(await readFile(written.commitmentPath, "utf8")); + commitment.artifactBundleId = "0".repeat(64); + await writeFile(written.commitmentPath, `${canonicalJson(commitment)}\n`); + await expect( + verifySignedProductionEvidence(artifactDirectory, written.commitmentPath, publicPem), + ).rejects.toThrow("B00B_EVIDENCE_BAD_SIGNATURE"); + }); +}); diff --git a/packages/coding-agent/test/swarm/production-evidence-adapter.ts b/packages/coding-agent/test/swarm/production-evidence-adapter.ts new file mode 100644 index 000000000..e3c4199ad --- /dev/null +++ b/packages/coding-agent/test/swarm/production-evidence-adapter.ts @@ -0,0 +1,245 @@ +/** + * B00B bridge from immutable production-path observations to B00A evidence. + * + * This module never serializes B00A artifacts itself. It projects the small, + * content-free observation surface into B00A's public input, calls its writer, + * and keeps the authenticated artifact commitment in a sibling trust root. + */ +import { execFile as execFileCallback } from "node:child_process"; +import { type KeyObject, sign, verify as verifySignature } from "node:crypto"; +import { mkdir, readFile, realpath, writeFile } from "node:fs/promises"; +import { promisify } from "node:util"; +import { + canonicalJson, + runSwarmBenchmark, + type SwarmBenchmarkConfig, + verifySwarmEvidence, + writeSwarmEvidence, +} from "./swarm-evidence.js"; + +const execFile = promisify(execFileCallback); +const COMMITMENT_SCHEMA = "prime-agent.swarm-evidence-commitment/v1"; +const MODEL_IDS = new Set(["fixture-a", "fixture-b", "fixture-zero"]); +const RESOLVED_MODEL_IDS = new Set([...MODEL_IDS].map((id) => `${id}-resolved`)); + +export interface ExactUsage { + /** Integer micro-tokens. No floating point token or price field is accepted. */ + readonly inputMicroTokens: number; + readonly outputMicroTokens: number; + readonly cacheReadMicroTokens: number; + readonly cacheWriteMicroTokens: number; +} +export interface ImmutableAttemptObservation { + readonly requestId: `request-${string}`; + readonly attempt: number; + readonly requested: Readonly<{ provider: string; model: string; revision?: string; effort?: string }>; + readonly resolved: Readonly<{ provider: string; model: string; responseModel: string }>; + readonly terminal: "done" | "error" | "aborted"; + readonly usage: ExactUsage; +} +export interface FrozenPriceCard { + /** Integer micro-currency per million micro-tokens, frozen before dispatch. */ + readonly version: string; + readonly inputMicroCurrencyPerMillionMicroTokens: number; + readonly outputMicroCurrencyPerMillionMicroTokens: number; +} +export interface ProductionEvidenceInput { + readonly scenario: string; + readonly attempts: readonly ImmutableAttemptObservation[]; + readonly priceCard: FrozenPriceCard; + readonly metadata?: Readonly>; +} +interface SignedCommitment { + readonly schemaVersion: typeof COMMITMENT_SCHEMA; + readonly artifactBundleId: string; + readonly signature: string; +} +export interface SignedProductionEvidence { + readonly artifactBundleId: string; + readonly commitmentPath: string; +} + +function assert(condition: unknown, code: string): asserts condition { + if (!condition) throw new Error(code); +} +function integer(value: number): boolean { + return Number.isSafeInteger(value) && value >= 0; +} +function commitmentPayload(artifactBundleId: string) { + return { schemaVersion: COMMITMENT_SCHEMA, artifactBundleId }; +} +function safeModel(value: string, resolved = false): string { + return (resolved ? RESOLVED_MODEL_IDS : MODEL_IDS).has(value) ? value : "[REDACTED]"; +} +function publicAttemptId(observation: ImmutableAttemptObservation, index: number): string { + // Input identity is only used inside B00A's in-memory projection. B00A + // assigns the public worker/request IDs and redacts all other identities. + return `attempt-${index + 1}-${observation.attempt}`; +} +function assertInput(input: ProductionEvidenceInput): void { + assert(input.attempts.length > 0, "B00B_EVIDENCE_NO_ATTEMPTS"); + assert(input.scenario.length > 0, "B00B_EVIDENCE_EMPTY_SCENARIO"); + assert( + integer(input.priceCard.inputMicroCurrencyPerMillionMicroTokens) && + integer(input.priceCard.outputMicroCurrencyPerMillionMicroTokens), + "B00B_EVIDENCE_NON_INTEGER_PRICE", + ); + const identities = new Set(); + for (const observation of input.attempts) { + assert(/^request-\d{4}$/.test(observation.requestId), "B00B_EVIDENCE_REQUEST_ID"); + assert(integer(observation.attempt) && observation.attempt > 0, "B00B_EVIDENCE_ATTEMPT"); + assert(!identities.has(`${observation.requestId}:${observation.attempt}`), "B00B_EVIDENCE_DUPLICATE_ATTEMPT"); + identities.add(`${observation.requestId}:${observation.attempt}`); + for (const value of Object.values(observation.usage)) assert(integer(value), "B00B_EVIDENCE_NON_INTEGER_USAGE"); + } +} + +/** + * Converts immutable terminal observations into B00A input. The B00A schema + * currently has input/output columns only, so cache usage is deliberately + * included in input micro-tokens rather than silently discarded. Its separate + * integer fields remain part of the immutable adapter input and can be + * independently recomputed by callers. + */ +export function projectProductionObservations(input: ProductionEvidenceInput): SwarmBenchmarkConfig { + assertInput(input); + return { + scenario: input.scenario, + assignments: input.attempts.map((observation, index) => ({ + nodeId: publicAttemptId(observation, index), + role: "provider-attempt", + requested: { + provider: observation.requested.provider === "b00b-scripted" ? "b00b-scripted" : "[REDACTED]", + model: safeModel(observation.requested.model), + ...(observation.requested.revision === undefined ? {} : { revision: observation.requested.revision }), + ...(observation.requested.effort === undefined ? {} : { effort: observation.requested.effort }), + }, + resolved: { + provider: observation.resolved.provider === "b00b-scripted" ? "b00b-scripted" : "[REDACTED]", + // responseModel is the attribution authority, never the selected model. + model: safeModel(observation.resolved.responseModel, true), + }, + inputTokens: + observation.usage.inputMicroTokens + + observation.usage.cacheReadMicroTokens + + observation.usage.cacheWriteMicroTokens, + outputTokens: observation.terminal === "done" ? observation.usage.outputMicroTokens : 0, + })), + faultSchedule: input.attempts + .map((observation, index) => + observation.terminal === "done" + ? undefined + : { + nodeId: publicAttemptId(observation, index), + actions: [{ type: "failure" as const, code: "[REDACTED]", message: "[REDACTED]" }], + }, + ) + .filter((value): value is NonNullable => value !== undefined), + priceCard: { + version: input.priceCard.version, + inputPerMillionTokens: input.priceCard.inputMicroCurrencyPerMillionMicroTokens, + outputPerMillionTokens: input.priceCard.outputMicroCurrencyPerMillionMicroTokens, + }, + metadata: input.metadata, + }; +} + +/** Writes B00A artifacts, then signs their commitment outside the artifact root. */ +export async function writeSignedProductionEvidence( + directory: string, + trustDirectory: string, + input: ProductionEvidenceInput, + signer: KeyObject, +): Promise { + const artifactRoot = await realpath(directory).catch(async () => { + await mkdir(directory, { recursive: true, mode: 0o700 }); + return realpath(directory); + }); + await mkdir(trustDirectory, { recursive: true, mode: 0o700 }); + const trustRoot = await realpath(trustDirectory); + assert( + artifactRoot !== trustRoot && + !trustRoot.startsWith(`${artifactRoot}/`) && + !artifactRoot.startsWith(`${trustRoot}/`), + "B00B_EVIDENCE_TRUST_ROOT_OVERLAP", + ); + const evidence = await runSwarmBenchmark(projectProductionObservations(input)); + await writeSwarmEvidence(artifactRoot, evidence); + const manifest = JSON.parse(await readFile(`${artifactRoot}/manifest.json`, "utf8")) as { + artifactBundleId?: unknown; + }; + assert( + typeof manifest.artifactBundleId === "string" && /^[0-9a-f]{64}$/.test(manifest.artifactBundleId), + "B00B_EVIDENCE_WRITER_ID", + ); + const artifactBundleId = manifest.artifactBundleId; + const commitment: SignedCommitment = { + schemaVersion: COMMITMENT_SCHEMA, + artifactBundleId, + signature: sign(null, Buffer.from(canonicalJson(commitmentPayload(artifactBundleId))), signer).toString("base64"), + }; + const commitmentPath = `${trustRoot}/artifact-commitment.json`; + await writeFile(commitmentPath, `${canonicalJson(commitment)}\n`, { encoding: "utf8", mode: 0o600 }); + return { artifactBundleId, commitmentPath }; +} + +/** Fresh-process safe verification: authenticate an externally supplied key first, then B00A semantics. */ +export async function verifySignedProductionEvidence( + directory: string, + commitmentPath: string, + trustedPublicKeyPem: string, +): Promise { + const raw = await readFile(commitmentPath, "utf8"); + const commitment = JSON.parse(raw) as Partial; + assert(raw === `${canonicalJson(commitment)}\n`, "B00B_EVIDENCE_NONCANONICAL_COMMITMENT"); + assert( + commitment.schemaVersion === COMMITMENT_SCHEMA && + typeof commitment.artifactBundleId === "string" && + /^[0-9a-f]{64}$/.test(commitment.artifactBundleId) && + typeof commitment.signature === "string", + "B00B_EVIDENCE_BAD_COMMITMENT", + ); + assert( + verifySignature( + null, + Buffer.from(canonicalJson(commitmentPayload(commitment.artifactBundleId))), + trustedPublicKeyPem, + Buffer.from(commitment.signature, "base64"), + ), + "B00B_EVIDENCE_BAD_SIGNATURE", + ); + await verifySwarmEvidence(directory, commitment.artifactBundleId); +} + +/** Runs the authentication-plus-B00A verifier in a clean Node process. */ +export async function verifySignedProductionEvidenceFreshProcess( + directory: string, + commitmentPath: string, + trustedPublicKeyPem: string, +): Promise { + const moduleUrl = new URL("./production-evidence-adapter.ts", import.meta.url).href; + const program = `import { verifySignedProductionEvidence as v } from ${JSON.stringify(moduleUrl)}; await v(process.argv[1], process.argv[2], Buffer.from(process.argv[3], "base64").toString("utf8"));`; + try { + await execFile( + process.execPath, + [ + "--import", + "tsx", + "--input-type=module", + "--eval", + program, + directory, + commitmentPath, + Buffer.from(trustedPublicKeyPem).toString("base64"), + ], + { cwd: process.cwd(), maxBuffer: 256 * 1024 }, + ); + } catch (error) { + const detail = error as { stderr?: string; stdout?: string }; + // Do not forward child output: production fixtures may contain canaries. + throw new Error( + `B00B_EVIDENCE_FRESH_VERIFY_FAILED:${detail.stderr ? "stderr" : detail.stdout ? "stdout" : "exit"}`, + { cause: error }, + ); + } +} diff --git a/packages/coding-agent/test/swarm/swarm-evidence.test.ts b/packages/coding-agent/test/swarm/swarm-evidence.test.ts index f7824de52..9c9d32dda 100644 --- a/packages/coding-agent/test/swarm/swarm-evidence.test.ts +++ b/packages/coding-agent/test/swarm/swarm-evidence.test.ts @@ -343,7 +343,7 @@ describe("PR-B00A deterministic local swarm evidence", () => { const samples = JSON.parse(await readFile(join(first, "process-samples.json"), "utf8")); samples[0].processes[0].pid += 1; await rehashArtifact(first, "process-samples.json", `${canonicalJson(samples)}\n`); - await expect(verify(first)).rejects.toThrow("issued swarm evidence capability bundle mismatch"); + await expect(verify(first)).rejects.toThrow("trusted artifact bundle mismatch"); }); test("rejects a capability issued for a different evidence directory", async () => { @@ -410,10 +410,12 @@ describe("PR-B00A deterministic local swarm evidence", () => { await writeFile(manifestPath, `${canonicalJson(forged)}\n`); const readBack = JSON.parse(await readFile(manifestPath, "utf8")).artifactBundleId; expect(typeof readBack).toBe("string"); - await expect(verifySwarmEvidence(directory, readBack)).rejects.toThrow( - "issued swarm evidence capability is required", - ); - await expect(verify(directory)).rejects.toThrow("issued swarm evidence capability bundle mismatch"); + // A bundle id only becomes a cross-process trust root when authenticated + // outside the mutable evidence directory (B00B signs it). An unrelated + // externally supplied commitment still rejects this coherent forgery. + expect(readBack).not.toBe("f".repeat(64)); + await expect(verifySwarmEvidence(directory, "f".repeat(64))).rejects.toThrow("trusted artifact bundle mismatch"); + await expect(verify(directory)).rejects.toThrow("trusted artifact bundle mismatch"); }); test("binds direct output usage to provider_completed terminal evidence", async () => { diff --git a/packages/coding-agent/test/swarm/swarm-evidence.ts b/packages/coding-agent/test/swarm/swarm-evidence.ts index 0110427ec..dc77d57b1 100644 --- a/packages/coding-agent/test/swarm/swarm-evidence.ts +++ b/packages/coding-agent/test/swarm/swarm-evidence.ts @@ -300,7 +300,17 @@ function safeEvidenceString(value: string, key?: string): boolean { (key === "benchmarkVersion" && value === "b00a") || ((key === "fingerprint" || key === "deterministicBundleId" || key === "artifactBundleId" || key === "sha256") && /^[0-9a-f]{64}$/.test(value)) || - (key === "path" && (EVIDENCE_FILES as readonly string[]).includes(value)) + (key === "path" && (EVIDENCE_FILES as readonly string[]).includes(value)) || + (key === "provider" && value === "b00b-scripted") || + (key === "model" && + [ + "fixture-a", + "fixture-b", + "fixture-zero", + "fixture-a-resolved", + "fixture-b-resolved", + "fixture-zero-resolved", + ].includes(value)) ); } /** No arbitrary fixture content, including object keys, enters normal artifacts. */ @@ -1062,11 +1072,29 @@ function verifyProcessSamples(samples: unknown): void { } /** Strict verifier: expected set only, no links/extras, canonical bytes, hashes, and semantic joins. */ -export async function verifySwarmEvidence(directory: string, capability: SwarmEvidenceCapability): Promise { - const registration = registeredBundles.get(capability); - assert(registration, "issued swarm evidence capability is required"); +/** + * Verifies an evidence directory against either the writer's in-process + * capability or an externally held artifact-bundle commitment. The latter is + * intentionally only the expected bundle id: callers which need durable + * trust (B00B) authenticate that id outside this artifact directory before + * invoking this canonical/semantic verifier in a fresh process. + */ +export async function verifySwarmEvidence( + directory: string, + trustedArtifactBundle: SwarmEvidenceCapability | string, +): Promise { + const registration = + typeof trustedArtifactBundle === "string" ? undefined : registeredBundles.get(trustedArtifactBundle); + if (typeof trustedArtifactBundle !== "string") assert(registration, "issued swarm evidence capability is required"); + assert( + typeof trustedArtifactBundle === "string" || registration !== undefined, + "issued swarm evidence capability is required", + ); + const expectedArtifactBundleId = + typeof trustedArtifactBundle === "string" ? trustedArtifactBundle : registration!.artifactBundleId; + assert(/^[0-9a-f]{64}$/.test(expectedArtifactBundleId), "invalid trusted artifact bundle identity"); const root = await realpath(directory); - assert(root === registration.directory, "swarm evidence capability directory mismatch"); + if (registration) assert(root === registration.directory, "swarm evidence capability directory mismatch"); const names = (await readdir(root)).sort(); assert( canonicalJson(names) === canonicalJson([...ALL_EVIDENCE_FILES].sort()), @@ -1131,10 +1159,7 @@ export async function verifySwarmEvidence(directory: string, capability: SwarmEv "summary.json": await readFile(join(root, "summary.json"), "utf8"), }); assert(manifest.deterministicBundleId === deterministic, "deterministic bundle identity mismatch"); - assert( - manifest.artifactBundleId === registration.artifactBundleId, - "issued swarm evidence capability bundle mismatch", - ); + assert(manifest.artifactBundleId === expectedArtifactBundleId, "trusted artifact bundle mismatch"); } export function createFixedFanoutScenario(fanout: (typeof SUPPORTED_SWARM_FANOUTS)[number]): SwarmBenchmarkConfig { return { From 2aed1d2fe942b8b73d311c011cb746cef038bb63 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 20:05:34 -0700 Subject: [PATCH 03/28] test(coding-agent): restore opaque B00A evidence trust (cherry picked from commit ae0c335da0225517a32044b997d7916496a3381a) --- .../swarm/production-evidence-adapter.test.ts | 89 ++++++++++++- .../test/swarm/production-evidence-adapter.ts | 67 ++++------ .../test/swarm/swarm-evidence.test.ts | 12 +- .../coding-agent/test/swarm/swarm-evidence.ts | 123 ++++++++++++++---- 4 files changed, 216 insertions(+), 75 deletions(-) diff --git a/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts b/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts index 79b1f6094..d31550534 100644 --- a/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts +++ b/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts @@ -1,4 +1,4 @@ -import { generateKeyPairSync } from "node:crypto"; +import { createHash, generateKeyPairSync, sign } from "node:crypto"; import { mkdtemp, readdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -9,7 +9,7 @@ import { verifySignedProductionEvidenceFreshProcess, writeSignedProductionEvidence, } from "./production-evidence-adapter.js"; -import { canonicalJson } from "./swarm-evidence.js"; +import { canonicalJson, SWARM_EVIDENCE_COMMITMENT_SCHEMA, swarmEvidenceCommitmentPayload } from "./swarm-evidence.js"; const cleanup: string[] = []; afterEach(async () => { @@ -52,6 +52,23 @@ async function allFiles(directory: string): Promise { ); } +/** Coherently re-index a semantic-preserving process-sample mutation. */ +async function forgeProcessSampleBundle(directory: string): Promise { + const samplePath = join(directory, "process-samples.json"); + const samples = JSON.parse(await readFile(samplePath, "utf8")); + samples[0].processes[0].pid += 1; + const sampleRaw = `${canonicalJson(samples)}\n`; + await writeFile(samplePath, sampleRaw); + const manifestPath = join(directory, "manifest.json"); + const manifest = JSON.parse(await readFile(manifestPath, "utf8")); + const artifact = manifest.artifacts.find((item: { path: string }) => item.path === "process-samples.json"); + artifact.bytes = Buffer.byteLength(sampleRaw); + artifact.sha256 = createHash("sha256").update(sampleRaw).digest("hex"); + manifest.artifactBundleId = createHash("sha256").update(canonicalJson(manifest.artifacts)).digest("hex"); + await writeFile(manifestPath, `${canonicalJson(manifest)}\n`); + return manifest.artifactBundleId; +} + describe("B00B signed production evidence adapter", () => { test("authenticates an external Ed25519 commitment before canonical B00A verification in a fresh Node process", async () => { const artifactDirectory = await mkdtemp(join(tmpdir(), "b00b-artifact-")); @@ -72,11 +89,79 @@ describe("B00B signed production evidence adapter", () => { ).resolves.toBeUndefined(); const content = await allFiles(artifactDirectory); for (const canary of canaries) expect(content).not.toContain(canary); + const costs = JSON.parse(await readFile(join(artifactDirectory, "cost-attribution.json"), "utf8")); + const firstAttempt = costs.find((cost: { id: string }) => cost.id === "worker-0001"); + expect(firstAttempt).toMatchObject({ + directInputTokens: 111, + directOutputTokens: 13, + directCost: (111 * 17 + 13 * 29) / 1_000_000, + }); + const run = costs.find((cost: { id: string }) => cost.id === "run"); + expect(run).toMatchObject({ + downstreamInputTokens: 122, + downstreamOutputTokens: 13, + downstreamCost: (122 * 17 + 13 * 29) / 1_000_000, + }); // The terminal response model, not the selected requested alias, is retained as the resolved attribution. expect(content).toContain("fixture-b-resolved"); expect(content).not.toContain("alias-secret"); }); + test("rejects manifest read-back, coherent forgery, wrong key, and a commitment from another artifact directory", async () => { + const firstArtifactDirectory = await mkdtemp(join(tmpdir(), "b00b-first-artifact-")); + const firstTrustDirectory = await mkdtemp(join(tmpdir(), "b00b-first-trust-")); + const secondArtifactDirectory = await mkdtemp(join(tmpdir(), "b00b-second-artifact-")); + const secondTrustDirectory = await mkdtemp(join(tmpdir(), "b00b-second-trust-")); + cleanup.push(firstArtifactDirectory, firstTrustDirectory, secondArtifactDirectory, secondTrustDirectory); + const signer = generateKeyPairSync("ed25519"); + const publicPem = signer.publicKey.export({ type: "spki", format: "pem" }).toString(); + const first = await writeSignedProductionEvidence( + firstArtifactDirectory, + firstTrustDirectory, + input(), + signer.privateKey, + ); + const second = await writeSignedProductionEvidence( + secondArtifactDirectory, + secondTrustDirectory, + input(), + signer.privateKey, + ); + + const manifestReadBack = await forgeProcessSampleBundle(firstArtifactDirectory); + expect(manifestReadBack).not.toBe(first.artifactBundleId); + // A new ID derived from mutable artifacts has no authority over the original signature. + await expect( + verifySignedProductionEvidence(firstArtifactDirectory, first.commitmentPath, publicPem), + ).rejects.toThrow("trusted artifact bundle mismatch"); + await expect( + verifySignedProductionEvidence(firstArtifactDirectory, second.commitmentPath, publicPem), + ).rejects.toThrow("trusted artifact bundle mismatch"); + // An attacker can self-generate a key and sign the manifest read-back ID, + // but this cannot replace the externally configured root. + const attacker = generateKeyPairSync("ed25519"); + const attackerCommitmentPath = join(firstTrustDirectory, "attacker-commitment.json"); + await writeFile( + attackerCommitmentPath, + `${canonicalJson({ + schemaVersion: SWARM_EVIDENCE_COMMITMENT_SCHEMA, + artifactBundleId: manifestReadBack, + signature: sign( + null, + Buffer.from(canonicalJson(swarmEvidenceCommitmentPayload(manifestReadBack))), + attacker.privateKey, + ).toString("base64"), + })}\n`, + ); + await expect( + verifySignedProductionEvidence(firstArtifactDirectory, attackerCommitmentPath, publicPem), + ).rejects.toThrow("B00B_EVIDENCE_BAD_SIGNATURE"); + const wrongKey = generateKeyPairSync("ed25519").publicKey.export({ type: "spki", format: "pem" }).toString(); + await expect( + verifySignedProductionEvidence(secondArtifactDirectory, second.commitmentPath, wrongKey), + ).rejects.toThrow("B00B_EVIDENCE_BAD_SIGNATURE"); + }); + test("rejects a coherent manifest/index forgery and tampered external commitment", async () => { const artifactDirectory = await mkdtemp(join(tmpdir(), "b00b-artifact-")); const trustDirectory = await mkdtemp(join(tmpdir(), "b00b-trust-")); diff --git a/packages/coding-agent/test/swarm/production-evidence-adapter.ts b/packages/coding-agent/test/swarm/production-evidence-adapter.ts index e3c4199ad..eba02c8fa 100644 --- a/packages/coding-agent/test/swarm/production-evidence-adapter.ts +++ b/packages/coding-agent/test/swarm/production-evidence-adapter.ts @@ -6,19 +6,22 @@ * and keeps the authenticated artifact commitment in a sibling trust root. */ import { execFile as execFileCallback } from "node:child_process"; -import { type KeyObject, sign, verify as verifySignature } from "node:crypto"; +import { type KeyObject, sign } from "node:crypto"; import { mkdir, readFile, realpath, writeFile } from "node:fs/promises"; import { promisify } from "node:util"; import { + artifactBundleIdForSwarmEvidenceCapability, canonicalJson, + createSwarmEvidenceTrustRoot, runSwarmBenchmark, + SWARM_EVIDENCE_COMMITMENT_SCHEMA, type SwarmBenchmarkConfig, - verifySwarmEvidence, + swarmEvidenceCommitmentPayload, + verifyAuthenticatedSwarmEvidence, writeSwarmEvidence, } from "./swarm-evidence.js"; const execFile = promisify(execFileCallback); -const COMMITMENT_SCHEMA = "prime-agent.swarm-evidence-commitment/v1"; const MODEL_IDS = new Set(["fixture-a", "fixture-b", "fixture-zero"]); const RESOLVED_MODEL_IDS = new Set([...MODEL_IDS].map((id) => `${id}-resolved`)); @@ -49,11 +52,6 @@ export interface ProductionEvidenceInput { readonly priceCard: FrozenPriceCard; readonly metadata?: Readonly>; } -interface SignedCommitment { - readonly schemaVersion: typeof COMMITMENT_SCHEMA; - readonly artifactBundleId: string; - readonly signature: string; -} export interface SignedProductionEvidence { readonly artifactBundleId: string; readonly commitmentPath: string; @@ -65,9 +63,6 @@ function assert(condition: unknown, code: string): asserts condition { function integer(value: number): boolean { return Number.isSafeInteger(value) && value >= 0; } -function commitmentPayload(artifactBundleId: string) { - return { schemaVersion: COMMITMENT_SCHEMA, artifactBundleId }; -} function safeModel(value: string, resolved = false): string { return (resolved ? RESOLVED_MODEL_IDS : MODEL_IDS).has(value) ? value : "[REDACTED]"; } @@ -164,51 +159,35 @@ export async function writeSignedProductionEvidence( "B00B_EVIDENCE_TRUST_ROOT_OVERLAP", ); const evidence = await runSwarmBenchmark(projectProductionObservations(input)); - await writeSwarmEvidence(artifactRoot, evidence); - const manifest = JSON.parse(await readFile(`${artifactRoot}/manifest.json`, "utf8")) as { - artifactBundleId?: unknown; - }; - assert( - typeof manifest.artifactBundleId === "string" && /^[0-9a-f]{64}$/.test(manifest.artifactBundleId), - "B00B_EVIDENCE_WRITER_ID", - ); - const artifactBundleId = manifest.artifactBundleId; - const commitment: SignedCommitment = { - schemaVersion: COMMITMENT_SCHEMA, + const writerCapability = await writeSwarmEvidence(artifactRoot, evidence); + // This value is taken from the writer's opaque registration, never manifest.json. + const artifactBundleId = artifactBundleIdForSwarmEvidenceCapability(writerCapability); + const commitment = { + schemaVersion: SWARM_EVIDENCE_COMMITMENT_SCHEMA, artifactBundleId, - signature: sign(null, Buffer.from(canonicalJson(commitmentPayload(artifactBundleId))), signer).toString("base64"), + signature: sign( + null, + Buffer.from(canonicalJson(swarmEvidenceCommitmentPayload(artifactBundleId))), + signer, + ).toString("base64"), }; const commitmentPath = `${trustRoot}/artifact-commitment.json`; await writeFile(commitmentPath, `${canonicalJson(commitment)}\n`, { encoding: "utf8", mode: 0o600 }); return { artifactBundleId, commitmentPath }; } -/** Fresh-process safe verification: authenticate an externally supplied key first, then B00A semantics. */ +/** + * Fresh-process safe verification. The supplied public key is registered as an + * opaque trust root before the B00B verifier authenticates the commitment. + */ export async function verifySignedProductionEvidence( directory: string, commitmentPath: string, trustedPublicKeyPem: string, ): Promise { - const raw = await readFile(commitmentPath, "utf8"); - const commitment = JSON.parse(raw) as Partial; - assert(raw === `${canonicalJson(commitment)}\n`, "B00B_EVIDENCE_NONCANONICAL_COMMITMENT"); - assert( - commitment.schemaVersion === COMMITMENT_SCHEMA && - typeof commitment.artifactBundleId === "string" && - /^[0-9a-f]{64}$/.test(commitment.artifactBundleId) && - typeof commitment.signature === "string", - "B00B_EVIDENCE_BAD_COMMITMENT", - ); - assert( - verifySignature( - null, - Buffer.from(canonicalJson(commitmentPayload(commitment.artifactBundleId))), - trustedPublicKeyPem, - Buffer.from(commitment.signature, "base64"), - ), - "B00B_EVIDENCE_BAD_SIGNATURE", - ); - await verifySwarmEvidence(directory, commitment.artifactBundleId); + const commitmentRaw = await readFile(commitmentPath, "utf8"); + const trustRoot = createSwarmEvidenceTrustRoot(trustedPublicKeyPem); + await verifyAuthenticatedSwarmEvidence(directory, commitmentRaw, trustRoot); } /** Runs the authentication-plus-B00A verifier in a clean Node process. */ diff --git a/packages/coding-agent/test/swarm/swarm-evidence.test.ts b/packages/coding-agent/test/swarm/swarm-evidence.test.ts index 9c9d32dda..f7824de52 100644 --- a/packages/coding-agent/test/swarm/swarm-evidence.test.ts +++ b/packages/coding-agent/test/swarm/swarm-evidence.test.ts @@ -343,7 +343,7 @@ describe("PR-B00A deterministic local swarm evidence", () => { const samples = JSON.parse(await readFile(join(first, "process-samples.json"), "utf8")); samples[0].processes[0].pid += 1; await rehashArtifact(first, "process-samples.json", `${canonicalJson(samples)}\n`); - await expect(verify(first)).rejects.toThrow("trusted artifact bundle mismatch"); + await expect(verify(first)).rejects.toThrow("issued swarm evidence capability bundle mismatch"); }); test("rejects a capability issued for a different evidence directory", async () => { @@ -410,12 +410,10 @@ describe("PR-B00A deterministic local swarm evidence", () => { await writeFile(manifestPath, `${canonicalJson(forged)}\n`); const readBack = JSON.parse(await readFile(manifestPath, "utf8")).artifactBundleId; expect(typeof readBack).toBe("string"); - // A bundle id only becomes a cross-process trust root when authenticated - // outside the mutable evidence directory (B00B signs it). An unrelated - // externally supplied commitment still rejects this coherent forgery. - expect(readBack).not.toBe("f".repeat(64)); - await expect(verifySwarmEvidence(directory, "f".repeat(64))).rejects.toThrow("trusted artifact bundle mismatch"); - await expect(verify(directory)).rejects.toThrow("trusted artifact bundle mismatch"); + await expect(verifySwarmEvidence(directory, readBack)).rejects.toThrow( + "issued swarm evidence capability is required", + ); + await expect(verify(directory)).rejects.toThrow("issued swarm evidence capability bundle mismatch"); }); test("binds direct output usage to provider_completed terminal evidence", async () => { diff --git a/packages/coding-agent/test/swarm/swarm-evidence.ts b/packages/coding-agent/test/swarm/swarm-evidence.ts index dc77d57b1..a4c0a73fb 100644 --- a/packages/coding-agent/test/swarm/swarm-evidence.ts +++ b/packages/coding-agent/test/swarm/swarm-evidence.ts @@ -7,7 +7,7 @@ */ import { execFileSync } from "node:child_process"; -import { createHash } from "node:crypto"; +import { createHash, createPublicKey, type KeyObject, verify as verifySignature } from "node:crypto"; import { chmod, lstat, mkdir, readdir, readFile, realpath, stat, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { performance } from "node:perf_hooks"; @@ -171,6 +171,49 @@ function issueSwarmEvidenceCapability(): SwarmEvidenceCapability { return Object.freeze({}) as SwarmEvidenceCapability; } +/** An opaque root created only from an externally supplied Ed25519 public key. */ +declare const swarmEvidenceTrustRootBrand: unique symbol; +export type SwarmEvidenceTrustRoot = { readonly [swarmEvidenceTrustRootBrand]: true }; +const registeredTrustRoots = new WeakMap(); +export const SWARM_EVIDENCE_COMMITMENT_SCHEMA = "prime-agent.swarm-evidence-commitment/v1"; +export interface SignedSwarmEvidenceCommitment { + readonly schemaVersion: typeof SWARM_EVIDENCE_COMMITMENT_SCHEMA; + readonly artifactBundleId: string; + readonly signature: string; +} + +/** + * Registers a verifier trust root. The caller must provide this public key out + * of band: an artifact directory has no authority to manufacture this object. + */ +export function createSwarmEvidenceTrustRoot(publicKeyPem: string): SwarmEvidenceTrustRoot { + let publicKey: KeyObject; + try { + publicKey = createPublicKey(publicKeyPem); + } catch { + throw new Error("invalid swarm evidence public key"); + } + assert(publicKey.asymmetricKeyType === "ed25519", "swarm evidence trust root must be Ed25519"); + const root = Object.freeze({}) as SwarmEvidenceTrustRoot; + registeredTrustRoots.set(root, publicKey); + return root; +} + +/** Checked accessor: use the writer-issued identity; never read it back from mutable artifacts. */ +export function artifactBundleIdForSwarmEvidenceCapability(capability: SwarmEvidenceCapability): string { + const registration = registeredBundles.get(capability); + assert(registration, "issued swarm evidence capability is required"); + return registration.artifactBundleId; +} + +export function swarmEvidenceCommitmentPayload(artifactBundleId: string): { + schemaVersion: typeof SWARM_EVIDENCE_COMMITMENT_SCHEMA; + artifactBundleId: string; +} { + assert(/^[0-9a-f]{64}$/.test(artifactBundleId), "invalid trusted artifact bundle identity"); + return { schemaVersion: SWARM_EVIDENCE_COMMITMENT_SCHEMA, artifactBundleId }; +} + /** Canonical JSON rejects values which JSON.stringify silently changes. */ export function canonicalJson(value: unknown): string { if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value); @@ -1072,29 +1115,9 @@ function verifyProcessSamples(samples: unknown): void { } /** Strict verifier: expected set only, no links/extras, canonical bytes, hashes, and semantic joins. */ -/** - * Verifies an evidence directory against either the writer's in-process - * capability or an externally held artifact-bundle commitment. The latter is - * intentionally only the expected bundle id: callers which need durable - * trust (B00B) authenticate that id outside this artifact directory before - * invoking this canonical/semantic verifier in a fresh process. - */ -export async function verifySwarmEvidence( - directory: string, - trustedArtifactBundle: SwarmEvidenceCapability | string, -): Promise { - const registration = - typeof trustedArtifactBundle === "string" ? undefined : registeredBundles.get(trustedArtifactBundle); - if (typeof trustedArtifactBundle !== "string") assert(registration, "issued swarm evidence capability is required"); - assert( - typeof trustedArtifactBundle === "string" || registration !== undefined, - "issued swarm evidence capability is required", - ); - const expectedArtifactBundleId = - typeof trustedArtifactBundle === "string" ? trustedArtifactBundle : registration!.artifactBundleId; +async function verifyExpectedSwarmEvidence(directory: string, expectedArtifactBundleId: string): Promise { assert(/^[0-9a-f]{64}$/.test(expectedArtifactBundleId), "invalid trusted artifact bundle identity"); const root = await realpath(directory); - if (registration) assert(root === registration.directory, "swarm evidence capability directory mismatch"); const names = (await readdir(root)).sort(); assert( canonicalJson(names) === canonicalJson([...ALL_EVIDENCE_FILES].sort()), @@ -1161,6 +1184,62 @@ export async function verifySwarmEvidence( assert(manifest.deterministicBundleId === deterministic, "deterministic bundle identity mismatch"); assert(manifest.artifactBundleId === expectedArtifactBundleId, "trusted artifact bundle mismatch"); } + +/** B00A accepts only an issued in-process writer capability. */ +export async function verifySwarmEvidence(directory: string, capability: SwarmEvidenceCapability): Promise { + const registration = registeredBundles.get(capability); + assert(registration, "issued swarm evidence capability is required"); + const root = await realpath(directory); + assert(root === registration.directory, "swarm evidence capability directory mismatch"); + try { + await verifyExpectedSwarmEvidence(root, registration.artifactBundleId); + } catch (error) { + if (error instanceof Error && error.message === "trusted artifact bundle mismatch") + throw new Error("issued swarm evidence capability bundle mismatch", { cause: error }); + throw error; + } +} + +/** + * B00B fresh-process entry point. It authenticates canonical commitment bytes + * against an explicitly registered public-key root before entering the shared + * expected-ID semantic verifier. No artifact-derived string is a trust input. + */ +export async function verifyAuthenticatedSwarmEvidence( + directory: string, + commitmentRaw: string, + trustRoot: SwarmEvidenceTrustRoot, +): Promise { + const publicKey = registeredTrustRoots.get(trustRoot); + assert(publicKey, "registered swarm evidence trust root is required"); + const commitment = parseCanonicalJson( + commitmentRaw, + "artifact commitment", + ) as Partial; + assert( + commitment.schemaVersion === SWARM_EVIDENCE_COMMITMENT_SCHEMA && + typeof commitment.artifactBundleId === "string" && + /^[0-9a-f]{64}$/.test(commitment.artifactBundleId) && + typeof commitment.signature === "string", + "invalid swarm evidence commitment", + ); + let signature: Buffer; + try { + signature = Buffer.from(commitment.signature, "base64"); + } catch { + throw new Error("invalid swarm evidence commitment signature"); + } + assert( + verifySignature( + null, + Buffer.from(canonicalJson(swarmEvidenceCommitmentPayload(commitment.artifactBundleId))), + publicKey, + signature, + ), + "B00B_EVIDENCE_BAD_SIGNATURE", + ); + await verifyExpectedSwarmEvidence(directory, commitment.artifactBundleId); +} export function createFixedFanoutScenario(fanout: (typeof SUPPORTED_SWARM_FANOUTS)[number]): SwarmBenchmarkConfig { return { scenario: `fixed-fanout-${fanout}`, From 06a5b5450489f1009977028514bb30ed7e6d5f20 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 20:15:19 -0700 Subject: [PATCH 04/28] test(coding-agent): harden B00B economics and provenance (cherry picked from commit acd8de0c8fa800db8ec46cc03c1b93127f647526) --- .../swarm/production-evidence-adapter.test.ts | 147 +++++++++++++- .../test/swarm/production-evidence-adapter.ts | 58 ++++-- .../test/swarm/swarm-evidence.test.ts | 30 +-- .../coding-agent/test/swarm/swarm-evidence.ts | 181 +++++++++++++----- 4 files changed, 323 insertions(+), 93 deletions(-) diff --git a/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts b/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts index d31550534..ffed17b6b 100644 --- a/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts +++ b/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts @@ -5,11 +5,19 @@ import { join } from "node:path"; import { afterEach, describe, expect, test } from "vitest"; import { type ProductionEvidenceInput, + projectProductionObservations, verifySignedProductionEvidence, verifySignedProductionEvidenceFreshProcess, writeSignedProductionEvidence, } from "./production-evidence-adapter.js"; -import { canonicalJson, SWARM_EVIDENCE_COMMITMENT_SCHEMA, swarmEvidenceCommitmentPayload } from "./swarm-evidence.js"; +import { + COST_NUMERATOR_SCALE, + canonicalJson, + createSwarmEvidenceTrustRoot, + SWARM_EVIDENCE_COMMITMENT_SCHEMA, + swarmEvidenceCommitmentPayload, + verifyAuthenticatedSwarmEvidence, +} from "./swarm-evidence.js"; const cleanup: string[] = []; afterEach(async () => { @@ -31,15 +39,39 @@ function input(): ProductionEvidenceInput { requestId: "request-0001", attempt: 1, requested: { provider: "b00b-scripted", model: "fixture-a", revision: "alias-secret", effort: "high" }, - resolved: { provider: "b00b-scripted", model: "fixture-a", responseModel: "fixture-b-resolved" }, + resolved: { + api: "b00b-scripted", + provider: "b00b-scripted", + model: "fixture-a", + responseModel: "fixture-b-resolved", + }, terminal: "done", usage: { inputMicroTokens: 101, outputMicroTokens: 13, cacheReadMicroTokens: 7, cacheWriteMicroTokens: 3 }, }, + { + // A failed retry remains a separately authenticated attempt even at zero usage. + requestId: "request-0001", + attempt: 2, + requested: { provider: "b00b-scripted", model: "fixture-zero" }, + resolved: { + api: "b00b-scripted", + provider: "b00b-scripted", + model: "fixture-zero", + responseModel: "fixture-zero-resolved", + }, + terminal: "error", + usage: { inputMicroTokens: 0, outputMicroTokens: 99, cacheReadMicroTokens: 0, cacheWriteMicroTokens: 0 }, + }, { requestId: "request-0002", attempt: 1, requested: { provider: "b00b-scripted", model: "fixture-zero" }, - resolved: { provider: "b00b-scripted", model: "fixture-zero", responseModel: "fixture-zero-resolved" }, + resolved: { + api: "b00b-scripted", + provider: "b00b-scripted", + model: "fixture-zero", + responseModel: "fixture-zero-resolved", + }, terminal: "aborted", usage: { inputMicroTokens: 9, outputMicroTokens: 99, cacheReadMicroTokens: 2, cacheWriteMicroTokens: 0 }, }, @@ -47,9 +79,25 @@ function input(): ProductionEvidenceInput { }; } async function allFiles(directory: string): Promise { - return (await Promise.all((await readdir(directory)).map((name) => readFile(join(directory, name), "utf8")))).join( - "\n", - ); + const contents: string[] = []; + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) contents.push(await allFiles(path)); + else if (entry.isFile()) contents.push(await readFile(path, "utf8")); + } + return contents.join("\n"); +} +function privacyVariants(value: string): readonly string[] { + const unicodeEscaped = [...value] + .map((character) => `\\u${character.codePointAt(0)!.toString(16).padStart(4, "0")}`) + .join(""); + return [value, value.normalize("NFC"), value.normalize("NFKC"), unicodeEscaped]; +} +function expectNoCanaryLeak(chunks: readonly string[]): void { + const joined = chunks.join(""); + for (const canary of canaries) + for (const variant of privacyVariants(canary)) + expect(joined.normalize("NFKC")).not.toContain(variant.normalize("NFKC")); } /** Coherently re-index a semantic-preserving process-sample mutation. */ @@ -88,23 +136,34 @@ describe("B00B signed production evidence adapter", () => { verifySignedProductionEvidenceFreshProcess(artifactDirectory, written.commitmentPath, publicPem), ).resolves.toBeUndefined(); const content = await allFiles(artifactDirectory); - for (const canary of canaries) expect(content).not.toContain(canary); + expectNoCanaryLeak([content]); const costs = JSON.parse(await readFile(join(artifactDirectory, "cost-attribution.json"), "utf8")); const firstAttempt = costs.find((cost: { id: string }) => cost.id === "worker-0001"); expect(firstAttempt).toMatchObject({ directInputTokens: 111, directOutputTokens: 13, - directCost: (111 * 17 + 13 * 29) / 1_000_000, + directCostNumerator: 111 * 17 + 13 * 29, }); + expect(COST_NUMERATOR_SCALE).toBe(1_000_000); const run = costs.find((cost: { id: string }) => cost.id === "run"); expect(run).toMatchObject({ downstreamInputTokens: 122, downstreamOutputTokens: 13, - downstreamCost: (122 * 17 + 13 * 29) / 1_000_000, + downstreamCostNumerator: 122 * 17 + 13 * 29, }); // The terminal response model, not the selected requested alias, is retained as the resolved attribution. expect(content).toContain("fixture-b-resolved"); + expect(content).toContain('"api":"b00b-scripted"'); + expect(content).toContain('"responseModel":"fixture-b-resolved"'); expect(content).not.toContain("alias-secret"); + const manifest = JSON.parse(await readFile(join(artifactDirectory, "manifest.json"), "utf8")); + expect(manifest.assignments.map((assignment: { attemptId: string }) => assignment.attemptId)).toEqual([ + "attempt-0001-01", + "attempt-0001-02", + "attempt-0002-01", + ]); + const retry = manifest.assignments[1]; + expect(retry.resolved).toMatchObject({ model: "fixture-zero", responseModel: "fixture-zero-resolved" }); }); test("rejects manifest read-back, coherent forgery, wrong key, and a commitment from another artifact directory", async () => { @@ -184,4 +243,74 @@ describe("B00B signed production evidence adapter", () => { verifySignedProductionEvidence(artifactDirectory, written.commitmentPath, publicPem), ).rejects.toThrow("B00B_EVIDENCE_BAD_SIGNATURE"); }); + test("keeps zero-price, cache, done/error/abort, and retry economics in exact numerators", () => { + const source = input(); + const zero: ProductionEvidenceInput = { + ...source, + priceCard: { + ...source.priceCard, + inputMicroCurrencyPerMillionMicroTokens: 0, + outputMicroCurrencyPerMillionMicroTokens: 0, + }, + }; + const projected = projectProductionObservations(zero); + expect(projected.assignments).toHaveLength(3); + expect(projected.assignments.map((assignment) => assignment.inputTokens)).toEqual([111, 0, 11]); + expect(projected.assignments.map((assignment) => assignment.outputTokens)).toEqual([13, 0, 0]); + expect(projected.assignments.map((assignment) => assignment.attemptId)).toEqual([ + "attempt-0001-01", + "attempt-0001-02", + "attempt-0002-01", + ]); + expect(projected.priceCard).toMatchObject({ inputPerMillionTokens: 0, outputPerMillionTokens: 0 }); + }); + + test("privacy scans recursive normal outputs and captured console/stderr in normalized, escaped, and split forms", async () => { + const artifactDirectory = await mkdtemp(join(tmpdir(), "b00b-privacy-artifact-")); + const trustDirectory = await mkdtemp(join(tmpdir(), "b00b-privacy-trust-")); + cleanup.push(artifactDirectory, trustDirectory); + const keys = generateKeyPairSync("ed25519"); + const capturedConsole: string[] = []; + const capturedStderr: string[] = []; + const originalError = console.error; + const originalWrite = process.stderr.write; + console.error = (...values: unknown[]) => capturedConsole.push(values.map(String).join(" ")); + process.stderr.write = ((chunk: unknown) => { + capturedStderr.push(String(chunk)); + return true; + }) as typeof process.stderr.write; + try { + await writeSignedProductionEvidence(artifactDirectory, trustDirectory, input(), keys.privateKey); + } finally { + console.error = originalError; + process.stderr.write = originalWrite; + } + expectNoCanaryLeak([await allFiles(artifactDirectory), ...capturedConsole, ...capturedStderr]); + }); + test("rejects fabricated trust roots and malformed or noncanonical commitments", async () => { + const artifactDirectory = await mkdtemp(join(tmpdir(), "b00b-root-artifact-")); + const trustDirectory = await mkdtemp(join(tmpdir(), "b00b-root-trust-")); + cleanup.push(artifactDirectory, trustDirectory); + const keys = generateKeyPairSync("ed25519"); + const written = await writeSignedProductionEvidence(artifactDirectory, trustDirectory, input(), keys.privateKey); + const publicPem = keys.publicKey.export({ type: "spki", format: "pem" }).toString(); + const commitment = await readFile(written.commitmentPath, "utf8"); + await expect( + verifyAuthenticatedSwarmEvidence( + artifactDirectory, + commitment, + {} as ReturnType, + ), + ).rejects.toThrow("registered swarm evidence trust root is required"); + await expect( + verifyAuthenticatedSwarmEvidence(artifactDirectory, `${commitment} `, createSwarmEvidenceTrustRoot(publicPem)), + ).rejects.toThrow("non-canonical JSON: artifact commitment"); + await expect( + verifyAuthenticatedSwarmEvidence( + artifactDirectory, + `${canonicalJson({ schemaVersion: SWARM_EVIDENCE_COMMITMENT_SCHEMA, artifactBundleId: written.artifactBundleId, signature: "%%%" })}\n`, + createSwarmEvidenceTrustRoot(publicPem), + ), + ).rejects.toThrow("B00B_EVIDENCE_BAD_SIGNATURE"); + }); }); diff --git a/packages/coding-agent/test/swarm/production-evidence-adapter.ts b/packages/coding-agent/test/swarm/production-evidence-adapter.ts index eba02c8fa..3ba6bb0fe 100644 --- a/packages/coding-agent/test/swarm/production-evidence-adapter.ts +++ b/packages/coding-agent/test/swarm/production-evidence-adapter.ts @@ -23,7 +23,8 @@ import { const execFile = promisify(execFileCallback); const MODEL_IDS = new Set(["fixture-a", "fixture-b", "fixture-zero"]); -const RESOLVED_MODEL_IDS = new Set([...MODEL_IDS].map((id) => `${id}-resolved`)); +const RESPONSE_MODEL_IDS = new Set([...MODEL_IDS].map((id) => `${id}-resolved`)); +const PROVIDER = "b00b-scripted"; export interface ExactUsage { /** Integer micro-tokens. No floating point token or price field is accepted. */ @@ -36,7 +37,7 @@ export interface ImmutableAttemptObservation { readonly requestId: `request-${string}`; readonly attempt: number; readonly requested: Readonly<{ provider: string; model: string; revision?: string; effort?: string }>; - readonly resolved: Readonly<{ provider: string; model: string; responseModel: string }>; + readonly resolved: Readonly<{ api: string; provider: string; model: string; responseModel: string }>; readonly terminal: "done" | "error" | "aborted"; readonly usage: ExactUsage; } @@ -63,13 +64,11 @@ function assert(condition: unknown, code: string): asserts condition { function integer(value: number): boolean { return Number.isSafeInteger(value) && value >= 0; } -function safeModel(value: string, resolved = false): string { - return (resolved ? RESOLVED_MODEL_IDS : MODEL_IDS).has(value) ? value : "[REDACTED]"; +function safeModel(value: string, responseModel = false): string { + return (responseModel ? RESPONSE_MODEL_IDS : MODEL_IDS).has(value) ? value : "[REDACTED]"; } -function publicAttemptId(observation: ImmutableAttemptObservation, index: number): string { - // Input identity is only used inside B00A's in-memory projection. B00A - // assigns the public worker/request IDs and redacts all other identities. - return `attempt-${index + 1}-${observation.attempt}`; +function publicAttemptId(observation: ImmutableAttemptObservation): string { + return `attempt-${observation.requestId.slice("request-".length)}-${String(observation.attempt).padStart(2, "0")}`; } function assertInput(input: ProductionEvidenceInput): void { assert(input.attempts.length > 0, "B00B_EVIDENCE_NO_ATTEMPTS"); @@ -86,33 +85,52 @@ function assertInput(input: ProductionEvidenceInput): void { assert(!identities.has(`${observation.requestId}:${observation.attempt}`), "B00B_EVIDENCE_DUPLICATE_ATTEMPT"); identities.add(`${observation.requestId}:${observation.attempt}`); for (const value of Object.values(observation.usage)) assert(integer(value), "B00B_EVIDENCE_NON_INTEGER_USAGE"); + assert( + Boolean(observation.requested.provider && observation.requested.model), + "B00B_EVIDENCE_REQUESTED_PROVENANCE", + ); + assert( + Boolean( + observation.resolved.api && + observation.resolved.provider && + observation.resolved.model && + observation.resolved.responseModel, + ), + "B00B_EVIDENCE_RESOLVED_PROVENANCE", + ); } } /** * Converts immutable terminal observations into B00A input. The B00A schema - * currently has input/output columns only, so cache usage is deliberately - * included in input micro-tokens rather than silently discarded. Its separate - * integer fields remain part of the immutable adapter input and can be - * independently recomputed by callers. + * records only integer usage: cache read/write are included in direct input, + * terminal error/abort output is zero, and every retry attempt is a distinct + * stable assignment. B00A stores exact cost numerators over its documented + * 1,000,000 scale, never binary floating-point money. */ export function projectProductionObservations(input: ProductionEvidenceInput): SwarmBenchmarkConfig { assertInput(input); return { scenario: input.scenario, assignments: input.attempts.map((observation, index) => ({ - nodeId: publicAttemptId(observation, index), + nodeId: `attempt-worker-${String(index + 1).padStart(4, "0")}`, role: "provider-attempt", + requestId: observation.requestId, + attempt: observation.attempt, + attemptId: publicAttemptId(observation), requested: { - provider: observation.requested.provider === "b00b-scripted" ? "b00b-scripted" : "[REDACTED]", + provider: observation.requested.provider === PROVIDER ? PROVIDER : "[REDACTED]", model: safeModel(observation.requested.model), - ...(observation.requested.revision === undefined ? {} : { revision: observation.requested.revision }), - ...(observation.requested.effort === undefined ? {} : { effort: observation.requested.effort }), + // revision/effort remain explicitly present but content-free. + ...(observation.requested.revision === undefined ? {} : { revision: "[REDACTED]" }), + ...(observation.requested.effort === undefined ? {} : { effort: "[REDACTED]" }), }, resolved: { - provider: observation.resolved.provider === "b00b-scripted" ? "b00b-scripted" : "[REDACTED]", - // responseModel is the attribution authority, never the selected model. - model: safeModel(observation.resolved.responseModel, true), + api: observation.resolved.api === PROVIDER ? PROVIDER : "[REDACTED]", + provider: observation.resolved.provider === PROVIDER ? PROVIDER : "[REDACTED]", + model: safeModel(observation.resolved.model), + // responseModel, not selected resolved model, is the attribution authority. + responseModel: safeModel(observation.resolved.responseModel, true), }, inputTokens: observation.usage.inputMicroTokens + @@ -125,7 +143,7 @@ export function projectProductionObservations(input: ProductionEvidenceInput): S observation.terminal === "done" ? undefined : { - nodeId: publicAttemptId(observation, index), + nodeId: `attempt-worker-${String(index + 1).padStart(4, "0")}`, actions: [{ type: "failure" as const, code: "[REDACTED]", message: "[REDACTED]" }], }, ) diff --git a/packages/coding-agent/test/swarm/swarm-evidence.test.ts b/packages/coding-agent/test/swarm/swarm-evidence.test.ts index f7824de52..8c834756f 100644 --- a/packages/coding-agent/test/swarm/swarm-evidence.test.ts +++ b/packages/coding-agent/test/swarm/swarm-evidence.test.ts @@ -135,8 +135,8 @@ describe("PR-B00A deterministic local swarm evidence", () => { const lead = evidence.costAttribution.find((cost) => cost.id === "role-0001"); const run = evidence.costAttribution.find((cost) => cost.id === "run"); expect(lead?.downstreamInputTokens).toBe(64); - expect(run).toMatchObject({ kind: "run", directCost: 0 }); - expect(run?.downstreamCost).toBeGreaterThan(0); + expect(run).toMatchObject({ kind: "run", directCostNumerator: 0 }); + expect(run?.downstreamCostNumerator).toBeGreaterThan(0); }); test("accepts an empty process sample when the platform sampler has no visible processes", async () => { const directory = await mkdtemp(join(tmpdir(), "prime-agent-b00a-empty-processes-")); @@ -248,7 +248,7 @@ describe("PR-B00A deterministic local swarm evidence", () => { const directory = await evidenceDirectory(1); const costsPath = join(directory, "cost-attribution.json"); const costs = JSON.parse(await readFile(costsPath, "utf8")); - costs.find((cost: { kind: string }) => cost.kind === "node").directCost = 7; + costs.find((cost: { kind: string }) => cost.kind === "node").directCostNumerator = 7; const costsRaw = `${canonicalJson(costs)} `; await writeFile(costsPath, costsRaw); @@ -319,17 +319,17 @@ describe("PR-B00A deterministic local swarm evidence", () => { const costs = JSON.parse(await readFile(costsPath, "utf8")); const node = costs.find((cost: { kind: string }) => cost.kind === "node"); node.directInputTokens = 999; - node.directCost = (999 + node.directOutputTokens * 2) / 1_000_000; + node.directCostNumerator = 999 + node.directOutputTokens * 2; node.downstreamInputTokens = 999; - node.downstreamCost = node.directCost; + node.downstreamCostNumerator = node.directCostNumerator; const role = costs.find((cost: { kind: string }) => cost.kind === "role"); role.directInputTokens = 999; role.downstreamInputTokens = 999; - role.directCost = node.directCost; - role.downstreamCost = node.directCost; + role.directCostNumerator = node.directCostNumerator; + role.downstreamCostNumerator = node.directCostNumerator; const run = costs.find((cost: { kind: string }) => cost.kind === "run"); run.downstreamInputTokens = 999; - run.downstreamCost = node.directCost; + run.downstreamCostNumerator = node.directCostNumerator; await rehashArtifact(directory, "cost-attribution.json", `${canonicalJson(costs)}\n`); await expect(verify(directory)).rejects.toThrow("assignment input usage mismatch"); }); @@ -377,8 +377,8 @@ describe("PR-B00A deterministic local swarm evidence", () => { row.directOutputTokens = row.kind === "run" ? 0 : 999; row.downstreamInputTokens = 999; row.downstreamOutputTokens = 999; - row.directCost = row.kind === "run" ? 0 : 999 / 1_000_000 + (999 * 2) / 1_000_000; - row.downstreamCost = 999 / 1_000_000 + (999 * 2) / 1_000_000; + row.directCostNumerator = row.kind === "run" ? 0 : 999 + 999 * 2; + row.downstreamCostNumerator = 999 + 999 * 2; } await rehashArtifact(directory, "events.jsonl", `${events.map(canonicalJson).join("\n")}\n`); await rehashArtifact( @@ -422,17 +422,17 @@ describe("PR-B00A deterministic local swarm evidence", () => { const costs = JSON.parse(await readFile(costsPath, "utf8")); const node = costs.find((cost: { kind: string }) => cost.kind === "node"); node.directOutputTokens = 999; - node.directCost = (node.directInputTokens + 999 * 2) / 1_000_000; + node.directCostNumerator = node.directInputTokens + 999 * 2; node.downstreamOutputTokens = 999; - node.downstreamCost = node.directCost; + node.downstreamCostNumerator = node.directCostNumerator; const role = costs.find((cost: { kind: string }) => cost.kind === "role"); role.directOutputTokens = 999; role.downstreamOutputTokens = 999; - role.directCost = node.directCost; - role.downstreamCost = node.directCost; + role.directCostNumerator = node.directCostNumerator; + role.downstreamCostNumerator = node.directCostNumerator; const run = costs.find((cost: { kind: string }) => cost.kind === "run"); run.downstreamOutputTokens = 999; - run.downstreamCost = node.directCost; + run.downstreamCostNumerator = node.directCostNumerator; await rehashArtifact(directory, "cost-attribution.json", `${canonicalJson(costs)}\n`); await expect(verify(directory)).rejects.toThrow("terminal output usage mismatch"); }); diff --git a/packages/coding-agent/test/swarm/swarm-evidence.ts b/packages/coding-agent/test/swarm/swarm-evidence.ts index a4c0a73fb..2f7e04a17 100644 --- a/packages/coding-agent/test/swarm/swarm-evidence.ts +++ b/packages/coding-agent/test/swarm/swarm-evidence.ts @@ -14,7 +14,12 @@ import { performance } from "node:perf_hooks"; export const SUPPORTED_SWARM_FANOUTS = [1, 4, 16, 64] as const; export const SWARM_EVIDENCE_SCHEMA_VERSION = "prime-agent.swarm-evidence/v1"; -const MICRO_TOKENS = 1_000_000; +/** + * Cost amounts are exact safe-integer numerators over this fixed denominator. + * `costNumerator / COST_NUMERATOR_SCALE` is presentation only; every signed + * artifact invariant operates exclusively on the numerator. + */ +export const COST_NUMERATOR_SCALE = 1_000_000; const REDACTED = "[REDACTED]"; const EVIDENCE_FILES = [ "cost-attribution.json", @@ -47,17 +52,29 @@ export interface FakeProviderFaultSchedule { readonly nodeId: string; readonly actions: readonly FakeProviderAction[]; } +export interface RequestedModelProvenance { + readonly provider: string; + readonly model: string; + readonly revision?: string; + readonly effort?: string; +} +/** Response model is the attribution authority; selected resolved model is retained separately. */ +export interface ResolvedModelProvenance { + readonly api: string; + readonly provider: string; + readonly model: string; + readonly responseModel: string; +} export interface AssignmentSpec { readonly nodeId: string; readonly parentNodeId?: string; readonly role: string; - readonly requested: { - readonly provider: string; - readonly model: string; - readonly revision?: string; - readonly effort?: string; - }; - readonly resolved?: AssignmentSpec["requested"]; + /** Stable public linkage for a request and its individual retry attempt. */ + readonly requestId?: string; + readonly attempt?: number; + readonly attemptId?: string; + readonly requested: RequestedModelProvenance; + readonly resolved?: ResolvedModelProvenance; readonly inputTokens?: number; readonly outputTokens?: number; } @@ -129,10 +146,10 @@ export interface CostAttribution { readonly kind: "node" | "role" | "run"; readonly directInputTokens: number; readonly directOutputTokens: number; - readonly directCost: number; + readonly directCostNumerator: number; readonly downstreamInputTokens: number; readonly downstreamOutputTokens: number; - readonly downstreamCost: number; + readonly downstreamCostNumerator: number; } export interface EvidenceArtifact { readonly path: (typeof EVIDENCE_FILES)[number]; @@ -285,6 +302,10 @@ const SAFE_EVIDENCE_KEYS = new Set([ "model", "revision", "effort", + "api", + "responseModel", + "attempt", + "attemptId", "inputTokens", "outputTokens", "actions", @@ -312,10 +333,10 @@ const SAFE_EVIDENCE_KEYS = new Set([ "kind", "directInputTokens", "directOutputTokens", - "directCost", + "directCostNumerator", "downstreamInputTokens", "downstreamOutputTokens", - "downstreamCost", + "downstreamCostNumerator", "admitted", "started", "completed", @@ -331,6 +352,7 @@ function safeEvidenceString(value: string, key?: string): boolean { return ( (key === "nodeId" && /^worker-\d{4}$/.test(value)) || (key === "requestId" && /^request-\d{4}$/.test(value)) || + (key === "attemptId" && /^attempt-\d{4}-\d{2}$/.test(value)) || (key === "parentNodeId" && (value === "root" || /^worker-\d{4}$/.test(value))) || ((key === "id" || key === "role") && (value === "run" || /^worker-\d{4}$/.test(value) || /^role-\d{4}$/.test(value))) || @@ -344,8 +366,9 @@ function safeEvidenceString(value: string, key?: string): boolean { ((key === "fingerprint" || key === "deterministicBundleId" || key === "artifactBundleId" || key === "sha256") && /^[0-9a-f]{64}$/.test(value)) || (key === "path" && (EVIDENCE_FILES as readonly string[]).includes(value)) || - (key === "provider" && value === "b00b-scripted") || - (key === "model" && + ((key === "provider" || key === "api") && value === "b00b-scripted") || + ((key === "revision" || key === "effort") && value === REDACTED) || + ((key === "model" || key === "responseModel") && [ "fixture-a", "fixture-b", @@ -397,8 +420,17 @@ function assertContentFree(value: unknown, key?: string, untrustedObjectKeys = f ); } } -function money(tokens: number, pricePerMillion: number): number { - return (tokens * pricePerMillion) / MICRO_TOKENS; +/** Exact numerator with denominator COST_NUMERATOR_SCALE; never use decimal money in evidence invariants. */ +function costNumerator(tokens: number, pricePerMillionTokens: number): number { + const numerator = tokens * pricePerMillionTokens; + assert(isSafeInteger(numerator), "cost numerator exceeds safe integer range"); + return numerator; +} +/** Sum authenticated integer fields without ever crossing into binary decimal money. */ +function exactSum(values: readonly number[]): number { + const total = values.reduce((sum, value) => sum + value, 0); + assert(isSafeInteger(total), "exact accounting sum exceeds safe integer range"); + return total; } function validate(config: SwarmBenchmarkConfig): void { assert(config.scenario.trim().length > 0, "scenario must not be empty"); @@ -424,6 +456,12 @@ function validate(config: SwarmBenchmarkConfig): void { assignment.outputTokens === undefined || isSafeInteger(assignment.outputTokens), "output tokens must be non-negative safe integers", ); + if (assignment.requestId !== undefined) + assert(/^request-\d{4}$/.test(assignment.requestId), "request IDs must be stable public IDs"); + if (assignment.attempt !== undefined) + assert(isSafeInteger(assignment.attempt) && assignment.attempt > 0, "attempt must be positive"); + if (assignment.attemptId !== undefined) + assert(/^attempt-\d{4}-\d{2}$/.test(assignment.attemptId), "attempt IDs must be stable public IDs"); } for (const schedule of config.faultSchedule ?? []) { assert( @@ -457,6 +495,9 @@ function publicConfig(config: SwarmBenchmarkConfig): Omit `request-${nodeId.slice("worker-".length)}`; + const requestFor = (assignment: AssignmentSpec) => + assignment.requestId ?? `request-${assignment.nodeId.slice("worker-".length)}`; const record = (type: EventType, nodeId: string, detail?: Readonly>) => events.push({ sequence: ++sequence, elapsedMilliseconds: performance.now() - startedAt, type, nodeId, - requestId: requestFor(nodeId), + requestId: requestFor(publicAssignments.find((assignment) => assignment.nodeId === nodeId)!), ...(detail === undefined ? {} : { detail }), }); const sample = (phase: ProcessSample["phase"]) => { @@ -621,20 +663,24 @@ export async function runSwarmBenchmark(config: SwarmBenchmarkConfig): Promise calculate(candidate.assignment.nodeId)); const directInputTokens = result.assignment.inputTokens ?? 32; const directOutputTokens = result.outputTokens; - const directCost = - money(directInputTokens, config.priceCard.inputPerMillionTokens) + - money(directOutputTokens, config.priceCard.outputPerMillionTokens); + const directCostNumerator = + costNumerator(directInputTokens, config.priceCard.inputPerMillionTokens) + + costNumerator(directOutputTokens, config.priceCard.outputPerMillionTokens); const attribution = { id, kind: "node" as const, directInputTokens, directOutputTokens, - directCost, - downstreamInputTokens: - directInputTokens + children.reduce((sum, child) => sum + child.downstreamInputTokens, 0), - downstreamOutputTokens: - directOutputTokens + children.reduce((sum, child) => sum + child.downstreamOutputTokens, 0), - downstreamCost: directCost + children.reduce((sum, child) => sum + child.downstreamCost, 0), + directCostNumerator, + downstreamInputTokens: exactSum([directInputTokens, ...children.map((child) => child.downstreamInputTokens)]), + downstreamOutputTokens: exactSum([ + directOutputTokens, + ...children.map((child) => child.downstreamOutputTokens), + ]), + downstreamCostNumerator: exactSum([ + directCostNumerator, + ...children.map((child) => child.downstreamCostNumerator), + ]), }; costs.set(id, attribution); return attribution; @@ -662,16 +708,16 @@ export async function runSwarmBenchmark(config: SwarmBenchmarkConfig): Promise result.assignment.role === role) .map((result) => calculate(result.assignment.nodeId)); const sum = (items: readonly CostAttribution[], key: keyof CostAttribution) => - items.reduce((total, item) => total + (item[key] as number), 0); + exactSum(items.map((item) => item[key] as number)); return { id: role, kind: "role" as const, directInputTokens: sum(direct, "directInputTokens"), directOutputTokens: sum(direct, "directOutputTokens"), - directCost: sum(direct, "directCost"), + directCostNumerator: sum(direct, "directCostNumerator"), downstreamInputTokens: sum([...included.values()], "directInputTokens"), downstreamOutputTokens: sum([...included.values()], "directOutputTokens"), - downstreamCost: sum([...included.values()], "directCost"), + downstreamCostNumerator: sum([...included.values()], "directCostNumerator"), }; }); const roots = results @@ -682,10 +728,10 @@ export async function runSwarmBenchmark(config: SwarmBenchmarkConfig): Promise sum + item.downstreamInputTokens, 0), - downstreamOutputTokens: roots.reduce((sum, item) => sum + item.downstreamOutputTokens, 0), - downstreamCost: roots.reduce((sum, item) => sum + item.downstreamCost, 0), + directCostNumerator: 0, + downstreamInputTokens: exactSum(roots.map((item) => item.downstreamInputTokens)), + downstreamOutputTokens: exactSum(roots.map((item) => item.downstreamOutputTokens)), + downstreamCostNumerator: exactSum(roots.map((item) => item.downstreamCostNumerator)), }; const firstTerminal = events.findIndex( (event) => event.type === "provider_completed" || event.type === "provider_failure", @@ -828,6 +874,29 @@ function requireManifest(manifest: unknown): asserts manifest is SwarmManifest & "invalid artifact bundle identity", ); assertContentFree(manifest); + const attemptIds = new Set(); + for (const assignment of manifest.assignments as Record[]) { + assert(isRecord(assignment) && isRecord(assignment.requested), "invalid assignment provenance"); + for (const key of ["provider", "model"]) + assert(typeof assignment.requested[key] === "string", `missing requested ${key}`); + if (assignment.resolved !== undefined) { + assert(isRecord(assignment.resolved), "invalid resolved provenance"); + for (const key of ["api", "provider", "model", "responseModel"]) + assert(typeof assignment.resolved[key] === "string", `missing resolved ${key}`); + } + if (assignment.attemptId !== undefined) { + const attemptId = assignment.attemptId; + assert( + typeof assignment.requestId === "string" && + isSafeInteger(assignment.attempt) && + typeof attemptId === "string" && + /^attempt-\d{4}-\d{2}$/.test(attemptId) && + !attemptIds.has(attemptId), + "invalid or duplicate request attempt identity", + ); + attemptIds.add(attemptId); + } + } const source = { schemaVersion: manifest.schemaVersion, benchmarkVersion: manifest.benchmarkVersion, @@ -841,9 +910,9 @@ function requireManifest(manifest: unknown): asserts manifest is SwarmManifest & } function verifyEvents(events: readonly unknown[], oracle: readonly unknown[], assignments: readonly unknown[]): void { assert(events.length === oracle.length && events.length > 0, "event/oracle length mismatch"); - const nodeIds = new Set( - (assignments as readonly Record[]).map((assignment) => assignment.nodeId).filter(isString), - ); + const assignmentRows = assignments as readonly Record[]; + const nodeIds = new Set(assignmentRows.map((assignment) => assignment.nodeId).filter(isString)); + const assignmentByNode = new Map(assignmentRows.map((assignment) => [assignment.nodeId as string, assignment])); let previousSequence = 0; const byNode = new Map[]>(); for (let index = 0; index < events.length; index++) { @@ -870,10 +939,11 @@ function verifyEvents(events: readonly unknown[], oracle: readonly unknown[], as (EVENT_TYPES as readonly unknown[]).includes(event.type), "invalid event timing/type", ); + const eventAssignment = assignmentByNode.get(event.nodeId as string); assert( typeof event.nodeId === "string" && nodeIds.has(event.nodeId) && - event.requestId === `request-${event.nodeId.slice("worker-".length)}`, + event.requestId === (eventAssignment?.requestId ?? `request-${event.nodeId.slice("worker-".length)}`), "invalid event identity", ); assert( @@ -895,6 +965,14 @@ function verifyEvents(events: readonly unknown[], oracle: readonly unknown[], as break; case "provider_request_started": exactDetail(["role", "requested", "resolved"]); + assert( + isRecord(detail) && + canonicalJson(detail.role) === canonicalJson(eventAssignment?.role) && + canonicalJson(detail.requested) === canonicalJson(eventAssignment?.requested) && + canonicalJson(detail.resolved) === + canonicalJson(eventAssignment?.resolved ?? eventAssignment?.requested), + "event provenance mismatch", + ); break; case "progress": exactDetail(["message"]); @@ -990,8 +1068,8 @@ function verifyCosts( ids.add(row.id); for (const key of ["directInputTokens", "directOutputTokens", "downstreamInputTokens", "downstreamOutputTokens"]) assert(isSafeInteger(row[key]), `invalid ${key}`); - for (const key of ["directCost", "downstreamCost"]) - assert(typeof row[key] === "number" && Number.isFinite(row[key]), `invalid ${key}`); + for (const key of ["directCostNumerator", "downstreamCostNumerator"]) + assert(isSafeInteger(row[key]), `invalid ${key}`); } const nodes = rows.filter((row) => row.kind === "node"); const assignmentRows = assignments as readonly Record[]; @@ -1017,20 +1095,21 @@ function verifyCosts( `terminal output usage mismatch: ${node.id}`, ); assert( - node.directCost === - money(node.directInputTokens as number, inputPrice) + money(node.directOutputTokens as number, outputPrice), + node.directCostNumerator === + costNumerator(node.directInputTokens as number, inputPrice) + + costNumerator(node.directOutputTokens as number, outputPrice), "direct economics mismatch", ); const children = assignmentRows .filter((assignment) => assignment.parentNodeId === node.id) .map((assignment) => nodeById.get(assignment.nodeId as string)); assert(children.every(isRecord), "missing child cost"); - for (const suffix of ["InputTokens", "OutputTokens", "Cost"] as const) { + for (const suffix of ["InputTokens", "OutputTokens", "CostNumerator"] as const) { const direct = node[`direct${suffix}`]; assert(typeof direct === "number", `invalid direct cost field: ${suffix}`); assert( node[`downstream${suffix}`] === - direct + children.reduce((sum, child) => sum + (child[`downstream${suffix}`] as number), 0), + exactSum([direct as number, ...children.map((child) => child[`downstream${suffix}`] as number)]), `node tree invariant failed: ${node.id}:${suffix}`, ); } @@ -1056,26 +1135,30 @@ function verifyCosts( .filter((assignment) => assignment.role === role.id) .map((assignment) => nodeById.get(assignment.nodeId as string)!); const included = new Map(direct.flatMap((node) => descendants(node.id as string)).map((node) => [node.id, node])); - for (const suffix of ["InputTokens", "OutputTokens", "Cost"] as const) { + for (const suffix of ["InputTokens", "OutputTokens", "CostNumerator"] as const) { assert( - role[`direct${suffix}`] === direct.reduce((sum, node) => sum + (node[`direct${suffix}`] as number), 0), + role[`direct${suffix}`] === exactSum(direct.map((node) => node[`direct${suffix}`] as number)), `role direct invariant failed: ${role.id}:${suffix}`, ); assert( role[`downstream${suffix}`] === - [...included.values()].reduce((sum, node) => sum + (node[`direct${suffix}`] as number), 0), + exactSum([...included.values()].map((node) => node[`direct${suffix}`] as number)), `role tree invariant failed: ${role.id}:${suffix}`, ); } } const run = rows.find((row) => row.id === "run" && row.kind === "run"); assert(run, "missing run cost"); + assert( + run.directInputTokens === 0 && run.directOutputTokens === 0 && run.directCostNumerator === 0, + "run direct invariant failed", + ); const roots = nodes.filter( (node) => !assignmentRows.find((assignment) => assignment.nodeId === node.id)?.parentNodeId, ); - for (const suffix of ["InputTokens", "OutputTokens", "Cost"] as const) + for (const suffix of ["InputTokens", "OutputTokens", "CostNumerator"] as const) assert( - run[`downstream${suffix}`] === roots.reduce((sum, node) => sum + (node[`downstream${suffix}`] as number), 0), + run[`downstream${suffix}`] === exactSum(roots.map((node) => node[`downstream${suffix}`] as number)), `run tree invariant failed: ${suffix}`, ); } From ae4c15fc2599008ab75e96d78e0b7a7c68f43514 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 20:25:44 -0700 Subject: [PATCH 05/28] test(coding-agent): exercise B00B real RLM child admission (cherry picked from commit 32a9570861fad3a6e3dc22e3253dd89ed4223e2e) --- .../swarm-production-integration.test.ts | 286 +++++++++++++++++- 1 file changed, 283 insertions(+), 3 deletions(-) diff --git a/packages/coding-agent/test/swarm/swarm-production-integration.test.ts b/packages/coding-agent/test/swarm/swarm-production-integration.test.ts index 0cc6cc59e..5e03e2849 100644 --- a/packages/coding-agent/test/swarm/swarm-production-integration.test.ts +++ b/packages/coding-agent/test/swarm/swarm-production-integration.test.ts @@ -1,17 +1,29 @@ /** Production-path coverage for the B00B test-only scripted provider. */ +import { generateKeyPairSync } from "node:crypto"; import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { Agent } from "@earendil-works/pi-agent-core"; import { Type } from "@earendil-works/pi-ai"; -import { afterEach, describe, expect, test } from "vitest"; +import { afterEach, describe, expect, test, vi } from "vitest"; import { AgentSession } from "../../src/core/agent-session.js"; +import { + type AgentSessionRuntime, + type CreateAgentSessionRuntimeFactory, + createAgentSessionFromServices, + createAgentSessionRuntime, + createAgentSessionServices, +} from "../../src/core/agent-session-runtime.js"; import { AuthStorage } from "../../src/core/auth-storage.js"; import { convertToLlm } from "../../src/core/messages.js"; import { ModelRegistry } from "../../src/core/model-registry.js"; import { SessionManager } from "../../src/core/session-manager.js"; import { SettingsManager } from "../../src/core/settings-manager.js"; import { createTestResourceLoader } from "../utilities.js"; +import { + verifySignedProductionEvidenceFreshProcess, + writeSignedProductionEvidence, +} from "./production-evidence-adapter.js"; import { createBarrierScriptedProvider, type ProviderScript } from "./production-scripted-provider.js"; const cleanups: Array<() => Promise | void> = []; @@ -44,9 +56,9 @@ const canaries = [ function provider(scripts: Record, expected: readonly string[]) { const registered = createBarrierScriptedProvider({ - api: "b00b-scripted-api", + api: "b00b-scripted", provider: "b00b-scripted", - barrier: { expected, timeoutMs: 2_000 }, + barrier: { expected, timeoutMs: 10_000 }, models: [ { id: "fixture-a", @@ -90,6 +102,91 @@ async function readTree(directory: string): Promise { return (await Promise.all(names.map((name) => readFile(join(directory, name), "utf8")))).join("\n"); } +function providerRegistration(models: ReturnType["models"]) { + return { + baseUrl: models[0]!.baseUrl, + apiKey: "fixture-key", + api: models[0]!.api, + models: models.map((model) => ({ + id: model.id, + name: model.name, + api: model.api, + reasoning: model.reasoning, + input: model.input, + cost: model.cost, + contextWindow: model.contextWindow, + maxTokens: model.maxTokens, + baseUrl: model.baseUrl, + })), + }; +} + +/** + * Builds the same in-process runtime host used by production sessions. In + * particular, children are created by AgentSessionRuntime rather than injected + * AgentSession fixtures, so this reaches runRlmChild -> runtime -> agent-loop. + */ +async function runtimeForRlmFixture( + fixture: ReturnType, + directory: string, +): Promise { + const authStorage = AuthStorage.inMemory(); + const rootModel = fixture.models[0]!; + authStorage.setRuntimeApiKey(rootModel.provider, "fixture-key"); + const settingsManager = SettingsManager.inMemory({ retry: { enabled: false } }); + const registration = providerRegistration(fixture.models); + const createRuntime: CreateAgentSessionRuntimeFactory = async (runtimeOptions) => { + const services = await createAgentSessionServices({ + cwd: runtimeOptions.cwd, + agentDir: directory, + authStorage, + settingsManager, + telemetryDisabled: true, + resourceLoaderOptions: { + extensionFactories: [ + (pi) => { + pi.registerProvider(rootModel.provider, registration); + }, + ], + noSkills: true, + noPromptTemplates: true, + noThemes: true, + }, + }); + const result = await createAgentSessionFromServices({ + services, + sessionManager: runtimeOptions.sessionManager, + sessionStartEvent: runtimeOptions.sessionStartEvent, + ...runtimeOptions.sessionOptions, + }); + return { ...result, services, diagnostics: services.diagnostics }; + }; + return createAgentSessionRuntime(createRuntime, { + cwd: directory, + agentDir: directory, + sessionManager: SessionManager.create(directory, join(directory, "sessions")), + sessionOptions: { + model: rootModel, + noTools: "all", + includeGoals: false, + rlmDepth: 0, + rlmMaxDepth: 1, + }, + }); +} + +function requestIds(fanout: number, offset = 10): string[] { + return Array.from({ length: fanout }, (_, index) => `request-${String(index + offset).padStart(4, "0")}`); +} + +async function waitForTerminals(fixture: ReturnType, count: number): Promise { + await vi.waitFor( + () => + expect(fixture.observations().filter((observation) => observation.eventKinds.length > 0)).toHaveLength(count), + { timeout: 10_000, interval: 10 }, + ); +} + describe("B00B production scripted provider", () => { test("registers through the real AI registry and holds a 1/4 fanout only as an observation barrier", async () => { const ids = ["request-0001", "request-0002", "request-0003", "request-0004"] as const; @@ -251,6 +348,189 @@ describe("B00B production scripted provider", () => { expect(observed.filter((item) => item.requestId === ids[0])[0]?.eventKinds).toHaveLength(1); }); + test.each([1, 4, 16, 64])( + "admits a real RLM fanout of %i children before the provider barrier opens", + async (fanout) => { + const ids = requestIds(fanout, fanout === 1 ? 20 : fanout * 100); + const fixture = provider( + Object.fromEntries(ids.map((id) => [id, [simple(id, { waitForRelease: true })]])), + ids, + ); + const directory = await mkdtemp(join(tmpdir(), `b00b-rlm-${fanout}-`)); + const runtime = await runtimeForRlmFixture(fixture, directory); + cleanups.push(async () => { + await runtime.dispose(); + await rm(directory, { recursive: true, force: true }); + }); + + const handles = await Promise.all( + ids.map((id, index) => + runtime.session.runRlmChild(id, { + name: `worker-${String(index + 1).padStart(4, "0")}`, + model: `${fixture.models[index % fixture.models.length]!.provider}/${fixture.models[index % fixture.models.length]!.id}`, + }), + ), + ); + // Admission is detached: every handle returns before an entry is allowed + // to leave its observation latch. This is deliberately not a fanout + // semaphore or permit queue. + expect(handles).toHaveLength(fanout); + expect(new Set(handles.map((handle) => handle.rlm_child_id)).size).toBe(fanout); + await fixture.open; + const entered = fixture.observations(); + expect(entered).toHaveLength(fanout); + expect(entered.map((entry) => entry.requestId).sort()).toEqual([...ids].sort()); + expect(entered.every((entry) => entry.eventKinds.length === 0 && entry.attempt === 1)).toBe(true); + expect(entered.map((entry) => entry.sequence)).toEqual( + Array.from({ length: fanout }, (_, index) => index + 1), + ); + + // Fast siblings complete while the held first request has not emitted a + // provider event. This proves the latch observes independently admitted + // streams rather than serializing their execution. + fixture.release(ids.slice(1)); + if (fanout > 1) await waitForTerminals(fixture, fanout - 1); + expect(fixture.observations().find((entry) => entry.requestId === ids[0])?.eventKinds).toEqual([]); + fixture.release([ids[0]!]); + await waitForTerminals(fixture, fanout); + expect(fixture.observations().every((entry) => entry.terminal === "done")).toBe(true); + }, + 20_000, + ); + + test("uses the RLM runtime child host for cancel, real scripted 429, and sibling isolation", async () => { + const ids = ["request-0701", "request-0702", "request-0703"] as const; + const fixture = provider( + { + [ids[0]]: [simple(ids[0], { waitForRelease: true })], + [ids[1]]: [ + { + requestId: ids[1], + waitForRelease: true, + upstreamStatus: 429, + errorCode: "upstream-429", + usage: usage(23, 0, 5, 0), + }, + ], + [ids[2]]: [ + simple(ids[2], { waitForRelease: true, responseModel: "fixture-zero-resolved", usage: usage(3, 2) }), + ], + }, + ids, + ); + const directory = await mkdtemp(join(tmpdir(), "b00b-rlm-isolation-")); + const runtime = await runtimeForRlmFixture(fixture, directory); + cleanups.push(async () => { + await runtime.dispose(); + await rm(directory, { recursive: true, force: true }); + }); + const handles = await Promise.all( + ids.map((id, index) => + runtime.session.runRlmChild(id, { + name: `worker-${String(index + 701).padStart(4, "0")}`, + model: `${fixture.models[index]!.provider}/${fixture.models[index]!.id}`, + }), + ), + ); + await fixture.open; + expect(fixture.observations().every((entry) => entry.eventKinds.length === 0)).toBe(true); + await vi.waitFor(() => expect(runtime.session.getRlmChildSession(handles[0]!.rlm_child_id)).toBeDefined()); + expect(runtime.session.cancelRlmChildRun(handles[0]!.rlm_child_id, "B00B_CANCELLED")).toBe(true); + fixture.release([ids[1], ids[2]]); + await waitForTerminals(fixture, 3); + const observed = fixture.observations(); + expect(observed.find((entry) => entry.requestId === ids[0])).toMatchObject({ + terminal: "aborted", + signalAborted: true, + eventKinds: ["error"], + }); + expect(observed.find((entry) => entry.requestId === ids[1])).toMatchObject({ + upstreamStatus: 429, + terminal: "error", + eventKinds: ["error"], + }); + expect(observed.find((entry) => entry.requestId === ids[2])).toMatchObject({ + terminal: "done", + responseModel: "fixture-zero-resolved", + }); + // The fixture’s disabled retry setting is an explicit per-child policy: + // no synthetic local 429 and no shared client-side limiter intervene. + expect(observed.filter((entry) => entry.requestId === ids[1])).toHaveLength(1); + }); + + test("projects immutable real RLM observations into signed B00A evidence and verifies in a fresh process", async () => { + const id = "request-0801"; + const fixture = provider( + { + [id]: [ + simple(id, { waitForRelease: true, responseModel: "fixture-b-resolved", usage: usage(101, 13, 7, 3) }), + ], + }, + [id], + ); + const directory = await mkdtemp(join(tmpdir(), "b00b-rlm-evidence-")); + const artifactDirectory = await mkdtemp(join(tmpdir(), "b00b-rlm-artifact-")); + const trustDirectory = await mkdtemp(join(tmpdir(), "b00b-rlm-trust-")); + const runtime = await runtimeForRlmFixture(fixture, directory); + cleanups.push(async () => { + await runtime.dispose(); + await rm(directory, { recursive: true, force: true }); + await rm(artifactDirectory, { recursive: true, force: true }); + await rm(trustDirectory, { recursive: true, force: true }); + }); + const [handle] = await Promise.all([ + runtime.session.runRlmChild(id, { name: "worker-0801", model: `${fixture.models[1]!.provider}/fixture-b` }), + ]); + expect(handle).toMatchObject({ model: `${fixture.models[1]!.provider}/fixture-b` }); + await fixture.open; + fixture.release([id]); + await waitForTerminals(fixture, 1); + const observation = fixture.observations()[0]!; + expect(observation.requested).toMatchObject({ provider: "b00b-scripted", model: "fixture-b" }); + expect(observation.responseModel).toBe("fixture-b-resolved"); + expect(observation.usage).toMatchObject({ input: 101, output: 13, cacheRead: 7, cacheWrite: 3 }); + const keys = generateKeyPairSync("ed25519"); + const written = await writeSignedProductionEvidence( + artifactDirectory, + trustDirectory, + { + scenario: "rlm-real-path", + priceCard: { + version: "fixture-price-card-v1", + inputMicroCurrencyPerMillionMicroTokens: 17, + outputMicroCurrencyPerMillionMicroTokens: 29, + }, + attempts: [ + { + requestId: observation.requestId as `request-${string}`, + attempt: observation.attempt, + requested: { provider: observation.requested.provider, model: observation.requested.model }, + resolved: { + api: "b00b-scripted", + provider: observation.requested.provider, + model: observation.requested.model, + responseModel: observation.responseModel!, + }, + terminal: observation.terminal, + usage: { + inputMicroTokens: observation.usage!.input, + outputMicroTokens: observation.usage!.output, + cacheReadMicroTokens: observation.usage!.cacheRead, + cacheWriteMicroTokens: observation.usage!.cacheWrite, + }, + }, + ], + }, + keys.privateKey, + ); + await verifySignedProductionEvidenceFreshProcess( + artifactDirectory, + written.commitmentPath, + keys.publicKey.export({ type: "spki", format: "pem" }).toString(), + ); + expect(written.artifactBundleId).toMatch(/^[a-f0-9]{64}$/); + }); + test("runs through AgentSession.promptAndWait with a registered provider and writes no canary or network fixture", async () => { const id = "request-0009"; const fixture = provider({ [id]: [simple(id, { waitForRelease: true, responseModel: "fixture-a-resolved" })] }, [ From 317fc04c296fed4c6b7ae381d6c87acababe08e0 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 20:01:00 -0700 Subject: [PATCH 06/28] test: cover daemon production dispatch backpressure (cherry picked from commit d7d5dfd630a0bf35620c63847e2d27b21e029ec4) --- .../swarm/daemon-production-dispatch.test.ts | 351 ++++++++++++++++++ 1 file changed, 351 insertions(+) create mode 100644 packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts diff --git a/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts b/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts new file mode 100644 index 000000000..5907efb65 --- /dev/null +++ b/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts @@ -0,0 +1,351 @@ +/** + * Real supervisor/worker dispatch coverage. The HTTP fixture is deliberately + * local: workers use the production OpenAI-completions transport, while the + * test observes request entry without installing a provider in either worker. + */ +import { type ChildProcess, spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:http"; +import { createConnection } from "node:net"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { afterEach, describe, expect, test } from "vitest"; +import { ENV_AGENT_DIR } from "../../src/config.js"; +import { DaemonClient } from "../../src/modes/daemon/daemon-client.js"; +import type { DaemonEventCursor, DaemonOutbound, DaemonResponse } from "../../src/modes/daemon/daemon-protocol.js"; +import type { SessionSummary } from "../../src/modes/daemon/daemon-session-list.js"; + +const cliPath = resolve(__dirname, "../../src/cli.ts"); +const tsxPath = resolve(__dirname, "../../../../node_modules/tsx/dist/cli.mjs"); +const resources: Array<() => Promise | void> = []; + +afterEach(async () => { + while (resources.length) await resources.pop()?.(); +}); + +function pause(ms: number): Promise { + return new Promise((resolveDelay) => setTimeout(resolveDelay, ms)); +} + +async function eventually(predicate: () => boolean, code: string, timeoutMs = 15_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return; + await pause(20); + } + throw new Error(code); +} + +function summary(value: unknown): SessionSummary { + if (!value || typeof value !== "object") throw new Error("B00B_MISSING_SESSION_SUMMARY"); + return value as SessionSummary; +} + +function active(summaryValue: SessionSummary): string { + return summaryValue.activeSessionId ?? summaryValue.id; +} + +function requestId(body: string): string { + const match = /request-\d{4}/.exec(body); + if (!match) throw new Error("B00B_LOCAL_FIXTURE_MISSING_REQUEST_ID"); + return match[0]; +} + +interface LocalProvider { + readonly url: string; + readonly entered: readonly string[]; + close(): Promise; +} + +/** + * This is a real HTTP/SSE upstream from the worker's perspective. It is not a + * model limiter: each POST enters immediately and receives its own scripted + * response. 429 is an actual upstream HTTP response, never a local result. + */ +async function localProvider(): Promise { + const entered: string[] = []; + const server = createServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk: string) => { + body += chunk; + }); + request.on("end", () => { + let id: string; + try { + id = requestId(body); + } catch { + response.writeHead(400).end("B00B_BAD_LOCAL_REQUEST"); + return; + } + entered.push(id); + if (id === "request-0003") { + response.writeHead(429, { "content-type": "application/json", "retry-after": "0" }); + response.end(JSON.stringify({ error: { message: "fixture upstream 429", type: "rate_limit_error" } })); + return; + } + const emitSuccess = () => { + if (request.destroyed || response.destroyed) return; + response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + // The large body deliberately exceeds a socket high-water mark. A paused + // attachment must be resynced instead of accumulating unbounded events. + const content = id === "request-0001" ? `${"x".repeat(512 * 1024)} fast-tail` : "cancelled-root-content"; + const event = (value: unknown) => response.write(`data: ${JSON.stringify(value)}\n\n`); + event({ + id: `fixture-${id}`, + model: "fixture-resolved", + choices: [{ index: 0, delta: { role: "assistant", content }, finish_reason: null }], + }); + event({ + id: `fixture-${id}`, + model: "fixture-resolved", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 7, completion_tokens: 11, total_tokens: 18 }, + }); + response.end("data: [DONE]\n\n"); + }; + // Keep this upstream request in flight until its root's real abort signal + // closes the transport; no sibling shares this timer. + if (id === "request-0002") setTimeout(emitSuccess, 2_000); + else emitSuccess(); + }); + }); + await new Promise((resolveListen) => server.listen(0, "127.0.0.1", resolveListen)); + const address = server.address(); + if (!address || typeof address === "string") throw new Error("B00B_LOCAL_FIXTURE_NO_PORT"); + return { + url: `http://127.0.0.1:${address.port}/v1`, + entered, + close: () => new Promise((resolveClose) => server.close(() => resolveClose())), + }; +} + +function spawnSupervisor(agentDir: string, socketPath: string, cwd: string): ChildProcess { + const child = spawn(process.execPath, [tsxPath, cliPath, "--mode", "daemon", "--daemon-socket", socketPath], { + cwd, + env: { + ...process.env, + [ENV_AGENT_DIR]: agentDir, + TSX_TSCONFIG_PATH: resolve(__dirname, "../../../../tsconfig.json"), + PRIME_AGENT_INTERNAL_DAEMON_WORKER: undefined, + PRIME_AGENT_INTERNAL_DAEMON_WORKER_TOKEN: undefined, + PRIME_AGENT_INTERNAL_DAEMON_WORKER_ACTIVE_SESSION_ID: undefined, + PRIME_AGENT_INTERNAL_DAEMON_SUPERVISOR_SOCKET: undefined, + PRIME_AGENT_INTERNAL_DAEMON_WORKER_RECOVERY_JOURNAL: undefined, + PRIME_AGENT_INTERNAL_DAEMON_WORKER_STARTUP_GATE_FD: undefined, + PRIME_AGENT_INTERNAL_SESSION_LEASES_ENABLED: undefined, + PRIME_AGENT_INTERNAL_SESSION_LEASE_OWNER_ID: undefined, + }, + stdio: ["ignore", "ignore", "pipe"], + }); + let stderr = ""; + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString("utf8"); + }); + Object.assign(child, { b00bStderr: () => stderr }); + resources.push(() => { + if (child.exitCode === null) child.kill("SIGTERM"); + }); + return child; +} + +async function connect(socketPath: string, child: ChildProcess): Promise { + const deadline = Date.now() + 15_000; + let lastError = ""; + while (Date.now() < deadline) { + if (child.exitCode !== null) throw new Error("B00B_SUPERVISOR_EXITED"); + const client = new DaemonClient(socketPath); + try { + await client.connect(200); + await client.waitForHello(1_000); + return client; + } catch (error) { + lastError = String(error); + client.close(); + await pause(25); + } + } + throw new Error( + `B00B_SUPERVISOR_CONNECT_TIMEOUT ${lastError} ${(child as ChildProcess & { b00bStderr?: () => string }).b00bStderr?.() ?? ""}`, + ); +} + +async function attachThenPause(socketPath: string, activeSessionId: string): Promise<{ cursor: DaemonEventCursor }> { + const socket = createConnection(socketPath); + resources.push(() => { + socket.destroy(); + }); + const first = await new Promise((resolveLine, rejectLine) => { + let buffered = ""; + const timeout = setTimeout(() => rejectLine(new Error("B00B_BLOCKED_ATTACH_TIMEOUT")), 5_000); + socket.on("error", rejectLine); + socket.on("data", (chunk: Buffer) => { + buffered += chunk.toString("utf8"); + const newline = buffered.indexOf("\n"); + if (newline < 0) return; + const line = buffered.slice(0, newline); + buffered = buffered.slice(newline + 1); + const decoded = JSON.parse(line) as DaemonResponse | { type: "daemon_hello" }; + if (decoded.type === "daemon_hello") return; + clearTimeout(timeout); + socket.pause(); // Known cursor reached. Do not drain this attachment. + resolveLine(decoded); + }); + socket.on("connect", () => { + socket.write( + `${JSON.stringify({ + type: "command", + id: "blocked-attach", + clientId: "b00b-blocked", + protocol: { name: "prime-agent.daemon", version: 7 }, + command: { + type: "attach", + activeSessionId, + capabilities: ["attach_snapshot", "event_sequence"], + }, + })}\n`, + ); + }); + }); + if (!first.success || !first.data || typeof first.data !== "object") + throw new Error(`B00B_BLOCKED_ATTACH_FAILED ${JSON.stringify(first)}`); + const cursor = (first.data as { lastEventCursor?: DaemonEventCursor }).lastEventCursor; + if (!cursor) throw new Error("B00B_BLOCKED_ATTACH_NO_CURSOR"); + return { cursor }; +} + +describe("B00B real daemon production dispatch", () => { + test("isolates paused attachment, cancellation, and upstream 429 across real supervisor workers", async () => { + const root = mkdtempSync(join(tmpdir(), "b00b-daemon-")); + resources.push(async () => { + for (let attempt = 0; attempt < 20; attempt++) { + try { + rmSync(root, { recursive: true, force: true, maxRetries: 1, retryDelay: 25 }); + return; + } catch { + await pause(50); + } + } + throw new Error("B00B_TEMP_CLEANUP_FAILED"); + }); + const agentDir = join(root, "agent"); + const projectDir = join(root, "project"); + const socketPath = join(tmpdir(), `b00b-${process.pid}-${randomUUID().slice(0, 8)}.sock`); + mkdirSync(agentDir, { recursive: true }); + mkdirSync(projectDir, { recursive: true }); + const upstream = await localProvider(); + resources.push(() => upstream.close()); + writeFileSync( + join(agentDir, "models.json"), + JSON.stringify({ + providers: { + "b00b-local": { + baseUrl: upstream.url, + apiKey: "fixture-key", + api: "openai-completions", + models: [{ id: "fixture-a", api: "openai-completions", reasoning: false, input: ["text"] }], + }, + }, + }), + ); + const supervisor = spawnSupervisor(agentDir, socketPath, projectDir); + const control = await connect(socketPath, supervisor); + resources.push(() => control.close()); + const create = async (name: string) => { + const result = await control.request({ + type: "create", + name, + config: { + cwd: projectDir, + agentDir, + provider: "b00b-local", + model: "fixture-a", + noTools: true, + noExtensions: true, + noSkills: true, + }, + }); + if (!result.success) throw new Error("B00B_CREATE_ROOT_FAILED"); + return summary(result.data); + }; + const fast = await create("fast-root"); + const cancelled = await create("cancelled-root"); + const rateLimited = await create("rate-limited-root"); + expect(new Set([fast.workerPid, cancelled.workerPid, rateLimited.workerPid]).size).toBe(3); + + const blocked = await attachThenPause(socketPath, active(fast)); + const draining = await connect(socketPath, supervisor); + resources.push(() => draining.close()); + const drainEvents: Extract[] = []; + draining.onMessage((message) => { + if (message.type === "session_event" && message.activeSessionId === active(fast)) drainEvents.push(message); + }); + const attached = await draining.request({ + type: "attach", + activeSessionId: active(fast), + capabilities: ["attach_snapshot", "event_sequence"], + }); + expect(attached.success).toBe(true); + + const dispatch = (session: SessionSummary, id: string) => + control.request({ type: "prompt", activeSessionId: active(session), message: id }, 10_000); + const admissions = await Promise.all([ + dispatch(fast, "request-0001"), + dispatch(cancelled, "request-0002"), + dispatch(rateLimited, "request-0003"), + ]); + expect(admissions.every((item) => item.success)).toBe(true); + await eventually(() => new Set(upstream.entered).size === 3, "B00B_PROVIDER_ENTRY_TIMEOUT"); + expect(upstream.entered.filter((id) => id === "request-0001")).toHaveLength(1); + expect(upstream.entered.filter((id) => id === "request-0002")).toHaveLength(1); + // Existing per-request retry behavior may re-enter only the upstream-429 root. + expect(upstream.entered.filter((id) => id === "request-0003").length).toBeGreaterThanOrEqual(1); + + // Abort and HTTP 429 are root-local. They cannot prevent the draining + // attachment's independent root from reaching a normal terminal. + const aborted = await control.request({ type: "abort", activeSessionId: active(cancelled) }); + expect(aborted.success).toBe(true); + const idle = await control.request({ type: "wait_for_idle", activeSessionId: active(fast) }, 30_000); + expect(idle.success).toBe(true); + await eventually( + () => drainEvents.some((event) => event.type === "session_event" && event.event.type === "message_end"), + "B00B_DRAINING_ATTACHMENT_DID_NOT_COMPLETE", + ); + const ordered = drainEvents + .map((event) => event.meta?.sequence) + .filter((sequence): sequence is number => sequence !== undefined); + expect(ordered).toEqual([...ordered].sort((left, right) => left - right)); + expect(ordered.length).toBeGreaterThan(2); + + // Reattach from the cursor known before the paused write. The supervisor + // supplies a bounded snapshot/replay rather than a per-attachment model queue. + const catchup = await connect(socketPath, supervisor); + resources.push(() => catchup.close()); + const resynced = await catchup.request({ + type: "attach", + activeSessionId: active(fast), + capabilities: ["attach_snapshot", "event_sequence"], + resumeCursor: { activeSessionId: active(fast), ...blocked.cursor }, + }); + if (!resynced.success || !resynced.data || typeof resynced.data !== "object") + throw new Error("B00B_CATCHUP_FAILED"); + const catchupData = resynced.data as { snapshot?: { messages?: unknown[] }; replay?: { toSequence?: number } }; + expect(catchupData.snapshot?.messages?.length).toBeGreaterThanOrEqual(2); + expect(catchupData.replay?.toSequence).toBeGreaterThanOrEqual(blocked.cursor.sequence); + + const cancelledIdle = await control.request( + { type: "wait_for_idle", activeSessionId: active(cancelled) }, + 30_000, + ); + expect(cancelledIdle.success).toBe(true); + const cancelledMessages = await control.request({ type: "get_messages", activeSessionId: active(cancelled) }); + expect(JSON.stringify(cancelledMessages)).not.toContain("cancelled-root-content"); + // The provider saw a genuine status-429 request while fast completed; + // no test code implements a permit, semaphore, or fabricated response. + expect(upstream.entered).toContain("request-0003"); + + const shutdown = await control.request({ type: "shutdown" }, 10_000); + expect(shutdown.success).toBe(true); + }, 60_000); +}); From 441d1cadd867077150c2c21a48aa7c171966d9d1 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 20:36:36 -0700 Subject: [PATCH 07/28] test: harden daemon production dispatch coverage (cherry picked from commit d279037bba89be299d0ba8a63893f963a6df6157) --- packages/coding-agent/package.json | 2 +- .../swarm/daemon-production-dispatch.test.ts | 104 ++++++++++++++++-- 2 files changed, 98 insertions(+), 8 deletions(-) diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index bb0570e40..6555f0f9f 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -40,7 +40,7 @@ "test": "vitest --run", "test:ci": "tsx src/core/kernel/bootstrap-cli.ts && vitest --run --exclude test/daemon-supervisor-process.test.ts", "test:process": "vitest --run test/daemon-supervisor-process.test.ts", - "test:process-stress": "vitest --run --tagsFilter process-stress test/daemon-supervisor-process.test.ts", + "test:process-stress": "env -u PRIME_AGENT_INTERNAL_DAEMON_WORKER -u PRIME_AGENT_INTERNAL_DAEMON_WORKER_TOKEN -u PRIME_AGENT_INTERNAL_DAEMON_WORKER_ACTIVE_SESSION_ID -u PRIME_AGENT_INTERNAL_DAEMON_SUPERVISOR_SOCKET -u PRIME_AGENT_INTERNAL_DAEMON_WORKER_RECOVERY_JOURNAL -u PRIME_AGENT_INTERNAL_DAEMON_WORKER_STARTUP_GATE_FD -u PRIME_AGENT_INTERNAL_ORPHAN_PROCESS_JOURNAL -u PRIME_AGENT_INTERNAL_SESSION_LEASES -u PRIME_AGENT_INTERNAL_SESSION_LEASE_OWNER_ID vitest --run --tagsFilter process-stress test/daemon-supervisor-process.test.ts", "postinstall": "node postinstall.cjs", "prepublishOnly": "npm run clean && npm run build", "bundle": "node scripts/bundle.mjs", diff --git a/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts b/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts index 5907efb65..e6c655494 100644 --- a/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts +++ b/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts @@ -5,7 +5,16 @@ */ import { type ChildProcess, spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; -import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; import { createServer } from "node:http"; import { createConnection } from "node:net"; import { tmpdir } from "node:os"; @@ -37,6 +46,43 @@ async function eventually(predicate: () => boolean, code: string, timeoutMs = 15 throw new Error(code); } +async function waitForProcessGone(pid: number): Promise { + await eventually(() => { + try { + process.kill(pid, 0); + return false; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ESRCH"; + } + }, `B00B_WORKER_${pid}_SURVIVED`); +} + +function recursiveFiles(directory: string): string[] { + if (!existsSync(directory)) return []; + const result: string[] = []; + for (const entry of readdirSync(directory)) { + const path = join(directory, entry); + try { + if (lstatSync(path).isDirectory()) result.push(...recursiveFiles(path)); + else result.push(path); + } catch { + // Supervisor cleanup may atomically rename/remove a descriptor mid-scan. + } + } + return result; +} + +function assertNoFixtureKey(texts: readonly string[], key: string): void { + const normalizedKey = key.replace(/[^a-zA-Z0-9]/g, "").toLowerCase(); + for (const text of texts) { + const decoded = text.replace(/\\u([\dA-Fa-f]{4})/g, (_, hex: string) => + String.fromCharCode(Number.parseInt(hex, 16)), + ); + expect(decoded).not.toContain(key); + expect(decoded.replace(/[^a-zA-Z0-9]/g, "").toLowerCase()).not.toContain(normalizedKey); + } +} + function summary(value: unknown): SessionSummary { if (!value || typeof value !== "object") throw new Error("B00B_MISSING_SESSION_SUMMARY"); return value as SessionSummary; @@ -55,6 +101,7 @@ function requestId(body: string): string { interface LocalProvider { readonly url: string; readonly entered: readonly string[]; + readonly maxInFlight: number; close(): Promise; } @@ -65,6 +112,8 @@ interface LocalProvider { */ async function localProvider(): Promise { const entered: string[] = []; + let inFlight = 0; + let maxInFlight = 0; const server = createServer((request, response) => { let body = ""; request.setEncoding("utf8"); @@ -79,10 +128,15 @@ async function localProvider(): Promise { response.writeHead(400).end("B00B_BAD_LOCAL_REQUEST"); return; } + entered.push(id); - if (id === "request-0003") { + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + const attempt = entered.filter((entry) => entry === id).length; + if (id === "request-0003" && attempt === 1) { response.writeHead(429, { "content-type": "application/json", "retry-after": "0" }); response.end(JSON.stringify({ error: { message: "fixture upstream 429", type: "rate_limit_error" } })); + inFlight -= 1; return; } const emitSuccess = () => { @@ -103,7 +157,9 @@ async function localProvider(): Promise { choices: [{ index: 0, delta: {}, finish_reason: "stop" }], usage: { prompt_tokens: 7, completion_tokens: 11, total_tokens: 18 }, }); + response.end("data: [DONE]\n\n"); + inFlight -= 1; }; // Keep this upstream request in flight until its root's real abort signal // closes the transport; no sibling shares this timer. @@ -113,10 +169,15 @@ async function localProvider(): Promise { }); await new Promise((resolveListen) => server.listen(0, "127.0.0.1", resolveListen)); const address = server.address(); - if (!address || typeof address === "string") throw new Error("B00B_LOCAL_FIXTURE_NO_PORT"); + + if (!address || typeof address === "string" || address.address !== "127.0.0.1") + throw new Error("B00B_LOCAL_FIXTURE_NOT_LOOPBACK_ONLY"); return { url: `http://127.0.0.1:${address.port}/v1`, entered, + get maxInFlight() { + return maxInFlight; + }, close: () => new Promise((resolveClose) => server.close(() => resolveClose())), }; } @@ -242,7 +303,7 @@ describe("B00B real daemon production dispatch", () => { providers: { "b00b-local": { baseUrl: upstream.url, - apiKey: "fixture-key", + apiKey: "fixture-key-B00B-canary", api: "openai-completions", models: [{ id: "fixture-a", api: "openai-completions", reasoning: false, input: ["text"] }], }, @@ -272,7 +333,11 @@ describe("B00B real daemon production dispatch", () => { const fast = await create("fast-root"); const cancelled = await create("cancelled-root"); const rateLimited = await create("rate-limited-root"); - expect(new Set([fast.workerPid, cancelled.workerPid, rateLimited.workerPid]).size).toBe(3); + + const workerPids = [fast.workerPid, cancelled.workerPid, rateLimited.workerPid]; + expect(workerPids.every((pid): pid is number => typeof pid === "number" && pid > 1)).toBe(true); + const concreteWorkerPids = workerPids as number[]; + expect(new Set(concreteWorkerPids).size).toBe(3); const blocked = await attachThenPause(socketPath, active(fast)); const draining = await connect(socketPath, supervisor); @@ -298,9 +363,11 @@ describe("B00B real daemon production dispatch", () => { expect(admissions.every((item) => item.success)).toBe(true); await eventually(() => new Set(upstream.entered).size === 3, "B00B_PROVIDER_ENTRY_TIMEOUT"); expect(upstream.entered.filter((id) => id === "request-0001")).toHaveLength(1); + expect(upstream.entered.filter((id) => id === "request-0002")).toHaveLength(1); - // Existing per-request retry behavior may re-enter only the upstream-429 root. - expect(upstream.entered.filter((id) => id === "request-0003").length).toBeGreaterThanOrEqual(1); + + // All three independently created workers enter the real upstream before + // cancellation; a provider-side barrier is not faked by the client. // Abort and HTTP 429 are root-local. They cannot prevent the draining // attachment's independent root from reaching a normal terminal. @@ -334,6 +401,19 @@ describe("B00B real daemon production dispatch", () => { expect(catchupData.snapshot?.messages?.length).toBeGreaterThanOrEqual(2); expect(catchupData.replay?.toSequence).toBeGreaterThanOrEqual(blocked.cursor.sequence); + await eventually( + () => upstream.entered.filter((id) => id === "request-0003").length === 2, + "B00B_429_RETRY_TIMEOUT", + ); + const rateLimitedIdle = await control.request( + { type: "wait_for_idle", activeSessionId: active(rateLimited) }, + 30_000, + ); + expect(rateLimitedIdle.success).toBe(true); + const rateLimitedMessages = await control.request({ type: "get_messages", activeSessionId: active(rateLimited) }); + expect(JSON.stringify(rateLimitedMessages)).toContain("fixture-resolved"); + expect(upstream.entered.filter((id) => id === "request-0003")).toHaveLength(2); + const cancelledIdle = await control.request( { type: "wait_for_idle", activeSessionId: active(cancelled) }, 30_000, @@ -345,7 +425,17 @@ describe("B00B real daemon production dispatch", () => { // no test code implements a permit, semaphore, or fabricated response. expect(upstream.entered).toContain("request-0003"); + const capturedTexts = [ + (supervisor as ChildProcess & { b00bStderr?: () => string }).b00bStderr?.() ?? "", + ...recursiveFiles(agentDir) + .filter((path) => path !== join(agentDir, "models.json")) + .map((path) => readFileSync(path, "utf8")), + ]; + assertNoFixtureKey(capturedTexts, "fixture-key-B00B-canary"); const shutdown = await control.request({ type: "shutdown" }, 10_000); expect(shutdown.success).toBe(true); + await Promise.all(concreteWorkerPids.map((pid) => waitForProcessGone(pid))); + await eventually(() => !existsSync(socketPath), "B00B_SUPERVISOR_SOCKET_SURVIVED"); + expect(recursiveFiles(join(agentDir, "daemon-workers")).filter((path) => path.endsWith(".tmp"))).toEqual([]); }, 60_000); }); From 37c23c08a5701ae942f1ee356cb6392b3a17b515 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 21:08:48 -0700 Subject: [PATCH 08/28] test: harden B00B daemon dispatch fixture (cherry picked from commit 9c66fccce4b5a1b265c621cd165d7f6e238d6932) --- .../swarm/daemon-production-dispatch.test.ts | 556 +++++++++++------- 1 file changed, 345 insertions(+), 211 deletions(-) diff --git a/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts b/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts index e6c655494..3ea971dd7 100644 --- a/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts +++ b/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts @@ -3,7 +3,7 @@ * local: workers use the production OpenAI-completions transport, while the * test observes request entry without installing a provider in either worker. */ -import { type ChildProcess, spawn } from "node:child_process"; +import { type ChildProcess, execFileSync, spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; import { existsSync, @@ -16,7 +16,7 @@ import { writeFileSync, } from "node:fs"; import { createServer } from "node:http"; -import { createConnection } from "node:net"; +import { createConnection, type Socket } from "node:net"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { afterEach, describe, expect, test } from "vitest"; @@ -28,6 +28,7 @@ import type { SessionSummary } from "../../src/modes/daemon/daemon-session-list. const cliPath = resolve(__dirname, "../../src/cli.ts"); const tsxPath = resolve(__dirname, "../../../../node_modules/tsx/dist/cli.mjs"); const resources: Array<() => Promise | void> = []; +const completedRoots: string[] = []; afterEach(async () => { while (resources.length) await resources.pop()?.(); @@ -57,32 +58,97 @@ async function waitForProcessGone(pid: number): Promise { }, `B00B_WORKER_${pid}_SURVIVED`); } -function recursiveFiles(directory: string): string[] { +function recursiveNormalFiles(directory: string): string[] { if (!existsSync(directory)) return []; const result: string[] = []; for (const entry of readdirSync(directory)) { const path = join(directory, entry); try { - if (lstatSync(path).isDirectory()) result.push(...recursiveFiles(path)); - else result.push(path); + const stat = lstatSync(path); + if (stat.isDirectory()) result.push(...recursiveNormalFiles(path)); + else if (stat.isFile()) result.push(path); } catch { - // Supervisor cleanup may atomically rename/remove a descriptor mid-scan. + // Cleanup can atomically rename/remove an entry while an assertion scans it. } } return result; } +function recursivePaths(directory: string): string[] { + if (!existsSync(directory)) return []; + const result: string[] = []; + for (const entry of readdirSync(directory)) { + const path = join(directory, entry); + result.push(path); + try { + if (lstatSync(path).isDirectory()) result.push(...recursivePaths(path)); + } catch { + // See recursiveNormalFiles. + } + } + return result; +} + +async function removeTempRoot(root: string): Promise { + for (let attempt = 0; attempt < 20; attempt++) { + try { + rmSync(root, { recursive: true, force: true, maxRetries: 1, retryDelay: 25 }); + return; + } catch { + await pause(50); + } + } + throw new Error("B00B_TEMP_CLEANUP_FAILED"); +} +function cwdPidsUnder(roots: readonly string[]): number[] { + let output: string; + try { + output = execFileSync("lsof", ["-n", "-Fpn", "-a", "-d", "cwd"], { encoding: "utf8" }); + } catch (error) { + const status = (error as { status?: unknown }).status; + if (status === 1) return []; + throw error; + } + let pid: number | undefined; + const matching = new Set(); + for (const line of output.split("\n")) { + if (line.startsWith("p")) pid = Number.parseInt(line.slice(1), 10); + if (!line.startsWith("n") || pid === undefined) continue; + const cwd = line.slice(1).replace(/\s+\(deleted\)$/, ""); + if (roots.some((root) => cwd === root || cwd.startsWith(`${root}/`))) matching.add(pid); + } + return [...matching]; +} + +function assertNoRunResidue(roots: readonly string[]): void { + expect(roots.every((root) => !existsSync(root))).toBe(true); + expect(cwdPidsUnder(roots)).toEqual([]); + const runNames = new Set(roots.map((root) => root.slice(root.lastIndexOf("/") + 1))); + expect(readdirSync(tmpdir()).filter((entry) => runNames.has(entry))).toEqual([]); +} + function assertNoFixtureKey(texts: readonly string[], key: string): void { const normalizedKey = key.replace(/[^a-zA-Z0-9]/g, "").toLowerCase(); + const escapedKey = [...key] + .map((character) => `\\u${character.charCodeAt(0).toString(16).padStart(4, "0")}`) + .join(""); for (const text of texts) { - const decoded = text.replace(/\\u([\dA-Fa-f]{4})/g, (_, hex: string) => - String.fromCharCode(Number.parseInt(hex, 16)), - ); + const decoded = text + .replace(/\\u\{([\dA-Fa-f]+)\}/g, (_, hex: string) => String.fromCodePoint(Number.parseInt(hex, 16))) + .replace(/\\u([\dA-Fa-f]{4})/g, (_, hex: string) => String.fromCharCode(Number.parseInt(hex, 16))) + .replace(/\\x([\dA-Fa-f]{2})/g, (_, hex: string) => String.fromCharCode(Number.parseInt(hex, 16))); + const normalized = decoded.replace(/[^a-zA-Z0-9]/g, "").toLowerCase(); + expect(text).not.toContain(key); + expect(text).not.toContain(escapedKey); expect(decoded).not.toContain(key); - expect(decoded.replace(/[^a-zA-Z0-9]/g, "").toLowerCase()).not.toContain(normalizedKey); + expect(normalized).not.toContain(normalizedKey); + // Catch a key serialized as fragments with punctuation/whitespace between characters. + const splitKey = [...key] + .map((character) => character.replace(/[\^$.*+?()[\]{}|]/g, "\\$&")) + .join("[^a-zA-Z0-9]*"); + expect(decoded).not.toMatch(new RegExp(splitKey, "i")); } } - function summary(value: unknown): SessionSummary { if (!value || typeof value !== "object") throw new Error("B00B_MISSING_SESSION_SUMMARY"); return value as SessionSummary; @@ -102,6 +168,7 @@ interface LocalProvider { readonly url: string; readonly entered: readonly string[]; readonly maxInFlight: number; + release(ids: readonly string[]): void; close(): Promise; } @@ -110,10 +177,32 @@ interface LocalProvider { * model limiter: each POST enters immediately and receives its own scripted * response. 429 is an actual upstream HTTP response, never a local result. */ -async function localProvider(): Promise { +async function localProvider(canary: string): Promise { const entered: string[] = []; let inFlight = 0; let maxInFlight = 0; + const released = new Set(); + const releaseWaiters = new Set<() => void>(); + const sockets = new Set(); + let closePromise: Promise | undefined; + const waitForRelease = (id: string, response: import("node:http").ServerResponse): Promise => + new Promise((resolveRelease) => { + if (released.has(id)) { + resolveRelease(true); + return; + } + const release = () => finish(true); + const closed = () => finish(false); + const finish = (wasReleased: boolean) => { + releaseWaiters.delete(release); + response.off("close", closed); + resolveRelease(wasReleased); + }; + releaseWaiters.add(release); + // Worker cancellation destroys the response; it never relies on request.destroyed, + // which can be true after an otherwise usable async request body is read. + response.once("close", closed); + }); const server = createServer((request, response) => { let body = ""; request.setEncoding("utf8"); @@ -121,52 +210,62 @@ async function localProvider(): Promise { body += chunk; }); request.on("end", () => { - let id: string; - try { - id = requestId(body); - } catch { - response.writeHead(400).end("B00B_BAD_LOCAL_REQUEST"); - return; - } - - entered.push(id); - inFlight += 1; - maxInFlight = Math.max(maxInFlight, inFlight); - const attempt = entered.filter((entry) => entry === id).length; - if (id === "request-0003" && attempt === 1) { - response.writeHead(429, { "content-type": "application/json", "retry-after": "0" }); - response.end(JSON.stringify({ error: { message: "fixture upstream 429", type: "rate_limit_error" } })); - inFlight -= 1; - return; - } - const emitSuccess = () => { - if (request.destroyed || response.destroyed) return; - response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); - // The large body deliberately exceeds a socket high-water mark. A paused - // attachment must be resynced instead of accumulating unbounded events. - const content = id === "request-0001" ? `${"x".repeat(512 * 1024)} fast-tail` : "cancelled-root-content"; - const event = (value: unknown) => response.write(`data: ${JSON.stringify(value)}\n\n`); - event({ - id: `fixture-${id}`, - model: "fixture-resolved", - choices: [{ index: 0, delta: { role: "assistant", content }, finish_reason: null }], - }); - event({ - id: `fixture-${id}`, - model: "fixture-resolved", - choices: [{ index: 0, delta: {}, finish_reason: "stop" }], - usage: { prompt_tokens: 7, completion_tokens: 11, total_tokens: 18 }, - }); - - response.end("data: [DONE]\n\n"); - inFlight -= 1; - }; - // Keep this upstream request in flight until its root's real abort signal - // closes the transport; no sibling shares this timer. - if (id === "request-0002") setTimeout(emitSuccess, 2_000); - else emitSuccess(); + void (async () => { + let id: string; + try { + id = requestId(body); + } catch { + response.writeHead(400).end("B00B_BAD_LOCAL_REQUEST"); + return; + } + if (request.headers.authorization !== `Bearer ${canary}`) { + response.writeHead(401).end("B00B_BAD_LOCAL_AUTHORIZATION"); + return; + } + entered.push(id); + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + const attempt = entered.filter((entry) => entry === id).length; + try { + if (!(await waitForRelease(id, response))) return; + if (response.destroyed || response.writableEnded) return; + if (id === "request-0003" && attempt === 1) { + response.writeHead(429, { "content-type": "application/json", "retry-after": "0" }); + response.end( + JSON.stringify({ error: { message: "fixture upstream 429", type: "rate_limit_error" } }), + ); + return; + } + if (id === "request-0002") await pause(2_000); + if (response.destroyed || response.writableEnded) return; + response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + const content = id === "request-0001" ? `${"x".repeat(512 * 1024)} fast-tail` : "cancelled-root-content"; + const event = (value: unknown) => response.write(`data: ${JSON.stringify(value)}\n\n`); + event({ + id: `fixture-${id}`, + model: "fixture-resolved", + choices: [{ index: 0, delta: { role: "assistant", content }, finish_reason: null }], + }); + event({ + id: `fixture-${id}`, + model: "fixture-resolved", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 7, completion_tokens: 11, total_tokens: 18 }, + }); + response.end("data: [DONE]\n\n"); + } finally { + inFlight -= 1; + } + })().catch(() => { + if (!response.headersSent) response.writeHead(500); + response.end("B00B_LOCAL_FIXTURE_FAILURE"); + }); }); }); + server.on("connection", (socket) => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + }); await new Promise((resolveListen) => server.listen(0, "127.0.0.1", resolveListen)); const address = server.address(); @@ -178,16 +277,28 @@ async function localProvider(): Promise { get maxInFlight() { return maxInFlight; }, - close: () => new Promise((resolveClose) => server.close(() => resolveClose())), + release: (ids) => { + for (const id of ids) released.add(id); + for (const waiter of releaseWaiters) waiter(); + releaseWaiters.clear(); + }, + close: () => { + if (closePromise) return closePromise; + closePromise = new Promise((resolveClose) => { + for (const socket of sockets) socket.destroy(); + server.close(() => resolveClose()); + }); + return closePromise; + }, }; } - -function spawnSupervisor(agentDir: string, socketPath: string, cwd: string): ChildProcess { +function spawnSupervisor(agentDir: string, socketPath: string, cwd: string, canary: string): ChildProcess { const child = spawn(process.execPath, [tsxPath, cliPath, "--mode", "daemon", "--daemon-socket", socketPath], { cwd, env: { ...process.env, [ENV_AGENT_DIR]: agentDir, + B00B_FIXTURE_KEY: canary, TSX_TSCONFIG_PATH: resolve(__dirname, "../../../../tsconfig.json"), PRIME_AGENT_INTERNAL_DAEMON_WORKER: undefined, PRIME_AGENT_INTERNAL_DAEMON_WORKER_TOKEN: undefined, @@ -232,7 +343,10 @@ async function connect(socketPath: string, child: ChildProcess): Promise { +async function attachThenPause( + socketPath: string, + activeSessionId: string, +): Promise<{ cursor: DaemonEventCursor; close(): void }> { const socket = createConnection(socketPath); resources.push(() => { socket.destroy(); @@ -273,169 +387,189 @@ async function attachThenPause(socketPath: string, activeSessionId: string): Pro throw new Error(`B00B_BLOCKED_ATTACH_FAILED ${JSON.stringify(first)}`); const cursor = (first.data as { lastEventCursor?: DaemonEventCursor }).lastEventCursor; if (!cursor) throw new Error("B00B_BLOCKED_ATTACH_NO_CURSOR"); - return { cursor }; + return { cursor, close: () => socket.destroy() }; } describe("B00B real daemon production dispatch", () => { - test("isolates paused attachment, cancellation, and upstream 429 across real supervisor workers", async () => { - const root = mkdtempSync(join(tmpdir(), "b00b-daemon-")); - resources.push(async () => { - for (let attempt = 0; attempt < 20; attempt++) { - try { - rmSync(root, { recursive: true, force: true, maxRetries: 1, retryDelay: 25 }); - return; - } catch { - await pause(50); - } - } - throw new Error("B00B_TEMP_CLEANUP_FAILED"); - }); - const agentDir = join(root, "agent"); - const projectDir = join(root, "project"); - const socketPath = join(tmpdir(), `b00b-${process.pid}-${randomUUID().slice(0, 8)}.sock`); - mkdirSync(agentDir, { recursive: true }); - mkdirSync(projectDir, { recursive: true }); - const upstream = await localProvider(); - resources.push(() => upstream.close()); - writeFileSync( - join(agentDir, "models.json"), - JSON.stringify({ - providers: { - "b00b-local": { - baseUrl: upstream.url, - apiKey: "fixture-key-B00B-canary", - api: "openai-completions", - models: [{ id: "fixture-a", api: "openai-completions", reasoning: false, input: ["text"] }], + test.each([1, 2, 3])( + "isolates paused attachment, cancellation, and upstream 429 across real supervisor workers (run %i)", + async () => { + const root = mkdtempSync(join(tmpdir(), "b00b-daemon-")); + const canary = `fixture-key-B00B-${randomUUID()}`; + resources.push(() => removeTempRoot(root)); + const agentDir = join(root, "agent"); + const projectDir = join(root, "project"); + const socketPath = join(tmpdir(), `b00b-${process.pid}-${randomUUID().slice(0, 8)}.sock`); + mkdirSync(agentDir, { recursive: true }); + mkdirSync(projectDir, { recursive: true }); + const upstream = await localProvider(canary); + resources.push(() => upstream.close()); + writeFileSync( + join(agentDir, "models.json"), + JSON.stringify({ + providers: { + "b00b-local": { + baseUrl: upstream.url, + apiKey: "B00B_FIXTURE_KEY", + api: "openai-completions", + models: [{ id: "fixture-a", api: "openai-completions", reasoning: false, input: ["text"] }], + }, }, - }, - }), - ); - const supervisor = spawnSupervisor(agentDir, socketPath, projectDir); - const control = await connect(socketPath, supervisor); - resources.push(() => control.close()); - const create = async (name: string) => { - const result = await control.request({ - type: "create", - name, - config: { - cwd: projectDir, - agentDir, - provider: "b00b-local", - model: "fixture-a", - noTools: true, - noExtensions: true, - noSkills: true, - }, - }); - if (!result.success) throw new Error("B00B_CREATE_ROOT_FAILED"); - return summary(result.data); - }; - const fast = await create("fast-root"); - const cancelled = await create("cancelled-root"); - const rateLimited = await create("rate-limited-root"); + }), + ); + const supervisor = spawnSupervisor(agentDir, socketPath, projectDir, canary); + const control = await connect(socketPath, supervisor); + resources.push(() => control.close()); + const create = async (name: string) => { + const result = await control.request({ + type: "create", + name, + config: { + cwd: projectDir, + agentDir, + provider: "b00b-local", + model: "fixture-a", + noTools: true, + noExtensions: true, + noSkills: true, + }, + }); + if (!result.success) throw new Error("B00B_CREATE_ROOT_FAILED"); + return summary(result.data); + }; + const fast = await create("fast-root"); + const cancelled = await create("cancelled-root"); + const rateLimited = await create("rate-limited-root"); - const workerPids = [fast.workerPid, cancelled.workerPid, rateLimited.workerPid]; - expect(workerPids.every((pid): pid is number => typeof pid === "number" && pid > 1)).toBe(true); - const concreteWorkerPids = workerPids as number[]; - expect(new Set(concreteWorkerPids).size).toBe(3); + const workerPids = [fast.workerPid, cancelled.workerPid, rateLimited.workerPid]; + expect(workerPids.every((pid): pid is number => typeof pid === "number" && pid > 1)).toBe(true); + const concreteWorkerPids = workerPids as number[]; + expect(new Set(concreteWorkerPids).size).toBe(3); - const blocked = await attachThenPause(socketPath, active(fast)); - const draining = await connect(socketPath, supervisor); - resources.push(() => draining.close()); - const drainEvents: Extract[] = []; - draining.onMessage((message) => { - if (message.type === "session_event" && message.activeSessionId === active(fast)) drainEvents.push(message); - }); - const attached = await draining.request({ - type: "attach", - activeSessionId: active(fast), - capabilities: ["attach_snapshot", "event_sequence"], - }); - expect(attached.success).toBe(true); + const blocked = await attachThenPause(socketPath, active(fast)); + const draining = await connect(socketPath, supervisor); + resources.push(() => draining.close()); + const drainEvents: Extract[] = []; + draining.onMessage((message) => { + if (message.type === "session_event" && message.activeSessionId === active(fast)) drainEvents.push(message); + }); + const attached = await draining.request({ + type: "attach", + activeSessionId: active(fast), + capabilities: ["attach_snapshot", "event_sequence"], + }); + expect(attached.success).toBe(true); - const dispatch = (session: SessionSummary, id: string) => - control.request({ type: "prompt", activeSessionId: active(session), message: id }, 10_000); - const admissions = await Promise.all([ - dispatch(fast, "request-0001"), - dispatch(cancelled, "request-0002"), - dispatch(rateLimited, "request-0003"), - ]); - expect(admissions.every((item) => item.success)).toBe(true); - await eventually(() => new Set(upstream.entered).size === 3, "B00B_PROVIDER_ENTRY_TIMEOUT"); - expect(upstream.entered.filter((id) => id === "request-0001")).toHaveLength(1); + const dispatch = (session: SessionSummary, id: string) => + control.request({ type: "prompt", activeSessionId: active(session), message: id }, 10_000); + const admissions = await Promise.all([ + dispatch(fast, "request-0001"), + dispatch(cancelled, "request-0002"), + dispatch(rateLimited, "request-0003"), + ]); + expect(admissions.every((item) => item.success)).toBe(true); + await eventually(() => new Set(upstream.entered).size === 3, "B00B_PROVIDER_ENTRY_TIMEOUT"); + // Before any fixture release, all three independent production HTTP requests overlap. + expect(upstream.maxInFlight).toBeGreaterThanOrEqual(3); + expect(upstream.entered.filter((id) => id === "request-0001")).toHaveLength(1); + expect(upstream.entered.filter((id) => id === "request-0002")).toHaveLength(1); + expect(upstream.entered.filter((id) => id === "request-0003")).toHaveLength(1); + // Abort is root-local; only then independently release fast and genuine 429. + const aborted = await control.request({ type: "abort", activeSessionId: active(cancelled) }); + expect(aborted.success).toBe(true); + upstream.release(["request-0001", "request-0003"]); + const idle = await control.request({ type: "wait_for_idle", activeSessionId: active(fast) }, 30_000); + expect(idle.success).toBe(true); + await eventually( + () => drainEvents.some((event) => event.type === "session_event" && event.event.type === "message_end"), + "B00B_DRAINING_ATTACHMENT_DID_NOT_COMPLETE", + ); + const ordered = drainEvents + .map((event) => event.meta?.sequence) + .filter((sequence): sequence is number => sequence !== undefined); + expect(ordered).toEqual([...ordered].sort((left, right) => left - right)); + expect(ordered.length).toBeGreaterThan(2); - expect(upstream.entered.filter((id) => id === "request-0002")).toHaveLength(1); + // Reattach from the cursor known before the paused write. The supervisor + // supplies a bounded snapshot/replay rather than a per-attachment model queue. + const catchup = await connect(socketPath, supervisor); + resources.push(() => catchup.close()); + const resynced = await catchup.request({ + type: "attach", + activeSessionId: active(fast), + capabilities: ["attach_snapshot", "event_sequence"], + resumeCursor: { activeSessionId: active(fast), ...blocked.cursor }, + }); + if (!resynced.success || !resynced.data || typeof resynced.data !== "object") + throw new Error("B00B_CATCHUP_FAILED"); + const catchupData = resynced.data as { snapshot?: { messages?: unknown[] }; replay?: { toSequence?: number } }; + expect(catchupData.snapshot?.messages?.length).toBeGreaterThanOrEqual(2); + expect(catchupData.replay?.toSequence).toBeGreaterThanOrEqual(blocked.cursor.sequence); - // All three independently created workers enter the real upstream before - // cancellation; a provider-side barrier is not faked by the client. + await eventually( + () => upstream.entered.filter((id) => id === "request-0003").length === 2, + "B00B_429_RETRY_TIMEOUT", + ); + upstream.release(["request-0003"]); + const rateLimitedIdle = await control.request( + { type: "wait_for_idle", activeSessionId: active(rateLimited) }, + 30_000, + ); + expect(rateLimitedIdle.success).toBe(true); + const rateLimitedMessages = await control.request({ + type: "get_messages", + activeSessionId: active(rateLimited), + }); + expect(JSON.stringify(rateLimitedMessages)).toContain("fixture-resolved"); + expect(upstream.entered.filter((id) => id === "request-0003")).toHaveLength(2); - // Abort and HTTP 429 are root-local. They cannot prevent the draining - // attachment's independent root from reaching a normal terminal. - const aborted = await control.request({ type: "abort", activeSessionId: active(cancelled) }); - expect(aborted.success).toBe(true); - const idle = await control.request({ type: "wait_for_idle", activeSessionId: active(fast) }, 30_000); - expect(idle.success).toBe(true); - await eventually( - () => drainEvents.some((event) => event.type === "session_event" && event.event.type === "message_end"), - "B00B_DRAINING_ATTACHMENT_DID_NOT_COMPLETE", - ); - const ordered = drainEvents - .map((event) => event.meta?.sequence) - .filter((sequence): sequence is number => sequence !== undefined); - expect(ordered).toEqual([...ordered].sort((left, right) => left - right)); - expect(ordered.length).toBeGreaterThan(2); + const cancelledIdle = await control.request( + { type: "wait_for_idle", activeSessionId: active(cancelled) }, + 30_000, + ); + expect(cancelledIdle.success).toBe(true); + const cancelledMessages = await control.request({ type: "get_messages", activeSessionId: active(cancelled) }); + expect(JSON.stringify(cancelledMessages)).not.toContain("cancelled-root-content"); + // The provider saw a genuine status-429 request while fast completed; + // no test code implements a permit, semaphore, or fabricated response. + expect(upstream.entered).toContain("request-0003"); - // Reattach from the cursor known before the paused write. The supervisor - // supplies a bounded snapshot/replay rather than a per-attachment model queue. - const catchup = await connect(socketPath, supervisor); - resources.push(() => catchup.close()); - const resynced = await catchup.request({ - type: "attach", - activeSessionId: active(fast), - capabilities: ["attach_snapshot", "event_sequence"], - resumeCursor: { activeSessionId: active(fast), ...blocked.cursor }, - }); - if (!resynced.success || !resynced.data || typeof resynced.data !== "object") - throw new Error("B00B_CATCHUP_FAILED"); - const catchupData = resynced.data as { snapshot?: { messages?: unknown[] }; replay?: { toSequence?: number } }; - expect(catchupData.snapshot?.messages?.length).toBeGreaterThanOrEqual(2); - expect(catchupData.replay?.toSequence).toBeGreaterThanOrEqual(blocked.cursor.sequence); + // Explicitly release every local client, including the deliberately paused raw socket, + // before asking the supervisor to stop accepting work. + blocked.close(); + catchup.close(); + draining.close(); + const shutdown = await control.request({ type: "shutdown" }, 10_000); + expect(shutdown.success).toBe(true); + control.close(); - await eventually( - () => upstream.entered.filter((id) => id === "request-0003").length === 2, - "B00B_429_RETRY_TIMEOUT", - ); - const rateLimitedIdle = await control.request( - { type: "wait_for_idle", activeSessionId: active(rateLimited) }, - 30_000, - ); - expect(rateLimitedIdle.success).toBe(true); - const rateLimitedMessages = await control.request({ type: "get_messages", activeSessionId: active(rateLimited) }); - expect(JSON.stringify(rateLimitedMessages)).toContain("fixture-resolved"); - expect(upstream.entered.filter((id) => id === "request-0003")).toHaveLength(2); + const supervisorPid = supervisor.pid; + expect(typeof supervisorPid).toBe("number"); + await Promise.all([ + waitForProcessGone(supervisorPid as number), + ...concreteWorkerPids.map((pid) => waitForProcessGone(pid)), + ]); + await eventually(() => !existsSync(socketPath), "B00B_SUPERVISOR_SOCKET_SURVIVED"); - const cancelledIdle = await control.request( - { type: "wait_for_idle", activeSessionId: active(cancelled) }, - 30_000, - ); - expect(cancelledIdle.success).toBe(true); - const cancelledMessages = await control.request({ type: "get_messages", activeSessionId: active(cancelled) }); - expect(JSON.stringify(cancelledMessages)).not.toContain("cancelled-root-content"); - // The provider saw a genuine status-429 request while fast completed; - // no test code implements a permit, semaphore, or fabricated response. - expect(upstream.entered).toContain("request-0003"); + const artifactPaths = recursivePaths(root).filter((path) => + /(?:^|[/\\])[^/\\]*(?:recovery|orphan|\.tmp)[^/\\]*$/i.test(path), + ); + expect(artifactPaths).toEqual([]); + const capturedTexts = [ + (supervisor as ChildProcess & { b00bStderr?: () => string }).b00bStderr?.() ?? "", + ...recursiveNormalFiles(root).map((path) => readFileSync(path, "utf8")), + ]; + assertNoFixtureKey(capturedTexts, canary); + await upstream.close(); + await removeTempRoot(root); + completedRoots.push(root); + assertNoRunResidue([root]); + }, + 60_000, + ); - const capturedTexts = [ - (supervisor as ChildProcess & { b00bStderr?: () => string }).b00bStderr?.() ?? "", - ...recursiveFiles(agentDir) - .filter((path) => path !== join(agentDir, "models.json")) - .map((path) => readFileSync(path, "utf8")), - ]; - assertNoFixtureKey(capturedTexts, "fixture-key-B00B-canary"); - const shutdown = await control.request({ type: "shutdown" }, 10_000); - expect(shutdown.success).toBe(true); - await Promise.all(concreteWorkerPids.map((pid) => waitForProcessGone(pid))); - await eventually(() => !existsSync(socketPath), "B00B_SUPERVISOR_SOCKET_SURVIVED"); - expect(recursiveFiles(join(agentDir, "daemon-workers")).filter((path) => path.endsWith(".tmp"))).toEqual([]); - }, 60_000); + test("leaves no b00b-daemon cwd process or directory after all repeated runs", () => { + expect(completedRoots).toHaveLength(3); + assertNoRunResidue(completedRoots); + }); }); From ec11f1ce9701d437f8232051c09c0e262004924f Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 21:16:15 -0700 Subject: [PATCH 09/28] test: prove B00B daemon dispatch backpressure (cherry picked from commit 6dd1e53734ea74897159fa62a3fafcc92fe69245) --- .../swarm/daemon-production-dispatch.test.ts | 217 +++++++++++++++--- 1 file changed, 191 insertions(+), 26 deletions(-) diff --git a/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts b/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts index 3ea971dd7..4b7e461c1 100644 --- a/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts +++ b/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts @@ -164,14 +164,60 @@ function requestId(body: string): string { return match[0]; } +interface ProviderAttempt { + readonly requestId: string; + readonly rootIdentity: string; + readonly attempt: number; + readonly enteredAt: number; + responseStatus?: number; + responseAt?: number; + responseEndedAt?: number; + requestAbortedAt?: number; + requestClosedAt?: number; + responseClosedAt?: number; +} + interface LocalProvider { readonly url: string; readonly entered: readonly string[]; + readonly attempts: readonly ProviderAttempt[]; readonly maxInFlight: number; release(ids: readonly string[]): void; close(): Promise; } +function rootIdentity(body: string): string { + const match = /b00b-root:([^"\s]+)/.exec(body); + if (!match) throw new Error("B00B_LOCAL_FIXTURE_MISSING_ROOT_IDENTITY"); + return match[1]; +} + +/** Test-only preload: observes, but never changes, the real Socket.write result. */ +function createSocketWriteObserver(root: string): { preloadPath: string; tracePath: string } { + const preloadPath = join(root, "socket-write-observer.cjs"); + const tracePath = join(root, "socket-write-0600.log"); + writeFileSync( + preloadPath, + `const { appendFileSync } = require("node:fs"); +const { Socket } = require("node:net"); +const trace = process.env.B00B_SOCKET_WRITE_TRACE; +// Workers inherit NODE_OPTIONS, but their role env exists before preload evaluation. +if (trace && !process.env.PRIME_AGENT_INTERNAL_DAEMON_WORKER) { + const realWrite = Socket.prototype.write; + Socket.prototype.write = function (...args) { + const accepted = realWrite.apply(this, args); + const wire = args[0]; + const text = Buffer.isBuffer(wire) ? wire.toString("utf8") : String(wire); + if (!accepted && text.includes('"type":"session_event"')) { + try { appendFileSync(trace, "0600 " + JSON.stringify({ writableLength: this.writableLength, bytes: Buffer.byteLength(text) }) + "\\n"); } catch {} + } + return accepted; + }; +}`, + ); + return { preloadPath, tracePath }; +} + /** * This is a real HTTP/SSE upstream from the worker's perspective. It is not a * model limiter: each POST enters immediately and receives its own scripted @@ -179,26 +225,24 @@ interface LocalProvider { */ async function localProvider(canary: string): Promise { const entered: string[] = []; + const attempts: ProviderAttempt[] = []; let inFlight = 0; let maxInFlight = 0; - const released = new Set(); - const releaseWaiters = new Set<() => void>(); + const releaseWaiters = new Map void>>(); const sockets = new Set(); let closePromise: Promise | undefined; const waitForRelease = (id: string, response: import("node:http").ServerResponse): Promise => new Promise((resolveRelease) => { - if (released.has(id)) { - resolveRelease(true); - return; - } const release = () => finish(true); const closed = () => finish(false); const finish = (wasReleased: boolean) => { - releaseWaiters.delete(release); + releaseWaiters.get(id)?.delete(release); response.off("close", closed); resolveRelease(wasReleased); }; - releaseWaiters.add(release); + const waiters = releaseWaiters.get(id) ?? new Set<() => void>(); + waiters.add(release); + releaseWaiters.set(id, waiters); // Worker cancellation destroys the response; it never relies on request.destroyed, // which can be true after an otherwise usable async request body is read. response.once("close", closed); @@ -222,14 +266,38 @@ async function localProvider(canary: string): Promise { response.writeHead(401).end("B00B_BAD_LOCAL_AUTHORIZATION"); return; } + let identity: string; + try { + identity = rootIdentity(body); + } catch { + response.writeHead(400).end("B00B_BAD_ROOT_IDENTITY"); + return; + } entered.push(id); + const record: ProviderAttempt = { + requestId: id, + rootIdentity: identity, + attempt: entered.filter((entry) => entry === id).length, + enteredAt: Date.now(), + }; + attempts.push(record); + request.once("aborted", () => { + record.requestAbortedAt = Date.now(); + }); + request.once("close", () => { + record.requestClosedAt = Date.now(); + }); + response.once("close", () => { + record.responseClosedAt = Date.now(); + }); inFlight += 1; maxInFlight = Math.max(maxInFlight, inFlight); - const attempt = entered.filter((entry) => entry === id).length; try { if (!(await waitForRelease(id, response))) return; if (response.destroyed || response.writableEnded) return; - if (id === "request-0003" && attempt === 1) { + if (id === "request-0003" && record.attempt === 1) { + record.responseStatus = 429; + record.responseAt = Date.now(); response.writeHead(429, { "content-type": "application/json", "retry-after": "0" }); response.end( JSON.stringify({ error: { message: "fixture upstream 429", type: "rate_limit_error" } }), @@ -238,6 +306,8 @@ async function localProvider(canary: string): Promise { } if (id === "request-0002") await pause(2_000); if (response.destroyed || response.writableEnded) return; + record.responseStatus = 200; + record.responseAt = Date.now(); response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); const content = id === "request-0001" ? `${"x".repeat(512 * 1024)} fast-tail` : "cancelled-root-content"; const event = (value: unknown) => response.write(`data: ${JSON.stringify(value)}\n\n`); @@ -253,6 +323,7 @@ async function localProvider(canary: string): Promise { usage: { prompt_tokens: 7, completion_tokens: 11, total_tokens: 18 }, }); response.end("data: [DONE]\n\n"); + record.responseEndedAt = Date.now(); } finally { inFlight -= 1; } @@ -274,13 +345,14 @@ async function localProvider(canary: string): Promise { return { url: `http://127.0.0.1:${address.port}/v1`, entered, + attempts, get maxInFlight() { return maxInFlight; }, release: (ids) => { - for (const id of ids) released.add(id); - for (const waiter of releaseWaiters) waiter(); - releaseWaiters.clear(); + // Release only attempts already at the named provider barrier. A retry + // remains independently held until this method is called again. + for (const id of ids) for (const waiter of [...(releaseWaiters.get(id) ?? [])]) waiter(); }, close: () => { if (closePromise) return closePromise; @@ -292,13 +364,21 @@ async function localProvider(canary: string): Promise { }, }; } -function spawnSupervisor(agentDir: string, socketPath: string, cwd: string, canary: string): ChildProcess { +function spawnSupervisor( + agentDir: string, + socketPath: string, + cwd: string, + canary: string, + observer: { preloadPath: string; tracePath: string }, +): ChildProcess { const child = spawn(process.execPath, [tsxPath, cliPath, "--mode", "daemon", "--daemon-socket", socketPath], { cwd, env: { ...process.env, [ENV_AGENT_DIR]: agentDir, B00B_FIXTURE_KEY: canary, + B00B_SOCKET_WRITE_TRACE: observer.tracePath, + NODE_OPTIONS: `${process.env.NODE_OPTIONS ?? ""} --require ${observer.preloadPath}`.trim(), TSX_TSCONFIG_PATH: resolve(__dirname, "../../../../tsconfig.json"), PRIME_AGENT_INTERNAL_DAEMON_WORKER: undefined, PRIME_AGENT_INTERNAL_DAEMON_WORKER_TOKEN: undefined, @@ -404,6 +484,7 @@ describe("B00B real daemon production dispatch", () => { mkdirSync(projectDir, { recursive: true }); const upstream = await localProvider(canary); resources.push(() => upstream.close()); + const observer = createSocketWriteObserver(root); writeFileSync( join(agentDir, "models.json"), JSON.stringify({ @@ -417,7 +498,7 @@ describe("B00B real daemon production dispatch", () => { }, }), ); - const supervisor = spawnSupervisor(agentDir, socketPath, projectDir, canary); + const supervisor = spawnSupervisor(agentDir, socketPath, projectDir, canary, observer); const control = await connect(socketPath, supervisor); resources.push(() => control.close()); const create = async (name: string) => { @@ -449,9 +530,11 @@ describe("B00B real daemon production dispatch", () => { const blocked = await attachThenPause(socketPath, active(fast)); const draining = await connect(socketPath, supervisor); resources.push(() => draining.close()); - const drainEvents: Extract[] = []; + const drainEvents: Array<{ event: Extract; observedAt: number }> = + []; draining.onMessage((message) => { - if (message.type === "session_event" && message.activeSessionId === active(fast)) drainEvents.push(message); + if (message.type === "session_event" && message.activeSessionId === active(fast)) + drainEvents.push({ event: message, observedAt: Date.now() }); }); const attached = await draining.request({ type: "attach", @@ -460,8 +543,28 @@ describe("B00B real daemon production dispatch", () => { }); expect(attached.success).toBe(true); + const cancelledObserver = await connect(socketPath, supervisor); + resources.push(() => cancelledObserver.close()); + const cancelledEvents: Array<{ + event: Extract; + observedAt: number; + }> = []; + cancelledObserver.onMessage((message) => { + if (message.type === "session_event" && message.activeSessionId === active(cancelled)) + cancelledEvents.push({ event: message, observedAt: Date.now() }); + }); + const cancelledAttached = await cancelledObserver.request({ + type: "attach", + activeSessionId: active(cancelled), + capabilities: ["attach_snapshot", "event_sequence"], + }); + expect(cancelledAttached.success).toBe(true); + const dispatch = (session: SessionSummary, id: string) => - control.request({ type: "prompt", activeSessionId: active(session), message: id }, 10_000); + control.request( + { type: "prompt", activeSessionId: active(session), message: `${id} b00b-root:${active(session)}` }, + 10_000, + ); const admissions = await Promise.all([ dispatch(fast, "request-0001"), dispatch(cancelled, "request-0002"), @@ -474,21 +577,73 @@ describe("B00B real daemon production dispatch", () => { expect(upstream.entered.filter((id) => id === "request-0001")).toHaveLength(1); expect(upstream.entered.filter((id) => id === "request-0002")).toHaveLength(1); expect(upstream.entered.filter((id) => id === "request-0003")).toHaveLength(1); - // Abort is root-local; only then independently release fast and genuine 429. + // Abort is root-local. Its open response must actually be closed upstream, + // rather than merely suppressing a locally continuing model result. const aborted = await control.request({ type: "abort", activeSessionId: active(cancelled) }); expect(aborted.success).toBe(true); - upstream.release(["request-0001", "request-0003"]); + await eventually(() => { + const cancelledAttempt = upstream.attempts.find((attempt) => attempt.requestId === "request-0002"); + return Boolean(cancelledAttempt?.requestAbortedAt || cancelledAttempt?.responseClosedAt); + }, "B00B_CANCEL_DID_NOT_CLOSE_UPSTREAM"); + const cancelledAttempt = upstream.attempts.find((attempt) => attempt.requestId === "request-0002"); + expect(cancelledAttempt).toMatchObject({ + requestId: "request-0002", + rootIdentity: active(cancelled), + attempt: 1, + }); + + // The first rate-root attempt genuinely returns 429. Its second attempt is + // held at the fixture barrier, so a sibling must finish while it backs off. + upstream.release(["request-0003"]); + await eventually( + () => + upstream.attempts.some( + (attempt) => + attempt.requestId === "request-0003" && attempt.attempt === 1 && attempt.responseStatus === 429, + ), + "B00B_GENUINE_429_NOT_OBSERVED", + ); + await eventually( + () => upstream.attempts.some((attempt) => attempt.requestId === "request-0003" && attempt.attempt === 2), + "B00B_429_RETRY_TIMEOUT", + ); + upstream.release(["request-0001"]); const idle = await control.request({ type: "wait_for_idle", activeSessionId: active(fast) }, 30_000); expect(idle.success).toBe(true); await eventually( - () => drainEvents.some((event) => event.type === "session_event" && event.event.type === "message_end"), + () => drainEvents.some(({ event }) => event.event.type === "message_end"), "B00B_DRAINING_ATTACHMENT_DID_NOT_COMPLETE", ); const ordered = drainEvents - .map((event) => event.meta?.sequence) + .map(({ event }) => event.meta?.sequence) .filter((sequence): sequence is number => sequence !== undefined); expect(ordered).toEqual([...ordered].sort((left, right) => left - right)); expect(ordered.length).toBeGreaterThan(2); + const fastCompletedAt = Date.now(); + const rateFirst = upstream.attempts.find( + (attempt) => attempt.requestId === "request-0003" && attempt.attempt === 1, + ); + const rateSecond = upstream.attempts.find( + (attempt) => attempt.requestId === "request-0003" && attempt.attempt === 2, + ); + expect(rateFirst).toMatchObject({ rootIdentity: active(rateLimited), responseStatus: 429 }); + expect(rateSecond).toMatchObject({ rootIdentity: active(rateLimited) }); + expect(rateFirst?.responseAt).toBeTypeOf("number"); + expect(rateSecond?.enteredAt).toBeLessThanOrEqual(fastCompletedAt); + expect(rateSecond?.responseEndedAt).toBeUndefined(); + + // The paused raw client caused a natural real net.Socket.write false in + // the supervisor. The preload only records its return and writableLength. + const falseWrites = existsSync(observer.tracePath) + ? readFileSync(observer.tracePath, "utf8") + .split("\n") + .filter((line) => line.startsWith("0600 ")) + .map((line) => JSON.parse(line.slice(5)) as { writableLength: number; bytes: number }) + : []; + expect(falseWrites.length).toBeGreaterThanOrEqual(1); + expect(falseWrites.length).toBeLessThanOrEqual(8); + expect(Math.max(...falseWrites.map((entry) => entry.writableLength))).toBeLessThanOrEqual(2 * 1024 * 1024); + expect(Math.max(...falseWrites.map((entry) => entry.bytes))).toBeLessThanOrEqual(2 * 1024 * 1024); // Reattach from the cursor known before the paused write. The supervisor // supplies a bounded snapshot/replay rather than a per-attachment model queue. @@ -506,10 +661,6 @@ describe("B00B real daemon production dispatch", () => { expect(catchupData.snapshot?.messages?.length).toBeGreaterThanOrEqual(2); expect(catchupData.replay?.toSequence).toBeGreaterThanOrEqual(blocked.cursor.sequence); - await eventually( - () => upstream.entered.filter((id) => id === "request-0003").length === 2, - "B00B_429_RETRY_TIMEOUT", - ); upstream.release(["request-0003"]); const rateLimitedIdle = await control.request( { type: "wait_for_idle", activeSessionId: active(rateLimited) }, @@ -522,6 +673,7 @@ describe("B00B real daemon production dispatch", () => { }); expect(JSON.stringify(rateLimitedMessages)).toContain("fixture-resolved"); expect(upstream.entered.filter((id) => id === "request-0003")).toHaveLength(2); + expect(rateSecond?.responseEndedAt).toBeGreaterThan(fastCompletedAt); const cancelledIdle = await control.request( { type: "wait_for_idle", activeSessionId: active(cancelled) }, @@ -530,6 +682,19 @@ describe("B00B real daemon production dispatch", () => { expect(cancelledIdle.success).toBe(true); const cancelledMessages = await control.request({ type: "get_messages", activeSessionId: active(cancelled) }); expect(JSON.stringify(cancelledMessages)).not.toContain("cancelled-root-content"); + const cancelledTerminals = cancelledEvents.filter( + ({ event }) => + event.event.type === "message_end" && + (event.event.message as { stopReason?: string }).stopReason === "aborted", + ); + expect(cancelledTerminals).toHaveLength(1); + const cancelledTerminalSequence = cancelledTerminals[0]?.event.meta?.sequence ?? -1; + expect( + cancelledEvents.filter( + ({ event }) => + event.event.type === "message_update" && (event.meta?.sequence ?? -1) > cancelledTerminalSequence, + ), + ).toHaveLength(0); // The provider saw a genuine status-429 request while fast completed; // no test code implements a permit, semaphore, or fabricated response. expect(upstream.entered).toContain("request-0003"); From b2660880cd94af0e4dbfcc923c7ab7b086d92cc3 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 21:22:17 -0700 Subject: [PATCH 10/28] test: make b00b cwd residue check portable on Linux (cherry picked from commit d860c1dac5021d2fe8d05db8708353f3b44bf295) --- .../swarm/daemon-production-dispatch.test.ts | 73 +++++++++++++++++-- 1 file changed, 68 insertions(+), 5 deletions(-) diff --git a/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts b/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts index 4b7e461c1..f2e152642 100644 --- a/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts +++ b/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts @@ -12,7 +12,9 @@ import { mkdtempSync, readdirSync, readFileSync, + readlinkSync, rmSync, + symlinkSync, writeFileSync, } from "node:fs"; import { createServer } from "node:http"; @@ -100,13 +102,45 @@ async function removeTempRoot(root: string): Promise { } throw new Error("B00B_TEMP_CLEANUP_FAILED"); } -function cwdPidsUnder(roots: readonly string[]): number[] { +function cwdUnderRoots(cwd: string, roots: readonly string[]): boolean { + const normalizedCwd = cwd.replace(/\s+\(deleted\)$/, ""); + return roots.some((root) => normalizedCwd === root || normalizedCwd.startsWith(`${root}/`)); +} + +/** Linux has a portable cwd handle for every visible process; do not require lsof in the pinned image. */ +function cwdPidsUnderProc(roots: readonly string[], procRoot = "/proc"): number[] { + let entries: string[]; + try { + entries = readdirSync(procRoot); + } catch (error) { + throw new Error(`B00B_PROC_ROOT_UNREADABLE ${procRoot}: ${(error as Error).message}`); + } + + const matching = new Set(); + for (const entry of entries) { + if (!/^\d+$/.test(entry)) continue; + const pid = Number.parseInt(entry, 10); + try { + if (cwdUnderRoots(readlinkSync(join(procRoot, entry, "cwd")), roots)) matching.add(pid); + } catch (error) { + // Processes can exit, or their cwd can be inaccessible, between readdir and readlink. + const code = (error as NodeJS.ErrnoException).code; + if (code === "ENOENT" || code === "EACCES" || code === "EPERM") continue; + throw new Error(`B00B_PROC_CWD_UNREADABLE ${join(procRoot, entry, "cwd")}: ${(error as Error).message}`); + } + } + return [...matching]; +} + +function cwdPidsUnderDarwin(roots: readonly string[]): number[] { let output: string; try { output = execFileSync("lsof", ["-n", "-Fpn", "-a", "-d", "cwd"], { encoding: "utf8" }); } catch (error) { - const status = (error as { status?: unknown }).status; - if (status === 1) return []; + const errno = error as NodeJS.ErrnoException & { status?: unknown }; + if (errno.status === 1) return []; + if (errno.code === "ENOENT") + throw new Error("B00B_LSOF_UNAVAILABLE_ON_DARWIN: install lsof to enforce the cwd residue assertion"); throw error; } let pid: number | undefined; @@ -114,12 +148,17 @@ function cwdPidsUnder(roots: readonly string[]): number[] { for (const line of output.split("\n")) { if (line.startsWith("p")) pid = Number.parseInt(line.slice(1), 10); if (!line.startsWith("n") || pid === undefined) continue; - const cwd = line.slice(1).replace(/\s+\(deleted\)$/, ""); - if (roots.some((root) => cwd === root || cwd.startsWith(`${root}/`))) matching.add(pid); + if (cwdUnderRoots(line.slice(1), roots)) matching.add(pid); } return [...matching]; } +function cwdPidsUnder(roots: readonly string[]): number[] { + if (process.platform === "linux") return cwdPidsUnderProc(roots); + if (process.platform === "darwin") return cwdPidsUnderDarwin(roots); + throw new Error(`B00B_CWD_RESIDUE_CHECK_UNSUPPORTED_PLATFORM: ${process.platform}`); +} + function assertNoRunResidue(roots: readonly string[]): void { expect(roots.every((root) => !existsSync(root))).toBe(true); expect(cwdPidsUnder(roots)).toEqual([]); @@ -471,6 +510,30 @@ async function attachThenPause( } describe("B00B real daemon production dispatch", () => { + test("enumerates Linux-style proc cwd links without lsof", () => { + const procRoot = mkdtempSync(join(tmpdir(), "b00b-proc-")); + try { + const matchingRoot = join(procRoot, "matching-root"); + const otherRoot = join(procRoot, "other-root"); + mkdirSync(matchingRoot); + mkdirSync(otherRoot); + mkdirSync(join(procRoot, "101")); + mkdirSync(join(procRoot, "202")); + // A process may exit after /proc is listed, leaving no cwd link. + mkdirSync(join(procRoot, "303")); + mkdirSync(join(procRoot, "404")); + writeFileSync(join(procRoot, "not-a-pid"), "ignored"); + // proc cwd entries are symlinks; Linux appends this suffix for a deleted cwd. + symlinkSync(matchingRoot, join(procRoot, "101", "cwd")); + symlinkSync(otherRoot, join(procRoot, "202", "cwd")); + symlinkSync(`${matchingRoot} (deleted)`, join(procRoot, "404", "cwd")); + expect(cwdPidsUnderProc([matchingRoot], procRoot)).toEqual([101, 404]); + expect(() => cwdPidsUnderProc([], join(procRoot, "missing"))).toThrow("B00B_PROC_ROOT_UNREADABLE"); + } finally { + rmSync(procRoot, { recursive: true, force: true }); + } + }); + test.each([1, 2, 3])( "isolates paused attachment, cancellation, and upstream 429 across real supervisor workers (run %i)", async () => { From 696158d9286bbe3b5e0ad9b9be4a891b65a770e3 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 21:32:14 -0700 Subject: [PATCH 11/28] test: keep daemon proof within swarm scope (cherry picked from commit d672b051a636f26f661b2b70b49a39f78fa61612) --- packages/coding-agent/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 6555f0f9f..bb0570e40 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -40,7 +40,7 @@ "test": "vitest --run", "test:ci": "tsx src/core/kernel/bootstrap-cli.ts && vitest --run --exclude test/daemon-supervisor-process.test.ts", "test:process": "vitest --run test/daemon-supervisor-process.test.ts", - "test:process-stress": "env -u PRIME_AGENT_INTERNAL_DAEMON_WORKER -u PRIME_AGENT_INTERNAL_DAEMON_WORKER_TOKEN -u PRIME_AGENT_INTERNAL_DAEMON_WORKER_ACTIVE_SESSION_ID -u PRIME_AGENT_INTERNAL_DAEMON_SUPERVISOR_SOCKET -u PRIME_AGENT_INTERNAL_DAEMON_WORKER_RECOVERY_JOURNAL -u PRIME_AGENT_INTERNAL_DAEMON_WORKER_STARTUP_GATE_FD -u PRIME_AGENT_INTERNAL_ORPHAN_PROCESS_JOURNAL -u PRIME_AGENT_INTERNAL_SESSION_LEASES -u PRIME_AGENT_INTERNAL_SESSION_LEASE_OWNER_ID vitest --run --tagsFilter process-stress test/daemon-supervisor-process.test.ts", + "test:process-stress": "vitest --run --tagsFilter process-stress test/daemon-supervisor-process.test.ts", "postinstall": "node postinstall.cjs", "prepublishOnly": "npm run clean && npm run build", "bundle": "node scripts/bundle.mjs", From cabe19441539f2522199719ea2e994031b1cc012 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 19:52:53 -0700 Subject: [PATCH 12/28] test(coding-agent): add supervised RSS campaign fixtures (cherry picked from commit 4ecb14d0bc41fc4c9e99d769c2c044e7f7a93d3e) --- .../test/swarm/rss-campaign-worker.ts | 136 ++++++ .../test/swarm/run-production-rss-campaign.ts | 439 ++++++++++++++++++ 2 files changed, 575 insertions(+) create mode 100644 packages/coding-agent/test/swarm/rss-campaign-worker.ts create mode 100644 packages/coding-agent/test/swarm/run-production-rss-campaign.ts diff --git a/packages/coding-agent/test/swarm/rss-campaign-worker.ts b/packages/coding-agent/test/swarm/rss-campaign-worker.ts new file mode 100644 index 000000000..445ecfd66 --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-campaign-worker.ts @@ -0,0 +1,136 @@ +/** + * Disposable, test-only child supervisor for the PR-B00B RSS campaign. + * It deliberately has no provider imports, network client, daemon listener, or + * persistent state. The parent owns this process group and measures it. + */ +import { mkdtemp, mkdir, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { spawn, type ChildProcess } from "node:child_process"; + +interface WorkerOptions { + fanout: number; + allocationMiB: number; + scratch: string; + fixtureCommand?: string; + fixtureArgs: readonly string[]; +} + +type WorkerMessage = + | { type: "boundary"; phase: "started" | "barrier-held" | "terminals" | "cleanup"; allocatedBytes: number } + | { type: "result"; completed: number; failed: number; allocatedBytes: number }; + +function option(name: string): string | undefined { + const index = process.argv.indexOf(name); + return index === -1 ? undefined : process.argv[index + 1]; +} + +function positiveInteger(name: string, fallback?: number): number { + const value = option(name) ?? (fallback === undefined ? undefined : String(fallback)); + const parsed = value === undefined ? Number.NaN : Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 1) throw new Error(`invalid_${name.slice(2)}`); + return parsed; +} + +function options(): WorkerOptions { + const fanout = positiveInteger("--fanout"); + const allocationMiB = positiveInteger("--allocation-mib", 1); + const scratch = option("--scratch") ?? join(tmpdir(), "b00b-rss"); + const fixtureCommand = option("--fixture-command"); + const fixtureArgs: string[] = []; + for (let index = 0; index < process.argv.length; index += 1) { + if (process.argv[index] === "--fixture-arg") { + const value = process.argv[index + 1]; + if (value === undefined) throw new Error("invalid_fixture_arg"); + fixtureArgs.push(value); + index += 1; + } + } + return { fanout, allocationMiB, scratch, fixtureCommand, fixtureArgs }; +} + +function safeEnvironment(worker: number, fanout: number, allocationBytes: number): NodeJS.ProcessEnv { + const inherited = process.env; + const environment: NodeJS.ProcessEnv = { + B00B_WORKER_INDEX: String(worker), + B00B_FANOUT: String(fanout), + B00B_FIXTURE_ALLOCATION_BYTES: String(allocationBytes), + LANG: "C", + LC_ALL: "C", + }; + for (const key of ["PATH", "HOME", "TMPDIR", "TMP", "TEMP", "SystemRoot", "ComSpec"]) { + if (inherited[key]) environment[key] = inherited[key]; + } + return environment; +} + +// This fixture is intentionally local and deterministic. An integration can +// replace it with --fixture-command/--fixture-arg without the launcher ever +// serializing that command, its arguments, or its output into campaign data. +const BUILTIN_FIXTURE = [ + "const bytes=Number(process.env.B00B_FIXTURE_ALLOCATION_BYTES||0);", + "const b=Buffer.allocUnsafe(bytes);for(let i=0;iprocess.exit(0),50);", +].join(""); + +function runFixture(config: WorkerOptions, worker: number, allocationBytes: number): Promise { + const command = config.fixtureCommand ?? process.execPath; + const args = config.fixtureCommand ? [...config.fixtureArgs] : ["-e", BUILTIN_FIXTURE]; + return new Promise((resolve) => { + let child: ChildProcess; + try { + child = spawn(command, args, { + cwd: process.cwd(), + detached: false, + env: safeEnvironment(worker, config.fanout, allocationBytes), + stdio: "ignore", + }); + } catch { + resolve(false); + return; + } + child.once("error", () => resolve(false)); + child.once("exit", (code, signal) => resolve(code === 0 && signal === null)); + }); +} + +function send(message: WorkerMessage): void { + process.send?.(message); +} + +function pause(milliseconds: number): Promise { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +async function main(): Promise { + const config = options(); + const allocationBytes = config.allocationMiB * 1024 * 1024; + // The supervisor allocation and each fixture allocation are touched so RSS + // has a deliberate, numeric-only allocation proof. + let allocation = Buffer.allocUnsafe(allocationBytes); + for (let index = 0; index < allocation.length; index += 4096) allocation[index] = 1; + const runtimeRoot = await mkdtemp(join(config.scratch, "b00b-rss-")); + try { + await Promise.all([mkdir(join(runtimeRoot, "agent")), mkdir(join(runtimeRoot, "socket")), mkdir(join(runtimeRoot, "output"))]); + send({ type: "boundary", phase: "started", allocatedBytes: allocationBytes }); + const fixtures = Array.from({ length: config.fanout }, (_, index) => runFixture(config, index + 1, allocationBytes)); + // All fixture entries are dispatched before this boundary. This is an + // observation boundary, never a permit, queue, or admission limiter. + send({ type: "boundary", phase: "barrier-held", allocatedBytes: allocationBytes * (config.fanout + 1) }); + const results = await Promise.all(fixtures); + const completed = results.filter(Boolean).length; + send({ type: "boundary", phase: "terminals", allocatedBytes: allocationBytes * (config.fanout + 1) }); + // Hold the terminal boundary long enough for the 20 Hz parent sampler to + // capture it; this is post-terminal observation only, not admission. + await pause(100); + allocation = Buffer.alloc(0); + global.gc?.(); + await rm(runtimeRoot, { force: true, recursive: true }); + send({ type: "boundary", phase: "cleanup", allocatedBytes: 0 }); + send({ type: "result", completed, failed: config.fanout - completed, allocatedBytes: 0 }); + } finally { + await rm(runtimeRoot, { force: true, recursive: true }); + } +} + +await main(); diff --git a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts new file mode 100644 index 000000000..fcd2f2c7a --- /dev/null +++ b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts @@ -0,0 +1,439 @@ +/** + * Fresh-process, test-only RSS campaign launcher for PR-B00B. + * + * It has no product import, provider credential, network client, or resident + * daemon. Every measured cell owns a newly spawned Unix process group. A later + * real-provider fixture can be supplied with --fixture-command; its command, + * arguments, stdout, stderr, and environment are deliberately not archived. + */ +import { createHash } from "node:crypto"; +import { chmod, mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { cpus, platform, release, totalmem } from "node:os"; +import { basename, dirname, join } from "node:path"; +import { spawn, type ChildProcess } from "node:child_process"; + +const FANOUTS = [1, 4, 16, 64] as const; +const WORKER = new URL("./rss-campaign-worker.ts", import.meta.url); +const MIN_INTERVAL_MS = 50; +const DEFAULT_TIMEOUT_MS = 60_000; +const SCHEMA_VERSION = 1; + +type SupportedPlatform = "linux" | "darwin"; +type Phase = "baseline" | "started" | "barrier-held" | "terminals" | "cleanup" | "final"; +type Status = "complete" | "failed" | "timed_out" | "unsupported"; + +interface ProcessRecord { + pid: number; + ppid: number; + start: number; + rssKiB: number; +} + +interface ProcessSample { + phase: Phase; + monotonicMs: number; + totalRssKiB: number; + processes: readonly ProcessRecord[]; +} + +interface BoundaryMessage { + type: "boundary"; + phase: Exclude; + allocatedBytes: number; +} + +interface ResultMessage { + type: "result"; + completed: number; + failed: number; + allocatedBytes: number; +} + +type WorkerMessage = BoundaryMessage | ResultMessage; + +interface Repetition { + schemaVersion: number; + kind: "b00b-rss-repetition"; + status: Status; + fanout: number; + repetition: number; + warmup: boolean; + sampler: { source: "proc-status" | "ps"; intervalMs: number; sharedPages: "summed-per-process" } | null; + reasonCode: number | null; + baselineRssKiB: number | null; + peakRssKiB: number | null; + terminalRssKiB: number | null; + finalRssKiB: number | null; + allocatedBytes: number; + completed: number; + failed: number; + timedOut: boolean; + samples: readonly ProcessSample[]; +} + +interface Config { + fanouts: readonly number[]; + repetitions: number; + output: string; + intervalMs: number; + timeoutMs: number; + platformRequired?: string; + fixtureCommand?: string; + fixtureArgs: readonly string[]; + allocationMiB: number; +} + +function option(name: string): string | undefined { + const index = process.argv.indexOf(name); + return index < 0 ? undefined : process.argv[index + 1]; +} + +function safeInteger(name: string, fallback: number, minimum: number): number { + const parsed = Number(option(name) ?? fallback); + if (!Number.isSafeInteger(parsed) || parsed < minimum) throw new Error(`invalid_${name.slice(2)}`); + return parsed; +} + +function parseFanouts(value: string | undefined): readonly number[] { + if (!value) return FANOUTS; + const values = value.split(",").map(Number); + if (!values.length || values.some((value) => !FANOUTS.includes(value as (typeof FANOUTS)[number]))) { + throw new Error("invalid_fanout"); + } + return [...new Set(values)]; +} + +function config(): Config { + const intervalMs = safeInteger("--interval-ms", MIN_INTERVAL_MS, MIN_INTERVAL_MS); + const fixtureArgs: string[] = []; + for (let index = 0; index < process.argv.length; index += 1) { + if (process.argv[index] === "--fixture-arg") { + const argument = process.argv[index + 1]; + if (argument === undefined) throw new Error("invalid_fixture_arg"); + fixtureArgs.push(argument); + index += 1; + } + } + return { + fanouts: parseFanouts(option("--fanout")), + repetitions: safeInteger("--repetitions", 3, 1), + output: option("--output") ?? "b00b-rss-artifacts", + intervalMs, + timeoutMs: safeInteger("--timeout-ms", DEFAULT_TIMEOUT_MS, 1), + platformRequired: option("--platform-required"), + fixtureCommand: option("--fixture-command"), + fixtureArgs, + allocationMiB: safeInteger("--allocation-mib", 1, 1), + }; +} + +function monotonicMs(): number { + return Number(process.hrtime.bigint() / 1_000_000n); +} + +function sha256(value: string | Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +function canonical(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + const object = value as Record; + return `{${Object.keys(object) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonical(object[key])}`) + .join(",")}}`; +} + +async function writeOwnerFile(path: string, content: string): Promise { + await writeFile(path, content, { encoding: "utf8", mode: 0o600 }); + await chmod(path, 0o600); +} + +async function procStartAndRss(pid: number): Promise { + try { + const [statLine, status] = await Promise.all([ + readFile(`/proc/${pid}/stat`, "utf8"), + readFile(`/proc/${pid}/status`, "utf8"), + ]); + const close = statLine.lastIndexOf(")"); + const fields = statLine.slice(close + 2).trim().split(/\s+/); + const ppid = Number(fields[1]); // field 4 after removing pid/comm + const start = Number(fields[19]); // field 22 after removing pid/comm + const rss = /^VmRSS:\s+(\d+)\s+kB$/m.exec(status)?.[1]; + if (!Number.isSafeInteger(ppid) || !Number.isSafeInteger(start) || rss === undefined) return undefined; + return { pid, ppid, start, rssKiB: Number(rss) }; + } catch { + return undefined; + } +} + +async function linuxSnapshot(rootPid: number): Promise { + let entries: string[]; + try { + entries = await readdir("/proc"); + } catch { + return []; + } + const records = (await Promise.all(entries.filter((entry) => /^\d+$/.test(entry)).map((entry) => procStartAndRss(Number(entry))))).filter( + (record): record is ProcessRecord => record !== undefined, + ); + const descendants = new Set([rootPid]); + let changed = true; + while (changed) { + changed = false; + for (const record of records) { + if (descendants.has(record.ppid) && !descendants.has(record.pid)) { + descendants.add(record.pid); + changed = true; + } + } + } + return records.filter((record) => descendants.has(record.pid)).sort((left, right) => left.pid - right.pid); +} + +async function macSnapshot(rootPid: number): Promise { + const ps = spawn("ps", ["-axo", "pid=,ppid=,rss=,lstart="], { stdio: ["ignore", "pipe", "ignore"] }); + let text = ""; + ps.stdout?.setEncoding("utf8"); + ps.stdout?.on("data", (chunk: string) => { + text += chunk; + }); + const exited = await new Promise((resolve) => { + ps.once("error", () => resolve(false)); + ps.once("exit", (code) => resolve(code === 0)); + }); + if (!exited) return []; + const records: ProcessRecord[] = []; + for (const line of text.split("\n")) { + const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(.+?)\s*$/.exec(line); + if (!match) continue; + const start = Date.parse(match[4]); + if (Number.isNaN(start)) continue; + records.push({ pid: Number(match[1]), ppid: Number(match[2]), rssKiB: Number(match[3]), start }); + } + const descendants = new Set([rootPid]); + let changed = true; + while (changed) { + changed = false; + for (const record of records) { + if (descendants.has(record.ppid) && !descendants.has(record.pid)) { + descendants.add(record.pid); + changed = true; + } + } + } + return records.filter((record) => descendants.has(record.pid)).sort((left, right) => left.pid - right.pid); +} + +async function snapshot(kind: SupportedPlatform, rootPid: number): Promise { + return kind === "linux" ? linuxSnapshot(rootPid) : macSnapshot(rootPid); +} + +async function collectorAvailable(kind: SupportedPlatform): Promise { + const own = (await snapshot(kind, process.pid)).find((record) => record.pid === process.pid); + return own !== undefined && own.rssKiB >= 0 && Number.isSafeInteger(own.start); +} + +function total(records: readonly ProcessRecord[]): number { + return records.reduce((sum, record) => sum + record.rssKiB, 0); +} + +function sample(phase: Phase, records: readonly ProcessRecord[]): ProcessSample { + return { phase, monotonicMs: monotonicMs(), totalRssKiB: total(records), processes: records }; +} + +async function identityMatches(kind: SupportedPlatform, record: ProcessRecord): Promise { + const current = (await snapshot(kind, record.pid)).find((candidate) => candidate.pid === record.pid); + return current?.start === record.start; +} + +async function reapOwnGroup(kind: SupportedPlatform, leader?: ProcessRecord): Promise { + if (!leader || !(await identityMatches(kind, leader))) return; + try { + process.kill(-leader.pid, "SIGTERM"); + } catch { + return; + } + await new Promise((resolve) => setTimeout(resolve, 250)); + if (await identityMatches(kind, leader)) { + try { + process.kill(-leader.pid, "SIGKILL"); + } catch { + // The group may have cleanly exited between the identity check and kill. + } + } +} + +function workerArguments(config: Config, fanout: number, scratch: string): string[] { + const args = ["--expose-gc", ...process.execArgv, WORKER.pathname, "--fanout", String(fanout), "--allocation-mib", String(config.allocationMiB), "--scratch", scratch]; + if (config.fixtureCommand) args.push("--fixture-command", config.fixtureCommand); + for (const fixtureArg of config.fixtureArgs) args.push("--fixture-arg", fixtureArg); + return args; +} + +async function runCell(config: Config, kind: SupportedPlatform, fanout: number, repetition: number, warmup: boolean, scratch: string): Promise { + const samples: ProcessSample[] = [sample("baseline", [])]; + let leader: ProcessRecord | undefined; + let child: ChildProcess | undefined; + let timer: NodeJS.Timeout | undefined; + let timedOut = false; + let completed = 0; + let failed = fanout; + let allocatedBytes = 0; + let resolved = false; + const finish = async (status: Status): Promise => { + if (timer) clearInterval(timer); + await reapOwnGroup(kind, leader); + const final = sample("final", leader ? await snapshot(kind, leader.pid) : []); + samples.push(final); + const byPhase = (phase: Phase) => samples.filter((entry) => entry.phase === phase).at(-1)?.totalRssKiB ?? null; + const active = samples.filter((entry) => entry.phase !== "baseline" && entry.phase !== "final"); + return { + schemaVersion: SCHEMA_VERSION, + kind: "b00b-rss-repetition", + status, + fanout, + repetition, + warmup, + sampler: { source: kind === "linux" ? "proc-status" : "ps", intervalMs: config.intervalMs, sharedPages: "summed-per-process" }, + reasonCode: status === "complete" ? null : status === "timed_out" ? 1 : 2, + baselineRssKiB: 0, + peakRssKiB: active.length ? Math.max(...active.map((entry) => entry.totalRssKiB)) : null, + terminalRssKiB: byPhase("terminals"), + finalRssKiB: final.totalRssKiB, + allocatedBytes, + completed, + failed, + timedOut, + samples, + }; + }; + return new Promise((resolve) => { + const settle = async (status: Status) => { + if (resolved) return; + resolved = true; + resolve(await finish(status)); + }; + try { + child = spawn(process.execPath, workerArguments(config, fanout, scratch), { + cwd: process.cwd(), + detached: process.platform !== "win32", + env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, LANG: "C", LC_ALL: "C" }, + serialization: "json", + stdio: ["ignore", "ignore", "ignore", "ipc"], + }); + } catch { + void settle("failed"); + return; + } + const pid = child.pid; + if (!pid) { + void settle("failed"); + return; + } + void snapshot(kind, pid).then((records) => { + leader = records.find((record) => record.pid === pid); + samples.push(sample("started", records)); + }); + timer = setInterval(() => { + if (!leader || resolved) return; + void snapshot(kind, leader.pid).then((records) => samples.push(sample("started", records))); + }, config.intervalMs); + const timeout = setTimeout(() => { + timedOut = true; + void settle("timed_out"); + }, config.timeoutMs); + child.once("error", () => { + clearTimeout(timeout); + void settle("failed"); + }); + child.on("message", (message: WorkerMessage) => { + if (message.type === "result") { + completed = message.completed; + failed = message.failed; + allocatedBytes = Math.max(allocatedBytes, message.allocatedBytes); + return; + } + allocatedBytes = Math.max(allocatedBytes, message.allocatedBytes); + if (leader) void snapshot(kind, leader.pid).then((records) => samples.push(sample(message.phase, records))); + }); + child.once("exit", (code, signal) => { + clearTimeout(timeout); + void settle(code === 0 && signal === null && completed === fanout ? "complete" : "failed"); + }); + }); +} + +function percentile(values: readonly number[], proportion: number): number | null { + if (!values.length) return null; + const sorted = [...values].sort((left, right) => left - right); + return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(proportion * sorted.length) - 1))]; +} + +function summary(repetitions: readonly Repetition[]): Record { + const complete = repetitions.filter((entry) => entry.status === "complete"); + const peak = complete.map((entry) => (entry.peakRssKiB ?? 0) - (entry.baselineRssKiB ?? 0)); + const final = complete.map((entry) => (entry.finalRssKiB ?? 0) - (entry.baselineRssKiB ?? 0)); + return { + count: complete.length, + peakMinKiB: percentile(peak, 0), peakMedianKiB: percentile(peak, 0.5), peakP95KiB: percentile(peak, 0.95), peakMaxKiB: percentile(peak, 1), + finalMinKiB: percentile(final, 0), finalMedianKiB: percentile(final, 0.5), finalP95KiB: percentile(final, 0.95), finalMaxKiB: percentile(final, 1), + }; +} + +async function gitSha(): Promise { + const git = spawn("git", ["rev-parse", "HEAD"], { stdio: ["ignore", "pipe", "ignore"] }); + let value = ""; + git.stdout?.setEncoding("utf8"); git.stdout?.on("data", (chunk: string) => { value += chunk; }); + const code = await new Promise((resolve) => git.once("exit", resolve)); + const sha = value.trim(); + return code === 0 && /^[a-f0-9]{40}$/.test(sha) ? sha : null; +} + +async function hashTree(directory: string): Promise { + const names = (await readdir(directory)).filter((name) => name !== "manifest.json").sort(); + return Promise.all(names.map(async (name) => { + const content = await readFile(join(directory, name)); + return { name, sha256: sha256(content), bytes: content.byteLength }; + })); +} + +async function main(): Promise { + const settings = config(); + const current = platform(); + if (settings.platformRequired && settings.platformRequired !== current) throw new Error("platform_required_mismatch"); + const platformKind: SupportedPlatform | undefined = current === "linux" || current === "darwin" ? current : undefined; + // Do not produce a plausible-looking zero-RSS cell if the native counter is + // absent or inaccessible. The capability is checked before any campaign work. + const kind = platformKind && (await collectorAvailable(platformKind)) ? platformKind : undefined; + await mkdir(settings.output, { recursive: true, mode: 0o700 }); + await chmod(settings.output, 0o700); + const scratch = join(dirname(settings.output), ".b00b-rss-scratch"); + await mkdir(scratch, { recursive: true, mode: 0o700 }); + await chmod(scratch, 0o700); + const runs: Repetition[] = []; + if (!kind) { + for (const fanout of settings.fanouts) runs.push({ schemaVersion: SCHEMA_VERSION, kind: "b00b-rss-repetition", status: "unsupported", fanout, repetition: 0, warmup: true, sampler: null, reasonCode: 3, baselineRssKiB: null, peakRssKiB: null, terminalRssKiB: null, finalRssKiB: null, allocatedBytes: 0, completed: 0, failed: fanout, timedOut: false, samples: [] }); + } else { + for (const fanout of settings.fanouts) { + runs.push(await runCell(settings, kind, fanout, 0, true, scratch)); + for (let repetition = 1; repetition <= settings.repetitions; repetition += 1) runs.push(await runCell(settings, kind, fanout, repetition, false, scratch)); + } + } + await rm(scratch, { force: true, recursive: true }); + for (const run of runs) await writeOwnerFile(join(settings.output, `run-${run.fanout}-${run.repetition}-${run.warmup ? 0 : 1}.json`), `${canonical(run)}\n`); + const manifest = { + schemaVersion: SCHEMA_VERSION, kind: "b00b-rss-campaign", platform: current, release: release(), node: process.version, cpuCount: cpus().length, memoryBytes: totalmem(), gitSha: await gitSha(), + collector: kind === "linux" ? "proc-status" : kind === "darwin" ? "ps" : "unsupported", intervalMs: settings.intervalMs, timeoutMs: settings.timeoutMs, + fanouts: settings.fanouts, repetitions: settings.repetitions, warmups: 1, allocationMiB: settings.allocationMiB, + // Command/env/provider payloads intentionally are absent. This bit only states whether an external fixture was used. + externalFixture: settings.fixtureCommand !== undefined, + summaries: settings.fanouts.map((fanout) => ({ fanout, ...summary(runs.filter((run) => run.fanout === fanout && !run.warmup)) })), + files: await hashTree(settings.output), + }; + await writeOwnerFile(join(settings.output, "manifest.json"), `${canonical(manifest)}\n`); + console.log(`b00b-rss: ${runs.filter((run) => run.status === "complete" && !run.warmup).length} completed cells`); +} + +await main(); From 4be3dd19c1ca42ef74ee84aab5f97992943482a5 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 20:04:56 -0700 Subject: [PATCH 13/28] fix(coding-agent): harden RSS campaign cleanup (cherry picked from commit 1a051af313aeb5ebec9d331fdc6a4b88035c16f0) --- .../test/swarm/rss-campaign-worker.ts | 75 ++-- .../test/swarm/rss-campaign.test.ts | 89 +++++ .../test/swarm/run-production-rss-campaign.ts | 357 +++++++++--------- 3 files changed, 298 insertions(+), 223 deletions(-) create mode 100644 packages/coding-agent/test/swarm/rss-campaign.test.ts diff --git a/packages/coding-agent/test/swarm/rss-campaign-worker.ts b/packages/coding-agent/test/swarm/rss-campaign-worker.ts index 445ecfd66..65751cb70 100644 --- a/packages/coding-agent/test/swarm/rss-campaign-worker.ts +++ b/packages/coding-agent/test/swarm/rss-campaign-worker.ts @@ -1,12 +1,12 @@ /** - * Disposable, test-only child supervisor for the PR-B00B RSS campaign. + * Disposable, test-only child supervisor for the B00B RSS campaign. * It deliberately has no provider imports, network client, daemon listener, or - * persistent state. The parent owns this process group and measures it. + * persistent state. The parent owns this process group and measures it. */ -import { mkdtemp, mkdir, rm } from "node:fs/promises"; +import { spawn, type ChildProcess } from "node:child_process"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { spawn, type ChildProcess } from "node:child_process"; interface WorkerOptions { fanout: number; @@ -17,7 +17,7 @@ interface WorkerOptions { } type WorkerMessage = - | { type: "boundary"; phase: "started" | "barrier-held" | "terminals" | "cleanup"; allocatedBytes: number } + | { type: "boundary"; phase: "started" | "barrier-held" | "terminals" | "cleanup"; allocatedBytes: number; memberPids: readonly number[] } | { type: "result"; completed: number; failed: number; allocatedBytes: number }; function option(name: string): string | undefined { @@ -64,34 +64,38 @@ function safeEnvironment(worker: number, fanout: number, allocationBytes: number return environment; } -// This fixture is intentionally local and deterministic. An integration can -// replace it with --fixture-command/--fixture-arg without the launcher ever -// serializing that command, its arguments, or its output into campaign data. const BUILTIN_FIXTURE = [ "const bytes=Number(process.env.B00B_FIXTURE_ALLOCATION_BYTES||0);", "const b=Buffer.allocUnsafe(bytes);for(let i=0;iprocess.exit(0),50);", ].join(""); -function runFixture(config: WorkerOptions, worker: number, allocationBytes: number): Promise { +interface Fixture { + pid: number; + exit: Promise; +} + +function launchFixture(config: WorkerOptions, worker: number, allocationBytes: number): Fixture | undefined { const command = config.fixtureCommand ?? process.execPath; const args = config.fixtureCommand ? [...config.fixtureArgs] : ["-e", BUILTIN_FIXTURE]; - return new Promise((resolve) => { - let child: ChildProcess; - try { - child = spawn(command, args, { - cwd: process.cwd(), - detached: false, - env: safeEnvironment(worker, config.fanout, allocationBytes), - stdio: "ignore", - }); - } catch { - resolve(false); - return; - } - child.once("error", () => resolve(false)); - child.once("exit", (code, signal) => resolve(code === 0 && signal === null)); - }); + try { + const child: ChildProcess = spawn(command, args, { + cwd: process.cwd(), + detached: false, + env: safeEnvironment(worker, config.fanout, allocationBytes), + stdio: "ignore", + }); + if (!child.pid) return undefined; + return { + pid: child.pid, + exit: new Promise((resolve) => { + child.once("error", () => resolve(false)); + child.once("exit", (code, signal) => resolve(code === 0 && signal === null)); + }), + }; + } catch { + return undefined; + } } function send(message: WorkerMessage): void { @@ -105,28 +109,25 @@ function pause(milliseconds: number): Promise { async function main(): Promise { const config = options(); const allocationBytes = config.allocationMiB * 1024 * 1024; - // The supervisor allocation and each fixture allocation are touched so RSS - // has a deliberate, numeric-only allocation proof. let allocation = Buffer.allocUnsafe(allocationBytes); for (let index = 0; index < allocation.length; index += 4096) allocation[index] = 1; const runtimeRoot = await mkdtemp(join(config.scratch, "b00b-rss-")); try { await Promise.all([mkdir(join(runtimeRoot, "agent")), mkdir(join(runtimeRoot, "socket")), mkdir(join(runtimeRoot, "output"))]); - send({ type: "boundary", phase: "started", allocatedBytes: allocationBytes }); - const fixtures = Array.from({ length: config.fanout }, (_, index) => runFixture(config, index + 1, allocationBytes)); - // All fixture entries are dispatched before this boundary. This is an - // observation boundary, never a permit, queue, or admission limiter. - send({ type: "boundary", phase: "barrier-held", allocatedBytes: allocationBytes * (config.fanout + 1) }); - const results = await Promise.all(fixtures); + const fixtures = Array.from({ length: config.fanout }, (_, index) => launchFixture(config, index + 1, allocationBytes)); + const memberPids = fixtures.flatMap((fixture) => (fixture ? [fixture.pid] : [])); + send({ type: "boundary", phase: "started", allocatedBytes: allocationBytes, memberPids }); + // Every fixture is dispatched before this observation boundary. It is never + // a permit, queue, semaphore, or admission limiter. + send({ type: "boundary", phase: "barrier-held", allocatedBytes: allocationBytes * (config.fanout + 1), memberPids }); + const results = await Promise.all(fixtures.map((fixture) => fixture?.exit ?? Promise.resolve(false))); const completed = results.filter(Boolean).length; - send({ type: "boundary", phase: "terminals", allocatedBytes: allocationBytes * (config.fanout + 1) }); - // Hold the terminal boundary long enough for the 20 Hz parent sampler to - // capture it; this is post-terminal observation only, not admission. + send({ type: "boundary", phase: "terminals", allocatedBytes: allocationBytes * (config.fanout + 1), memberPids }); await pause(100); allocation = Buffer.alloc(0); global.gc?.(); await rm(runtimeRoot, { force: true, recursive: true }); - send({ type: "boundary", phase: "cleanup", allocatedBytes: 0 }); + send({ type: "boundary", phase: "cleanup", allocatedBytes: 0, memberPids }); send({ type: "result", completed, failed: config.fanout - completed, allocatedBytes: 0 }); } finally { await rm(runtimeRoot, { force: true, recursive: true }); diff --git a/packages/coding-agent/test/swarm/rss-campaign.test.ts b/packages/coding-agent/test/swarm/rss-campaign.test.ts new file mode 100644 index 000000000..66baeb468 --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-campaign.test.ts @@ -0,0 +1,89 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, it } from "vitest"; + +const execute = promisify(execFile); +const launcher = fileURLToPath(new URL("./run-production-rss-campaign.ts", import.meta.url)); +const tsx = fileURLToPath(new URL("../../../../node_modules/tsx/dist/cli.mjs", import.meta.url)); +const temporary: string[] = []; + +afterEach(async () => { + await Promise.all(temporary.splice(0).map((path) => rm(path, { force: true, recursive: true }))); +}); + +async function directory(label: string): Promise { + const path = await mkdtemp(join(tmpdir(), `b00b-rss-${label}-`)); + temporary.push(path); + return path; +} + +async function campaign(output: string, args: readonly string[]): Promise { + await execute(process.execPath, [tsx, launcher, "--output", output, ...args], { cwd: fileURLToPath(new URL("../../../../", import.meta.url)), timeout: 30_000 }); +} + +async function run(output: string, fanout: number, repetition: number): Promise> { + return JSON.parse(await readFile(join(output, `run-${fanout}-${repetition}-1.json`), "utf8")) as Record; +} + +describe("B00B RSS campaign", () => { + it("writes complete structured dry artifacts rather than zero-looking macOS data", async () => { + const output = join(await directory("dry"), "output"); + await campaign(output, ["--fanout", "1", "--repetitions", "2"]); + const first = await run(output, 1, 1); + const second = await run(output, 1, 2); + if (process.platform === "darwin") { + expect(first.status).toBe("unsupported"); + expect(second.status).toBe("unsupported"); + expect(first.sampler).toBeNull(); + expect(first.finalRssKiB).toBeNull(); + } + const mode = (await stat(join(output, "manifest.json"))).mode & 0o777; + expect(mode).toBe(0o600); + }); + + it.skipIf(process.platform !== "linux")("reaps a SIGTERM-ignoring descendant after its group leader exits", async () => { + const root = await directory("reap"); + const output = join(root, "output"); + const pidFile = join(root, "fixture.pid"); + const fixture = join(root, "ignore-term.cjs"); + await writeFile(fixture, "require('fs').writeFileSync(process.argv[2], String(process.pid));process.on('SIGTERM',()=>{});setInterval(()=>{},1000);", { mode: 0o700 }); + await campaign(output, ["--fanout", "1", "--repetitions", "1", "--timeout-ms", "500", "--fixture-command", process.execPath, "--fixture-arg", fixture, "--fixture-arg", pidFile]); + const pid = Number(await readFile(pidFile, "utf8")); + const result = await run(output, 1, 1); + expect(result.status).toBe("timed_out"); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(() => process.kill(pid, 0)).toThrow(); + }); + + it.skipIf(process.platform !== "linux")("records a real 20 Hz fanout-64 cadence or marks the cell failed", async () => { + const output = join(await directory("cadence"), "output"); + await campaign(output, ["--fanout", "64", "--repetitions", "1", "--interval-ms", "50"]); + const result = await run(output, 64, 1); + if (result.status === "complete") { + const periodic = (result.samples as { phase: string; monotonicMs: number }[]).filter((sample) => sample.phase === "started"); + const gaps = periodic.slice(1).map((sample, index) => sample.monotonicMs - periodic[index].monotonicMs); + expect(gaps.length).toBeGreaterThan(0); + expect(Math.max(...gaps)).toBeLessThanOrEqual(50); + } else { + expect(result.reasonCode).toBe(4); + } + }); + + it("never archives a fixture secret, command, or argument", async () => { + const root = await directory("secret"); + const output = join(root, "output"); + const secret = "B00B_RSS_SECRET_4f85d5c7"; + const fixture = join(root, "secret-fixture.cjs"); + await writeFile(fixture, "setTimeout(()=>process.exit(0),10)"); + await campaign(output, ["--fanout", "1", "--repetitions", "1", "--fixture-command", process.execPath, "--fixture-arg", fixture, "--fixture-arg", secret]); + const artifact = await Promise.all(["run-1-0-0.json", "run-1-1-1.json", "manifest.json"].map((name) => readFile(join(output, name), "utf8"))); + for (const content of artifact) { + expect(content).not.toContain(secret); + expect(content).not.toContain(fixture); + } + }); +}); diff --git a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts index fcd2f2c7a..062c24e21 100644 --- a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts +++ b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts @@ -6,25 +6,28 @@ * real-provider fixture can be supplied with --fixture-command; its command, * arguments, stdout, stderr, and environment are deliberately not archived. */ +import { spawn, type ChildProcess } from "node:child_process"; import { createHash } from "node:crypto"; import { chmod, mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; import { cpus, platform, release, totalmem } from "node:os"; -import { basename, dirname, join } from "node:path"; -import { spawn, type ChildProcess } from "node:child_process"; +import { dirname, join } from "node:path"; const FANOUTS = [1, 4, 16, 64] as const; const WORKER = new URL("./rss-campaign-worker.ts", import.meta.url); const MIN_INTERVAL_MS = 50; const DEFAULT_TIMEOUT_MS = 60_000; +const REAP_GRACE_MS = 250; +const REAP_VERIFY_MS = 1_000; const SCHEMA_VERSION = 1; -type SupportedPlatform = "linux" | "darwin"; +type SupportedPlatform = "linux"; type Phase = "baseline" | "started" | "barrier-held" | "terminals" | "cleanup" | "final"; type Status = "complete" | "failed" | "timed_out" | "unsupported"; interface ProcessRecord { pid: number; ppid: number; + pgid: number; start: number; rssKiB: number; } @@ -40,6 +43,7 @@ interface BoundaryMessage { type: "boundary"; phase: Exclude; allocatedBytes: number; + memberPids: readonly number[]; } interface ResultMessage { @@ -58,7 +62,7 @@ interface Repetition { fanout: number; repetition: number; warmup: boolean; - sampler: { source: "proc-status" | "ps"; intervalMs: number; sharedPages: "summed-per-process" } | null; + sampler: { source: "proc-status"; intervalMs: number; sharedPages: "summed-per-process" } | null; reasonCode: number | null; baselineRssKiB: number | null; peakRssKiB: number | null; @@ -83,6 +87,12 @@ interface Config { allocationMiB: number; } +interface GroupOwnership { + pgid: number; + leader: ProcessRecord; + members: Map; +} + function option(name: string): string | undefined { const index = process.argv.indexOf(name); return index < 0 ? undefined : process.argv[index + 1]; @@ -97,9 +107,7 @@ function safeInteger(name: string, fallback: number, minimum: number): number { function parseFanouts(value: string | undefined): readonly number[] { if (!value) return FANOUTS; const values = value.split(",").map(Number); - if (!values.length || values.some((value) => !FANOUTS.includes(value as (typeof FANOUTS)[number]))) { - throw new Error("invalid_fanout"); - } + if (!values.length || values.some((value) => !FANOUTS.includes(value as (typeof FANOUTS)[number]))) throw new Error("invalid_fanout"); return [...new Set(values)]; } @@ -139,10 +147,7 @@ function canonical(value: unknown): string { if (value === null || typeof value !== "object") return JSON.stringify(value); if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; const object = value as Record; - return `{${Object.keys(object) - .sort() - .map((key) => `${JSON.stringify(key)}:${canonical(object[key])}`) - .join(",")}}`; + return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonical(object[key])}`).join(",")}}`; } async function writeOwnerFile(path: string, content: string): Promise { @@ -150,89 +155,36 @@ async function writeOwnerFile(path: string, content: string): Promise { await chmod(path, 0o600); } -async function procStartAndRss(pid: number): Promise { +async function procRecord(pid: number): Promise { try { - const [statLine, status] = await Promise.all([ - readFile(`/proc/${pid}/stat`, "utf8"), - readFile(`/proc/${pid}/status`, "utf8"), - ]); + const [statLine, status] = await Promise.all([readFile(`/proc/${pid}/stat`, "utf8"), readFile(`/proc/${pid}/status`, "utf8")]); const close = statLine.lastIndexOf(")"); const fields = statLine.slice(close + 2).trim().split(/\s+/); const ppid = Number(fields[1]); // field 4 after removing pid/comm + const pgid = Number(fields[2]); // field 5 after removing pid/comm const start = Number(fields[19]); // field 22 after removing pid/comm const rss = /^VmRSS:\s+(\d+)\s+kB$/m.exec(status)?.[1]; - if (!Number.isSafeInteger(ppid) || !Number.isSafeInteger(start) || rss === undefined) return undefined; - return { pid, ppid, start, rssKiB: Number(rss) }; + if (![ppid, pgid, start].every(Number.isSafeInteger) || rss === undefined) return undefined; + return { pid, ppid, pgid, start, rssKiB: Number(rss) }; } catch { return undefined; } } -async function linuxSnapshot(rootPid: number): Promise { +async function groupSnapshot(pgid: number): Promise { let entries: string[]; try { entries = await readdir("/proc"); } catch { return []; } - const records = (await Promise.all(entries.filter((entry) => /^\d+$/.test(entry)).map((entry) => procStartAndRss(Number(entry))))).filter( - (record): record is ProcessRecord => record !== undefined, - ); - const descendants = new Set([rootPid]); - let changed = true; - while (changed) { - changed = false; - for (const record of records) { - if (descendants.has(record.ppid) && !descendants.has(record.pid)) { - descendants.add(record.pid); - changed = true; - } - } - } - return records.filter((record) => descendants.has(record.pid)).sort((left, right) => left.pid - right.pid); + const records = await Promise.all(entries.filter((entry) => /^\d+$/.test(entry)).map((entry) => procRecord(Number(entry)))); + return records.filter((record): record is ProcessRecord => record?.pgid === pgid).sort((left, right) => left.pid - right.pid); } -async function macSnapshot(rootPid: number): Promise { - const ps = spawn("ps", ["-axo", "pid=,ppid=,rss=,lstart="], { stdio: ["ignore", "pipe", "ignore"] }); - let text = ""; - ps.stdout?.setEncoding("utf8"); - ps.stdout?.on("data", (chunk: string) => { - text += chunk; - }); - const exited = await new Promise((resolve) => { - ps.once("error", () => resolve(false)); - ps.once("exit", (code) => resolve(code === 0)); - }); - if (!exited) return []; - const records: ProcessRecord[] = []; - for (const line of text.split("\n")) { - const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(.+?)\s*$/.exec(line); - if (!match) continue; - const start = Date.parse(match[4]); - if (Number.isNaN(start)) continue; - records.push({ pid: Number(match[1]), ppid: Number(match[2]), rssKiB: Number(match[3]), start }); - } - const descendants = new Set([rootPid]); - let changed = true; - while (changed) { - changed = false; - for (const record of records) { - if (descendants.has(record.ppid) && !descendants.has(record.pid)) { - descendants.add(record.pid); - changed = true; - } - } - } - return records.filter((record) => descendants.has(record.pid)).sort((left, right) => left.pid - right.pid); -} - -async function snapshot(kind: SupportedPlatform, rootPid: number): Promise { - return kind === "linux" ? linuxSnapshot(rootPid) : macSnapshot(rootPid); -} - -async function collectorAvailable(kind: SupportedPlatform): Promise { - const own = (await snapshot(kind, process.pid)).find((record) => record.pid === process.pid); - return own !== undefined && own.rssKiB >= 0 && Number.isSafeInteger(own.start); +async function collectorAvailable(): Promise { + const own = await procRecord(process.pid); + return own !== undefined && own.rssKiB >= 0 && Number.isSafeInteger(own.start) && Number.isSafeInteger(own.pgid); } function total(records: readonly ProcessRecord[]): number { @@ -240,128 +192,152 @@ function total(records: readonly ProcessRecord[]): number { } function sample(phase: Phase, records: readonly ProcessRecord[]): ProcessSample { + // This timestamp is taken only after the native collection finished. return { phase, monotonicMs: monotonicMs(), totalRssKiB: total(records), processes: records }; } -async function identityMatches(kind: SupportedPlatform, record: ProcessRecord): Promise { - const current = (await snapshot(kind, record.pid)).find((candidate) => candidate.pid === record.pid); - return current?.start === record.start; +function sameIdentity(left: ProcessRecord, right: ProcessRecord): boolean { + return left.pid === right.pid && left.start === right.start && left.pgid === right.pgid; } -async function reapOwnGroup(kind: SupportedPlatform, leader?: ProcessRecord): Promise { - if (!leader || !(await identityMatches(kind, leader))) return; - try { - process.kill(-leader.pid, "SIGTERM"); - } catch { - return; - } - await new Promise((resolve) => setTimeout(resolve, 250)); - if (await identityMatches(kind, leader)) { +function remember(ownership: GroupOwnership, records: readonly ProcessRecord[]): void { + for (const record of records) ownership.members.set(record.pid, record); +} + +function hasOwnedAnchor(ownership: GroupOwnership, records: readonly ProcessRecord[]): boolean { + return records.some((record) => { + const known = ownership.members.get(record.pid); + return known !== undefined && sameIdentity(known, record); + }); +} + +function pause(milliseconds: number): Promise { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +async function reapOwnGroup(ownership?: GroupOwnership): Promise { + if (!ownership) return true; + const signalOwnedGroup = async (signal: NodeJS.Signals): Promise => { + const records = await groupSnapshot(ownership.pgid); + // A negative PID can affect a reused PGID. Signal only when a process whose + // PID, start tick, and PGID we captured is still anchoring this exact group. + if (!hasOwnedAnchor(ownership, records)) return records.length === 0; + remember(ownership, records); try { - process.kill(-leader.pid, "SIGKILL"); + process.kill(-ownership.pgid, signal); + return true; } catch { - // The group may have cleanly exited between the identity check and kill. + return false; } - } + }; + if (!(await signalOwnedGroup("SIGTERM"))) return false; + await pause(REAP_GRACE_MS); + let records = await groupSnapshot(ownership.pgid); + if (records.length === 0) return true; + if (!(await signalOwnedGroup("SIGKILL"))) return false; + const deadline = monotonicMs() + REAP_VERIFY_MS; + do { + await pause(10); + records = await groupSnapshot(ownership.pgid); + } while (records.length > 0 && monotonicMs() < deadline); + return records.length === 0; } -function workerArguments(config: Config, fanout: number, scratch: string): string[] { - const args = ["--expose-gc", ...process.execArgv, WORKER.pathname, "--fanout", String(fanout), "--allocation-mib", String(config.allocationMiB), "--scratch", scratch]; - if (config.fixtureCommand) args.push("--fixture-command", config.fixtureCommand); - for (const fixtureArg of config.fixtureArgs) args.push("--fixture-arg", fixtureArg); +function workerArguments(settings: Config, fanout: number, scratch: string): string[] { + const args = ["--expose-gc", ...process.execArgv, WORKER.pathname, "--fanout", String(fanout), "--allocation-mib", String(settings.allocationMiB), "--scratch", scratch]; + if (settings.fixtureCommand) args.push("--fixture-command", settings.fixtureCommand); + for (const fixtureArg of settings.fixtureArgs) args.push("--fixture-arg", fixtureArg); return args; } -async function runCell(config: Config, kind: SupportedPlatform, fanout: number, repetition: number, warmup: boolean, scratch: string): Promise { +function unsupportedRun(fanout: number, repetition: number, warmup: boolean): Repetition { + return { schemaVersion: SCHEMA_VERSION, kind: "b00b-rss-repetition", status: "unsupported", fanout, repetition, warmup, sampler: null, reasonCode: 3, baselineRssKiB: null, peakRssKiB: null, terminalRssKiB: null, finalRssKiB: null, allocatedBytes: 0, completed: 0, failed: fanout, timedOut: false, samples: [] }; +} + +async function runCell(settings: Config, fanout: number, repetition: number, warmup: boolean, scratch: string): Promise { const samples: ProcessSample[] = [sample("baseline", [])]; - let leader: ProcessRecord | undefined; + let ownership: GroupOwnership | undefined; let child: ChildProcess | undefined; - let timer: NodeJS.Timeout | undefined; + let stopped = false; let timedOut = false; let completed = 0; let failed = fanout; let allocatedBytes = 0; - let resolved = false; - const finish = async (status: Status): Promise => { - if (timer) clearInterval(timer); - await reapOwnGroup(kind, leader); - const final = sample("final", leader ? await snapshot(kind, leader.pid) : []); - samples.push(final); - const byPhase = (phase: Phase) => samples.filter((entry) => entry.phase === phase).at(-1)?.totalRssKiB ?? null; - const active = samples.filter((entry) => entry.phase !== "baseline" && entry.phase !== "final"); - return { - schemaVersion: SCHEMA_VERSION, - kind: "b00b-rss-repetition", - status, - fanout, - repetition, - warmup, - sampler: { source: kind === "linux" ? "proc-status" : "ps", intervalMs: config.intervalMs, sharedPages: "summed-per-process" }, - reasonCode: status === "complete" ? null : status === "timed_out" ? 1 : 2, - baselineRssKiB: 0, - peakRssKiB: active.length ? Math.max(...active.map((entry) => entry.totalRssKiB)) : null, - terminalRssKiB: byPhase("terminals"), - finalRssKiB: final.totalRssKiB, - allocatedBytes, - completed, - failed, - timedOut, - samples, - }; + let cadenceFailed = false; + let queue = Promise.resolve(); + let lastPeriodicSample: number | undefined; + let periodicSamples = 0; + const pendingMemberPids = new Set(); + const enqueue = (phase: Phase, memberPids: readonly number[] = []): Promise => { + for (const pid of memberPids) pendingMemberPids.add(pid); + queue = queue.then(async () => { + if (!ownership) return; + // The worker supplies its direct fixture PIDs at each boundary. Preserve + // their PID/start/PGID identities before a timeout can make the leader exit. + const announced = await Promise.all([...pendingMemberPids].map(procRecord)); + remember(ownership, announced.filter((record): record is ProcessRecord => record?.pgid === ownership.pgid)); + const records = await groupSnapshot(ownership.pgid); + remember(ownership, records); + const entry = sample(phase, records); + if (phase === "started") { + if (lastPeriodicSample !== undefined && entry.monotonicMs - lastPeriodicSample > settings.intervalMs) cadenceFailed = true; + lastPeriodicSample = entry.monotonicMs; + periodicSamples += 1; + } + samples.push(entry); + }); + return queue; }; return new Promise((resolve) => { - const settle = async (status: Status) => { - if (resolved) return; - resolved = true; - resolve(await finish(status)); + let timer: NodeJS.Timeout | undefined; + let timeout: NodeJS.Timeout | undefined; + const settle = async (requested: Status): Promise => { + if (stopped) return; + stopped = true; + if (timer) clearInterval(timer); + if (timeout) clearTimeout(timeout); + await queue; + if (requested === "complete" && periodicSamples < 2) cadenceFailed = true; + const reaped = await reapOwnGroup(ownership); + const finalRecords = ownership ? await groupSnapshot(ownership.pgid) : []; + samples.push(sample("final", finalRecords)); + const status = requested === "complete" && (!reaped || cadenceFailed) ? "failed" : requested; + const byPhase = (phase: Phase) => samples.filter((entry) => entry.phase === phase).at(-1)?.totalRssKiB ?? null; + const active = samples.filter((entry) => entry.phase !== "baseline" && entry.phase !== "final"); + resolve({ + schemaVersion: SCHEMA_VERSION, kind: "b00b-rss-repetition", status, fanout, repetition, warmup, + sampler: { source: "proc-status", intervalMs: settings.intervalMs, sharedPages: "summed-per-process" }, + reasonCode: status === "complete" ? null : timedOut ? 1 : cadenceFailed ? 4 : 2, + baselineRssKiB: 0, peakRssKiB: active.length ? Math.max(...active.map((entry) => entry.totalRssKiB)) : null, + terminalRssKiB: byPhase("terminals"), finalRssKiB: total(finalRecords), allocatedBytes, completed, failed, timedOut, samples, + }); }; try { - child = spawn(process.execPath, workerArguments(config, fanout, scratch), { - cwd: process.cwd(), - detached: process.platform !== "win32", + child = spawn(process.execPath, workerArguments(settings, fanout, scratch), { + cwd: process.cwd(), detached: true, env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, LANG: "C", LC_ALL: "C" }, - serialization: "json", - stdio: ["ignore", "ignore", "ignore", "ipc"], + serialization: "json", stdio: ["ignore", "ignore", "ignore", "ipc"], }); } catch { void settle("failed"); return; } const pid = child.pid; - if (!pid) { - void settle("failed"); - return; - } - void snapshot(kind, pid).then((records) => { - leader = records.find((record) => record.pid === pid); - samples.push(sample("started", records)); - }); - timer = setInterval(() => { - if (!leader || resolved) return; - void snapshot(kind, leader.pid).then((records) => samples.push(sample("started", records))); - }, config.intervalMs); - const timeout = setTimeout(() => { - timedOut = true; - void settle("timed_out"); - }, config.timeoutMs); - child.once("error", () => { - clearTimeout(timeout); - void settle("failed"); + if (!pid) { void settle("failed"); return; } + void procRecord(pid).then((leader) => { + if (!leader || leader.pgid !== pid || stopped) { void settle("failed"); return; } + ownership = { pgid: leader.pgid, leader, members: new Map([[leader.pid, leader]]) }; + void enqueue("started"); + timer = setInterval(() => { if (!stopped) void enqueue("started"); }, settings.intervalMs); }); + timeout = setTimeout(() => { timedOut = true; void settle("timed_out"); }, settings.timeoutMs); + child.once("error", () => void settle("failed")); child.on("message", (message: WorkerMessage) => { - if (message.type === "result") { - completed = message.completed; - failed = message.failed; - allocatedBytes = Math.max(allocatedBytes, message.allocatedBytes); - return; - } + if (message.type === "result") { completed = message.completed; failed = message.failed; allocatedBytes = Math.max(allocatedBytes, message.allocatedBytes); return; } allocatedBytes = Math.max(allocatedBytes, message.allocatedBytes); - if (leader) void snapshot(kind, leader.pid).then((records) => samples.push(sample(message.phase, records))); - }); - child.once("exit", (code, signal) => { - clearTimeout(timeout); - void settle(code === 0 && signal === null && completed === fanout ? "complete" : "failed"); + void enqueue(message.phase, message.memberPids); }); + child.once("exit", (code, signal) => void settle(code === 0 && signal === null && completed === fanout ? "complete" : "failed")); }); } @@ -375,17 +351,14 @@ function summary(repetitions: readonly Repetition[]): Record entry.status === "complete"); const peak = complete.map((entry) => (entry.peakRssKiB ?? 0) - (entry.baselineRssKiB ?? 0)); const final = complete.map((entry) => (entry.finalRssKiB ?? 0) - (entry.baselineRssKiB ?? 0)); - return { - count: complete.length, - peakMinKiB: percentile(peak, 0), peakMedianKiB: percentile(peak, 0.5), peakP95KiB: percentile(peak, 0.95), peakMaxKiB: percentile(peak, 1), - finalMinKiB: percentile(final, 0), finalMedianKiB: percentile(final, 0.5), finalP95KiB: percentile(final, 0.95), finalMaxKiB: percentile(final, 1), - }; + return { count: complete.length, peakMinKiB: percentile(peak, 0), peakMedianKiB: percentile(peak, 0.5), peakP95KiB: percentile(peak, 0.95), peakMaxKiB: percentile(peak, 1), finalMinKiB: percentile(final, 0), finalMedianKiB: percentile(final, 0.5), finalP95KiB: percentile(final, 0.95), finalMaxKiB: percentile(final, 1) }; } async function gitSha(): Promise { const git = spawn("git", ["rev-parse", "HEAD"], { stdio: ["ignore", "pipe", "ignore"] }); let value = ""; - git.stdout?.setEncoding("utf8"); git.stdout?.on("data", (chunk: string) => { value += chunk; }); + git.stdout?.setEncoding("utf8"); + git.stdout?.on("data", (chunk: string) => { value += chunk; }); const code = await new Promise((resolve) => git.once("exit", resolve)); const sha = value.trim(); return code === 0 && /^[a-f0-9]{40}$/.test(sha) ? sha : null; @@ -394,40 +367,52 @@ async function gitSha(): Promise { async function hashTree(directory: string): Promise { const names = (await readdir(directory)).filter((name) => name !== "manifest.json").sort(); return Promise.all(names.map(async (name) => { - const content = await readFile(join(directory, name)); + const path = join(directory, name); + if (!(await stat(path)).isFile()) throw new Error("output_contains_non_file"); + const content = await readFile(path); return { name, sha256: sha256(content), bytes: content.byteLength }; })); } +async function freshOutput(directory: string): Promise { + try { + const info = await stat(directory); + if (!info.isDirectory() || (await readdir(directory)).length > 0) throw new Error("output_must_be_new_or_empty"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + await mkdir(directory, { mode: 0o700 }); + } + await chmod(directory, 0o700); +} + async function main(): Promise { const settings = config(); const current = platform(); - if (settings.platformRequired && settings.platformRequired !== current) throw new Error("platform_required_mismatch"); - const platformKind: SupportedPlatform | undefined = current === "linux" || current === "darwin" ? current : undefined; - // Do not produce a plausible-looking zero-RSS cell if the native counter is - // absent or inaccessible. The capability is checked before any campaign work. - const kind = platformKind && (await collectorAvailable(platformKind)) ? platformKind : undefined; - await mkdir(settings.output, { recursive: true, mode: 0o700 }); - await chmod(settings.output, 0o700); - const scratch = join(dirname(settings.output), ".b00b-rss-scratch"); + const platformMatches = !settings.platformRequired || settings.platformRequired === current; + // Darwin ps lstart is wall-clock, whole-second data. It cannot establish the + // PID/start identity needed before destructive negative-PID signals, so macOS + // is explicitly unsupported rather than pretending its lstart values are safe. + const kind: SupportedPlatform | undefined = platformMatches && current === "linux" && (await collectorAvailable()) ? "linux" : undefined; + await freshOutput(settings.output); + const scratch = join(dirname(settings.output), `.b00b-rss-scratch-${process.pid}`); await mkdir(scratch, { recursive: true, mode: 0o700 }); await chmod(scratch, 0o700); const runs: Repetition[] = []; - if (!kind) { - for (const fanout of settings.fanouts) runs.push({ schemaVersion: SCHEMA_VERSION, kind: "b00b-rss-repetition", status: "unsupported", fanout, repetition: 0, warmup: true, sampler: null, reasonCode: 3, baselineRssKiB: null, peakRssKiB: null, terminalRssKiB: null, finalRssKiB: null, allocatedBytes: 0, completed: 0, failed: fanout, timedOut: false, samples: [] }); - } else { - for (const fanout of settings.fanouts) { - runs.push(await runCell(settings, kind, fanout, 0, true, scratch)); - for (let repetition = 1; repetition <= settings.repetitions; repetition += 1) runs.push(await runCell(settings, kind, fanout, repetition, false, scratch)); + for (const fanout of settings.fanouts) { + if (!kind) { + runs.push(unsupportedRun(fanout, 0, true)); + for (let repetition = 1; repetition <= settings.repetitions; repetition += 1) runs.push(unsupportedRun(fanout, repetition, false)); + continue; } + runs.push(await runCell(settings, fanout, 0, true, scratch)); + for (let repetition = 1; repetition <= settings.repetitions; repetition += 1) runs.push(await runCell(settings, fanout, repetition, false, scratch)); } await rm(scratch, { force: true, recursive: true }); for (const run of runs) await writeOwnerFile(join(settings.output, `run-${run.fanout}-${run.repetition}-${run.warmup ? 0 : 1}.json`), `${canonical(run)}\n`); const manifest = { schemaVersion: SCHEMA_VERSION, kind: "b00b-rss-campaign", platform: current, release: release(), node: process.version, cpuCount: cpus().length, memoryBytes: totalmem(), gitSha: await gitSha(), - collector: kind === "linux" ? "proc-status" : kind === "darwin" ? "ps" : "unsupported", intervalMs: settings.intervalMs, timeoutMs: settings.timeoutMs, + collector: kind === "linux" ? "proc-status" : "unsupported", intervalMs: settings.intervalMs, timeoutMs: settings.timeoutMs, fanouts: settings.fanouts, repetitions: settings.repetitions, warmups: 1, allocationMiB: settings.allocationMiB, - // Command/env/provider payloads intentionally are absent. This bit only states whether an external fixture was used. externalFixture: settings.fixtureCommand !== undefined, summaries: settings.fanouts.map((fanout) => ({ fanout, ...summary(runs.filter((run) => run.fanout === fanout && !run.warmup)) })), files: await hashTree(settings.output), From 6b0dbc7efee21b84767fd7e8bdd9fb76fa737e5b Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 20:14:41 -0700 Subject: [PATCH 14/28] style(coding-agent): format RSS campaign files (cherry picked from commit 27de42170303d6942af9443af8827c553a246c74) --- .../test/swarm/rss-campaign-worker.ts | 26 ++- .../test/swarm/rss-campaign.test.ts | 104 ++++++--- .../test/swarm/run-production-rss-campaign.ts | 201 ++++++++++++++---- 3 files changed, 256 insertions(+), 75 deletions(-) diff --git a/packages/coding-agent/test/swarm/rss-campaign-worker.ts b/packages/coding-agent/test/swarm/rss-campaign-worker.ts index 65751cb70..c90d14893 100644 --- a/packages/coding-agent/test/swarm/rss-campaign-worker.ts +++ b/packages/coding-agent/test/swarm/rss-campaign-worker.ts @@ -3,7 +3,7 @@ * It deliberately has no provider imports, network client, daemon listener, or * persistent state. The parent owns this process group and measures it. */ -import { spawn, type ChildProcess } from "node:child_process"; +import { type ChildProcess, spawn } from "node:child_process"; import { mkdir, mkdtemp, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -17,7 +17,12 @@ interface WorkerOptions { } type WorkerMessage = - | { type: "boundary"; phase: "started" | "barrier-held" | "terminals" | "cleanup"; allocatedBytes: number; memberPids: readonly number[] } + | { + type: "boundary"; + phase: "started" | "barrier-held" | "terminals" | "cleanup"; + allocatedBytes: number; + memberPids: readonly number[]; + } | { type: "result"; completed: number; failed: number; allocatedBytes: number }; function option(name: string): string | undefined { @@ -113,13 +118,24 @@ async function main(): Promise { for (let index = 0; index < allocation.length; index += 4096) allocation[index] = 1; const runtimeRoot = await mkdtemp(join(config.scratch, "b00b-rss-")); try { - await Promise.all([mkdir(join(runtimeRoot, "agent")), mkdir(join(runtimeRoot, "socket")), mkdir(join(runtimeRoot, "output"))]); - const fixtures = Array.from({ length: config.fanout }, (_, index) => launchFixture(config, index + 1, allocationBytes)); + await Promise.all([ + mkdir(join(runtimeRoot, "agent")), + mkdir(join(runtimeRoot, "socket")), + mkdir(join(runtimeRoot, "output")), + ]); + const fixtures = Array.from({ length: config.fanout }, (_, index) => + launchFixture(config, index + 1, allocationBytes), + ); const memberPids = fixtures.flatMap((fixture) => (fixture ? [fixture.pid] : [])); send({ type: "boundary", phase: "started", allocatedBytes: allocationBytes, memberPids }); // Every fixture is dispatched before this observation boundary. It is never // a permit, queue, semaphore, or admission limiter. - send({ type: "boundary", phase: "barrier-held", allocatedBytes: allocationBytes * (config.fanout + 1), memberPids }); + send({ + type: "boundary", + phase: "barrier-held", + allocatedBytes: allocationBytes * (config.fanout + 1), + memberPids, + }); const results = await Promise.all(fixtures.map((fixture) => fixture?.exit ?? Promise.resolve(false))); const completed = results.filter(Boolean).length; send({ type: "boundary", phase: "terminals", allocatedBytes: allocationBytes * (config.fanout + 1), memberPids }); diff --git a/packages/coding-agent/test/swarm/rss-campaign.test.ts b/packages/coding-agent/test/swarm/rss-campaign.test.ts index 66baeb468..4e5bd4112 100644 --- a/packages/coding-agent/test/swarm/rss-campaign.test.ts +++ b/packages/coding-agent/test/swarm/rss-campaign.test.ts @@ -22,11 +22,17 @@ async function directory(label: string): Promise { } async function campaign(output: string, args: readonly string[]): Promise { - await execute(process.execPath, [tsx, launcher, "--output", output, ...args], { cwd: fileURLToPath(new URL("../../../../", import.meta.url)), timeout: 30_000 }); + await execute(process.execPath, [tsx, launcher, "--output", output, ...args], { + cwd: fileURLToPath(new URL("../../../../", import.meta.url)), + timeout: 30_000, + }); } async function run(output: string, fanout: number, repetition: number): Promise> { - return JSON.parse(await readFile(join(output, `run-${fanout}-${repetition}-1.json`), "utf8")) as Record; + return JSON.parse(await readFile(join(output, `run-${fanout}-${repetition}-1.json`), "utf8")) as Record< + string, + unknown + >; } describe("B00B RSS campaign", () => { @@ -45,33 +51,58 @@ describe("B00B RSS campaign", () => { expect(mode).toBe(0o600); }); - it.skipIf(process.platform !== "linux")("reaps a SIGTERM-ignoring descendant after its group leader exits", async () => { - const root = await directory("reap"); - const output = join(root, "output"); - const pidFile = join(root, "fixture.pid"); - const fixture = join(root, "ignore-term.cjs"); - await writeFile(fixture, "require('fs').writeFileSync(process.argv[2], String(process.pid));process.on('SIGTERM',()=>{});setInterval(()=>{},1000);", { mode: 0o700 }); - await campaign(output, ["--fanout", "1", "--repetitions", "1", "--timeout-ms", "500", "--fixture-command", process.execPath, "--fixture-arg", fixture, "--fixture-arg", pidFile]); - const pid = Number(await readFile(pidFile, "utf8")); - const result = await run(output, 1, 1); - expect(result.status).toBe("timed_out"); - await new Promise((resolve) => setTimeout(resolve, 100)); - expect(() => process.kill(pid, 0)).toThrow(); - }); + it.skipIf(process.platform !== "linux")( + "reaps a SIGTERM-ignoring descendant after its group leader exits", + async () => { + const root = await directory("reap"); + const output = join(root, "output"); + const pidFile = join(root, "fixture.pid"); + const fixture = join(root, "ignore-term.cjs"); + await writeFile( + fixture, + "require('fs').writeFileSync(process.argv[2], String(process.pid));process.on('SIGTERM',()=>{});setInterval(()=>{},1000);", + { mode: 0o700 }, + ); + await campaign(output, [ + "--fanout", + "1", + "--repetitions", + "1", + "--timeout-ms", + "500", + "--fixture-command", + process.execPath, + "--fixture-arg", + fixture, + "--fixture-arg", + pidFile, + ]); + const pid = Number(await readFile(pidFile, "utf8")); + const result = await run(output, 1, 1); + expect(result.status).toBe("timed_out"); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(() => process.kill(pid, 0)).toThrow(); + }, + ); - it.skipIf(process.platform !== "linux")("records a real 20 Hz fanout-64 cadence or marks the cell failed", async () => { - const output = join(await directory("cadence"), "output"); - await campaign(output, ["--fanout", "64", "--repetitions", "1", "--interval-ms", "50"]); - const result = await run(output, 64, 1); - if (result.status === "complete") { - const periodic = (result.samples as { phase: string; monotonicMs: number }[]).filter((sample) => sample.phase === "started"); - const gaps = periodic.slice(1).map((sample, index) => sample.monotonicMs - periodic[index].monotonicMs); - expect(gaps.length).toBeGreaterThan(0); - expect(Math.max(...gaps)).toBeLessThanOrEqual(50); - } else { - expect(result.reasonCode).toBe(4); - } - }); + it.skipIf(process.platform !== "linux")( + "records a real 20 Hz fanout-64 cadence or marks the cell failed", + async () => { + const output = join(await directory("cadence"), "output"); + await campaign(output, ["--fanout", "64", "--repetitions", "1", "--interval-ms", "50"]); + const result = await run(output, 64, 1); + if (result.status === "complete") { + const periodic = (result.samples as { phase: string; monotonicMs: number }[]).filter( + (sample) => sample.phase === "started", + ); + const gaps = periodic.slice(1).map((sample, index) => sample.monotonicMs - periodic[index].monotonicMs); + expect(gaps.length).toBeGreaterThan(0); + expect(Math.max(...gaps)).toBeLessThanOrEqual(50); + } else { + expect(result.reasonCode).toBe(4); + } + }, + ); it("never archives a fixture secret, command, or argument", async () => { const root = await directory("secret"); @@ -79,8 +110,21 @@ describe("B00B RSS campaign", () => { const secret = "B00B_RSS_SECRET_4f85d5c7"; const fixture = join(root, "secret-fixture.cjs"); await writeFile(fixture, "setTimeout(()=>process.exit(0),10)"); - await campaign(output, ["--fanout", "1", "--repetitions", "1", "--fixture-command", process.execPath, "--fixture-arg", fixture, "--fixture-arg", secret]); - const artifact = await Promise.all(["run-1-0-0.json", "run-1-1-1.json", "manifest.json"].map((name) => readFile(join(output, name), "utf8"))); + await campaign(output, [ + "--fanout", + "1", + "--repetitions", + "1", + "--fixture-command", + process.execPath, + "--fixture-arg", + fixture, + "--fixture-arg", + secret, + ]); + const artifact = await Promise.all( + ["run-1-0-0.json", "run-1-1-1.json", "manifest.json"].map((name) => readFile(join(output, name), "utf8")), + ); for (const content of artifact) { expect(content).not.toContain(secret); expect(content).not.toContain(fixture); diff --git a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts index 062c24e21..a05b86208 100644 --- a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts +++ b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts @@ -6,9 +6,9 @@ * real-provider fixture can be supplied with --fixture-command; its command, * arguments, stdout, stderr, and environment are deliberately not archived. */ -import { spawn, type ChildProcess } from "node:child_process"; +import { type ChildProcess, spawn } from "node:child_process"; import { createHash } from "node:crypto"; -import { chmod, mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises"; +import { chmod, mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; import { cpus, platform, release, totalmem } from "node:os"; import { dirname, join } from "node:path"; @@ -107,7 +107,8 @@ function safeInteger(name: string, fallback: number, minimum: number): number { function parseFanouts(value: string | undefined): readonly number[] { if (!value) return FANOUTS; const values = value.split(",").map(Number); - if (!values.length || values.some((value) => !FANOUTS.includes(value as (typeof FANOUTS)[number]))) throw new Error("invalid_fanout"); + if (!values.length || values.some((value) => !FANOUTS.includes(value as (typeof FANOUTS)[number]))) + throw new Error("invalid_fanout"); return [...new Set(values)]; } @@ -147,7 +148,10 @@ function canonical(value: unknown): string { if (value === null || typeof value !== "object") return JSON.stringify(value); if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; const object = value as Record; - return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonical(object[key])}`).join(",")}}`; + return `{${Object.keys(object) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonical(object[key])}`) + .join(",")}}`; } async function writeOwnerFile(path: string, content: string): Promise { @@ -157,9 +161,15 @@ async function writeOwnerFile(path: string, content: string): Promise { async function procRecord(pid: number): Promise { try { - const [statLine, status] = await Promise.all([readFile(`/proc/${pid}/stat`, "utf8"), readFile(`/proc/${pid}/status`, "utf8")]); + const [statLine, status] = await Promise.all([ + readFile(`/proc/${pid}/stat`, "utf8"), + readFile(`/proc/${pid}/status`, "utf8"), + ]); const close = statLine.lastIndexOf(")"); - const fields = statLine.slice(close + 2).trim().split(/\s+/); + const fields = statLine + .slice(close + 2) + .trim() + .split(/\s+/); const ppid = Number(fields[1]); // field 4 after removing pid/comm const pgid = Number(fields[2]); // field 5 after removing pid/comm const start = Number(fields[19]); // field 22 after removing pid/comm @@ -178,8 +188,12 @@ async function groupSnapshot(pgid: number): Promise { } catch { return []; } - const records = await Promise.all(entries.filter((entry) => /^\d+$/.test(entry)).map((entry) => procRecord(Number(entry)))); - return records.filter((record): record is ProcessRecord => record?.pgid === pgid).sort((left, right) => left.pid - right.pid); + const records = await Promise.all( + entries.filter((entry) => /^\d+$/.test(entry)).map((entry) => procRecord(Number(entry))), + ); + return records + .filter((record): record is ProcessRecord => record?.pgid === pgid) + .sort((left, right) => left.pid - right.pid); } async function collectorAvailable(): Promise { @@ -244,17 +258,51 @@ async function reapOwnGroup(ownership?: GroupOwnership): Promise { } function workerArguments(settings: Config, fanout: number, scratch: string): string[] { - const args = ["--expose-gc", ...process.execArgv, WORKER.pathname, "--fanout", String(fanout), "--allocation-mib", String(settings.allocationMiB), "--scratch", scratch]; + const args = [ + "--expose-gc", + ...process.execArgv, + WORKER.pathname, + "--fanout", + String(fanout), + "--allocation-mib", + String(settings.allocationMiB), + "--scratch", + scratch, + ]; if (settings.fixtureCommand) args.push("--fixture-command", settings.fixtureCommand); for (const fixtureArg of settings.fixtureArgs) args.push("--fixture-arg", fixtureArg); return args; } function unsupportedRun(fanout: number, repetition: number, warmup: boolean): Repetition { - return { schemaVersion: SCHEMA_VERSION, kind: "b00b-rss-repetition", status: "unsupported", fanout, repetition, warmup, sampler: null, reasonCode: 3, baselineRssKiB: null, peakRssKiB: null, terminalRssKiB: null, finalRssKiB: null, allocatedBytes: 0, completed: 0, failed: fanout, timedOut: false, samples: [] }; + return { + schemaVersion: SCHEMA_VERSION, + kind: "b00b-rss-repetition", + status: "unsupported", + fanout, + repetition, + warmup, + sampler: null, + reasonCode: 3, + baselineRssKiB: null, + peakRssKiB: null, + terminalRssKiB: null, + finalRssKiB: null, + allocatedBytes: 0, + completed: 0, + failed: fanout, + timedOut: false, + samples: [], + }; } -async function runCell(settings: Config, fanout: number, repetition: number, warmup: boolean, scratch: string): Promise { +async function runCell( + settings: Config, + fanout: number, + repetition: number, + warmup: boolean, + scratch: string, +): Promise { const samples: ProcessSample[] = [sample("baseline", [])]; let ownership: GroupOwnership | undefined; let child: ChildProcess | undefined; @@ -275,12 +323,16 @@ async function runCell(settings: Config, fanout: number, repetition: number, war // The worker supplies its direct fixture PIDs at each boundary. Preserve // their PID/start/PGID identities before a timeout can make the leader exit. const announced = await Promise.all([...pendingMemberPids].map(procRecord)); - remember(ownership, announced.filter((record): record is ProcessRecord => record?.pgid === ownership.pgid)); + remember( + ownership, + announced.filter((record): record is ProcessRecord => record?.pgid === ownership.pgid), + ); const records = await groupSnapshot(ownership.pgid); remember(ownership, records); const entry = sample(phase, records); if (phase === "started") { - if (lastPeriodicSample !== undefined && entry.monotonicMs - lastPeriodicSample > settings.intervalMs) cadenceFailed = true; + if (lastPeriodicSample !== undefined && entry.monotonicMs - lastPeriodicSample > settings.intervalMs) + cadenceFailed = true; lastPeriodicSample = entry.monotonicMs; periodicSamples += 1; } @@ -305,39 +357,72 @@ async function runCell(settings: Config, fanout: number, repetition: number, war const byPhase = (phase: Phase) => samples.filter((entry) => entry.phase === phase).at(-1)?.totalRssKiB ?? null; const active = samples.filter((entry) => entry.phase !== "baseline" && entry.phase !== "final"); resolve({ - schemaVersion: SCHEMA_VERSION, kind: "b00b-rss-repetition", status, fanout, repetition, warmup, + schemaVersion: SCHEMA_VERSION, + kind: "b00b-rss-repetition", + status, + fanout, + repetition, + warmup, sampler: { source: "proc-status", intervalMs: settings.intervalMs, sharedPages: "summed-per-process" }, reasonCode: status === "complete" ? null : timedOut ? 1 : cadenceFailed ? 4 : 2, - baselineRssKiB: 0, peakRssKiB: active.length ? Math.max(...active.map((entry) => entry.totalRssKiB)) : null, - terminalRssKiB: byPhase("terminals"), finalRssKiB: total(finalRecords), allocatedBytes, completed, failed, timedOut, samples, + baselineRssKiB: 0, + peakRssKiB: active.length ? Math.max(...active.map((entry) => entry.totalRssKiB)) : null, + terminalRssKiB: byPhase("terminals"), + finalRssKiB: total(finalRecords), + allocatedBytes, + completed, + failed, + timedOut, + samples, }); }; try { child = spawn(process.execPath, workerArguments(settings, fanout, scratch), { - cwd: process.cwd(), detached: true, + cwd: process.cwd(), + detached: true, env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, LANG: "C", LC_ALL: "C" }, - serialization: "json", stdio: ["ignore", "ignore", "ignore", "ipc"], + serialization: "json", + stdio: ["ignore", "ignore", "ignore", "ipc"], }); } catch { void settle("failed"); return; } const pid = child.pid; - if (!pid) { void settle("failed"); return; } + if (!pid) { + void settle("failed"); + return; + } void procRecord(pid).then((leader) => { - if (!leader || leader.pgid !== pid || stopped) { void settle("failed"); return; } + if (!leader || leader.pgid !== pid || stopped) { + void settle("failed"); + return; + } ownership = { pgid: leader.pgid, leader, members: new Map([[leader.pid, leader]]) }; void enqueue("started"); - timer = setInterval(() => { if (!stopped) void enqueue("started"); }, settings.intervalMs); + timer = setInterval(() => { + if (!stopped) void enqueue("started"); + }, settings.intervalMs); }); - timeout = setTimeout(() => { timedOut = true; void settle("timed_out"); }, settings.timeoutMs); + timeout = setTimeout(() => { + timedOut = true; + void settle("timed_out"); + }, settings.timeoutMs); child.once("error", () => void settle("failed")); child.on("message", (message: WorkerMessage) => { - if (message.type === "result") { completed = message.completed; failed = message.failed; allocatedBytes = Math.max(allocatedBytes, message.allocatedBytes); return; } + if (message.type === "result") { + completed = message.completed; + failed = message.failed; + allocatedBytes = Math.max(allocatedBytes, message.allocatedBytes); + return; + } allocatedBytes = Math.max(allocatedBytes, message.allocatedBytes); void enqueue(message.phase, message.memberPids); }); - child.once("exit", (code, signal) => void settle(code === 0 && signal === null && completed === fanout ? "complete" : "failed")); + child.once( + "exit", + (code, signal) => void settle(code === 0 && signal === null && completed === fanout ? "complete" : "failed"), + ); }); } @@ -351,14 +436,26 @@ function summary(repetitions: readonly Repetition[]): Record entry.status === "complete"); const peak = complete.map((entry) => (entry.peakRssKiB ?? 0) - (entry.baselineRssKiB ?? 0)); const final = complete.map((entry) => (entry.finalRssKiB ?? 0) - (entry.baselineRssKiB ?? 0)); - return { count: complete.length, peakMinKiB: percentile(peak, 0), peakMedianKiB: percentile(peak, 0.5), peakP95KiB: percentile(peak, 0.95), peakMaxKiB: percentile(peak, 1), finalMinKiB: percentile(final, 0), finalMedianKiB: percentile(final, 0.5), finalP95KiB: percentile(final, 0.95), finalMaxKiB: percentile(final, 1) }; + return { + count: complete.length, + peakMinKiB: percentile(peak, 0), + peakMedianKiB: percentile(peak, 0.5), + peakP95KiB: percentile(peak, 0.95), + peakMaxKiB: percentile(peak, 1), + finalMinKiB: percentile(final, 0), + finalMedianKiB: percentile(final, 0.5), + finalP95KiB: percentile(final, 0.95), + finalMaxKiB: percentile(final, 1), + }; } async function gitSha(): Promise { const git = spawn("git", ["rev-parse", "HEAD"], { stdio: ["ignore", "pipe", "ignore"] }); let value = ""; git.stdout?.setEncoding("utf8"); - git.stdout?.on("data", (chunk: string) => { value += chunk; }); + git.stdout?.on("data", (chunk: string) => { + value += chunk; + }); const code = await new Promise((resolve) => git.once("exit", resolve)); const sha = value.trim(); return code === 0 && /^[a-f0-9]{40}$/.test(sha) ? sha : null; @@ -366,12 +463,14 @@ async function gitSha(): Promise { async function hashTree(directory: string): Promise { const names = (await readdir(directory)).filter((name) => name !== "manifest.json").sort(); - return Promise.all(names.map(async (name) => { - const path = join(directory, name); - if (!(await stat(path)).isFile()) throw new Error("output_contains_non_file"); - const content = await readFile(path); - return { name, sha256: sha256(content), bytes: content.byteLength }; - })); + return Promise.all( + names.map(async (name) => { + const path = join(directory, name); + if (!(await stat(path)).isFile()) throw new Error("output_contains_non_file"); + const content = await readFile(path); + return { name, sha256: sha256(content), bytes: content.byteLength }; + }), + ); } async function freshOutput(directory: string): Promise { @@ -392,7 +491,8 @@ async function main(): Promise { // Darwin ps lstart is wall-clock, whole-second data. It cannot establish the // PID/start identity needed before destructive negative-PID signals, so macOS // is explicitly unsupported rather than pretending its lstart values are safe. - const kind: SupportedPlatform | undefined = platformMatches && current === "linux" && (await collectorAvailable()) ? "linux" : undefined; + const kind: SupportedPlatform | undefined = + platformMatches && current === "linux" && (await collectorAvailable()) ? "linux" : undefined; await freshOutput(settings.output); const scratch = join(dirname(settings.output), `.b00b-rss-scratch-${process.pid}`); await mkdir(scratch, { recursive: true, mode: 0o700 }); @@ -401,20 +501,41 @@ async function main(): Promise { for (const fanout of settings.fanouts) { if (!kind) { runs.push(unsupportedRun(fanout, 0, true)); - for (let repetition = 1; repetition <= settings.repetitions; repetition += 1) runs.push(unsupportedRun(fanout, repetition, false)); + for (let repetition = 1; repetition <= settings.repetitions; repetition += 1) + runs.push(unsupportedRun(fanout, repetition, false)); continue; } runs.push(await runCell(settings, fanout, 0, true, scratch)); - for (let repetition = 1; repetition <= settings.repetitions; repetition += 1) runs.push(await runCell(settings, fanout, repetition, false, scratch)); + for (let repetition = 1; repetition <= settings.repetitions; repetition += 1) + runs.push(await runCell(settings, fanout, repetition, false, scratch)); } await rm(scratch, { force: true, recursive: true }); - for (const run of runs) await writeOwnerFile(join(settings.output, `run-${run.fanout}-${run.repetition}-${run.warmup ? 0 : 1}.json`), `${canonical(run)}\n`); + for (const run of runs) + await writeOwnerFile( + join(settings.output, `run-${run.fanout}-${run.repetition}-${run.warmup ? 0 : 1}.json`), + `${canonical(run)}\n`, + ); const manifest = { - schemaVersion: SCHEMA_VERSION, kind: "b00b-rss-campaign", platform: current, release: release(), node: process.version, cpuCount: cpus().length, memoryBytes: totalmem(), gitSha: await gitSha(), - collector: kind === "linux" ? "proc-status" : "unsupported", intervalMs: settings.intervalMs, timeoutMs: settings.timeoutMs, - fanouts: settings.fanouts, repetitions: settings.repetitions, warmups: 1, allocationMiB: settings.allocationMiB, + schemaVersion: SCHEMA_VERSION, + kind: "b00b-rss-campaign", + platform: current, + release: release(), + node: process.version, + cpuCount: cpus().length, + memoryBytes: totalmem(), + gitSha: await gitSha(), + collector: kind === "linux" ? "proc-status" : "unsupported", + intervalMs: settings.intervalMs, + timeoutMs: settings.timeoutMs, + fanouts: settings.fanouts, + repetitions: settings.repetitions, + warmups: 1, + allocationMiB: settings.allocationMiB, externalFixture: settings.fixtureCommand !== undefined, - summaries: settings.fanouts.map((fanout) => ({ fanout, ...summary(runs.filter((run) => run.fanout === fanout && !run.warmup)) })), + summaries: settings.fanouts.map((fanout) => ({ + fanout, + ...summary(runs.filter((run) => run.fanout === fanout && !run.warmup)), + })), files: await hashTree(settings.output), }; await writeOwnerFile(join(settings.output, "manifest.json"), `${canonical(manifest)}\n`); From 940e4fbd8d70c3a198a668e35455c5cee523d924 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 20:27:17 -0700 Subject: [PATCH 15/28] fix(coding-agent): capture RSS group ownership in sampler (cherry picked from commit 0b43304d5821ea59ff85b1a11fb5109b82be6ead) --- .../test/swarm/run-production-rss-campaign.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts index a05b86208..437420f2a 100644 --- a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts +++ b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts @@ -319,16 +319,17 @@ async function runCell( const enqueue = (phase: Phase, memberPids: readonly number[] = []): Promise => { for (const pid of memberPids) pendingMemberPids.add(pid); queue = queue.then(async () => { - if (!ownership) return; + const currentOwnership = ownership; + if (!currentOwnership) return; // The worker supplies its direct fixture PIDs at each boundary. Preserve // their PID/start/PGID identities before a timeout can make the leader exit. const announced = await Promise.all([...pendingMemberPids].map(procRecord)); remember( - ownership, - announced.filter((record): record is ProcessRecord => record?.pgid === ownership.pgid), + currentOwnership, + announced.filter((record): record is ProcessRecord => record?.pgid === currentOwnership.pgid), ); - const records = await groupSnapshot(ownership.pgid); - remember(ownership, records); + const records = await groupSnapshot(currentOwnership.pgid); + remember(currentOwnership, records); const entry = sample(phase, records); if (phase === "started") { if (lastPeriodicSample !== undefined && entry.monotonicMs - lastPeriodicSample > settings.intervalMs) From 86595ac3e4fc3c5e235f5c036087fa304df1b6e8 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 20:37:02 -0700 Subject: [PATCH 16/28] fix(test): add RSS sampler jitter headroom (cherry picked from commit 6f1368010ea16193d7ec60f373cf3d0e820118d4) --- .../test/swarm/rss-campaign-cadence.ts | 15 +++ .../test/swarm/rss-campaign.test.ts | 55 +++++++-- .../test/swarm/run-production-rss-campaign.ts | 111 ++++++++++++------ 3 files changed, 132 insertions(+), 49 deletions(-) create mode 100644 packages/coding-agent/test/swarm/rss-campaign-cadence.ts diff --git a/packages/coding-agent/test/swarm/rss-campaign-cadence.ts b/packages/coding-agent/test/swarm/rss-campaign-cadence.ts new file mode 100644 index 000000000..d47736d0a --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-campaign-cadence.ts @@ -0,0 +1,15 @@ +export const MAX_RSS_SAMPLE_GAP_MS = 50; +export const DEFAULT_RSS_REQUESTED_PERIOD_MS = 25; + +export interface CadenceValidation { + maxObservedGapMs: number | null; + valid: boolean; +} + +/** Validates the unmodified monotonic timestamps emitted after each collection. */ +export function validateRssSampleCadence(timestamps: readonly number[]): CadenceValidation { + if (timestamps.length < 2) return { maxObservedGapMs: null, valid: false }; + const gaps = timestamps.slice(1).map((timestamp, index) => timestamp - timestamps[index]!); + const maxObservedGapMs = Math.max(...gaps); + return { maxObservedGapMs, valid: maxObservedGapMs <= MAX_RSS_SAMPLE_GAP_MS }; +} diff --git a/packages/coding-agent/test/swarm/rss-campaign.test.ts b/packages/coding-agent/test/swarm/rss-campaign.test.ts index 4e5bd4112..938b11f4f 100644 --- a/packages/coding-agent/test/swarm/rss-campaign.test.ts +++ b/packages/coding-agent/test/swarm/rss-campaign.test.ts @@ -5,6 +5,7 @@ import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { afterEach, describe, expect, it } from "vitest"; +import { validateRssSampleCadence } from "./rss-campaign-cadence.js"; const execute = promisify(execFile); const launcher = fileURLToPath(new URL("./run-production-rss-campaign.ts", import.meta.url)); @@ -85,22 +86,52 @@ describe("B00B RSS campaign", () => { }, ); + it("validates unchanged sample timestamps at the 50 ms max-gap contract", () => { + expect(validateRssSampleCadence([0, 50, 100]).valid).toBe(true); + expect(validateRssSampleCadence([0, 25, 76])).toEqual({ maxObservedGapMs: 51, valid: false }); + }); + it.skipIf(process.platform !== "linux")( - "records a real 20 Hz fanout-64 cadence or marks the cell failed", + "uses 25 ms jitter headroom while recording the 50 ms cadence contract", async () => { const output = join(await directory("cadence"), "output"); - await campaign(output, ["--fanout", "64", "--repetitions", "1", "--interval-ms", "50"]); + await campaign(output, ["--fanout", "64", "--repetitions", "1"]); const result = await run(output, 64, 1); - if (result.status === "complete") { - const periodic = (result.samples as { phase: string; monotonicMs: number }[]).filter( - (sample) => sample.phase === "started", - ); - const gaps = periodic.slice(1).map((sample, index) => sample.monotonicMs - periodic[index].monotonicMs); - expect(gaps.length).toBeGreaterThan(0); - expect(Math.max(...gaps)).toBeLessThanOrEqual(50); - } else { - expect(result.reasonCode).toBe(4); - } + const sampler = result.sampler as { + requestedPeriodMs: number; + maxGapMs: number; + maxObservedGapMs: number | null; + }; + expect(sampler.requestedPeriodMs).toBe(25); + expect(sampler.maxGapMs).toBe(50); + if (result.status === "complete") expect(sampler.maxObservedGapMs).toBeLessThanOrEqual(50); + else expect(result.reasonCode).toBe(4); + }, + ); + + it.skipIf(process.platform !== "linux")( + "fails rather than relabeling samples when injected scheduler jitter exceeds 50 ms", + async () => { + const output = join(await directory("cadence-jitter"), "output"); + await campaign(output, [ + "--fanout", + "1", + "--repetitions", + "1", + "--test-scheduler-jitter-ms", + "26", + "--fixture-command", + process.execPath, + "--fixture-arg", + "-e", + "--fixture-arg", + "setTimeout(()=>process.exit(0),150)", + ]); + const result = await run(output, 1, 1); + expect(result.status).toBe("failed"); + expect(result.reasonCode).toBe(4); + expect((result.sampler as { requestedPeriodMs: number }).requestedPeriodMs).toBe(51); + expect((result.sampler as { maxObservedGapMs: number }).maxObservedGapMs).toBeGreaterThan(50); }, ); diff --git a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts index 437420f2a..fefe0b974 100644 --- a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts +++ b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts @@ -11,14 +11,18 @@ import { createHash } from "node:crypto"; import { chmod, mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; import { cpus, platform, release, totalmem } from "node:os"; import { dirname, join } from "node:path"; +import { + DEFAULT_RSS_REQUESTED_PERIOD_MS, + MAX_RSS_SAMPLE_GAP_MS, + validateRssSampleCadence, +} from "./rss-campaign-cadence.js"; const FANOUTS = [1, 4, 16, 64] as const; const WORKER = new URL("./rss-campaign-worker.ts", import.meta.url); -const MIN_INTERVAL_MS = 50; const DEFAULT_TIMEOUT_MS = 60_000; const REAP_GRACE_MS = 250; const REAP_VERIFY_MS = 1_000; -const SCHEMA_VERSION = 1; +const SCHEMA_VERSION = 2; type SupportedPlatform = "linux"; type Phase = "baseline" | "started" | "barrier-held" | "terminals" | "cleanup" | "final"; @@ -62,7 +66,13 @@ interface Repetition { fanout: number; repetition: number; warmup: boolean; - sampler: { source: "proc-status"; intervalMs: number; sharedPages: "summed-per-process" } | null; + sampler: { + source: "proc-status"; + requestedPeriodMs: number; + maxGapMs: number; + maxObservedGapMs: number | null; + sharedPages: "summed-per-process"; + } | null; reasonCode: number | null; baselineRssKiB: number | null; peakRssKiB: number | null; @@ -79,7 +89,7 @@ interface Config { fanouts: readonly number[]; repetitions: number; output: string; - intervalMs: number; + requestedPeriodMs: number; timeoutMs: number; platformRequired?: string; fixtureCommand?: string; @@ -113,7 +123,8 @@ function parseFanouts(value: string | undefined): readonly number[] { } function config(): Config { - const intervalMs = safeInteger("--interval-ms", MIN_INTERVAL_MS, MIN_INTERVAL_MS); + const maxGapMs = safeInteger("--interval-ms", MAX_RSS_SAMPLE_GAP_MS, MAX_RSS_SAMPLE_GAP_MS); + if (maxGapMs !== MAX_RSS_SAMPLE_GAP_MS) throw new Error("interval_ms_must_be_50"); const fixtureArgs: string[] = []; for (let index = 0; index < process.argv.length; index += 1) { if (process.argv[index] === "--fixture-arg") { @@ -127,7 +138,7 @@ function config(): Config { fanouts: parseFanouts(option("--fanout")), repetitions: safeInteger("--repetitions", 3, 1), output: option("--output") ?? "b00b-rss-artifacts", - intervalMs, + requestedPeriodMs: DEFAULT_RSS_REQUESTED_PERIOD_MS + safeInteger("--test-scheduler-jitter-ms", 0, 0), timeoutMs: safeInteger("--timeout-ms", DEFAULT_TIMEOUT_MS, 1), platformRequired: option("--platform-required"), fixtureCommand: option("--fixture-command"), @@ -159,28 +170,49 @@ async function writeOwnerFile(path: string, content: string): Promise { await chmod(path, 0o600); } -async function procRecord(pid: number): Promise { +interface ProcessIdentity { + pid: number; + ppid: number; + pgid: number; + start: number; +} + +function parseProcessIdentity(pid: number, statLine: string): ProcessIdentity | undefined { + const close = statLine.lastIndexOf(")"); + const fields = statLine + .slice(close + 2) + .trim() + .split(/\s+/); + const ppid = Number(fields[1]); // field 4 after removing pid/comm + const pgid = Number(fields[2]); // field 5 after removing pid/comm + const start = Number(fields[19]); // field 22 after removing pid/comm + if (![ppid, pgid, start].every(Number.isSafeInteger)) return undefined; + return { pid, ppid, pgid, start }; +} + +async function procIdentity(pid: number): Promise { + try { + return parseProcessIdentity(pid, await readFile(`/proc/${pid}/stat`, "utf8")); + } catch { + return undefined; + } +} + +async function procRecordForIdentity(identity: ProcessIdentity): Promise { try { - const [statLine, status] = await Promise.all([ - readFile(`/proc/${pid}/stat`, "utf8"), - readFile(`/proc/${pid}/status`, "utf8"), - ]); - const close = statLine.lastIndexOf(")"); - const fields = statLine - .slice(close + 2) - .trim() - .split(/\s+/); - const ppid = Number(fields[1]); // field 4 after removing pid/comm - const pgid = Number(fields[2]); // field 5 after removing pid/comm - const start = Number(fields[19]); // field 22 after removing pid/comm + const status = await readFile(`/proc/${identity.pid}/status`, "utf8"); const rss = /^VmRSS:\s+(\d+)\s+kB$/m.exec(status)?.[1]; - if (![ppid, pgid, start].every(Number.isSafeInteger) || rss === undefined) return undefined; - return { pid, ppid, pgid, start, rssKiB: Number(rss) }; + return rss === undefined ? undefined : { ...identity, rssKiB: Number(rss) }; } catch { return undefined; } } +async function procRecord(pid: number): Promise { + const identity = await procIdentity(pid); + return identity ? procRecordForIdentity(identity) : undefined; +} + async function groupSnapshot(pgid: number): Promise { let entries: string[]; try { @@ -188,11 +220,15 @@ async function groupSnapshot(pgid: number): Promise { } catch { return []; } - const records = await Promise.all( - entries.filter((entry) => /^\d+$/.test(entry)).map((entry) => procRecord(Number(entry))), + // Scanning all /proc stat files identifies the process group without reading + // every status file; VmRSS is read only for members of this measured group. + const identities = await Promise.all( + entries.filter((entry) => /^\d+$/.test(entry)).map((entry) => procIdentity(Number(entry))), ); + const members = identities.filter((identity): identity is ProcessIdentity => identity?.pgid === pgid); + const records = await Promise.all(members.map(procRecordForIdentity)); return records - .filter((record): record is ProcessRecord => record?.pgid === pgid) + .filter((record): record is ProcessRecord => record !== undefined) .sort((left, right) => left.pid - right.pid); } @@ -311,10 +347,7 @@ async function runCell( let completed = 0; let failed = fanout; let allocatedBytes = 0; - let cadenceFailed = false; let queue = Promise.resolve(); - let lastPeriodicSample: number | undefined; - let periodicSamples = 0; const pendingMemberPids = new Set(); const enqueue = (phase: Phase, memberPids: readonly number[] = []): Promise => { for (const pid of memberPids) pendingMemberPids.add(pid); @@ -331,12 +364,6 @@ async function runCell( const records = await groupSnapshot(currentOwnership.pgid); remember(currentOwnership, records); const entry = sample(phase, records); - if (phase === "started") { - if (lastPeriodicSample !== undefined && entry.monotonicMs - lastPeriodicSample > settings.intervalMs) - cadenceFailed = true; - lastPeriodicSample = entry.monotonicMs; - periodicSamples += 1; - } samples.push(entry); }); return queue; @@ -350,7 +377,10 @@ async function runCell( if (timer) clearInterval(timer); if (timeout) clearTimeout(timeout); await queue; - if (requested === "complete" && periodicSamples < 2) cadenceFailed = true; + const cadence = validateRssSampleCadence( + samples.filter((entry) => entry.phase === "started").map((entry) => entry.monotonicMs), + ); + const cadenceFailed = !cadence.valid; const reaped = await reapOwnGroup(ownership); const finalRecords = ownership ? await groupSnapshot(ownership.pgid) : []; samples.push(sample("final", finalRecords)); @@ -364,7 +394,13 @@ async function runCell( fanout, repetition, warmup, - sampler: { source: "proc-status", intervalMs: settings.intervalMs, sharedPages: "summed-per-process" }, + sampler: { + source: "proc-status", + requestedPeriodMs: settings.requestedPeriodMs, + maxGapMs: MAX_RSS_SAMPLE_GAP_MS, + maxObservedGapMs: cadence.maxObservedGapMs, + sharedPages: "summed-per-process", + }, reasonCode: status === "complete" ? null : timedOut ? 1 : cadenceFailed ? 4 : 2, baselineRssKiB: 0, peakRssKiB: active.length ? Math.max(...active.map((entry) => entry.totalRssKiB)) : null, @@ -403,7 +439,7 @@ async function runCell( void enqueue("started"); timer = setInterval(() => { if (!stopped) void enqueue("started"); - }, settings.intervalMs); + }, settings.requestedPeriodMs); }); timeout = setTimeout(() => { timedOut = true; @@ -526,7 +562,8 @@ async function main(): Promise { memoryBytes: totalmem(), gitSha: await gitSha(), collector: kind === "linux" ? "proc-status" : "unsupported", - intervalMs: settings.intervalMs, + requestedPeriodMs: settings.requestedPeriodMs, + maxGapMs: MAX_RSS_SAMPLE_GAP_MS, timeoutMs: settings.timeoutMs, fanouts: settings.fanouts, repetitions: settings.repetitions, From 5947277e1cce336022913c3b6cb6c9deb2588b41 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 20:52:55 -0700 Subject: [PATCH 17/28] fix(test): gate RSS worker on ownership handshake (cherry picked from commit 5474a1ec731793e1ad57263f984da5acc29f0ed1) --- .../test/swarm/rss-campaign-worker.ts | 22 +++- .../test/swarm/rss-campaign.test.ts | 39 ++++++ .../test/swarm/run-production-rss-campaign.ts | 117 +++++++++++++++--- 3 files changed, 157 insertions(+), 21 deletions(-) diff --git a/packages/coding-agent/test/swarm/rss-campaign-worker.ts b/packages/coding-agent/test/swarm/rss-campaign-worker.ts index c90d14893..668d13ed1 100644 --- a/packages/coding-agent/test/swarm/rss-campaign-worker.ts +++ b/packages/coding-agent/test/swarm/rss-campaign-worker.ts @@ -14,6 +14,7 @@ interface WorkerOptions { scratch: string; fixtureCommand?: string; fixtureArgs: readonly string[]; + testIgnoreTerm: boolean; } type WorkerMessage = @@ -51,7 +52,14 @@ function options(): WorkerOptions { index += 1; } } - return { fanout, allocationMiB, scratch, fixtureCommand, fixtureArgs }; + return { + fanout, + allocationMiB, + scratch, + fixtureCommand, + fixtureArgs, + testIgnoreTerm: process.argv.includes("--test-ignore-term"), + }; } function safeEnvironment(worker: number, fanout: number, allocationBytes: number): NodeJS.ProcessEnv { @@ -111,8 +119,20 @@ function pause(milliseconds: number): Promise { return new Promise((resolve) => setTimeout(resolve, milliseconds)); } +function awaitRelease(): Promise { + return new Promise((resolve) => { + process.once("message", (message: unknown) => { + if ((message as { type?: unknown })?.type === "release") resolve(); + }); + }); +} + async function main(): Promise { const config = options(); + // No allocation, fixture, or descendant may exist before the parent has + // authenticated this leader's PID/start/PGID and explicitly releases us. + await awaitRelease(); + if (config.testIgnoreTerm) process.on("SIGTERM", () => {}); const allocationBytes = config.allocationMiB * 1024 * 1024; let allocation = Buffer.allocUnsafe(allocationBytes); for (let index = 0; index < allocation.length; index += 4096) allocation[index] = 1; diff --git a/packages/coding-agent/test/swarm/rss-campaign.test.ts b/packages/coding-agent/test/swarm/rss-campaign.test.ts index 938b11f4f..4cf3a65ee 100644 --- a/packages/coding-agent/test/swarm/rss-campaign.test.ts +++ b/packages/coding-agent/test/swarm/rss-campaign.test.ts @@ -86,6 +86,45 @@ describe("B00B RSS campaign", () => { }, ); + it.skipIf(process.platform !== "linux")( + "does not arm timeout or release descendants before exact ownership capture", + async () => { + const root = await directory("delayed-ownership"); + const output = join(root, "output"); + const pidFile = join(root, "fixture.pid"); + const fixture = join(root, "ignore-term.cjs"); + await writeFile( + fixture, + "require('fs').writeFileSync(process.argv[2],String(process.pid));process.on('SIGTERM',()=>{});setInterval(()=>{},1000);", + { mode: 0o700 }, + ); + await campaign(output, [ + "--fanout", + "1", + "--repetitions", + "1", + "--timeout-ms", + "1", + "--test-identity-capture-delay-ms", + "25", + "--test-ignore-term", + "--fixture-command", + process.execPath, + "--fixture-arg", + fixture, + "--fixture-arg", + pidFile, + ]); + const pid = Number(await readFile(pidFile, "utf8")); + const result = await run(output, 1, 1); + expect(result.status).toBe("timed_out"); + expect(result.timedOut).toBe(true); + expect(result.finalRssKiB).toBe(0); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(() => process.kill(pid, 0)).toThrow(); + }, + ); + it("validates unchanged sample timestamps at the 50 ms max-gap contract", () => { expect(validateRssSampleCadence([0, 50, 100]).valid).toBe(true); expect(validateRssSampleCadence([0, 25, 76])).toEqual({ maxObservedGapMs: 51, valid: false }); diff --git a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts index fefe0b974..d42fac8a4 100644 --- a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts +++ b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts @@ -95,6 +95,9 @@ interface Config { fixtureCommand?: string; fixtureArgs: readonly string[]; allocationMiB: number; + // Test-only delay which makes the pre-release ownership window deterministic. + identityCaptureDelayMs: number; + testIgnoreTerm: boolean; } interface GroupOwnership { @@ -144,6 +147,8 @@ function config(): Config { fixtureCommand: option("--fixture-command"), fixtureArgs, allocationMiB: safeInteger("--allocation-mib", 1, 1), + identityCaptureDelayMs: safeInteger("--test-identity-capture-delay-ms", 0, 0), + testIgnoreTerm: process.argv.includes("--test-ignore-term"), }; } @@ -307,6 +312,7 @@ function workerArguments(settings: Config, fanout: number, scratch: string): str ]; if (settings.fixtureCommand) args.push("--fixture-command", settings.fixtureCommand); for (const fixtureArg of settings.fixtureArgs) args.push("--fixture-arg", fixtureArg); + if (settings.testIgnoreTerm) args.push("--test-ignore-term"); return args; } @@ -363,15 +369,43 @@ async function runCell( ); const records = await groupSnapshot(currentOwnership.pgid); remember(currentOwnership, records); - const entry = sample(phase, records); - samples.push(entry); + samples.push(sample(phase, records)); }); return queue; }; + const reapDirectChild = async (): Promise => { + // Before release the worker protocol has not allocated or spawned anything. + // Never use a negative PGID without an authenticated /proc identity anchor. + const direct = child; + if (!direct || direct.exitCode !== null || direct.signalCode !== null) return; + try { + direct.kill("SIGKILL"); + } catch { + return; + } + await new Promise((resolve) => { + if (direct.exitCode !== null || direct.signalCode !== null) resolve(); + else direct.once("exit", () => resolve()); + }); + }; return new Promise((resolve) => { let timer: NodeJS.Timeout | undefined; let timeout: NodeJS.Timeout | undefined; - const settle = async (requested: Status): Promise => { + let executionStarted = false; + let settle: (requested: Status) => Promise; + const startExecution = (): void => { + if (executionStarted || stopped) return; + executionStarted = true; + void enqueue("started"); + timer = setInterval(() => { + if (!stopped) void enqueue("started"); + }, settings.requestedPeriodMs); + timeout = setTimeout(() => { + timedOut = true; + void settle("timed_out"); + }, settings.timeoutMs); + }; + settle = async (requested: Status): Promise => { if (stopped) return; stopped = true; if (timer) clearInterval(timer); @@ -381,10 +415,49 @@ async function runCell( samples.filter((entry) => entry.phase === "started").map((entry) => entry.monotonicMs), ); const cadenceFailed = !cadence.valid; + + // A failed startup is deliberately not represented as an empty owned + // group: no exact PID/start/PGID anchor was ever authenticated. + if (!ownership) { + await reapDirectChild(); + resolve({ + schemaVersion: SCHEMA_VERSION, + kind: "b00b-rss-repetition", + status: "failed", + fanout, + repetition, + warmup, + sampler: { + source: "proc-status", + requestedPeriodMs: settings.requestedPeriodMs, + maxGapMs: MAX_RSS_SAMPLE_GAP_MS, + maxObservedGapMs: cadence.maxObservedGapMs, + sharedPages: "summed-per-process", + }, + reasonCode: 2, + baselineRssKiB: 0, + peakRssKiB: null, + terminalRssKiB: null, + finalRssKiB: null, + allocatedBytes: 0, + completed: 0, + failed: fanout, + timedOut: false, + samples, + }); + return; + } + const reaped = await reapOwnGroup(ownership); - const finalRecords = ownership ? await groupSnapshot(ownership.pgid) : []; + const finalRecords = await groupSnapshot(ownership.pgid); samples.push(sample("final", finalRecords)); - const status = requested === "complete" && (!reaped || cadenceFailed) ? "failed" : requested; + const emptyOwnedGroup = reaped && finalRecords.length === 0; + const status = + requested === "complete" && emptyOwnedGroup && !cadenceFailed + ? "complete" + : requested === "timed_out" && emptyOwnedGroup && !cadenceFailed + ? "timed_out" + : "failed"; const byPhase = (phase: Phase) => samples.filter((entry) => entry.phase === phase).at(-1)?.totalRssKiB ?? null; const active = samples.filter((entry) => entry.phase !== "baseline" && entry.phase !== "final"); resolve({ @@ -430,23 +503,9 @@ async function runCell( void settle("failed"); return; } - void procRecord(pid).then((leader) => { - if (!leader || leader.pgid !== pid || stopped) { - void settle("failed"); - return; - } - ownership = { pgid: leader.pgid, leader, members: new Map([[leader.pid, leader]]) }; - void enqueue("started"); - timer = setInterval(() => { - if (!stopped) void enqueue("started"); - }, settings.requestedPeriodMs); - }); - timeout = setTimeout(() => { - timedOut = true; - void settle("timed_out"); - }, settings.timeoutMs); child.once("error", () => void settle("failed")); child.on("message", (message: WorkerMessage) => { + if (message.type === "boundary") startExecution(); if (message.type === "result") { completed = message.completed; failed = message.failed; @@ -460,6 +519,24 @@ async function runCell( "exit", (code, signal) => void settle(code === 0 && signal === null && completed === fanout ? "complete" : "failed"), ); + void (async () => { + if (settings.identityCaptureDelayMs) await pause(settings.identityCaptureDelayMs); + const leader = await procRecord(pid); + if (!leader || leader.pgid !== pid || stopped) { + void settle("failed"); + return; + } + ownership = { pgid: leader.pgid, leader, members: new Map([[leader.pid, leader]]) }; + // The worker remains gated until this release. Its first boundary proves + // execution began, at which point startExecution arms the timeout. + try { + child?.send({ type: "release" }, (error) => { + if (error && !stopped) void settle("failed"); + }); + } catch { + void settle("failed"); + } + })(); }); } From a5ed92ebc15b84528dfdd8c3bf784d9eee857e6f Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 20:55:19 -0700 Subject: [PATCH 18/28] test(coding-agent): stabilize ownership timeout race (cherry picked from commit 68132ed1e7ea6b83ee00868d0ea5ebe35e21b37a) --- packages/coding-agent/test/swarm/rss-campaign.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/test/swarm/rss-campaign.test.ts b/packages/coding-agent/test/swarm/rss-campaign.test.ts index 4cf3a65ee..e020a1268 100644 --- a/packages/coding-agent/test/swarm/rss-campaign.test.ts +++ b/packages/coding-agent/test/swarm/rss-campaign.test.ts @@ -104,9 +104,9 @@ describe("B00B RSS campaign", () => { "--repetitions", "1", "--timeout-ms", - "1", + "250", "--test-identity-capture-delay-ms", - "25", + "500", "--test-ignore-term", "--fixture-command", process.execPath, From 497b79c208d423f19374f37388bb46562c0840ca Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 21:06:13 -0700 Subject: [PATCH 19/28] fix(rss): fail closed on unavailable proc scans (cherry picked from commit cc8f12b7c8659e9f0473ef86ac4f06c8e4fd733d) --- .../test/swarm/rss-campaign.test.ts | 38 ++++++ .../test/swarm/run-production-rss-campaign.ts | 124 ++++++++++++++---- 2 files changed, 134 insertions(+), 28 deletions(-) diff --git a/packages/coding-agent/test/swarm/rss-campaign.test.ts b/packages/coding-agent/test/swarm/rss-campaign.test.ts index e020a1268..e1d706651 100644 --- a/packages/coding-agent/test/swarm/rss-campaign.test.ts +++ b/packages/coding-agent/test/swarm/rss-campaign.test.ts @@ -86,6 +86,44 @@ describe("B00B RSS campaign", () => { }, ); + it.skipIf(process.platform !== "linux")( + "fails closed when the final scan is unavailable after a TERM-ignoring descendant", + async () => { + const root = await directory("final-scan-failure"); + const output = join(root, "output"); + const pidFile = join(root, "fixture.pid"); + const fixture = join(root, "ignore-term.cjs"); + await writeFile( + fixture, + "require('fs').writeFileSync(process.argv[2],String(process.pid));process.on('SIGTERM',()=>{});setInterval(()=>{},1000);", + { mode: 0o700 }, + ); + await campaign(output, [ + "--fanout", + "1", + "--repetitions", + "1", + "--timeout-ms", + "500", + "--test-fail-final-scan", + "--fixture-command", + process.execPath, + "--fixture-arg", + fixture, + "--fixture-arg", + pidFile, + ]); + const pid = Number(await readFile(pidFile, "utf8")); + const result = await run(output, 1, 1); + expect(result.status).toBe("failed"); + expect(result.reasonCode).toBe(5); + expect(result.finalRssKiB).toBeNull(); + expect((result.samples as { phase: string }[]).some((sample) => sample.phase === "final")).toBe(false); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(() => process.kill(pid, 0)).toThrow(); + }, + ); + it.skipIf(process.platform !== "linux")( "does not arm timeout or release descendants before exact ownership capture", async () => { diff --git a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts index d42fac8a4..63224097e 100644 --- a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts +++ b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts @@ -98,6 +98,8 @@ interface Config { // Test-only delay which makes the pre-release ownership window deterministic. identityCaptureDelayMs: number; testIgnoreTerm: boolean; + // Test-only deterministic final-observation fault injection. + testFailFinalScan: boolean; } interface GroupOwnership { @@ -149,6 +151,7 @@ function config(): Config { allocationMiB: safeInteger("--allocation-mib", 1, 1), identityCaptureDelayMs: safeInteger("--test-identity-capture-delay-ms", 0, 0), testIgnoreTerm: process.argv.includes("--test-ignore-term"), + testFailFinalScan: process.argv.includes("--test-fail-final-scan"), }; } @@ -218,23 +221,64 @@ async function procRecord(pid: number): Promise { return identity ? procRecordForIdentity(identity) : undefined; } -async function groupSnapshot(pgid: number): Promise { +type GroupSnapshot = + | { kind: "empty" } + | { kind: "records"; records: readonly ProcessRecord[] } + | { kind: "unavailable" }; + +function snapshotRecords(snapshot: GroupSnapshot): readonly ProcessRecord[] | undefined { + if (snapshot.kind === "unavailable") return undefined; + return snapshot.kind === "empty" ? [] : snapshot.records; +} + +function exitedDuringScan(error: unknown): boolean { + const code = (error as NodeJS.ErrnoException).code; + return code === "ENOENT" || code === "ESRCH"; +} + +/** + * Collect a complete group view. An empty records result is a positive claim: + * every numeric /proc entry was read and no member of this PGID remained. + * Any non-disappearance read or parse failure makes the entire scan unusable; + * collapsing it to [] could authorize a signal or a false zero-RSS result. + */ +async function groupSnapshot(pgid: number, injectFailure = false): Promise { + if (injectFailure) return { kind: "unavailable" }; let entries: string[]; try { entries = await readdir("/proc"); } catch { - return []; + return { kind: "unavailable" }; } - // Scanning all /proc stat files identifies the process group without reading - // every status file; VmRSS is read only for members of this measured group. - const identities = await Promise.all( - entries.filter((entry) => /^\d+$/.test(entry)).map((entry) => procIdentity(Number(entry))), - ); - const members = identities.filter((identity): identity is ProcessIdentity => identity?.pgid === pgid); - const records = await Promise.all(members.map(procRecordForIdentity)); - return records - .filter((record): record is ProcessRecord => record !== undefined) - .sort((left, right) => left.pid - right.pid); + const identities: ProcessIdentity[] = []; + for (const entry of entries) { + if (!/^\d+$/.test(entry)) continue; + const pid = Number(entry); + try { + const identity = parseProcessIdentity(pid, await readFile(`/proc/${pid}/stat`, "utf8")); + if (!identity) return { kind: "unavailable" }; + identities.push(identity); + } catch (error) { + // A process which vanished between readdir and stat cannot be a live, + // unobserved group member. Every other unreadable numeric entry fails closed. + if (!exitedDuringScan(error)) return { kind: "unavailable" }; + } + } + const records: ProcessRecord[] = []; + for (const identity of identities) { + if (identity.pgid !== pgid) continue; + try { + const status = await readFile(`/proc/${identity.pid}/status`, "utf8"); + const rss = /^VmRSS:\s+(\d+)\s+kB$/m.exec(status)?.[1]; + if (rss === undefined) return { kind: "unavailable" }; + records.push({ ...identity, rssKiB: Number(rss) }); + } catch (error) { + // Only a confirmed disappearance is safe to omit after membership was found. + if (!exitedDuringScan(error)) return { kind: "unavailable" }; + } + } + const sorted = records.sort((left, right) => left.pid - right.pid); + return sorted.length === 0 ? { kind: "empty" } : { kind: "records", records: sorted }; } async function collectorAvailable(): Promise { @@ -270,10 +314,21 @@ function pause(milliseconds: number): Promise { return new Promise((resolve) => setTimeout(resolve, milliseconds)); } -async function reapOwnGroup(ownership?: GroupOwnership): Promise { - if (!ownership) return true; +interface ReapResult { + reaped: boolean; + collectionFailed: boolean; +} + +async function reapOwnGroup(ownership?: GroupOwnership): Promise { + if (!ownership) return { reaped: true, collectionFailed: false }; + let collectionFailed = false; const signalOwnedGroup = async (signal: NodeJS.Signals): Promise => { - const records = await groupSnapshot(ownership.pgid); + const snapshot = await groupSnapshot(ownership.pgid); + const records = snapshotRecords(snapshot); + if (!records) { + collectionFailed = true; + return false; + } // A negative PID can affect a reused PGID. Signal only when a process whose // PID, start tick, and PGID we captured is still anchoring this exact group. if (!hasOwnedAnchor(ownership, records)) return records.length === 0; @@ -285,17 +340,21 @@ async function reapOwnGroup(ownership?: GroupOwnership): Promise { return false; } }; - if (!(await signalOwnedGroup("SIGTERM"))) return false; + if (!(await signalOwnedGroup("SIGTERM"))) return { reaped: false, collectionFailed }; await pause(REAP_GRACE_MS); - let records = await groupSnapshot(ownership.pgid); - if (records.length === 0) return true; - if (!(await signalOwnedGroup("SIGKILL"))) return false; + let snapshot = await groupSnapshot(ownership.pgid); + let records = snapshotRecords(snapshot); + if (!records) return { reaped: false, collectionFailed: true }; + if (records.length === 0) return { reaped: true, collectionFailed }; + if (!(await signalOwnedGroup("SIGKILL"))) return { reaped: false, collectionFailed }; const deadline = monotonicMs() + REAP_VERIFY_MS; do { await pause(10); - records = await groupSnapshot(ownership.pgid); + snapshot = await groupSnapshot(ownership.pgid); + records = snapshotRecords(snapshot); + if (!records) return { reaped: false, collectionFailed: true }; } while (records.length > 0 && monotonicMs() < deadline); - return records.length === 0; + return { reaped: records.length === 0, collectionFailed }; } function workerArguments(settings: Config, fanout: number, scratch: string): string[] { @@ -353,6 +412,7 @@ async function runCell( let completed = 0; let failed = fanout; let allocatedBytes = 0; + let collectorFailed = false; let queue = Promise.resolve(); const pendingMemberPids = new Set(); const enqueue = (phase: Phase, memberPids: readonly number[] = []): Promise => { @@ -367,7 +427,12 @@ async function runCell( currentOwnership, announced.filter((record): record is ProcessRecord => record?.pgid === currentOwnership.pgid), ); - const records = await groupSnapshot(currentOwnership.pgid); + const snapshot = await groupSnapshot(currentOwnership.pgid); + const records = snapshotRecords(snapshot); + if (!records) { + collectorFailed = true; + return; + } remember(currentOwnership, records); samples.push(sample(phase, records)); }); @@ -448,10 +513,13 @@ async function runCell( return; } - const reaped = await reapOwnGroup(ownership); - const finalRecords = await groupSnapshot(ownership.pgid); - samples.push(sample("final", finalRecords)); - const emptyOwnedGroup = reaped && finalRecords.length === 0; + const reap = await reapOwnGroup(ownership); + const finalSnapshot = await groupSnapshot(ownership.pgid, settings.testFailFinalScan); + const finalRecords = snapshotRecords(finalSnapshot); + const finalCollectionFailed = finalRecords === undefined; + if (finalRecords) samples.push(sample("final", finalRecords)); + const collectionFailure = collectorFailed || reap.collectionFailed || finalCollectionFailed; + const emptyOwnedGroup = !collectionFailure && reap.reaped && finalRecords?.length === 0; const status = requested === "complete" && emptyOwnedGroup && !cadenceFailed ? "complete" @@ -474,11 +542,11 @@ async function runCell( maxObservedGapMs: cadence.maxObservedGapMs, sharedPages: "summed-per-process", }, - reasonCode: status === "complete" ? null : timedOut ? 1 : cadenceFailed ? 4 : 2, + reasonCode: status === "complete" ? null : collectionFailure ? 5 : timedOut ? 1 : cadenceFailed ? 4 : 2, baselineRssKiB: 0, peakRssKiB: active.length ? Math.max(...active.map((entry) => entry.totalRssKiB)) : null, terminalRssKiB: byPhase("terminals"), - finalRssKiB: total(finalRecords), + finalRssKiB: collectionFailure ? null : total(finalRecords ?? []), allocatedBytes, completed, failed, From 964d69193c34d825c05ca3afeb3b54d5155e57e2 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 21:16:12 -0700 Subject: [PATCH 20/28] fix(rss): retry transient proc scan outages (cherry picked from commit 32c2af11823ae36ecd3bd683db26248fa5e2aec1) --- .../test/swarm/rss-campaign.test.ts | 16 +++++++ .../test/swarm/run-production-rss-campaign.ts | 45 +++++++++++++++---- 2 files changed, 53 insertions(+), 8 deletions(-) diff --git a/packages/coding-agent/test/swarm/rss-campaign.test.ts b/packages/coding-agent/test/swarm/rss-campaign.test.ts index e1d706651..ec3c9bcd2 100644 --- a/packages/coding-agent/test/swarm/rss-campaign.test.ts +++ b/packages/coding-agent/test/swarm/rss-campaign.test.ts @@ -124,6 +124,22 @@ describe("B00B RSS campaign", () => { }, ); + it.skipIf(process.platform !== "linux")( + "retries one unavailable final scan and still requires a positive empty final snapshot", + async () => { + const output = join(await directory("final-scan-retry"), "output"); + await campaign(output, ["--fanout", "1", "--repetitions", "1", "--test-fail-final-scan-once"]); + const result = await run(output, 1, 1); + expect(result.status).toBe("complete"); + expect(result.reasonCode).toBeNull(); + expect(result.finalRssKiB).toBe(0); + expect((result.samples as { phase: string; totalRssKiB: number }[]).at(-1)).toMatchObject({ + phase: "final", + totalRssKiB: 0, + }); + }, + ); + it.skipIf(process.platform !== "linux")( "does not arm timeout or release descendants before exact ownership capture", async () => { diff --git a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts index 63224097e..3c9366359 100644 --- a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts +++ b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts @@ -22,6 +22,8 @@ const WORKER = new URL("./rss-campaign-worker.ts", import.meta.url); const DEFAULT_TIMEOUT_MS = 60_000; const REAP_GRACE_MS = 250; const REAP_VERIFY_MS = 1_000; +const SCAN_ATTEMPTS = 3; +const SCAN_RETRY_DELAY_MS = 2; const SCHEMA_VERSION = 2; type SupportedPlatform = "linux"; @@ -100,6 +102,8 @@ interface Config { testIgnoreTerm: boolean; // Test-only deterministic final-observation fault injection. testFailFinalScan: boolean; + // Test-only one-shot final-observation fault injection for retry coverage. + testFailFinalScanOnce: boolean; } interface GroupOwnership { @@ -152,6 +156,7 @@ function config(): Config { identityCaptureDelayMs: safeInteger("--test-identity-capture-delay-ms", 0, 0), testIgnoreTerm: process.argv.includes("--test-ignore-term"), testFailFinalScan: process.argv.includes("--test-fail-final-scan"), + testFailFinalScanOnce: process.argv.includes("--test-fail-final-scan-once"), }; } @@ -281,6 +286,23 @@ async function groupSnapshot(pgid: number, injectFailure = false): Promise boolean = () => false, +): Promise { + for (let attempt = 0; attempt < SCAN_ATTEMPTS; attempt += 1) { + const snapshot = await groupSnapshot(pgid, shouldInjectFailure(attempt)); + if (snapshot.kind !== "unavailable") return snapshot; + if (attempt + 1 < SCAN_ATTEMPTS) await pause(SCAN_RETRY_DELAY_MS); + } + return { kind: "unavailable" }; +} + async function collectorAvailable(): Promise { const own = await procRecord(process.pid); return own !== undefined && own.rssKiB >= 0 && Number.isSafeInteger(own.start) && Number.isSafeInteger(own.pgid); @@ -323,7 +345,7 @@ async function reapOwnGroup(ownership?: GroupOwnership): Promise { if (!ownership) return { reaped: true, collectionFailed: false }; let collectionFailed = false; const signalOwnedGroup = async (signal: NodeJS.Signals): Promise => { - const snapshot = await groupSnapshot(ownership.pgid); + const snapshot = await groupSnapshotWithRetries(ownership.pgid); const records = snapshotRecords(snapshot); if (!records) { collectionFailed = true; @@ -342,19 +364,23 @@ async function reapOwnGroup(ownership?: GroupOwnership): Promise { }; if (!(await signalOwnedGroup("SIGTERM"))) return { reaped: false, collectionFailed }; await pause(REAP_GRACE_MS); - let snapshot = await groupSnapshot(ownership.pgid); + let snapshot = await groupSnapshotWithRetries(ownership.pgid); let records = snapshotRecords(snapshot); if (!records) return { reaped: false, collectionFailed: true }; if (records.length === 0) return { reaped: true, collectionFailed }; if (!(await signalOwnedGroup("SIGKILL"))) return { reaped: false, collectionFailed }; const deadline = monotonicMs() + REAP_VERIFY_MS; - do { + for (;;) { await pause(10); - snapshot = await groupSnapshot(ownership.pgid); + snapshot = await groupSnapshotWithRetries(ownership.pgid); records = snapshotRecords(snapshot); - if (!records) return { reaped: false, collectionFailed: true }; - } while (records.length > 0 && monotonicMs() < deadline); - return { reaped: records.length === 0, collectionFailed }; + if (records?.length === 0) return { reaped: true, collectionFailed }; + if (records === undefined) collectionFailed = true; + // After SIGKILL, a short /proc outage need not decide the result. Keep + // checking through the existing verification deadline, but only a later + // complete empty scan can establish a successful reap/final zero. + if (monotonicMs() >= deadline) return { reaped: false, collectionFailed }; + } } function workerArguments(settings: Config, fanout: number, scratch: string): string[] { @@ -514,7 +540,10 @@ async function runCell( } const reap = await reapOwnGroup(ownership); - const finalSnapshot = await groupSnapshot(ownership.pgid, settings.testFailFinalScan); + const finalSnapshot = await groupSnapshotWithRetries( + ownership.pgid, + (attempt) => settings.testFailFinalScan || (settings.testFailFinalScanOnce && attempt === 0), + ); const finalRecords = snapshotRecords(finalSnapshot); const finalCollectionFailed = finalRecords === undefined; if (finalRecords) samples.push(sample("final", finalRecords)); From 6ef5c27ea0e7d91cf40028586fc65b0880dfaba3 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 21:24:04 -0700 Subject: [PATCH 21/28] fix(rss): retain zombie group members at zero rss (cherry picked from commit 7fd57674c21ba6c8a46d87c31f8fcb5317934f57) --- .../coding-agent/test/swarm/rss-proc.test.ts | 26 ++++++++++ packages/coding-agent/test/swarm/rss-proc.ts | 50 ++++++++++++++++++ .../test/swarm/run-production-rss-campaign.ts | 52 +++++-------------- 3 files changed, 88 insertions(+), 40 deletions(-) create mode 100644 packages/coding-agent/test/swarm/rss-proc.test.ts create mode 100644 packages/coding-agent/test/swarm/rss-proc.ts diff --git a/packages/coding-agent/test/swarm/rss-proc.test.ts b/packages/coding-agent/test/swarm/rss-proc.test.ts new file mode 100644 index 000000000..82f53747d --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-proc.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { parseProcessStat, processRecordFromStatus } from "./rss-proc.js"; + +function stat(state: string): string { + const fields = Array.from({ length: 20 }, () => "0"); + fields[0] = state; + fields[1] = "17"; + fields[2] = "23"; + fields[19] = "456"; + return `123 (worker name) ${fields.join(" ")}`; +} + +describe("Linux proc RSS records", () => { + it("parses the stat state field and keeps a zombie without VmRSS at zero RSS", () => { + const zombie = parseProcessStat(123, stat("Z")); + expect(zombie).toEqual({ pid: 123, ppid: 17, pgid: 23, start: 456, state: "Z" }); + const record = processRecordFromStatus(zombie!, "Name:\tworker\nState:\tZ (zombie)\n"); + expect(record).toEqual({ pid: 123, ppid: 17, pgid: 23, start: 456, rssKiB: 0 }); + expect(record).not.toHaveProperty("state"); + }); + + it("fails closed when a non-zombie lacks VmRSS", () => { + const running = parseProcessStat(123, stat("S")); + expect(processRecordFromStatus(running!, "Name:\tworker\nState:\tS (sleeping)\n")).toBeUndefined(); + }); +}); diff --git a/packages/coding-agent/test/swarm/rss-proc.ts b/packages/coding-agent/test/swarm/rss-proc.ts new file mode 100644 index 000000000..9029e3503 --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-proc.ts @@ -0,0 +1,50 @@ +export interface ProcessIdentity { + pid: number; + ppid: number; + pgid: number; + start: number; +} + +export interface ProcessStat extends ProcessIdentity { + state: string; +} + +/** The persisted artifact shape deliberately excludes transient procfs state. */ +export interface ProcessRecord extends ProcessIdentity { + rssKiB: number; +} + +/** + * Parses Linux /proc/PID/stat after the parenthesized comm field, which can + * itself contain spaces and closing parentheses. + */ +export function parseProcessStat(pid: number, statLine: string): ProcessStat | undefined { + const close = statLine.lastIndexOf(")"); + if (close < 0) return undefined; + const fields = statLine + .slice(close + 1) + .trim() + .split(/\s+/); + const state = fields[0]; // field 3 + const ppid = Number(fields[1]); // field 4 + const pgid = Number(fields[2]); // field 5 + const start = Number(fields[19]); // field 22 + if (!state || state.length !== 1 || ![ppid, pgid, start].every(Number.isSafeInteger)) return undefined; + return { pid, ppid, pgid, start, state }; +} + +/** + * A zombie has no address space, so Linux omits VmRSS from its status file. + * Retaining it at zero keeps ownership/reaping conservative until it vanishes. + */ +export function processRecordFromStatus(stat: ProcessStat, status: string): ProcessRecord | undefined { + const rss = /^VmRSS:\s+(\d+)\s+kB$/m.exec(status)?.[1]; + if (rss === undefined && stat.state !== "Z") return undefined; + return { + pid: stat.pid, + ppid: stat.ppid, + pgid: stat.pgid, + start: stat.start, + rssKiB: rss === undefined ? 0 : Number(rss), + }; +} diff --git a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts index 3c9366359..580387601 100644 --- a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts +++ b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts @@ -16,6 +16,7 @@ import { MAX_RSS_SAMPLE_GAP_MS, validateRssSampleCadence, } from "./rss-campaign-cadence.js"; +import { type ProcessRecord, type ProcessStat, parseProcessStat, processRecordFromStatus } from "./rss-proc.js"; const FANOUTS = [1, 4, 16, 64] as const; const WORKER = new URL("./rss-campaign-worker.ts", import.meta.url); @@ -30,14 +31,6 @@ type SupportedPlatform = "linux"; type Phase = "baseline" | "started" | "barrier-held" | "terminals" | "cleanup" | "final"; type Status = "complete" | "failed" | "timed_out" | "unsupported"; -interface ProcessRecord { - pid: number; - ppid: number; - pgid: number; - start: number; - rssKiB: number; -} - interface ProcessSample { phase: Phase; monotonicMs: number; @@ -183,39 +176,17 @@ async function writeOwnerFile(path: string, content: string): Promise { await chmod(path, 0o600); } -interface ProcessIdentity { - pid: number; - ppid: number; - pgid: number; - start: number; -} - -function parseProcessIdentity(pid: number, statLine: string): ProcessIdentity | undefined { - const close = statLine.lastIndexOf(")"); - const fields = statLine - .slice(close + 2) - .trim() - .split(/\s+/); - const ppid = Number(fields[1]); // field 4 after removing pid/comm - const pgid = Number(fields[2]); // field 5 after removing pid/comm - const start = Number(fields[19]); // field 22 after removing pid/comm - if (![ppid, pgid, start].every(Number.isSafeInteger)) return undefined; - return { pid, ppid, pgid, start }; -} - -async function procIdentity(pid: number): Promise { +async function procIdentity(pid: number): Promise { try { - return parseProcessIdentity(pid, await readFile(`/proc/${pid}/stat`, "utf8")); + return parseProcessStat(pid, await readFile(`/proc/${pid}/stat`, "utf8")); } catch { return undefined; } } -async function procRecordForIdentity(identity: ProcessIdentity): Promise { +async function procRecordForIdentity(identity: ProcessStat): Promise { try { - const status = await readFile(`/proc/${identity.pid}/status`, "utf8"); - const rss = /^VmRSS:\s+(\d+)\s+kB$/m.exec(status)?.[1]; - return rss === undefined ? undefined : { ...identity, rssKiB: Number(rss) }; + return processRecordFromStatus(identity, await readFile(`/proc/${identity.pid}/status`, "utf8")); } catch { return undefined; } @@ -255,12 +226,12 @@ async function groupSnapshot(pgid: number, injectFailure = false): Promise Date: Sun, 9 Aug 2026 21:30:48 -0700 Subject: [PATCH 22/28] fix(rss): retry transient proc snapshots to deadline (cherry picked from commit 334682199513ad552834af462e1e2794adcd1019) --- .../test/swarm/rss-snapshot-retry.test.ts | 60 +++++++++++++++++++ .../test/swarm/rss-snapshot-retry.ts | 36 +++++++++++ .../test/swarm/run-production-rss-campaign.ts | 21 ++++--- 3 files changed, 108 insertions(+), 9 deletions(-) create mode 100644 packages/coding-agent/test/swarm/rss-snapshot-retry.test.ts create mode 100644 packages/coding-agent/test/swarm/rss-snapshot-retry.ts diff --git a/packages/coding-agent/test/swarm/rss-snapshot-retry.test.ts b/packages/coding-agent/test/swarm/rss-snapshot-retry.test.ts new file mode 100644 index 000000000..96b9319c2 --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-snapshot-retry.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { retryUnavailableSnapshot } from "./rss-snapshot-retry.js"; + +interface FakeClock { + time: number; + pauses: number[]; + now(): number; + pause(milliseconds: number): Promise; +} + +function clock(): FakeClock { + return { + time: 0, + pauses: [], + now() { + return this.time; + }, + async pause(milliseconds) { + this.pauses.push(milliseconds); + this.time += milliseconds; + }, + }; +} + +describe("RSS proc snapshot retry window", () => { + it("accepts a coherent snapshot after a deterministic transient sequence beyond three attempts", async () => { + const fake = clock(); + let attempts = 0; + const result = await retryUnavailableSnapshot( + async () => (attempts++ < 8 ? "unavailable" : "coherent"), + (snapshot) => snapshot === "unavailable", + fake, + 20, + 2, + ); + expect(result).toBe("coherent"); + expect(attempts).toBe(9); + expect(fake.pauses).toEqual([2, 2, 2, 2, 2, 2, 2, 2]); + expect(fake.time).toBe(16); + }); + + it("returns persistent unavailability at the monotonic deadline without inventing an empty snapshot", async () => { + const fake = clock(); + let attempts = 0; + const result = await retryUnavailableSnapshot( + async () => { + attempts += 1; + return "unavailable"; + }, + (snapshot) => snapshot === "unavailable", + fake, + 20, + 2, + ); + expect(result).toBe("unavailable"); + expect(attempts).toBe(10); + expect(fake.pauses).toHaveLength(10); + expect(fake.time).toBe(20); + }); +}); diff --git a/packages/coding-agent/test/swarm/rss-snapshot-retry.ts b/packages/coding-agent/test/swarm/rss-snapshot-retry.ts new file mode 100644 index 000000000..16eae07c5 --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-snapshot-retry.ts @@ -0,0 +1,36 @@ +/** The retry budget is deliberately shorter than the 50 ms sample-gap contract. */ +export const RSS_SCAN_RETRY_WINDOW_MS = 20; +export const RSS_SCAN_RETRY_DELAY_MS = 2; + +export interface RetryClock { + now(): number; + pause(milliseconds: number): Promise; +} + +/** + * Retries only unavailable snapshots within one monotonic convergence window. + * A returned value is accepted only if the collector itself declared it + * coherent; this helper never converts an unavailable result into an empty one. + */ +export async function retryUnavailableSnapshot( + collect: (attempt: number) => Promise, + isUnavailable: (snapshot: T) => boolean, + clock: RetryClock, + windowMs = RSS_SCAN_RETRY_WINDOW_MS, + delayMs = RSS_SCAN_RETRY_DELAY_MS, +): Promise { + const deadline = clock.now() + windowMs; + let attempt = 0; + let lastUnavailable: T | undefined; + for (;;) { + // Do not begin a further full scan once the convergence budget expired. + if (attempt > 0 && clock.now() >= deadline) return lastUnavailable!; + const snapshot = await collect(attempt); + attempt += 1; + if (!isUnavailable(snapshot)) return snapshot; + lastUnavailable = snapshot; + const remaining = deadline - clock.now(); + if (remaining <= 0) return snapshot; + await clock.pause(Math.min(delayMs, remaining)); + } +} diff --git a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts index 580387601..4dc0c4a8b 100644 --- a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts +++ b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts @@ -17,14 +17,13 @@ import { validateRssSampleCadence, } from "./rss-campaign-cadence.js"; import { type ProcessRecord, type ProcessStat, parseProcessStat, processRecordFromStatus } from "./rss-proc.js"; +import { RSS_SCAN_RETRY_DELAY_MS, RSS_SCAN_RETRY_WINDOW_MS, retryUnavailableSnapshot } from "./rss-snapshot-retry.js"; const FANOUTS = [1, 4, 16, 64] as const; const WORKER = new URL("./rss-campaign-worker.ts", import.meta.url); const DEFAULT_TIMEOUT_MS = 60_000; const REAP_GRACE_MS = 250; const REAP_VERIFY_MS = 1_000; -const SCAN_ATTEMPTS = 3; -const SCAN_RETRY_DELAY_MS = 2; const SCHEMA_VERSION = 2; type SupportedPlatform = "linux"; @@ -267,12 +266,16 @@ async function groupSnapshotWithRetries( pgid: number, shouldInjectFailure: (attempt: number) => boolean = () => false, ): Promise { - for (let attempt = 0; attempt < SCAN_ATTEMPTS; attempt += 1) { - const snapshot = await groupSnapshot(pgid, shouldInjectFailure(attempt)); - if (snapshot.kind !== "unavailable") return snapshot; - if (attempt + 1 < SCAN_ATTEMPTS) await pause(SCAN_RETRY_DELAY_MS); - } - return { kind: "unavailable" }; + // Fork/exec children can briefly have a stat record but no VmRSS. Retry + // complete scans through this short convergence window, never individual + // records: a successful return is always one coherent full-group view. + return retryUnavailableSnapshot( + (attempt) => groupSnapshot(pgid, shouldInjectFailure(attempt)), + (snapshot) => snapshot.kind === "unavailable", + { now: monotonicMs, pause }, + RSS_SCAN_RETRY_WINDOW_MS, + RSS_SCAN_RETRY_DELAY_MS, + ); } async function collectorAvailable(): Promise { @@ -425,7 +428,7 @@ async function runCell( currentOwnership, announced.filter((record): record is ProcessRecord => record?.pgid === currentOwnership.pgid), ); - const snapshot = await groupSnapshot(currentOwnership.pgid); + const snapshot = await groupSnapshotWithRetries(currentOwnership.pgid); const records = snapshotRecords(snapshot); if (!records) { collectorFailed = true; From d781743f0fbd9a0be2dba58f4498547917370846 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 21:37:08 -0700 Subject: [PATCH 23/28] fix(coding-agent): retain no-mm proc members (cherry picked from commit ab170645d8390f36f6ff4b7aa17fd4802bc7db9a) --- .../coding-agent/test/swarm/rss-proc.test.ts | 37 +++++++++++++++---- packages/coding-agent/test/swarm/rss-proc.ts | 30 +++++++++------ .../test/swarm/run-production-rss-campaign.ts | 4 +- 3 files changed, 51 insertions(+), 20 deletions(-) diff --git a/packages/coding-agent/test/swarm/rss-proc.test.ts b/packages/coding-agent/test/swarm/rss-proc.test.ts index 82f53747d..d0afe90de 100644 --- a/packages/coding-agent/test/swarm/rss-proc.test.ts +++ b/packages/coding-agent/test/swarm/rss-proc.test.ts @@ -10,17 +10,40 @@ function stat(state: string): string { return `123 (worker name) ${fields.join(" ")}`; } +function processStat(state: string) { + const parsed = parseProcessStat(123, stat(state)); + expect(parsed).toEqual({ pid: 123, ppid: 17, pgid: 23, start: 456, state }); + return parsed!; +} + describe("Linux proc RSS records", () => { - it("parses the stat state field and keeps a zombie without VmRSS at zero RSS", () => { - const zombie = parseProcessStat(123, stat("Z")); - expect(zombie).toEqual({ pid: 123, ppid: 17, pgid: 23, start: 456, state: "Z" }); - const record = processRecordFromStatus(zombie!, "Name:\tworker\nState:\tZ (zombie)\n"); + it.each(["S", "R"])("keeps a %s process with no mm at zero RSS", (state) => { + const record = processRecordFromStatus(processStat(state), `Name:\tworker\nState:\t${state} (running)\n`); expect(record).toEqual({ pid: 123, ppid: 17, pgid: 23, start: 456, rssKiB: 0 }); expect(record).not.toHaveProperty("state"); }); - it("fails closed when a non-zombie lacks VmRSS", () => { - const running = parseProcessStat(123, stat("S")); - expect(processRecordFromStatus(running!, "Name:\tworker\nState:\tS (sleeping)\n")).toBeUndefined(); + it("keeps a zombie without an mm at zero RSS", () => { + expect(processRecordFromStatus(processStat("Z"), "Name:\tworker\nState:\tZ (zombie)\n")).toMatchObject({ + rssKiB: 0, + }); + }); + + it.each([ + ["missing", "Name:\tworker\n"], + ["mismatched", "Name:\tworker\nState:\tR (running)\n"], + ["malformed", "Name:\tworker\nState:\tS not-a-linux-state\n"], + ])("fails closed for %s status state", (_case, status) => { + expect(processRecordFromStatus(processStat("S"), status)).toBeUndefined(); + }); + + it("fails closed when an mm field exists but VmRSS is absent", () => { + const status = "Name:\tworker\nState:\tS (sleeping)\nVmSize:\t1024 kB\n"; + expect(processRecordFromStatus(processStat("S"), status)).toBeUndefined(); + }); + + it("parses VmRSS from a state-validated status file", () => { + const status = "Name:\tworker\nState:\tS (sleeping)\nVmSize:\t1024 kB\nVmRSS:\t512 kB\n"; + expect(processRecordFromStatus(processStat("S"), status)).toMatchObject({ rssKiB: 512 }); }); }); diff --git a/packages/coding-agent/test/swarm/rss-proc.ts b/packages/coding-agent/test/swarm/rss-proc.ts index 9029e3503..1ab2d13c5 100644 --- a/packages/coding-agent/test/swarm/rss-proc.ts +++ b/packages/coding-agent/test/swarm/rss-proc.ts @@ -34,17 +34,25 @@ export function parseProcessStat(pid: number, statLine: string): ProcessStat | u } /** - * A zombie has no address space, so Linux omits VmRSS from its status file. - * Retaining it at zero keeps ownership/reaping conservative until it vanishes. + * A process without an mm has no Vm* fields. Linux exposes this both for + * zombies and briefly for fork/exec children, which are conservatively kept + * as zero-RSS records after the status state confirms the stat identity. */ export function processRecordFromStatus(stat: ProcessStat, status: string): ProcessRecord | undefined { - const rss = /^VmRSS:\s+(\d+)\s+kB$/m.exec(status)?.[1]; - if (rss === undefined && stat.state !== "Z") return undefined; - return { - pid: stat.pid, - ppid: stat.ppid, - pgid: stat.pgid, - start: stat.start, - rssKiB: rss === undefined ? 0 : Number(rss), - }; + const lines = status.split(/\r?\n/); + const stateLines = lines.filter((line) => line.startsWith("State:")); + const state = stateLines.length === 1 ? /^State:\s+(\S)(?:\s+\([^()]*\))?\s*$/.exec(stateLines[0])?.[1] : undefined; + if (state !== stat.state) return undefined; + + const vmLines = lines.filter((line) => line.startsWith("Vm")); + const rssLines = vmLines.filter((line) => line.startsWith("VmRSS:")); + if (rssLines.length === 0) { + return vmLines.length === 0 + ? { pid: stat.pid, ppid: stat.ppid, pgid: stat.pgid, start: stat.start, rssKiB: 0 } + : undefined; + } + if (rssLines.length !== 1) return undefined; + const rss = /^VmRSS:\s+(\d+)\s+kB\s*$/.exec(rssLines[0])?.[1]; + if (rss === undefined) return undefined; + return { pid: stat.pid, ppid: stat.ppid, pgid: stat.pgid, start: stat.start, rssKiB: Number(rss) }; } diff --git a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts index 4dc0c4a8b..d713d8226 100644 --- a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts +++ b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts @@ -244,8 +244,8 @@ async function groupSnapshot(pgid: number, injectFailure = false): Promise Date: Sun, 9 Aug 2026 21:41:59 -0700 Subject: [PATCH 24/28] fix(rss): confirm proc identity after status read (cherry picked from commit 7c2485ae54ad9ae31a193fcef8dcf32bfcb17ffc) --- .../coding-agent/test/swarm/rss-proc.test.ts | 57 ++++++++++++++----- packages/coding-agent/test/swarm/rss-proc.ts | 26 ++++++++- .../test/swarm/run-production-rss-campaign.ts | 12 +++- 3 files changed, 75 insertions(+), 20 deletions(-) diff --git a/packages/coding-agent/test/swarm/rss-proc.test.ts b/packages/coding-agent/test/swarm/rss-proc.test.ts index d0afe90de..e3ebb0e0c 100644 --- a/packages/coding-agent/test/swarm/rss-proc.test.ts +++ b/packages/coding-agent/test/swarm/rss-proc.test.ts @@ -1,49 +1,76 @@ import { describe, expect, it } from "vitest"; -import { parseProcessStat, processRecordFromStatus } from "./rss-proc.js"; +import { hasStableProcessIdentity, parseProcessStat, processRecordFromStatus } from "./rss-proc.js"; -function stat(state: string): string { +function stat(state: string, ppid = 17, pgid = 23, start = 456): string { const fields = Array.from({ length: 20 }, () => "0"); fields[0] = state; - fields[1] = "17"; - fields[2] = "23"; - fields[19] = "456"; + fields[1] = String(ppid); + fields[2] = String(pgid); + fields[19] = String(start); return `123 (worker name) ${fields.join(" ")}`; } -function processStat(state: string) { - const parsed = parseProcessStat(123, stat(state)); - expect(parsed).toEqual({ pid: 123, ppid: 17, pgid: 23, start: 456, state }); +function processStat(state: string, ppid = 17, pgid = 23, start = 456, pid = 123) { + const parsed = parseProcessStat(pid, stat(state, ppid, pgid, start)); + expect(parsed).toEqual({ pid, ppid, pgid, start, state }); return parsed!; } +function recordFromStatus(initial: ReturnType, status: string, confirmation = initial) { + return processRecordFromStatus(initial, status, confirmation); +} + describe("Linux proc RSS records", () => { it.each(["S", "R"])("keeps a %s process with no mm at zero RSS", (state) => { - const record = processRecordFromStatus(processStat(state), `Name:\tworker\nState:\t${state} (running)\n`); + const initial = processStat(state); + const record = recordFromStatus(initial, `Name:\tworker\nState:\t${state} (running)\n`); expect(record).toEqual({ pid: 123, ppid: 17, pgid: 23, start: 456, rssKiB: 0 }); expect(record).not.toHaveProperty("state"); }); + it("accepts a legitimate R-to-S state change after confirming stable external identity", () => { + const initial = processStat("R", 17, 23, 456); + const confirmation = processStat("S", 99, 23, 456); + expect(recordFromStatus(initial, "Name:\tworker\nState:\tS (sleeping)\n", confirmation)).toEqual({ + pid: 123, + ppid: 17, + pgid: 23, + start: 456, + rssKiB: 0, + }); + }); + it("keeps a zombie without an mm at zero RSS", () => { - expect(processRecordFromStatus(processStat("Z"), "Name:\tworker\nState:\tZ (zombie)\n")).toMatchObject({ + expect(recordFromStatus(processStat("Z"), "Name:\tworker\nState:\tZ (zombie)\n")).toMatchObject({ rssKiB: 0, }); }); it.each([ ["missing", "Name:\tworker\n"], - ["mismatched", "Name:\tworker\nState:\tR (running)\n"], ["malformed", "Name:\tworker\nState:\tS not-a-linux-state\n"], + ["invalid Linux state", "Name:\tworker\nState:\tQ (not a task state)\n"], ])("fails closed for %s status state", (_case, status) => { - expect(processRecordFromStatus(processStat("S"), status)).toBeUndefined(); + expect(recordFromStatus(processStat("S"), status)).toBeUndefined(); + }); + + it.each([ + ["PID", processStat("S", 17, 23, 456, 124)], + ["start time", processStat("S", 17, 23, 457)], + ["process group", processStat("S", 17, 24, 456)], + ])("fails the scan when the confirmation has a mismatched %s", (_case, confirmation) => { + const initial = processStat("S"); + expect(hasStableProcessIdentity(initial, confirmation)).toBe(false); + expect(recordFromStatus(initial, "Name:\tworker\nState:\tS (sleeping)\n", confirmation)).toBeUndefined(); }); it("fails closed when an mm field exists but VmRSS is absent", () => { const status = "Name:\tworker\nState:\tS (sleeping)\nVmSize:\t1024 kB\n"; - expect(processRecordFromStatus(processStat("S"), status)).toBeUndefined(); + expect(recordFromStatus(processStat("S"), status)).toBeUndefined(); }); - it("parses VmRSS from a state-validated status file", () => { + it("parses VmRSS from a status file with stable external identity", () => { const status = "Name:\tworker\nState:\tS (sleeping)\nVmSize:\t1024 kB\nVmRSS:\t512 kB\n"; - expect(processRecordFromStatus(processStat("S"), status)).toMatchObject({ rssKiB: 512 }); + expect(recordFromStatus(processStat("S"), status)).toMatchObject({ rssKiB: 512 }); }); }); diff --git a/packages/coding-agent/test/swarm/rss-proc.ts b/packages/coding-agent/test/swarm/rss-proc.ts index 1ab2d13c5..7cbe524b5 100644 --- a/packages/coding-agent/test/swarm/rss-proc.ts +++ b/packages/coding-agent/test/swarm/rss-proc.ts @@ -9,6 +9,19 @@ export interface ProcessStat extends ProcessIdentity { state: string; } +// Linux task-state letters emitted by /proc/PID/status. +const LINUX_PROCESS_STATES = new Set(["R", "S", "D", "Z", "T", "t", "W", "X", "x", "K", "P", "I"]); + +/** + * Confirms that two reads identify the same process-group member. State and + * parent PID are deliberately transient and therefore are not identity. + */ +export function hasStableProcessIdentity(initial: ProcessStat, confirmation: ProcessStat): boolean { + return ( + initial.pid === confirmation.pid && initial.start === confirmation.start && initial.pgid === confirmation.pgid + ); +} + /** The persisted artifact shape deliberately excludes transient procfs state. */ export interface ProcessRecord extends ProcessIdentity { rssKiB: number; @@ -36,13 +49,20 @@ export function parseProcessStat(pid: number, statLine: string): ProcessStat | u /** * A process without an mm has no Vm* fields. Linux exposes this both for * zombies and briefly for fork/exec children, which are conservatively kept - * as zero-RSS records after the status state confirms the stat identity. + * as zero-RSS records after an external stat reread confirms the identity. + * State is only a status-file integrity check: it may change between reads. */ -export function processRecordFromStatus(stat: ProcessStat, status: string): ProcessRecord | undefined { +export function processRecordFromStatus( + stat: ProcessStat, + status: string, + confirmation: ProcessStat, +): ProcessRecord | undefined { + if (!hasStableProcessIdentity(stat, confirmation)) return undefined; + const lines = status.split(/\r?\n/); const stateLines = lines.filter((line) => line.startsWith("State:")); const state = stateLines.length === 1 ? /^State:\s+(\S)(?:\s+\([^()]*\))?\s*$/.exec(stateLines[0])?.[1] : undefined; - if (state !== stat.state) return undefined; + if (state === undefined || !LINUX_PROCESS_STATES.has(state)) return undefined; const vmLines = lines.filter((line) => line.startsWith("Vm")); const rssLines = vmLines.filter((line) => line.startsWith("VmRSS:")); diff --git a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts index d713d8226..d56ba4522 100644 --- a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts +++ b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts @@ -185,7 +185,11 @@ async function procIdentity(pid: number): Promise { async function procRecordForIdentity(identity: ProcessStat): Promise { try { - return processRecordFromStatus(identity, await readFile(`/proc/${identity.pid}/status`, "utf8")); + const status = await readFile(`/proc/${identity.pid}/status`, "utf8"); + // State and PPID can change while status is read. A second stat read + // anchors the status to the original PID/start-time/process-group identity. + const confirmation = parseProcessStat(identity.pid, await readFile(`/proc/${identity.pid}/stat`, "utf8")); + return confirmation ? processRecordFromStatus(identity, status, confirmation) : undefined; } catch { return undefined; } @@ -243,7 +247,11 @@ async function groupSnapshot(pgid: number, injectFailure = false): Promise Date: Sun, 9 Aug 2026 22:04:18 -0700 Subject: [PATCH 25/28] test(coding-agent): forge empty process evidence safely --- .../swarm/production-evidence-adapter.test.ts | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts b/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts index ffed17b6b..13fd8a743 100644 --- a/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts +++ b/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts @@ -2,7 +2,7 @@ import { createHash, generateKeyPairSync, sign } from "node:crypto"; import { mkdtemp, readdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { afterEach, describe, expect, test } from "vitest"; +import { afterEach, describe, expect, test, vi } from "vitest"; import { type ProductionEvidenceInput, projectProductionObservations, @@ -14,6 +14,7 @@ import { COST_NUMERATOR_SCALE, canonicalJson, createSwarmEvidenceTrustRoot, + currentProcessSampler, SWARM_EVIDENCE_COMMITMENT_SCHEMA, swarmEvidenceCommitmentPayload, verifyAuthenticatedSwarmEvidence, @@ -104,7 +105,12 @@ function expectNoCanaryLeak(chunks: readonly string[]): void { async function forgeProcessSampleBundle(directory: string): Promise { const samplePath = join(directory, "process-samples.json"); const samples = JSON.parse(await readFile(samplePath, "utf8")); - samples[0].processes[0].pid += 1; + const firstSample = samples[0]; + const firstProcess = firstSample.processes[0]; + if (firstProcess) firstProcess.pid += 1; + // B00A's default sampler legitimately returns no processes on some hosts. + // A zero-RSS process remains schema-valid and leaves the sample total intact. + else firstSample.processes.push({ pid: 1, rssBytes: 0 }); const sampleRaw = `${canonicalJson(samples)}\n`; await writeFile(samplePath, sampleRaw); const manifestPath = join(directory, "manifest.json"); @@ -221,6 +227,49 @@ describe("B00B signed production evidence adapter", () => { ).rejects.toThrow("B00B_EVIDENCE_BAD_SIGNATURE"); }); + test("coherently forges an empty default B00A sample but cannot satisfy the original external commitment", async () => { + const artifactDirectory = await mkdtemp(join(tmpdir(), "b00b-empty-artifact-")); + const trustDirectory = await mkdtemp(join(tmpdir(), "b00b-empty-trust-")); + cleanup.push(artifactDirectory, trustDirectory); + const sampler = vi.spyOn(currentProcessSampler, "sample").mockReturnValue([]); + const signer = generateKeyPairSync("ed25519"); + const publicPem = signer.publicKey.export({ type: "spki", format: "pem" }).toString(); + let written: Awaited>; + try { + written = await writeSignedProductionEvidence(artifactDirectory, trustDirectory, input(), signer.privateKey); + } finally { + sampler.mockRestore(); + } + const originalSamples = JSON.parse(await readFile(join(artifactDirectory, "process-samples.json"), "utf8")); + expect(originalSamples[0].processes).toEqual([]); + expect(originalSamples[0].totalRssBytes).toBe(0); + const forgedBundleId = await forgeProcessSampleBundle(artifactDirectory); + const forgedSamples = JSON.parse(await readFile(join(artifactDirectory, "process-samples.json"), "utf8")); + expect(forgedSamples[0]).toMatchObject({ processes: [{ pid: 1, rssBytes: 0 }], totalRssBytes: 0 }); + // The forged artifact remains B00A-canonical when re-indexed against its new identity. + const attacker = generateKeyPairSync("ed25519"); + const attackerCommitmentPath = join(trustDirectory, "attacker-commitment.json"); + await writeFile( + attackerCommitmentPath, + `${canonicalJson({ + schemaVersion: SWARM_EVIDENCE_COMMITMENT_SCHEMA, + artifactBundleId: forgedBundleId, + signature: sign( + null, + Buffer.from(canonicalJson(swarmEvidenceCommitmentPayload(forgedBundleId))), + attacker.privateKey, + ).toString("base64"), + })}\n`, + ); + const attackerPublicPem = attacker.publicKey.export({ type: "spki", format: "pem" }).toString(); + await expect( + verifySignedProductionEvidence(artifactDirectory, attackerCommitmentPath, attackerPublicPem), + ).resolves.toBeUndefined(); + await expect( + verifySignedProductionEvidence(artifactDirectory, written.commitmentPath, publicPem), + ).rejects.toThrow("trusted artifact bundle mismatch"); + }); + test("rejects a coherent manifest/index forgery and tampered external commitment", async () => { const artifactDirectory = await mkdtemp(join(tmpdir(), "b00b-artifact-")); const trustDirectory = await mkdtemp(join(tmpdir(), "b00b-trust-")); From 7b9c404c0a3a4f0f501962092a3cda7762e794ed Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 22:33:30 -0700 Subject: [PATCH 26/28] test(coding-agent): harden B00B production gate --- .../swarm/daemon-production-dispatch.test.ts | 4 +- .../swarm/production-scripted-provider.ts | 58 +++++++++++-------- .../test/swarm/rss-campaign.test.ts | 8 +++ .../test/swarm/rss-child-exec-args.ts | 15 +++++ .../test/swarm/run-production-rss-campaign.ts | 3 +- .../swarm-production-integration.test.ts | 39 ++++++++++++- 6 files changed, 98 insertions(+), 29 deletions(-) create mode 100644 packages/coding-agent/test/swarm/rss-child-exec-args.ts diff --git a/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts b/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts index f2e152642..bf56ceb90 100644 --- a/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts +++ b/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts @@ -17,7 +17,7 @@ import { symlinkSync, writeFileSync, } from "node:fs"; -import { createServer } from "node:http"; +import { createServer, type ServerResponse } from "node:http"; import { createConnection, type Socket } from "node:net"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; @@ -270,7 +270,7 @@ async function localProvider(canary: string): Promise { const releaseWaiters = new Map void>>(); const sockets = new Set(); let closePromise: Promise | undefined; - const waitForRelease = (id: string, response: import("node:http").ServerResponse): Promise => + const waitForRelease = (id: string, response: ServerResponse): Promise => new Promise((resolveRelease) => { const release = () => finish(true); const closed = () => finish(false); diff --git a/packages/coding-agent/test/swarm/production-scripted-provider.ts b/packages/coding-agent/test/swarm/production-scripted-provider.ts index 4a762c77f..c26486a94 100644 --- a/packages/coding-agent/test/swarm/production-scripted-provider.ts +++ b/packages/coding-agent/test/swarm/production-scripted-provider.ts @@ -178,51 +178,64 @@ function assertScript(script: ProviderScript, requestId: string): void { } /** Abort-aware gate. It owns one promise per request, so one abort cannot release a sibling. */ -function createBarrier(expected: readonly string[], timeoutMs: number) { +export function createBarrier(expected: readonly string[], timeoutMs: number) { const expectedSet = new Set(expected); if (expectedSet.size !== expected.length || expected.some((id) => !/^request-\d{4}$/.test(id))) { throw new Error("B00B_BAD_BARRIER_EXPECTED"); } let resolveOpen!: () => void; let rejectOpen!: (error: Error) => void; - let settled = false; + let openSettled = false; + let closed = false; const open = new Promise((resolve, reject) => { resolveOpen = resolve; rejectOpen = reject; }); const entered = new Set(); const released = new Set(); - const waiters = new Map void>(); + const waiters = new Map void>(); + const abortPendingWaiters = () => { + for (const waiter of waiters.values()) waiter("aborted"); + waiters.clear(); + }; + const rejectPendingOpen = (error: Error) => { + if (openSettled) return; + openSettled = true; + rejectOpen(error); + }; const timer = setTimeout(() => { - if (!settled) { - settled = true; - rejectOpen(new Error("B00B_BARRIER_TIMEOUT")); - } + if (closed) return; + closed = true; + rejectPendingOpen(new Error("B00B_BARRIER_TIMEOUT")); + abortPendingWaiters(); }, timeoutMs); const enteredRequest = (id: string) => { if (!expectedSet.has(id)) return; if (entered.has(id)) throw new Error("B00B_BARRIER_DUPLICATE"); entered.add(id); - if (entered.size === expectedSet.size && !settled) { - settled = true; + if (entered.size === expectedSet.size && !openSettled) { + openSettled = true; clearTimeout(timer); resolveOpen(); } }; const wait = (id: string, signal: AbortSignal | undefined) => new Promise<"released" | "aborted">((resolve) => { - if (released.has(id)) return resolve("released"); - const onAbort = () => { + let done = false; + const settle = (result: "released" | "aborted") => { + if (done) return; + done = true; signal?.removeEventListener("abort", onAbort); waiters.delete(id); - resolve("aborted"); + resolve(result); }; - if (signal?.aborted) return onAbort(); - waiters.set(id, () => { - signal?.removeEventListener("abort", onAbort); - resolve("released"); - }); + const onAbort = () => settle("aborted"); + if (closed || signal?.aborted) return settle("aborted"); + if (released.has(id)) return settle("released"); + waiters.set(id, settle); signal?.addEventListener("abort", onAbort, { once: true }); + // The abort may race listener registration in an implementation-specific host. + if (signal?.aborted) settle("aborted"); }); return { open, @@ -231,16 +244,15 @@ function createBarrier(expected: readonly string[], timeoutMs: number) { release(ids?: readonly string[]) { for (const id of ids ?? expected) { released.add(id); - waiters.get(id)?.(); - waiters.delete(id); + waiters.get(id)?.("released"); } }, close() { + if (closed) return; + closed = true; clearTimeout(timer); - if (!settled) { - settled = true; - rejectOpen(new Error("B00B_BARRIER_CLOSED")); - } + rejectPendingOpen(new Error("B00B_BARRIER_CLOSED")); + abortPendingWaiters(); }, }; } diff --git a/packages/coding-agent/test/swarm/rss-campaign.test.ts b/packages/coding-agent/test/swarm/rss-campaign.test.ts index ec3c9bcd2..49a16a865 100644 --- a/packages/coding-agent/test/swarm/rss-campaign.test.ts +++ b/packages/coding-agent/test/swarm/rss-campaign.test.ts @@ -6,6 +6,7 @@ import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { afterEach, describe, expect, it } from "vitest"; import { validateRssSampleCadence } from "./rss-campaign-cadence.js"; +import { childExecArgsWithTsxImport } from "./rss-child-exec-args.js"; const execute = promisify(execFile); const launcher = fileURLToPath(new URL("./run-production-rss-campaign.ts", import.meta.url)); @@ -37,6 +38,13 @@ async function run(output: string, fanout: number, repetition: number): Promise< } describe("B00B RSS campaign", () => { + it("preloads tsx for a TypeScript worker exactly once unless a loader is already selected", () => { + expect(childExecArgsWithTsxImport([])).toEqual(["--import", "tsx"]); + expect(childExecArgsWithTsxImport(["--trace-warnings"])).toEqual(["--trace-warnings", "--import", "tsx"]); + expect(childExecArgsWithTsxImport(["--import", "tsx"])).toEqual(["--import", "tsx"]); + expect(childExecArgsWithTsxImport(["--import=tsx"])).toEqual(["--import=tsx"]); + expect(childExecArgsWithTsxImport(["--loader", "custom-ts-loader"])).toEqual(["--loader", "custom-ts-loader"]); + }); it("writes complete structured dry artifacts rather than zero-looking macOS data", async () => { const output = join(await directory("dry"), "output"); await campaign(output, ["--fanout", "1", "--repetitions", "2"]); diff --git a/packages/coding-agent/test/swarm/rss-child-exec-args.ts b/packages/coding-agent/test/swarm/rss-child-exec-args.ts new file mode 100644 index 000000000..5c4760100 --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-child-exec-args.ts @@ -0,0 +1,15 @@ +/** + * Preserve a parent TypeScript runtime when present. Otherwise use tsx's Node + * preload so the disposable child can execute the .ts worker directly. + */ +export function childExecArgsWithTsxImport(execArgs: readonly string[]): string[] { + for (let index = 0; index < execArgs.length; index += 1) { + const argument = execArgs[index]!; + if (argument === "--import" && execArgs[index + 1] === "tsx") return [...execArgs]; + if (argument === "--import=tsx") return [...execArgs]; + // An explicit Node loader owns module loading for this child; do not + // stack tsx on top of a caller-selected TypeScript loader. + if (argument === "--loader" || argument.startsWith("--loader=")) return [...execArgs]; + } + return [...execArgs, "--import", "tsx"]; +} diff --git a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts index d56ba4522..9ccbfaf7b 100644 --- a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts +++ b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts @@ -16,6 +16,7 @@ import { MAX_RSS_SAMPLE_GAP_MS, validateRssSampleCadence, } from "./rss-campaign-cadence.js"; +import { childExecArgsWithTsxImport } from "./rss-child-exec-args.js"; import { type ProcessRecord, type ProcessStat, parseProcessStat, processRecordFromStatus } from "./rss-proc.js"; import { RSS_SCAN_RETRY_DELAY_MS, RSS_SCAN_RETRY_WINDOW_MS, retryUnavailableSnapshot } from "./rss-snapshot-retry.js"; @@ -369,7 +370,7 @@ async function reapOwnGroup(ownership?: GroupOwnership): Promise { function workerArguments(settings: Config, fanout: number, scratch: string): string[] { const args = [ "--expose-gc", - ...process.execArgv, + ...childExecArgsWithTsxImport(process.execArgv), WORKER.pathname, "--fanout", String(fanout), diff --git a/packages/coding-agent/test/swarm/swarm-production-integration.test.ts b/packages/coding-agent/test/swarm/swarm-production-integration.test.ts index 5e03e2849..48c4f8d61 100644 --- a/packages/coding-agent/test/swarm/swarm-production-integration.test.ts +++ b/packages/coding-agent/test/swarm/swarm-production-integration.test.ts @@ -3,7 +3,7 @@ import { generateKeyPairSync } from "node:crypto"; import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { Agent } from "@earendil-works/pi-agent-core"; +import { Agent, type AgentOptions } from "@earendil-works/pi-agent-core"; import { Type } from "@earendil-works/pi-ai"; import { afterEach, describe, expect, test, vi } from "vitest"; import { AgentSession } from "../../src/core/agent-session.js"; @@ -24,7 +24,7 @@ import { verifySignedProductionEvidenceFreshProcess, writeSignedProductionEvidence, } from "./production-evidence-adapter.js"; -import { createBarrierScriptedProvider, type ProviderScript } from "./production-scripted-provider.js"; +import { createBarrier, createBarrierScriptedProvider, type ProviderScript } from "./production-scripted-provider.js"; const cleanups: Array<() => Promise | void> = []; afterEach(async () => { @@ -92,7 +92,7 @@ function simple(requestId: string, options: Partial = {}): Provi } function agentFor( model: ReturnType["models"][number], - tools: NonNullable["tools"] = [], + tools: NonNullable["tools"] = [], ) { return new Agent({ getApiKey: () => "fixture-key", initialState: { model, systemPrompt: canaries[0], tools } }); } @@ -188,6 +188,39 @@ async function waitForTerminals(fixture: ReturnType, count: num } describe("B00B production scripted provider", () => { + test("settles every held waiter as aborted on barrier timeout and removes its abort listeners", async () => { + vi.useFakeTimers(); + try { + const barrier = createBarrier(["request-0091", "request-0092"], 100); + const first = new AbortController(); + const second = new AbortController(); + const firstWait = barrier.wait("request-0091", first.signal); + const secondWait = barrier.wait("request-0092", second.signal); + const rejectedOpen = expect(barrier.open).rejects.toThrow("B00B_BARRIER_TIMEOUT"); + await vi.advanceTimersByTimeAsync(100); + await expect(Promise.all([firstWait, secondWait])).resolves.toEqual(["aborted", "aborted"]); + await rejectedOpen; + // A stale abort listener would have a second settlement path after timeout. + first.abort(); + second.abort(); + } finally { + vi.useRealTimers(); + } + }); + + test("settles every held waiter as aborted when the provider closes", async () => { + const barrier = createBarrier(["request-0093", "request-0094"], 10_000); + const first = new AbortController(); + const second = new AbortController(); + const firstWait = barrier.wait("request-0093", first.signal); + const secondWait = barrier.wait("request-0094", second.signal); + const rejectedOpen = expect(barrier.open).rejects.toThrow("B00B_BARRIER_CLOSED"); + barrier.close(); + await expect(Promise.all([firstWait, secondWait])).resolves.toEqual(["aborted", "aborted"]); + await rejectedOpen; + first.abort(); + second.abort(); + }); test("registers through the real AI registry and holds a 1/4 fanout only as an observation barrier", async () => { const ids = ["request-0001", "request-0002", "request-0003", "request-0004"] as const; const fixture = provider(Object.fromEntries(ids.map((id) => [id, [simple(id, { waitForRelease: true })]])), ids); From 6e9354cc52dbd488bc7be74d28586ffe9f0a7c53 Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 22:44:27 -0700 Subject: [PATCH 27/28] test(coding-agent): settle barrier early waits safely --- .../test/swarm/production-scripted-provider.ts | 5 +++-- .../swarm/swarm-production-integration.test.ts | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/packages/coding-agent/test/swarm/production-scripted-provider.ts b/packages/coding-agent/test/swarm/production-scripted-provider.ts index c26486a94..d77c0a03b 100644 --- a/packages/coding-agent/test/swarm/production-scripted-provider.ts +++ b/packages/coding-agent/test/swarm/production-scripted-provider.ts @@ -222,14 +222,15 @@ export function createBarrier(expected: readonly string[], timeoutMs: number) { const wait = (id: string, signal: AbortSignal | undefined) => new Promise<"released" | "aborted">((resolve) => { let done = false; + let onAbort: (() => void) | undefined; const settle = (result: "released" | "aborted") => { if (done) return; done = true; - signal?.removeEventListener("abort", onAbort); + if (onAbort) signal?.removeEventListener("abort", onAbort); waiters.delete(id); resolve(result); }; - const onAbort = () => settle("aborted"); + onAbort = () => settle("aborted"); if (closed || signal?.aborted) return settle("aborted"); if (released.has(id)) return settle("released"); waiters.set(id, settle); diff --git a/packages/coding-agent/test/swarm/swarm-production-integration.test.ts b/packages/coding-agent/test/swarm/swarm-production-integration.test.ts index 48c4f8d61..1f6c78fdd 100644 --- a/packages/coding-agent/test/swarm/swarm-production-integration.test.ts +++ b/packages/coding-agent/test/swarm/swarm-production-integration.test.ts @@ -221,6 +221,23 @@ describe("B00B production scripted provider", () => { first.abort(); second.abort(); }); + test("settles wait calls made after release, close, or pre-abort without throwing", async () => { + const barrier = createBarrier(["request-0095", "request-0096"], 10_000); + const rejectedOpen = expect(barrier.open).rejects.toThrow("B00B_BARRIER_CLOSED"); + barrier.release(["request-0095"]); + await expect(barrier.wait("request-0095", undefined)).resolves.toBe("released"); + barrier.close(); + await rejectedOpen; + await expect(barrier.wait("request-0096", undefined)).resolves.toBe("aborted"); + + const preAborted = createBarrier(["request-0097"], 10_000); + const preAbortedOpen = expect(preAborted.open).rejects.toThrow("B00B_BARRIER_CLOSED"); + const controller = new AbortController(); + controller.abort(); + await expect(preAborted.wait("request-0097", controller.signal)).resolves.toBe("aborted"); + preAborted.close(); + await preAbortedOpen; + }); test("registers through the real AI registry and holds a 1/4 fanout only as an observation barrier", async () => { const ids = ["request-0001", "request-0002", "request-0003", "request-0004"] as const; const fixture = provider(Object.fromEntries(ids.map((id) => [id, [simple(id, { waitForRelease: true })]])), ids); From 9d9cf28d51490ef06efba738c3fff788463acdff Mon Sep 17 00:00:00 2001 From: Seth Date: Sun, 9 Aug 2026 23:38:03 -0700 Subject: [PATCH 28/28] test(coding-agent): decode RSS worker file URL --- .../coding-agent/test/swarm/run-production-rss-campaign.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts index 9ccbfaf7b..6e38929b1 100644 --- a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts +++ b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts @@ -11,6 +11,7 @@ import { createHash } from "node:crypto"; import { chmod, mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; import { cpus, platform, release, totalmem } from "node:os"; import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; import { DEFAULT_RSS_REQUESTED_PERIOD_MS, MAX_RSS_SAMPLE_GAP_MS, @@ -22,6 +23,7 @@ import { RSS_SCAN_RETRY_DELAY_MS, RSS_SCAN_RETRY_WINDOW_MS, retryUnavailableSnap const FANOUTS = [1, 4, 16, 64] as const; const WORKER = new URL("./rss-campaign-worker.ts", import.meta.url); +const WORKER_PATH = fileURLToPath(WORKER); const DEFAULT_TIMEOUT_MS = 60_000; const REAP_GRACE_MS = 250; const REAP_VERIFY_MS = 1_000; @@ -371,7 +373,7 @@ function workerArguments(settings: Config, fanout: number, scratch: string): str const args = [ "--expose-gc", ...childExecArgsWithTsxImport(process.execArgv), - WORKER.pathname, + WORKER_PATH, "--fanout", String(fanout), "--allocation-mib",