diff --git a/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts b/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts new file mode 100644 index 000000000..bf56ceb90 --- /dev/null +++ b/packages/coding-agent/test/swarm/daemon-production-dispatch.test.ts @@ -0,0 +1,803 @@ +/** + * Real supervisor/worker dispatch coverage. The HTTP fixture is deliberately + * local: workers use the production OpenAI-completions transport, while the + * test observes request entry without installing a provider in either worker. + */ +import { type ChildProcess, execFileSync, spawn } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { + existsSync, + lstatSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + readlinkSync, + rmSync, + symlinkSync, + writeFileSync, +} from "node:fs"; +import { createServer, type ServerResponse } from "node:http"; +import { createConnection, type Socket } from "node:net"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { afterEach, describe, expect, test } from "vitest"; +import { ENV_AGENT_DIR } from "../../src/config.js"; +import { DaemonClient } from "../../src/modes/daemon/daemon-client.js"; +import type { DaemonEventCursor, DaemonOutbound, DaemonResponse } from "../../src/modes/daemon/daemon-protocol.js"; +import type { SessionSummary } from "../../src/modes/daemon/daemon-session-list.js"; + +const cliPath = resolve(__dirname, "../../src/cli.ts"); +const tsxPath = resolve(__dirname, "../../../../node_modules/tsx/dist/cli.mjs"); +const resources: Array<() => Promise | void> = []; +const completedRoots: string[] = []; + +afterEach(async () => { + while (resources.length) await resources.pop()?.(); +}); + +function pause(ms: number): Promise { + return new Promise((resolveDelay) => setTimeout(resolveDelay, ms)); +} + +async function eventually(predicate: () => boolean, code: string, timeoutMs = 15_000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (predicate()) return; + await pause(20); + } + throw new Error(code); +} + +async function waitForProcessGone(pid: number): Promise { + await eventually(() => { + try { + process.kill(pid, 0); + return false; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "ESRCH"; + } + }, `B00B_WORKER_${pid}_SURVIVED`); +} + +function recursiveNormalFiles(directory: string): string[] { + if (!existsSync(directory)) return []; + const result: string[] = []; + for (const entry of readdirSync(directory)) { + const path = join(directory, entry); + try { + const stat = lstatSync(path); + if (stat.isDirectory()) result.push(...recursiveNormalFiles(path)); + else if (stat.isFile()) result.push(path); + } catch { + // Cleanup can atomically rename/remove an entry while an assertion scans it. + } + } + return result; +} + +function recursivePaths(directory: string): string[] { + if (!existsSync(directory)) return []; + const result: string[] = []; + for (const entry of readdirSync(directory)) { + const path = join(directory, entry); + result.push(path); + try { + if (lstatSync(path).isDirectory()) result.push(...recursivePaths(path)); + } catch { + // See recursiveNormalFiles. + } + } + return result; +} + +async function removeTempRoot(root: string): Promise { + for (let attempt = 0; attempt < 20; attempt++) { + try { + rmSync(root, { recursive: true, force: true, maxRetries: 1, retryDelay: 25 }); + return; + } catch { + await pause(50); + } + } + throw new Error("B00B_TEMP_CLEANUP_FAILED"); +} +function cwdUnderRoots(cwd: string, roots: readonly string[]): boolean { + const normalizedCwd = cwd.replace(/\s+\(deleted\)$/, ""); + return roots.some((root) => normalizedCwd === root || normalizedCwd.startsWith(`${root}/`)); +} + +/** Linux has a portable cwd handle for every visible process; do not require lsof in the pinned image. */ +function cwdPidsUnderProc(roots: readonly string[], procRoot = "/proc"): number[] { + let entries: string[]; + try { + entries = readdirSync(procRoot); + } catch (error) { + throw new Error(`B00B_PROC_ROOT_UNREADABLE ${procRoot}: ${(error as Error).message}`); + } + + const matching = new Set(); + for (const entry of entries) { + if (!/^\d+$/.test(entry)) continue; + const pid = Number.parseInt(entry, 10); + try { + if (cwdUnderRoots(readlinkSync(join(procRoot, entry, "cwd")), roots)) matching.add(pid); + } catch (error) { + // Processes can exit, or their cwd can be inaccessible, between readdir and readlink. + const code = (error as NodeJS.ErrnoException).code; + if (code === "ENOENT" || code === "EACCES" || code === "EPERM") continue; + throw new Error(`B00B_PROC_CWD_UNREADABLE ${join(procRoot, entry, "cwd")}: ${(error as Error).message}`); + } + } + return [...matching]; +} + +function cwdPidsUnderDarwin(roots: readonly string[]): number[] { + let output: string; + try { + output = execFileSync("lsof", ["-n", "-Fpn", "-a", "-d", "cwd"], { encoding: "utf8" }); + } catch (error) { + const errno = error as NodeJS.ErrnoException & { status?: unknown }; + if (errno.status === 1) return []; + if (errno.code === "ENOENT") + throw new Error("B00B_LSOF_UNAVAILABLE_ON_DARWIN: install lsof to enforce the cwd residue assertion"); + throw error; + } + let pid: number | undefined; + const matching = new Set(); + for (const line of output.split("\n")) { + if (line.startsWith("p")) pid = Number.parseInt(line.slice(1), 10); + if (!line.startsWith("n") || pid === undefined) continue; + if (cwdUnderRoots(line.slice(1), roots)) matching.add(pid); + } + return [...matching]; +} + +function cwdPidsUnder(roots: readonly string[]): number[] { + if (process.platform === "linux") return cwdPidsUnderProc(roots); + if (process.platform === "darwin") return cwdPidsUnderDarwin(roots); + throw new Error(`B00B_CWD_RESIDUE_CHECK_UNSUPPORTED_PLATFORM: ${process.platform}`); +} + +function assertNoRunResidue(roots: readonly string[]): void { + expect(roots.every((root) => !existsSync(root))).toBe(true); + expect(cwdPidsUnder(roots)).toEqual([]); + const runNames = new Set(roots.map((root) => root.slice(root.lastIndexOf("/") + 1))); + expect(readdirSync(tmpdir()).filter((entry) => runNames.has(entry))).toEqual([]); +} + +function assertNoFixtureKey(texts: readonly string[], key: string): void { + const normalizedKey = key.replace(/[^a-zA-Z0-9]/g, "").toLowerCase(); + const escapedKey = [...key] + .map((character) => `\\u${character.charCodeAt(0).toString(16).padStart(4, "0")}`) + .join(""); + for (const text of texts) { + const decoded = text + .replace(/\\u\{([\dA-Fa-f]+)\}/g, (_, hex: string) => String.fromCodePoint(Number.parseInt(hex, 16))) + .replace(/\\u([\dA-Fa-f]{4})/g, (_, hex: string) => String.fromCharCode(Number.parseInt(hex, 16))) + .replace(/\\x([\dA-Fa-f]{2})/g, (_, hex: string) => String.fromCharCode(Number.parseInt(hex, 16))); + const normalized = decoded.replace(/[^a-zA-Z0-9]/g, "").toLowerCase(); + expect(text).not.toContain(key); + expect(text).not.toContain(escapedKey); + expect(decoded).not.toContain(key); + expect(normalized).not.toContain(normalizedKey); + // Catch a key serialized as fragments with punctuation/whitespace between characters. + const splitKey = [...key] + .map((character) => character.replace(/[\^$.*+?()[\]{}|]/g, "\\$&")) + .join("[^a-zA-Z0-9]*"); + expect(decoded).not.toMatch(new RegExp(splitKey, "i")); + } +} +function summary(value: unknown): SessionSummary { + if (!value || typeof value !== "object") throw new Error("B00B_MISSING_SESSION_SUMMARY"); + return value as SessionSummary; +} + +function active(summaryValue: SessionSummary): string { + return summaryValue.activeSessionId ?? summaryValue.id; +} + +function requestId(body: string): string { + const match = /request-\d{4}/.exec(body); + if (!match) throw new Error("B00B_LOCAL_FIXTURE_MISSING_REQUEST_ID"); + return match[0]; +} + +interface ProviderAttempt { + readonly requestId: string; + readonly rootIdentity: string; + readonly attempt: number; + readonly enteredAt: number; + responseStatus?: number; + responseAt?: number; + responseEndedAt?: number; + requestAbortedAt?: number; + requestClosedAt?: number; + responseClosedAt?: number; +} + +interface LocalProvider { + readonly url: string; + readonly entered: readonly string[]; + readonly attempts: readonly ProviderAttempt[]; + readonly maxInFlight: number; + release(ids: readonly string[]): void; + close(): Promise; +} + +function rootIdentity(body: string): string { + const match = /b00b-root:([^"\s]+)/.exec(body); + if (!match) throw new Error("B00B_LOCAL_FIXTURE_MISSING_ROOT_IDENTITY"); + return match[1]; +} + +/** Test-only preload: observes, but never changes, the real Socket.write result. */ +function createSocketWriteObserver(root: string): { preloadPath: string; tracePath: string } { + const preloadPath = join(root, "socket-write-observer.cjs"); + const tracePath = join(root, "socket-write-0600.log"); + writeFileSync( + preloadPath, + `const { appendFileSync } = require("node:fs"); +const { Socket } = require("node:net"); +const trace = process.env.B00B_SOCKET_WRITE_TRACE; +// Workers inherit NODE_OPTIONS, but their role env exists before preload evaluation. +if (trace && !process.env.PRIME_AGENT_INTERNAL_DAEMON_WORKER) { + const realWrite = Socket.prototype.write; + Socket.prototype.write = function (...args) { + const accepted = realWrite.apply(this, args); + const wire = args[0]; + const text = Buffer.isBuffer(wire) ? wire.toString("utf8") : String(wire); + if (!accepted && text.includes('"type":"session_event"')) { + try { appendFileSync(trace, "0600 " + JSON.stringify({ writableLength: this.writableLength, bytes: Buffer.byteLength(text) }) + "\\n"); } catch {} + } + return accepted; + }; +}`, + ); + return { preloadPath, tracePath }; +} + +/** + * This is a real HTTP/SSE upstream from the worker's perspective. It is not a + * model limiter: each POST enters immediately and receives its own scripted + * response. 429 is an actual upstream HTTP response, never a local result. + */ +async function localProvider(canary: string): Promise { + const entered: string[] = []; + const attempts: ProviderAttempt[] = []; + let inFlight = 0; + let maxInFlight = 0; + const releaseWaiters = new Map void>>(); + const sockets = new Set(); + let closePromise: Promise | undefined; + const waitForRelease = (id: string, response: ServerResponse): Promise => + new Promise((resolveRelease) => { + const release = () => finish(true); + const closed = () => finish(false); + const finish = (wasReleased: boolean) => { + releaseWaiters.get(id)?.delete(release); + response.off("close", closed); + resolveRelease(wasReleased); + }; + const waiters = releaseWaiters.get(id) ?? new Set<() => void>(); + waiters.add(release); + releaseWaiters.set(id, waiters); + // Worker cancellation destroys the response; it never relies on request.destroyed, + // which can be true after an otherwise usable async request body is read. + response.once("close", closed); + }); + const server = createServer((request, response) => { + let body = ""; + request.setEncoding("utf8"); + request.on("data", (chunk: string) => { + body += chunk; + }); + request.on("end", () => { + void (async () => { + let id: string; + try { + id = requestId(body); + } catch { + response.writeHead(400).end("B00B_BAD_LOCAL_REQUEST"); + return; + } + if (request.headers.authorization !== `Bearer ${canary}`) { + response.writeHead(401).end("B00B_BAD_LOCAL_AUTHORIZATION"); + return; + } + let identity: string; + try { + identity = rootIdentity(body); + } catch { + response.writeHead(400).end("B00B_BAD_ROOT_IDENTITY"); + return; + } + entered.push(id); + const record: ProviderAttempt = { + requestId: id, + rootIdentity: identity, + attempt: entered.filter((entry) => entry === id).length, + enteredAt: Date.now(), + }; + attempts.push(record); + request.once("aborted", () => { + record.requestAbortedAt = Date.now(); + }); + request.once("close", () => { + record.requestClosedAt = Date.now(); + }); + response.once("close", () => { + record.responseClosedAt = Date.now(); + }); + inFlight += 1; + maxInFlight = Math.max(maxInFlight, inFlight); + try { + if (!(await waitForRelease(id, response))) return; + if (response.destroyed || response.writableEnded) return; + if (id === "request-0003" && record.attempt === 1) { + record.responseStatus = 429; + record.responseAt = Date.now(); + response.writeHead(429, { "content-type": "application/json", "retry-after": "0" }); + response.end( + JSON.stringify({ error: { message: "fixture upstream 429", type: "rate_limit_error" } }), + ); + return; + } + if (id === "request-0002") await pause(2_000); + if (response.destroyed || response.writableEnded) return; + record.responseStatus = 200; + record.responseAt = Date.now(); + response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" }); + const content = id === "request-0001" ? `${"x".repeat(512 * 1024)} fast-tail` : "cancelled-root-content"; + const event = (value: unknown) => response.write(`data: ${JSON.stringify(value)}\n\n`); + event({ + id: `fixture-${id}`, + model: "fixture-resolved", + choices: [{ index: 0, delta: { role: "assistant", content }, finish_reason: null }], + }); + event({ + id: `fixture-${id}`, + model: "fixture-resolved", + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { prompt_tokens: 7, completion_tokens: 11, total_tokens: 18 }, + }); + response.end("data: [DONE]\n\n"); + record.responseEndedAt = Date.now(); + } finally { + inFlight -= 1; + } + })().catch(() => { + if (!response.headersSent) response.writeHead(500); + response.end("B00B_LOCAL_FIXTURE_FAILURE"); + }); + }); + }); + server.on("connection", (socket) => { + sockets.add(socket); + socket.once("close", () => sockets.delete(socket)); + }); + await new Promise((resolveListen) => server.listen(0, "127.0.0.1", resolveListen)); + const address = server.address(); + + if (!address || typeof address === "string" || address.address !== "127.0.0.1") + throw new Error("B00B_LOCAL_FIXTURE_NOT_LOOPBACK_ONLY"); + return { + url: `http://127.0.0.1:${address.port}/v1`, + entered, + attempts, + get maxInFlight() { + return maxInFlight; + }, + release: (ids) => { + // Release only attempts already at the named provider barrier. A retry + // remains independently held until this method is called again. + for (const id of ids) for (const waiter of [...(releaseWaiters.get(id) ?? [])]) waiter(); + }, + close: () => { + if (closePromise) return closePromise; + closePromise = new Promise((resolveClose) => { + for (const socket of sockets) socket.destroy(); + server.close(() => resolveClose()); + }); + return closePromise; + }, + }; +} +function spawnSupervisor( + agentDir: string, + socketPath: string, + cwd: string, + canary: string, + observer: { preloadPath: string; tracePath: string }, +): ChildProcess { + const child = spawn(process.execPath, [tsxPath, cliPath, "--mode", "daemon", "--daemon-socket", socketPath], { + cwd, + env: { + ...process.env, + [ENV_AGENT_DIR]: agentDir, + B00B_FIXTURE_KEY: canary, + B00B_SOCKET_WRITE_TRACE: observer.tracePath, + NODE_OPTIONS: `${process.env.NODE_OPTIONS ?? ""} --require ${observer.preloadPath}`.trim(), + TSX_TSCONFIG_PATH: resolve(__dirname, "../../../../tsconfig.json"), + PRIME_AGENT_INTERNAL_DAEMON_WORKER: undefined, + PRIME_AGENT_INTERNAL_DAEMON_WORKER_TOKEN: undefined, + PRIME_AGENT_INTERNAL_DAEMON_WORKER_ACTIVE_SESSION_ID: undefined, + PRIME_AGENT_INTERNAL_DAEMON_SUPERVISOR_SOCKET: undefined, + PRIME_AGENT_INTERNAL_DAEMON_WORKER_RECOVERY_JOURNAL: undefined, + PRIME_AGENT_INTERNAL_DAEMON_WORKER_STARTUP_GATE_FD: undefined, + PRIME_AGENT_INTERNAL_SESSION_LEASES_ENABLED: undefined, + PRIME_AGENT_INTERNAL_SESSION_LEASE_OWNER_ID: undefined, + }, + stdio: ["ignore", "ignore", "pipe"], + }); + let stderr = ""; + child.stderr?.on("data", (chunk: Buffer) => { + stderr += chunk.toString("utf8"); + }); + Object.assign(child, { b00bStderr: () => stderr }); + resources.push(() => { + if (child.exitCode === null) child.kill("SIGTERM"); + }); + return child; +} + +async function connect(socketPath: string, child: ChildProcess): Promise { + const deadline = Date.now() + 15_000; + let lastError = ""; + while (Date.now() < deadline) { + if (child.exitCode !== null) throw new Error("B00B_SUPERVISOR_EXITED"); + const client = new DaemonClient(socketPath); + try { + await client.connect(200); + await client.waitForHello(1_000); + return client; + } catch (error) { + lastError = String(error); + client.close(); + await pause(25); + } + } + throw new Error( + `B00B_SUPERVISOR_CONNECT_TIMEOUT ${lastError} ${(child as ChildProcess & { b00bStderr?: () => string }).b00bStderr?.() ?? ""}`, + ); +} + +async function attachThenPause( + socketPath: string, + activeSessionId: string, +): Promise<{ cursor: DaemonEventCursor; close(): void }> { + const socket = createConnection(socketPath); + resources.push(() => { + socket.destroy(); + }); + const first = await new Promise((resolveLine, rejectLine) => { + let buffered = ""; + const timeout = setTimeout(() => rejectLine(new Error("B00B_BLOCKED_ATTACH_TIMEOUT")), 5_000); + socket.on("error", rejectLine); + socket.on("data", (chunk: Buffer) => { + buffered += chunk.toString("utf8"); + const newline = buffered.indexOf("\n"); + if (newline < 0) return; + const line = buffered.slice(0, newline); + buffered = buffered.slice(newline + 1); + const decoded = JSON.parse(line) as DaemonResponse | { type: "daemon_hello" }; + if (decoded.type === "daemon_hello") return; + clearTimeout(timeout); + socket.pause(); // Known cursor reached. Do not drain this attachment. + resolveLine(decoded); + }); + socket.on("connect", () => { + socket.write( + `${JSON.stringify({ + type: "command", + id: "blocked-attach", + clientId: "b00b-blocked", + protocol: { name: "prime-agent.daemon", version: 7 }, + command: { + type: "attach", + activeSessionId, + capabilities: ["attach_snapshot", "event_sequence"], + }, + })}\n`, + ); + }); + }); + if (!first.success || !first.data || typeof first.data !== "object") + throw new Error(`B00B_BLOCKED_ATTACH_FAILED ${JSON.stringify(first)}`); + const cursor = (first.data as { lastEventCursor?: DaemonEventCursor }).lastEventCursor; + if (!cursor) throw new Error("B00B_BLOCKED_ATTACH_NO_CURSOR"); + return { cursor, close: () => socket.destroy() }; +} + +describe("B00B real daemon production dispatch", () => { + test("enumerates Linux-style proc cwd links without lsof", () => { + const procRoot = mkdtempSync(join(tmpdir(), "b00b-proc-")); + try { + const matchingRoot = join(procRoot, "matching-root"); + const otherRoot = join(procRoot, "other-root"); + mkdirSync(matchingRoot); + mkdirSync(otherRoot); + mkdirSync(join(procRoot, "101")); + mkdirSync(join(procRoot, "202")); + // A process may exit after /proc is listed, leaving no cwd link. + mkdirSync(join(procRoot, "303")); + mkdirSync(join(procRoot, "404")); + writeFileSync(join(procRoot, "not-a-pid"), "ignored"); + // proc cwd entries are symlinks; Linux appends this suffix for a deleted cwd. + symlinkSync(matchingRoot, join(procRoot, "101", "cwd")); + symlinkSync(otherRoot, join(procRoot, "202", "cwd")); + symlinkSync(`${matchingRoot} (deleted)`, join(procRoot, "404", "cwd")); + expect(cwdPidsUnderProc([matchingRoot], procRoot)).toEqual([101, 404]); + expect(() => cwdPidsUnderProc([], join(procRoot, "missing"))).toThrow("B00B_PROC_ROOT_UNREADABLE"); + } finally { + rmSync(procRoot, { recursive: true, force: true }); + } + }); + + test.each([1, 2, 3])( + "isolates paused attachment, cancellation, and upstream 429 across real supervisor workers (run %i)", + async () => { + const root = mkdtempSync(join(tmpdir(), "b00b-daemon-")); + const canary = `fixture-key-B00B-${randomUUID()}`; + resources.push(() => removeTempRoot(root)); + const agentDir = join(root, "agent"); + const projectDir = join(root, "project"); + const socketPath = join(tmpdir(), `b00b-${process.pid}-${randomUUID().slice(0, 8)}.sock`); + mkdirSync(agentDir, { recursive: true }); + mkdirSync(projectDir, { recursive: true }); + const upstream = await localProvider(canary); + resources.push(() => upstream.close()); + const observer = createSocketWriteObserver(root); + writeFileSync( + join(agentDir, "models.json"), + JSON.stringify({ + providers: { + "b00b-local": { + baseUrl: upstream.url, + apiKey: "B00B_FIXTURE_KEY", + api: "openai-completions", + models: [{ id: "fixture-a", api: "openai-completions", reasoning: false, input: ["text"] }], + }, + }, + }), + ); + const supervisor = spawnSupervisor(agentDir, socketPath, projectDir, canary, observer); + const control = await connect(socketPath, supervisor); + resources.push(() => control.close()); + const create = async (name: string) => { + const result = await control.request({ + type: "create", + name, + config: { + cwd: projectDir, + agentDir, + provider: "b00b-local", + model: "fixture-a", + noTools: true, + noExtensions: true, + noSkills: true, + }, + }); + if (!result.success) throw new Error("B00B_CREATE_ROOT_FAILED"); + return summary(result.data); + }; + const fast = await create("fast-root"); + const cancelled = await create("cancelled-root"); + const rateLimited = await create("rate-limited-root"); + + const workerPids = [fast.workerPid, cancelled.workerPid, rateLimited.workerPid]; + expect(workerPids.every((pid): pid is number => typeof pid === "number" && pid > 1)).toBe(true); + const concreteWorkerPids = workerPids as number[]; + expect(new Set(concreteWorkerPids).size).toBe(3); + + const blocked = await attachThenPause(socketPath, active(fast)); + const draining = await connect(socketPath, supervisor); + resources.push(() => draining.close()); + const drainEvents: Array<{ event: Extract; observedAt: number }> = + []; + draining.onMessage((message) => { + if (message.type === "session_event" && message.activeSessionId === active(fast)) + drainEvents.push({ event: message, observedAt: Date.now() }); + }); + const attached = await draining.request({ + type: "attach", + activeSessionId: active(fast), + capabilities: ["attach_snapshot", "event_sequence"], + }); + expect(attached.success).toBe(true); + + const cancelledObserver = await connect(socketPath, supervisor); + resources.push(() => cancelledObserver.close()); + const cancelledEvents: Array<{ + event: Extract; + observedAt: number; + }> = []; + cancelledObserver.onMessage((message) => { + if (message.type === "session_event" && message.activeSessionId === active(cancelled)) + cancelledEvents.push({ event: message, observedAt: Date.now() }); + }); + const cancelledAttached = await cancelledObserver.request({ + type: "attach", + activeSessionId: active(cancelled), + capabilities: ["attach_snapshot", "event_sequence"], + }); + expect(cancelledAttached.success).toBe(true); + + const dispatch = (session: SessionSummary, id: string) => + control.request( + { type: "prompt", activeSessionId: active(session), message: `${id} b00b-root:${active(session)}` }, + 10_000, + ); + const admissions = await Promise.all([ + dispatch(fast, "request-0001"), + dispatch(cancelled, "request-0002"), + dispatch(rateLimited, "request-0003"), + ]); + expect(admissions.every((item) => item.success)).toBe(true); + await eventually(() => new Set(upstream.entered).size === 3, "B00B_PROVIDER_ENTRY_TIMEOUT"); + // Before any fixture release, all three independent production HTTP requests overlap. + expect(upstream.maxInFlight).toBeGreaterThanOrEqual(3); + expect(upstream.entered.filter((id) => id === "request-0001")).toHaveLength(1); + expect(upstream.entered.filter((id) => id === "request-0002")).toHaveLength(1); + expect(upstream.entered.filter((id) => id === "request-0003")).toHaveLength(1); + // Abort is root-local. Its open response must actually be closed upstream, + // rather than merely suppressing a locally continuing model result. + const aborted = await control.request({ type: "abort", activeSessionId: active(cancelled) }); + expect(aborted.success).toBe(true); + await eventually(() => { + const cancelledAttempt = upstream.attempts.find((attempt) => attempt.requestId === "request-0002"); + return Boolean(cancelledAttempt?.requestAbortedAt || cancelledAttempt?.responseClosedAt); + }, "B00B_CANCEL_DID_NOT_CLOSE_UPSTREAM"); + const cancelledAttempt = upstream.attempts.find((attempt) => attempt.requestId === "request-0002"); + expect(cancelledAttempt).toMatchObject({ + requestId: "request-0002", + rootIdentity: active(cancelled), + attempt: 1, + }); + + // The first rate-root attempt genuinely returns 429. Its second attempt is + // held at the fixture barrier, so a sibling must finish while it backs off. + upstream.release(["request-0003"]); + await eventually( + () => + upstream.attempts.some( + (attempt) => + attempt.requestId === "request-0003" && attempt.attempt === 1 && attempt.responseStatus === 429, + ), + "B00B_GENUINE_429_NOT_OBSERVED", + ); + await eventually( + () => upstream.attempts.some((attempt) => attempt.requestId === "request-0003" && attempt.attempt === 2), + "B00B_429_RETRY_TIMEOUT", + ); + upstream.release(["request-0001"]); + const idle = await control.request({ type: "wait_for_idle", activeSessionId: active(fast) }, 30_000); + expect(idle.success).toBe(true); + await eventually( + () => drainEvents.some(({ event }) => event.event.type === "message_end"), + "B00B_DRAINING_ATTACHMENT_DID_NOT_COMPLETE", + ); + const ordered = drainEvents + .map(({ event }) => event.meta?.sequence) + .filter((sequence): sequence is number => sequence !== undefined); + expect(ordered).toEqual([...ordered].sort((left, right) => left - right)); + expect(ordered.length).toBeGreaterThan(2); + const fastCompletedAt = Date.now(); + const rateFirst = upstream.attempts.find( + (attempt) => attempt.requestId === "request-0003" && attempt.attempt === 1, + ); + const rateSecond = upstream.attempts.find( + (attempt) => attempt.requestId === "request-0003" && attempt.attempt === 2, + ); + expect(rateFirst).toMatchObject({ rootIdentity: active(rateLimited), responseStatus: 429 }); + expect(rateSecond).toMatchObject({ rootIdentity: active(rateLimited) }); + expect(rateFirst?.responseAt).toBeTypeOf("number"); + expect(rateSecond?.enteredAt).toBeLessThanOrEqual(fastCompletedAt); + expect(rateSecond?.responseEndedAt).toBeUndefined(); + + // The paused raw client caused a natural real net.Socket.write false in + // the supervisor. The preload only records its return and writableLength. + const falseWrites = existsSync(observer.tracePath) + ? readFileSync(observer.tracePath, "utf8") + .split("\n") + .filter((line) => line.startsWith("0600 ")) + .map((line) => JSON.parse(line.slice(5)) as { writableLength: number; bytes: number }) + : []; + expect(falseWrites.length).toBeGreaterThanOrEqual(1); + expect(falseWrites.length).toBeLessThanOrEqual(8); + expect(Math.max(...falseWrites.map((entry) => entry.writableLength))).toBeLessThanOrEqual(2 * 1024 * 1024); + expect(Math.max(...falseWrites.map((entry) => entry.bytes))).toBeLessThanOrEqual(2 * 1024 * 1024); + + // Reattach from the cursor known before the paused write. The supervisor + // supplies a bounded snapshot/replay rather than a per-attachment model queue. + const catchup = await connect(socketPath, supervisor); + resources.push(() => catchup.close()); + const resynced = await catchup.request({ + type: "attach", + activeSessionId: active(fast), + capabilities: ["attach_snapshot", "event_sequence"], + resumeCursor: { activeSessionId: active(fast), ...blocked.cursor }, + }); + if (!resynced.success || !resynced.data || typeof resynced.data !== "object") + throw new Error("B00B_CATCHUP_FAILED"); + const catchupData = resynced.data as { snapshot?: { messages?: unknown[] }; replay?: { toSequence?: number } }; + expect(catchupData.snapshot?.messages?.length).toBeGreaterThanOrEqual(2); + expect(catchupData.replay?.toSequence).toBeGreaterThanOrEqual(blocked.cursor.sequence); + + upstream.release(["request-0003"]); + const rateLimitedIdle = await control.request( + { type: "wait_for_idle", activeSessionId: active(rateLimited) }, + 30_000, + ); + expect(rateLimitedIdle.success).toBe(true); + const rateLimitedMessages = await control.request({ + type: "get_messages", + activeSessionId: active(rateLimited), + }); + expect(JSON.stringify(rateLimitedMessages)).toContain("fixture-resolved"); + expect(upstream.entered.filter((id) => id === "request-0003")).toHaveLength(2); + expect(rateSecond?.responseEndedAt).toBeGreaterThan(fastCompletedAt); + + const cancelledIdle = await control.request( + { type: "wait_for_idle", activeSessionId: active(cancelled) }, + 30_000, + ); + expect(cancelledIdle.success).toBe(true); + const cancelledMessages = await control.request({ type: "get_messages", activeSessionId: active(cancelled) }); + expect(JSON.stringify(cancelledMessages)).not.toContain("cancelled-root-content"); + const cancelledTerminals = cancelledEvents.filter( + ({ event }) => + event.event.type === "message_end" && + (event.event.message as { stopReason?: string }).stopReason === "aborted", + ); + expect(cancelledTerminals).toHaveLength(1); + const cancelledTerminalSequence = cancelledTerminals[0]?.event.meta?.sequence ?? -1; + expect( + cancelledEvents.filter( + ({ event }) => + event.event.type === "message_update" && (event.meta?.sequence ?? -1) > cancelledTerminalSequence, + ), + ).toHaveLength(0); + // The provider saw a genuine status-429 request while fast completed; + // no test code implements a permit, semaphore, or fabricated response. + expect(upstream.entered).toContain("request-0003"); + + // Explicitly release every local client, including the deliberately paused raw socket, + // before asking the supervisor to stop accepting work. + blocked.close(); + catchup.close(); + draining.close(); + const shutdown = await control.request({ type: "shutdown" }, 10_000); + expect(shutdown.success).toBe(true); + control.close(); + + const supervisorPid = supervisor.pid; + expect(typeof supervisorPid).toBe("number"); + await Promise.all([ + waitForProcessGone(supervisorPid as number), + ...concreteWorkerPids.map((pid) => waitForProcessGone(pid)), + ]); + await eventually(() => !existsSync(socketPath), "B00B_SUPERVISOR_SOCKET_SURVIVED"); + + const artifactPaths = recursivePaths(root).filter((path) => + /(?:^|[/\\])[^/\\]*(?:recovery|orphan|\.tmp)[^/\\]*$/i.test(path), + ); + expect(artifactPaths).toEqual([]); + const capturedTexts = [ + (supervisor as ChildProcess & { b00bStderr?: () => string }).b00bStderr?.() ?? "", + ...recursiveNormalFiles(root).map((path) => readFileSync(path, "utf8")), + ]; + assertNoFixtureKey(capturedTexts, canary); + await upstream.close(); + await removeTempRoot(root); + completedRoots.push(root); + assertNoRunResidue([root]); + }, + 60_000, + ); + + test("leaves no b00b-daemon cwd process or directory after all repeated runs", () => { + expect(completedRoots).toHaveLength(3); + assertNoRunResidue(completedRoots); + }); +}); diff --git a/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts b/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts new file mode 100644 index 000000000..13fd8a743 --- /dev/null +++ b/packages/coding-agent/test/swarm/production-evidence-adapter.test.ts @@ -0,0 +1,365 @@ +import { createHash, generateKeyPairSync, sign } from "node:crypto"; +import { mkdtemp, readdir, readFile, realpath, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { + type ProductionEvidenceInput, + projectProductionObservations, + verifySignedProductionEvidence, + verifySignedProductionEvidenceFreshProcess, + writeSignedProductionEvidence, +} from "./production-evidence-adapter.js"; +import { + COST_NUMERATOR_SCALE, + canonicalJson, + createSwarmEvidenceTrustRoot, + currentProcessSampler, + SWARM_EVIDENCE_COMMITMENT_SCHEMA, + swarmEvidenceCommitmentPayload, + verifyAuthenticatedSwarmEvidence, +} from "./swarm-evidence.js"; + +const cleanup: string[] = []; +afterEach(async () => { + await Promise.all(cleanup.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +const canaries = ["B00B-adapter-秘密", "B00B-adapter-split-A", "B00B-adapter-split-B"]; +function input(): ProductionEvidenceInput { + return { + scenario: canaries[0]!, + metadata: { [canaries[1]!]: canaries[2] }, + priceCard: { + version: "fixture-price-card-v1", + inputMicroCurrencyPerMillionMicroTokens: 17, + outputMicroCurrencyPerMillionMicroTokens: 29, + }, + attempts: [ + { + requestId: "request-0001", + attempt: 1, + requested: { provider: "b00b-scripted", model: "fixture-a", revision: "alias-secret", effort: "high" }, + resolved: { + api: "b00b-scripted", + provider: "b00b-scripted", + model: "fixture-a", + responseModel: "fixture-b-resolved", + }, + terminal: "done", + usage: { inputMicroTokens: 101, outputMicroTokens: 13, cacheReadMicroTokens: 7, cacheWriteMicroTokens: 3 }, + }, + { + // A failed retry remains a separately authenticated attempt even at zero usage. + requestId: "request-0001", + attempt: 2, + requested: { provider: "b00b-scripted", model: "fixture-zero" }, + resolved: { + api: "b00b-scripted", + provider: "b00b-scripted", + model: "fixture-zero", + responseModel: "fixture-zero-resolved", + }, + terminal: "error", + usage: { inputMicroTokens: 0, outputMicroTokens: 99, cacheReadMicroTokens: 0, cacheWriteMicroTokens: 0 }, + }, + { + requestId: "request-0002", + attempt: 1, + requested: { provider: "b00b-scripted", model: "fixture-zero" }, + resolved: { + api: "b00b-scripted", + provider: "b00b-scripted", + model: "fixture-zero", + responseModel: "fixture-zero-resolved", + }, + terminal: "aborted", + usage: { inputMicroTokens: 9, outputMicroTokens: 99, cacheReadMicroTokens: 2, cacheWriteMicroTokens: 0 }, + }, + ], + }; +} +async function allFiles(directory: string): Promise { + const contents: string[] = []; + for (const entry of await readdir(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) contents.push(await allFiles(path)); + else if (entry.isFile()) contents.push(await readFile(path, "utf8")); + } + return contents.join("\n"); +} +function privacyVariants(value: string): readonly string[] { + const unicodeEscaped = [...value] + .map((character) => `\\u${character.codePointAt(0)!.toString(16).padStart(4, "0")}`) + .join(""); + return [value, value.normalize("NFC"), value.normalize("NFKC"), unicodeEscaped]; +} +function expectNoCanaryLeak(chunks: readonly string[]): void { + const joined = chunks.join(""); + for (const canary of canaries) + for (const variant of privacyVariants(canary)) + expect(joined.normalize("NFKC")).not.toContain(variant.normalize("NFKC")); +} + +/** Coherently re-index a semantic-preserving process-sample mutation. */ +async function forgeProcessSampleBundle(directory: string): Promise { + const samplePath = join(directory, "process-samples.json"); + const samples = JSON.parse(await readFile(samplePath, "utf8")); + const firstSample = samples[0]; + const firstProcess = firstSample.processes[0]; + if (firstProcess) firstProcess.pid += 1; + // B00A's default sampler legitimately returns no processes on some hosts. + // A zero-RSS process remains schema-valid and leaves the sample total intact. + else firstSample.processes.push({ pid: 1, rssBytes: 0 }); + const sampleRaw = `${canonicalJson(samples)}\n`; + await writeFile(samplePath, sampleRaw); + const manifestPath = join(directory, "manifest.json"); + const manifest = JSON.parse(await readFile(manifestPath, "utf8")); + const artifact = manifest.artifacts.find((item: { path: string }) => item.path === "process-samples.json"); + artifact.bytes = Buffer.byteLength(sampleRaw); + artifact.sha256 = createHash("sha256").update(sampleRaw).digest("hex"); + manifest.artifactBundleId = createHash("sha256").update(canonicalJson(manifest.artifacts)).digest("hex"); + await writeFile(manifestPath, `${canonicalJson(manifest)}\n`); + return manifest.artifactBundleId; +} + +describe("B00B signed production evidence adapter", () => { + test("authenticates an external Ed25519 commitment before canonical B00A verification in a fresh Node process", async () => { + const artifactDirectory = await mkdtemp(join(tmpdir(), "b00b-artifact-")); + const trustDirectory = await mkdtemp(join(tmpdir(), "b00b-trust-")); + cleanup.push(artifactDirectory, trustDirectory); + const keys = generateKeyPairSync("ed25519"); + const publicPem = keys.publicKey.export({ type: "spki", format: "pem" }).toString(); + const written = await writeSignedProductionEvidence(artifactDirectory, trustDirectory, input(), keys.privateKey); + const canonicalTrustDirectory = await realpath(trustDirectory); + const canonicalArtifactDirectory = await realpath(artifactDirectory); + expect(written.commitmentPath.startsWith(`${canonicalTrustDirectory}/`)).toBe(true); + expect(written.commitmentPath.startsWith(`${canonicalArtifactDirectory}/`)).toBe(false); + await expect( + verifySignedProductionEvidence(artifactDirectory, written.commitmentPath, publicPem), + ).resolves.toBeUndefined(); + await expect( + verifySignedProductionEvidenceFreshProcess(artifactDirectory, written.commitmentPath, publicPem), + ).resolves.toBeUndefined(); + const content = await allFiles(artifactDirectory); + expectNoCanaryLeak([content]); + const costs = JSON.parse(await readFile(join(artifactDirectory, "cost-attribution.json"), "utf8")); + const firstAttempt = costs.find((cost: { id: string }) => cost.id === "worker-0001"); + expect(firstAttempt).toMatchObject({ + directInputTokens: 111, + directOutputTokens: 13, + directCostNumerator: 111 * 17 + 13 * 29, + }); + expect(COST_NUMERATOR_SCALE).toBe(1_000_000); + const run = costs.find((cost: { id: string }) => cost.id === "run"); + expect(run).toMatchObject({ + downstreamInputTokens: 122, + downstreamOutputTokens: 13, + downstreamCostNumerator: 122 * 17 + 13 * 29, + }); + // The terminal response model, not the selected requested alias, is retained as the resolved attribution. + expect(content).toContain("fixture-b-resolved"); + expect(content).toContain('"api":"b00b-scripted"'); + expect(content).toContain('"responseModel":"fixture-b-resolved"'); + expect(content).not.toContain("alias-secret"); + const manifest = JSON.parse(await readFile(join(artifactDirectory, "manifest.json"), "utf8")); + expect(manifest.assignments.map((assignment: { attemptId: string }) => assignment.attemptId)).toEqual([ + "attempt-0001-01", + "attempt-0001-02", + "attempt-0002-01", + ]); + const retry = manifest.assignments[1]; + expect(retry.resolved).toMatchObject({ model: "fixture-zero", responseModel: "fixture-zero-resolved" }); + }); + + test("rejects manifest read-back, coherent forgery, wrong key, and a commitment from another artifact directory", async () => { + const firstArtifactDirectory = await mkdtemp(join(tmpdir(), "b00b-first-artifact-")); + const firstTrustDirectory = await mkdtemp(join(tmpdir(), "b00b-first-trust-")); + const secondArtifactDirectory = await mkdtemp(join(tmpdir(), "b00b-second-artifact-")); + const secondTrustDirectory = await mkdtemp(join(tmpdir(), "b00b-second-trust-")); + cleanup.push(firstArtifactDirectory, firstTrustDirectory, secondArtifactDirectory, secondTrustDirectory); + const signer = generateKeyPairSync("ed25519"); + const publicPem = signer.publicKey.export({ type: "spki", format: "pem" }).toString(); + const first = await writeSignedProductionEvidence( + firstArtifactDirectory, + firstTrustDirectory, + input(), + signer.privateKey, + ); + const second = await writeSignedProductionEvidence( + secondArtifactDirectory, + secondTrustDirectory, + input(), + signer.privateKey, + ); + + const manifestReadBack = await forgeProcessSampleBundle(firstArtifactDirectory); + expect(manifestReadBack).not.toBe(first.artifactBundleId); + // A new ID derived from mutable artifacts has no authority over the original signature. + await expect( + verifySignedProductionEvidence(firstArtifactDirectory, first.commitmentPath, publicPem), + ).rejects.toThrow("trusted artifact bundle mismatch"); + await expect( + verifySignedProductionEvidence(firstArtifactDirectory, second.commitmentPath, publicPem), + ).rejects.toThrow("trusted artifact bundle mismatch"); + // An attacker can self-generate a key and sign the manifest read-back ID, + // but this cannot replace the externally configured root. + const attacker = generateKeyPairSync("ed25519"); + const attackerCommitmentPath = join(firstTrustDirectory, "attacker-commitment.json"); + await writeFile( + attackerCommitmentPath, + `${canonicalJson({ + schemaVersion: SWARM_EVIDENCE_COMMITMENT_SCHEMA, + artifactBundleId: manifestReadBack, + signature: sign( + null, + Buffer.from(canonicalJson(swarmEvidenceCommitmentPayload(manifestReadBack))), + attacker.privateKey, + ).toString("base64"), + })}\n`, + ); + await expect( + verifySignedProductionEvidence(firstArtifactDirectory, attackerCommitmentPath, publicPem), + ).rejects.toThrow("B00B_EVIDENCE_BAD_SIGNATURE"); + const wrongKey = generateKeyPairSync("ed25519").publicKey.export({ type: "spki", format: "pem" }).toString(); + await expect( + verifySignedProductionEvidence(secondArtifactDirectory, second.commitmentPath, wrongKey), + ).rejects.toThrow("B00B_EVIDENCE_BAD_SIGNATURE"); + }); + + test("coherently forges an empty default B00A sample but cannot satisfy the original external commitment", async () => { + const artifactDirectory = await mkdtemp(join(tmpdir(), "b00b-empty-artifact-")); + const trustDirectory = await mkdtemp(join(tmpdir(), "b00b-empty-trust-")); + cleanup.push(artifactDirectory, trustDirectory); + const sampler = vi.spyOn(currentProcessSampler, "sample").mockReturnValue([]); + const signer = generateKeyPairSync("ed25519"); + const publicPem = signer.publicKey.export({ type: "spki", format: "pem" }).toString(); + let written: Awaited>; + try { + written = await writeSignedProductionEvidence(artifactDirectory, trustDirectory, input(), signer.privateKey); + } finally { + sampler.mockRestore(); + } + const originalSamples = JSON.parse(await readFile(join(artifactDirectory, "process-samples.json"), "utf8")); + expect(originalSamples[0].processes).toEqual([]); + expect(originalSamples[0].totalRssBytes).toBe(0); + const forgedBundleId = await forgeProcessSampleBundle(artifactDirectory); + const forgedSamples = JSON.parse(await readFile(join(artifactDirectory, "process-samples.json"), "utf8")); + expect(forgedSamples[0]).toMatchObject({ processes: [{ pid: 1, rssBytes: 0 }], totalRssBytes: 0 }); + // The forged artifact remains B00A-canonical when re-indexed against its new identity. + const attacker = generateKeyPairSync("ed25519"); + const attackerCommitmentPath = join(trustDirectory, "attacker-commitment.json"); + await writeFile( + attackerCommitmentPath, + `${canonicalJson({ + schemaVersion: SWARM_EVIDENCE_COMMITMENT_SCHEMA, + artifactBundleId: forgedBundleId, + signature: sign( + null, + Buffer.from(canonicalJson(swarmEvidenceCommitmentPayload(forgedBundleId))), + attacker.privateKey, + ).toString("base64"), + })}\n`, + ); + const attackerPublicPem = attacker.publicKey.export({ type: "spki", format: "pem" }).toString(); + await expect( + verifySignedProductionEvidence(artifactDirectory, attackerCommitmentPath, attackerPublicPem), + ).resolves.toBeUndefined(); + await expect( + verifySignedProductionEvidence(artifactDirectory, written.commitmentPath, publicPem), + ).rejects.toThrow("trusted artifact bundle mismatch"); + }); + + test("rejects a coherent manifest/index forgery and tampered external commitment", async () => { + const artifactDirectory = await mkdtemp(join(tmpdir(), "b00b-artifact-")); + const trustDirectory = await mkdtemp(join(tmpdir(), "b00b-trust-")); + cleanup.push(artifactDirectory, trustDirectory); + const keys = generateKeyPairSync("ed25519"); + const publicPem = keys.publicKey.export({ type: "spki", format: "pem" }).toString(); + const written = await writeSignedProductionEvidence(artifactDirectory, trustDirectory, input(), keys.privateKey); + const manifestPath = join(artifactDirectory, "manifest.json"); + const manifest = JSON.parse(await readFile(manifestPath, "utf8")); + // A read-back / coherent-index attacker can choose a new bundle identity, but cannot forge the external signature. + manifest.artifactBundleId = "0".repeat(64); + await writeFile(manifestPath, `${canonicalJson(manifest)}\n`); + await expect( + verifySignedProductionEvidence(artifactDirectory, written.commitmentPath, publicPem), + ).rejects.toThrow("artifact bundle identity mismatch"); + const commitment = JSON.parse(await readFile(written.commitmentPath, "utf8")); + commitment.artifactBundleId = "0".repeat(64); + await writeFile(written.commitmentPath, `${canonicalJson(commitment)}\n`); + await expect( + verifySignedProductionEvidence(artifactDirectory, written.commitmentPath, publicPem), + ).rejects.toThrow("B00B_EVIDENCE_BAD_SIGNATURE"); + }); + test("keeps zero-price, cache, done/error/abort, and retry economics in exact numerators", () => { + const source = input(); + const zero: ProductionEvidenceInput = { + ...source, + priceCard: { + ...source.priceCard, + inputMicroCurrencyPerMillionMicroTokens: 0, + outputMicroCurrencyPerMillionMicroTokens: 0, + }, + }; + const projected = projectProductionObservations(zero); + expect(projected.assignments).toHaveLength(3); + expect(projected.assignments.map((assignment) => assignment.inputTokens)).toEqual([111, 0, 11]); + expect(projected.assignments.map((assignment) => assignment.outputTokens)).toEqual([13, 0, 0]); + expect(projected.assignments.map((assignment) => assignment.attemptId)).toEqual([ + "attempt-0001-01", + "attempt-0001-02", + "attempt-0002-01", + ]); + expect(projected.priceCard).toMatchObject({ inputPerMillionTokens: 0, outputPerMillionTokens: 0 }); + }); + + test("privacy scans recursive normal outputs and captured console/stderr in normalized, escaped, and split forms", async () => { + const artifactDirectory = await mkdtemp(join(tmpdir(), "b00b-privacy-artifact-")); + const trustDirectory = await mkdtemp(join(tmpdir(), "b00b-privacy-trust-")); + cleanup.push(artifactDirectory, trustDirectory); + const keys = generateKeyPairSync("ed25519"); + const capturedConsole: string[] = []; + const capturedStderr: string[] = []; + const originalError = console.error; + const originalWrite = process.stderr.write; + console.error = (...values: unknown[]) => capturedConsole.push(values.map(String).join(" ")); + process.stderr.write = ((chunk: unknown) => { + capturedStderr.push(String(chunk)); + return true; + }) as typeof process.stderr.write; + try { + await writeSignedProductionEvidence(artifactDirectory, trustDirectory, input(), keys.privateKey); + } finally { + console.error = originalError; + process.stderr.write = originalWrite; + } + expectNoCanaryLeak([await allFiles(artifactDirectory), ...capturedConsole, ...capturedStderr]); + }); + test("rejects fabricated trust roots and malformed or noncanonical commitments", async () => { + const artifactDirectory = await mkdtemp(join(tmpdir(), "b00b-root-artifact-")); + const trustDirectory = await mkdtemp(join(tmpdir(), "b00b-root-trust-")); + cleanup.push(artifactDirectory, trustDirectory); + const keys = generateKeyPairSync("ed25519"); + const written = await writeSignedProductionEvidence(artifactDirectory, trustDirectory, input(), keys.privateKey); + const publicPem = keys.publicKey.export({ type: "spki", format: "pem" }).toString(); + const commitment = await readFile(written.commitmentPath, "utf8"); + await expect( + verifyAuthenticatedSwarmEvidence( + artifactDirectory, + commitment, + {} as ReturnType, + ), + ).rejects.toThrow("registered swarm evidence trust root is required"); + await expect( + verifyAuthenticatedSwarmEvidence(artifactDirectory, `${commitment} `, createSwarmEvidenceTrustRoot(publicPem)), + ).rejects.toThrow("non-canonical JSON: artifact commitment"); + await expect( + verifyAuthenticatedSwarmEvidence( + artifactDirectory, + `${canonicalJson({ schemaVersion: SWARM_EVIDENCE_COMMITMENT_SCHEMA, artifactBundleId: written.artifactBundleId, signature: "%%%" })}\n`, + createSwarmEvidenceTrustRoot(publicPem), + ), + ).rejects.toThrow("B00B_EVIDENCE_BAD_SIGNATURE"); + }); +}); diff --git a/packages/coding-agent/test/swarm/production-evidence-adapter.ts b/packages/coding-agent/test/swarm/production-evidence-adapter.ts new file mode 100644 index 000000000..3ba6bb0fe --- /dev/null +++ b/packages/coding-agent/test/swarm/production-evidence-adapter.ts @@ -0,0 +1,242 @@ +/** + * B00B bridge from immutable production-path observations to B00A evidence. + * + * This module never serializes B00A artifacts itself. It projects the small, + * content-free observation surface into B00A's public input, calls its writer, + * and keeps the authenticated artifact commitment in a sibling trust root. + */ +import { execFile as execFileCallback } from "node:child_process"; +import { type KeyObject, sign } from "node:crypto"; +import { mkdir, readFile, realpath, writeFile } from "node:fs/promises"; +import { promisify } from "node:util"; +import { + artifactBundleIdForSwarmEvidenceCapability, + canonicalJson, + createSwarmEvidenceTrustRoot, + runSwarmBenchmark, + SWARM_EVIDENCE_COMMITMENT_SCHEMA, + type SwarmBenchmarkConfig, + swarmEvidenceCommitmentPayload, + verifyAuthenticatedSwarmEvidence, + writeSwarmEvidence, +} from "./swarm-evidence.js"; + +const execFile = promisify(execFileCallback); +const MODEL_IDS = new Set(["fixture-a", "fixture-b", "fixture-zero"]); +const RESPONSE_MODEL_IDS = new Set([...MODEL_IDS].map((id) => `${id}-resolved`)); +const PROVIDER = "b00b-scripted"; + +export interface ExactUsage { + /** Integer micro-tokens. No floating point token or price field is accepted. */ + readonly inputMicroTokens: number; + readonly outputMicroTokens: number; + readonly cacheReadMicroTokens: number; + readonly cacheWriteMicroTokens: number; +} +export interface ImmutableAttemptObservation { + readonly requestId: `request-${string}`; + readonly attempt: number; + readonly requested: Readonly<{ provider: string; model: string; revision?: string; effort?: string }>; + readonly resolved: Readonly<{ api: string; provider: string; model: string; responseModel: string }>; + readonly terminal: "done" | "error" | "aborted"; + readonly usage: ExactUsage; +} +export interface FrozenPriceCard { + /** Integer micro-currency per million micro-tokens, frozen before dispatch. */ + readonly version: string; + readonly inputMicroCurrencyPerMillionMicroTokens: number; + readonly outputMicroCurrencyPerMillionMicroTokens: number; +} +export interface ProductionEvidenceInput { + readonly scenario: string; + readonly attempts: readonly ImmutableAttemptObservation[]; + readonly priceCard: FrozenPriceCard; + readonly metadata?: Readonly>; +} +export interface SignedProductionEvidence { + readonly artifactBundleId: string; + readonly commitmentPath: string; +} + +function assert(condition: unknown, code: string): asserts condition { + if (!condition) throw new Error(code); +} +function integer(value: number): boolean { + return Number.isSafeInteger(value) && value >= 0; +} +function safeModel(value: string, responseModel = false): string { + return (responseModel ? RESPONSE_MODEL_IDS : MODEL_IDS).has(value) ? value : "[REDACTED]"; +} +function publicAttemptId(observation: ImmutableAttemptObservation): string { + return `attempt-${observation.requestId.slice("request-".length)}-${String(observation.attempt).padStart(2, "0")}`; +} +function assertInput(input: ProductionEvidenceInput): void { + assert(input.attempts.length > 0, "B00B_EVIDENCE_NO_ATTEMPTS"); + assert(input.scenario.length > 0, "B00B_EVIDENCE_EMPTY_SCENARIO"); + assert( + integer(input.priceCard.inputMicroCurrencyPerMillionMicroTokens) && + integer(input.priceCard.outputMicroCurrencyPerMillionMicroTokens), + "B00B_EVIDENCE_NON_INTEGER_PRICE", + ); + const identities = new Set(); + for (const observation of input.attempts) { + assert(/^request-\d{4}$/.test(observation.requestId), "B00B_EVIDENCE_REQUEST_ID"); + assert(integer(observation.attempt) && observation.attempt > 0, "B00B_EVIDENCE_ATTEMPT"); + assert(!identities.has(`${observation.requestId}:${observation.attempt}`), "B00B_EVIDENCE_DUPLICATE_ATTEMPT"); + identities.add(`${observation.requestId}:${observation.attempt}`); + for (const value of Object.values(observation.usage)) assert(integer(value), "B00B_EVIDENCE_NON_INTEGER_USAGE"); + assert( + Boolean(observation.requested.provider && observation.requested.model), + "B00B_EVIDENCE_REQUESTED_PROVENANCE", + ); + assert( + Boolean( + observation.resolved.api && + observation.resolved.provider && + observation.resolved.model && + observation.resolved.responseModel, + ), + "B00B_EVIDENCE_RESOLVED_PROVENANCE", + ); + } +} + +/** + * Converts immutable terminal observations into B00A input. The B00A schema + * records only integer usage: cache read/write are included in direct input, + * terminal error/abort output is zero, and every retry attempt is a distinct + * stable assignment. B00A stores exact cost numerators over its documented + * 1,000,000 scale, never binary floating-point money. + */ +export function projectProductionObservations(input: ProductionEvidenceInput): SwarmBenchmarkConfig { + assertInput(input); + return { + scenario: input.scenario, + assignments: input.attempts.map((observation, index) => ({ + nodeId: `attempt-worker-${String(index + 1).padStart(4, "0")}`, + role: "provider-attempt", + requestId: observation.requestId, + attempt: observation.attempt, + attemptId: publicAttemptId(observation), + requested: { + provider: observation.requested.provider === PROVIDER ? PROVIDER : "[REDACTED]", + model: safeModel(observation.requested.model), + // revision/effort remain explicitly present but content-free. + ...(observation.requested.revision === undefined ? {} : { revision: "[REDACTED]" }), + ...(observation.requested.effort === undefined ? {} : { effort: "[REDACTED]" }), + }, + resolved: { + api: observation.resolved.api === PROVIDER ? PROVIDER : "[REDACTED]", + provider: observation.resolved.provider === PROVIDER ? PROVIDER : "[REDACTED]", + model: safeModel(observation.resolved.model), + // responseModel, not selected resolved model, is the attribution authority. + responseModel: safeModel(observation.resolved.responseModel, true), + }, + inputTokens: + observation.usage.inputMicroTokens + + observation.usage.cacheReadMicroTokens + + observation.usage.cacheWriteMicroTokens, + outputTokens: observation.terminal === "done" ? observation.usage.outputMicroTokens : 0, + })), + faultSchedule: input.attempts + .map((observation, index) => + observation.terminal === "done" + ? undefined + : { + nodeId: `attempt-worker-${String(index + 1).padStart(4, "0")}`, + actions: [{ type: "failure" as const, code: "[REDACTED]", message: "[REDACTED]" }], + }, + ) + .filter((value): value is NonNullable => value !== undefined), + priceCard: { + version: input.priceCard.version, + inputPerMillionTokens: input.priceCard.inputMicroCurrencyPerMillionMicroTokens, + outputPerMillionTokens: input.priceCard.outputMicroCurrencyPerMillionMicroTokens, + }, + metadata: input.metadata, + }; +} + +/** Writes B00A artifacts, then signs their commitment outside the artifact root. */ +export async function writeSignedProductionEvidence( + directory: string, + trustDirectory: string, + input: ProductionEvidenceInput, + signer: KeyObject, +): Promise { + const artifactRoot = await realpath(directory).catch(async () => { + await mkdir(directory, { recursive: true, mode: 0o700 }); + return realpath(directory); + }); + await mkdir(trustDirectory, { recursive: true, mode: 0o700 }); + const trustRoot = await realpath(trustDirectory); + assert( + artifactRoot !== trustRoot && + !trustRoot.startsWith(`${artifactRoot}/`) && + !artifactRoot.startsWith(`${trustRoot}/`), + "B00B_EVIDENCE_TRUST_ROOT_OVERLAP", + ); + const evidence = await runSwarmBenchmark(projectProductionObservations(input)); + const writerCapability = await writeSwarmEvidence(artifactRoot, evidence); + // This value is taken from the writer's opaque registration, never manifest.json. + const artifactBundleId = artifactBundleIdForSwarmEvidenceCapability(writerCapability); + const commitment = { + schemaVersion: SWARM_EVIDENCE_COMMITMENT_SCHEMA, + artifactBundleId, + signature: sign( + null, + Buffer.from(canonicalJson(swarmEvidenceCommitmentPayload(artifactBundleId))), + signer, + ).toString("base64"), + }; + const commitmentPath = `${trustRoot}/artifact-commitment.json`; + await writeFile(commitmentPath, `${canonicalJson(commitment)}\n`, { encoding: "utf8", mode: 0o600 }); + return { artifactBundleId, commitmentPath }; +} + +/** + * Fresh-process safe verification. The supplied public key is registered as an + * opaque trust root before the B00B verifier authenticates the commitment. + */ +export async function verifySignedProductionEvidence( + directory: string, + commitmentPath: string, + trustedPublicKeyPem: string, +): Promise { + const commitmentRaw = await readFile(commitmentPath, "utf8"); + const trustRoot = createSwarmEvidenceTrustRoot(trustedPublicKeyPem); + await verifyAuthenticatedSwarmEvidence(directory, commitmentRaw, trustRoot); +} + +/** Runs the authentication-plus-B00A verifier in a clean Node process. */ +export async function verifySignedProductionEvidenceFreshProcess( + directory: string, + commitmentPath: string, + trustedPublicKeyPem: string, +): Promise { + const moduleUrl = new URL("./production-evidence-adapter.ts", import.meta.url).href; + const program = `import { verifySignedProductionEvidence as v } from ${JSON.stringify(moduleUrl)}; await v(process.argv[1], process.argv[2], Buffer.from(process.argv[3], "base64").toString("utf8"));`; + try { + await execFile( + process.execPath, + [ + "--import", + "tsx", + "--input-type=module", + "--eval", + program, + directory, + commitmentPath, + Buffer.from(trustedPublicKeyPem).toString("base64"), + ], + { cwd: process.cwd(), maxBuffer: 256 * 1024 }, + ); + } catch (error) { + const detail = error as { stderr?: string; stdout?: string }; + // Do not forward child output: production fixtures may contain canaries. + throw new Error( + `B00B_EVIDENCE_FRESH_VERIFY_FAILED:${detail.stderr ? "stderr" : detail.stdout ? "stdout" : "exit"}`, + { cause: error }, + ); + } +} diff --git a/packages/coding-agent/test/swarm/production-scripted-provider.ts b/packages/coding-agent/test/swarm/production-scripted-provider.ts new file mode 100644 index 000000000..d77c0a03b --- /dev/null +++ b/packages/coding-agent/test/swarm/production-scripted-provider.ts @@ -0,0 +1,437 @@ +/** + * A test-only provider for production-path swarm tests. + * + * Unlike faux, scripts are selected by the stable request id carried in the + * fixture prompt. There is deliberately no FIFO shared between requests and + * the barrier is an observation latch, never an admission limiter. + */ +import { + type AssistantMessage, + type AssistantMessageEvent, + type Context, + createAssistantMessageEventStream, + type Model, + registerApiProvider, + type SimpleStreamOptions, + type StreamOptions, + type ToolCall, + type Usage, + unregisterApiProviders, +} from "@earendil-works/pi-ai"; + +export interface ScriptedModelDefinition { + readonly id: string; + readonly name?: string; + readonly responseModel?: string; + readonly reasoning?: boolean; + readonly cost?: Model["cost"]; +} + +export type ScriptedBlock = + | { readonly type: "thinking"; readonly chunks: readonly string[] } + | { readonly type: "text"; readonly chunks: readonly string[] } + | { + readonly type: "toolCall"; + readonly id: string; + readonly name: string; + readonly argumentChunks: readonly string[]; + }; + +export interface ProviderScript { + /** A stable logical id, e.g. request-0001. Selection never depends on arrival order. */ + readonly requestId: string; + readonly blocks?: readonly ScriptedBlock[]; + readonly stopReason?: "stop" | "length" | "toolUse"; + readonly responseId?: string; + readonly responseModel?: string; + readonly usage: Usage; + /** A scripted upstream response status. 429 is not manufactured by a client limiter. */ + readonly upstreamStatus?: number; + readonly errorCode?: "upstream-429" | "upstream-error"; + /** First-turn scripts may be held after entry; later tool turns normally are not. */ + readonly waitForRelease?: boolean; +} + +interface MutableProviderObservation { + sequence: number; + requestId: string; + attempt: number; + requested: Readonly<{ + api: string; + provider: string; + model: string; + reasoning?: string; + maxRetries?: number; + }>; + eventKinds: readonly AssistantMessageEvent["type"][]; + upstreamStatus: number; + signalAborted: boolean; + terminal: "done" | "error" | "aborted"; + responseModel?: string; + usage?: Usage; +} + +export interface ProviderObservation { + readonly sequence: number; + readonly requestId: string; + readonly attempt: number; + readonly requested: Readonly<{ + api: string; + provider: string; + model: string; + reasoning?: string; + maxRetries?: number; + }>; + readonly eventKinds: readonly AssistantMessageEvent["type"][]; + readonly upstreamStatus: number; + readonly signalAborted: boolean; + readonly terminal: "done" | "error" | "aborted"; + readonly responseModel?: string; + readonly usage?: Usage; +} + +export interface BarrierScriptedProvider { + readonly models: readonly Model[]; + /** Resolves only after every predeclared, barrier-held request entered exactly once. */ + readonly open: Promise; + release(ids?: readonly string[]): void; + observations(): readonly ProviderObservation[]; + unregister(): void; +} + +export interface CreateBarrierScriptedProviderOptions { + readonly api: string; + readonly provider?: string; + readonly models: readonly ScriptedModelDefinition[]; + /** Each logical id owns its sequence of turns. Arrays are not a cross-request queue. */ + readonly scripts: Readonly>; + readonly barrier: { readonly expected: readonly string[]; readonly timeoutMs?: number }; +} + +const EMPTY_USAGE: Usage = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 0, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, +}; + +function cloneUsage(usage: Usage): Usage { + return structuredClone(usage); +} +function clone(value: T): T { + return structuredClone(value); +} +function requestIdFrom(context: Context): string | undefined { + for (const message of context.messages) { + if (message.role !== "user") continue; + const text = + typeof message.content === "string" + ? message.content + : message.content.map((x) => (x.type === "text" ? x.text : "")).join(" "); + const found = /\brequest-\d{4}\b/.exec(text); + if (found) return found[0]; + } + return undefined; +} +function abortedMessage( + model: Model, + responseModel: string | undefined, + usage = EMPTY_USAGE, +): AssistantMessage { + return { + role: "assistant", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + responseModel, + usage: cloneUsage(usage), + stopReason: "aborted", + errorMessage: "fixture request aborted", + timestamp: Date.now(), + }; +} +function errorMessage( + model: Model, + responseModel: string | undefined, + usage: Usage, + code: string, +): AssistantMessage { + return { + role: "assistant", + content: [], + api: model.api, + provider: model.provider, + model: model.id, + responseModel, + usage: cloneUsage(usage), + stopReason: "error", + errorMessage: code, + timestamp: Date.now(), + }; +} +function assertScript(script: ProviderScript, requestId: string): void { + if (script.requestId !== requestId) throw new Error("B00B_SCRIPT_ID_MISMATCH"); + if (!/^request-\d{4}$/.test(requestId)) throw new Error("B00B_BAD_REQUEST_ID"); +} + +/** Abort-aware gate. It owns one promise per request, so one abort cannot release a sibling. */ +export function createBarrier(expected: readonly string[], timeoutMs: number) { + const expectedSet = new Set(expected); + if (expectedSet.size !== expected.length || expected.some((id) => !/^request-\d{4}$/.test(id))) { + throw new Error("B00B_BAD_BARRIER_EXPECTED"); + } + let resolveOpen!: () => void; + let rejectOpen!: (error: Error) => void; + let openSettled = false; + let closed = false; + const open = new Promise((resolve, reject) => { + resolveOpen = resolve; + rejectOpen = reject; + }); + const entered = new Set(); + const released = new Set(); + const waiters = new Map void>(); + const abortPendingWaiters = () => { + for (const waiter of waiters.values()) waiter("aborted"); + waiters.clear(); + }; + const rejectPendingOpen = (error: Error) => { + if (openSettled) return; + openSettled = true; + rejectOpen(error); + }; + const timer = setTimeout(() => { + if (closed) return; + closed = true; + rejectPendingOpen(new Error("B00B_BARRIER_TIMEOUT")); + abortPendingWaiters(); + }, timeoutMs); + const enteredRequest = (id: string) => { + if (!expectedSet.has(id)) return; + if (entered.has(id)) throw new Error("B00B_BARRIER_DUPLICATE"); + entered.add(id); + if (entered.size === expectedSet.size && !openSettled) { + openSettled = true; + clearTimeout(timer); + resolveOpen(); + } + }; + const wait = (id: string, signal: AbortSignal | undefined) => + new Promise<"released" | "aborted">((resolve) => { + let done = false; + let onAbort: (() => void) | undefined; + const settle = (result: "released" | "aborted") => { + if (done) return; + done = true; + if (onAbort) signal?.removeEventListener("abort", onAbort); + waiters.delete(id); + resolve(result); + }; + onAbort = () => settle("aborted"); + if (closed || signal?.aborted) return settle("aborted"); + if (released.has(id)) return settle("released"); + waiters.set(id, settle); + signal?.addEventListener("abort", onAbort, { once: true }); + // The abort may race listener registration in an implementation-specific host. + if (signal?.aborted) settle("aborted"); + }); + return { + open, + entered: enteredRequest, + wait, + release(ids?: readonly string[]) { + for (const id of ids ?? expected) { + released.add(id); + waiters.get(id)?.("released"); + } + }, + close() { + if (closed) return; + closed = true; + clearTimeout(timer); + rejectPendingOpen(new Error("B00B_BARRIER_CLOSED")); + abortPendingWaiters(); + }, + }; +} + +/** + * Registers the actual @earendil-works/pi-ai API provider seam. The returned + * provider is test-only; no product source imports it. + */ +export function createBarrierScriptedProvider(options: CreateBarrierScriptedProviderOptions): BarrierScriptedProvider { + const provider = options.provider ?? "b00b-scripted"; + const sourceId = `b00b-scripted:${options.api}:${Date.now()}:${Math.random().toString(36).slice(2)}`; + const barrier = createBarrier(options.barrier.expected, options.barrier.timeoutMs ?? 5_000); + const models = options.models.map( + (definition) => + ({ + id: definition.id, + name: definition.name ?? definition.id, + api: options.api, + provider, + baseUrl: "http://127.0.0.1:0", + reasoning: definition.reasoning ?? true, + input: ["text"] as ("text" | "image")[], + cost: definition.cost ?? { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 128_000, + maxTokens: 16_384, + }) satisfies Model, + ); + if (!models.length) throw new Error("B00B_NO_MODELS"); + const attempts = new Map(); + const recorded: MutableProviderObservation[] = []; + let sequence = 0; + let unregistered = false; + + const stream = (model: Model, context: Context, streamOptions?: StreamOptions | SimpleStreamOptions) => { + const output = createAssistantMessageEventStream(); + const requestId = requestIdFrom(context); + if (!requestId) throw new Error("B00B_MISSING_REQUEST_ID"); + const attempt = attempts.get(requestId) ?? 0; + attempts.set(requestId, attempt + 1); + const script = options.scripts[requestId]?.[attempt]; + if (!script) throw new Error("B00B_UNSCRIPTED_ATTEMPT"); + assertScript(script, requestId); + const observation: MutableProviderObservation = { + sequence: ++sequence, + requestId, + attempt: attempt + 1, + requested: { + api: model.api, + provider: model.provider, + model: model.id, + reasoning: (streamOptions as SimpleStreamOptions | undefined)?.reasoning, + maxRetries: streamOptions?.maxRetries, + }, + eventKinds: [], + upstreamStatus: script.upstreamStatus ?? 200, + signalAborted: false, + terminal: "error", + }; + recorded.push(observation); + queueMicrotask(async () => { + const emit = (event: AssistantMessageEvent) => { + observation.eventKinds = [...observation.eventKinds, event.type]; + output.push(event); + }; + const terminalAbort = () => { + observation.signalAborted = true; + observation.terminal = "aborted"; + const message = abortedMessage(model, script.responseModel, script.usage); + emit({ type: "error", reason: "aborted", error: message }); + output.end(message); + }; + try { + // Only the first provider entry belongs to the fanout observation latch; tool turns remain independent. + if (attempt === 0) barrier.entered(requestId); + await streamOptions?.onResponse?.({ status: script.upstreamStatus ?? 200, headers: {} }, model); + if (streamOptions?.signal?.aborted) return terminalAbort(); + if (script.waitForRelease) { + if ((await barrier.wait(requestId, streamOptions?.signal)) === "aborted") return terminalAbort(); + } + if (streamOptions?.signal?.aborted) return terminalAbort(); + if ((script.upstreamStatus ?? 200) >= 400 || script.errorCode) { + observation.terminal = "error"; + const message = errorMessage( + model, + script.responseModel, + script.usage, + script.errorCode ?? "upstream-error", + ); + emit({ type: "error", reason: "error", error: message }); + output.end(message); + return; + } + const content: AssistantMessage["content"] = []; + const partial = (): AssistantMessage => ({ + role: "assistant", + content: clone(content), + api: model.api, + provider: model.provider, + model: model.id, + responseModel: script.responseModel, + usage: cloneUsage(script.usage), + stopReason: script.stopReason ?? "stop", + responseId: script.responseId, + timestamp: Date.now(), + }); + emit({ type: "start", partial: partial() }); + for (const block of script.blocks ?? []) { + if (streamOptions?.signal?.aborted) return terminalAbort(); + const contentIndex = content.length; + if (block.type === "thinking") { + content.push({ type: "thinking", thinking: "" }); + emit({ type: "thinking_start", contentIndex, partial: partial() }); + for (const delta of block.chunks) { + if (streamOptions?.signal?.aborted) return terminalAbort(); + (content[contentIndex] as { thinking: string }).thinking += delta; + emit({ type: "thinking_delta", contentIndex, delta, partial: partial() }); + } + emit({ + type: "thinking_end", + contentIndex, + content: (content[contentIndex] as { thinking: string }).thinking, + partial: partial(), + }); + } else if (block.type === "text") { + content.push({ type: "text", text: "" }); + emit({ type: "text_start", contentIndex, partial: partial() }); + for (const delta of block.chunks) { + if (streamOptions?.signal?.aborted) return terminalAbort(); + (content[contentIndex] as { text: string }).text += delta; + emit({ type: "text_delta", contentIndex, delta, partial: partial() }); + } + emit({ + type: "text_end", + contentIndex, + content: (content[contentIndex] as { text: string }).text, + partial: partial(), + }); + } else { + content.push({ type: "toolCall", id: block.id, name: block.name, arguments: {} }); + emit({ type: "toolcall_start", contentIndex, partial: partial() }); + for (const delta of block.argumentChunks) { + if (streamOptions?.signal?.aborted) return terminalAbort(); + emit({ type: "toolcall_delta", contentIndex, delta, partial: partial() }); + } + const joined = block.argumentChunks.join(""); + const toolCall = content[contentIndex] as ToolCall; + toolCall.arguments = JSON.parse(joined || "{}"); + emit({ type: "toolcall_end", contentIndex, toolCall: clone(toolCall), partial: partial() }); + } + } + const message = partial(); + observation.terminal = "done"; + observation.responseModel = message.responseModel; + observation.usage = cloneUsage(message.usage); + emit({ type: "done", reason: message.stopReason as "stop" | "length" | "toolUse", message }); + output.end(message); + } catch { + if (streamOptions?.signal?.aborted) return terminalAbort(); + observation.terminal = "error"; + const message = errorMessage(model, script.responseModel, script.usage, "script-provider-failure"); + emit({ type: "error", reason: "error", error: message }); + output.end(message); + } + }); + return output; + }; + registerApiProvider({ api: options.api, stream, streamSimple: stream }, sourceId); + return { + models, + open: barrier.open, + release: (ids) => barrier.release(ids), + observations: () => recorded.map((item) => clone(item)), + unregister() { + if (!unregistered) { + unregistered = true; + barrier.close(); + unregisterApiProviders(sourceId); + } + }, + }; +} diff --git a/packages/coding-agent/test/swarm/rss-campaign-cadence.ts b/packages/coding-agent/test/swarm/rss-campaign-cadence.ts new file mode 100644 index 000000000..d47736d0a --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-campaign-cadence.ts @@ -0,0 +1,15 @@ +export const MAX_RSS_SAMPLE_GAP_MS = 50; +export const DEFAULT_RSS_REQUESTED_PERIOD_MS = 25; + +export interface CadenceValidation { + maxObservedGapMs: number | null; + valid: boolean; +} + +/** Validates the unmodified monotonic timestamps emitted after each collection. */ +export function validateRssSampleCadence(timestamps: readonly number[]): CadenceValidation { + if (timestamps.length < 2) return { maxObservedGapMs: null, valid: false }; + const gaps = timestamps.slice(1).map((timestamp, index) => timestamp - timestamps[index]!); + const maxObservedGapMs = Math.max(...gaps); + return { maxObservedGapMs, valid: maxObservedGapMs <= MAX_RSS_SAMPLE_GAP_MS }; +} diff --git a/packages/coding-agent/test/swarm/rss-campaign-worker.ts b/packages/coding-agent/test/swarm/rss-campaign-worker.ts new file mode 100644 index 000000000..668d13ed1 --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-campaign-worker.ts @@ -0,0 +1,173 @@ +/** + * Disposable, test-only child supervisor for the B00B RSS campaign. + * It deliberately has no provider imports, network client, daemon listener, or + * persistent state. The parent owns this process group and measures it. + */ +import { type ChildProcess, spawn } from "node:child_process"; +import { mkdir, mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +interface WorkerOptions { + fanout: number; + allocationMiB: number; + scratch: string; + fixtureCommand?: string; + fixtureArgs: readonly string[]; + testIgnoreTerm: boolean; +} + +type WorkerMessage = + | { + type: "boundary"; + phase: "started" | "barrier-held" | "terminals" | "cleanup"; + allocatedBytes: number; + memberPids: readonly number[]; + } + | { type: "result"; completed: number; failed: number; allocatedBytes: number }; + +function option(name: string): string | undefined { + const index = process.argv.indexOf(name); + return index === -1 ? undefined : process.argv[index + 1]; +} + +function positiveInteger(name: string, fallback?: number): number { + const value = option(name) ?? (fallback === undefined ? undefined : String(fallback)); + const parsed = value === undefined ? Number.NaN : Number(value); + if (!Number.isSafeInteger(parsed) || parsed < 1) throw new Error(`invalid_${name.slice(2)}`); + return parsed; +} + +function options(): WorkerOptions { + const fanout = positiveInteger("--fanout"); + const allocationMiB = positiveInteger("--allocation-mib", 1); + const scratch = option("--scratch") ?? join(tmpdir(), "b00b-rss"); + const fixtureCommand = option("--fixture-command"); + const fixtureArgs: string[] = []; + for (let index = 0; index < process.argv.length; index += 1) { + if (process.argv[index] === "--fixture-arg") { + const value = process.argv[index + 1]; + if (value === undefined) throw new Error("invalid_fixture_arg"); + fixtureArgs.push(value); + index += 1; + } + } + return { + fanout, + allocationMiB, + scratch, + fixtureCommand, + fixtureArgs, + testIgnoreTerm: process.argv.includes("--test-ignore-term"), + }; +} + +function safeEnvironment(worker: number, fanout: number, allocationBytes: number): NodeJS.ProcessEnv { + const inherited = process.env; + const environment: NodeJS.ProcessEnv = { + B00B_WORKER_INDEX: String(worker), + B00B_FANOUT: String(fanout), + B00B_FIXTURE_ALLOCATION_BYTES: String(allocationBytes), + LANG: "C", + LC_ALL: "C", + }; + for (const key of ["PATH", "HOME", "TMPDIR", "TMP", "TEMP", "SystemRoot", "ComSpec"]) { + if (inherited[key]) environment[key] = inherited[key]; + } + return environment; +} + +const BUILTIN_FIXTURE = [ + "const bytes=Number(process.env.B00B_FIXTURE_ALLOCATION_BYTES||0);", + "const b=Buffer.allocUnsafe(bytes);for(let i=0;iprocess.exit(0),50);", +].join(""); + +interface Fixture { + pid: number; + exit: Promise; +} + +function launchFixture(config: WorkerOptions, worker: number, allocationBytes: number): Fixture | undefined { + const command = config.fixtureCommand ?? process.execPath; + const args = config.fixtureCommand ? [...config.fixtureArgs] : ["-e", BUILTIN_FIXTURE]; + try { + const child: ChildProcess = spawn(command, args, { + cwd: process.cwd(), + detached: false, + env: safeEnvironment(worker, config.fanout, allocationBytes), + stdio: "ignore", + }); + if (!child.pid) return undefined; + return { + pid: child.pid, + exit: new Promise((resolve) => { + child.once("error", () => resolve(false)); + child.once("exit", (code, signal) => resolve(code === 0 && signal === null)); + }), + }; + } catch { + return undefined; + } +} + +function send(message: WorkerMessage): void { + process.send?.(message); +} + +function pause(milliseconds: number): Promise { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +function awaitRelease(): Promise { + return new Promise((resolve) => { + process.once("message", (message: unknown) => { + if ((message as { type?: unknown })?.type === "release") resolve(); + }); + }); +} + +async function main(): Promise { + const config = options(); + // No allocation, fixture, or descendant may exist before the parent has + // authenticated this leader's PID/start/PGID and explicitly releases us. + await awaitRelease(); + if (config.testIgnoreTerm) process.on("SIGTERM", () => {}); + const allocationBytes = config.allocationMiB * 1024 * 1024; + let allocation = Buffer.allocUnsafe(allocationBytes); + for (let index = 0; index < allocation.length; index += 4096) allocation[index] = 1; + const runtimeRoot = await mkdtemp(join(config.scratch, "b00b-rss-")); + try { + await Promise.all([ + mkdir(join(runtimeRoot, "agent")), + mkdir(join(runtimeRoot, "socket")), + mkdir(join(runtimeRoot, "output")), + ]); + const fixtures = Array.from({ length: config.fanout }, (_, index) => + launchFixture(config, index + 1, allocationBytes), + ); + const memberPids = fixtures.flatMap((fixture) => (fixture ? [fixture.pid] : [])); + send({ type: "boundary", phase: "started", allocatedBytes: allocationBytes, memberPids }); + // Every fixture is dispatched before this observation boundary. It is never + // a permit, queue, semaphore, or admission limiter. + send({ + type: "boundary", + phase: "barrier-held", + allocatedBytes: allocationBytes * (config.fanout + 1), + memberPids, + }); + const results = await Promise.all(fixtures.map((fixture) => fixture?.exit ?? Promise.resolve(false))); + const completed = results.filter(Boolean).length; + send({ type: "boundary", phase: "terminals", allocatedBytes: allocationBytes * (config.fanout + 1), memberPids }); + await pause(100); + allocation = Buffer.alloc(0); + global.gc?.(); + await rm(runtimeRoot, { force: true, recursive: true }); + send({ type: "boundary", phase: "cleanup", allocatedBytes: 0, memberPids }); + send({ type: "result", completed, failed: config.fanout - completed, allocatedBytes: 0 }); + } finally { + await rm(runtimeRoot, { force: true, recursive: true }); + } +} + +await main(); diff --git a/packages/coding-agent/test/swarm/rss-campaign.test.ts b/packages/coding-agent/test/swarm/rss-campaign.test.ts new file mode 100644 index 000000000..49a16a865 --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-campaign.test.ts @@ -0,0 +1,265 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, it } from "vitest"; +import { validateRssSampleCadence } from "./rss-campaign-cadence.js"; +import { childExecArgsWithTsxImport } from "./rss-child-exec-args.js"; + +const execute = promisify(execFile); +const launcher = fileURLToPath(new URL("./run-production-rss-campaign.ts", import.meta.url)); +const tsx = fileURLToPath(new URL("../../../../node_modules/tsx/dist/cli.mjs", import.meta.url)); +const temporary: string[] = []; + +afterEach(async () => { + await Promise.all(temporary.splice(0).map((path) => rm(path, { force: true, recursive: true }))); +}); + +async function directory(label: string): Promise { + const path = await mkdtemp(join(tmpdir(), `b00b-rss-${label}-`)); + temporary.push(path); + return path; +} + +async function campaign(output: string, args: readonly string[]): Promise { + await execute(process.execPath, [tsx, launcher, "--output", output, ...args], { + cwd: fileURLToPath(new URL("../../../../", import.meta.url)), + timeout: 30_000, + }); +} + +async function run(output: string, fanout: number, repetition: number): Promise> { + return JSON.parse(await readFile(join(output, `run-${fanout}-${repetition}-1.json`), "utf8")) as Record< + string, + unknown + >; +} + +describe("B00B RSS campaign", () => { + it("preloads tsx for a TypeScript worker exactly once unless a loader is already selected", () => { + expect(childExecArgsWithTsxImport([])).toEqual(["--import", "tsx"]); + expect(childExecArgsWithTsxImport(["--trace-warnings"])).toEqual(["--trace-warnings", "--import", "tsx"]); + expect(childExecArgsWithTsxImport(["--import", "tsx"])).toEqual(["--import", "tsx"]); + expect(childExecArgsWithTsxImport(["--import=tsx"])).toEqual(["--import=tsx"]); + expect(childExecArgsWithTsxImport(["--loader", "custom-ts-loader"])).toEqual(["--loader", "custom-ts-loader"]); + }); + it("writes complete structured dry artifacts rather than zero-looking macOS data", async () => { + const output = join(await directory("dry"), "output"); + await campaign(output, ["--fanout", "1", "--repetitions", "2"]); + const first = await run(output, 1, 1); + const second = await run(output, 1, 2); + if (process.platform === "darwin") { + expect(first.status).toBe("unsupported"); + expect(second.status).toBe("unsupported"); + expect(first.sampler).toBeNull(); + expect(first.finalRssKiB).toBeNull(); + } + const mode = (await stat(join(output, "manifest.json"))).mode & 0o777; + expect(mode).toBe(0o600); + }); + + it.skipIf(process.platform !== "linux")( + "reaps a SIGTERM-ignoring descendant after its group leader exits", + async () => { + const root = await directory("reap"); + const output = join(root, "output"); + const pidFile = join(root, "fixture.pid"); + const fixture = join(root, "ignore-term.cjs"); + await writeFile( + fixture, + "require('fs').writeFileSync(process.argv[2], String(process.pid));process.on('SIGTERM',()=>{});setInterval(()=>{},1000);", + { mode: 0o700 }, + ); + await campaign(output, [ + "--fanout", + "1", + "--repetitions", + "1", + "--timeout-ms", + "500", + "--fixture-command", + process.execPath, + "--fixture-arg", + fixture, + "--fixture-arg", + pidFile, + ]); + const pid = Number(await readFile(pidFile, "utf8")); + const result = await run(output, 1, 1); + expect(result.status).toBe("timed_out"); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(() => process.kill(pid, 0)).toThrow(); + }, + ); + + it.skipIf(process.platform !== "linux")( + "fails closed when the final scan is unavailable after a TERM-ignoring descendant", + async () => { + const root = await directory("final-scan-failure"); + const output = join(root, "output"); + const pidFile = join(root, "fixture.pid"); + const fixture = join(root, "ignore-term.cjs"); + await writeFile( + fixture, + "require('fs').writeFileSync(process.argv[2],String(process.pid));process.on('SIGTERM',()=>{});setInterval(()=>{},1000);", + { mode: 0o700 }, + ); + await campaign(output, [ + "--fanout", + "1", + "--repetitions", + "1", + "--timeout-ms", + "500", + "--test-fail-final-scan", + "--fixture-command", + process.execPath, + "--fixture-arg", + fixture, + "--fixture-arg", + pidFile, + ]); + const pid = Number(await readFile(pidFile, "utf8")); + const result = await run(output, 1, 1); + expect(result.status).toBe("failed"); + expect(result.reasonCode).toBe(5); + expect(result.finalRssKiB).toBeNull(); + expect((result.samples as { phase: string }[]).some((sample) => sample.phase === "final")).toBe(false); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(() => process.kill(pid, 0)).toThrow(); + }, + ); + + it.skipIf(process.platform !== "linux")( + "retries one unavailable final scan and still requires a positive empty final snapshot", + async () => { + const output = join(await directory("final-scan-retry"), "output"); + await campaign(output, ["--fanout", "1", "--repetitions", "1", "--test-fail-final-scan-once"]); + const result = await run(output, 1, 1); + expect(result.status).toBe("complete"); + expect(result.reasonCode).toBeNull(); + expect(result.finalRssKiB).toBe(0); + expect((result.samples as { phase: string; totalRssKiB: number }[]).at(-1)).toMatchObject({ + phase: "final", + totalRssKiB: 0, + }); + }, + ); + + it.skipIf(process.platform !== "linux")( + "does not arm timeout or release descendants before exact ownership capture", + async () => { + const root = await directory("delayed-ownership"); + const output = join(root, "output"); + const pidFile = join(root, "fixture.pid"); + const fixture = join(root, "ignore-term.cjs"); + await writeFile( + fixture, + "require('fs').writeFileSync(process.argv[2],String(process.pid));process.on('SIGTERM',()=>{});setInterval(()=>{},1000);", + { mode: 0o700 }, + ); + await campaign(output, [ + "--fanout", + "1", + "--repetitions", + "1", + "--timeout-ms", + "250", + "--test-identity-capture-delay-ms", + "500", + "--test-ignore-term", + "--fixture-command", + process.execPath, + "--fixture-arg", + fixture, + "--fixture-arg", + pidFile, + ]); + const pid = Number(await readFile(pidFile, "utf8")); + const result = await run(output, 1, 1); + expect(result.status).toBe("timed_out"); + expect(result.timedOut).toBe(true); + expect(result.finalRssKiB).toBe(0); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(() => process.kill(pid, 0)).toThrow(); + }, + ); + + it("validates unchanged sample timestamps at the 50 ms max-gap contract", () => { + expect(validateRssSampleCadence([0, 50, 100]).valid).toBe(true); + expect(validateRssSampleCadence([0, 25, 76])).toEqual({ maxObservedGapMs: 51, valid: false }); + }); + + it.skipIf(process.platform !== "linux")( + "uses 25 ms jitter headroom while recording the 50 ms cadence contract", + async () => { + const output = join(await directory("cadence"), "output"); + await campaign(output, ["--fanout", "64", "--repetitions", "1"]); + const result = await run(output, 64, 1); + const sampler = result.sampler as { + requestedPeriodMs: number; + maxGapMs: number; + maxObservedGapMs: number | null; + }; + expect(sampler.requestedPeriodMs).toBe(25); + expect(sampler.maxGapMs).toBe(50); + if (result.status === "complete") expect(sampler.maxObservedGapMs).toBeLessThanOrEqual(50); + else expect(result.reasonCode).toBe(4); + }, + ); + + it.skipIf(process.platform !== "linux")( + "fails rather than relabeling samples when injected scheduler jitter exceeds 50 ms", + async () => { + const output = join(await directory("cadence-jitter"), "output"); + await campaign(output, [ + "--fanout", + "1", + "--repetitions", + "1", + "--test-scheduler-jitter-ms", + "26", + "--fixture-command", + process.execPath, + "--fixture-arg", + "-e", + "--fixture-arg", + "setTimeout(()=>process.exit(0),150)", + ]); + const result = await run(output, 1, 1); + expect(result.status).toBe("failed"); + expect(result.reasonCode).toBe(4); + expect((result.sampler as { requestedPeriodMs: number }).requestedPeriodMs).toBe(51); + expect((result.sampler as { maxObservedGapMs: number }).maxObservedGapMs).toBeGreaterThan(50); + }, + ); + + it("never archives a fixture secret, command, or argument", async () => { + const root = await directory("secret"); + const output = join(root, "output"); + const secret = "B00B_RSS_SECRET_4f85d5c7"; + const fixture = join(root, "secret-fixture.cjs"); + await writeFile(fixture, "setTimeout(()=>process.exit(0),10)"); + await campaign(output, [ + "--fanout", + "1", + "--repetitions", + "1", + "--fixture-command", + process.execPath, + "--fixture-arg", + fixture, + "--fixture-arg", + secret, + ]); + const artifact = await Promise.all( + ["run-1-0-0.json", "run-1-1-1.json", "manifest.json"].map((name) => readFile(join(output, name), "utf8")), + ); + for (const content of artifact) { + expect(content).not.toContain(secret); + expect(content).not.toContain(fixture); + } + }); +}); diff --git a/packages/coding-agent/test/swarm/rss-child-exec-args.ts b/packages/coding-agent/test/swarm/rss-child-exec-args.ts new file mode 100644 index 000000000..5c4760100 --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-child-exec-args.ts @@ -0,0 +1,15 @@ +/** + * Preserve a parent TypeScript runtime when present. Otherwise use tsx's Node + * preload so the disposable child can execute the .ts worker directly. + */ +export function childExecArgsWithTsxImport(execArgs: readonly string[]): string[] { + for (let index = 0; index < execArgs.length; index += 1) { + const argument = execArgs[index]!; + if (argument === "--import" && execArgs[index + 1] === "tsx") return [...execArgs]; + if (argument === "--import=tsx") return [...execArgs]; + // An explicit Node loader owns module loading for this child; do not + // stack tsx on top of a caller-selected TypeScript loader. + if (argument === "--loader" || argument.startsWith("--loader=")) return [...execArgs]; + } + return [...execArgs, "--import", "tsx"]; +} diff --git a/packages/coding-agent/test/swarm/rss-proc.test.ts b/packages/coding-agent/test/swarm/rss-proc.test.ts new file mode 100644 index 000000000..e3ebb0e0c --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-proc.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, it } from "vitest"; +import { hasStableProcessIdentity, parseProcessStat, processRecordFromStatus } from "./rss-proc.js"; + +function stat(state: string, ppid = 17, pgid = 23, start = 456): string { + const fields = Array.from({ length: 20 }, () => "0"); + fields[0] = state; + fields[1] = String(ppid); + fields[2] = String(pgid); + fields[19] = String(start); + return `123 (worker name) ${fields.join(" ")}`; +} + +function processStat(state: string, ppid = 17, pgid = 23, start = 456, pid = 123) { + const parsed = parseProcessStat(pid, stat(state, ppid, pgid, start)); + expect(parsed).toEqual({ pid, ppid, pgid, start, state }); + return parsed!; +} + +function recordFromStatus(initial: ReturnType, status: string, confirmation = initial) { + return processRecordFromStatus(initial, status, confirmation); +} + +describe("Linux proc RSS records", () => { + it.each(["S", "R"])("keeps a %s process with no mm at zero RSS", (state) => { + const initial = processStat(state); + const record = recordFromStatus(initial, `Name:\tworker\nState:\t${state} (running)\n`); + expect(record).toEqual({ pid: 123, ppid: 17, pgid: 23, start: 456, rssKiB: 0 }); + expect(record).not.toHaveProperty("state"); + }); + + it("accepts a legitimate R-to-S state change after confirming stable external identity", () => { + const initial = processStat("R", 17, 23, 456); + const confirmation = processStat("S", 99, 23, 456); + expect(recordFromStatus(initial, "Name:\tworker\nState:\tS (sleeping)\n", confirmation)).toEqual({ + pid: 123, + ppid: 17, + pgid: 23, + start: 456, + rssKiB: 0, + }); + }); + + it("keeps a zombie without an mm at zero RSS", () => { + expect(recordFromStatus(processStat("Z"), "Name:\tworker\nState:\tZ (zombie)\n")).toMatchObject({ + rssKiB: 0, + }); + }); + + it.each([ + ["missing", "Name:\tworker\n"], + ["malformed", "Name:\tworker\nState:\tS not-a-linux-state\n"], + ["invalid Linux state", "Name:\tworker\nState:\tQ (not a task state)\n"], + ])("fails closed for %s status state", (_case, status) => { + expect(recordFromStatus(processStat("S"), status)).toBeUndefined(); + }); + + it.each([ + ["PID", processStat("S", 17, 23, 456, 124)], + ["start time", processStat("S", 17, 23, 457)], + ["process group", processStat("S", 17, 24, 456)], + ])("fails the scan when the confirmation has a mismatched %s", (_case, confirmation) => { + const initial = processStat("S"); + expect(hasStableProcessIdentity(initial, confirmation)).toBe(false); + expect(recordFromStatus(initial, "Name:\tworker\nState:\tS (sleeping)\n", confirmation)).toBeUndefined(); + }); + + it("fails closed when an mm field exists but VmRSS is absent", () => { + const status = "Name:\tworker\nState:\tS (sleeping)\nVmSize:\t1024 kB\n"; + expect(recordFromStatus(processStat("S"), status)).toBeUndefined(); + }); + + it("parses VmRSS from a status file with stable external identity", () => { + const status = "Name:\tworker\nState:\tS (sleeping)\nVmSize:\t1024 kB\nVmRSS:\t512 kB\n"; + expect(recordFromStatus(processStat("S"), status)).toMatchObject({ rssKiB: 512 }); + }); +}); diff --git a/packages/coding-agent/test/swarm/rss-proc.ts b/packages/coding-agent/test/swarm/rss-proc.ts new file mode 100644 index 000000000..7cbe524b5 --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-proc.ts @@ -0,0 +1,78 @@ +export interface ProcessIdentity { + pid: number; + ppid: number; + pgid: number; + start: number; +} + +export interface ProcessStat extends ProcessIdentity { + state: string; +} + +// Linux task-state letters emitted by /proc/PID/status. +const LINUX_PROCESS_STATES = new Set(["R", "S", "D", "Z", "T", "t", "W", "X", "x", "K", "P", "I"]); + +/** + * Confirms that two reads identify the same process-group member. State and + * parent PID are deliberately transient and therefore are not identity. + */ +export function hasStableProcessIdentity(initial: ProcessStat, confirmation: ProcessStat): boolean { + return ( + initial.pid === confirmation.pid && initial.start === confirmation.start && initial.pgid === confirmation.pgid + ); +} + +/** The persisted artifact shape deliberately excludes transient procfs state. */ +export interface ProcessRecord extends ProcessIdentity { + rssKiB: number; +} + +/** + * Parses Linux /proc/PID/stat after the parenthesized comm field, which can + * itself contain spaces and closing parentheses. + */ +export function parseProcessStat(pid: number, statLine: string): ProcessStat | undefined { + const close = statLine.lastIndexOf(")"); + if (close < 0) return undefined; + const fields = statLine + .slice(close + 1) + .trim() + .split(/\s+/); + const state = fields[0]; // field 3 + const ppid = Number(fields[1]); // field 4 + const pgid = Number(fields[2]); // field 5 + const start = Number(fields[19]); // field 22 + if (!state || state.length !== 1 || ![ppid, pgid, start].every(Number.isSafeInteger)) return undefined; + return { pid, ppid, pgid, start, state }; +} + +/** + * A process without an mm has no Vm* fields. Linux exposes this both for + * zombies and briefly for fork/exec children, which are conservatively kept + * as zero-RSS records after an external stat reread confirms the identity. + * State is only a status-file integrity check: it may change between reads. + */ +export function processRecordFromStatus( + stat: ProcessStat, + status: string, + confirmation: ProcessStat, +): ProcessRecord | undefined { + if (!hasStableProcessIdentity(stat, confirmation)) return undefined; + + const lines = status.split(/\r?\n/); + const stateLines = lines.filter((line) => line.startsWith("State:")); + const state = stateLines.length === 1 ? /^State:\s+(\S)(?:\s+\([^()]*\))?\s*$/.exec(stateLines[0])?.[1] : undefined; + if (state === undefined || !LINUX_PROCESS_STATES.has(state)) return undefined; + + const vmLines = lines.filter((line) => line.startsWith("Vm")); + const rssLines = vmLines.filter((line) => line.startsWith("VmRSS:")); + if (rssLines.length === 0) { + return vmLines.length === 0 + ? { pid: stat.pid, ppid: stat.ppid, pgid: stat.pgid, start: stat.start, rssKiB: 0 } + : undefined; + } + if (rssLines.length !== 1) return undefined; + const rss = /^VmRSS:\s+(\d+)\s+kB\s*$/.exec(rssLines[0])?.[1]; + if (rss === undefined) return undefined; + return { pid: stat.pid, ppid: stat.ppid, pgid: stat.pgid, start: stat.start, rssKiB: Number(rss) }; +} diff --git a/packages/coding-agent/test/swarm/rss-snapshot-retry.test.ts b/packages/coding-agent/test/swarm/rss-snapshot-retry.test.ts new file mode 100644 index 000000000..96b9319c2 --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-snapshot-retry.test.ts @@ -0,0 +1,60 @@ +import { describe, expect, it } from "vitest"; +import { retryUnavailableSnapshot } from "./rss-snapshot-retry.js"; + +interface FakeClock { + time: number; + pauses: number[]; + now(): number; + pause(milliseconds: number): Promise; +} + +function clock(): FakeClock { + return { + time: 0, + pauses: [], + now() { + return this.time; + }, + async pause(milliseconds) { + this.pauses.push(milliseconds); + this.time += milliseconds; + }, + }; +} + +describe("RSS proc snapshot retry window", () => { + it("accepts a coherent snapshot after a deterministic transient sequence beyond three attempts", async () => { + const fake = clock(); + let attempts = 0; + const result = await retryUnavailableSnapshot( + async () => (attempts++ < 8 ? "unavailable" : "coherent"), + (snapshot) => snapshot === "unavailable", + fake, + 20, + 2, + ); + expect(result).toBe("coherent"); + expect(attempts).toBe(9); + expect(fake.pauses).toEqual([2, 2, 2, 2, 2, 2, 2, 2]); + expect(fake.time).toBe(16); + }); + + it("returns persistent unavailability at the monotonic deadline without inventing an empty snapshot", async () => { + const fake = clock(); + let attempts = 0; + const result = await retryUnavailableSnapshot( + async () => { + attempts += 1; + return "unavailable"; + }, + (snapshot) => snapshot === "unavailable", + fake, + 20, + 2, + ); + expect(result).toBe("unavailable"); + expect(attempts).toBe(10); + expect(fake.pauses).toHaveLength(10); + expect(fake.time).toBe(20); + }); +}); diff --git a/packages/coding-agent/test/swarm/rss-snapshot-retry.ts b/packages/coding-agent/test/swarm/rss-snapshot-retry.ts new file mode 100644 index 000000000..16eae07c5 --- /dev/null +++ b/packages/coding-agent/test/swarm/rss-snapshot-retry.ts @@ -0,0 +1,36 @@ +/** The retry budget is deliberately shorter than the 50 ms sample-gap contract. */ +export const RSS_SCAN_RETRY_WINDOW_MS = 20; +export const RSS_SCAN_RETRY_DELAY_MS = 2; + +export interface RetryClock { + now(): number; + pause(milliseconds: number): Promise; +} + +/** + * Retries only unavailable snapshots within one monotonic convergence window. + * A returned value is accepted only if the collector itself declared it + * coherent; this helper never converts an unavailable result into an empty one. + */ +export async function retryUnavailableSnapshot( + collect: (attempt: number) => Promise, + isUnavailable: (snapshot: T) => boolean, + clock: RetryClock, + windowMs = RSS_SCAN_RETRY_WINDOW_MS, + delayMs = RSS_SCAN_RETRY_DELAY_MS, +): Promise { + const deadline = clock.now() + windowMs; + let attempt = 0; + let lastUnavailable: T | undefined; + for (;;) { + // Do not begin a further full scan once the convergence budget expired. + if (attempt > 0 && clock.now() >= deadline) return lastUnavailable!; + const snapshot = await collect(attempt); + attempt += 1; + if (!isUnavailable(snapshot)) return snapshot; + lastUnavailable = snapshot; + const remaining = deadline - clock.now(); + if (remaining <= 0) return snapshot; + await clock.pause(Math.min(delayMs, remaining)); + } +} diff --git a/packages/coding-agent/test/swarm/run-production-rss-campaign.ts b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts new file mode 100644 index 000000000..6e38929b1 --- /dev/null +++ b/packages/coding-agent/test/swarm/run-production-rss-campaign.ts @@ -0,0 +1,743 @@ +/** + * Fresh-process, test-only RSS campaign launcher for PR-B00B. + * + * It has no product import, provider credential, network client, or resident + * daemon. Every measured cell owns a newly spawned Unix process group. A later + * real-provider fixture can be supplied with --fixture-command; its command, + * arguments, stdout, stderr, and environment are deliberately not archived. + */ +import { type ChildProcess, spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { chmod, mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; +import { cpus, platform, release, totalmem } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + DEFAULT_RSS_REQUESTED_PERIOD_MS, + MAX_RSS_SAMPLE_GAP_MS, + validateRssSampleCadence, +} from "./rss-campaign-cadence.js"; +import { childExecArgsWithTsxImport } from "./rss-child-exec-args.js"; +import { type ProcessRecord, type ProcessStat, parseProcessStat, processRecordFromStatus } from "./rss-proc.js"; +import { RSS_SCAN_RETRY_DELAY_MS, RSS_SCAN_RETRY_WINDOW_MS, retryUnavailableSnapshot } from "./rss-snapshot-retry.js"; + +const FANOUTS = [1, 4, 16, 64] as const; +const WORKER = new URL("./rss-campaign-worker.ts", import.meta.url); +const WORKER_PATH = fileURLToPath(WORKER); +const DEFAULT_TIMEOUT_MS = 60_000; +const REAP_GRACE_MS = 250; +const REAP_VERIFY_MS = 1_000; +const SCHEMA_VERSION = 2; + +type SupportedPlatform = "linux"; +type Phase = "baseline" | "started" | "barrier-held" | "terminals" | "cleanup" | "final"; +type Status = "complete" | "failed" | "timed_out" | "unsupported"; + +interface ProcessSample { + phase: Phase; + monotonicMs: number; + totalRssKiB: number; + processes: readonly ProcessRecord[]; +} + +interface BoundaryMessage { + type: "boundary"; + phase: Exclude; + allocatedBytes: number; + memberPids: readonly number[]; +} + +interface ResultMessage { + type: "result"; + completed: number; + failed: number; + allocatedBytes: number; +} + +type WorkerMessage = BoundaryMessage | ResultMessage; + +interface Repetition { + schemaVersion: number; + kind: "b00b-rss-repetition"; + status: Status; + fanout: number; + repetition: number; + warmup: boolean; + sampler: { + source: "proc-status"; + requestedPeriodMs: number; + maxGapMs: number; + maxObservedGapMs: number | null; + sharedPages: "summed-per-process"; + } | null; + reasonCode: number | null; + baselineRssKiB: number | null; + peakRssKiB: number | null; + terminalRssKiB: number | null; + finalRssKiB: number | null; + allocatedBytes: number; + completed: number; + failed: number; + timedOut: boolean; + samples: readonly ProcessSample[]; +} + +interface Config { + fanouts: readonly number[]; + repetitions: number; + output: string; + requestedPeriodMs: number; + timeoutMs: number; + platformRequired?: string; + fixtureCommand?: string; + fixtureArgs: readonly string[]; + allocationMiB: number; + // Test-only delay which makes the pre-release ownership window deterministic. + identityCaptureDelayMs: number; + testIgnoreTerm: boolean; + // Test-only deterministic final-observation fault injection. + testFailFinalScan: boolean; + // Test-only one-shot final-observation fault injection for retry coverage. + testFailFinalScanOnce: boolean; +} + +interface GroupOwnership { + pgid: number; + leader: ProcessRecord; + members: Map; +} + +function option(name: string): string | undefined { + const index = process.argv.indexOf(name); + return index < 0 ? undefined : process.argv[index + 1]; +} + +function safeInteger(name: string, fallback: number, minimum: number): number { + const parsed = Number(option(name) ?? fallback); + if (!Number.isSafeInteger(parsed) || parsed < minimum) throw new Error(`invalid_${name.slice(2)}`); + return parsed; +} + +function parseFanouts(value: string | undefined): readonly number[] { + if (!value) return FANOUTS; + const values = value.split(",").map(Number); + if (!values.length || values.some((value) => !FANOUTS.includes(value as (typeof FANOUTS)[number]))) + throw new Error("invalid_fanout"); + return [...new Set(values)]; +} + +function config(): Config { + const maxGapMs = safeInteger("--interval-ms", MAX_RSS_SAMPLE_GAP_MS, MAX_RSS_SAMPLE_GAP_MS); + if (maxGapMs !== MAX_RSS_SAMPLE_GAP_MS) throw new Error("interval_ms_must_be_50"); + const fixtureArgs: string[] = []; + for (let index = 0; index < process.argv.length; index += 1) { + if (process.argv[index] === "--fixture-arg") { + const argument = process.argv[index + 1]; + if (argument === undefined) throw new Error("invalid_fixture_arg"); + fixtureArgs.push(argument); + index += 1; + } + } + return { + fanouts: parseFanouts(option("--fanout")), + repetitions: safeInteger("--repetitions", 3, 1), + output: option("--output") ?? "b00b-rss-artifacts", + requestedPeriodMs: DEFAULT_RSS_REQUESTED_PERIOD_MS + safeInteger("--test-scheduler-jitter-ms", 0, 0), + timeoutMs: safeInteger("--timeout-ms", DEFAULT_TIMEOUT_MS, 1), + platformRequired: option("--platform-required"), + fixtureCommand: option("--fixture-command"), + fixtureArgs, + allocationMiB: safeInteger("--allocation-mib", 1, 1), + identityCaptureDelayMs: safeInteger("--test-identity-capture-delay-ms", 0, 0), + testIgnoreTerm: process.argv.includes("--test-ignore-term"), + testFailFinalScan: process.argv.includes("--test-fail-final-scan"), + testFailFinalScanOnce: process.argv.includes("--test-fail-final-scan-once"), + }; +} + +function monotonicMs(): number { + return Number(process.hrtime.bigint() / 1_000_000n); +} + +function sha256(value: string | Uint8Array): string { + return createHash("sha256").update(value).digest("hex"); +} + +function canonical(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + const object = value as Record; + return `{${Object.keys(object) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonical(object[key])}`) + .join(",")}}`; +} + +async function writeOwnerFile(path: string, content: string): Promise { + await writeFile(path, content, { encoding: "utf8", mode: 0o600 }); + await chmod(path, 0o600); +} + +async function procIdentity(pid: number): Promise { + try { + return parseProcessStat(pid, await readFile(`/proc/${pid}/stat`, "utf8")); + } catch { + return undefined; + } +} + +async function procRecordForIdentity(identity: ProcessStat): Promise { + try { + const status = await readFile(`/proc/${identity.pid}/status`, "utf8"); + // State and PPID can change while status is read. A second stat read + // anchors the status to the original PID/start-time/process-group identity. + const confirmation = parseProcessStat(identity.pid, await readFile(`/proc/${identity.pid}/stat`, "utf8")); + return confirmation ? processRecordFromStatus(identity, status, confirmation) : undefined; + } catch { + return undefined; + } +} + +async function procRecord(pid: number): Promise { + const identity = await procIdentity(pid); + return identity ? procRecordForIdentity(identity) : undefined; +} + +type GroupSnapshot = + | { kind: "empty" } + | { kind: "records"; records: readonly ProcessRecord[] } + | { kind: "unavailable" }; + +function snapshotRecords(snapshot: GroupSnapshot): readonly ProcessRecord[] | undefined { + if (snapshot.kind === "unavailable") return undefined; + return snapshot.kind === "empty" ? [] : snapshot.records; +} + +function exitedDuringScan(error: unknown): boolean { + const code = (error as NodeJS.ErrnoException).code; + return code === "ENOENT" || code === "ESRCH"; +} + +/** + * Collect a complete group view. An empty records result is a positive claim: + * every numeric /proc entry was read and no member of this PGID remained. + * Any non-disappearance read or parse failure makes the entire scan unusable; + * collapsing it to [] could authorize a signal or a false zero-RSS result. + */ +async function groupSnapshot(pgid: number, injectFailure = false): Promise { + if (injectFailure) return { kind: "unavailable" }; + let entries: string[]; + try { + entries = await readdir("/proc"); + } catch { + return { kind: "unavailable" }; + } + const identities: ProcessStat[] = []; + for (const entry of entries) { + if (!/^\d+$/.test(entry)) continue; + const pid = Number(entry); + try { + const identity = parseProcessStat(pid, await readFile(`/proc/${pid}/stat`, "utf8")); + if (!identity) return { kind: "unavailable" }; + identities.push(identity); + } catch (error) { + // A process which vanished between readdir and stat cannot be a live, + // unobserved group member. Every other unreadable numeric entry fails closed. + if (!exitedDuringScan(error)) return { kind: "unavailable" }; + } + } + const records: ProcessRecord[] = []; + for (const identity of identities) { + if (identity.pgid !== pgid) continue; + try { + const status = await readFile(`/proc/${identity.pid}/status`, "utf8"); + // State and PPID are not stable identity. Re-read stat after status so + // PID reuse or a process-group move makes the entire scan unavailable. + const confirmation = parseProcessStat(identity.pid, await readFile(`/proc/${identity.pid}/stat`, "utf8")); + const record = confirmation ? processRecordFromStatus(identity, status, confirmation) : undefined; + // A coherently read status without any Vm* fields denotes a process + // without an mm and is retained as a zero-RSS owned record. + if (!record) return { kind: "unavailable" }; + records.push(record); + } catch (error) { + // Only a confirmed disappearance is safe to omit after membership was found. + if (!exitedDuringScan(error)) return { kind: "unavailable" }; + } + } + const sorted = records.sort((left, right) => left.pid - right.pid); + return sorted.length === 0 ? { kind: "empty" } : { kind: "records", records: sorted }; +} + +/** + * A /proc scan is only accepted when it completes coherently. Short-lived + * procfs read/parse races are retried, but a retry never turns an unavailable + * scan into an empty one without a later positive complete scan. + */ +async function groupSnapshotWithRetries( + pgid: number, + shouldInjectFailure: (attempt: number) => boolean = () => false, +): Promise { + // Fork/exec children can briefly have a stat record but no VmRSS. Retry + // complete scans through this short convergence window, never individual + // records: a successful return is always one coherent full-group view. + return retryUnavailableSnapshot( + (attempt) => groupSnapshot(pgid, shouldInjectFailure(attempt)), + (snapshot) => snapshot.kind === "unavailable", + { now: monotonicMs, pause }, + RSS_SCAN_RETRY_WINDOW_MS, + RSS_SCAN_RETRY_DELAY_MS, + ); +} + +async function collectorAvailable(): Promise { + const own = await procRecord(process.pid); + return own !== undefined && own.rssKiB >= 0 && Number.isSafeInteger(own.start) && Number.isSafeInteger(own.pgid); +} + +function total(records: readonly ProcessRecord[]): number { + return records.reduce((sum, record) => sum + record.rssKiB, 0); +} + +function sample(phase: Phase, records: readonly ProcessRecord[]): ProcessSample { + // This timestamp is taken only after the native collection finished. + return { phase, monotonicMs: monotonicMs(), totalRssKiB: total(records), processes: records }; +} + +function sameIdentity(left: ProcessRecord, right: ProcessRecord): boolean { + return left.pid === right.pid && left.start === right.start && left.pgid === right.pgid; +} + +function remember(ownership: GroupOwnership, records: readonly ProcessRecord[]): void { + for (const record of records) ownership.members.set(record.pid, record); +} + +function hasOwnedAnchor(ownership: GroupOwnership, records: readonly ProcessRecord[]): boolean { + return records.some((record) => { + const known = ownership.members.get(record.pid); + return known !== undefined && sameIdentity(known, record); + }); +} + +function pause(milliseconds: number): Promise { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +interface ReapResult { + reaped: boolean; + collectionFailed: boolean; +} + +async function reapOwnGroup(ownership?: GroupOwnership): Promise { + if (!ownership) return { reaped: true, collectionFailed: false }; + let collectionFailed = false; + const signalOwnedGroup = async (signal: NodeJS.Signals): Promise => { + const snapshot = await groupSnapshotWithRetries(ownership.pgid); + const records = snapshotRecords(snapshot); + if (!records) { + collectionFailed = true; + return false; + } + // A negative PID can affect a reused PGID. Signal only when a process whose + // PID, start tick, and PGID we captured is still anchoring this exact group. + if (!hasOwnedAnchor(ownership, records)) return records.length === 0; + remember(ownership, records); + try { + process.kill(-ownership.pgid, signal); + return true; + } catch { + return false; + } + }; + if (!(await signalOwnedGroup("SIGTERM"))) return { reaped: false, collectionFailed }; + await pause(REAP_GRACE_MS); + let snapshot = await groupSnapshotWithRetries(ownership.pgid); + let records = snapshotRecords(snapshot); + if (!records) return { reaped: false, collectionFailed: true }; + if (records.length === 0) return { reaped: true, collectionFailed }; + if (!(await signalOwnedGroup("SIGKILL"))) return { reaped: false, collectionFailed }; + const deadline = monotonicMs() + REAP_VERIFY_MS; + for (;;) { + await pause(10); + snapshot = await groupSnapshotWithRetries(ownership.pgid); + records = snapshotRecords(snapshot); + if (records?.length === 0) return { reaped: true, collectionFailed }; + if (records === undefined) collectionFailed = true; + // After SIGKILL, a short /proc outage need not decide the result. Keep + // checking through the existing verification deadline, but only a later + // complete empty scan can establish a successful reap/final zero. + if (monotonicMs() >= deadline) return { reaped: false, collectionFailed }; + } +} + +function workerArguments(settings: Config, fanout: number, scratch: string): string[] { + const args = [ + "--expose-gc", + ...childExecArgsWithTsxImport(process.execArgv), + WORKER_PATH, + "--fanout", + String(fanout), + "--allocation-mib", + String(settings.allocationMiB), + "--scratch", + scratch, + ]; + if (settings.fixtureCommand) args.push("--fixture-command", settings.fixtureCommand); + for (const fixtureArg of settings.fixtureArgs) args.push("--fixture-arg", fixtureArg); + if (settings.testIgnoreTerm) args.push("--test-ignore-term"); + return args; +} + +function unsupportedRun(fanout: number, repetition: number, warmup: boolean): Repetition { + return { + schemaVersion: SCHEMA_VERSION, + kind: "b00b-rss-repetition", + status: "unsupported", + fanout, + repetition, + warmup, + sampler: null, + reasonCode: 3, + baselineRssKiB: null, + peakRssKiB: null, + terminalRssKiB: null, + finalRssKiB: null, + allocatedBytes: 0, + completed: 0, + failed: fanout, + timedOut: false, + samples: [], + }; +} + +async function runCell( + settings: Config, + fanout: number, + repetition: number, + warmup: boolean, + scratch: string, +): Promise { + const samples: ProcessSample[] = [sample("baseline", [])]; + let ownership: GroupOwnership | undefined; + let child: ChildProcess | undefined; + let stopped = false; + let timedOut = false; + let completed = 0; + let failed = fanout; + let allocatedBytes = 0; + let collectorFailed = false; + let queue = Promise.resolve(); + const pendingMemberPids = new Set(); + const enqueue = (phase: Phase, memberPids: readonly number[] = []): Promise => { + for (const pid of memberPids) pendingMemberPids.add(pid); + queue = queue.then(async () => { + const currentOwnership = ownership; + if (!currentOwnership) return; + // The worker supplies its direct fixture PIDs at each boundary. Preserve + // their PID/start/PGID identities before a timeout can make the leader exit. + const announced = await Promise.all([...pendingMemberPids].map(procRecord)); + remember( + currentOwnership, + announced.filter((record): record is ProcessRecord => record?.pgid === currentOwnership.pgid), + ); + const snapshot = await groupSnapshotWithRetries(currentOwnership.pgid); + const records = snapshotRecords(snapshot); + if (!records) { + collectorFailed = true; + return; + } + remember(currentOwnership, records); + samples.push(sample(phase, records)); + }); + return queue; + }; + const reapDirectChild = async (): Promise => { + // Before release the worker protocol has not allocated or spawned anything. + // Never use a negative PGID without an authenticated /proc identity anchor. + const direct = child; + if (!direct || direct.exitCode !== null || direct.signalCode !== null) return; + try { + direct.kill("SIGKILL"); + } catch { + return; + } + await new Promise((resolve) => { + if (direct.exitCode !== null || direct.signalCode !== null) resolve(); + else direct.once("exit", () => resolve()); + }); + }; + return new Promise((resolve) => { + let timer: NodeJS.Timeout | undefined; + let timeout: NodeJS.Timeout | undefined; + let executionStarted = false; + let settle: (requested: Status) => Promise; + const startExecution = (): void => { + if (executionStarted || stopped) return; + executionStarted = true; + void enqueue("started"); + timer = setInterval(() => { + if (!stopped) void enqueue("started"); + }, settings.requestedPeriodMs); + timeout = setTimeout(() => { + timedOut = true; + void settle("timed_out"); + }, settings.timeoutMs); + }; + settle = async (requested: Status): Promise => { + if (stopped) return; + stopped = true; + if (timer) clearInterval(timer); + if (timeout) clearTimeout(timeout); + await queue; + const cadence = validateRssSampleCadence( + samples.filter((entry) => entry.phase === "started").map((entry) => entry.monotonicMs), + ); + const cadenceFailed = !cadence.valid; + + // A failed startup is deliberately not represented as an empty owned + // group: no exact PID/start/PGID anchor was ever authenticated. + if (!ownership) { + await reapDirectChild(); + resolve({ + schemaVersion: SCHEMA_VERSION, + kind: "b00b-rss-repetition", + status: "failed", + fanout, + repetition, + warmup, + sampler: { + source: "proc-status", + requestedPeriodMs: settings.requestedPeriodMs, + maxGapMs: MAX_RSS_SAMPLE_GAP_MS, + maxObservedGapMs: cadence.maxObservedGapMs, + sharedPages: "summed-per-process", + }, + reasonCode: 2, + baselineRssKiB: 0, + peakRssKiB: null, + terminalRssKiB: null, + finalRssKiB: null, + allocatedBytes: 0, + completed: 0, + failed: fanout, + timedOut: false, + samples, + }); + return; + } + + const reap = await reapOwnGroup(ownership); + const finalSnapshot = await groupSnapshotWithRetries( + ownership.pgid, + (attempt) => settings.testFailFinalScan || (settings.testFailFinalScanOnce && attempt === 0), + ); + const finalRecords = snapshotRecords(finalSnapshot); + const finalCollectionFailed = finalRecords === undefined; + if (finalRecords) samples.push(sample("final", finalRecords)); + const collectionFailure = collectorFailed || reap.collectionFailed || finalCollectionFailed; + const emptyOwnedGroup = !collectionFailure && reap.reaped && finalRecords?.length === 0; + const status = + requested === "complete" && emptyOwnedGroup && !cadenceFailed + ? "complete" + : requested === "timed_out" && emptyOwnedGroup && !cadenceFailed + ? "timed_out" + : "failed"; + const byPhase = (phase: Phase) => samples.filter((entry) => entry.phase === phase).at(-1)?.totalRssKiB ?? null; + const active = samples.filter((entry) => entry.phase !== "baseline" && entry.phase !== "final"); + resolve({ + schemaVersion: SCHEMA_VERSION, + kind: "b00b-rss-repetition", + status, + fanout, + repetition, + warmup, + sampler: { + source: "proc-status", + requestedPeriodMs: settings.requestedPeriodMs, + maxGapMs: MAX_RSS_SAMPLE_GAP_MS, + maxObservedGapMs: cadence.maxObservedGapMs, + sharedPages: "summed-per-process", + }, + reasonCode: status === "complete" ? null : collectionFailure ? 5 : timedOut ? 1 : cadenceFailed ? 4 : 2, + baselineRssKiB: 0, + peakRssKiB: active.length ? Math.max(...active.map((entry) => entry.totalRssKiB)) : null, + terminalRssKiB: byPhase("terminals"), + finalRssKiB: collectionFailure ? null : total(finalRecords ?? []), + allocatedBytes, + completed, + failed, + timedOut, + samples, + }); + }; + try { + child = spawn(process.execPath, workerArguments(settings, fanout, scratch), { + cwd: process.cwd(), + detached: true, + env: { PATH: process.env.PATH, HOME: process.env.HOME, TMPDIR: process.env.TMPDIR, LANG: "C", LC_ALL: "C" }, + serialization: "json", + stdio: ["ignore", "ignore", "ignore", "ipc"], + }); + } catch { + void settle("failed"); + return; + } + const pid = child.pid; + if (!pid) { + void settle("failed"); + return; + } + child.once("error", () => void settle("failed")); + child.on("message", (message: WorkerMessage) => { + if (message.type === "boundary") startExecution(); + if (message.type === "result") { + completed = message.completed; + failed = message.failed; + allocatedBytes = Math.max(allocatedBytes, message.allocatedBytes); + return; + } + allocatedBytes = Math.max(allocatedBytes, message.allocatedBytes); + void enqueue(message.phase, message.memberPids); + }); + child.once( + "exit", + (code, signal) => void settle(code === 0 && signal === null && completed === fanout ? "complete" : "failed"), + ); + void (async () => { + if (settings.identityCaptureDelayMs) await pause(settings.identityCaptureDelayMs); + const leader = await procRecord(pid); + if (!leader || leader.pgid !== pid || stopped) { + void settle("failed"); + return; + } + ownership = { pgid: leader.pgid, leader, members: new Map([[leader.pid, leader]]) }; + // The worker remains gated until this release. Its first boundary proves + // execution began, at which point startExecution arms the timeout. + try { + child?.send({ type: "release" }, (error) => { + if (error && !stopped) void settle("failed"); + }); + } catch { + void settle("failed"); + } + })(); + }); +} + +function percentile(values: readonly number[], proportion: number): number | null { + if (!values.length) return null; + const sorted = [...values].sort((left, right) => left - right); + return sorted[Math.min(sorted.length - 1, Math.max(0, Math.ceil(proportion * sorted.length) - 1))]; +} + +function summary(repetitions: readonly Repetition[]): Record { + const complete = repetitions.filter((entry) => entry.status === "complete"); + const peak = complete.map((entry) => (entry.peakRssKiB ?? 0) - (entry.baselineRssKiB ?? 0)); + const final = complete.map((entry) => (entry.finalRssKiB ?? 0) - (entry.baselineRssKiB ?? 0)); + return { + count: complete.length, + peakMinKiB: percentile(peak, 0), + peakMedianKiB: percentile(peak, 0.5), + peakP95KiB: percentile(peak, 0.95), + peakMaxKiB: percentile(peak, 1), + finalMinKiB: percentile(final, 0), + finalMedianKiB: percentile(final, 0.5), + finalP95KiB: percentile(final, 0.95), + finalMaxKiB: percentile(final, 1), + }; +} + +async function gitSha(): Promise { + const git = spawn("git", ["rev-parse", "HEAD"], { stdio: ["ignore", "pipe", "ignore"] }); + let value = ""; + git.stdout?.setEncoding("utf8"); + git.stdout?.on("data", (chunk: string) => { + value += chunk; + }); + const code = await new Promise((resolve) => git.once("exit", resolve)); + const sha = value.trim(); + return code === 0 && /^[a-f0-9]{40}$/.test(sha) ? sha : null; +} + +async function hashTree(directory: string): Promise { + const names = (await readdir(directory)).filter((name) => name !== "manifest.json").sort(); + return Promise.all( + names.map(async (name) => { + const path = join(directory, name); + if (!(await stat(path)).isFile()) throw new Error("output_contains_non_file"); + const content = await readFile(path); + return { name, sha256: sha256(content), bytes: content.byteLength }; + }), + ); +} + +async function freshOutput(directory: string): Promise { + try { + const info = await stat(directory); + if (!info.isDirectory() || (await readdir(directory)).length > 0) throw new Error("output_must_be_new_or_empty"); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + await mkdir(directory, { mode: 0o700 }); + } + await chmod(directory, 0o700); +} + +async function main(): Promise { + const settings = config(); + const current = platform(); + const platformMatches = !settings.platformRequired || settings.platformRequired === current; + // Darwin ps lstart is wall-clock, whole-second data. It cannot establish the + // PID/start identity needed before destructive negative-PID signals, so macOS + // is explicitly unsupported rather than pretending its lstart values are safe. + const kind: SupportedPlatform | undefined = + platformMatches && current === "linux" && (await collectorAvailable()) ? "linux" : undefined; + await freshOutput(settings.output); + const scratch = join(dirname(settings.output), `.b00b-rss-scratch-${process.pid}`); + await mkdir(scratch, { recursive: true, mode: 0o700 }); + await chmod(scratch, 0o700); + const runs: Repetition[] = []; + for (const fanout of settings.fanouts) { + if (!kind) { + runs.push(unsupportedRun(fanout, 0, true)); + for (let repetition = 1; repetition <= settings.repetitions; repetition += 1) + runs.push(unsupportedRun(fanout, repetition, false)); + continue; + } + runs.push(await runCell(settings, fanout, 0, true, scratch)); + for (let repetition = 1; repetition <= settings.repetitions; repetition += 1) + runs.push(await runCell(settings, fanout, repetition, false, scratch)); + } + await rm(scratch, { force: true, recursive: true }); + for (const run of runs) + await writeOwnerFile( + join(settings.output, `run-${run.fanout}-${run.repetition}-${run.warmup ? 0 : 1}.json`), + `${canonical(run)}\n`, + ); + const manifest = { + schemaVersion: SCHEMA_VERSION, + kind: "b00b-rss-campaign", + platform: current, + release: release(), + node: process.version, + cpuCount: cpus().length, + memoryBytes: totalmem(), + gitSha: await gitSha(), + collector: kind === "linux" ? "proc-status" : "unsupported", + requestedPeriodMs: settings.requestedPeriodMs, + maxGapMs: MAX_RSS_SAMPLE_GAP_MS, + timeoutMs: settings.timeoutMs, + fanouts: settings.fanouts, + repetitions: settings.repetitions, + warmups: 1, + allocationMiB: settings.allocationMiB, + externalFixture: settings.fixtureCommand !== undefined, + summaries: settings.fanouts.map((fanout) => ({ + fanout, + ...summary(runs.filter((run) => run.fanout === fanout && !run.warmup)), + })), + files: await hashTree(settings.output), + }; + await writeOwnerFile(join(settings.output, "manifest.json"), `${canonical(manifest)}\n`); + console.log(`b00b-rss: ${runs.filter((run) => run.status === "complete" && !run.warmup).length} completed cells`); +} + +await main(); diff --git a/packages/coding-agent/test/swarm/swarm-evidence.test.ts b/packages/coding-agent/test/swarm/swarm-evidence.test.ts index f7824de52..8c834756f 100644 --- a/packages/coding-agent/test/swarm/swarm-evidence.test.ts +++ b/packages/coding-agent/test/swarm/swarm-evidence.test.ts @@ -135,8 +135,8 @@ describe("PR-B00A deterministic local swarm evidence", () => { const lead = evidence.costAttribution.find((cost) => cost.id === "role-0001"); const run = evidence.costAttribution.find((cost) => cost.id === "run"); expect(lead?.downstreamInputTokens).toBe(64); - expect(run).toMatchObject({ kind: "run", directCost: 0 }); - expect(run?.downstreamCost).toBeGreaterThan(0); + expect(run).toMatchObject({ kind: "run", directCostNumerator: 0 }); + expect(run?.downstreamCostNumerator).toBeGreaterThan(0); }); test("accepts an empty process sample when the platform sampler has no visible processes", async () => { const directory = await mkdtemp(join(tmpdir(), "prime-agent-b00a-empty-processes-")); @@ -248,7 +248,7 @@ describe("PR-B00A deterministic local swarm evidence", () => { const directory = await evidenceDirectory(1); const costsPath = join(directory, "cost-attribution.json"); const costs = JSON.parse(await readFile(costsPath, "utf8")); - costs.find((cost: { kind: string }) => cost.kind === "node").directCost = 7; + costs.find((cost: { kind: string }) => cost.kind === "node").directCostNumerator = 7; const costsRaw = `${canonicalJson(costs)} `; await writeFile(costsPath, costsRaw); @@ -319,17 +319,17 @@ describe("PR-B00A deterministic local swarm evidence", () => { const costs = JSON.parse(await readFile(costsPath, "utf8")); const node = costs.find((cost: { kind: string }) => cost.kind === "node"); node.directInputTokens = 999; - node.directCost = (999 + node.directOutputTokens * 2) / 1_000_000; + node.directCostNumerator = 999 + node.directOutputTokens * 2; node.downstreamInputTokens = 999; - node.downstreamCost = node.directCost; + node.downstreamCostNumerator = node.directCostNumerator; const role = costs.find((cost: { kind: string }) => cost.kind === "role"); role.directInputTokens = 999; role.downstreamInputTokens = 999; - role.directCost = node.directCost; - role.downstreamCost = node.directCost; + role.directCostNumerator = node.directCostNumerator; + role.downstreamCostNumerator = node.directCostNumerator; const run = costs.find((cost: { kind: string }) => cost.kind === "run"); run.downstreamInputTokens = 999; - run.downstreamCost = node.directCost; + run.downstreamCostNumerator = node.directCostNumerator; await rehashArtifact(directory, "cost-attribution.json", `${canonicalJson(costs)}\n`); await expect(verify(directory)).rejects.toThrow("assignment input usage mismatch"); }); @@ -377,8 +377,8 @@ describe("PR-B00A deterministic local swarm evidence", () => { row.directOutputTokens = row.kind === "run" ? 0 : 999; row.downstreamInputTokens = 999; row.downstreamOutputTokens = 999; - row.directCost = row.kind === "run" ? 0 : 999 / 1_000_000 + (999 * 2) / 1_000_000; - row.downstreamCost = 999 / 1_000_000 + (999 * 2) / 1_000_000; + row.directCostNumerator = row.kind === "run" ? 0 : 999 + 999 * 2; + row.downstreamCostNumerator = 999 + 999 * 2; } await rehashArtifact(directory, "events.jsonl", `${events.map(canonicalJson).join("\n")}\n`); await rehashArtifact( @@ -422,17 +422,17 @@ describe("PR-B00A deterministic local swarm evidence", () => { const costs = JSON.parse(await readFile(costsPath, "utf8")); const node = costs.find((cost: { kind: string }) => cost.kind === "node"); node.directOutputTokens = 999; - node.directCost = (node.directInputTokens + 999 * 2) / 1_000_000; + node.directCostNumerator = node.directInputTokens + 999 * 2; node.downstreamOutputTokens = 999; - node.downstreamCost = node.directCost; + node.downstreamCostNumerator = node.directCostNumerator; const role = costs.find((cost: { kind: string }) => cost.kind === "role"); role.directOutputTokens = 999; role.downstreamOutputTokens = 999; - role.directCost = node.directCost; - role.downstreamCost = node.directCost; + role.directCostNumerator = node.directCostNumerator; + role.downstreamCostNumerator = node.directCostNumerator; const run = costs.find((cost: { kind: string }) => cost.kind === "run"); run.downstreamOutputTokens = 999; - run.downstreamCost = node.directCost; + run.downstreamCostNumerator = node.directCostNumerator; await rehashArtifact(directory, "cost-attribution.json", `${canonicalJson(costs)}\n`); await expect(verify(directory)).rejects.toThrow("terminal output usage mismatch"); }); diff --git a/packages/coding-agent/test/swarm/swarm-evidence.ts b/packages/coding-agent/test/swarm/swarm-evidence.ts index 0110427ec..2f7e04a17 100644 --- a/packages/coding-agent/test/swarm/swarm-evidence.ts +++ b/packages/coding-agent/test/swarm/swarm-evidence.ts @@ -7,14 +7,19 @@ */ import { execFileSync } from "node:child_process"; -import { createHash } from "node:crypto"; +import { createHash, createPublicKey, type KeyObject, verify as verifySignature } from "node:crypto"; import { chmod, lstat, mkdir, readdir, readFile, realpath, stat, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { performance } from "node:perf_hooks"; export const SUPPORTED_SWARM_FANOUTS = [1, 4, 16, 64] as const; export const SWARM_EVIDENCE_SCHEMA_VERSION = "prime-agent.swarm-evidence/v1"; -const MICRO_TOKENS = 1_000_000; +/** + * Cost amounts are exact safe-integer numerators over this fixed denominator. + * `costNumerator / COST_NUMERATOR_SCALE` is presentation only; every signed + * artifact invariant operates exclusively on the numerator. + */ +export const COST_NUMERATOR_SCALE = 1_000_000; const REDACTED = "[REDACTED]"; const EVIDENCE_FILES = [ "cost-attribution.json", @@ -47,17 +52,29 @@ export interface FakeProviderFaultSchedule { readonly nodeId: string; readonly actions: readonly FakeProviderAction[]; } +export interface RequestedModelProvenance { + readonly provider: string; + readonly model: string; + readonly revision?: string; + readonly effort?: string; +} +/** Response model is the attribution authority; selected resolved model is retained separately. */ +export interface ResolvedModelProvenance { + readonly api: string; + readonly provider: string; + readonly model: string; + readonly responseModel: string; +} export interface AssignmentSpec { readonly nodeId: string; readonly parentNodeId?: string; readonly role: string; - readonly requested: { - readonly provider: string; - readonly model: string; - readonly revision?: string; - readonly effort?: string; - }; - readonly resolved?: AssignmentSpec["requested"]; + /** Stable public linkage for a request and its individual retry attempt. */ + readonly requestId?: string; + readonly attempt?: number; + readonly attemptId?: string; + readonly requested: RequestedModelProvenance; + readonly resolved?: ResolvedModelProvenance; readonly inputTokens?: number; readonly outputTokens?: number; } @@ -129,10 +146,10 @@ export interface CostAttribution { readonly kind: "node" | "role" | "run"; readonly directInputTokens: number; readonly directOutputTokens: number; - readonly directCost: number; + readonly directCostNumerator: number; readonly downstreamInputTokens: number; readonly downstreamOutputTokens: number; - readonly downstreamCost: number; + readonly downstreamCostNumerator: number; } export interface EvidenceArtifact { readonly path: (typeof EVIDENCE_FILES)[number]; @@ -171,6 +188,49 @@ function issueSwarmEvidenceCapability(): SwarmEvidenceCapability { return Object.freeze({}) as SwarmEvidenceCapability; } +/** An opaque root created only from an externally supplied Ed25519 public key. */ +declare const swarmEvidenceTrustRootBrand: unique symbol; +export type SwarmEvidenceTrustRoot = { readonly [swarmEvidenceTrustRootBrand]: true }; +const registeredTrustRoots = new WeakMap(); +export const SWARM_EVIDENCE_COMMITMENT_SCHEMA = "prime-agent.swarm-evidence-commitment/v1"; +export interface SignedSwarmEvidenceCommitment { + readonly schemaVersion: typeof SWARM_EVIDENCE_COMMITMENT_SCHEMA; + readonly artifactBundleId: string; + readonly signature: string; +} + +/** + * Registers a verifier trust root. The caller must provide this public key out + * of band: an artifact directory has no authority to manufacture this object. + */ +export function createSwarmEvidenceTrustRoot(publicKeyPem: string): SwarmEvidenceTrustRoot { + let publicKey: KeyObject; + try { + publicKey = createPublicKey(publicKeyPem); + } catch { + throw new Error("invalid swarm evidence public key"); + } + assert(publicKey.asymmetricKeyType === "ed25519", "swarm evidence trust root must be Ed25519"); + const root = Object.freeze({}) as SwarmEvidenceTrustRoot; + registeredTrustRoots.set(root, publicKey); + return root; +} + +/** Checked accessor: use the writer-issued identity; never read it back from mutable artifacts. */ +export function artifactBundleIdForSwarmEvidenceCapability(capability: SwarmEvidenceCapability): string { + const registration = registeredBundles.get(capability); + assert(registration, "issued swarm evidence capability is required"); + return registration.artifactBundleId; +} + +export function swarmEvidenceCommitmentPayload(artifactBundleId: string): { + schemaVersion: typeof SWARM_EVIDENCE_COMMITMENT_SCHEMA; + artifactBundleId: string; +} { + assert(/^[0-9a-f]{64}$/.test(artifactBundleId), "invalid trusted artifact bundle identity"); + return { schemaVersion: SWARM_EVIDENCE_COMMITMENT_SCHEMA, artifactBundleId }; +} + /** Canonical JSON rejects values which JSON.stringify silently changes. */ export function canonicalJson(value: unknown): string { if (value === null || typeof value === "boolean" || typeof value === "string") return JSON.stringify(value); @@ -242,6 +302,10 @@ const SAFE_EVIDENCE_KEYS = new Set([ "model", "revision", "effort", + "api", + "responseModel", + "attempt", + "attemptId", "inputTokens", "outputTokens", "actions", @@ -269,10 +333,10 @@ const SAFE_EVIDENCE_KEYS = new Set([ "kind", "directInputTokens", "directOutputTokens", - "directCost", + "directCostNumerator", "downstreamInputTokens", "downstreamOutputTokens", - "downstreamCost", + "downstreamCostNumerator", "admitted", "started", "completed", @@ -288,6 +352,7 @@ function safeEvidenceString(value: string, key?: string): boolean { return ( (key === "nodeId" && /^worker-\d{4}$/.test(value)) || (key === "requestId" && /^request-\d{4}$/.test(value)) || + (key === "attemptId" && /^attempt-\d{4}-\d{2}$/.test(value)) || (key === "parentNodeId" && (value === "root" || /^worker-\d{4}$/.test(value))) || ((key === "id" || key === "role") && (value === "run" || /^worker-\d{4}$/.test(value) || /^role-\d{4}$/.test(value))) || @@ -300,7 +365,18 @@ function safeEvidenceString(value: string, key?: string): boolean { (key === "benchmarkVersion" && value === "b00a") || ((key === "fingerprint" || key === "deterministicBundleId" || key === "artifactBundleId" || key === "sha256") && /^[0-9a-f]{64}$/.test(value)) || - (key === "path" && (EVIDENCE_FILES as readonly string[]).includes(value)) + (key === "path" && (EVIDENCE_FILES as readonly string[]).includes(value)) || + ((key === "provider" || key === "api") && value === "b00b-scripted") || + ((key === "revision" || key === "effort") && value === REDACTED) || + ((key === "model" || key === "responseModel") && + [ + "fixture-a", + "fixture-b", + "fixture-zero", + "fixture-a-resolved", + "fixture-b-resolved", + "fixture-zero-resolved", + ].includes(value)) ); } /** No arbitrary fixture content, including object keys, enters normal artifacts. */ @@ -344,8 +420,17 @@ function assertContentFree(value: unknown, key?: string, untrustedObjectKeys = f ); } } -function money(tokens: number, pricePerMillion: number): number { - return (tokens * pricePerMillion) / MICRO_TOKENS; +/** Exact numerator with denominator COST_NUMERATOR_SCALE; never use decimal money in evidence invariants. */ +function costNumerator(tokens: number, pricePerMillionTokens: number): number { + const numerator = tokens * pricePerMillionTokens; + assert(isSafeInteger(numerator), "cost numerator exceeds safe integer range"); + return numerator; +} +/** Sum authenticated integer fields without ever crossing into binary decimal money. */ +function exactSum(values: readonly number[]): number { + const total = values.reduce((sum, value) => sum + value, 0); + assert(isSafeInteger(total), "exact accounting sum exceeds safe integer range"); + return total; } function validate(config: SwarmBenchmarkConfig): void { assert(config.scenario.trim().length > 0, "scenario must not be empty"); @@ -371,6 +456,12 @@ function validate(config: SwarmBenchmarkConfig): void { assignment.outputTokens === undefined || isSafeInteger(assignment.outputTokens), "output tokens must be non-negative safe integers", ); + if (assignment.requestId !== undefined) + assert(/^request-\d{4}$/.test(assignment.requestId), "request IDs must be stable public IDs"); + if (assignment.attempt !== undefined) + assert(isSafeInteger(assignment.attempt) && assignment.attempt > 0, "attempt must be positive"); + if (assignment.attemptId !== undefined) + assert(/^attempt-\d{4}-\d{2}$/.test(assignment.attemptId), "attempt IDs must be stable public IDs"); } for (const schedule of config.faultSchedule ?? []) { assert( @@ -404,6 +495,9 @@ function publicConfig(config: SwarmBenchmarkConfig): Omit `request-${nodeId.slice("worker-".length)}`; + const requestFor = (assignment: AssignmentSpec) => + assignment.requestId ?? `request-${assignment.nodeId.slice("worker-".length)}`; const record = (type: EventType, nodeId: string, detail?: Readonly>) => events.push({ sequence: ++sequence, elapsedMilliseconds: performance.now() - startedAt, type, nodeId, - requestId: requestFor(nodeId), + requestId: requestFor(publicAssignments.find((assignment) => assignment.nodeId === nodeId)!), ...(detail === undefined ? {} : { detail }), }); const sample = (phase: ProcessSample["phase"]) => { @@ -568,20 +663,24 @@ export async function runSwarmBenchmark(config: SwarmBenchmarkConfig): Promise calculate(candidate.assignment.nodeId)); const directInputTokens = result.assignment.inputTokens ?? 32; const directOutputTokens = result.outputTokens; - const directCost = - money(directInputTokens, config.priceCard.inputPerMillionTokens) + - money(directOutputTokens, config.priceCard.outputPerMillionTokens); + const directCostNumerator = + costNumerator(directInputTokens, config.priceCard.inputPerMillionTokens) + + costNumerator(directOutputTokens, config.priceCard.outputPerMillionTokens); const attribution = { id, kind: "node" as const, directInputTokens, directOutputTokens, - directCost, - downstreamInputTokens: - directInputTokens + children.reduce((sum, child) => sum + child.downstreamInputTokens, 0), - downstreamOutputTokens: - directOutputTokens + children.reduce((sum, child) => sum + child.downstreamOutputTokens, 0), - downstreamCost: directCost + children.reduce((sum, child) => sum + child.downstreamCost, 0), + directCostNumerator, + downstreamInputTokens: exactSum([directInputTokens, ...children.map((child) => child.downstreamInputTokens)]), + downstreamOutputTokens: exactSum([ + directOutputTokens, + ...children.map((child) => child.downstreamOutputTokens), + ]), + downstreamCostNumerator: exactSum([ + directCostNumerator, + ...children.map((child) => child.downstreamCostNumerator), + ]), }; costs.set(id, attribution); return attribution; @@ -609,16 +708,16 @@ export async function runSwarmBenchmark(config: SwarmBenchmarkConfig): Promise result.assignment.role === role) .map((result) => calculate(result.assignment.nodeId)); const sum = (items: readonly CostAttribution[], key: keyof CostAttribution) => - items.reduce((total, item) => total + (item[key] as number), 0); + exactSum(items.map((item) => item[key] as number)); return { id: role, kind: "role" as const, directInputTokens: sum(direct, "directInputTokens"), directOutputTokens: sum(direct, "directOutputTokens"), - directCost: sum(direct, "directCost"), + directCostNumerator: sum(direct, "directCostNumerator"), downstreamInputTokens: sum([...included.values()], "directInputTokens"), downstreamOutputTokens: sum([...included.values()], "directOutputTokens"), - downstreamCost: sum([...included.values()], "directCost"), + downstreamCostNumerator: sum([...included.values()], "directCostNumerator"), }; }); const roots = results @@ -629,10 +728,10 @@ export async function runSwarmBenchmark(config: SwarmBenchmarkConfig): Promise sum + item.downstreamInputTokens, 0), - downstreamOutputTokens: roots.reduce((sum, item) => sum + item.downstreamOutputTokens, 0), - downstreamCost: roots.reduce((sum, item) => sum + item.downstreamCost, 0), + directCostNumerator: 0, + downstreamInputTokens: exactSum(roots.map((item) => item.downstreamInputTokens)), + downstreamOutputTokens: exactSum(roots.map((item) => item.downstreamOutputTokens)), + downstreamCostNumerator: exactSum(roots.map((item) => item.downstreamCostNumerator)), }; const firstTerminal = events.findIndex( (event) => event.type === "provider_completed" || event.type === "provider_failure", @@ -775,6 +874,29 @@ function requireManifest(manifest: unknown): asserts manifest is SwarmManifest & "invalid artifact bundle identity", ); assertContentFree(manifest); + const attemptIds = new Set(); + for (const assignment of manifest.assignments as Record[]) { + assert(isRecord(assignment) && isRecord(assignment.requested), "invalid assignment provenance"); + for (const key of ["provider", "model"]) + assert(typeof assignment.requested[key] === "string", `missing requested ${key}`); + if (assignment.resolved !== undefined) { + assert(isRecord(assignment.resolved), "invalid resolved provenance"); + for (const key of ["api", "provider", "model", "responseModel"]) + assert(typeof assignment.resolved[key] === "string", `missing resolved ${key}`); + } + if (assignment.attemptId !== undefined) { + const attemptId = assignment.attemptId; + assert( + typeof assignment.requestId === "string" && + isSafeInteger(assignment.attempt) && + typeof attemptId === "string" && + /^attempt-\d{4}-\d{2}$/.test(attemptId) && + !attemptIds.has(attemptId), + "invalid or duplicate request attempt identity", + ); + attemptIds.add(attemptId); + } + } const source = { schemaVersion: manifest.schemaVersion, benchmarkVersion: manifest.benchmarkVersion, @@ -788,9 +910,9 @@ function requireManifest(manifest: unknown): asserts manifest is SwarmManifest & } function verifyEvents(events: readonly unknown[], oracle: readonly unknown[], assignments: readonly unknown[]): void { assert(events.length === oracle.length && events.length > 0, "event/oracle length mismatch"); - const nodeIds = new Set( - (assignments as readonly Record[]).map((assignment) => assignment.nodeId).filter(isString), - ); + const assignmentRows = assignments as readonly Record[]; + const nodeIds = new Set(assignmentRows.map((assignment) => assignment.nodeId).filter(isString)); + const assignmentByNode = new Map(assignmentRows.map((assignment) => [assignment.nodeId as string, assignment])); let previousSequence = 0; const byNode = new Map[]>(); for (let index = 0; index < events.length; index++) { @@ -817,10 +939,11 @@ function verifyEvents(events: readonly unknown[], oracle: readonly unknown[], as (EVENT_TYPES as readonly unknown[]).includes(event.type), "invalid event timing/type", ); + const eventAssignment = assignmentByNode.get(event.nodeId as string); assert( typeof event.nodeId === "string" && nodeIds.has(event.nodeId) && - event.requestId === `request-${event.nodeId.slice("worker-".length)}`, + event.requestId === (eventAssignment?.requestId ?? `request-${event.nodeId.slice("worker-".length)}`), "invalid event identity", ); assert( @@ -842,6 +965,14 @@ function verifyEvents(events: readonly unknown[], oracle: readonly unknown[], as break; case "provider_request_started": exactDetail(["role", "requested", "resolved"]); + assert( + isRecord(detail) && + canonicalJson(detail.role) === canonicalJson(eventAssignment?.role) && + canonicalJson(detail.requested) === canonicalJson(eventAssignment?.requested) && + canonicalJson(detail.resolved) === + canonicalJson(eventAssignment?.resolved ?? eventAssignment?.requested), + "event provenance mismatch", + ); break; case "progress": exactDetail(["message"]); @@ -937,8 +1068,8 @@ function verifyCosts( ids.add(row.id); for (const key of ["directInputTokens", "directOutputTokens", "downstreamInputTokens", "downstreamOutputTokens"]) assert(isSafeInteger(row[key]), `invalid ${key}`); - for (const key of ["directCost", "downstreamCost"]) - assert(typeof row[key] === "number" && Number.isFinite(row[key]), `invalid ${key}`); + for (const key of ["directCostNumerator", "downstreamCostNumerator"]) + assert(isSafeInteger(row[key]), `invalid ${key}`); } const nodes = rows.filter((row) => row.kind === "node"); const assignmentRows = assignments as readonly Record[]; @@ -964,20 +1095,21 @@ function verifyCosts( `terminal output usage mismatch: ${node.id}`, ); assert( - node.directCost === - money(node.directInputTokens as number, inputPrice) + money(node.directOutputTokens as number, outputPrice), + node.directCostNumerator === + costNumerator(node.directInputTokens as number, inputPrice) + + costNumerator(node.directOutputTokens as number, outputPrice), "direct economics mismatch", ); const children = assignmentRows .filter((assignment) => assignment.parentNodeId === node.id) .map((assignment) => nodeById.get(assignment.nodeId as string)); assert(children.every(isRecord), "missing child cost"); - for (const suffix of ["InputTokens", "OutputTokens", "Cost"] as const) { + for (const suffix of ["InputTokens", "OutputTokens", "CostNumerator"] as const) { const direct = node[`direct${suffix}`]; assert(typeof direct === "number", `invalid direct cost field: ${suffix}`); assert( node[`downstream${suffix}`] === - direct + children.reduce((sum, child) => sum + (child[`downstream${suffix}`] as number), 0), + exactSum([direct as number, ...children.map((child) => child[`downstream${suffix}`] as number)]), `node tree invariant failed: ${node.id}:${suffix}`, ); } @@ -1003,26 +1135,30 @@ function verifyCosts( .filter((assignment) => assignment.role === role.id) .map((assignment) => nodeById.get(assignment.nodeId as string)!); const included = new Map(direct.flatMap((node) => descendants(node.id as string)).map((node) => [node.id, node])); - for (const suffix of ["InputTokens", "OutputTokens", "Cost"] as const) { + for (const suffix of ["InputTokens", "OutputTokens", "CostNumerator"] as const) { assert( - role[`direct${suffix}`] === direct.reduce((sum, node) => sum + (node[`direct${suffix}`] as number), 0), + role[`direct${suffix}`] === exactSum(direct.map((node) => node[`direct${suffix}`] as number)), `role direct invariant failed: ${role.id}:${suffix}`, ); assert( role[`downstream${suffix}`] === - [...included.values()].reduce((sum, node) => sum + (node[`direct${suffix}`] as number), 0), + exactSum([...included.values()].map((node) => node[`direct${suffix}`] as number)), `role tree invariant failed: ${role.id}:${suffix}`, ); } } const run = rows.find((row) => row.id === "run" && row.kind === "run"); assert(run, "missing run cost"); + assert( + run.directInputTokens === 0 && run.directOutputTokens === 0 && run.directCostNumerator === 0, + "run direct invariant failed", + ); const roots = nodes.filter( (node) => !assignmentRows.find((assignment) => assignment.nodeId === node.id)?.parentNodeId, ); - for (const suffix of ["InputTokens", "OutputTokens", "Cost"] as const) + for (const suffix of ["InputTokens", "OutputTokens", "CostNumerator"] as const) assert( - run[`downstream${suffix}`] === roots.reduce((sum, node) => sum + (node[`downstream${suffix}`] as number), 0), + run[`downstream${suffix}`] === exactSum(roots.map((node) => node[`downstream${suffix}`] as number)), `run tree invariant failed: ${suffix}`, ); } @@ -1062,11 +1198,9 @@ function verifyProcessSamples(samples: unknown): void { } /** Strict verifier: expected set only, no links/extras, canonical bytes, hashes, and semantic joins. */ -export async function verifySwarmEvidence(directory: string, capability: SwarmEvidenceCapability): Promise { - const registration = registeredBundles.get(capability); - assert(registration, "issued swarm evidence capability is required"); +async function verifyExpectedSwarmEvidence(directory: string, expectedArtifactBundleId: string): Promise { + assert(/^[0-9a-f]{64}$/.test(expectedArtifactBundleId), "invalid trusted artifact bundle identity"); const root = await realpath(directory); - assert(root === registration.directory, "swarm evidence capability directory mismatch"); const names = (await readdir(root)).sort(); assert( canonicalJson(names) === canonicalJson([...ALL_EVIDENCE_FILES].sort()), @@ -1131,10 +1265,63 @@ export async function verifySwarmEvidence(directory: string, capability: SwarmEv "summary.json": await readFile(join(root, "summary.json"), "utf8"), }); assert(manifest.deterministicBundleId === deterministic, "deterministic bundle identity mismatch"); + assert(manifest.artifactBundleId === expectedArtifactBundleId, "trusted artifact bundle mismatch"); +} + +/** B00A accepts only an issued in-process writer capability. */ +export async function verifySwarmEvidence(directory: string, capability: SwarmEvidenceCapability): Promise { + const registration = registeredBundles.get(capability); + assert(registration, "issued swarm evidence capability is required"); + const root = await realpath(directory); + assert(root === registration.directory, "swarm evidence capability directory mismatch"); + try { + await verifyExpectedSwarmEvidence(root, registration.artifactBundleId); + } catch (error) { + if (error instanceof Error && error.message === "trusted artifact bundle mismatch") + throw new Error("issued swarm evidence capability bundle mismatch", { cause: error }); + throw error; + } +} + +/** + * B00B fresh-process entry point. It authenticates canonical commitment bytes + * against an explicitly registered public-key root before entering the shared + * expected-ID semantic verifier. No artifact-derived string is a trust input. + */ +export async function verifyAuthenticatedSwarmEvidence( + directory: string, + commitmentRaw: string, + trustRoot: SwarmEvidenceTrustRoot, +): Promise { + const publicKey = registeredTrustRoots.get(trustRoot); + assert(publicKey, "registered swarm evidence trust root is required"); + const commitment = parseCanonicalJson( + commitmentRaw, + "artifact commitment", + ) as Partial; + assert( + commitment.schemaVersion === SWARM_EVIDENCE_COMMITMENT_SCHEMA && + typeof commitment.artifactBundleId === "string" && + /^[0-9a-f]{64}$/.test(commitment.artifactBundleId) && + typeof commitment.signature === "string", + "invalid swarm evidence commitment", + ); + let signature: Buffer; + try { + signature = Buffer.from(commitment.signature, "base64"); + } catch { + throw new Error("invalid swarm evidence commitment signature"); + } assert( - manifest.artifactBundleId === registration.artifactBundleId, - "issued swarm evidence capability bundle mismatch", + verifySignature( + null, + Buffer.from(canonicalJson(swarmEvidenceCommitmentPayload(commitment.artifactBundleId))), + publicKey, + signature, + ), + "B00B_EVIDENCE_BAD_SIGNATURE", ); + await verifyExpectedSwarmEvidence(directory, commitment.artifactBundleId); } export function createFixedFanoutScenario(fanout: (typeof SUPPORTED_SWARM_FANOUTS)[number]): SwarmBenchmarkConfig { return { diff --git a/packages/coding-agent/test/swarm/swarm-production-integration.test.ts b/packages/coding-agent/test/swarm/swarm-production-integration.test.ts new file mode 100644 index 000000000..1f6c78fdd --- /dev/null +++ b/packages/coding-agent/test/swarm/swarm-production-integration.test.ts @@ -0,0 +1,635 @@ +/** Production-path coverage for the B00B test-only scripted provider. */ +import { generateKeyPairSync } from "node:crypto"; +import { mkdtemp, readdir, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Agent, type AgentOptions } from "@earendil-works/pi-agent-core"; +import { Type } from "@earendil-works/pi-ai"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { AgentSession } from "../../src/core/agent-session.js"; +import { + type AgentSessionRuntime, + type CreateAgentSessionRuntimeFactory, + createAgentSessionFromServices, + createAgentSessionRuntime, + createAgentSessionServices, +} from "../../src/core/agent-session-runtime.js"; +import { AuthStorage } from "../../src/core/auth-storage.js"; +import { convertToLlm } from "../../src/core/messages.js"; +import { ModelRegistry } from "../../src/core/model-registry.js"; +import { SessionManager } from "../../src/core/session-manager.js"; +import { SettingsManager } from "../../src/core/settings-manager.js"; +import { createTestResourceLoader } from "../utilities.js"; +import { + verifySignedProductionEvidenceFreshProcess, + writeSignedProductionEvidence, +} from "./production-evidence-adapter.js"; +import { createBarrier, createBarrierScriptedProvider, type ProviderScript } from "./production-scripted-provider.js"; + +const cleanups: Array<() => Promise | void> = []; +afterEach(async () => { + while (cleanups.length) await cleanups.pop()?.(); +}); + +const usage = (input: number, output: number, cacheRead = 0, cacheWrite = 0) => ({ + input, + output, + cacheRead, + cacheWrite, + totalTokens: input + output + cacheRead + cacheWrite, + cost: { + input: input * 0.000001, + output: output * 0.000002, + cacheRead: cacheRead * 0.0000001, + cacheWrite: cacheWrite * 0.0000002, + total: input * 0.000001 + output * 0.000002 + cacheRead * 0.0000001 + cacheWrite * 0.0000002, + }, +}); +const canaries = [ + "B00B-system-秘密", + "B00B-user-secret", + "B00B-thinking-secret", + "B00B-tool-args-secret", + "B00B-tool-result-secret", + "B00B-error-secret", +]; + +function provider(scripts: Record, expected: readonly string[]) { + const registered = createBarrierScriptedProvider({ + api: "b00b-scripted", + provider: "b00b-scripted", + barrier: { expected, timeoutMs: 10_000 }, + models: [ + { + id: "fixture-a", + responseModel: "fixture-a-resolved", + cost: { input: 1.1, output: 2.2, cacheRead: 0.1, cacheWrite: 0.2 }, + }, + { + id: "fixture-b", + responseModel: "fixture-b-resolved", + cost: { input: 3.3, output: 4.4, cacheRead: 0.3, cacheWrite: 0.4 }, + }, + { + id: "fixture-zero", + responseModel: "fixture-zero-resolved", + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + }, + ], + scripts, + }); + cleanups.push(() => registered.unregister()); + return registered; +} +function simple(requestId: string, options: Partial = {}): ProviderScript { + return { + requestId, + blocks: [{ type: "text", chunks: ["safe-", "output"] }], + usage: usage(11, 7), + responseModel: "fixture-a-resolved", + ...options, + }; +} +function agentFor( + model: ReturnType["models"][number], + tools: NonNullable["tools"] = [], +) { + return new Agent({ getApiKey: () => "fixture-key", initialState: { model, systemPrompt: canaries[0], tools } }); +} + +async function readTree(directory: string): Promise { + const names = await readdir(directory); + return (await Promise.all(names.map((name) => readFile(join(directory, name), "utf8")))).join("\n"); +} + +function providerRegistration(models: ReturnType["models"]) { + return { + baseUrl: models[0]!.baseUrl, + apiKey: "fixture-key", + api: models[0]!.api, + models: models.map((model) => ({ + id: model.id, + name: model.name, + api: model.api, + reasoning: model.reasoning, + input: model.input, + cost: model.cost, + contextWindow: model.contextWindow, + maxTokens: model.maxTokens, + baseUrl: model.baseUrl, + })), + }; +} + +/** + * Builds the same in-process runtime host used by production sessions. In + * particular, children are created by AgentSessionRuntime rather than injected + * AgentSession fixtures, so this reaches runRlmChild -> runtime -> agent-loop. + */ +async function runtimeForRlmFixture( + fixture: ReturnType, + directory: string, +): Promise { + const authStorage = AuthStorage.inMemory(); + const rootModel = fixture.models[0]!; + authStorage.setRuntimeApiKey(rootModel.provider, "fixture-key"); + const settingsManager = SettingsManager.inMemory({ retry: { enabled: false } }); + const registration = providerRegistration(fixture.models); + const createRuntime: CreateAgentSessionRuntimeFactory = async (runtimeOptions) => { + const services = await createAgentSessionServices({ + cwd: runtimeOptions.cwd, + agentDir: directory, + authStorage, + settingsManager, + telemetryDisabled: true, + resourceLoaderOptions: { + extensionFactories: [ + (pi) => { + pi.registerProvider(rootModel.provider, registration); + }, + ], + noSkills: true, + noPromptTemplates: true, + noThemes: true, + }, + }); + const result = await createAgentSessionFromServices({ + services, + sessionManager: runtimeOptions.sessionManager, + sessionStartEvent: runtimeOptions.sessionStartEvent, + ...runtimeOptions.sessionOptions, + }); + return { ...result, services, diagnostics: services.diagnostics }; + }; + return createAgentSessionRuntime(createRuntime, { + cwd: directory, + agentDir: directory, + sessionManager: SessionManager.create(directory, join(directory, "sessions")), + sessionOptions: { + model: rootModel, + noTools: "all", + includeGoals: false, + rlmDepth: 0, + rlmMaxDepth: 1, + }, + }); +} + +function requestIds(fanout: number, offset = 10): string[] { + return Array.from({ length: fanout }, (_, index) => `request-${String(index + offset).padStart(4, "0")}`); +} + +async function waitForTerminals(fixture: ReturnType, count: number): Promise { + await vi.waitFor( + () => + expect(fixture.observations().filter((observation) => observation.eventKinds.length > 0)).toHaveLength(count), + { timeout: 10_000, interval: 10 }, + ); +} + +describe("B00B production scripted provider", () => { + test("settles every held waiter as aborted on barrier timeout and removes its abort listeners", async () => { + vi.useFakeTimers(); + try { + const barrier = createBarrier(["request-0091", "request-0092"], 100); + const first = new AbortController(); + const second = new AbortController(); + const firstWait = barrier.wait("request-0091", first.signal); + const secondWait = barrier.wait("request-0092", second.signal); + const rejectedOpen = expect(barrier.open).rejects.toThrow("B00B_BARRIER_TIMEOUT"); + await vi.advanceTimersByTimeAsync(100); + await expect(Promise.all([firstWait, secondWait])).resolves.toEqual(["aborted", "aborted"]); + await rejectedOpen; + // A stale abort listener would have a second settlement path after timeout. + first.abort(); + second.abort(); + } finally { + vi.useRealTimers(); + } + }); + + test("settles every held waiter as aborted when the provider closes", async () => { + const barrier = createBarrier(["request-0093", "request-0094"], 10_000); + const first = new AbortController(); + const second = new AbortController(); + const firstWait = barrier.wait("request-0093", first.signal); + const secondWait = barrier.wait("request-0094", second.signal); + const rejectedOpen = expect(barrier.open).rejects.toThrow("B00B_BARRIER_CLOSED"); + barrier.close(); + await expect(Promise.all([firstWait, secondWait])).resolves.toEqual(["aborted", "aborted"]); + await rejectedOpen; + first.abort(); + second.abort(); + }); + test("settles wait calls made after release, close, or pre-abort without throwing", async () => { + const barrier = createBarrier(["request-0095", "request-0096"], 10_000); + const rejectedOpen = expect(barrier.open).rejects.toThrow("B00B_BARRIER_CLOSED"); + barrier.release(["request-0095"]); + await expect(barrier.wait("request-0095", undefined)).resolves.toBe("released"); + barrier.close(); + await rejectedOpen; + await expect(barrier.wait("request-0096", undefined)).resolves.toBe("aborted"); + + const preAborted = createBarrier(["request-0097"], 10_000); + const preAbortedOpen = expect(preAborted.open).rejects.toThrow("B00B_BARRIER_CLOSED"); + const controller = new AbortController(); + controller.abort(); + await expect(preAborted.wait("request-0097", controller.signal)).resolves.toBe("aborted"); + preAborted.close(); + await preAbortedOpen; + }); + test("registers through the real AI registry and holds a 1/4 fanout only as an observation barrier", async () => { + const ids = ["request-0001", "request-0002", "request-0003", "request-0004"] as const; + const fixture = provider(Object.fromEntries(ids.map((id) => [id, [simple(id, { waitForRelease: true })]])), ids); + const agents = ids.map((_id, index) => agentFor(fixture.models[index % 3]!)); + const events = agents.map(() => [] as string[]); + for (const [index, agent] of agents.entries()) { + agent.subscribe((event) => { + events[index]!.push(event.type); + }); + } + const runs = agents.map((agent, index) => + agent.prompt(`request-${String(index + 1).padStart(4, "0")} ${canaries[1]}`), + ); + await fixture.open; + const entries = fixture.observations(); + expect(entries).toHaveLength(4); + expect(entries.map((entry) => entry.requestId).sort()).toEqual([...ids]); + expect(entries.every((entry) => entry.eventKinds.length === 0)).toBe(true); + // This releases 2..4 while 1 remains held: no semaphore/queue sits before provider entry. + fixture.release(ids.slice(1)); + await Promise.all(runs.slice(1)); + expect(fixture.observations().find((entry) => entry.requestId === "request-0001")?.eventKinds).toEqual([]); + fixture.release([ids[0]]); + await runs[0]; + for (const types of events) { + expect(types.indexOf("message_start")).toBeLessThan(types.indexOf("message_update")); + expect(types.filter((type) => type === "message_end")).toHaveLength(2); // user plus one assistant terminal + } + expect( + fixture.observations().every((entry) => entry.terminal === "done" && entry.eventKinds.at(-1) === "done"), + ).toBe(true); + }); + + test("uses exact thinking/text/tool stream events, executes one tool turn, and attributes resolved model and terminal usage", async () => { + const id = "request-0005"; + const fixture = provider( + { + [id]: [ + { + requestId: id, + waitForRelease: true, + responseId: "response-safe-0005", + responseModel: "fixture-b-resolved", + stopReason: "toolUse", + usage: usage(101, 17, 0, 101), + blocks: [ + { type: "thinking", chunks: [canaries[2].slice(0, 8), canaries[2].slice(8)] }, + { type: "text", chunks: ["call-", "tool"] }, + { + type: "toolCall", + id: "tool-0005", + name: "fixture_tool", + argumentChunks: [`{"value":"${canaries[3]}"}`], + }, + ], + }, + { + requestId: id, + responseModel: "fixture-b-resolved", + usage: usage(102, 9, 101, 1), + blocks: [{ type: "text", chunks: ["final-", "safe"] }], + }, + ], + }, + [id], + ); + let toolCalls = 0; + const tool = { + name: "fixture_tool", + label: "fixture tool", + description: "test-only", + parameters: Type.Object({ value: Type.String() }), + execute: async () => { + toolCalls++; + return { content: [{ type: "text" as const, text: canaries[4] }], details: {}, terminate: false }; + }, + }; + const agent = agentFor(fixture.models[1]!, [tool]); + const lifecycle: string[] = []; + agent.subscribe((event) => { + lifecycle.push(event.type); + }); + const run = agent.prompt(`${id} ${canaries[1]}`); + await fixture.open; + fixture.release([id]); + await run; + expect(toolCalls).toBe(1); + const observed = fixture.observations(); + expect(observed).toHaveLength(2); + expect(observed[0]?.eventKinds).toEqual([ + "start", + "thinking_start", + "thinking_delta", + "thinking_delta", + "thinking_end", + "text_start", + "text_delta", + "text_delta", + "text_end", + "toolcall_start", + "toolcall_delta", + "toolcall_end", + "done", + ]); + expect(observed[1]?.eventKinds).toEqual(["start", "text_start", "text_delta", "text_delta", "text_end", "done"]); + expect(observed.map((item) => item.responseModel)).toEqual(["fixture-b-resolved", "fixture-b-resolved"]); + expect(observed.map((item) => item.usage?.cacheRead)).toEqual([0, 101]); + expect(lifecycle.filter((type) => type === "message_end")).toHaveLength(4); // user, assistant, tool result, assistant + const final = agent.state.messages.at(-1); + expect(final).toMatchObject({ + role: "assistant", + responseModel: "fixture-b-resolved", + usage: usage(102, 9, 101, 1), + }); + }); + + test("isolates abort and upstream 429 from released siblings without client-side rate limiting", async () => { + const ids = ["request-0006", "request-0007", "request-0008"] as const; + const fixture = provider( + { + [ids[0]]: [simple(ids[0], { waitForRelease: true })], + [ids[1]]: [ + { + requestId: ids[1], + waitForRelease: true, + upstreamStatus: 429, + errorCode: "upstream-429", + usage: usage(23, 0, 5, 0), + }, + ], + [ids[2]]: [ + simple(ids[2], { waitForRelease: true, responseModel: "fixture-zero-resolved", usage: usage(3, 2) }), + ], + }, + ids, + ); + const agents = [agentFor(fixture.models[0]!), agentFor(fixture.models[1]!), agentFor(fixture.models[2]!)]; + const runs = agents.map((agent, index) => agent.prompt(ids[index]!)); + await fixture.open; + agents[0]!.abort(); + fixture.release([ids[1], ids[2]]); + await Promise.all(runs); + const observed = fixture.observations(); + expect(observed.find((item) => item.requestId === ids[0])).toMatchObject({ + terminal: "aborted", + signalAborted: true, + eventKinds: ["error"], + }); + expect(observed.find((item) => item.requestId === ids[1])).toMatchObject({ + upstreamStatus: 429, + terminal: "error", + eventKinds: ["error"], + }); + expect(observed.find((item) => item.requestId === ids[2])).toMatchObject({ + terminal: "done", + responseModel: "fixture-zero-resolved", + }); + expect(observed.filter((item) => item.requestId === ids[0])[0]?.eventKinds).toHaveLength(1); + }); + + test.each([1, 4, 16, 64])( + "admits a real RLM fanout of %i children before the provider barrier opens", + async (fanout) => { + const ids = requestIds(fanout, fanout === 1 ? 20 : fanout * 100); + const fixture = provider( + Object.fromEntries(ids.map((id) => [id, [simple(id, { waitForRelease: true })]])), + ids, + ); + const directory = await mkdtemp(join(tmpdir(), `b00b-rlm-${fanout}-`)); + const runtime = await runtimeForRlmFixture(fixture, directory); + cleanups.push(async () => { + await runtime.dispose(); + await rm(directory, { recursive: true, force: true }); + }); + + const handles = await Promise.all( + ids.map((id, index) => + runtime.session.runRlmChild(id, { + name: `worker-${String(index + 1).padStart(4, "0")}`, + model: `${fixture.models[index % fixture.models.length]!.provider}/${fixture.models[index % fixture.models.length]!.id}`, + }), + ), + ); + // Admission is detached: every handle returns before an entry is allowed + // to leave its observation latch. This is deliberately not a fanout + // semaphore or permit queue. + expect(handles).toHaveLength(fanout); + expect(new Set(handles.map((handle) => handle.rlm_child_id)).size).toBe(fanout); + await fixture.open; + const entered = fixture.observations(); + expect(entered).toHaveLength(fanout); + expect(entered.map((entry) => entry.requestId).sort()).toEqual([...ids].sort()); + expect(entered.every((entry) => entry.eventKinds.length === 0 && entry.attempt === 1)).toBe(true); + expect(entered.map((entry) => entry.sequence)).toEqual( + Array.from({ length: fanout }, (_, index) => index + 1), + ); + + // Fast siblings complete while the held first request has not emitted a + // provider event. This proves the latch observes independently admitted + // streams rather than serializing their execution. + fixture.release(ids.slice(1)); + if (fanout > 1) await waitForTerminals(fixture, fanout - 1); + expect(fixture.observations().find((entry) => entry.requestId === ids[0])?.eventKinds).toEqual([]); + fixture.release([ids[0]!]); + await waitForTerminals(fixture, fanout); + expect(fixture.observations().every((entry) => entry.terminal === "done")).toBe(true); + }, + 20_000, + ); + + test("uses the RLM runtime child host for cancel, real scripted 429, and sibling isolation", async () => { + const ids = ["request-0701", "request-0702", "request-0703"] as const; + const fixture = provider( + { + [ids[0]]: [simple(ids[0], { waitForRelease: true })], + [ids[1]]: [ + { + requestId: ids[1], + waitForRelease: true, + upstreamStatus: 429, + errorCode: "upstream-429", + usage: usage(23, 0, 5, 0), + }, + ], + [ids[2]]: [ + simple(ids[2], { waitForRelease: true, responseModel: "fixture-zero-resolved", usage: usage(3, 2) }), + ], + }, + ids, + ); + const directory = await mkdtemp(join(tmpdir(), "b00b-rlm-isolation-")); + const runtime = await runtimeForRlmFixture(fixture, directory); + cleanups.push(async () => { + await runtime.dispose(); + await rm(directory, { recursive: true, force: true }); + }); + const handles = await Promise.all( + ids.map((id, index) => + runtime.session.runRlmChild(id, { + name: `worker-${String(index + 701).padStart(4, "0")}`, + model: `${fixture.models[index]!.provider}/${fixture.models[index]!.id}`, + }), + ), + ); + await fixture.open; + expect(fixture.observations().every((entry) => entry.eventKinds.length === 0)).toBe(true); + await vi.waitFor(() => expect(runtime.session.getRlmChildSession(handles[0]!.rlm_child_id)).toBeDefined()); + expect(runtime.session.cancelRlmChildRun(handles[0]!.rlm_child_id, "B00B_CANCELLED")).toBe(true); + fixture.release([ids[1], ids[2]]); + await waitForTerminals(fixture, 3); + const observed = fixture.observations(); + expect(observed.find((entry) => entry.requestId === ids[0])).toMatchObject({ + terminal: "aborted", + signalAborted: true, + eventKinds: ["error"], + }); + expect(observed.find((entry) => entry.requestId === ids[1])).toMatchObject({ + upstreamStatus: 429, + terminal: "error", + eventKinds: ["error"], + }); + expect(observed.find((entry) => entry.requestId === ids[2])).toMatchObject({ + terminal: "done", + responseModel: "fixture-zero-resolved", + }); + // The fixture’s disabled retry setting is an explicit per-child policy: + // no synthetic local 429 and no shared client-side limiter intervene. + expect(observed.filter((entry) => entry.requestId === ids[1])).toHaveLength(1); + }); + + test("projects immutable real RLM observations into signed B00A evidence and verifies in a fresh process", async () => { + const id = "request-0801"; + const fixture = provider( + { + [id]: [ + simple(id, { waitForRelease: true, responseModel: "fixture-b-resolved", usage: usage(101, 13, 7, 3) }), + ], + }, + [id], + ); + const directory = await mkdtemp(join(tmpdir(), "b00b-rlm-evidence-")); + const artifactDirectory = await mkdtemp(join(tmpdir(), "b00b-rlm-artifact-")); + const trustDirectory = await mkdtemp(join(tmpdir(), "b00b-rlm-trust-")); + const runtime = await runtimeForRlmFixture(fixture, directory); + cleanups.push(async () => { + await runtime.dispose(); + await rm(directory, { recursive: true, force: true }); + await rm(artifactDirectory, { recursive: true, force: true }); + await rm(trustDirectory, { recursive: true, force: true }); + }); + const [handle] = await Promise.all([ + runtime.session.runRlmChild(id, { name: "worker-0801", model: `${fixture.models[1]!.provider}/fixture-b` }), + ]); + expect(handle).toMatchObject({ model: `${fixture.models[1]!.provider}/fixture-b` }); + await fixture.open; + fixture.release([id]); + await waitForTerminals(fixture, 1); + const observation = fixture.observations()[0]!; + expect(observation.requested).toMatchObject({ provider: "b00b-scripted", model: "fixture-b" }); + expect(observation.responseModel).toBe("fixture-b-resolved"); + expect(observation.usage).toMatchObject({ input: 101, output: 13, cacheRead: 7, cacheWrite: 3 }); + const keys = generateKeyPairSync("ed25519"); + const written = await writeSignedProductionEvidence( + artifactDirectory, + trustDirectory, + { + scenario: "rlm-real-path", + priceCard: { + version: "fixture-price-card-v1", + inputMicroCurrencyPerMillionMicroTokens: 17, + outputMicroCurrencyPerMillionMicroTokens: 29, + }, + attempts: [ + { + requestId: observation.requestId as `request-${string}`, + attempt: observation.attempt, + requested: { provider: observation.requested.provider, model: observation.requested.model }, + resolved: { + api: "b00b-scripted", + provider: observation.requested.provider, + model: observation.requested.model, + responseModel: observation.responseModel!, + }, + terminal: observation.terminal, + usage: { + inputMicroTokens: observation.usage!.input, + outputMicroTokens: observation.usage!.output, + cacheReadMicroTokens: observation.usage!.cacheRead, + cacheWriteMicroTokens: observation.usage!.cacheWrite, + }, + }, + ], + }, + keys.privateKey, + ); + await verifySignedProductionEvidenceFreshProcess( + artifactDirectory, + written.commitmentPath, + keys.publicKey.export({ type: "spki", format: "pem" }).toString(), + ); + expect(written.artifactBundleId).toMatch(/^[a-f0-9]{64}$/); + }); + + test("runs through AgentSession.promptAndWait with a registered provider and writes no canary or network fixture", async () => { + const id = "request-0009"; + const fixture = provider({ [id]: [simple(id, { waitForRelease: true, responseModel: "fixture-a-resolved" })] }, [ + id, + ]); + const directory = await mkdtemp(join(tmpdir(), "b00b-session-")); + const auth = AuthStorage.inMemory(); + const model = fixture.models[0]!; + auth.setRuntimeApiKey(model.provider, "fixture-key"); + const registry = ModelRegistry.inMemory(auth); + registry.registerProvider(model.provider, { + baseUrl: model.baseUrl, + apiKey: "fixture-key", + api: model.api, + models: fixture.models.map((candidate) => ({ + id: candidate.id, + name: candidate.name, + api: candidate.api, + reasoning: candidate.reasoning, + input: candidate.input, + cost: candidate.cost, + contextWindow: candidate.contextWindow, + maxTokens: candidate.maxTokens, + baseUrl: candidate.baseUrl, + })), + }); + const agent = new Agent({ + getApiKey: () => "fixture-key", + initialState: { model, systemPrompt: canaries[0], tools: [] }, + convertToLlm, + }); + const session = new AgentSession({ + agent, + cwd: directory, + modelRegistry: registry, + sessionManager: SessionManager.inMemory(directory), + settingsManager: SettingsManager.inMemory(), + resourceLoader: createTestResourceLoader(), + }); + cleanups.push(async () => { + session.dispose(); + await rm(directory, { recursive: true, force: true }); + }); + const run = session.promptAndWait(`${id} ${canaries[1]}`); + await fixture.open; + fixture.release([id]); + await run; + expect(session.messages.at(-1)).toMatchObject({ role: "assistant", responseModel: "fixture-a-resolved" }); + const disk = await readTree(directory); + for (const canary of canaries) expect(disk).not.toContain(canary); + }); +});