diff --git a/packages/coding-agent/src/cli/daemon-ps.ts b/packages/coding-agent/src/cli/daemon-ps.ts index 75020d755..0758e9a6e 100644 --- a/packages/coding-agent/src/cli/daemon-ps.ts +++ b/packages/coding-agent/src/cli/daemon-ps.ts @@ -934,8 +934,11 @@ async function forceStopTrackedWorkers( const failures: string[] = []; for (const worker of findTrackedWorkers(supervisorSocketPath)) { const { descriptor } = worker; - const pid = descriptor.pid!; - let cleanupWorkerRecords = await stopTrackedProcess(pid, descriptor.processStartId, assertAdmission); + const process = descriptor.process; + // Legacy and processless descriptors are display/recovery evidence, never a kill target. + if (!process) continue; + const { pid, processStartId } = process; + let cleanupWorkerRecords = await stopTrackedProcess(pid, processStartId, assertAdmission); if (!cleanupWorkerRecords) { failures.push(`could not safely stop worker ${descriptor.workerId} (pid ${pid})`); } @@ -1034,9 +1037,11 @@ function isTrackedWorkerDescriptor(value: unknown): value is DaemonWorkerDescrip descriptor.lifecycle !== "passivated" && typeof descriptor.supervisorSocketPath === "string" && typeof descriptor.workerId === "string" && - Number.isInteger(descriptor.pid) && - (descriptor.pid ?? 0) > 0 && - (descriptor.processStartId === undefined || typeof descriptor.processStartId === "string") && + !!descriptor.process && + Number.isInteger(descriptor.process.pid) && + descriptor.process.pid > 0 && + typeof descriptor.process.processStartId === "string" && + !!descriptor.process.processStartId && typeof descriptor.socketPath === "string" && typeof descriptor.recoveryJournalPath === "string" ); diff --git a/packages/coding-agent/src/core/agent-session-runtime.ts b/packages/coding-agent/src/core/agent-session-runtime.ts index 198b628f9..eb3870634 100644 --- a/packages/coding-agent/src/core/agent-session-runtime.ts +++ b/packages/coding-agent/src/core/agent-session-runtime.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import { copyFileSync, existsSync, mkdirSync } from "node:fs"; import { basename, join, resolve } from "node:path"; import type { AgentSession } from "./agent-session.js"; @@ -56,6 +57,10 @@ export interface AgentSessionRuntimeMetadata { parentSessionId?: string; parentSessionFile?: string; rlmChildId?: string; + /** Daemon worker incarnation; absent for legacy and inline top-level runtimes. */ + generation?: string; + /** Required for C01-created subagent runtimes; internal only. */ + assignmentId?: string; rlmParentNodeId?: string; /** Runtime restored from an already-persisted completed registry entry. */ rehydratedCompleted?: boolean; @@ -90,6 +95,8 @@ export class AgentSessionRuntime implements SubagentRuntimeHost { private beforeSessionInvalidate?: () => void; private subagentRuntimeHost?: SubagentRuntimeHost; private subagentRuntimes = new Map(); + /** Assignment currently owning each compatibility child-id map entry. */ + private subagentRuntimeAssignments = new Map(); private disposePromise?: Promise; constructor( @@ -298,6 +305,7 @@ export class AgentSessionRuntime implements SubagentRuntimeHost { private async disposeSubagentRuntimes(): Promise { const runtimes = [...this.subagentRuntimes.values()]; this.subagentRuntimes.clear(); + this.subagentRuntimeAssignments.clear(); let disposeError: unknown; for (const runtime of runtimes) { try { @@ -340,6 +348,7 @@ export class AgentSessionRuntime implements SubagentRuntimeHost { rlmDepth: options.rlmDepth, }); } + const assignmentId = options.assignmentId ?? randomUUID(); const runtime = await this.scopedBuild(() => createAgentSessionRuntime(this.createRuntime, { cwd: sessionManager.getCwd(), @@ -369,6 +378,7 @@ export class AgentSessionRuntime implements SubagentRuntimeHost { parentSessionId: options.parentSession.sessionId, parentSessionFile: options.parentSession.sessionFile, rlmChildId: options.id, + assignmentId: assignmentId, rlmParentNodeId: options.rlmParentNodeId, prompt: options.prompt, spawnCode: options.spawnCode, @@ -377,9 +387,13 @@ export class AgentSessionRuntime implements SubagentRuntimeHost { }), ); this.subagentRuntimes.set(options.id, runtime); + this.subagentRuntimeAssignments.set(options.id, assignmentId); try { await runtime.session.bindExtensions({}); - if (options.parentSession.getRlmChildRunStatus(options.id) === "cancelled") { + if ( + this.subagentRuntimeAssignments.get(options.id) !== assignmentId || + options.parentSession.getRlmChildRunStatus(options.id) === "cancelled" + ) { throw new Error("RLM subagent startup was cancelled"); } if (runtime.session.sessionName !== options.sessionName) { @@ -387,27 +401,35 @@ export class AgentSessionRuntime implements SubagentRuntimeHost { } options.onSessionPublished?.(runtime.session); } catch (error) { - this.subagentRuntimes.delete(options.id); + if ( + this.subagentRuntimes.get(options.id) === runtime && + this.subagentRuntimeAssignments.get(options.id) === assignmentId + ) { + this.subagentRuntimes.delete(options.id); + this.subagentRuntimeAssignments.delete(options.id); + } await runtime.dispose(); throw error; } return runtime; } - async deleteRlmSubagentRuntime(childId: string, session: AgentSession): Promise { + async deleteRlmSubagentRuntime(childId: string, childSession?: AgentSession, assignmentId?: string): Promise { const runtime = this.subagentRuntimes.get(childId); - if (!runtime) { - await session.disposeAsync(); + const currentAssignment = this.subagentRuntimeAssignments.get(childId); + // Inline runtimes have no durable daemon registry. Preserve direct delete + // compatibility, but a named C01 assignment fences stale callbacks. + if (!runtime || (assignmentId !== undefined && currentAssignment !== assignmentId)) { + await childSession?.disposeAsync(); return; } this.subagentRuntimes.delete(childId); - const shouldDisposeStaleSession = runtime.session !== session; + this.subagentRuntimeAssignments.delete(childId); + const shouldDisposeStaleSession = !!childSession && runtime.session !== childSession; try { await runtime.dispose(); } finally { - if (shouldDisposeStaleSession) { - await session.disposeAsync(); - } + if (shouldDisposeStaleSession) await childSession?.disposeAsync(); } } diff --git a/packages/coding-agent/src/core/agent-session.ts b/packages/coding-agent/src/core/agent-session.ts index e99462eab..25fdd1e35 100644 --- a/packages/coding-agent/src/core/agent-session.ts +++ b/packages/coding-agent/src/core/agent-session.ts @@ -739,6 +739,27 @@ export interface SessionActionRecoverySnapshot { actions: SessionActionRecoveryAction[]; } +/** Reject recovery data the scheduler cannot admit before it is allowed to mutate state. */ +export function validateSessionActionRecoverySnapshot( + snapshot: SessionActionRecoverySnapshot, + existingActionIds: Iterable = [], +): void { + if (snapshot.formatVersion !== SESSION_ACTION_RECOVERY_FORMAT_VERSION) { + throw new Error(`Unsupported session action recovery format version: ${snapshot.formatVersion}`); + } + const actionIds = new Set(existingActionIds); + for (const recovered of snapshot.actions) { + if (actionIds.has(recovered.id)) throw new Error(`Duplicate session action id: ${recovered.id}`); + actionIds.add(recovered.id); + if ( + recovered.payload.kind === "turn" && + recovered.payload.records.some((record) => record.ownerActionId !== recovered.id) + ) { + throw new Error(`Session action ${recovered.id} has invalid delivery correlation`); + } + } +} + function cloneCustomMessage(message: CustomMessage): CustomMessage { return { ...message, @@ -931,6 +952,8 @@ type AutonomousRuntimeSnapshot = Pick< interface RlmChildRun { id: string; + /** UUID attempt fence, minted before this run is visible to any host. */ + assignmentId: string; prompt: string; sessionName: string; sessionDir: string; @@ -1216,6 +1239,8 @@ export class AgentSession { // Inline mode keeps finished child sessions so the inspector can still read them; // the daemon does the same by leaving the child session resident in its registry. private _rlmChildSessions = new Map(); + private _rlmChildSessionAssignments = new Map(); + // Tombstones are attempt-scoped: a late A deletion must never hide B. private _deletedRlmChildIds = new Set(); // Failed explicit deletes stay hidden from listings but retain their original // selector so a later delete can retry cleanup without orphaning the runtime. @@ -1224,12 +1249,14 @@ export class AgentSession { string, { subagent: RlmSubagentRegistryEntry; + assignmentId?: string; promise: Promise; } >(); // Kept alive for retained children so nested updates (e.g. a grandchild cancel) // still forward to root; torn down when the retained child is disposed. private _rlmChildUnsubscribes = new Map void>(); + private _rlmChildUnsubscribeAssignments = new Map(); /** Latest recap for this session, written by the daemon summarizer; read by a parent to label its child snapshots. */ private _currentRecap?: string; @@ -4902,20 +4929,18 @@ export class AgentSession { }); } - async restoreSessionActions(snapshot: SessionActionRecoverySnapshot): Promise { - if (snapshot.formatVersion !== SESSION_ACTION_RECOVERY_FORMAT_VERSION) { - throw new Error(`Unsupported session action recovery format version: ${snapshot.formatVersion}`); - } - const actionIds = new Set(this._actionStore.ownedActions().map((action) => action.id)); + /** Validates recovery data against immutable snapshot and current scheduler invariants. */ + validateSessionActionRecoverySnapshot(snapshot: SessionActionRecoverySnapshot): void { + validateSessionActionRecoverySnapshot( + snapshot, + this._actionStore.ownedActions().map((action) => action.id), + ); + } + + /** Restores queued work and returns the exact durable IDs admitted to the scheduler. */ + async restoreSessionActions(snapshot: SessionActionRecoverySnapshot): Promise { + this.validateSessionActionRecoverySnapshot(snapshot); const actions = snapshot.actions.map((recovered): QueuedSessionAction => { - if (actionIds.has(recovered.id)) throw new Error(`Duplicate session action id: ${recovered.id}`); - actionIds.add(recovered.id); - if ( - recovered.payload.kind === "turn" && - recovered.payload.records.some((record) => record.ownerActionId !== recovered.id) - ) { - throw new Error(`Session action ${recovered.id} has invalid delivery correlation`); - } const payload: PreparedTurnPayload | PreparedCommandPayload = recovered.payload.kind === "turn" ? { @@ -4984,7 +5009,7 @@ export class AgentSession { }; }); for (const action of actions) this._admitSessionInput(action, { restore: true }); - return actions.length; + return actions.map((action) => action.id); } private _restoreSessionCommand( @@ -8928,6 +8953,7 @@ export class AgentSession { private _createRlmSubagentRuntimeOptions(options: { id: string; + assignmentId: string; prompt: string; sessionName: string; spawnCode?: string; @@ -8937,6 +8963,7 @@ export class AgentSession { return { parentSession: this, id: options.id, + assignmentId: options.assignmentId, prompt: options.prompt, sessionName: options.sessionName, spawnCode: options.spawnCode, @@ -9103,7 +9130,7 @@ export class AgentSession { const subagents: RlmListSubagentsResult["subagents"] = []; const recorded = new Set(); for (const run of this._activeRlmChildRuns.values()) { - if (this._deletingRlmChildren.has(run.id) || run.detachedDeletion || run.status === "cancelled") { + if (this._isRlmChildDeleting(run.id, run.assignmentId) || run.detachedDeletion || run.status === "cancelled") { continue; } const daemonChild = daemonChildren.get(run.id); @@ -9119,7 +9146,7 @@ export class AgentSession { } for (const [childId, childSession] of this._rlmChildSessions) { if ( - this._deletingRlmChildren.has(childId) || + this._isRlmChildDeleting(childId, this._rlmChildSessionAssignments.get(childId)) || recorded.has(childId) || this._rlmChildCleanupFailures.has(childId) ) { @@ -9144,8 +9171,8 @@ export class AgentSession { for (const [childId, daemonChild] of daemonChildren) { if ( recorded.has(childId) || - this._deletingRlmChildren.has(childId) || - this._deletedRlmChildIds.has(childId) || + this._isRlmChildDeleting(childId, this._rlmChildSessionAssignments.get(childId)) || + this._deletedRlmChildIds.has(this._rlmAssignmentKey(childId, this._currentRlmAssignment(childId))) || this._rlmChildCleanupFailures.has(childId) || !daemonChild.sessionDir ) { @@ -9296,39 +9323,63 @@ export class AgentSession { subagent: RlmSubagentRegistryEntry, startDeletion: () => Promise, ): Promise { - const existing = this._deletingRlmChildren.get(subagent.rlm_child_id); + const assignmentId = this._currentRlmAssignment(subagent.rlm_child_id); + const key = this._rlmAssignmentKey(subagent.rlm_child_id, assignmentId); + const existing = this._deletingRlmChildren.get(key); if (existing) return existing.promise; const deletion = Promise.resolve().then(startDeletion); - this._deletingRlmChildren.set(subagent.rlm_child_id, { - subagent, - promise: deletion, - }); + this._deletingRlmChildren.set(key, { subagent, assignmentId, promise: deletion }); try { return await deletion; } finally { - if (this._deletingRlmChildren.get(subagent.rlm_child_id)?.promise === deletion) { - this._deletingRlmChildren.delete(subagent.rlm_child_id); - } + if (this._deletingRlmChildren.get(key)?.promise === deletion) this._deletingRlmChildren.delete(key); } } - private _deleteRlmSubagentSession(childId: string, session?: AgentSession): Promise { + private _deleteRlmSubagentSession(childId: string, assignmentId?: string, session?: AgentSession): Promise { if (this._subagentRuntimeHost) { - return this._subagentRuntimeHost.deleteRlmSubagentRuntime(childId, session); + // Preserve the established assignment-aware host ABI. A daemon host treats + // assignment-less deletes as explicit durable migration, never as callback authority. + return this._subagentRuntimeHost.assignmentIdentityFenced + ? this._subagentRuntimeHost.deleteRlmSubagentRuntime(childId, session, assignmentId) + : this._subagentRuntimeHost.deleteRlmSubagentRuntime(childId, session); } return session?.disposeAsync() ?? Promise.resolve(); } - private _removeRlmSubagentTracking(childId: string, run?: RlmChildRun): void { - run?.unsubscribe?.(); - this._rlmChildUnsubscribes.get(childId)?.(); - this._rlmChildUnsubscribes.delete(childId); - this._rlmChildSessions.delete(childId); - this._rlmChildCleanupFailures.delete(childId); - if (!run || this._activeRlmChildRuns.get(childId) === run) { + private _rlmAssignmentKey(childId: string, assignmentId?: string): string { + return `${childId}\u0000${assignmentId ?? "legacy"}`; + } + + private _currentRlmAssignment(childId: string): string | undefined { + return this._activeRlmChildRuns.get(childId)?.assignmentId ?? this._rlmChildSessionAssignments.get(childId); + } + + private _isRlmChildDeleting(childId: string, assignmentId?: string): boolean { + return this._deletingRlmChildren.has(this._rlmAssignmentKey(childId, assignmentId)); + } + + private _removeRlmSubagentTracking(childId: string, run?: RlmChildRun, expectedAssignmentId?: string): void { + const assignmentId = expectedAssignmentId ?? run?.assignmentId ?? this._rlmChildSessionAssignments.get(childId); + // A callback which cannot name its assignment is a legacy/display path and has + // no authority to mutate a C01 child incarnation. + if (!assignmentId) return; + if (run && run.assignmentId !== assignmentId) return; + if (this._rlmChildUnsubscribeAssignments.get(childId) === assignmentId) { + this._rlmChildUnsubscribes.get(childId)?.(); + this._rlmChildUnsubscribes.delete(childId); + this._rlmChildUnsubscribeAssignments.delete(childId); + } + if (this._rlmChildSessionAssignments.get(childId) === assignmentId) { + this._rlmChildSessions.delete(childId); + this._rlmChildSessionAssignments.delete(childId); + this._rlmChildCleanupFailures.delete(childId); + } + if (this._activeRlmChildRuns.get(childId)?.assignmentId === assignmentId) { this._activeRlmChildRuns.delete(childId); } if (run) { + run.unsubscribe?.(); run.abort = noopRlmChildAbort; run.unsubscribe = undefined; run.session = undefined; @@ -9360,30 +9411,40 @@ export class AgentSession { } const liveSession = run.session; if (run.status === "error" && !liveSession && run.settled) { - this._deletedRlmChildIds.add(childId); - this._removeRlmSubagentTracking(childId, run); + this._deletedRlmChildIds.add(this._rlmAssignmentKey(childId, run.assignmentId)); + this._removeRlmSubagentTracking(childId, run, run.assignmentId); return { subagent }; } if (liveSession) { try { - await this._deleteRlmSubagentSession(childId, liveSession); + await this._deleteRlmSubagentSession(childId, run.assignmentId, liveSession); } catch (error) { if (this._disposed || this._disposing) { - this._removeRlmSubagentTracking(childId, run); + this._removeRlmSubagentTracking(childId, run, run.assignmentId); void liveSession.disposeAsync().catch(() => undefined); throw error; } - this._rlmChildSessions.set(childId, liveSession); - this._rlmChildCleanupFailures.set(childId, subagent); - if (run.unsubscribe) this._rlmChildUnsubscribes.set(childId, run.unsubscribe); - this._activeRlmChildRuns.delete(childId); + // Preserve the failed cleanup only if this assignment is still current. + // A late A delete must never re-install A over a reused B selector. + if (this._activeRlmChildRuns.get(childId) === run) { + this._rlmChildSessions.set(childId, liveSession); + this._rlmChildSessionAssignments.set(childId, run.assignmentId); + this._rlmChildCleanupFailures.set(childId, subagent); + if (run.unsubscribe) { + this._rlmChildUnsubscribes.set(childId, run.unsubscribe); + this._rlmChildUnsubscribeAssignments.set(childId, run.assignmentId); + } + this._activeRlmChildRuns.delete(childId); + } run.abort = noopRlmChildAbort; run.unsubscribe = undefined; run.session = undefined; throw error; } - this._deletedRlmChildIds.add(childId); - this._removeRlmSubagentTracking(childId, run); + if (this._activeRlmChildRuns.get(childId) === run) { + this._deletedRlmChildIds.add(this._rlmAssignmentKey(childId, run.assignmentId)); + this._removeRlmSubagentTracking(childId, run, run.assignmentId); + } return { subagent }; } @@ -9391,25 +9452,38 @@ export class AgentSession { // deletion immediately, but retain the cancelled run as a hidden tombstone // until startup settles so selectors cannot be reused underneath it. run.detachedDeletion = subagent; - this._deletedRlmChildIds.add(childId); + if (this._activeRlmChildRuns.get(childId) === run) { + this._deletedRlmChildIds.add(this._rlmAssignmentKey(childId, run.assignmentId)); + } return { subagent }; } this._emitRlmSubagentRemoval(subagent); const retained = this._rlmChildSessions.get(childId); + // Capture before the host await: an old callback may synchronously change + // a compatibility map, but it must not make this explicit deletion forget + // the incarnation it admitted. + const retainedAssignmentId = this._rlmChildSessionAssignments.get(childId); try { - await this._deleteRlmSubagentSession(childId, retained); + await this._deleteRlmSubagentSession(childId, retainedAssignmentId, retained); } catch (error) { if (this._disposed || this._disposing) { - this._removeRlmSubagentTracking(childId); + this._removeRlmSubagentTracking(childId, undefined, this._rlmChildSessionAssignments.get(childId)); void retained?.disposeAsync().catch(() => undefined); } else { this._rlmChildCleanupFailures.set(childId, subagent); } throw error; } - this._deletedRlmChildIds.add(childId); - this._removeRlmSubagentTracking(childId); + if (retainedAssignmentId) { + this._deletedRlmChildIds.add(this._rlmAssignmentKey(childId, retainedAssignmentId)); + this._removeRlmSubagentTracking(childId, undefined, retainedAssignmentId); + } else { + // Display-only legacy hosts never expose an assignment. This explicit + // user deletion still hides their public row, but it grants no authority + // to any asynchronous callback because none can name this legacy key. + this._deletedRlmChildIds.add(this._rlmAssignmentKey(childId)); + } return { subagent }; } @@ -9419,13 +9493,45 @@ export class AgentSession { * the child) when the parent is already tearing down, so the caller can drop the * matching event forwarder too. */ - registerRlmChildSession(childId: string, session: AgentSession, unsubscribe?: () => void): boolean { + registerRlmChildSession( + childId: string, + session: AgentSession, + unsubscribe?: () => void, + assignmentId?: string, + ): boolean { + const run = this._activeRlmChildRuns.get(childId); + // Older in-process hosts may register a freshly restored child directly. + // Mint an internal assignment for that synchronous compatibility path; all + // asynchronous C01 callbacks supply their captured assignment explicitly. + const expectedAssignmentId = assignmentId ?? run?.assignmentId ?? randomUUID(); + const retainedAssignmentId = this._rlmChildSessionAssignments.get(childId); + // A live run must name its immutable assignment. Hydration has no live run, + // but it has already persisted a fresh assignment; it may bind only an empty + // slot (or its own prior binding), never a selector reused by another child. + if ( + !expectedAssignmentId || + (run + ? run.assignmentId !== expectedAssignmentId + : retainedAssignmentId !== undefined && retainedAssignmentId !== expectedAssignmentId) + ) + return false; // A child can finish concurrently while the parent is (or has) torn down; don't // resurrect the map (it would never be disposed), just drop the child now. - if (this._deletingRlmChildren.has(childId) || this._deletedRlmChildIds.has(childId)) { + if ( + this._isRlmChildDeleting(childId, expectedAssignmentId) || + this._deletedRlmChildIds.has(this._rlmAssignmentKey(childId, expectedAssignmentId)) + ) { return false; } - if (this._subagentRuntimeHost?.completeRlmSubagentRuntime?.(childId, session) === false) { + const completeRuntime = this._subagentRuntimeHost?.completeRlmSubagentRuntime; + // Pre-C01 embedded hosts accepted (childId, session). They are display-only + // compatibility shims; daemon-owned hosts always receive and verify assignmentId. + const completionResult = completeRuntime + ? this._subagentRuntimeHost?.assignmentIdentityFenced + ? completeRuntime(childId, session, expectedAssignmentId) + : completeRuntime(childId, session) + : undefined; + if (completionResult === false) { return false; } if (this._disposed || this._disposing) { @@ -9433,25 +9539,55 @@ export class AgentSession { return false; } this._rlmChildSessions.set(childId, session); + this._rlmChildSessionAssignments.set(childId, expectedAssignmentId); if (unsubscribe) { this._rlmChildUnsubscribes.set(childId, unsubscribe); + this._rlmChildUnsubscribeAssignments.set(childId, expectedAssignmentId); } return true; } + /** + * Bind a just-hydrated daemon child to its persisted immutable assignment without + * altering the historical register callback arity. The session identity guard + * prevents an A hydration continuation from rebinding a replacement B. + */ + rebindRlmChildSessionAssignment(childId: string, session: AgentSession, assignmentId: string): boolean { + if (this._rlmChildSessions.get(childId) !== session) return false; + // Hydration can resume after a selector has been rebound. The same session + // object is not sufficient authority: only the assignment which installed it + // may refresh its binding. + const currentAssignmentId = this._rlmChildSessionAssignments.get(childId); + if (currentAssignmentId !== undefined && currentAssignmentId !== assignmentId) return false; + this._rlmChildSessionAssignments.set(childId, assignmentId); + if (this._rlmChildUnsubscribes.has(childId)) this._rlmChildUnsubscribeAssignments.set(childId, assignmentId); + return true; + } + /** Stop retaining an idle daemon child without deleting its durable registry row. */ - releaseRlmChildSession(childId: string, session: AgentSession): (() => void) | false { + releaseRlmChildSession(childId: string, session: AgentSession, assignmentId?: string): (() => void) | false { const run = this._activeRlmChildRuns.get(childId); - if (run?.session === session && run.status === "done") { + const expectedAssignmentId = assignmentId ?? run?.assignmentId ?? this._rlmChildSessionAssignments.get(childId); + if (!expectedAssignmentId) return false; + if (run?.session === session && run.status === "done" && run.assignmentId === expectedAssignmentId) { const unsubscribe = run.unsubscribe ?? noopRlmChildEventUnsubscribe; run.unsubscribe = undefined; - this._activeRlmChildRuns.delete(childId); + if (this._activeRlmChildRuns.get(childId)?.assignmentId === expectedAssignmentId) + this._activeRlmChildRuns.delete(childId); return unsubscribe; } - if (this._rlmChildSessions.get(childId) !== session) return false; + if ( + this._rlmChildSessions.get(childId) !== session || + this._rlmChildSessionAssignments.get(childId) !== expectedAssignmentId + ) + return false; const unsubscribe = this._rlmChildUnsubscribes.get(childId) ?? noopRlmChildEventUnsubscribe; - this._rlmChildUnsubscribes.delete(childId); + if (this._rlmChildUnsubscribeAssignments.get(childId) === expectedAssignmentId) { + this._rlmChildUnsubscribes.delete(childId); + this._rlmChildUnsubscribeAssignments.delete(childId); + } this._rlmChildSessions.delete(childId); + this._rlmChildSessionAssignments.delete(childId); return unsubscribe; } @@ -9649,6 +9785,7 @@ export class AgentSession { let childSession: AgentSession | undefined; const run: RlmChildRun = { id: childNodeId, + assignmentId: randomUUID(), prompt, sessionName, sessionDir: childSessionDir, @@ -9662,6 +9799,13 @@ export class AgentSession { }; this._activeRlmChildRuns.set(run.id, run); const emitChildUpdate = () => { + const activeOwner = + this._activeRlmChildRuns.get(run.id) === run && + this._activeRlmChildRuns.get(run.id)?.assignmentId === run.assignmentId; + const retainedOwner = + this._rlmChildSessions.get(run.id) === childSession && + this._rlmChildSessionAssignments.get(run.id) === run.assignmentId; + if (!activeOwner && !retainedOwner) return; const childModel = childSession?.model ?? modelSelection.model; this._emit({ type: "rlm_child_update", @@ -9689,7 +9833,11 @@ export class AgentSession { const publishChildSession = (child: AgentSession) => { childSession = child; - if (this._activeRlmChildRuns.get(run.id) !== run) return; + if ( + this._activeRlmChildRuns.get(run.id) !== run || + this._activeRlmChildRuns.get(run.id)?.assignmentId !== run.assignmentId + ) + return; run.session = child; run.abort = () => void child.abort(); run.publication.resolve(); @@ -9697,6 +9845,7 @@ export class AgentSession { const subagentOptions: CreateRlmSubagentRuntimeOptions = { ...this._createRlmSubagentRuntimeOptions({ id: childNodeId, + assignmentId: run.assignmentId, prompt, sessionName, spawnCode, @@ -9707,6 +9856,11 @@ export class AgentSession { }; const deliverTerminalMessageToParent = async (message: CustomMessage): Promise => { + if ( + this._activeRlmChildRuns.get(run.id) !== run || + this._activeRlmChildRuns.get(run.id)?.assignmentId !== run.assignmentId + ) + return; const childController = childSession?._agentMessageController; if (childController) { try { @@ -9742,6 +9896,15 @@ export class AgentSession { run.status = "running"; emitChildUpdate(); const unsubscribeChildEvents = child.subscribe((event) => { + // Once retained, the active-run record is intentionally gone. Keep + // projecting this exact child session, but never let A's subscription + // observe a replacement B that reused the selector. + const activeOwner = + this._activeRlmChildRuns.get(run.id) === run && run.assignmentId === subagentOptions.assignmentId; + const retainedOwner = + this._rlmChildSessions.get(run.id) === child && + this._rlmChildSessionAssignments.get(run.id) === run.assignmentId; + if (!activeOwner && !retainedOwner) return; if (event.type === "rlm_child_update") { this._emit(event); return; @@ -9846,7 +10009,7 @@ export class AgentSession { }), ); } - if (!this.registerRlmChildSession(run.id, child)) { + if (!this.registerRlmChildSession(run.id, child, undefined, run.assignmentId)) { if (childRuntime && this._subagentRuntimeHost?.releaseRlmSubagentRuntime) { await this._subagentRuntimeHost .releaseRlmSubagentRuntime(childRuntime, subagentOptions, "error") @@ -9893,8 +10056,10 @@ export class AgentSession { run.status === "cancelled" ? "cancelled" : "error", ); if (run.status === "cancelled" && !this._disposed && !this._disposing) { - this._deletedRlmChildIds.add(run.id); - this._removeRlmSubagentTracking(run.id); + if (this._activeRlmChildRuns.get(run.id) === run) { + this._deletedRlmChildIds.add(this._rlmAssignmentKey(run.id, run.assignmentId)); + this._removeRlmSubagentTracking(run.id, run, run.assignmentId); + } } } catch { await childSession?.disposeAsync().catch(() => undefined); @@ -9902,13 +10067,25 @@ export class AgentSession { } else if (!run.detachedDeletion) { try { if (childRuntime && this._subagentRuntimeHost) { - await this._subagentRuntimeHost.deleteRlmSubagentRuntime(run.id, childRuntime.session); + await (this._subagentRuntimeHost.assignmentIdentityFenced + ? this._subagentRuntimeHost.deleteRlmSubagentRuntime( + run.id, + childRuntime.session, + run.assignmentId, + ) + : this._subagentRuntimeHost.deleteRlmSubagentRuntime(run.id, childRuntime.session)); } else if (childSession) { await childSession.disposeAsync(); } if (run.status === "cancelled" && !this._disposed && !this._disposing) { - this._deletedRlmChildIds.add(run.id); - this._removeRlmSubagentTracking(run.id); + const activeOwner = this._activeRlmChildRuns.get(run.id) === run; + const retainedOwner = + this._rlmChildSessions.get(run.id) === childRuntime?.session && + this._rlmChildSessionAssignments.get(run.id) === run.assignmentId; + if (activeOwner || retainedOwner) { + this._deletedRlmChildIds.add(this._rlmAssignmentKey(run.id, run.assignmentId)); + this._removeRlmSubagentTracking(run.id, activeOwner ? run : undefined, run.assignmentId); + } } } catch { // A failed best-effort retry remains available through the retained cleanup maps. @@ -9917,18 +10094,32 @@ export class AgentSession { } finally { if (run.detachedDeletion && childRuntime) { try { - await this._deleteRlmSubagentSession(run.id, childRuntime.session); + await this._deleteRlmSubagentSession(run.id, run.assignmentId, childRuntime.session); + // A retry can complete after its first failed delete retained the + // child. Remove only that captured session/assignment, never B. + if ( + this._rlmChildSessions.get(run.id) === childRuntime.session && + this._rlmChildSessionAssignments.get(run.id) === run.assignmentId + ) { + this._deletedRlmChildIds.add(this._rlmAssignmentKey(run.id, run.assignmentId)); + this._removeRlmSubagentTracking(run.id, undefined, run.assignmentId); + } } catch { - if (!this._disposed && !this._disposing) { + if (!this._disposed && !this._disposing && this._activeRlmChildRuns.get(run.id) === run) { this._rlmChildSessions.set(run.id, childRuntime.session); + this._rlmChildSessionAssignments.set(run.id, run.assignmentId); this._rlmChildCleanupFailures.set(run.id, run.detachedDeletion); } } } if (this._activeRlmChildRuns.get(run.id) === run) { if (this._rlmChildSessions.has(run.id)) { - this._activeRlmChildRuns.delete(run.id); - if (run.unsubscribe) this._rlmChildUnsubscribes.set(run.id, run.unsubscribe); + if (this._activeRlmChildRuns.get(run.id)?.assignmentId === run.assignmentId) + this._activeRlmChildRuns.delete(run.id); + if (run.unsubscribe) { + this._rlmChildUnsubscribes.set(run.id, run.unsubscribe); + this._rlmChildUnsubscribeAssignments.set(run.id, run.assignmentId); + } run.abort = noopRlmChildAbort; run.unsubscribe = undefined; run.session = undefined; @@ -10269,6 +10460,11 @@ export class AgentSession { return this._retryPromise !== undefined; } + /** Durable IDs of every action that has not reached its own terminal release. */ + get unfinishedActionIds(): readonly string[] { + return this._actionStore.unfinishedActions().map((action) => action.id); + } + /** Whether an accepted prompt is still running or waiting for retry completion. */ get hasAcceptedPromptInFlight(): boolean { return this._actionStore diff --git a/packages/coding-agent/src/core/rlm-runtime.ts b/packages/coding-agent/src/core/rlm-runtime.ts index e89472fce..a01591dce 100644 --- a/packages/coding-agent/src/core/rlm-runtime.ts +++ b/packages/coding-agent/src/core/rlm-runtime.ts @@ -198,13 +198,23 @@ export function createRlmDeleteSubagentHostHandler(handler: RlmDeleteSubagentHan }; } +export const RLM_CHILD_TERMINAL_STATUSES = ["done", "error", "cancelled"] as const; +export type RlmChildTerminalStatus = (typeof RLM_CHILD_TERMINAL_STATUSES)[number]; +export function isRlmChildTerminalStatus(value: unknown): value is RlmChildTerminalStatus { + return typeof value === "string" && (RLM_CHILD_TERMINAL_STATUSES as readonly string[]).includes(value); +} + export interface RlmSubagentRuntime { session: AgentSession; + /** Immutable attempt identity; never expose this in the public child catalog. */ + assignmentId?: string; } export interface CreateRlmSubagentRuntimeOptions { parentSession: AgentSession; id: string; + /** Minted by the parent before this child is published. */ + assignmentId?: string; prompt: string; sessionName: string; sessionDir: string; @@ -227,9 +237,11 @@ export interface CreateRlmSubagentRuntimeOptions { } export interface SubagentRuntimeHost { + /** Opt-in daemon adapter: legacy embedded hosts retain their historical callback arity. */ + readonly assignmentIdentityFenced?: true; createRlmSubagentRuntime(options: CreateRlmSubagentRuntimeOptions): Promise; /** Persist host-owned completion before the child becomes passivation-eligible. */ - completeRlmSubagentRuntime?(childId: string, session: AgentSession): boolean; + completeRlmSubagentRuntime?(childId: string, session: AgentSession, assignmentId?: string): boolean; /** Release a host-owned child after its detached initial task settles. */ releaseRlmSubagentRuntime?: ( runtime: RlmSubagentRuntime, @@ -237,6 +249,6 @@ export interface SubagentRuntimeHost { status: "done" | "error" | "cancelled", ) => Promise; /** Close or remove the host-owned child; session is absent when a persisted child is still passive. */ - deleteRlmSubagentRuntime(childId: string, session?: AgentSession): Promise; + deleteRlmSubagentRuntime(childId: string, session?: AgentSession, assignmentId?: string): Promise; disposeRlmSubagentRuntimes?(): Promise; } diff --git a/packages/coding-agent/src/modes/daemon/daemon-lifecycle-identity.ts b/packages/coding-agent/src/modes/daemon/daemon-lifecycle-identity.ts new file mode 100644 index 000000000..fda85aff8 --- /dev/null +++ b/packages/coding-agent/src/modes/daemon/daemon-lifecycle-identity.ts @@ -0,0 +1,42 @@ +import { getProcessStartId } from "../../core/session-lease.js"; + +/** Closed daemon-owned lifecycle values. Disk input is untrusted. */ +export const DAEMON_WORKER_LIFECYCLES = ["starting", "ready", "recovering", "failed", "passivated"] as const; +export type DaemonWorkerLifecycle = (typeof DAEMON_WORKER_LIFECYCLES)[number]; +export function isDaemonWorkerLifecycle(value: unknown): value is DaemonWorkerLifecycle { + return typeof value === "string" && (DAEMON_WORKER_LIFECYCLES as readonly string[]).includes(value); +} + +/** Execution terminals, deliberately distinct from public registry presentation status. */ +export const RLM_CHILD_TERMINAL_STATUSES = ["done", "error", "cancelled"] as const; +export type RlmChildTerminalStatus = (typeof RLM_CHILD_TERMINAL_STATUSES)[number]; +export function isRlmChildTerminalStatus(value: unknown): value is RlmChildTerminalStatus { + return typeof value === "string" && (RLM_CHILD_TERMINAL_STATUSES as readonly string[]).includes(value); +} + +export interface ProcessIdentity { + pid: number; + processStartId: string; +} + +export interface OperationIdentity { + operationId: string; + generation: string; +} + +// UUIDs written by C01 use crypto.randomUUID(). Require canonical RFC-4122 text +// when accepting an identity from disk rather than allowing arbitrary selectors. +const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; +export function assertFreshUuid(value: unknown): value is string { + return typeof value === "string" && UUID_RE.test(value); +} + +/** A PID alone is never a process identity. Unreadable start IDs fail closed. */ +export function isCurrentProcessIdentity(identity: ProcessIdentity): boolean { + if (!Number.isInteger(identity.pid) || identity.pid <= 0 || !identity.processStartId) return false; + try { + return getProcessStartId(identity.pid) === identity.processStartId; + } catch { + return false; + } +} diff --git a/packages/coding-agent/src/modes/daemon/daemon-mode.ts b/packages/coding-agent/src/modes/daemon/daemon-mode.ts index dfebcdf61..e07ef1825 100644 --- a/packages/coding-agent/src/modes/daemon/daemon-mode.ts +++ b/packages/coding-agent/src/modes/daemon/daemon-mode.ts @@ -140,6 +140,7 @@ import { DaemonClient } from "./daemon-client.js"; import { filterClientEnv, withClientEnv } from "./daemon-client-env.js"; import { deserializeDaemonError, serializeDaemonError } from "./daemon-errors.js"; import { bindActiveSessionState } from "./daemon-extension-binding.js"; +import { assertFreshUuid, type OperationIdentity } from "./daemon-lifecycle-identity.js"; import { createDaemonEventMeta, createDaemonReplayInfo, @@ -190,6 +191,7 @@ import { import { assertDaemonSupervisorOwnerCurrent, isDaemonShutdownAdmissionActive } from "./daemon-supervisor-ownership.js"; import { DAEMON_WORKER_ACTIVE_SESSION_ID_ENV, + DAEMON_WORKER_GENERATION_ENV, DAEMON_WORKER_RECOVERY_JOURNAL_ENV, DAEMON_WORKER_ROLE_ENV, DAEMON_WORKER_SUPERVISOR_SOCKET_ENV, @@ -207,7 +209,7 @@ import { SNAPSHOT_TARGET_CHUNK_BYTES, type SnapshotTranscriptChunkSource, } from "./snapshot-transcript-cache.js"; -import { WorkerRecoveryJournal } from "./worker-recovery-journal.js"; +import { WorkerRecoveryJournal, type WorkerRecoveryOperation } from "./worker-recovery-journal.js"; export interface DaemonModeOptions { socketPath?: string; @@ -377,6 +379,8 @@ const RLM_SUBAGENT_REGISTRY_FILE = "rlm-subagents.jsonl"; interface PersistedRlmSubagentRegistryEntry { type: "rlm_subagent"; childId: string; + /** Undefined only for legacy display rows; C01 callbacks must never bind one. */ + assignmentId?: string; sessionName: string; sessionDir: string; sessionFile: string; @@ -403,6 +407,12 @@ type PassiveRlmSubagent = PassiveRlmRoot & { chain: PersistedRlmSubagentRegistryEntry[]; }; +interface RecoveryOperationToken { + operation: WorkerRecoveryOperation; + identity: OperationIdentity; + sequence: number; +} + class RuntimeOpenCancelledError extends Error {} class BoundSessionUnavailableError extends Error {} @@ -454,11 +464,17 @@ export class AgentDaemon { * coalesces every close by transient activeSessionId, while this one identifies the * passivation-only close reason by the durable identity needed by hydration/opening. */ - private readonly passivatingSessions = new Map>(); + private readonly passivatingSessions = new Map< + string, + { promise: Promise; state: ActiveSessionState; generation: string; operation: OperationIdentity } + >(); private readonly closingSessions = new Map< string, { promise: Promise; + state: ActiveSessionState; + generation: string; + operation: OperationIdentity; reason: DaemonSessionClosedReason; descendants: Set; reasonUpgrade?: Promise; @@ -478,6 +494,10 @@ export class AgentDaemon { { activeSessionId: string; admissionId: string; + /** Captured before the prompt flow can await; never reassigned. */ + operation: OperationIdentity; + state?: ActiveSessionState; + stateGeneration?: string; controller?: AbortController; status: "waiting" | "owned" | "cancelled"; } @@ -520,6 +540,24 @@ export class AgentDaemon { }, ); private readonly recoveryJournal?: WorkerRecoveryJournal; + // Recovery identities belong to individual admitted operations. The queues below + // are scheduler queues, not a mutable family-current map: each element retains + // the immutable token that was durably begun for that particular operation. + private readonly recoveryTurnOperations = new WeakMap< + ActiveSessionState, + { pending: RecoveryOperationToken[]; active: RecoveryOperationToken[][] } + >(); + /** + * Restored actions do not have a command response lifetime: they remain queued + * until their individual scheduler actions terminally settle or are cancelled. + */ + private readonly restoredActionRecoveries = new WeakMap< + ActiveSessionState, + Array<{ tokensByActionId: Map }> + >(); + private readonly recoveryEventOperations = new WeakMap>(); + private recoveryOperationSequence = 0; + private readonly workerGeneration: string; constructor( private readonly socketPath: string, @@ -533,6 +571,7 @@ export class AgentDaemon { ? AgentCronJobStore.forSessionArtifacts() : new AgentCronJobStore(getCronJobsPath(this.agentDir)); this.restoreActiveSessionId = options.worker?.restoreActiveSessionId; + this.workerGeneration = process.env[DAEMON_WORKER_GENERATION_ENV] ?? randomUUID(); const recoveryJournalPath = process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; if (options.worker && recoveryJournalPath) { this.recoveryJournal = new WorkerRecoveryJournal(recoveryJournalPath); @@ -885,6 +924,37 @@ export class AgentDaemon { return join(artifactDir, RLM_SUBAGENT_REGISTRY_FILE); } + private rlmAssignmentKey(entry: Pick): string { + // Missing assignment is a legacy display identity and is deliberately never + // equal to a C01 UUID callback identity. + return `${entry.childId}\u0000${entry.assignmentId ?? "legacy"}`; + } + + /** + * Registry entries retain one terminal state per durable assignment, ordered + * by the assignment's first durable publication. A later terminal update for + * old A must not make A the public incarnation after B reuses its childId. + */ + private currentRlmSubagentRegistryEntry( + entries: readonly PersistedRlmSubagentRegistryEntry[], + childId: string, + ): PersistedRlmSubagentRegistryEntry | undefined { + for (let index = entries.length - 1; index >= 0; index--) { + const entry = entries[index]; + if (entry?.childId === childId) return entry; + } + return undefined; + } + + /** The sole public/passive row for each reused child selector. */ + private currentLiveRlmSubagentRegistryEntries( + entries: readonly PersistedRlmSubagentRegistryEntry[], + ): PersistedRlmSubagentRegistryEntry[] { + const currentByChildId = new Map(); + for (const entry of entries) currentByChildId.set(entry.childId, entry); + return [...currentByChildId.values()].filter((entry) => entry.status !== "deleted"); + } + private appendRlmSubagentRegistryEntry( parentState: ActiveSessionState, entry: PersistedRlmSubagentRegistryEntry, @@ -917,6 +987,7 @@ export class AgentDaemon { parentState: ActiveSessionState, input: { childId: string; + assignmentId: string; sessionName: string; sessionDir: string; sessionFile: string; @@ -934,6 +1005,7 @@ export class AgentDaemon { return this.appendRlmSubagentRegistryEntry(parentState, { type: "rlm_subagent", childId: input.childId, + assignmentId: input.assignmentId, sessionName: input.sessionName, sessionDir: input.sessionDir, sessionFile: input.sessionFile, @@ -951,9 +1023,13 @@ export class AgentDaemon { }); } - private async recordRlmSubagentDeletion(parentState: ActiveSessionState, childId: string): Promise { + private async recordRlmSubagentDeletion( + parentState: ActiveSessionState, + childId: string, + assignmentId: string, + ): Promise { const latest = (await this.readLatestRlmSubagentRegistry(parentState, true)).find( - (entry) => entry.childId === childId, + (entry) => entry.childId === childId && entry.assignmentId === assignmentId, ); if (!latest || latest.status === "deleted") { return; @@ -1015,18 +1091,55 @@ export class AgentDaemon { typeof entry.sessionFile !== "string" || (entry.status !== "running" && entry.status !== "completed" && entry.status !== "deleted") || (entry.rlmDepth !== undefined && (!Number.isSafeInteger(entry.rlmDepth) || entry.rlmDepth < 0)) || - (entry.rlmMaxDepth !== undefined && (!Number.isSafeInteger(entry.rlmMaxDepth) || entry.rlmMaxDepth < 0)) + (entry.rlmMaxDepth !== undefined && + (!Number.isSafeInteger(entry.rlmMaxDepth) || entry.rlmMaxDepth < 0)) || + (entry.assignmentId !== undefined && + (typeof entry.assignmentId !== "string" || + !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test( + entry.assignmentId, + ))) ) { continue; } - latest.set(entry.childId, entry as PersistedRlmSubagentRegistryEntry); + latest.set( + this.rlmAssignmentKey(entry as PersistedRlmSubagentRegistryEntry), + entry as PersistedRlmSubagentRegistryEntry, + ); } catch (error) { this.log( `ignored malformed RLM subagent registry entry: ${error instanceof Error ? error.message : String(error)}`, ); } } - return [...latest.values()]; + const entries = [...latest.values()]; + // Once an explicit hydrate has durably rebound an old display-only row, + // suppress that legacy duplicate from catalog traversal. Its late callbacks + // still cannot match the assigned row because callback matching is exact. + return entries.filter( + (entry) => + entry.assignmentId !== undefined || + !entries.some((candidate) => candidate.childId === entry.childId && candidate.assignmentId !== undefined), + ); + } + + /** Read only selectable catalog incarnations; exact lifecycle operations use the full reader above. */ + private async readCurrentLiveRlmSubagentRegistryPath( + path: string | undefined, + throwOnReadError = false, + ): Promise { + return this.currentLiveRlmSubagentRegistryEntries( + await this.readLatestRlmSubagentRegistryPath(path, throwOnReadError), + ); + } + + private async readCurrentLiveRlmSubagentRegistry( + parentState: ActiveSessionState, + throwOnReadError = false, + ): Promise { + return this.readCurrentLiveRlmSubagentRegistryPath( + this.rlmSubagentRegistryPath(parentState.runtime.session), + throwOnReadError, + ); } private rlmSubagentRegistryPathForEntry(entry: PersistedRlmSubagentRegistryEntry, info: SessionInfo): string { @@ -1062,7 +1175,7 @@ export class AgentDaemon { passive.push({ ...root, entry, info, chain }); await visit( root, - await this.readLatestRlmSubagentRegistryPath(this.rlmSubagentRegistryPathForEntry(entry, info)), + await this.readCurrentLiveRlmSubagentRegistryPath(this.rlmSubagentRegistryPathForEntry(entry, info)), chain, visited, ); @@ -1077,7 +1190,7 @@ export class AgentDaemon { residentRootPaths.add(parentPath); await visit( { rootParentState: parentState }, - await this.readLatestRlmSubagentRegistry(parentState), + await this.readCurrentLiveRlmSubagentRegistry(parentState), [], new Set([parentPath]), ); @@ -1087,7 +1200,12 @@ export class AgentDaemon { if (inactiveLifecycleForSession(rootInfo) !== "live" || residentRootPaths.has(rootPath)) continue; const registryPath = this.rlmSubagentRegistryPathForInfo(rootInfo); if (!existsSync(registryPath)) continue; - await visit({ rootInfo }, await this.readLatestRlmSubagentRegistryPath(registryPath), [], new Set([rootPath])); + await visit( + { rootInfo }, + await this.readCurrentLiveRlmSubagentRegistryPath(registryPath), + [], + new Set([rootPath]), + ); } return passive; } @@ -2209,26 +2327,31 @@ export class AgentDaemon { private createSubagentRuntimeHost(parentState: ActiveSessionState): SubagentRuntimeHost { return { + assignmentIdentityFenced: true, createRlmSubagentRuntime: async (options) => this.createRlmSubagentRuntime(parentState, options), - completeRlmSubagentRuntime: (childId, session) => { + completeRlmSubagentRuntime: (childId, childSession, assignmentId) => { + if (!assignmentId || !childSession) return false; const state = [...this.sessions.values()].find( (candidate) => candidate.runtime.metadata.kind === "subagent" && candidate.runtime.metadata.parentActiveSessionId === parentState.activeSessionId && candidate.runtime.metadata.rlmChildId === childId && - candidate.runtime.session === session, + candidate.runtime.metadata.assignmentId === assignmentId && + candidate.runtime.session === childSession, ); - if (!state?.runtime.session.sessionFile) return false; + if (!state?.runtime.session.sessionFile || this.sessions.get(parentState.activeSessionId) !== parentState) + return false; if (state.runtime.metadata.rehydratedCompleted) return true; const metadata = state.runtime.metadata; - const model = session.model; + const model = childSession.model; return this.recordRlmSubagentRegistryEntry(parentState, { childId, - sessionName: session.sessionName ?? childId, + assignmentId, + sessionName: childSession.sessionName ?? childId, sessionDir: metadata.sessionDir ?? dirname(state.runtime.session.sessionFile), sessionFile: state.runtime.session.sessionFile, - rlmDepth: session.rlmDepth, - rlmMaxDepth: session.rlmMaxDepth, + rlmDepth: childSession.rlmDepth, + rlmMaxDepth: childSession.rlmMaxDepth, rlmParentNodeId: metadata.rlmParentNodeId, prompt: metadata.prompt && metadata.prompt.length <= 4096 ? metadata.prompt : undefined, spawnCode: metadata.spawnCode, @@ -2238,12 +2361,19 @@ export class AgentDaemon { }); }, releaseRlmSubagentRuntime: async (runtime, options, status) => { + const assignmentId = runtime.assignmentId ?? options.assignmentId; + // A callback without an assignment is a legacy display path and may only + // dispose its own unpublished runtime, never mutate daemon state. + if (!assignmentId || this.sessions.get(parentState.activeSessionId) !== parentState) { + await runtime.session.disposeAsync(); + return; + } // Persist the deletion boundary first, but never let a registry failure // strand the cancelled child as a stale resident session. let deletionError: unknown; if (status === "cancelled") { try { - await this.recordRlmSubagentDeletion(parentState, options.id); + await this.recordRlmSubagentDeletion(parentState, options.id, assignmentId); } catch (error) { deletionError = error; } @@ -2253,43 +2383,96 @@ export class AgentDaemon { candidate.runtime.metadata.kind === "subagent" && candidate.runtime.metadata.parentActiveSessionId === parentState.activeSessionId && candidate.runtime.metadata.rlmChildId === options.id && + candidate.runtime.metadata.assignmentId === assignmentId && candidate.runtime.session === runtime.session, ); - if (state) { + if (state && this.sessions.get(state.activeSessionId) === state) { await this.closeSession(state, status === "cancelled" ? "killed" : "completed"); } else { await runtime.session.disposeAsync(); } if (deletionError !== undefined) throw deletionError; }, - deleteRlmSubagentRuntime: async (childId, session) => { + deleteRlmSubagentRuntime: async (childId, childSession, requestedAssignmentId) => { + // Public catalog entries intentionally hide assignment IDs. An explicit + // delete is therefore the one legacy operation permitted to resolve its + // durable row internally. A missing legacy ID is rebound *before* delete; + // asynchronous callbacks still require a named assignment and cannot use it. + let assignmentId = requestedAssignmentId; + // Read the durable boundary before any destructive action. Errors are + // intentionally authoritative: a failed read must not turn an ambiguous + // delete into an in-memory close. + const persistedEntries = await this.readLatestRlmSubagentRegistry(parentState, true); + const currentParent = this.sessions.get(parentState.activeSessionId); + if (currentParent && currentParent !== parentState) { + await childSession?.disposeAsync(); + return; + } + let persisted = assignmentId + ? persistedEntries.find((entry) => entry.childId === childId && entry.assignmentId === assignmentId) + : this.currentRlmSubagentRegistryEntry(persistedEntries, childId); + // Legacy catalog deletion resolves the newest durable incarnation, even + // if an older reused childId remains completed on disk. Explicit IDs + // above deliberately retain exact-assignment authority. + if (!assignmentId && persisted) { + if (!persisted.assignmentId) { + assignmentId = randomUUID(); + if ( + !this.recordRlmSubagentRegistryEntry(parentState, { + childId: persisted.childId, + assignmentId, + sessionName: persisted.sessionName, + sessionDir: persisted.sessionDir, + sessionFile: persisted.sessionFile, + rlmDepth: persisted.rlmDepth ?? 1, + rlmMaxDepth: persisted.rlmMaxDepth ?? parentState.runtime.session.rlmMaxDepth, + rlmParentNodeId: persisted.rlmParentNodeId, + prompt: persisted.prompt, + spawnCode: persisted.spawnCode, + model: persisted.model, + status: persisted.status, + createdAt: persisted.createdAt, + }) + ) + throw new Error(`Failed to bind legacy RLM subagent ${childId} for deletion`); + persisted = { ...persisted, assignmentId }; + } else assignmentId = persisted.assignmentId; + } + // Assignment-less resident children are old host data. They remain + // deletable only by this explicit (not callback) path; do not invent a + // durable row for a fixture/legacy runtime that was never published. const state = [...this.sessions.values()].find( (candidate) => candidate.runtime.metadata.kind === "subagent" && candidate.runtime.metadata.parentActiveSessionId === parentState.activeSessionId && - candidate.runtime.metadata.rlmChildId === childId, - ); - const persisted = (await this.readLatestRlmSubagentRegistry(parentState, true)).find( - (entry) => entry.childId === childId, + candidate.runtime.metadata.rlmChildId === childId && + (assignmentId === undefined || candidate.runtime.metadata.assignmentId === assignmentId), ); + if (!assignmentId && !state) { + await childSession?.disposeAsync(); + return; + } const childSessionFile = persisted?.sessionFile ?? state?.runtime.session.sessionFile; - // Persist the deletion boundary before tearing down the runtime. As with a - // resident child, deletion keeps its transcript and artifact tree on disk. - await this.recordRlmSubagentDeletion(parentState, childId); - const staleSession = state && session && state.runtime.session !== session ? session : undefined; + // Awaiting disk I/O must not give an old assignment authority over a replacement. + if ( + (this.sessions.get(parentState.activeSessionId) !== undefined && + this.sessions.get(parentState.activeSessionId) !== parentState) || + (state && this.sessions.get(state.activeSessionId) !== state) + ) { + await childSession?.disposeAsync(); + return; + } + if (assignmentId) await this.recordRlmSubagentDeletion(parentState, childId, assignmentId); + const staleSession = + state && childSession && state.runtime.session !== childSession ? childSession : undefined; try { - if (state) { + if (state && this.sessions.get(state.activeSessionId) === state) await this.closeSession(state, "killed", false); - } else { - await session?.disposeAsync(); - } + else await childSession?.disposeAsync(); } finally { await staleSession?.disposeAsync(); } - // A killed close can join a passivation close that already skipped killed cleanup. - if (childSessionFile) { - this.cancelScheduledJobsForSessionFile(childSessionFile); - } + if (childSessionFile) this.cancelScheduledJobsForSessionFile(childSessionFile); }, disposeRlmSubagentRuntimes: async () => { const cascadeError = await this.closeChildSessions(parentState, "replaced"); @@ -2304,6 +2487,12 @@ export class AgentDaemon { parentState: ActiveSessionState, options: CreateRlmSubagentRuntimeOptions, ): Promise { + // Assignment is mandatory authority for every daemon-hosted incarnation. + // Legacy callers that omit it are adapted by minting here; untrusted values + // are never written because readers correctly reject them. + const assignmentId = assertFreshUuid(options.assignmentId) ? options.assignmentId : randomUUID(); + const assignedOptions = assignmentId === options.assignmentId ? options : { ...options, assignmentId }; + options = assignedOptions; const sessionManager = SessionManager.create(options.parentSession.sessionManager.getCwd(), options.sessionDir); sessionManager.newSession({ parentSession: options.parentSession.sessionFile, @@ -2369,6 +2558,7 @@ export class AgentDaemon { parentSessionId: options.parentSession.sessionId, parentSessionFile: options.parentSession.sessionFile, rlmChildId: options.id, + assignmentId: options.assignmentId, rlmParentNodeId: options.rlmParentNodeId, prompt: options.prompt, spawnCode: options.spawnCode, @@ -2391,6 +2581,7 @@ export class AgentDaemon { if (runtime.session.sessionFile) { this.recordRlmSubagentRegistryEntry(parentState, { childId: options.id, + assignmentId, sessionName: options.sessionName, sessionDir: options.sessionDir, sessionFile: runtime.session.sessionFile, @@ -2469,12 +2660,22 @@ export class AgentDaemon { const sessionKey = resolve(sessionFile); const parentActiveSessionId = metadata.parentActiveSessionId; const childId = metadata.rlmChildId; + const assignmentId = metadata.assignmentId; + // Legacy resident children remain readable but cannot let an asynchronous + // passivation callback unbind a newer same-selector assignment. + if (!assignmentId) return false; const existing = this.passivatingSessions.get(sessionKey); if (existing) { - await existing; + await existing.promise; return false; } + const passivationOperation: OperationIdentity = { operationId: randomUUID(), generation: state.eventGeneration }; const snapshot = selectedSnapshot ?? (await this.sessionPassivationSnapshot(state)); + if ( + this.sessions.get(state.activeSessionId) !== state || + state.eventGeneration !== passivationOperation.generation + ) + return false; if (!canPassivateSession(snapshot, idleEvictionMinutes, now)) return false; // Publish the durable identity before running the close so opens and lazy @@ -2495,7 +2696,11 @@ export class AgentDaemon { const idleMinutes = Math.floor((now - snapshot.lastActivityAt) / 60_000); // Detach parent tracking before the standard graceful runtime disposal. The // registry/catalog rows remain the sole passive representation after close. - const unsubscribeChild = parentState.runtime.session.releaseRlmChildSession(childId, state.runtime.session); + const unsubscribeChild = parentState.runtime.session.releaseRlmChildSession( + childId, + state.runtime.session, + assignmentId, + ); if (!unsubscribeChild) { return; } @@ -2507,8 +2712,21 @@ export class AgentDaemon { if ( this.sessions.get(state.activeSessionId) === state && this.sessions.get(parentActiveSessionId) === parentState && - parentState.runtime.session.registerRlmChildSession(childId, state.runtime.session, unsubscribeChild) + parentState.runtime.session.registerRlmChildSession( + childId, + state.runtime.session, + unsubscribeChild, + assignmentId, + ) ) { + // The adapter is deliberately optional so old embedded/test hosts observe + // the historical register arity. Real AgentSession instances bind the + // durable assignment immediately after that compatibility call. + parentState.runtime.session.rebindRlmChildSessionAssignment?.( + childId, + state.runtime.session, + assignmentId, + ); throw error; } unsubscribeChild(); @@ -2519,14 +2737,18 @@ export class AgentDaemon { `Passivated idle child sessionId=${state.runtime.session.sessionId} name=${JSON.stringify(state.runtime.session.sessionName ?? "")} idleMinutes=${idleMinutes}`, ); }); - this.passivatingSessions.set(sessionKey, passivation); + const passivationJoin = { + promise: passivation, + state, + generation: passivationOperation.generation, + operation: passivationOperation, + }; + this.passivatingSessions.set(sessionKey, passivationJoin); try { await passivation; return this.sessions.get(state.activeSessionId) !== state; } finally { - if (this.passivatingSessions.get(sessionKey) === passivation) { - this.passivatingSessions.delete(sessionKey); - } + if (this.passivatingSessions.get(sessionKey) === passivationJoin) this.passivatingSessions.delete(sessionKey); } } @@ -2555,7 +2777,7 @@ export class AgentDaemon { } private findPassivationBySessionFile(sessionFile: string): Promise | undefined { - return this.passivatingSessions.get(resolve(sessionFile)); + return this.passivatingSessions.get(resolve(sessionFile))?.promise; } private async waitForPassivation(sessionFile: string): Promise { @@ -2581,7 +2803,8 @@ export class AgentDaemon { if ( resident && resident.runtime.metadata.kind === "subagent" && - resident.runtime.metadata.rlmChildId === passive.entry.childId + resident.runtime.metadata.rlmChildId === passive.entry.childId && + resident.runtime.metadata.assignmentId === passive.entry.assignmentId ) { return this.waitForBoundSession(resident); } @@ -2609,8 +2832,12 @@ export class AgentDaemon { hydrated = await this.rehydrateCompletedRlmSubagent(hydratingParent, entry, activeSessionId, clientEnv); } catch (error) { const passivation = this.findPassivationBySessionFile(entry.sessionFile); - if (error instanceof BoundSessionUnavailableError && passivation) { - await passivation.catch(() => {}); + if (error instanceof BoundSessionUnavailableError) { + // A close can publish after the pre-hydration wait and can finish + // before its durable passivation join is observed here. In either + // case, re-walk the passive chain rather than returning the closing + // incarnation to the caller. + if (passivation) await passivation.catch(() => {}); return restartAfterParentChange(hydratingParent); } if (isResident(hydratingParent)) throw error; @@ -2636,6 +2863,31 @@ export class AgentDaemon { if (this.updateRestart !== undefined) { throw new BoundSessionUnavailableError("Daemon is preparing an update restart"); } + // Assignment-less journal rows are intentionally display-only. Before a + // hydrate can create callbacks, append a new immutable assignment row; a late + // legacy callback can therefore never acquire authority over this runtime. + if (!entry.assignmentId) { + const assignmentId = randomUUID(); + if ( + !this.recordRlmSubagentRegistryEntry(parentState, { + childId: entry.childId, + assignmentId, + sessionName: entry.sessionName, + sessionDir: entry.sessionDir, + sessionFile: entry.sessionFile, + rlmDepth: entry.rlmDepth ?? 1, + rlmMaxDepth: entry.rlmMaxDepth ?? parentState.runtime.session.rlmMaxDepth, + rlmParentNodeId: entry.rlmParentNodeId, + prompt: entry.prompt, + spawnCode: entry.spawnCode, + model: entry.model, + status: entry.status, + createdAt: entry.createdAt, + }) + ) + throw new Error(`Cannot bind legacy RLM subagent ${entry.childId} without a durable assignment`); + entry = { ...entry, assignmentId }; + } const sessionKey = resolve(entry.sessionFile); const reservation = this.reservingSessionOpens.get(sessionKey); if (reservation) { @@ -2645,14 +2897,22 @@ export class AgentDaemon { const pending = this.openingSessions.get(sessionKey); if (pending) { const state = await pending; - if (state.runtime.metadata.kind !== "subagent" || state.runtime.metadata.rlmChildId !== entry.childId) { + if ( + state.runtime.metadata.kind !== "subagent" || + state.runtime.metadata.rlmChildId !== entry.childId || + state.runtime.metadata.assignmentId !== entry.assignmentId + ) { if (this.openingSessions.get(sessionKey) === pending) this.openingSessions.delete(sessionKey); return this.rehydrateCompletedRlmSubagent(parentState, entry, restoreActiveSessionId, clientEnv); } return this.waitForBoundSession(state); } const existing = this.findSessionBySessionFile(entry.sessionFile); - if (existing?.runtime.metadata.kind === "subagent" && existing.runtime.metadata.rlmChildId === entry.childId) { + if ( + existing?.runtime.metadata.kind === "subagent" && + existing.runtime.metadata.rlmChildId === entry.childId && + existing.runtime.metadata.assignmentId === entry.assignmentId + ) { return this.waitForBoundSession(existing); } const hydration = (async () => { @@ -2767,6 +3027,7 @@ export class AgentDaemon { ? { parentSessionFile: parentState.runtime.session.sessionFile } : {}), rlmChildId: entry.childId, + assignmentId: entry.assignmentId, rlmParentNodeId: entry.rlmParentNodeId ?? entry.childId, rehydratedCompleted: true, ...(entry.prompt ? { prompt: entry.prompt } : {}), @@ -2788,7 +3049,23 @@ export class AgentDaemon { ); // The session transcript is authoritative for mutable metadata such as a // later user-assigned name; the registry value is only the spawn snapshot. - if (!parentState.runtime.session.registerRlmChildSession(entry.childId, runtime.session)) { + const registered = parentState.runtime.session.registerRlmChildSession( + entry.childId, + runtime.session, + undefined, + entry.assignmentId, + ); + // The explicit assignment is the authority boundary for lazy hydration. + // Keep the historical rebinding adapter only for an old session host that + // does not accept the fourth argument. + const assignmentBound = entry.assignmentId + ? parentState.runtime.session.rebindRlmChildSessionAssignment?.( + entry.childId, + runtime.session, + entry.assignmentId, + ) + : undefined; + if (!registered || assignmentBound === false) { await this.closeSession(state, "replaced"); throw new RuntimeOpenCancelledError(); } @@ -2796,7 +3073,11 @@ export class AgentDaemon { this.sessions.get(parentState.activeSessionId) !== parentState || this.closingSessions.has(parentState.activeSessionId) ) { - const unsubscribeChild = parentState.runtime.session.releaseRlmChildSession(entry.childId, runtime.session); + const unsubscribeChild = parentState.runtime.session.releaseRlmChildSession( + entry.childId, + runtime.session, + entry.assignmentId, + ); try { await this.closeSession(state, "replaced"); } finally { @@ -3101,6 +3382,7 @@ export class AgentDaemon { this.promptAdmissions.set(key, { activeSessionId: parsed.activeSessionId, admissionId: parsed.admissionId, + operation: { operationId: randomUUID(), generation: this.workerGeneration }, controller: new AbortController(), status: "waiting", }); @@ -3770,6 +4052,9 @@ export class AgentDaemon { this.promptAdmissions.delete(admissionKey); } }; + // Admission identity is allocated by the synchronous line parser. For + // legacy/no-id prompts we allocate it before this command reaches its + // first await below, then retain it until the matching turn_end. const commitAdmission = () => { if (admission?.status === "waiting") admission.status = "owned"; }; @@ -3777,10 +4062,17 @@ export class AgentDaemon { try { if (admission?.status === "cancelled") throw new PromptAdmissionCancelledError(); state = this.getBoundSessionState(command.activeSessionId); + if (admission) { + admission.state = state; + admission.stateGeneration = state.eventGeneration; + } } catch (error) { clearAdmission(); throw error; } + const identity = admission?.operation ?? { operationId: randomUUID(), generation: this.workerGeneration }; + const recoveryToken = this.beginWorkerRecoveryOperation(state, "prompt", identity); + this.queueWorkerRecoveryTurn(state, recoveryToken); const options: PromptOptions = { content: command.content, images: command.images, @@ -3802,17 +4094,26 @@ export class AgentDaemon { await this.promptWithAgentMessagePreparingGuard(state, command.message, { ...options, preflightResult: (didSucceed) => { - if (didSucceed) this.recordWorkerRecoveryState(state, "prompt_accepted", true); + if (didSucceed) { + // `prompt` was durably begun at admission. Do not replace + // its identity at acceptance: a delayed A acceptance must not + // overwrite a later B begin in the one-record crash journal. + } }, }); return success(command.id, command.type); } finally { + this.cancelQueuedWorkerRecoveryTurn(state, recoveryToken); clearAdmission(); } } let responseSent = false; let preflightRejected = false; + // `promptUntilAccepted` resolves once the session has admitted work, not + // once the queued turn has terminally ended. Keep its exact token + // queued after acceptance so a crash remains recoverable until turn_end. + let accepted = false; const sendSuccessResponse = () => { if (responseSent) return; responseSent = true; @@ -3827,7 +4128,10 @@ export class AgentDaemon { customMessage: command.customMessage, preflightResult: (didSucceed) => { if (didSucceed) { - this.recordWorkerRecoveryState(state, "prompt_accepted", true); + // `prompt` was durably begun at admission. Do not replace + // its identity at acceptance: a delayed A acceptance must not + // overwrite a later B begin in the one-record crash journal. + accepted = true; sendSuccessResponse(); } else { preflightRejected = true; @@ -3842,6 +4146,9 @@ export class AgentDaemon { const error = new Error("Prompt was not accepted by the session."); this.write(client, failure(command.id, "prompt", error, serializeDaemonError(error))); } else { + // Guard legacy prompt implementations that resolve accepted without + // invoking preflightResult. + accepted = true; sendSuccessResponse(); } }) @@ -3852,56 +4159,81 @@ export class AgentDaemon { this.write(client, failure(command.id, "prompt", error, serializeDaemonError(error))); } }) - .finally(clearAdmission); + .finally(() => { + // Only a rejected/cancelled preflight has no terminal turn. An + // accepted non-waiting prompt must retain this queued token until + // its own turn_start/turn_end pair consumes it. + if (!accepted) this.cancelQueuedWorkerRecoveryTurn(state, recoveryToken); + clearAdmission(); + }); return undefined; } case "steer": { const state = this.getBoundSessionState(command.activeSessionId); - if (command.expandPromptTemplates === false) { - await state.runtime.session.restoreSteeringMessage(command.message, command.images, { - queueKey: command.queueKey, - agentMessageId: command.agentMessageId, - content: command.content, - customMessage: command.customMessage, - prefixMessages: command.prefixMessages, - }); - } else { - await state.runtime.session.steer(command.message, command.images, { - queueKey: command.queueKey, - agentMessageId: command.agentMessageId, - resumeIfIdle: true, - }); + const recoveryToken = this.beginWorkerRecoveryOperation(state, "steer_queued"); + // A steer injected during a running turn has no future turn_start; bind + // it to that exact turn so its turn_end clears it. Idle steer awaits one. + if (!this.attachWorkerRecoveryToActiveTurn(state, recoveryToken)) { + this.queueWorkerRecoveryTurn(state, recoveryToken); + } + try { + if (command.expandPromptTemplates === false) { + await state.runtime.session.restoreSteeringMessage(command.message, command.images, { + queueKey: command.queueKey, + agentMessageId: command.agentMessageId, + content: command.content, + customMessage: command.customMessage, + prefixMessages: command.prefixMessages, + }); + } else { + await state.runtime.session.steer(command.message, command.images, { + queueKey: command.queueKey, + agentMessageId: command.agentMessageId, + resumeIfIdle: true, + }); + } + return success(command.id, "steer"); + } catch (error) { + this.cancelQueuedWorkerRecoveryTurn(state, recoveryToken); + throw error; } - this.recordWorkerRecoveryState(state, "steer_queued", true); - return success(command.id, "steer"); } case "follow_up": { const state = this.getBoundSessionState(command.activeSessionId); - let queued = true; - let admitted = true; - if (command.expandPromptTemplates === false) { - queued = await state.runtime.session.restoreFollowUpMessage(command.message, command.images, { - queueKey: command.queueKey, - agentMessageId: command.agentMessageId, - content: command.content, - customMessage: command.customMessage, - prefixMessages: command.prefixMessages, - }); - admitted = queued; - } else { - queued = await state.runtime.session.followUp(command.message, command.images, { - queueKey: command.queueKey, - agentMessageId: command.agentMessageId, - resumeIfIdle: true, - }); - admitted = queued; - } - if (admitted) { - this.recordWorkerRecoveryState(state, "follow_up_queued", true); + // Do not clear this at queue admission. A queued follow-up has not + // terminally run; turn_end will consume this exact FIFO scheduler token. + const recoveryToken = this.beginWorkerRecoveryOperation(state, "follow_up_queued"); + // Publish before calling the session: followUp may synchronously resume + // an idle action before its returned promise continuation runs. + this.queueWorkerRecoveryTurn(state, recoveryToken); + try { + let queued = true; + let admitted = true; + if (command.expandPromptTemplates === false) { + queued = await state.runtime.session.restoreFollowUpMessage(command.message, command.images, { + queueKey: command.queueKey, + agentMessageId: command.agentMessageId, + content: command.content, + customMessage: command.customMessage, + prefixMessages: command.prefixMessages, + }); + admitted = queued; + } else { + queued = await state.runtime.session.followUp(command.message, command.images, { + queueKey: command.queueKey, + agentMessageId: command.agentMessageId, + resumeIfIdle: true, + }); + admitted = queued; + } + if (!admitted) this.cancelQueuedWorkerRecoveryTurn(state, recoveryToken); + return success(command.id, "follow_up", { queued }); + } catch (error) { + this.cancelQueuedWorkerRecoveryTurn(state, recoveryToken); + throw error; } - return success(command.id, "follow_up", { queued }); } case "restore_next_turn": { @@ -3912,9 +4244,55 @@ export class AgentDaemon { case "restore_actions": { const state = this.getSessionState(command.activeSessionId); - const restored = await state.runtime.session.restoreSessionActions(command.snapshot); - if (restored > 0) this.recordWorkerRecoveryState(state, "actions_restored", true); - return success(command.id, "restore_actions", { restored }); + // Reject every snapshot-local invariant before allocating crash evidence. + // In particular, duplicate IDs would collapse a Map entry and make exact + // cleanup impossible if the scheduler rejects the snapshot. + state.runtime.session.validateSessionActionRecoverySnapshot(command.snapshot); + // Allocate one durable identity per declared action before restore can + // mutate the scheduler. This leaves crash evidence fail-closed if the + // worker dies between admission and its response. + const recoveryTokens = new Map(); + for (const action of command.snapshot.actions) { + recoveryTokens.set(action.id, this.beginWorkerRecoveryOperation(state, "actions_restored")); + } + const emptyRecoveryToken = + command.snapshot.actions.length === 0 + ? this.beginWorkerRecoveryOperation(state, "actions_restored") + : undefined; + try { + const restoredActionIds = await state.runtime.session.restoreSessionActions(command.snapshot); + const declaredActionIds = new Set(command.snapshot.actions.map((action) => action.id)); + if ( + new Set(restoredActionIds).size !== restoredActionIds.length || + restoredActionIds.some((actionId) => !declaredActionIds.has(actionId)) + ) { + throw new Error("Restored session action IDs do not match the recovery snapshot"); + } + if (emptyRecoveryToken) this.completeWorkerRecoveryOperation(state, emptyRecoveryToken); + const tokensByActionId = new Map(); + for (const actionId of restoredActionIds) { + const token = recoveryTokens.get(actionId); + if (!token) throw new Error(`Missing recovery token for restored action ${actionId}`); + tokensByActionId.set(actionId, token); + recoveryTokens.delete(actionId); + } + // The restore result identifies every admitted durable action. Bind its + // preallocated token by ID, never snapshot/result position or a global + // unfinished-count delta. This remains correct for partial restores. + if (tokensByActionId.size > 0) { + const recoveries = this.restoredActionRecoveries.get(state) ?? []; + recoveries.push({ tokensByActionId }); + this.restoredActionRecoveries.set(state, recoveries); + } + for (const token of recoveryTokens.values()) this.completeWorkerRecoveryOperation(state, token); + return success(command.id, "restore_actions", { restored: restoredActionIds.length }); + } catch (error) { + // Failure has no reliable action-to-token mapping. Clear only the exact + // identities begun by this call; another restore remains independent. + for (const token of recoveryTokens.values()) this.completeWorkerRecoveryOperation(state, token); + if (emptyRecoveryToken) this.completeWorkerRecoveryOperation(state, emptyRecoveryToken); + throw error; + } } case "append_custom_message": { @@ -5912,7 +6290,12 @@ export class AgentDaemon { this.abortSideQuestionsFor(client, state.activeSessionId); } const existingClose = this.closingSessions.get(state.activeSessionId); - if (existingClose) { + if ( + existingClose && + existingClose.state === state && + existingClose.generation === state.eventGeneration && + (this.sessions.get(state.activeSessionId) === state || this.sessions.get(state.activeSessionId) === undefined) + ) { const requestedReason = this.isStrongerCloseReason(reason, existingClose.reason) ? reason : existingClose.reason; @@ -5925,6 +6308,12 @@ export class AgentDaemon { closeFailed = true; } const reasonUpgrade = (existingClose.reasonUpgrade ?? Promise.resolve()).then(() => { + if ( + (this.sessions.get(state.activeSessionId) !== state && + this.sessions.get(state.activeSessionId) !== undefined) || + state.eventGeneration !== existingClose.generation + ) + return; if (!this.isStrongerCloseReason(requestedReason, existingClose.reason)) return; try { this.applyReasonUpgrade(state, existingClose.descendants, existingClose.reason, requestedReason); @@ -5943,10 +6332,11 @@ export class AgentDaemon { return; } const descendants = new Set(); + const operation: OperationIdentity = { operationId: randomUUID(), generation: state.eventGeneration }; const closePromise = Promise.resolve().then(() => - this.closeSessionOnce(state, reason, waitForAbort, cascadeChildren, descendants), + this.closeSessionOnce(state, reason, waitForAbort, cascadeChildren, descendants, operation), ); - const close = { promise: closePromise, reason, descendants }; + const close = { promise: closePromise, state, generation: operation.generation, operation, reason, descendants }; this.closingSessions.set(state.activeSessionId, close); try { await closePromise; @@ -6019,10 +6409,14 @@ export class AgentDaemon { waitForAbort: boolean, cascadeChildren: boolean, descendants: Set, + operation: OperationIdentity, ): Promise { - if (!this.sessions.has(state.activeSessionId)) { - return; - } + const current = () => + this.sessions.get(state.activeSessionId) === state && state.eventGeneration === operation.generation; + if (!current()) return; + // Begin before the first await; completion receives this exact immutable identity. + const recoveryIdentity = this.recordWorkerRecoveryState(state, `closed:${reason}`, true, undefined, operation); + if (!current()) return; if (reason === "killed") { this.cancelScheduledJobsForSession(state); } else if (reason !== "shutdown" && reason !== "update") { @@ -6034,6 +6428,7 @@ export class AgentDaemon { const cascadeError = cascadeChildren ? await this.closeChildSessions(state, reason, waitForAbort, descendants) : undefined; + if (!current()) return; // Empty draft (no messages, config, or jobs): discard rather than persist an // empty session file. Mirrors the detach-time discard so a config-bearing // draft closed via kill/completed is never wiped. @@ -6043,6 +6438,7 @@ export class AgentDaemon { // Clean shutdown leaves the session un-archived so it stays in the resume list. if (!keepsResumeEntry && !isEmptyDraftSession) { try { + if (!current()) return; this.archiveSession(state); } catch (error) { persistError = error; @@ -6063,14 +6459,18 @@ export class AgentDaemon { } else if (reason === "shutdown" || reason === "replaced") { await state.runtime.session.abort().catch(() => undefined); } - this.recordWorkerRecoveryState(state, `closed:${reason}`, false); + if (!current()) return; + this.recordWorkerRecoveryState(state, `closed:${reason}`, false, recoveryIdentity, operation); + if (!current()) return; state.unsubscribe?.(); + if (!current()) return; let disposeError: unknown; try { await state.runtime.dispose(); } catch (error) { disposeError = error; } + if (!current()) return; for (const client of state.clients) { abortClientSnapshotStreaming(client, state.activeSessionId); } @@ -6080,6 +6480,7 @@ export class AgentDaemon { removeDaemonClientSessionCapabilities(client, state.activeSessionId); } state.clients.clear(); + if (!current()) return; this.sessions.delete(state.activeSessionId); if (isEmptyDraftSession) { const sessionFile = state.runtime.session.sessionFile; @@ -6133,7 +6534,7 @@ export class AgentDaemon { void this.closeSession(state, "killed"); } if (RECOVERY_CHECKPOINT_EVENTS.has(eventType)) { - this.recordWorkerRecoveryState(state, eventType); + this.checkpointWorkerRecoveryEvent(state, eventType as WorkerRecoveryOperation); } } this.stampRlmChildActiveSessionId(message); @@ -6265,26 +6666,188 @@ export class AgentDaemon { } } - private recordWorkerRecoveryState(state: ActiveSessionState, operation: string, busyOverride?: boolean): void { - if (!this.recoveryJournal) { - return; - } + private beginWorkerRecoveryOperation( + state: ActiveSessionState, + operation: WorkerRecoveryOperation, + identity: OperationIdentity = { operationId: randomUUID(), generation: this.workerGeneration }, + ): RecoveryOperationToken { + const token: RecoveryOperationToken = { operation, identity, sequence: ++this.recoveryOperationSequence }; + this.writeWorkerRecoveryOperation(state, token, true); + return token; + } + + private completeWorkerRecoveryOperation(state: ActiveSessionState, token: RecoveryOperationToken): void { + // The journal is intentionally given the token captured at admission. It + // never looks up a later same-family identity, so A cannot clear B. + this.writeWorkerRecoveryOperation(state, token, false); + } + + private writeWorkerRecoveryOperation(state: ActiveSessionState, token: RecoveryOperationToken, busy: boolean): void { + if (!this.recoveryJournal) return; const session = state.runtime.session; - const busy = - busyOverride ?? (isActiveSessionBusy(state) || session.isRetrying || session.hasAcceptedPromptInFlight); try { this.recoveryJournal.record({ activeSessionId: state.activeSessionId, sessionId: session.sessionId, ...(session.sessionFile ? { sessionFile: session.sessionFile } : {}), busy, - operation, + operation: token.operation, + operationId: token.identity.operationId, + generation: token.identity.generation, }); } catch (error) { this.log(`could not checkpoint worker operation state: ${String(error)}`); } } + private recoveryTurnsFor(state: ActiveSessionState): { + pending: RecoveryOperationToken[]; + active: RecoveryOperationToken[][]; + } { + let turns = this.recoveryTurnOperations.get(state); + if (!turns) { + turns = { pending: [], active: [] }; + this.recoveryTurnOperations.set(state, turns); + } + return turns; + } + + private queueWorkerRecoveryTurn(state: ActiveSessionState, token: RecoveryOperationToken): void { + const turns = this.recoveryTurnsFor(state); + if (!turns.pending.includes(token) && !turns.active.some((turn) => turn.includes(token))) { + turns.pending.push(token); + } + } + + private attachWorkerRecoveryToActiveTurn(state: ActiveSessionState, token: RecoveryOperationToken): boolean { + const turn = this.recoveryTurnsFor(state).active.at(-1); + if (!turn) return false; + turn.push(token); + return true; + } + + private cancelQueuedWorkerRecoveryTurn(state: ActiveSessionState, token: RecoveryOperationToken): void { + const turns = this.recoveryTurnsFor(state); + let removed = false; + const pendingIndex = turns.pending.indexOf(token); + if (pendingIndex >= 0) { + turns.pending.splice(pendingIndex, 1); + removed = true; + } + // A steer admitted during an existing turn belongs to that active frame, + // rather than pending a future turn. Its rejection therefore must remove + // this exact token here, at the command boundary, not at an unrelated + // turn_end. Preserve every other frame/token (including nested turns). + for (const turn of turns.active) { + const activeIndex = turn.indexOf(token); + if (activeIndex >= 0) { + turn.splice(activeIndex, 1); + removed = true; + } + } + if (!removed) return; + this.completeWorkerRecoveryOperation(state, token); + } + + private settleRestoredActionRecoveries(state: ActiveSessionState): void { + const recoveries = this.restoredActionRecoveries.get(state); + if (!recoveries) return; + // The action store reports every queued, selected, running, failed, and + // cancelled action as unfinished until it reaches its own terminal release. + // Compare exact durable action IDs, so A's cancellation cannot terminally + // settle B merely because both restores share a session. + const unfinished = new Set(state.runtime.session.unfinishedActionIds); + for (const recovery of recoveries) { + for (const [actionId, token] of recovery.tokensByActionId) { + if (unfinished.has(actionId)) continue; + this.completeWorkerRecoveryOperation(state, token); + recovery.tokensByActionId.delete(actionId); + } + } + const pending = recoveries.filter((recovery) => recovery.tokensByActionId.size > 0); + if (pending.length > 0) this.restoredActionRecoveries.set(state, pending); + else this.restoredActionRecoveries.delete(state); + } + + private checkpointWorkerRecoveryEvent(state: ActiveSessionState, operation: WorkerRecoveryOperation): void { + if (operation === "session_action_update") this.settleRestoredActionRecoveries(state); + const match = /^(.*)_(start|end)$/.exec(operation); + if (!match) { + // These are observations inside a turn/tool, not independently live work. + // Preserve the fsync evidence without stranding a permanent busy record. + const token = this.beginWorkerRecoveryOperation(state, operation); + this.completeWorkerRecoveryOperation(state, token); + return; + } + const [, family, edge] = match; + if (family === "turn") { + const turns = this.recoveryTurnsFor(state); + if (edge === "start") { + const token = turns.pending.shift() ?? this.beginWorkerRecoveryOperation(state, operation); + turns.active.push([token]); + } else { + const tokens = turns.active.pop(); + for (const token of tokens ?? []) this.completeWorkerRecoveryOperation(state, token); + } + return; + } + let queues = this.recoveryEventOperations.get(state); + if (!queues) { + queues = new Map(); + this.recoveryEventOperations.set(state, queues); + } + const queue = queues.get(family) ?? []; + queues.set(family, queue); + if (edge === "start") queue.push(this.beginWorkerRecoveryOperation(state, operation)); + else { + const token = queue.pop(); + if (token) this.completeWorkerRecoveryOperation(state, token); + } + } + + /** Compatibility wrapper for isolated state observations and close callers. */ + private recordWorkerRecoveryState( + state: ActiveSessionState, + operation: WorkerRecoveryOperation, + busyOverride?: boolean, + exactIdentity?: { operation: WorkerRecoveryOperation; operationId: string }, + operationIdentity?: OperationIdentity, + ): { operation: WorkerRecoveryOperation; operationId: string } | undefined { + // The journal key is a worker attempt, never the session event/replay + // generation. Use this daemon worker incarnation for ready, ordinary + // events, and terminal callbacks alike. + const identity: OperationIdentity = { + operationId: operationIdentity?.operationId ?? exactIdentity?.operationId ?? randomUUID(), + generation: this.workerGeneration, + }; + const token: RecoveryOperationToken = { + operation: exactIdentity?.operation ?? operation, + identity, + sequence: ++this.recoveryOperationSequence, + }; + const session = state.runtime.session; + // A compatibility ready record has no lifecycle owner. Its busyness must + // reflect live runtime state, rather than defaulting to a sticky busy bit. + const busy = + busyOverride ?? + (session.isStreaming || + session.isCompacting || + session.isRetrying || + session.hasAcceptedPromptInFlight || + this.agentMessageAcceptingTargets.has(state.activeSessionId) || + (this.agentMessagePreparingTargets.get(state.activeSessionId) ?? 0) > 0); + if (!busy) { + // Compatibility records without an existing identity still need a + // durable non-busy checkpoint. Exact terminal callers must not invent a + // begin, because the journal itself is their stale-callback fence. + if (!exactIdentity) this.beginWorkerRecoveryOperation(state, token.operation, token.identity); + this.completeWorkerRecoveryOperation(state, token); + } else { + this.beginWorkerRecoveryOperation(state, token.operation, token.identity); + } + return { operation: token.operation, operationId: token.identity.operationId }; + } + private catchUpBackpressuredClient(client: DaemonSocketClient): Promise { if (client.catchupPromise) { return client.catchupPromise; diff --git a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts index 9c57e7720..4b2cd349b 100644 --- a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts +++ b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts @@ -61,6 +61,7 @@ import { CommandRecoveryJournal, createCommandIdempotencyKey } from "./command-r import { CompactAssistantStreamReconstructor, isCompactAssistantDelta } from "./compact-session-stream.js"; import { DAEMON_CATALOG_ROLE_ENV, DaemonCatalogClient } from "./daemon-catalog-process.js"; import { deserializeDaemonError, serializeDaemonError } from "./daemon-errors.js"; +import { assertFreshUuid, isCurrentProcessIdentity } from "./daemon-lifecycle-identity.js"; import { collectDaemonClientEnv, createDaemonEventMeta, @@ -123,6 +124,7 @@ import { type DaemonWorkerDescriptor, type DaemonWorkerFrameHeader, isDaemonWorkerLifecycle, + type ResidentDaemonWorkerDescriptor, SESSION_LEASE_OWNER_ID_ENV, SESSION_LEASES_ENABLED_ENV, } from "./daemon-worker-protocol.js"; @@ -152,6 +154,7 @@ const IDLE_EVICTION_DRAIN_TIMEOUT_MS = 5_000; const CHILD_PASSIVATION_PER_WORKER_CAP = 2; const SUPERVISOR_CONFIG_FILE_NAME = "supervisor-config"; const WORKER_STARTUP_GATE_FD = 3; +const C01_IDENTITY_FENCING_ENV = "PRIME_AGENT_ENABLE_C01_IDENTITY_FENCING"; const DAEMON_COMMAND_TYPES: ReadonlySet = new Set([ "ack_result", @@ -253,9 +256,14 @@ const DAEMON_COMMAND_TYPES: ReadonlySet = new Set([ "shutdown", ]); +type WorkerProcessIdentityState = "exact" | "dead" | "recycled" | "unreadable"; + interface ResidentWorker { - descriptor: DaemonWorkerDescriptor; + /** Normalized, generation-bearing runtime state; reader compatibility never escapes loading. */ + descriptor: ResidentDaemonWorkerDescriptor; descriptorPath: string; + /** Untrusted legacy lifecycle evidence is visible but cannot be routed or rewritten. */ + quarantined?: true; client?: DaemonWorkerClient; heartbeatSnapshot?: AgentConnectionHeartbeat[]; heartbeatSnapshotStale?: boolean; @@ -345,6 +353,17 @@ class SupervisorRecoveryCancelledError extends Error { readonly code = "supervisor_recovery_cancelled" as const; } +/** + * A recovery attempt publishes a replacement generation before it can connect + * or complete its create handshake. Keep that exact attempt's identity with + * its failure so the recovery loop can distinguish it from a real replacement + * that raced the old generation. + */ +const workerLaunchFailureAttempts = new WeakMap< + object, + { worker: ResidentWorker; generation: string; cleanupVerified: boolean } +>(); + class SnapshotLoadInvalidatedError extends Error {} function isSupervisorGenerationStale(error: unknown): boolean { @@ -378,7 +397,7 @@ function unrefDelay(ms: number): Promise { return new Promise((resolveDelay) => setTimeout(resolveDelay, ms).unref()); } -function commitWorkerStartupGate(gate: Writable): Promise { +function commitWorkerStartupGate(gate: Writable, generation: string): Promise { return new Promise((resolveCommit, rejectCommit) => { let settled = false; const finish = (error?: Error | null) => { @@ -395,7 +414,7 @@ function commitWorkerStartupGate(gate: Writable): Promise { const onError = (error: Error) => finish(error); gate.on("error", onError); gate.once("close", () => gate.off("error", onError)); - gate.end(DAEMON_WORKER_STARTUP_GATE_COMMIT, (error?: Error | null) => finish(error)); + gate.end(`${DAEMON_WORKER_STARTUP_GATE_COMMIT}${generation}\n`, (error?: Error | null) => finish(error)); }); } @@ -424,10 +443,17 @@ function isSessionSummary(value: unknown): value is SessionSummary { } function isDaemonWorkerDescriptor(value: unknown, socketPath: string): value is DaemonWorkerDescriptor { - if (!value || typeof value !== "object") { - return false; - } + if (!value || typeof value !== "object") return false; const descriptor = value as Partial; + const process = descriptor.process; + const validProcess = + !!process && + Number.isInteger(process.pid) && + process.pid > 0 && + typeof process.processStartId === "string" && + !!process.processStartId; + // Legacy records may be observed for conservative adoption only. They are never + // signal authority and are rewritten only after a fresh identity is observed. const validLegacyProcess = Number.isInteger(descriptor.pid) && (descriptor.pid ?? 0) > 0 && @@ -436,27 +462,31 @@ function isDaemonWorkerDescriptor(value: unknown, socketPath: string): value is // process identity it retains, however, must be a complete, valid pair: a // partial or object-shaped identity is untrusted input, not an invitation to // probe, signal, or passivate an arbitrary process. + const noLegacyProcess = descriptor.pid === undefined && descriptor.processStartId === undefined; + const validCompleteLegacyProcess = + process === undefined && + Number.isInteger(descriptor.pid) && + (descriptor.pid ?? 0) > 0 && + typeof descriptor.processStartId === "string" && + descriptor.processStartId.length > 0; const validRecoveringProcess = - (descriptor.pid === undefined && descriptor.processStartId === undefined) || - (Number.isInteger(descriptor.pid) && - (descriptor.pid ?? 0) > 0 && - typeof descriptor.processStartId === "string" && - descriptor.processStartId.length > 0); + (process === undefined && noLegacyProcess) || (validProcess && noLegacyProcess) || validCompleteLegacyProcess; const knownLifecycle = isDaemonWorkerLifecycle(descriptor.lifecycle); + const passivated = descriptor.lifecycle === "passivated"; + const validGeneration = descriptor.generation === undefined || assertFreshUuid(descriptor.generation); return ( descriptor.version === 1 && descriptor.supervisorSocketPath === socketPath && typeof descriptor.workerId === "string" && - // A passivated descriptor is intentionally processless. `recovering` is - // also processless after normalizing a legacy missing/unknown lifecycle: - // retaining it lets the next supervisor recover the root without treating - // a stale PID as safe to adopt or signal. Other lifecycle states still need - // a valid process identity, and unknown legacy states do too until their - // first normalization pass, so malformed input remains fail-closed. + // Early v1 records may lack a known lifecycle. Retain them only with a + // structurally valid process identity so load can normalize to recovery. + // Passivated rows and normalized processless recovering rows remain durable + // metadata, but no process field is signal authority until revalidated. (knownLifecycle - ? descriptor.lifecycle === "passivated" || - (descriptor.lifecycle === "recovering" ? validRecoveringProcess : validLegacyProcess) - : validLegacyProcess) && + ? passivated || + (descriptor.lifecycle === "recovering" ? validRecoveringProcess : validProcess || validLegacyProcess) + : validProcess || validLegacyProcess) && + validGeneration && (descriptor.ownerClientId === undefined || typeof descriptor.ownerClientId === "string") && typeof descriptor.socketPath === "string" && typeof descriptor.authenticationToken === "string" && @@ -469,7 +499,6 @@ function isDaemonWorkerDescriptor(value: unknown, socketPath: string): value is descriptor.createCommand.type === "create" ); } - function sessionSummariesFromResponse(response: DaemonResponse): SessionSummary[] { if (!response.success || !response.data || typeof response.data !== "object" || !("sessions" in response.data)) { throw new Error("Session worker returned an invalid list response"); @@ -653,6 +682,8 @@ export class DaemonSupervisor { private idleEvictionTimer?: ReturnType; private idleEvictionSweep?: Promise; private idleEvictionFence?: Promise; + // Private server-only incident escape hatch. It can relax callback rejection, never process identity/signal checks. + private readonly c01IdentityFencingEnabled = process.env[C01_IDENTITY_FENCING_ENV] !== "0"; constructor( private readonly socketPath: string, @@ -677,6 +708,11 @@ export class DaemonSupervisor { async start(): Promise { try { + if (!this.c01IdentityFencingEnabled) { + this.log( + `${C01_IDENTITY_FENCING_ENV}=0: callback identity rejection is temporarily disabled; process identity and signal safety remain enforced`, + ); + } const agentDir = this.defaultSessionConfig.agentDir; if (!agentDir) { throw new Error("Daemon supervisor config is missing agentDir"); @@ -700,7 +736,7 @@ export class DaemonSupervisor { this.commandJournal = new CommandRecoveryJournal(join(this.descriptorDir, "command-journal.jsonl")); await this.loadWorkerDescriptors(); const workersToAdopt = [...this.workers.values()].filter( - (worker) => worker.descriptor.lifecycle !== "passivated", + (worker) => !worker.quarantined && worker.descriptor.lifecycle !== "passivated", ); this.server = createServer((socket) => this.handleConnection(socket)); @@ -967,22 +1003,78 @@ export class DaemonSupervisor { if (name === SUPERVISOR_CONFIG_FILE_NAME || !name.endsWith(".json")) continue; const path = join(this.descriptorDir, name); try { - const descriptor: unknown = JSON.parse(readFileSync(path, "utf8")); - if (!isDaemonWorkerDescriptor(descriptor, this.socketPath)) continue; + const diskDescriptor: unknown = JSON.parse(readFileSync(path, "utf8")); + if (!isDaemonWorkerDescriptor(diskDescriptor, this.socketPath)) continue; + // Never mutate the parsed disk object: malformed lifecycle records are + // reader evidence, not a migration opportunity. + const descriptor = { + ...diskDescriptor, + ...(diskDescriptor.process ? { process: { ...diskDescriptor.process } } : {}), + } as DaemonWorkerDescriptor; const malformedLifecycle = !isDaemonWorkerLifecycle(descriptor.lifecycle); + let descriptorMigrated = false; + let descriptorPersisted = false; if (malformedLifecycle) { - // A legacy v1 descriptor can omit lifecycle (or contain a value from a - // newer writer). It is not evidence that the root is idle or stopped. - // Discard even a syntactically valid legacy PID before recovery: do not - // adopt, passivate, or signal a process based on malformed lifecycle. + // A missing or unknown lifecycle is neither process nor lifecycle + // authority. Keep a normalized in-memory view solely so operators can + // inspect it; no C01 path may adopt, wake, signal, or rewrite it. descriptor.lifecycle = "recovering"; + delete descriptor.process; delete descriptor.pid; delete descriptor.processStartId; + descriptor.generation = assertFreshUuid(descriptor.generation) ? descriptor.generation : randomUUID(); } descriptor.recoveryJournalPath ??= join(this.descriptorDir, `${descriptor.workerId}.recovery.jsonl`); descriptor.orphanProcessJournalPath ??= join(this.descriptorDir, `${descriptor.workerId}.orphans.jsonl`); + // Reader compatibility ends here. Normalize both old flat selectors and + // nested pre-C01 records before constructing a resident object. In + // particular, make this durable before any async summary/adoption work. + const alreadyPassivated = descriptor.lifecycle === "passivated"; + // A nested selector with no generation predates C01 just as a flat PID + // does. It becomes process authority only after its exact start ID can + // be observed again; otherwise it is raw migration evidence. + const legacyNestedIdentity = descriptor.process !== undefined && descriptor.generation === undefined; + if (alreadyPassivated) { + if (descriptor.process || descriptor.pid !== undefined || descriptor.processStartId !== undefined) { + delete descriptor.process; + delete descriptor.pid; + delete descriptor.processStartId; + descriptorMigrated = true; + } + } else if (!descriptor.process && descriptor.pid !== undefined) { + // A v1 flat PID is evidence only. Promote it only after observing the + // same live start ID; it is never signal authority before that point. + const observedStartId = getProcessStartId(descriptor.pid); + if ( + observedStartId && + (descriptor.processStartId === undefined || descriptor.processStartId === observedStartId) + ) { + descriptor.process = { pid: descriptor.pid, processStartId: observedStartId }; + } + delete descriptor.pid; + delete descriptor.processStartId; + descriptorMigrated = true; + } else if ( + legacyNestedIdentity && + descriptor.process !== undefined && + !isCurrentProcessIdentity(descriptor.process) + ) { + // Do not turn an unobservable pre-C01 nested PID into a durable + // generation-bearing recovery record. If passive classification later + // rejects it because work is recoverable, quarantine keeps the exact + // raw disk evidence for explicit repair instead. + delete descriptor.process; + descriptorMigrated = true; + } + if (!descriptor.generation) { + // Both legacy forms receive a fresh incarnation before resident state + // exists. A dead/processless row still needs it because explicit wake + // and its callbacks use the same resident representation. + descriptor.generation = randomUUID(); + descriptorMigrated = true; + } const worker: ResidentWorker = { - descriptor, + descriptor: descriptor as ResidentDaemonWorkerDescriptor, descriptorPath: path, summaries: new Map(), snapshotCache: new Map(), @@ -993,53 +1085,63 @@ export class DaemonSupervisor { stopRevision: 0, }; if (malformedLifecycle) { - // The descriptor remains visible to startup recovery, but malformed - // lifecycle is never allowed to reach passivation classification. - this.persistWorker(worker); + // Quarantine before any asynchronous classification. In particular, + // leave the exact disk bytes unchanged across every supervisor reload. + worker.quarantined = true; this.workers.set(descriptor.workerId, worker); continue; } // Never passivate a live process: adoption is the only safe way to - // reconnect work that may still be running. A legacy passivated v1 - // descriptor is deliberately migrated by discarding its stale identity. - const alreadyPassivated = descriptor.lifecycle === "passivated"; - // Old C00 records could claim to be passivated while retaining a PID. - // Discard it before *any* recovery classification, including malformed - // JSONL that must recover, so no later consumer can act on it. - if (alreadyPassivated) { - delete descriptor.pid; - delete descriptor.processStartId; - } + // reconnect work that may still be running. // A client-owned worker's launch environment is transient and deliberately // never persisted. A processless descriptor therefore cannot be safely // restarted at supervisor startup: only its owner can provide that env on a // subsequent attach. Keep it processless, visible, and wakeable by that path. - const ownerOwnedProcessless = descriptor.ownerClientId !== undefined && descriptor.pid === undefined; + const ownerOwnedProcessless = descriptor.ownerClientId !== undefined && descriptor.process === undefined; if (ownerOwnedProcessless && !descriptor.stopRequestedAt) { - // Fail closed even if the durable transcript is unreadable or has work - // pending. The owner attach path identifies this root from its descriptor - // and supplies launchEnv before it asks recovery to spawn anything. - descriptor.lifecycle = "passivated"; const passive = await this.passivatedSummaryForDescriptor(descriptor); + // Owner-owned C01 roots have no relaunch authority until the owner + // reconnects. That is distinct from a legacy selector whose identity + // could not be observed: if recovery work rejects passivation, retain + // the raw migration evidence instead of laundering it into a passive + // (and later processless recovering) C01 descriptor. + if (descriptorMigrated && !passive) { + worker.quarantined = true; + this.workers.set(descriptor.workerId, worker); + continue; + } + descriptor.lifecycle = "passivated"; if (passive) worker.summaries.set(descriptor.rootActiveSessionId, passive); this.persistWorker(worker); + descriptorPersisted = true; } else { const passive = descriptor.ownerClientId === undefined && !descriptor.stopRequestedAt && - (alreadyPassivated || descriptor.pid === undefined || !isProcessAlive(descriptor.pid)) + (alreadyPassivated || descriptor.process === undefined || !isProcessAlive(descriptor.process?.pid)) ? await this.passivatedSummaryForDescriptor(descriptor) : undefined; if (passive) { descriptor.lifecycle = "passivated"; + delete descriptor.process; delete descriptor.pid; delete descriptor.processStartId; worker.summaries.set(descriptor.rootActiveSessionId, passive); this.persistWorker(worker); + descriptorPersisted = true; } else { descriptor.lifecycle = "recovering"; } } + // A reader migration that cannot reach an explicitly passive state has + // lost process authority. It is evidence, not durable C01 recovery + // state: retain only a quarantined in-memory view and preserve raw disk. + if (descriptorMigrated && descriptor.process === undefined && descriptor.lifecycle !== "passivated") { + worker.quarantined = true; + this.workers.set(descriptor.workerId, worker); + continue; + } + if (descriptorMigrated && !descriptorPersisted) this.persistWorker(worker); this.workers.set(descriptor.workerId, worker); } catch (error) { this.log(`Ignoring invalid worker descriptor ${path}: ${String(error)}`); @@ -1117,9 +1219,42 @@ export class DaemonSupervisor { } private persistWorker(worker: ResidentWorker): void { - worker.descriptor.updatedAt = new Date().toISOString(); + if (worker.quarantined) { + throw new Error(`Refusing to rewrite quarantined worker ${worker.descriptor.workerId}`); + } + const { descriptor } = worker; + if (!assertFreshUuid(descriptor.generation)) { + throw new Error(`Refusing to persist worker ${descriptor.workerId} without a canonical generation`); + } + const identity = descriptor.process; + const hasNestedProcess = identity !== undefined; + if ( + hasNestedProcess && + (!identity || + !Number.isInteger(identity.pid) || + identity.pid <= 0 || + typeof identity.processStartId !== "string" || + !identity.processStartId) + ) { + throw new Error(`Refusing to persist worker ${descriptor.workerId} with an invalid process identity`); + } + // This is the C01 durable discriminator: only a deliberately passivated + // root may be processless. In particular, do not turn a pre-spawn recovery + // intent or failed launch into a durable recovering/failed descriptor. + if (descriptor.lifecycle !== "passivated" && !hasNestedProcess) { + throw new Error( + `Refusing to persist ${descriptor.lifecycle} worker ${descriptor.workerId} without a process identity`, + ); + } + if (descriptor.lifecycle === "passivated" && hasNestedProcess) { + throw new Error(`Refusing to persist passivated worker ${descriptor.workerId} with a process identity`); + } + descriptor.updatedAt = new Date().toISOString(); + // Keep the permissive v1 reader shape out of every C01 write, even if an + // untyped test/integration object accidentally reintroduces a legacy key. + const { pid: _legacyPid, processStartId: _legacyStartId, ...persisted } = descriptor; const tempPath = `${worker.descriptorPath}.${process.pid}.tmp`; - writeFileSync(tempPath, `${JSON.stringify(worker.descriptor, null, 2)}\n`, { mode: 0o600 }); + writeFileSync(tempPath, `${JSON.stringify(persisted, null, 2)}\n`, { mode: 0o600 }); chmodSync(tempPath, 0o600); renameSync(tempPath, worker.descriptorPath); } @@ -1689,8 +1824,8 @@ export class DaemonSupervisor { const finalizations = this.waitForStopFinalizations(worker); if (finalizations) await finalizations; if (worker.stopFinalized) throw new Error(`Session worker ${worker.descriptor.workerId} was stopped`); - const processless = worker.descriptor.lifecycle === "passivated" || worker.descriptor.pid === undefined; - if (!processless && isProcessAlive(worker.descriptor.pid!)) { + const processless = worker.descriptor.lifecycle === "passivated" || worker.descriptor.process === undefined; + if (!processless && isProcessAlive(worker.descriptor.process!.pid)) { throw new Error(`Session worker ${worker.descriptor.workerId} is still running; cannot retry its stop`); } worker.stopFailure = undefined; @@ -1700,7 +1835,7 @@ export class DaemonSupervisor { worker.descriptor.archiveOnStop = undefined; worker.descriptor.lifecycle = "recovering"; worker.descriptor.consecutiveFailures = 0; - this.persistWorker(worker); + // Retry shares wake's publication rule: launch publishes identity first. await this.recoverWorker(worker); if (this.workers.get(worker.descriptor.workerId)?.descriptor.lifecycle !== "ready") { throw new Error(worker.descriptor.lastError ?? "Session worker recovery failed"); @@ -2239,6 +2374,9 @@ export class DaemonSupervisor { } private async wakePassivatedWorker(worker: ResidentWorker): Promise { + if (worker.quarantined) { + throw new Error(`Session worker ${worker.descriptor.workerId} is quarantined pending lifecycle repair`); + } // Do not introduce an await when no stop exists: that would leave a gap in // which a concurrent stop could install its tombstone before this wake starts. const stopFence = this.stopFenceForWake(worker); @@ -2256,7 +2394,9 @@ export class DaemonSupervisor { worker.descriptor.archiveOnStop = undefined; worker.descriptor.lifecycle = "recovering"; worker.descriptor.consecutiveFailures = 0; - this.persistWorker(worker); + // This is only an in-memory launch intent. The first wake write is + // launchWorker's identity-bearing `starting` record; never crash with a + // processless recovering descriptor merely because a wake was admitted. await this.recoverWorker(worker); // recoverWorker mutates lifecycle through the normal launch/adoption // path; read it after await rather than retaining the narrowed value. @@ -2319,6 +2459,30 @@ export class DaemonSupervisor { await this.syncAgentPeers().catch((error) => this.log(`Could not synchronize agent peers: ${String(error)}`)); } + private recordWorkerLaunchFailure(error: unknown, worker: ResidentWorker, generation: string): void { + if (typeof error === "object" && error !== null) { + // Cleanup has not yet proved that a retry owns no live process. + workerLaunchFailureAttempts.set(error, { worker, generation, cleanupVerified: false }); + } + } + + private markWorkerLaunchFailureCleanupVerified(error: unknown, worker: ResidentWorker, generation: string): void { + if (typeof error !== "object" || error === null) return; + const attempt = workerLaunchFailureAttempts.get(error); + if (attempt?.worker === worker && attempt.generation === generation) { + attempt.cleanupVerified = true; + } + } + + private workerLaunchFailureAttempt( + error: unknown, + worker: ResidentWorker, + ): { generation: string; cleanupVerified: boolean } | undefined { + if (typeof error !== "object" || error === null) return undefined; + const attempt = workerLaunchFailureAttempts.get(error); + return attempt?.worker === worker ? attempt : undefined; + } + private async launchWorker( command: DaemonCreateCommand, existing?: ResidentWorker, @@ -2346,13 +2510,15 @@ export class DaemonSupervisor { const orphanProcessJournalPath = existing?.descriptor.orphanProcessJournalPath ?? join(this.descriptorDir, `${workerId}.orphans.jsonl`); const launch = createCliSubprocessLaunchSpec(["--mode", "daemon", "--daemon-socket", socketPath]); + // This gate is supervisor-only. Never leak an incident rollback control to a worker. + const workerEnvironment: NodeJS.ProcessEnv = { ...process.env, ...launchEnv }; + delete workerEnvironment[C01_IDENTITY_FENCING_ENV]; await this.assertRecoveryAllowed(); const child: ChildProcess = spawn(launch.command, launch.args, { cwd: createCommand.config?.cwd ?? process.cwd(), detached: true, env: createCliSubprocessEnv({ - ...process.env, - ...launchEnv, + ...workerEnvironment, [DAEMON_WORKER_ROLE_ENV]: "1", [DAEMON_WORKER_TOKEN_ENV]: token, [DAEMON_WORKER_ACTIVE_SESSION_ID_ENV]: rootActiveSessionId, @@ -2384,6 +2550,7 @@ export class DaemonSupervisor { let descriptorAssigned = false; let childPid: number; let childProcessStartId: string | undefined; + let workerGeneration: string; let worker: ResidentWorker; try { if (!child.pid) { @@ -2394,13 +2561,19 @@ export class DaemonSupervisor { } childPid = child.pid; childProcessStartId = getProcessStartId(childPid); + if (!childProcessStartId) { + throw new Error("Cannot safely launch daemon session worker without a process start identity"); + } + // This is deliberately after successful start-ID observation. The child + // cannot begin until the gate below forwards this committed value. + workerGeneration = randomUUID(); await this.assertRecoveryAllowed(); - const descriptor: DaemonWorkerDescriptor = { + const descriptor: ResidentDaemonWorkerDescriptor = { version: 1, workerId, - pid: childPid, - ...(childProcessStartId ? { processStartId: childProcessStartId } : {}), + process: { pid: childPid, processStartId: childProcessStartId }, + generation: workerGeneration, socketPath, recoveryJournalPath, orphanProcessJournalPath, @@ -2437,6 +2610,13 @@ export class DaemonSupervisor { if (startupGate instanceof Writable) { startupGate.destroy(); } + // Publication has not happened. This direct child object is ours, unlike a + // descriptor PID, so terminate it rather than leaving a gate-dependent orphan. + try { + child.kill("SIGTERM"); + } catch { + // It may have already observed the closed gate and exited. + } await childClosed; child.unref(); try { @@ -2456,7 +2636,7 @@ export class DaemonSupervisor { try { try { - await commitWorkerStartupGate(startupGate); + await commitWorkerStartupGate(startupGate, workerGeneration); } catch (error) { startupGate.destroy(); await childClosed; @@ -2465,6 +2645,9 @@ export class DaemonSupervisor { child.unref(); } const client = await this.connectWorker(worker, WORKER_CONNECT_TIMEOUT_MS); + if (!this.matchesCurrentWorker(worker, workerGeneration)) { + throw new Error(`Session worker ${workerId} launch was superseded`); + } const response = await client.request(withoutCommandId(createCommand), WORKER_REQUEST_TIMEOUT_MS); if (!response.success) { throw deserializeDaemonError(response); @@ -2473,6 +2656,9 @@ export class DaemonSupervisor { throw new Error("Session worker returned an invalid create response"); } const summary = response.data; + if (!this.matchesCurrentWorker(worker, workerGeneration)) { + throw new Error(`Session worker ${workerId} launch response was superseded`); + } if ((summary.activeSessionId ?? summary.id) !== rootActiveSessionId) { throw new Error("Session worker did not preserve its assigned active session id"); } @@ -2480,11 +2666,20 @@ export class DaemonSupervisor { worker.descriptor.rootSessionId = summary.sessionId; worker.descriptor.sessionFile = summary.sessionFile; await this.subscribeWorker(worker, rootActiveSessionId); + if (!this.matchesCurrentWorker(worker, workerGeneration)) { + throw new Error(`Session worker ${workerId} launch subscription was superseded`); + } await this.refreshWorkerSummaries(worker, true); - if (existing && (this.isWorkerRecoveryCancelled(worker) || worker.stopRevision !== recoveryStopRevision)) { - throw new Error(`Session worker ${workerId} recovery was cancelled`); + if ( + !this.matchesCurrentWorker(worker, workerGeneration) || + (existing && (this.isWorkerRecoveryCancelled(worker) || worker.stopRevision !== recoveryStopRevision)) + ) { + throw new Error(`Session worker ${workerId} launch was superseded or cancelled`); } await this.assertRecoveryAllowed(); + if (!this.matchesCurrentWorker(worker, workerGeneration)) { + throw new Error(`Session worker ${workerId} launch was superseded`); + } worker.descriptor.lifecycle = "ready"; worker.descriptor.consecutiveFailures = 0; worker.descriptor.lastError = undefined; @@ -2530,25 +2725,70 @@ export class DaemonSupervisor { throw error; } await this.assertRecoveryAllowed(); - const shouldResumeRecovery = + const ownsPublishedAttempt = existing !== undefined && + this.matchesCurrentWorker(worker, workerGeneration) && !this.shuttingDown && worker.descriptor.stopRequestedAt === undefined && worker.stopRevision === recoveryStopRevision; - await this.stopWorker(worker, existing === undefined, true, false, existing !== undefined).catch((stopError) => - this.log(`Could not stop failed worker ${workerId}: ${String(stopError)}`), - ); + // Tag before cleanup: cleanup may correctly restore a processless + // recovering descriptor, but the recovery loop must still know that this + // particular newly-published generation failed rather than was replaced. + if (ownsPublishedAttempt) { + this.recordWorkerLaunchFailure(error, worker, workerGeneration); + } + let failedWorkerStopped = false; + try { + await this.stopWorker(worker, existing === undefined, true, false, existing !== undefined); + failedWorkerStopped = true; + } catch (stopError) { + this.log(`Could not stop failed worker ${workerId}: ${String(stopError)}`); + // We cannot prove that this published process is gone. Preserve its + // current identity and durably mark the outcome failed; do not clear it, + // signal again, or silently leave a process-bearing `recovering` record. + if ( + ownsPublishedAttempt && + this.matchesCurrentWorker(worker, workerGeneration) && + worker.descriptor.stopRequestedAt === undefined && + worker.stopRevision === recoveryStopRevision + ) { + worker.descriptor.lifecycle = "failed"; + worker.descriptor.lastError = `Failed launch cleanup could not verify worker exit: ${ + stopError instanceof Error ? stopError.message : String(stopError) + }`; + try { + this.persistWorker(worker); + } catch (persistError) { + this.reportCleanupFailure(`failed worker launch ${workerId}`, persistError); + } + } + } if ( - shouldResumeRecovery && + ownsPublishedAttempt && + failedWorkerStopped && + // stopWorker removes its own completed generation. An absent map entry + // is therefore still ours; another resident object is a replacement + // and must never be overwritten by this stale recovery. + (this.workers.get(workerId) === undefined || this.matchesCurrentWorker(worker, workerGeneration)) && !this.shuttingDown && worker.descriptor.stopRequestedAt === undefined && worker.stopRevision === recoveryStopRevision ) { + this.markWorkerLaunchFailureCleanupVerified(error, worker, workerGeneration); await this.assertRecoveryAllowed(); + // stopWorker verified that this newly-published process is gone. Do not + // leave its now-stale identity as authority for the next retry. + delete worker.descriptor.process; + delete worker.descriptor.pid; + delete worker.descriptor.processStartId; worker.intentionalStop = false; + worker.stopFinalized = undefined; + worker.stopFailure = undefined; + // A recovering descriptor is durable only while it carries an exact + // process identity. This is intentionally in-memory state until the + // retry publishes its own process-bearing generation. worker.descriptor.lifecycle = "recovering"; this.workers.set(workerId, worker); - this.persistWorker(worker); } throw error; } @@ -2556,23 +2796,33 @@ export class DaemonSupervisor { private async connectWorker(worker: ResidentWorker, timeoutMs: number): Promise { const deadline = Date.now() + timeoutMs; + const generation = worker.descriptor.generation; + if (!generation || !this.matchesCurrentWorker(worker, generation)) { + throw new Error(`Session worker ${worker.descriptor.workerId} connect was superseded`); + } let lastError: unknown; while (Date.now() < deadline) { await this.assertRecoveryAllowed(); + if (!this.matchesCurrentWorker(worker, generation)) { + throw new Error(`Session worker ${worker.descriptor.workerId} connect was superseded`); + } const client = new DaemonWorkerClient(worker.descriptor.socketPath); try { await client.connect(Math.min(500, Math.max(50, deadline - Date.now()))); + if (!this.matchesCurrentWorker(worker, generation)) throw new Error("Worker connect superseded"); await client.waitForHello(1000); + if (!this.matchesCurrentWorker(worker, generation)) throw new Error("Worker hello superseded"); await client.authenticateWorker( worker.descriptor.authenticationToken, this.supervisorAuthenticationClaim(), 1000, ); await this.assertRecoveryAllowed(); - client.onFrame((frame) => this.handleWorkerFrame(worker, frame)); - client.onClose((error) => void this.handleWorkerClose(worker, client, error)); + if (!this.matchesCurrentWorker(worker, generation)) throw new Error("Worker authentication superseded"); worker.client?.close(); worker.client = client; + client.onFrame((frame) => this.handleWorkerFrame(worker, frame, generation, client)); + client.onClose((error) => void this.handleWorkerClose(worker, client, error, generation)); return client; } catch (error) { lastError = error; @@ -2606,12 +2856,77 @@ export class DaemonSupervisor { } } + /** + * C01 process fence. A missing start-id observation is deliberately not a + * death observation: process metadata can be transiently unreadable while a + * just-started worker is still live. Only ESRCH independently proves death. + */ + private classifyWorkerProcessIdentity(worker: ResidentWorker): WorkerProcessIdentityState { + if (worker.quarantined) return "unreadable"; + const identity = worker.descriptor.process; + if (!identity || !Number.isInteger(identity.pid) || identity.pid <= 0 || !identity.processStartId) { + return "unreadable"; + } + let observedProcessStartId: string | undefined; + try { + observedProcessStartId = getProcessStartId(identity.pid); + } catch { + return "unreadable"; + } + if (observedProcessStartId === identity.processStartId) return "exact"; + if (observedProcessStartId !== undefined) return "recycled"; + try { + process.kill(identity.pid, 0); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ESRCH") return "dead"; + } + return "unreadable"; + } + + private matchesCurrentWorker(worker: ResidentWorker, generation: string): boolean { + return this.workers.get(worker.descriptor.workerId) === worker && worker.descriptor.generation === generation; + } + + /** Callback fence only: the emergency gate never affects process/signal authority. */ + private acceptsWorkerCallback(worker: ResidentWorker, generation: string, client?: DaemonWorkerClient): boolean { + if (!this.c01IdentityFencingEnabled) return client === undefined || worker.client === client; + return this.matchesCurrentWorker(worker, generation) && (client === undefined || worker.client === client); + } + + private signalTrackedWorkerState( + worker: ResidentWorker, + generation: string, + signal: NodeJS.Signals, + ): WorkerProcessIdentityState { + if (!this.matchesCurrentWorker(worker, generation)) { + this.log(`Refusing ${signal}: stale worker generation for ${worker.descriptor.workerId}`); + return "unreadable"; + } + // Re-read immediately before signaling. In particular, an unreadable + // process-start value never becomes permission to signal or finalize. + const identity = worker.descriptor.process; + const state = this.classifyWorkerProcessIdentity(worker); + if (state !== "exact" || !identity) { + this.log(`Refusing ${signal}: ${state} process identity for ${worker.descriptor.workerId}`); + return state === "exact" ? "unreadable" : state; + } + signalProcessGroupOrProcess(identity.pid, signal); + return state; + } + + private signalTrackedWorker(worker: ResidentWorker, generation: string, signal: NodeJS.Signals): boolean { + return this.signalTrackedWorkerState(worker, generation, signal) === "exact"; + } + private async adoptOrRecoverWorker(worker: ResidentWorker): Promise { + const generation = worker.descriptor.generation; + if (!generation || !this.matchesCurrentWorker(worker, generation)) return; await this.assertRecoveryAllowed(); + if (!this.matchesCurrentWorker(worker, generation)) return; // A processless passivated descriptor (or a corrupt record that was // conservatively moved to recovery) has no PID authority. Never feed an // absent or stale identity into adoption/cleanup; recovery launches anew. - if (worker.descriptor.pid === undefined) { + if (worker.descriptor.process === undefined) { if (worker.descriptor.stopRequestedAt) { await this.stopWorker(worker, true, true, worker.descriptor.archiveOnStop === true); } else { @@ -2623,7 +2938,7 @@ export class DaemonSupervisor { try { // A tombstoned worker must not run long enough to elect another // supervisor while its intentional stop is being adopted. - signalProcessGroupOrProcess(worker.descriptor.pid!, "SIGKILL"); + this.signalTrackedWorker(worker, worker.descriptor.generation ?? "", "SIGKILL"); await this.stopWorker(worker, true, true, worker.descriptor.archiveOnStop === true); this.log(`Completed intentional stop for worker ${worker.descriptor.workerId} during supervisor adoption`); } catch (error) { @@ -2635,17 +2950,24 @@ export class DaemonSupervisor { return; } try { - if (!isProcessAlive(worker.descriptor.pid!)) { + if (!isProcessAlive(worker.descriptor.process!.pid)) { throw new Error("Session worker process is no longer running"); } - const observedProcessStartId = getProcessStartId(worker.descriptor.pid!); + const observedProcessStartId = getProcessStartId(worker.descriptor.process!.pid); + if (observedProcessStartId !== worker.descriptor.process!.processStartId) { + throw new Error("Session worker process identity changed before adoption"); + } await this.connectWorker(worker, 2000); + if (!this.matchesCurrentWorker(worker, generation)) return; await this.subscribeWorker(worker, worker.descriptor.rootActiveSessionId); + if (!this.matchesCurrentWorker(worker, generation)) return; await this.refreshWorkerSummaries(worker, true); - if (worker.descriptor.processStartId === undefined && observedProcessStartId) { - worker.descriptor.processStartId = observedProcessStartId; + if (!this.matchesCurrentWorker(worker, generation)) return; + if (worker.descriptor.process?.processStartId === undefined && observedProcessStartId) { + if (worker.descriptor.process) worker.descriptor.process.processStartId = observedProcessStartId; } await this.assertRecoveryAllowed(); + if (!this.matchesCurrentWorker(worker, generation)) return; worker.descriptor.lifecycle = "ready"; worker.descriptor.consecutiveFailures = 0; this.persistWorker(worker); @@ -2659,8 +2981,13 @@ export class DaemonSupervisor { } } - private async handleWorkerClose(worker: ResidentWorker, client: DaemonWorkerClient, error: Error): Promise { - if (worker.client !== client) { + private async handleWorkerClose( + worker: ResidentWorker, + client: DaemonWorkerClient, + error: Error, + generation?: string, + ): Promise { + if (generation !== undefined && !this.acceptsWorkerCallback(worker, generation, client)) { return; } worker.client = undefined; @@ -2703,7 +3030,14 @@ export class DaemonSupervisor { } return; } - if (!this.isWorkerRecoveryEligible(worker)) { + if ( + !this.isWorkerRecoveryEligible(worker) || + // The first callback fence deliberately consumed `client` by clearing it. + // Requiring worker.client === client here would strand this exact + // incarnation after assertRecoveryAllowed() yields. A duplicate close still + // fails the entry fence; this continuation needs only identity authority. + (generation !== undefined && !this.matchesCurrentWorker(worker, generation)) + ) { return; } worker.descriptor.lifecycle = "recovering"; @@ -2719,6 +3053,7 @@ export class DaemonSupervisor { private isWorkerRecoveryCandidate(worker: ResidentWorker): boolean { return ( + !worker.quarantined && !this.shuttingDown && !worker.intentionalStop && worker.descriptor.stopRequestedAt === undefined && @@ -2732,9 +3067,17 @@ export class DaemonSupervisor { if (worker.deferredRecovery) { return; } - worker.deferredRecovery = this.resumeDeferredWorkerRecovery(worker, disconnectError).finally(() => { - worker.deferredRecovery = undefined; + let deferred!: Promise; + deferred = this.resumeDeferredWorkerRecovery(worker, disconnectError).finally(() => { + // This field is a join handle, not a lifecycle authority. A successful + // recovery may publish a new generation before it settles; promise + // equality alone both releases this exact settled cycle and protects a + // newer deferred cycle that replaced it. + if (worker.deferredRecovery === deferred) { + worker.deferredRecovery = undefined; + } }); + worker.deferredRecovery = deferred; } private async resumeDeferredWorkerRecovery(worker: ResidentWorker, disconnectError: Error): Promise { @@ -2950,14 +3293,41 @@ export class DaemonSupervisor { } } + /** + * A wake/retry/owner-attach can intentionally be recovering in memory before + * spawn has observed a new process identity. If that launch fails, return to + * the only processless durable state instead of recording that transient + * intent as recovering or failed on disk. + */ + private persistProcesslessRecoveryFailure(worker: ResidentWorker, error: unknown): void { + if (worker.descriptor.process !== undefined) { + this.persistWorker(worker); + return; + } + worker.descriptor.lifecycle = "passivated"; + delete worker.descriptor.pid; + delete worker.descriptor.processStartId; + worker.descriptor.lastError = error instanceof Error ? error.message : String(error); + this.persistWorker(worker); + } + private async recoverWorker(worker: ResidentWorker): Promise { - if (this.isWorkerRecoveryCancelled(worker)) { + if (worker.quarantined) return; + let generation = worker.descriptor.generation; + // A live legacy selector can only occur in an unnormalized in-memory + // harness: loadWorkerDescriptors normalizes it before publication. Do not + // reconnect, replace, or signal it; preserve the conservative durable fail. + if (!generation && (worker.descriptor.process?.pid ?? worker.descriptor.pid) !== undefined) { + worker.descriptor.lifecycle = "failed"; + worker.descriptor.lastError = "Cannot recover a live legacy worker without a verified process identity"; + // This reader-only legacy evidence has no C01 nested identity. Leave its + // durable bytes untouched rather than attempting a forbidden failed write. return; } if ( worker.descriptor.ownerClientId && !worker.launchEnv && - (worker.descriptor.pid === undefined || !isProcessAlive(worker.descriptor.pid)) + (worker.descriptor.process === undefined || !isProcessAlive(worker.descriptor.process?.pid)) ) { // Never infer an owner environment or relaunch an owner-owned worker from // persisted state. This includes processless/passivated descriptors, whose @@ -2965,39 +3335,47 @@ export class DaemonSupervisor { // processless descriptor until its identity is removed: otherwise a later // recovery could probe or signal stale/recycled process metadata. worker.descriptor.lifecycle = "passivated"; + delete worker.descriptor.process; delete worker.descriptor.pid; delete worker.descriptor.processStartId; worker.descriptor.lastError = "Waiting for the owning client to reconnect"; this.persistWorker(worker); return; } + // All remaining recovery continuations are asynchronous and need a + // published incarnation to fence their post-await mutations. + if (!generation || this.isWorkerRecoveryCancelled(worker, generation)) return; if (worker.recovery) { return worker.recovery; } - worker.recovery = (async () => { + let recovery!: Promise; + recovery = (async () => { for (const [retryIndex, retryDelay] of WORKER_RETRY_DELAYS_MS.entries()) { await delay(retryDelay); - if (this.isWorkerRecoveryCancelled(worker)) { + if (this.isWorkerRecoveryCancelled(worker, generation)) { return; } try { await this.assertRecoveryAllowed(); - const pid = worker.descriptor.pid; + // A legacy PID is allowed only to establish that a live process exists + // and must not be replaced. It is never signal authority. + const pid = worker.descriptor.process?.pid ?? worker.descriptor.pid; const processAlive = pid !== undefined && isProcessAlive(pid); const observedProcessStartId = processAlive ? getProcessStartId(pid) : undefined; const processIdentityMatches = - worker.descriptor.processStartId === undefined || - observedProcessStartId === worker.descriptor.processStartId; + worker.descriptor.process !== undefined && + observedProcessStartId === worker.descriptor.process.processStartId; if (processAlive && processIdentityMatches) { try { await this.connectWorker(worker, 1500); await this.subscribeWorker(worker, worker.descriptor.rootActiveSessionId); await this.refreshWorkerSummaries(worker, true); - if (this.isWorkerRecoveryCancelled(worker)) { + if (this.isWorkerRecoveryCancelled(worker, generation)) { return; } - if (worker.descriptor.processStartId === undefined && observedProcessStartId) { - worker.descriptor.processStartId = observedProcessStartId; + if (worker.descriptor.process?.processStartId === undefined && observedProcessStartId) { + if (worker.descriptor.process) + worker.descriptor.process.processStartId = observedProcessStartId; } await this.assertRecoveryAllowed(); worker.descriptor.lifecycle = "ready"; @@ -3022,22 +3400,58 @@ export class DaemonSupervisor { } if ( processAlive && - (worker.descriptor.processStartId === undefined || observedProcessStartId === undefined) + (worker.descriptor.process?.processStartId === undefined || observedProcessStartId === undefined) ) { throw new Error( `Cannot safely replace live session worker ${worker.descriptor.workerId} without a verified process identity`, ); } const safeToKillWorkerProcess = - processAlive && processIdentityMatches && worker.descriptor.processStartId !== undefined; + processAlive && processIdentityMatches && worker.descriptor.process?.processStartId !== undefined; await this.recoverUncertainWorkerOperations(worker, safeToKillWorkerProcess); - if (this.isWorkerRecoveryCancelled(worker)) { + if (this.isWorkerRecoveryCancelled(worker, generation)) { return; } await this.launchWorker(worker.descriptor.createCommand, worker); return; } catch (error) { - if (isSupervisorRecoveryCancelled(error) || this.isWorkerRecoveryCancelled(worker)) { + // launchWorker may have atomically published a fresh generation before + // its spawn/handshake failed. That is this recovery's own failed + // attempt, not a cancellation of the older generation that entered + // this loop. A true replacement/stop never matches this exact tag. + const launchFailure = this.workerLaunchFailureAttempt(error, worker); + // A failed cleanup leaves a process-bearing failed descriptor. It is + // deliberately not retryable: this loop has no proof it owns a dead + // process and must not signal or replace it on the next pass. + if (launchFailure) { + if (launchFailure.generation !== worker.descriptor.generation) { + // A distinct, concurrently-published generation won. The + // tagged error cannot authorize any mutation of that worker. + return; + } + // The failed launch published this generation. Adopt it before + // returning so the finally block can release this completed + // recovery attempt rather than strand its stale promise. + generation = launchFailure.generation; + if (!launchFailure.cleanupVerified) { + return; + } + if (!this.isWorkerRecoveryCancelled(worker, generation)) { + // Keep diagnostics in memory until the next launch publishes a + // process-bearing descriptor. Persisting this processless recovery + // intent would make a crash-recoverable record without process + // identity authority. + worker.descriptor.consecutiveFailures++; + worker.descriptor.lastFailureAt = new Date().toISOString(); + worker.descriptor.lastError = error instanceof Error ? error.message : String(error); + // Cleanup proved the fresh process is gone. Keep its + // processless recovering state in memory and immediately + // proceed to the next retry; such a state is intentionally + // never persisted. + continue; + } + } + if (isSupervisorRecoveryCancelled(error) || this.isWorkerRecoveryCancelled(worker, generation)) { return; } try { @@ -3045,12 +3459,15 @@ export class DaemonSupervisor { } catch { return; } + if (this.isWorkerRecoveryCancelled(worker, generation)) return; worker.client?.close(); + if (this.isWorkerRecoveryCancelled(worker, generation)) return; worker.client = undefined; worker.descriptor.consecutiveFailures++; worker.descriptor.lastFailureAt = new Date().toISOString(); worker.descriptor.lastError = error instanceof Error ? error.message : String(error); - this.persistWorker(worker); + if (this.isWorkerRecoveryCancelled(worker, generation)) return; + this.persistProcesslessRecoveryFailure(worker, error); } } try { @@ -3058,32 +3475,43 @@ export class DaemonSupervisor { } catch { return; } - worker.descriptor.lifecycle = "failed"; - this.persistWorker(worker); + if (this.isWorkerRecoveryCancelled(worker, generation)) return; + worker.descriptor.lifecycle = worker.descriptor.process === undefined ? "passivated" : "failed"; + if (this.isWorkerRecoveryCancelled(worker, generation)) return; + this.persistProcesslessRecoveryFailure(worker, worker.descriptor.lastError ?? "Worker recovery failed"); await this.syncAgentPeers().catch(() => undefined); this.log(`Worker ${worker.descriptor.workerId} failed after three recovery attempts`); })().finally(() => { - worker.recovery = undefined; + // `recovery` is only a join handle. A retry can legitimately publish a + // newer generation before this cycle settles, so its pre-retry generation + // is not release authority. The resident-object and exact-promise fences + // release this completed cycle without clearing a replacement worker or a + // newer recovery cycle that took over the join slot. + if (this.workers.get(worker.descriptor.workerId) === worker && worker.recovery === recovery) { + worker.recovery = undefined; + } }); - return worker.recovery; + worker.recovery = recovery; + return recovery; } - private isWorkerRecoveryCancelled(worker: ResidentWorker): boolean { + private isWorkerRecoveryCancelled(worker: ResidentWorker, generation?: string): boolean { return ( this.shuttingDown || worker.intentionalStop || worker.descriptor.stopRequestedAt !== undefined || - this.workers.get(worker.descriptor.workerId) !== worker + this.workers.get(worker.descriptor.workerId) !== worker || + (generation !== undefined && worker.descriptor.generation !== generation) ); } private async recoverUncertainWorkerOperations(worker: ResidentWorker, killWorkerProcess = true): Promise { await this.assertRecoveryAllowed(); if (killWorkerProcess) { - signalProcessGroupOrProcess(worker.descriptor.pid!, "SIGKILL"); + this.signalTrackedWorker(worker, worker.descriptor.generation ?? "", "SIGKILL"); } const orphanProcessJournalPath = worker.descriptor.orphanProcessJournalPath; - const pid = worker.descriptor.pid; + const pid = worker.descriptor.process?.pid; // A processless passive record intentionally has no parent identity. Do // not use a legacy/stale parent PID to reap anything while waking it. if (orphanProcessJournalPath && pid !== undefined) { @@ -3144,13 +3572,18 @@ export class DaemonSupervisor { ), ); await this.assertRecoveryAllowed(); - for (const record of latest) { + // Only a validated v2 begin can be completed. v1 is intentionally + // conservative evidence and must never be "cleared" by invented IDs. + for (const record of uncertain) { + if (record.version !== 2) continue; journal.record({ activeSessionId: record.activeSessionId, sessionId: record.sessionId, ...(record.sessionFile ? { sessionFile: record.sessionFile } : {}), busy: false, - operation: "recovery_hold", + operation: record.operation, + operationId: record.operationId, + generation: record.generation, }); } this.log( @@ -3161,13 +3594,20 @@ export class DaemonSupervisor { } private async refreshWorkerSummaries(worker: ResidentWorker, recovery = false): Promise { - if (!worker.client) { - throw new Error("Session worker is not connected"); - } - const response = await worker.client.request({ type: "list" }, 5000); + const generation = worker.descriptor.generation; + const client = worker.client; + if (!client || !generation || !this.acceptsWorkerCallback(worker, generation, client)) { + throw new Error("Session worker is not connected or was superseded"); + } + const response = await client.request({ type: "list" }, 5000); + // The list request may have been held while a new incarnation was published. + // Do not let A overwrite B's summaries or durable descriptor. + if (!this.acceptsWorkerCallback(worker, generation, client)) return; const summaries = sessionSummariesFromResponse(response); + if (!this.acceptsWorkerCallback(worker, generation, client)) return; worker.summaries = new Map(summaries.map((summary) => [summary.activeSessionId ?? summary.id, summary])); for (const summary of summaries) { + if (!this.acceptsWorkerCallback(worker, generation, client)) return; const activeSessionId = summary.activeSessionId ?? summary.id; if (summary.streamingMessage?.role === "assistant") { this.streamReconstructor.seed(activeSessionId, summary.streamingMessage); @@ -3177,20 +3617,17 @@ export class DaemonSupervisor { } const root = worker.summaries.get(worker.descriptor.rootActiveSessionId); if (root) { - if (recovery) { - await this.assertRecoveryAllowed(); - } + if (recovery) await this.assertRecoveryAllowed(); + if (!this.acceptsWorkerCallback(worker, generation, client)) return; worker.descriptor.rootSessionId = root.sessionId; worker.descriptor.sessionFile = root.sessionFile; worker.descriptor.createCommand = { ...worker.descriptor.createCommand, sessionPath: root.sessionFile, continueRecent: false, - config: { - ...worker.descriptor.createCommand.config, - cwd: root.cwd, - }, + config: { ...worker.descriptor.createCommand.config, cwd: root.cwd }, }; + if (!this.acceptsWorkerCallback(worker, generation, client)) return; this.persistWorker(worker); } } @@ -3391,7 +3828,9 @@ export class DaemonSupervisor { attachedClients: [...this.clients].filter((client) => client.attachedActiveSessionIds.has(activeSessionId)) .length, workerState: worker.descriptor.lifecycle, - ...(worker.descriptor.lifecycle === "passivated" ? {} : { workerPid: worker.descriptor.pid! }), + // A recovering/processless descriptor is intentionally still routable metadata. + // Do not turn a missing identity into a PID read while reporting it. + ...(worker.descriptor.process ? { workerPid: worker.descriptor.process.pid } : {}), }; } @@ -3566,18 +4005,40 @@ export class DaemonSupervisor { command: DaemonCommand, timeoutMs = WORKER_REQUEST_TIMEOUT_MS, ): Promise { + let generation = worker.descriptor.generation; + if (!generation || !this.matchesCurrentWorker(worker, generation)) { + throw new Error(`Session worker ${worker.descriptor.workerId} was superseded`); + } if (this.commandExplicitlyWakesWorker(command)) { await this.wakePassivatedWorker(worker); + // Waking a passivated resident legitimately launches a new generation on + // the same object. Reacquire its published generation and client after the + // await, while still rejecting a different object that took this selector. + generation = worker.descriptor.generation; + if (!generation || !this.matchesCurrentWorker(worker, generation)) { + throw new Error(`Session worker ${worker.descriptor.workerId} was superseded`); + } } - if (!worker.client || worker.descriptor.lifecycle !== "ready") { + const client = worker.client; + if (!client || worker.descriptor.lifecycle !== "ready") { throw new Error(`Session worker is ${worker.descriptor.lifecycle}`); } - const response = await worker.client.request(withoutCommandId(command), timeoutMs); + const response = await client.request(withoutCommandId(command), timeoutMs); + // A request can finish after another assignment has claimed this public + // selector. Its result must not be surfaced through that new assignment. + if (!this.matchesCurrentWorker(worker, generation) || worker.client !== client) { + throw new Error(`Session worker ${worker.descriptor.workerId} was superseded`); + } if (command.type === "get_state" && response.success && isSessionSummary(response.data)) { return { ...response, id: command.id, data: this.publicSummary(worker, response.data) }; } if (command.type === "rename" && response.success && isSessionSummary(response.data)) { await this.refreshWorkerSummaries(worker); + // refreshWorkerSummaries fences its writes, but it intentionally returns + // quietly when stale. Revalidate before returning A's rename response. + if (!this.matchesCurrentWorker(worker, generation) || worker.client !== client) { + throw new Error(`Session worker ${worker.descriptor.workerId} was superseded`); + } return { ...response, id: command.id, data: this.publicSummary(worker, response.data) }; } return responseWithId(response, command.id); @@ -3608,7 +4069,8 @@ export class DaemonSupervisor { ownedWorker.descriptor.archiveOnStop = undefined; ownedWorker.descriptor.lifecycle = "recovering"; ownedWorker.descriptor.consecutiveFailures = 0; - this.persistWorker(ownedWorker); + // Owner attach is an explicit wake. Defer persistence until launch + // atomically publishes a canonical generation and process identity. await this.recoverWorker(ownedWorker); } } @@ -4060,7 +4522,16 @@ export class DaemonSupervisor { ); } - private handleWorkerFrame(worker: ResidentWorker, frame: PrivateFrame): void { + private handleWorkerFrame( + worker: ResidentWorker, + frame: PrivateFrame, + generation?: string, + client?: DaemonWorkerClient, + ): void { + if (generation !== undefined && client !== undefined && !this.acceptsWorkerCallback(worker, generation, client)) { + this.log(`Ignoring frame from stale worker callback ${worker.descriptor.workerId}`); + return; + } if (frame.header.kind !== "outbound") { return; } @@ -4882,7 +5353,7 @@ export class DaemonSupervisor { if (removeDescriptor) { this.persistWorkerStopTombstone(worker, archiveSession); } else { - worker.descriptor.lifecycle = "failed"; + worker.descriptor.lifecycle = worker.descriptor.process === undefined ? "passivated" : "failed"; worker.descriptor.lastError = failure.message; this.persistWorker(worker); } @@ -4907,6 +5378,9 @@ export class DaemonSupervisor { recoveryCleanup = false, directChild?: { child: ChildProcess; closed: Promise }, ): Promise { + if (worker.quarantined) { + throw new Error(`Session worker ${worker.descriptor.workerId} is quarantined pending lifecycle repair`); + } // Another independently dispatched stop may have completed between this // request being recorded and its turn to run. A dead worker cannot be // stopped twice, but a later archive request is still actionable from the @@ -4928,7 +5402,7 @@ export class DaemonSupervisor { } // A passivated descriptor is explicitly processless. Its old pid may have // been recycled while the supervisor was down, so never probe or signal it. - const processless = worker.descriptor.lifecycle === "passivated" || worker.descriptor.pid === undefined; + const processless = worker.descriptor.lifecycle === "passivated" || worker.descriptor.process === undefined; if (worker.ownerCleanupTimer) { clearTimeout(worker.ownerCleanupTimer); worker.ownerCleanupTimer = undefined; @@ -4941,7 +5415,10 @@ export class DaemonSupervisor { this.persistWorkerStopTombstone(worker, archiveSession); } else { worker.intentionalStop = true; - worker.descriptor.lifecycle = "recovering"; + // A stopped processless root remains a canonical passive routing + // record; recovery hand-off must not manufacture a processless + // recovering descriptor. + worker.descriptor.lifecycle = worker.descriptor.process === undefined ? "passivated" : "recovering"; this.persistWorker(worker); } } catch (error) { @@ -4984,31 +5461,41 @@ export class DaemonSupervisor { worker.client = undefined; } else if (directChild) { directChild.child.kill("SIGTERM"); - } else if (!processless && isProcessAlive(worker.descriptor.pid!)) { - signalProcessGroupOrProcess(worker.descriptor.pid!, "SIGTERM"); - } - const isWorkerProcessAlive = () => - processless - ? false - : directChild - ? directChild.child.exitCode === null && directChild.child.signalCode === null - : isProcessAlive(worker.descriptor.pid!); - const gracefulDeadline = Date.now() + (force ? 500 : 2000); - while (isWorkerProcessAlive() && Date.now() < gracefulDeadline) { - await delay(25); - } - if (force && isWorkerProcessAlive()) { + } else if (!processless) { + this.signalTrackedWorkerState(worker, worker.descriptor.generation, "SIGTERM"); + } + const processIdentityFailure = (state: WorkerProcessIdentityState) => + new Error( + `Session worker ${worker.descriptor.workerId} process identity is ${state}; retaining stop tombstone for retry`, + ); + const workerProcessState = (): WorkerProcessIdentityState => + directChild + ? directChild.child.exitCode === null && directChild.child.signalCode === null + ? "exact" + : "dead" + : processless + ? "dead" + : this.classifyWorkerProcessIdentity(worker); + const waitForWorkerStop = async (deadline: number): Promise => { + let state = workerProcessState(); + while (state === "exact" && Date.now() < deadline) { + await delay(25); + state = workerProcessState(); + } + if (state === "unreadable") throw processIdentityFailure(state); + return state; + }; + let processState = await waitForWorkerStop(Date.now() + (force ? 500 : 2000)); + if (force && processState === "exact") { if (directChild) { directChild.child.kill("SIGKILL"); } else { - signalProcessGroupOrProcess(worker.descriptor.pid!, "SIGKILL"); - } - const forceDeadline = Date.now() + 1000; - while (isWorkerProcessAlive() && Date.now() < forceDeadline) { - await delay(25); + const signalState = this.signalTrackedWorkerState(worker, worker.descriptor.generation, "SIGKILL"); + if (signalState === "unreadable") throw processIdentityFailure(signalState); } + processState = await waitForWorkerStop(Date.now() + 1000); } - if (isWorkerProcessAlive()) { + if (processState === "exact") { worker.intentionalStop = worker.descriptor.stopRequestedAt !== undefined; throw new Error(`Session worker ${worker.descriptor.workerId} did not stop${force ? " after SIGKILL" : ""}`); } @@ -5264,8 +5751,22 @@ export class DaemonSupervisor { cleanup(); } if (stopWorkers) { + // A malformed descriptor is retained as raw quarantine evidence. Its stop + // refusal must not prevent shutdown from stopping every healthy worker or + // releasing the daemon's global resources. await Promise.all( - [...this.workers.values()].map((worker) => this.stopWorker(worker, true, forceWorkers, true)), + [...this.workers.values()].map(async (worker) => { + if (worker.quarantined) { + this.reportCleanupFailure( + `quarantined session worker ${worker.descriptor.workerId}`, + new Error("Skipped stop to preserve quarantined lifecycle evidence"), + ); + return; + } + await this.runCleanupStep(`session worker ${worker.descriptor.workerId}`, () => + this.stopWorker(worker, true, forceWorkers, true), + ); + }), ); if (!this.hasPersistedWorkerDescriptors()) { rmSync(this.supervisorConfigPath, { force: true }); diff --git a/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts b/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts index 9b57acdbe..1b897c3c8 100644 --- a/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts +++ b/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts @@ -15,19 +15,22 @@ export const DAEMON_WORKER_TOKEN_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_TOKEN export const DAEMON_WORKER_ACTIVE_SESSION_ID_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_ACTIVE_SESSION_ID"; export const DAEMON_WORKER_SUPERVISOR_SOCKET_ENV = "PRIME_AGENT_INTERNAL_DAEMON_SUPERVISOR_SOCKET"; export const DAEMON_WORKER_RECOVERY_JOURNAL_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_RECOVERY_JOURNAL"; +/** Opaque supervisor-minted worker incarnation for recovery-journal v2. */ +export const DAEMON_WORKER_GENERATION_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_GENERATION"; export const DAEMON_WORKER_STARTUP_GATE_FD_ENV = "PRIME_AGENT_INTERNAL_DAEMON_WORKER_STARTUP_GATE_FD"; export const DAEMON_WORKER_STARTUP_GATE_COMMIT = "start\n"; /** * `passivated` descriptors retain a session's routing metadata without a worker * process. They are deliberately revived only by an explicit session operation. */ -export const DAEMON_WORKER_LIFECYCLES = ["starting", "ready", "recovering", "failed", "passivated"] as const; -export type DaemonWorkerLifecycle = (typeof DAEMON_WORKER_LIFECYCLES)[number]; +export { + DAEMON_WORKER_LIFECYCLES, + type DaemonWorkerLifecycle, + isDaemonWorkerLifecycle, + type ProcessIdentity, +} from "./daemon-lifecycle-identity.js"; -/** Durable descriptor states are untrusted input when read from disk. */ -export function isDaemonWorkerLifecycle(value: unknown): value is DaemonWorkerLifecycle { - return typeof value === "string" && (DAEMON_WORKER_LIFECYCLES as readonly string[]).includes(value); -} +import type { DaemonWorkerLifecycle, ProcessIdentity } from "./daemon-lifecycle-identity.js"; export type DaemonWorkerFrameHeader = | { @@ -98,12 +101,19 @@ export interface DaemonWorkerDescriptor { workerId: string; /** * Process identity for resident workers. Both fields are deliberately absent - * for passivated descriptors and may be absent on a recovering descriptor - * normalized from legacy lifecycle data. Legacy fields are accepted only while - * reading a non-passivated v1 descriptor; writers never retain them on a - * passivation. + * only for passivated descriptors in every C01 write. Reader-only legacy or + * malformed lifecycle evidence may be processless in memory, but is never + * rewritten as a non-passivated C01 descriptor. Legacy fields are accepted + * only while reading a non-passivated v1 descriptor; writers never retain + * them on a passivation. */ + /** Present only for a resident C01 worker. Legacy pid fields are reader-only. */ + process?: ProcessIdentity; + /** Fresh UUID for every launch/adoption; legacy records may not have one. */ + generation?: string; + /** @deprecated reader-only legacy v1 fields; never emitted by C01 writers. */ pid?: number; + /** @deprecated reader-only legacy v1 fields; never emitted by C01 writers. */ processStartId?: string; socketPath: string; recoveryJournalPath: string; @@ -128,6 +138,20 @@ export interface DaemonWorkerDescriptor { lastError?: string; } +/** + * Reader compatibility stays intentionally broad in DaemonWorkerDescriptor. + * Every C01 resident/new write instead uses this closed shape: it has a fresh + * generation and cannot carry the old PID selector fields. + */ +export interface ResidentDaemonWorkerDescriptor + extends Omit { + /** Every published resident/passivated C01 descriptor has an incarnation. */ + generation: string; + /** Legacy flat selectors are accepted only by the reader descriptor above. */ + pid?: never; + processStartId?: never; +} + export function isDaemonWorkerProcess(environment: NodeJS.ProcessEnv = process.env): boolean { return environment[DAEMON_WORKER_ROLE_ENV] === "1"; } @@ -148,9 +172,15 @@ export function waitForDaemonWorkerStartupGate(environment: NodeJS.ProcessEnv = } finally { closeSync(fd); } - if (marker !== DAEMON_WORKER_STARTUP_GATE_COMMIT) { + if (!marker.startsWith(DAEMON_WORKER_STARTUP_GATE_COMMIT)) { throw new Error("Daemon session worker startup was cancelled"); } + // The supervisor observes the child start identity before minting this value. + // Publishing it through the gate prevents the worker from journaling or + // callback-registration before it has the exact committed incarnation. + const generation = marker.slice(DAEMON_WORKER_STARTUP_GATE_COMMIT.length).trim(); + if (!generation) throw new Error("Daemon session worker startup omitted its generation"); + environment[DAEMON_WORKER_GENERATION_ENV] = generation; } export function requireDaemonWorkerAuthenticationToken(environment: NodeJS.ProcessEnv = process.env): string { diff --git a/packages/coding-agent/src/modes/daemon/worker-recovery-journal.ts b/packages/coding-agent/src/modes/daemon/worker-recovery-journal.ts index e765988eb..cbfeb2641 100644 --- a/packages/coding-agent/src/modes/daemon/worker-recovery-journal.ts +++ b/packages/coding-agent/src/modes/daemon/worker-recovery-journal.ts @@ -1,3 +1,4 @@ +import { randomUUID } from "node:crypto"; import { chmodSync, closeSync, @@ -6,12 +7,66 @@ import { openSync, readFileSync, renameSync, - writeFileSync, + unlinkSync, writeSync, } from "node:fs"; import { dirname } from "node:path"; +import { assertFreshUuid } from "./daemon-lifecycle-identity.js"; +/** Checkpoints emitted by the daemon worker. Keep the durable vocabulary closed. */ +export const WORKER_RECOVERY_OPERATIONS = [ + "ready", + "prompt", + "prompt_accepted", + "steer_queued", + "follow_up_queued", + "actions_restored", + "closed:killed", + "closed:shutdown", + "closed:completed", + "closed:replaced", + "closed:update", + "agent_start", + "agent_end", + "turn_start", + "turn_end", + "message_start", + "message_end", + "tool_execution_start", + "tool_execution_end", + "compaction_start", + "compaction_end", + "auto_retry_start", + "auto_retry_end", + "bash_start", + "bash_end", + "session_action_update", + "rlm_child_update", + "ipython_sent_agent_message", + "auth_stale", + "bash_output", + "goal_update", + "model_stream", + "tool_execution", + "recovery_hold", +] as const; +export type WorkerRecoveryOperation = (typeof WORKER_RECOVERY_OPERATIONS)[number]; + +/** The only writer format. operationId and generation fence a completion. */ export interface WorkerRecoveryRecord { + version: 2; + activeSessionId: string; + sessionId: string; + sessionFile?: string; + busy: boolean; + operation: WorkerRecoveryOperation; + operationId: string; + generation: string; + recordedAt: string; +} + +/** v1 evidence is intentionally readable, but cannot authorize v2 cleanup. */ +export interface LegacyWorkerRecoveryRecord { version: 1; activeSessionId: string; sessionId: string; @@ -20,83 +75,193 @@ export interface WorkerRecoveryRecord { operation: string; recordedAt: string; } +export type ReadWorkerRecoveryRecord = WorkerRecoveryRecord | LegacyWorkerRecoveryRecord; interface ParsedRecords { - latest: Map; + latest: Map; hasInvalidRecords: boolean; } -function parseRecords(path: string): ParsedRecords { - const latest = new Map(); +/** Narrow filesystem boundary so durability order and failure behavior are testable. */ +export interface WorkerRecoveryJournalFileSystem { + mkdirSync(path: string, options: { recursive: true; mode: number }): string | undefined; + readFileSync(path: string, encoding: "utf8"): string; + openSync(path: string, flags: string, mode?: number): number; + writeSync(fd: number, data: Uint8Array, offset: number, length: number): number; + fsyncSync(fd: number): void; + closeSync(fd: number): void; + chmodSync(path: string, mode: number): void; + renameSync(oldPath: string, newPath: string): void; + unlinkSync(path: string): void; +} + +export interface WorkerRecoveryJournalOptions { + fileSystem?: WorkerRecoveryJournalFileSystem; + makeTempPath?: (journalPath: string) => string; + platform?: NodeJS.Platform; +} + +const nativeFileSystem: WorkerRecoveryJournalFileSystem = { + mkdirSync, + readFileSync, + openSync, + writeSync, + fsyncSync, + closeSync, + chmodSync, + renameSync, + unlinkSync, +}; + +const v2Key = (record: Pick) => + `${record.activeSessionId}\u0000${record.generation}\u0000${record.operationId}`; +const legacyKey = (record: Pick) => + `legacy\u0000${record.activeSessionId}`; + +function isV2(value: unknown): value is WorkerRecoveryRecord { + if (!value || typeof value !== "object") return false; + const record = value as Partial; + return ( + record.version === 2 && + typeof record.activeSessionId === "string" && + typeof record.sessionId === "string" && + typeof record.busy === "boolean" && + typeof record.operation === "string" && + (WORKER_RECOVERY_OPERATIONS as readonly string[]).includes(record.operation) && + assertFreshUuid(record.operationId) && + assertFreshUuid(record.generation) && + typeof record.recordedAt === "string" && + (record.sessionFile === undefined || typeof record.sessionFile === "string") + ); +} + +function isV1(value: unknown): value is LegacyWorkerRecoveryRecord { + if (!value || typeof value !== "object") return false; + const record = value as Partial; + return ( + record.version === 1 && + typeof record.activeSessionId === "string" && + typeof record.sessionId === "string" && + typeof record.busy === "boolean" && + typeof record.operation === "string" && + typeof record.recordedAt === "string" && + (record.sessionFile === undefined || typeof record.sessionFile === "string") + ); +} + +function parseRecords( + path: string, + fileSystem: Pick = nativeFileSystem, +): ParsedRecords { + const latest = new Map(); let contents: string; try { - contents = readFileSync(path, "utf8"); + contents = fileSystem.readFileSync(path, "utf8"); } catch (error) { - if ((error as NodeJS.ErrnoException).code === "ENOENT") { - return { latest, hasInvalidRecords: false }; - } + if ((error as NodeJS.ErrnoException).code === "ENOENT") return { latest, hasInvalidRecords: false }; throw error; } let hasInvalidRecords = false; for (const line of contents.split("\n")) { - if (!line.trim()) { - continue; - } - let record: WorkerRecoveryRecord; + if (!line.trim()) continue; + let value: unknown; try { - record = JSON.parse(line) as WorkerRecoveryRecord; + value = JSON.parse(line); } catch { hasInvalidRecords = true; continue; } - if ( - record.version === 1 && - typeof record.activeSessionId === "string" && - typeof record.sessionId === "string" && - typeof record.busy === "boolean" && - typeof record.operation === "string" - ) { - latest.set(record.activeSessionId, record); - } else { - hasInvalidRecords = true; - } + if (isV2(value)) latest.set(v2Key(value), value); + else if (isV1(value)) latest.set(legacyKey(value), value); + else hasInvalidRecords = true; } return { latest, hasInvalidRecords }; } export class WorkerRecoveryJournal { - private readonly latest: Map; + private readonly latest: Map; private readonly hasInvalidRecords: boolean; + private readonly fileSystem: WorkerRecoveryJournalFileSystem; + private readonly makeTempPath: (journalPath: string) => string; + private readonly platform: NodeJS.Platform; - constructor(private readonly path: string) { - mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); - const parsed = parseRecords(path); + constructor( + private readonly path: string, + { + fileSystem = nativeFileSystem, + makeTempPath = (journalPath) => `${journalPath}.${process.pid}.${randomUUID()}.tmp`, + platform = process.platform, + }: WorkerRecoveryJournalOptions = {}, + ) { + this.fileSystem = fileSystem; + this.makeTempPath = makeTempPath; + this.platform = platform; + this.fileSystem.mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + const parsed = parseRecords(path, this.fileSystem); this.latest = parsed.latest; this.hasInvalidRecords = parsed.hasInvalidRecords; + // Upgrade journals written before completed v2 identities were pruned. + // A restart must not expose their historical terminal records forever. Raw + // malformed lines are fail-closed recovery evidence, however: rewriting a + // parsed subset would silently erase both that evidence and the unreadable + // signal on the next restart. + if (!this.hasInvalidRecords && [...this.latest.values()].some((record) => isV2(record) && !record.busy)) { + this.compact(); + } } + /** + * Append a fully validated v2 checkpoint. A non-busy record is a completion: + * it may replace only the busy record with precisely the same operation and + * process incarnation. This is the journal's authoritative stale-callback fence. + */ record(input: Omit): void { - const previous = this.latest.get(input.activeSessionId); + const record: WorkerRecoveryRecord = { version: 2, ...input, recordedAt: new Date().toISOString() }; + if (!isV2(record)) throw new Error("Invalid C01 recovery checkpoint"); + const key = v2Key(record); + const previous = this.latest.get(key); + // A completion is never an admission. It must replace a *busy* v2 record + // for the same stable active-session/generation/operation-ID key and operation. + // sessionId and sessionFile describe the session at each checkpoint, but are + // not authority: materialization, resume, or branching can change either while + // an admitted operation is still running. In particular, a random/new operation + // ID or a different operation must not manufacture a clear without its begin. if ( - previous?.busy === input.busy && - previous.operation === input.operation && - previous.sessionFile === input.sessionFile - ) { + !record.busy && + (!previous || + !isV2(previous) || + !previous.busy || + previous.operationId !== record.operationId || + previous.operation !== record.operation) + ) + return; + if ( + previous && + isV2(previous) && + previous.busy === record.busy && + previous.operation === record.operation && + previous.operationId === record.operationId && + previous.sessionFile === record.sessionFile + ) return; - } - const record: WorkerRecoveryRecord = { - version: 1, - ...input, - recordedAt: new Date().toISOString(), - }; this.append(record); - this.latest.set(record.activeSessionId, record); - if ([...this.latest.values()].every((entry) => !entry.busy)) { + this.latest.set(key, record); + // A terminal v2 operation is only a stale-callback fence while the append + // above is durable. It is not recovery evidence. Compact it immediately so + // operationId cardinality cannot turn completed work into unbounded journal + // or getLatest history. v1 remains conservative uncertainty; every busy v2 + // identity remains exact crash evidence. + if (!record.busy) { + // The terminal append is already durable. Do not roll it back in memory: + // restoring `previous` would let a later successful compaction rewrite its + // old busy record and resurrect completed work. compact() still propagates + // replacement failures to the caller, because it may not have durably + // retained the complete sibling evidence set. this.compact(); } } - getLatest(): WorkerRecoveryRecord[] { + getLatest(): ReadWorkerRecoveryRecord[] { return [...this.latest.values()]; } @@ -104,27 +269,101 @@ export class WorkerRecoveryJournal { return this.hasInvalidRecords; } - static readLatest(path: string): WorkerRecoveryRecord[] { + static readLatest(path: string): ReadWorkerRecoveryRecord[] { return [...parseRecords(path).latest.values()]; } private append(record: WorkerRecoveryRecord): void { - const descriptor = openSync(this.path, "a", 0o600); + const fd = this.fileSystem.openSync(this.path, "a", 0o600); try { - writeSync(descriptor, `${JSON.stringify(record)}\n`); - fsyncSync(descriptor); + this.writeAll(fd, `${JSON.stringify(record)}\n`); + this.fileSystem.fsyncSync(fd); } finally { - closeSync(descriptor); + this.fileSystem.closeSync(fd); } - chmodSync(this.path, 0o600); + this.fileSystem.chmodSync(this.path, 0o600); } private compact(): void { - const tempPath = `${this.path}.${process.pid}.tmp`; - writeFileSync(tempPath, `${[...this.latest.values()].map((record) => JSON.stringify(record)).join("\n")}\n`, { - mode: 0o600, - }); - chmodSync(tempPath, 0o600); - renameSync(tempPath, this.path); + // Parsed records are not a lossless representation of malformed input. Do + // not replace the journal while it contains any such raw evidence; otherwise + // a compaction would make future recovery appear safe merely by deleting it. + if (this.hasInvalidRecords) return; + // Completed v2 operations are deliberately omitted. Keeping their UUID-keyed + // terminal entries would make a long-lived idle worker retain one record per + // historical operation. v1 has no identity fence and is therefore preserved + // verbatim as uncertain legacy recovery evidence. + const retained = [...this.latest.entries()].filter(([, record]) => !isV2(record) || record.busy); + const contents = retained.map(([, record]) => JSON.stringify(record)).join("\n"); + this.replaceAtomically(contents ? `${contents}\n` : ""); + // Do not alter in-memory recovery evidence until its replacement is durable. + this.latest.clear(); + for (const [key, record] of retained) this.latest.set(key, record); + } + + private replaceAtomically(contents: string): void { + const tempPath = this.makeTempPath(this.path); + let tempFd: number | undefined; + let renamed = false; + try { + // Exclusive creation makes cleanup safe: this invocation owns this temp file. + tempFd = this.fileSystem.openSync(tempPath, "wx", 0o600); + this.fileSystem.chmodSync(tempPath, 0o600); + this.writeAll(tempFd, contents); + this.fileSystem.fsyncSync(tempFd); + this.fileSystem.closeSync(tempFd); + tempFd = undefined; + this.fileSystem.renameSync(tempPath, this.path); + renamed = true; + this.fsyncParentDirectory(); + } catch (error) { + if (tempFd !== undefined) { + try { + this.fileSystem.closeSync(tempFd); + } catch { + // The original write/sync/rename failure is the actionable failure. + } + } + if (!renamed) { + try { + this.fileSystem.unlinkSync(tempPath); + } catch { + // A unique, restrictive temp can be cleaned by a later operator; never touch the journal. + } + } + throw error; + } + } + + private writeAll(fd: number, contents: string): void { + const bytes = Buffer.from(contents); + for (let offset = 0; offset < bytes.length; ) { + const written = this.fileSystem.writeSync(fd, bytes, offset, bytes.length - offset); + if (!Number.isSafeInteger(written) || written <= 0 || written > bytes.length - offset) + throw new Error("Recovery journal write made no forward progress"); + offset += written; + } + } + + private fsyncParentDirectory(): void { + let directoryFd: number | undefined; + try { + directoryFd = this.fileSystem.openSync(dirname(this.path), "r"); + this.fileSystem.fsyncSync(directoryFd); + } catch (error) { + if (!this.isDirectoryFsyncUnsupported(error)) throw error; + } finally { + if (directoryFd !== undefined) this.fileSystem.closeSync(directoryFd); + } + } + + private isDirectoryFsyncUnsupported(error: unknown): boolean { + const code = (error as NodeJS.ErrnoException).code; + return ( + code === "ENOTSUP" || + code === "EOPNOTSUPP" || + code === "EINVAL" || + (this.platform === "win32" && (code === "EPERM" || code === "EISDIR")) + ); } } diff --git a/packages/coding-agent/test/agent-session-recursion.test.ts b/packages/coding-agent/test/agent-session-recursion.test.ts index f066f02b1..a31d3bcdc 100644 --- a/packages/coding-agent/test/agent-session-recursion.test.ts +++ b/packages/coding-agent/test/agent-session-recursion.test.ts @@ -110,6 +110,7 @@ interface CapturedCommReply { interface InspectableRlmRun { id: string; + assignmentId: string; sessionDir: string; abort: () => void; status: string; @@ -117,6 +118,7 @@ interface InspectableRlmRun { error?: string; detachedDeletion?: Awaited>["subagents"][number]; session?: AgentSession; + unsubscribe?: () => void; } interface InspectableRlmSession { @@ -130,7 +132,12 @@ interface InspectableRlmSession { >; _rlmChildCleanupFailures: Map>["subagents"][number]>; _rlmChildSessions: Map; + _rlmChildSessionAssignments: Map; + _deletedRlmChildIds: Set; _rlmChildUnsubscribes: Map void>; + _rlmChildUnsubscribeAssignments: Map; + _removeRlmSubagentTracking(childId: string, run?: InspectableRlmRun, assignmentId?: string): void; + _deleteRlmSubagentSession(childId: string, assignmentId?: string, session?: AgentSession): Promise; _createKernelHostHandlers(): HostRequestHandlers; _reapDeletedRlmSubagentRuntimesAfterCompaction(): Promise; } @@ -596,6 +603,50 @@ describe("AgentSession rlm recursion", () => { expect(childStatuses).toEqual(["cancelled"]); }); + it("keeps every held A callback from mutating a replacement B assignment", () => { + const childId = "held-assignment-child"; + const assignmentA = "held-assignment-A"; + const assignmentB = "held-assignment-B"; + const root = createSession(); + const childA = createSession({ rlmSessionDir: join(tempDir, "held-A") }); + const childB = createSession({ rlmSessionDir: join(tempDir, "held-B") }); + const unsubscribeA = vi.fn(); + const unsubscribeB = vi.fn(); + const internals = root as unknown as InspectableRlmSession; + + // A is admitted, then B takes the same public selector while all A callbacks + // are deliberately held. This models creation, terminal, usage/update, and + // unsubscribe continuations without allowing a stale callback to own B. + expect(root.registerRlmChildSession(childId, childA, unsubscribeA, assignmentA)).toBe(true); + internals._rlmChildSessions.set(childId, childB); + internals._rlmChildSessionAssignments.set(childId, assignmentB); + internals._rlmChildUnsubscribes.set(childId, unsubscribeB); + internals._rlmChildUnsubscribeAssignments.set(childId, assignmentB); + + // Held creation/terminal retention, release/delete, unsubscribe, and lazy + // hydration must all reject A, including when hydration has the same session + // object that B currently owns. + expect(root.registerRlmChildSession(childId, childA, unsubscribeA, assignmentA)).toBe(false); + expect(root.releaseRlmChildSession(childId, childA, assignmentA)).toBe(false); + expect(root.rebindRlmChildSessionAssignment(childId, childB, assignmentA)).toBe(false); + internals._removeRlmSubagentTracking(childId, undefined, assignmentA); + expect(internals._rlmChildSessions.get(childId)).toBe(childB); + expect(internals._rlmChildSessionAssignments.get(childId)).toBe(assignmentB); + expect(internals._rlmChildUnsubscribes.get(childId)).toBe(unsubscribeB); + expect(unsubscribeA).not.toHaveBeenCalled(); + expect(unsubscribeB).not.toHaveBeenCalled(); + + // A tombstone is scoped to A. It cannot hide B's map/listing incarnation, + // and B's terminal/delete cleanup cannot be swallowed by that old tombstone. + internals._deletedRlmChildIds.add(`${childId}\0${assignmentA}`); + expect(root.releaseRlmChildSession(childId, childB, assignmentB)).toBe(unsubscribeB); + expect(unsubscribeB).not.toHaveBeenCalled(); + expect(root.registerRlmChildSession(childId, childB, unsubscribeB, assignmentB)).toBe(true); + internals._removeRlmSubagentTracking(childId, undefined, assignmentB); + expect(internals._rlmChildSessions.has(childId)).toBe(false); + expect(internals._deletedRlmChildIds.has(`${childId}\0${assignmentA}`)).toBe(true); + }); + it("retries and releases failed retained child cleanup on the next compaction", async () => { const childId = "retained-retry-child"; const childDir = join(tempDir, childId); @@ -2137,9 +2188,9 @@ describe("AgentSession rlm recursion", () => { }); it("lets a stale kernel depth cap defer to the live host gate", () => { - const python = - process.env.PRIME_AGENT_KERNEL_PYTHON ?? join(homedir(), ".prime", "agent", "kernel-venv", "bin", "python"); - const runtime = join(process.cwd(), "..", "..", "prime-agent-runtime", "src"); + const defaultPython = join(homedir(), ".prime", "agent", "kernel-venv", "bin", "python"); + const python = process.env.PRIME_AGENT_KERNEL_PYTHON ?? (existsSync(defaultPython) ? defaultPython : "python3"); + const runtime = join(__dirname, "..", "..", "..", "prime-agent-runtime", "src"); const probe = spawnSync( python, ["-c", "import asyncio, rlm; rlm.Comm = None; asyncio.run(rlm.run('raised live cap'))"], @@ -2482,6 +2533,154 @@ describe("AgentSession rlm recursion", () => { expect(root.getRlmChildSession(spawned.rlm_child_id)).toBeUndefined(); }); + it("keeps replacement B's hosted lifecycle maps untouched when stale A settles", async () => { + let releaseHostedCreate!: () => void; + const hostedCreateGate = new Promise((resolve) => { + releaseHostedCreate = resolve; + }); + let markHostedCreateStarted!: () => void; + const hostedCreateStarted = new Promise((resolve) => { + markHostedCreateStarted = resolve; + }); + let releaseATerminal!: () => void; + const aTerminalGate = new Promise((resolve) => { + releaseATerminal = resolve; + }); + let markAPromptStarted!: () => void; + const aPromptStarted = new Promise((resolve) => { + markAPromptStarted = resolve; + }); + let markAReleased!: () => void; + const aReleased = new Promise((resolve) => { + markAReleased = resolve; + }); + const childA = createSession({ + rlmSessionDir: join(tempDir, "stale-a"), + streamFn: () => { + markAPromptStarted(); + const stream = createAssistantMessageEventStream(); + void aTerminalGate.then(() => { + stream.push({ type: "done", reason: "stop", message: assistantMessage("A terminal", usage(41, 17)) }); + }); + return stream; + }, + }); + const childB = createSession({ rlmSessionDir: join(tempDir, "replacement-b") }); + const releaseRuntime = vi.fn(async (runtime: { session: AgentSession }) => { + expect(runtime.session).toBe(childA); + markAReleased(); + await runtime.session.disposeAsync(); + }); + const deleteRuntime = vi.fn(async (_id: string, runtime: AgentSession | undefined) => { + expect(runtime).toBe(childA); + await runtime?.disposeAsync(); + }); + const root = createSession({ + depth: 0, + maxDepth: 2, + subagentRuntimeHost: { + createRlmSubagentRuntime: async () => { + markHostedCreateStarted(); + await hostedCreateGate; + return { session: childA }; + }, + assignmentIdentityFenced: true, + releaseRlmSubagentRuntime: releaseRuntime, + deleteRlmSubagentRuntime: deleteRuntime, + }, + }); + const usageAttribution = vi.spyOn(root.sessionManager, "appendChildUsageAttribution"); + const updates: unknown[] = []; + root.subscribe((event) => { + if (event.type === "rlm_child_update") updates.push(event); + }); + + const admittedA = await root.runRlmChild("A must become stale", { name: "shared-worker" }); + await hostedCreateStarted; + const internals = root as unknown as InspectableRlmSession; + const bUnsubscribe = vi.fn(); + const bRun: InspectableRlmRun = { + id: admittedA.rlm_child_id, + assignmentId: "assignment-B", + sessionDir: join(tempDir, "replacement-b"), + status: "done", + settled: true, + abort: vi.fn(), + session: childB, + }; + const aUnsubscribe = vi.fn(); + const staleARun: InspectableRlmRun = { + id: admittedA.rlm_child_id, + assignmentId: "assignment-A", + sessionDir: join(tempDir, "stale-a"), + status: "done", + settled: true, + abort: vi.fn(), + session: childA, + }; + staleARun.unsubscribe = aUnsubscribe; + const bTombstone = `${admittedA.rlm_child_id}\u0000assignment-B`; + internals._activeRlmChildRuns.set(admittedA.rlm_child_id, bRun); + internals._rlmChildSessions.set(admittedA.rlm_child_id, childB); + internals._rlmChildSessionAssignments.set(admittedA.rlm_child_id, "assignment-B"); + internals._rlmChildUnsubscribes.set(admittedA.rlm_child_id, bUnsubscribe); + internals._rlmChildUnsubscribeAssignments.set(admittedA.rlm_child_id, "assignment-B"); + internals._deletedRlmChildIds.add(bTombstone); + updates.length = 0; + const injectedTerminal = vi.fn(async () => undefined); + (root as unknown as { _promptInjectedMessage: typeof injectedTerminal })._promptInjectedMessage = + injectedTerminal; + + releaseHostedCreate(); + await aPromptStarted; + releaseATerminal(); + await aReleased; + + // A's hosted create, terminal delivery, release, event update, and usage + // attribution have all settled after B reuses its public selector. + expect(releaseRuntime).toHaveBeenCalledOnce(); + expect(injectedTerminal).not.toHaveBeenCalled(); + expect(usageAttribution).not.toHaveBeenCalled(); + expect(updates).toEqual([]); + expect(internals._activeRlmChildRuns.get(admittedA.rlm_child_id)).toBe(bRun); + expect(internals._rlmChildSessions.get(admittedA.rlm_child_id)).toBe(childB); + expect(internals._rlmChildSessionAssignments.get(admittedA.rlm_child_id)).toBe("assignment-B"); + expect(internals._rlmChildUnsubscribes.get(admittedA.rlm_child_id)).toBe(bUnsubscribe); + expect(internals._rlmChildUnsubscribeAssignments.get(admittedA.rlm_child_id)).toBe("assignment-B"); + expect(internals._deletedRlmChildIds.has(bTombstone)).toBe(true); + + // Stale A release, delete-cleanup, registration, and unsubscribe paths have + // no authority over B's assignment-keyed maps or B's tombstone. + expect(root.releaseRlmChildSession(admittedA.rlm_child_id, childA, "assignment-A")).toBe(false); + expect(root.registerRlmChildSession(admittedA.rlm_child_id, childA, vi.fn(), "assignment-A")).toBe(false); + await internals._deleteRlmSubagentSession(admittedA.rlm_child_id, "assignment-A", childA); + expect(deleteRuntime).toHaveBeenCalledWith(admittedA.rlm_child_id, childA, "assignment-A"); + internals._removeRlmSubagentTracking(admittedA.rlm_child_id, staleARun, "assignment-A"); + expect(aUnsubscribe).toHaveBeenCalledOnce(); + expect(bUnsubscribe).not.toHaveBeenCalled(); + expect(internals._activeRlmChildRuns.get(admittedA.rlm_child_id)).toBe(bRun); + expect(internals._rlmChildSessions.get(admittedA.rlm_child_id)).toBe(childB); + expect(internals._rlmChildUnsubscribes.get(admittedA.rlm_child_id)).toBe(bUnsubscribe); + expect(internals._deletedRlmChildIds.has(bTombstone)).toBe(true); + }); + + it("rejects stale lazy-hydration rebind A when B owns the reused child ID", () => { + const root = createSession(); + const childA = createSession({ rlmSessionDir: join(tempDir, "lazy-a") }); + const childB = createSession({ rlmSessionDir: join(tempDir, "lazy-b") }); + const internals = root as unknown as InspectableRlmSession; + const unsubscribeB = vi.fn(); + + expect(root.registerRlmChildSession("lazy-reused-child", childB, unsubscribeB, "assignment-B")).toBe(true); + // A hydration continuation is still holding its stale session object. It + // must not join B merely because the public child selector matches. + expect(root.rebindRlmChildSessionAssignment("lazy-reused-child", childA, "assignment-A")).toBe(false); + expect(internals._rlmChildSessions.get("lazy-reused-child")).toBe(childB); + expect(internals._rlmChildSessionAssignments.get("lazy-reused-child")).toBe("assignment-B"); + expect(internals._rlmChildUnsubscribes.get("lazy-reused-child")).toBe(unsubscribeB); + expect(internals._rlmChildUnsubscribeAssignments.get("lazy-reused-child")).toBe("assignment-B"); + }); + it("does not let completion retention resurrect a child being deleted", async () => { const root = createSession(); const spawned = await root.runRlmChild("fast child", { name: "fast-worker" }); diff --git a/packages/coding-agent/test/daemon-mode.test.ts b/packages/coding-agent/test/daemon-mode.test.ts index e8e6aed07..0b7e33caf 100644 --- a/packages/coding-agent/test/daemon-mode.test.ts +++ b/packages/coding-agent/test/daemon-mode.test.ts @@ -12,6 +12,7 @@ import { sessionNameReservationKey, } from "../src/core/agent-messages.js"; import type { AgentObserveController } from "../src/core/agent-observe.js"; +import type { SessionActionRecoverySnapshot } from "../src/core/agent-session.js"; import type { CreateAgentSessionRuntimeFactory } from "../src/core/agent-session-runtime.js"; import type { AgentCronJob, AgentCronJobStore } from "../src/core/cron-jobs.js"; import { @@ -43,7 +44,25 @@ import { failure, } from "../src/modes/daemon/daemon-protocol.js"; import type { SessionSummary } from "../src/modes/daemon/daemon-session-list.js"; -import { DAEMON_WORKER_SUPERVISOR_SOCKET_ENV } from "../src/modes/daemon/daemon-worker-protocol.js"; +import { + DAEMON_WORKER_RECOVERY_JOURNAL_ENV, + DAEMON_WORKER_SUPERVISOR_SOCKET_ENV, +} from "../src/modes/daemon/daemon-worker-protocol.js"; +import { WorkerRecoveryJournal } from "../src/modes/daemon/worker-recovery-journal.js"; + +function deferred(): { + promise: Promise; + resolve(value: T | PromiseLike): void; + reject(reason?: unknown): void; +} { + let resolve!: (value: T | PromiseLike) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; + }); + return { promise, resolve, reject }; +} describe("daemon mode helpers", () => { it("preserves envelope client identity while registering prompt admission", () => { @@ -58,6 +77,616 @@ describe("daemon mode helpers", () => { expect(client.id).toBe("public-client"); }); + it("keeps B's same-family recovery checkpoint until B's terminal event after A completes and restart", () => { + const root = mkdtempSync(join(tmpdir(), "prime-agent-operation-token-")); + const journalPath = join(root, "worker-recovery.jsonl"); + const previousJournal = process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + try { + process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = journalPath; + const daemon = new AgentDaemon("/tmp/prime-agent-operation-token.sock", { + defaultSessionConfig: { agentDir: root, cwd: root }, + createRuntime: vi.fn(), + worker: { authenticationToken: "test" }, + }); + const state = makeState("active"); + Object.assign(state, { eventGeneration: "11111111-1111-4111-8111-111111111111" }); + Object.assign(state.runtime, { session: { sessionId: "session" } }); + const internals = daemon as unknown as { + beginWorkerRecoveryOperation(state: ActiveSessionState, operation: "prompt"): unknown; + queueWorkerRecoveryTurn(state: ActiveSessionState, token: unknown): void; + checkpointWorkerRecoveryEvent(state: ActiveSessionState, operation: "turn_start" | "turn_end"): void; + }; + const tokenA = internals.beginWorkerRecoveryOperation(state, "prompt"); + internals.queueWorkerRecoveryTurn(state, tokenA); + const tokenB = internals.beginWorkerRecoveryOperation(state, "prompt"); + internals.queueWorkerRecoveryTurn(state, tokenB); + + // The scheduler starts/ends A while B is already admitted. A's terminal + // must clear only A's UUID; B remains durable crash evidence. + internals.checkpointWorkerRecoveryEvent(state, "turn_start"); + internals.checkpointWorkerRecoveryEvent(state, "turn_end"); + const afterA = WorkerRecoveryJournal.readLatest(journalPath); + expect(afterA).toEqual([expect.objectContaining({ busy: true, operation: "prompt" })]); + + // A fresh reader models a worker restart. It still sees B, then B's own + // terminal event clears it; no mutable operation-family current map exists. + expect(new WorkerRecoveryJournal(journalPath).getLatest()).toEqual(afterA); + internals.checkpointWorkerRecoveryEvent(state, "turn_start"); + internals.checkpointWorkerRecoveryEvent(state, "turn_end"); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual([]); + } finally { + if (previousJournal === undefined) delete process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + else process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = previousJournal; + rmSync(root, { recursive: true, force: true }); + } + }); + + it("clears an exact token after session materialization changes its checkpoint metadata", () => { + const root = mkdtempSync(join(tmpdir(), "prime-agent-materialized-recovery-")); + const journalPath = join(root, "worker-recovery.jsonl"); + const previousJournal = process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + try { + process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = journalPath; + const daemon = new AgentDaemon("/tmp/prime-agent-materialized-recovery.sock", { + defaultSessionConfig: { agentDir: root, cwd: root }, + createRuntime: vi.fn(), + worker: { authenticationToken: "test" }, + }); + const state = makeState("active"); + Object.assign(state, { eventGeneration: "11111111-1111-4111-8111-111111111111" }); + Object.assign(state.runtime, { session: { sessionId: "draft" } }); + const internals = daemon as unknown as { + beginWorkerRecoveryOperation(state: ActiveSessionState, operation: "prompt"): unknown; + completeWorkerRecoveryOperation(state: ActiveSessionState, token: unknown): void; + }; + + const token = internals.beginWorkerRecoveryOperation(state, "prompt"); + // writeWorkerRecoveryOperation re-reads the session at completion. A + // persisted draft can materialize or branch while this token is live. + Object.assign(state.runtime.session, { + sessionId: "materialized", + sessionFile: join(root, "materialized.jsonl"), + }); + internals.completeWorkerRecoveryOperation(state, token); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual([]); + } finally { + if (previousJournal === undefined) delete process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + else process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = previousJournal; + rmSync(root, { recursive: true, force: true }); + } + }); + + it("uses exact operation tokens for admission cancellation, active steer, rejection, and observation checkpoints", () => { + const root = mkdtempSync(join(tmpdir(), "prime-agent-operation-token-edges-")); + const journalPath = join(root, "worker-recovery.jsonl"); + const previousJournal = process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + try { + process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = journalPath; + const daemon = new AgentDaemon("/tmp/prime-agent-operation-token-edges.sock", { + defaultSessionConfig: { agentDir: root, cwd: root }, + createRuntime: vi.fn(), + worker: { authenticationToken: "test" }, + }); + const state = makeState("active"); + Object.assign(state, { eventGeneration: "11111111-1111-4111-8111-111111111111" }); + Object.assign(state.runtime, { session: { sessionId: "session" } }); + const internals = daemon as unknown as { + beginWorkerRecoveryOperation(state: ActiveSessionState, operation: "prompt" | "steer_queued"): unknown; + queueWorkerRecoveryTurn(state: ActiveSessionState, token: unknown): void; + cancelQueuedWorkerRecoveryTurn(state: ActiveSessionState, token: unknown): void; + attachWorkerRecoveryToActiveTurn(state: ActiveSessionState, token: unknown): boolean; + completeWorkerRecoveryOperation(state: ActiveSessionState, token: unknown): void; + checkpointWorkerRecoveryEvent( + state: ActiveSessionState, + operation: "turn_start" | "turn_end" | "session_action_update" | "rlm_child_update", + ): void; + }; + + // An async prompt finally cancels its own pending admission token. + const abandonedPrompt = internals.beginWorkerRecoveryOperation(state, "prompt"); + internals.queueWorkerRecoveryTurn(state, abandonedPrompt); + internals.cancelQueuedWorkerRecoveryTurn(state, abandonedPrompt); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual([]); + + // A steer while A's turn is active is attached to A, not queued for a + // nonexistent next turn. turn_end clears every exact token for that turn. + const turnA = internals.beginWorkerRecoveryOperation(state, "prompt"); + internals.queueWorkerRecoveryTurn(state, turnA); + internals.checkpointWorkerRecoveryEvent(state, "turn_start"); + const activeSteer = internals.beginWorkerRecoveryOperation(state, "steer_queued"); + expect(internals.attachWorkerRecoveryToActiveTurn(state, activeSteer)).toBe(true); + internals.checkpointWorkerRecoveryEvent(state, "turn_end"); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual([]); + + // A rejected paired command exact-clears the token it began. + const rejected = internals.beginWorkerRecoveryOperation(state, "steer_queued"); + internals.completeWorkerRecoveryOperation(state, rejected); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual([]); + + // Observation events retain durable evidence but cannot strand busy work. + internals.checkpointWorkerRecoveryEvent(state, "session_action_update"); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual([]); + internals.checkpointWorkerRecoveryEvent(state, "rlm_child_update"); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual([]); + } finally { + if (previousJournal === undefined) delete process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + else process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = previousJournal; + rmSync(root, { recursive: true, force: true }); + } + }); + + it("immediately clears a rejected active steer while its outer turn remains owned", async () => { + const root = mkdtempSync(join(tmpdir(), "prime-agent-rejected-active-steer-")); + const journalPath = join(root, "worker-recovery.jsonl"); + const previousJournal = process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + try { + process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = journalPath; + const daemon = new AgentDaemon("/tmp/prime-agent-rejected-active-steer.sock", { + defaultSessionConfig: { agentDir: root, cwd: root }, + createRuntime: vi.fn(), + worker: { authenticationToken: "test" }, + }); + const state = makeState("active"); + Object.assign(state, { eventGeneration: "11111111-1111-4111-8111-111111111111" }); + const checkpoint = Reflect.get(daemon, "checkpointWorkerRecoveryEvent").bind(daemon) as ( + state: ActiveSessionState, + event: "turn_start" | "turn_end", + ) => void; + const steerResult = deferred(); + state.runtime = { + ...state.runtime, + session: { sessionId: "session", steer: vi.fn(() => steerResult.promise) }, + } as never; + Reflect.get(daemon, "sessions").set("active", state); + const internals = daemon as unknown as { + beginWorkerRecoveryOperation(state: ActiveSessionState, operation: "prompt"): unknown; + queueWorkerRecoveryTurn(state: ActiveSessionState, token: unknown): void; + handleCommand(client: DaemonSocketClient, command: DaemonCommand): Promise; + }; + + // A owns the outer turn. B first becomes the observable current token, + // then fails while attached to A's active frame. + const outer = internals.beginWorkerRecoveryOperation(state, "prompt"); + internals.queueWorkerRecoveryTurn(state, outer); + checkpoint(state, "turn_start"); + const rejectedSteer = internals.handleCommand(makeClient("client", "active"), { + type: "steer", + activeSessionId: "active", + message: "reject", + }); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual( + expect.arrayContaining([expect.objectContaining({ busy: true, operation: "steer_queued" })]), + ); + steerResult.reject(new Error("steer rejected")); + await expect(rejectedSteer).rejects.toThrow("steer rejected"); + + // B is exact-cleared at its own rejection, and the still-live A is + // republished immediately rather than waiting for any turn terminal. + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual( + expect.arrayContaining([expect.objectContaining({ busy: true, operation: "prompt" })]), + ); + const afterRejection = readFileSync(journalPath, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)) as Array<{ busy: boolean; operation: string }>; + expect(afterRejection).toEqual([expect.objectContaining({ busy: true, operation: "prompt" })]); + + // An unrelated nested terminal cannot resurrect B; A remains owned until + // A's own terminal, which is the first point that clears it. + checkpoint(state, "turn_start"); + checkpoint(state, "turn_end"); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual( + expect.arrayContaining([expect.objectContaining({ busy: true, operation: "prompt" })]), + ); + checkpoint(state, "turn_end"); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual([]); + } finally { + if (previousJournal === undefined) delete process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + else process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = previousJournal; + rmSync(root, { recursive: true, force: true }); + } + }); + + it("retains an accepted non-waiting prompt token through its deferred terminal turn", async () => { + const root = mkdtempSync(join(tmpdir(), "prime-agent-accepted-prompt-")); + const journalPath = join(root, "worker-recovery.jsonl"); + const previousJournal = process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + try { + process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = journalPath; + const daemon = new AgentDaemon("/tmp/prime-agent-accepted-prompt.sock", { + defaultSessionConfig: { agentDir: root, cwd: root }, + createRuntime: vi.fn(), + worker: { authenticationToken: "test" }, + }); + const state = makeState("active"); + Object.assign(state, { eventGeneration: "11111111-1111-4111-8111-111111111111" }); + const terminal = deferred(); + state.runtime = { + ...state.runtime, + session: { + sessionId: "session", + isStreaming: false, + promptUntilAccepted: vi.fn( + async (_message: string, options?: { preflightResult?: (accepted: boolean) => void }) => { + options?.preflightResult?.(true); + await terminal.promise; + }, + ), + }, + } as never; + Reflect.get(daemon, "sessions").set("active", state); + const handle = Reflect.get(daemon, "handleCommand").bind(daemon) as ( + client: DaemonSocketClient, + command: DaemonCommand, + ) => Promise; + await handle(makeClient("client", "active"), { + type: "prompt", + activeSessionId: "active", + message: "queued", + streamingBehavior: "followUp", + }); + await vi.waitFor(() => + expect(WorkerRecoveryJournal.readLatest(journalPath)[0]).toMatchObject({ busy: true, operation: "prompt" }), + ); + terminal.resolve(); + await Promise.resolve(); + // Acceptance releases only the response/admission plumbing, never the queued token. + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual( + expect.arrayContaining([expect.objectContaining({ busy: true, operation: "prompt" })]), + ); + const checkpoint = Reflect.get(daemon, "checkpointWorkerRecoveryEvent").bind(daemon) as ( + state: ActiveSessionState, + event: "turn_start" | "turn_end", + ) => void; + checkpoint(state, "turn_start"); + checkpoint(state, "turn_end"); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual([]); + } finally { + if (previousJournal === undefined) delete process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + else process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = previousJournal; + rmSync(root, { recursive: true, force: true }); + } + }); + + it("pairs nested recovery events and turns in LIFO order", () => { + const root = mkdtempSync(join(tmpdir(), "prime-agent-lifo-recovery-")); + const journalPath = join(root, "worker-recovery.jsonl"); + const previousJournal = process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + try { + process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = journalPath; + const daemon = new AgentDaemon("/tmp/prime-agent-lifo-recovery.sock", { + defaultSessionConfig: { agentDir: root, cwd: root }, + createRuntime: vi.fn(), + worker: { authenticationToken: "test" }, + }); + const state = makeState("active"); + Object.assign(state, { eventGeneration: "11111111-1111-4111-8111-111111111111" }); + Object.assign(state.runtime, { session: { sessionId: "session" } }); + const internals = daemon as unknown as { + beginWorkerRecoveryOperation(state: ActiveSessionState, operation: "prompt" | "follow_up_queued"): unknown; + queueWorkerRecoveryTurn(state: ActiveSessionState, token: unknown): void; + checkpointWorkerRecoveryEvent( + state: ActiveSessionState, + event: "turn_start" | "turn_end" | "message_start" | "message_end", + ): void; + }; + const a = internals.beginWorkerRecoveryOperation(state, "prompt"); + const b = internals.beginWorkerRecoveryOperation(state, "follow_up_queued"); + internals.queueWorkerRecoveryTurn(state, a); + internals.queueWorkerRecoveryTurn(state, b); + internals.checkpointWorkerRecoveryEvent(state, "turn_start"); + internals.checkpointWorkerRecoveryEvent(state, "turn_start"); + internals.checkpointWorkerRecoveryEvent(state, "turn_end"); + // B is completed at B's edge and A is immediately republished because it + // remains active. The outer terminal subsequently clears A, not B. + const nestedTurnRecords = readFileSync(journalPath, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line)) as Array<{ + busy: boolean; + operation: string; + }>; + expect(nestedTurnRecords).toEqual([expect.objectContaining({ busy: true, operation: "prompt" })]); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual( + expect.arrayContaining([expect.objectContaining({ busy: true, operation: "prompt" })]), + ); + internals.checkpointWorkerRecoveryEvent(state, "turn_end"); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual([]); + internals.checkpointWorkerRecoveryEvent(state, "message_start"); + internals.checkpointWorkerRecoveryEvent(state, "message_start"); + internals.checkpointWorkerRecoveryEvent(state, "message_end"); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual( + expect.arrayContaining([expect.objectContaining({ busy: true, operation: "message_start" })]), + ); + internals.checkpointWorkerRecoveryEvent(state, "message_end"); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual([]); + } finally { + if (previousJournal === undefined) delete process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + else process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = previousJournal; + rmSync(root, { recursive: true, force: true }); + } + }); + + it("preserves active recovery evidence across instantaneous observations", () => { + const root = mkdtempSync(join(tmpdir(), "prime-agent-observation-recovery-")); + const journalPath = join(root, "worker-recovery.jsonl"); + const previousJournal = process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + try { + process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = journalPath; + const daemon = new AgentDaemon("/tmp/prime-agent-observation-recovery.sock", { + defaultSessionConfig: { agentDir: root, cwd: root }, + createRuntime: vi.fn(), + worker: { authenticationToken: "test" }, + }); + const state = makeState("active"); + Object.assign(state.runtime, { session: { sessionId: "session" } }); + const internals = daemon as unknown as { + beginWorkerRecoveryOperation(state: ActiveSessionState, operation: "prompt"): unknown; + queueWorkerRecoveryTurn(state: ActiveSessionState, token: unknown): void; + checkpointWorkerRecoveryEvent( + state: ActiveSessionState, + event: + | "turn_start" + | "turn_end" + | "tool_execution_start" + | "tool_execution_end" + | "session_action_update" + | "rlm_child_update", + ): void; + }; + const prompt = internals.beginWorkerRecoveryOperation(state, "prompt"); + internals.queueWorkerRecoveryTurn(state, prompt); + internals.checkpointWorkerRecoveryEvent(state, "turn_start"); + internals.checkpointWorkerRecoveryEvent(state, "tool_execution_start"); + + for (const observation of ["session_action_update", "rlm_child_update"] as const) { + internals.checkpointWorkerRecoveryEvent(state, observation); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual( + expect.arrayContaining([expect.objectContaining({ busy: true, operation: "tool_execution_start" })]), + ); + } + + internals.checkpointWorkerRecoveryEvent(state, "tool_execution_end"); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual( + expect.arrayContaining([expect.objectContaining({ busy: true, operation: "prompt" })]), + ); + internals.checkpointWorkerRecoveryEvent(state, "turn_end"); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual([]); + } finally { + if (previousJournal === undefined) delete process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + else process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = previousJournal; + rmSync(root, { recursive: true, force: true }); + } + }); + + it("uses the worker generation for ready and terminal recovery records", () => { + const root = mkdtempSync(join(tmpdir(), "prime-agent-worker-generation-")); + const journalPath = join(root, "worker-recovery.jsonl"); + const previousJournal = process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + const previousGeneration = process.env.PRIME_AGENT_INTERNAL_DAEMON_WORKER_GENERATION; + try { + process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = journalPath; + process.env.PRIME_AGENT_INTERNAL_DAEMON_WORKER_GENERATION = "22222222-2222-4222-8222-222222222222"; + const daemon = new AgentDaemon("/tmp/prime-agent-worker-generation.sock", { + defaultSessionConfig: { agentDir: root, cwd: root }, + createRuntime: vi.fn(), + worker: { authenticationToken: "test" }, + }); + const state = makeState("active"); + Object.assign(state, { eventGeneration: "11111111-1111-4111-8111-111111111111" }); + Object.assign(state.runtime, { session: { sessionId: "session" } }); + const internals = daemon as unknown as { + recordWorkerRecoveryState(state: ActiveSessionState, operation: "ready"): void; + beginWorkerRecoveryOperation(state: ActiveSessionState, operation: "prompt"): unknown; + completeWorkerRecoveryOperation(state: ActiveSessionState, token: unknown): void; + }; + internals.recordWorkerRecoveryState(state, "ready"); + const prompt = internals.beginWorkerRecoveryOperation(state, "prompt"); + internals.completeWorkerRecoveryOperation(state, prompt); + const latest = WorkerRecoveryJournal.readLatest(journalPath); + expect(latest).toEqual([]); + } finally { + if (previousJournal === undefined) delete process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + else process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = previousJournal; + if (previousGeneration === undefined) delete process.env.PRIME_AGENT_INTERNAL_DAEMON_WORKER_GENERATION; + else process.env.PRIME_AGENT_INTERNAL_DAEMON_WORKER_GENERATION = previousGeneration; + rmSync(root, { recursive: true, force: true }); + } + }); + + it("publishes a follow-up token before an admitted turn can synchronously start", async () => { + const root = mkdtempSync(join(tmpdir(), "prime-agent-follow-up-race-")); + const journalPath = join(root, "worker-recovery.jsonl"); + const previousJournal = process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + try { + process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = journalPath; + const daemon = new AgentDaemon("/tmp/prime-agent-follow-up-race.sock", { + defaultSessionConfig: { agentDir: root, cwd: root }, + createRuntime: vi.fn(), + worker: { authenticationToken: "test" }, + }); + const state = makeState("active"); + Object.assign(state, { eventGeneration: "11111111-1111-4111-8111-111111111111" }); + const checkpoint = Reflect.get(daemon, "checkpointWorkerRecoveryEvent").bind(daemon) as ( + state: ActiveSessionState, + event: "turn_start" | "turn_end", + ) => void; + const held = deferred(); + state.runtime = { + ...state.runtime, + session: { + sessionId: "session", + followUp: vi.fn(() => { + checkpoint(state, "turn_start"); + checkpoint(state, "turn_end"); + return held.promise; + }), + }, + } as never; + Reflect.get(daemon, "sessions").set("active", state); + const handle = Reflect.get(daemon, "handleCommand").bind(daemon) as ( + client: DaemonSocketClient, + command: DaemonCommand, + ) => Promise; + const pending = handle(makeClient("client", "active"), { + type: "follow_up", + activeSessionId: "active", + message: "race", + }); + // The turn terminal ran before the awaited followUp promise settled. + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual([]); + held.resolve(true); + await pending; + // No token was appended after the turn, so a later unrelated terminal cannot consume it. + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual([]); + } finally { + if (previousJournal === undefined) delete process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + else process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = previousJournal; + rmSync(root, { recursive: true, force: true }); + } + }); + + it("derives compatibility ready records from live busy state instead of stranding idle sessions", () => { + const root = mkdtempSync(join(tmpdir(), "prime-agent-ready-recovery-")); + const journalPath = join(root, "worker-recovery.jsonl"); + const previousJournal = process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + try { + process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = journalPath; + const daemon = new AgentDaemon("/tmp/prime-agent-ready-recovery.sock", { + defaultSessionConfig: { agentDir: root, cwd: root }, + createRuntime: vi.fn(), + worker: { authenticationToken: "test" }, + }); + const state = makeState("active"); + Object.assign(state.runtime, { + session: { + sessionId: "session", + isStreaming: false, + isCompacting: false, + isRetrying: false, + hasAcceptedPromptInFlight: false, + }, + }); + const record = Reflect.get(daemon, "recordWorkerRecoveryState").bind(daemon) as ( + state: ActiveSessionState, + operation: "ready", + ) => void; + record(state, "ready"); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual([]); + Object.assign(state.runtime.session, { isRetrying: true }); + record(state, "ready"); + expect(WorkerRecoveryJournal.readLatest(journalPath)).toEqual( + expect.arrayContaining([expect.objectContaining({ operation: "ready", busy: true })]), + ); + } finally { + if (previousJournal === undefined) delete process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + else process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = previousJournal; + rmSync(root, { recursive: true, force: true }); + } + }); + + it("keeps restored action identities until their own terminal or cancellation", async () => { + const root = mkdtempSync(join(tmpdir(), "prime-agent-restore-action-recovery-")); + const journalPath = join(root, "worker-recovery.jsonl"); + const previousJournal = process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + try { + process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = journalPath; + const daemon = new AgentDaemon("/tmp/prime-agent-restore-action-recovery.sock", { + defaultSessionConfig: { agentDir: root, cwd: root }, + createRuntime: vi.fn(), + worker: { authenticationToken: "test" }, + }); + const state = makeState("active"); + const actionIds = ["restored-a", "restored-b"]; + let unfinished = [...actionIds]; + const restoreSessionActions = vi.fn(async (snapshot: SessionActionRecoverySnapshot) => + snapshot.actions.map((action) => action.id), + ); + const validateSessionActionRecoverySnapshot = vi.fn((snapshot: SessionActionRecoverySnapshot) => { + const actionIds = new Set(); + for (const action of snapshot.actions) { + if (actionIds.has(action.id)) throw new Error(`Duplicate session action id: ${action.id}`); + actionIds.add(action.id); + } + }); + Object.assign(state.runtime, { + session: { + sessionId: "session", + validateSessionActionRecoverySnapshot, + get unfinishedActionCount() { + return unfinished.length; + }, + get unfinishedActionIds() { + return unfinished; + }, + restoreSessionActions, + }, + }); + Reflect.get(daemon, "sessions").set("active", state); + const handle = Reflect.get(daemon, "handleCommand").bind(daemon) as ( + client: DaemonSocketClient, + command: DaemonCommand, + ) => Promise; + const snapshot = { + formatVersion: 1 as const, + actions: actionIds.map((id) => ({ id })), + } as unknown as SessionActionRecoverySnapshot; + const duplicateSnapshot = { + formatVersion: 1 as const, + actions: [{ id: "duplicate" }, { id: "duplicate" }], + } as unknown as SessionActionRecoverySnapshot; + // Validate before durable allocation: duplicate IDs must neither invoke the + // mutating restore nor leave orphaned crash evidence visible after restart. + await expect( + handle(makeClient("client", "active"), { + type: "restore_actions", + activeSessionId: "active", + snapshot: duplicateSnapshot, + }), + ).rejects.toThrow("Duplicate session action id: duplicate"); + expect(restoreSessionActions).not.toHaveBeenCalled(); + expect(WorkerRecoveryJournal.readLatest(journalPath).filter((record) => record.busy)).toHaveLength(0); + expect(new WorkerRecoveryJournal(journalPath).getLatest()).toEqual([]); + // N=0 clears the exact admission synchronously; restore failure does the + // same, without touching a later successful restore's identities. + await handle(makeClient("client", "active"), { + type: "restore_actions", + activeSessionId: "active", + snapshot: { formatVersion: 1, actions: [] }, + }); + expect(WorkerRecoveryJournal.readLatest(journalPath).filter((record) => record.busy)).toHaveLength(0); + restoreSessionActions.mockRejectedValueOnce(new Error("restore failed")); + await expect( + handle(makeClient("client", "active"), { type: "restore_actions", activeSessionId: "active", snapshot }), + ).rejects.toThrow("restore failed"); + expect(WorkerRecoveryJournal.readLatest(journalPath).filter((record) => record.busy)).toHaveLength(0); + // Return a deliberately reordered partial result: action IDs, not array + // positions, must own their exact recovery identities. + restoreSessionActions.mockResolvedValueOnce(["restored-b"]); + await handle(makeClient("client", "active"), { type: "restore_actions", activeSessionId: "active", snapshot }); + const busyAfterPartialRestore = WorkerRecoveryJournal.readLatest(journalPath).filter((record) => record.busy); + expect(busyAfterPartialRestore).toEqual([expect.objectContaining({ operation: "actions_restored" })]); + unfinished = ["restored-a"]; + Reflect.get(daemon, "checkpointWorkerRecoveryEvent").call(daemon, state, "session_action_update"); + // restored-a was declared but not returned by this partial restore, so it + // cannot keep or clear restored-b's token. + expect(WorkerRecoveryJournal.readLatest(journalPath).filter((record) => record.busy)).toHaveLength(0); + unfinished = [...actionIds]; + await handle(makeClient("client", "active"), { type: "restore_actions", activeSessionId: "active", snapshot }); + expect(WorkerRecoveryJournal.readLatest(journalPath).filter((record) => record.busy)).toHaveLength(2); + unfinished = ["restored-b"]; + Reflect.get(daemon, "checkpointWorkerRecoveryEvent").call(daemon, state, "session_action_update"); + expect(WorkerRecoveryJournal.readLatest(journalPath).filter((record) => record.busy)).toHaveLength(1); + unfinished = []; + Reflect.get(daemon, "checkpointWorkerRecoveryEvent").call(daemon, state, "session_action_update"); + expect(WorkerRecoveryJournal.readLatest(journalPath).filter((record) => record.busy)).toHaveLength(0); + } finally { + if (previousJournal === undefined) delete process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV]; + else process.env[DAEMON_WORKER_RECOVERY_JOURNAL_ENV] = previousJournal; + rmSync(root, { recursive: true, force: true }); + } + }); + it("normalizes daemon session names before validation and persistence", async () => { const daemon = new AgentDaemon("/tmp/unused-daemon.sock", { defaultSessionConfig: { agentDir: "/tmp", cwd: "/tmp" }, @@ -657,15 +1286,21 @@ describe("daemon mode helpers", () => { }); const parentState = makeState("parent"); const childState = makeState("child", parentState.activeSessionId); + const assignmentId = "11111111-1111-4111-8111-111111111111"; Object.assign(childState.runtime.metadata, { kind: "subagent", parentActiveSessionId: parentState.activeSessionId, rlmChildId: "child-1", + assignmentId, }); let internals: { sessions: Map; closeSession: (state: ActiveSessionState, reason: "completed" | "killed") => Promise; - recordRlmSubagentDeletion(parentState: ActiveSessionState, childId: string): Promise; + recordRlmSubagentDeletion( + parentState: ActiveSessionState, + childId: string, + assignmentId: string, + ): Promise; createSubagentRuntimeHost(parentState: ActiveSessionState): SubagentRuntimeHost; }; const closeSession = vi.fn(async (state: ActiveSessionState) => { @@ -681,8 +1316,8 @@ describe("daemon mode helpers", () => { await internals .createSubagentRuntimeHost(parentState) .releaseRlmSubagentRuntime?.( - { session: childState.runtime.session }, - { id: "child-1" } as CreateRlmSubagentRuntimeOptions, + { session: childState.runtime.session, assignmentId }, + { id: "child-1", assignmentId } as CreateRlmSubagentRuntimeOptions, "error", ); @@ -692,11 +1327,11 @@ describe("daemon mode helpers", () => { await internals .createSubagentRuntimeHost(parentState) .releaseRlmSubagentRuntime?.( - { session: childState.runtime.session }, - { id: "child-1" } as CreateRlmSubagentRuntimeOptions, + { session: childState.runtime.session, assignmentId }, + { id: "child-1", assignmentId } as CreateRlmSubagentRuntimeOptions, "cancelled", ); - expect(recordDeletion).toHaveBeenCalledWith(parentState, "child-1"); + expect(recordDeletion).toHaveBeenCalledWith(parentState, "child-1", assignmentId); expect(recordDeletion.mock.invocationCallOrder[0]).toBeLessThan(closeSession.mock.invocationCallOrder[1]!); expect(closeSession).toHaveBeenLastCalledWith(childState, "killed"); expect(internals.sessions.has(childState.activeSessionId)).toBe(false); @@ -710,8 +1345,8 @@ describe("daemon mode helpers", () => { internals .createSubagentRuntimeHost(parentState) .releaseRlmSubagentRuntime?.( - { session: childState.runtime.session }, - { id: "child-1" } as CreateRlmSubagentRuntimeOptions, + { session: childState.runtime.session, assignmentId }, + { id: "child-1", assignmentId } as CreateRlmSubagentRuntimeOptions, "cancelled", ), ).rejects.toThrow("registry write failed"); @@ -719,6 +1354,273 @@ describe("daemon mode helpers", () => { expect(internals.sessions.has(childState.activeSessionId)).toBe(false); }); + it("fences stale assignment completion and deletion when a child selector is reused", async () => { + const daemon = new AgentDaemon("/tmp/prime-agent-assignment-fence.sock", { + defaultSessionConfig: { agentDir: "/tmp", cwd: "/tmp" }, + createRuntime: vi.fn(), + }); + const parent = makeState("parent"); + const assignmentA = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + const assignmentB = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + const childB = makeState("child-b", parent.activeSessionId); + Object.assign(childB.runtime.metadata, { + kind: "subagent", + parentActiveSessionId: parent.activeSessionId, + rlmChildId: "reused-child", + assignmentId: assignmentB, + }); + const internals = daemon as unknown as { + sessions: Map; + readLatestRlmSubagentRegistry: ReturnType; + createSubagentRuntimeHost(parentState: ActiveSessionState): SubagentRuntimeHost; + closeSession: ReturnType; + }; + internals.readLatestRlmSubagentRegistry = vi.fn(async () => []); + internals.sessions.set(parent.activeSessionId, parent); + internals.sessions.set(childB.activeSessionId, childB); + internals.closeSession = vi.fn(async () => undefined); + const host = internals.createSubagentRuntimeHost(parent); + + // A's completion cannot publish B's registry state, and A's delete cannot + // close or remove B even though the mock reused the same child id. + expect(host.completeRlmSubagentRuntime?.("reused-child", childB.runtime.session, assignmentA)).toBe(false); + await host.deleteRlmSubagentRuntime?.("reused-child", childB.runtime.session, assignmentA); + expect(internals.closeSession).not.toHaveBeenCalled(); + expect(internals.sessions.get(childB.activeSessionId)).toBe(childB); + }); + + it("rejects a stale lazy-hydration assignment rebind without joining B's reused selector", async () => { + const daemon = new AgentDaemon("/tmp/prime-agent-lazy-assignment-join.sock", { + defaultSessionConfig: { agentDir: "/tmp", cwd: "/tmp" }, + createRuntime: vi.fn(), + }); + const parent = makeState("parent"); + const childB = makeState("child-b", parent.activeSessionId); + const childA = makeState("child-a", parent.activeSessionId); + Object.assign(childB.runtime, { + metadata: { + kind: "subagent", + createdAt: 1, + parentActiveSessionId: parent.activeSessionId, + rlmChildId: "lazy-reused-child", + assignmentId: "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb", + }, + session: { sessionId: "B", sessionName: "B", sessionFile: "/tmp/B.jsonl" }, + }); + Object.assign(childA.runtime, { + metadata: { + kind: "subagent", + createdAt: 1, + parentActiveSessionId: parent.activeSessionId, + rlmChildId: "lazy-reused-child", + assignmentId: "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa", + }, + session: { sessionId: "A", sessionName: "A", sessionFile: "/tmp/A.jsonl" }, + }); + const internals = daemon as unknown as { + sessions: Map; + getOrHydrateBoundSessionState(id: string): Promise; + findPassiveRlmSubagent: ReturnType; + hydratePassiveRlmSubagent: ReturnType; + }; + internals.sessions.set(parent.activeSessionId, parent); + internals.sessions.set(childB.activeSessionId, childB); + // An obsolete A passive lookup must join the already-bound B session, + // rather than reopening/rebinding A by its shared child selector. + internals.findPassiveRlmSubagent = vi.fn(async () => undefined); + internals.hydratePassiveRlmSubagent = vi.fn(async () => childA); + await expect(internals.getOrHydrateBoundSessionState("lazy-reused-child")).resolves.toBe(childB); + expect(internals.hydratePassiveRlmSubagent).not.toHaveBeenCalled(); + expect(internals.sessions.get(childB.activeSessionId)).toBe(childB); + }); + + it("binds a legacy passive delete to a fresh assignment before persisting its exact tombstone", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "prime-agent-legacy-delete-assignment-")); + try { + const fixture = makePersistedRlmDaemonFixture(tempDir); + const registryPath = join(fixture.parentArtifactDir, "rlm-subagents.jsonl"); + const legacy = JSON.parse(readFileSync(registryPath, "utf8")) as Record; + delete legacy.assignmentId; + writeFileSync(registryPath, `${JSON.stringify(legacy)}\n`); + const internals = fixture.daemon as unknown as { + createRuntime(command: Extract): Promise; + createSubagentRuntimeHost(parent: ActiveSessionState): SubagentRuntimeHost; + }; + const parent = await internals.createRuntime({ type: "create", sessionPath: fixture.parentSessionFile }); + await internals.createSubagentRuntimeHost(parent).deleteRlmSubagentRuntime(fixture.childId); + const rows = readFileSync(registryPath, "utf8") + .trim() + .split(/\r?\n/) + .map((line) => JSON.parse(line) as { childId: string; assignmentId?: string; status: string }); + const bound = rows.at(-2); + const deleted = rows.at(-1); + expect(bound).toMatchObject({ childId: fixture.childId, status: "completed" }); + expect(bound?.assignmentId).toMatch(/^[0-9a-f-]{36}$/i); + expect(deleted).toMatchObject({ + childId: fixture.childId, + assignmentId: bound?.assignmentId, + status: "deleted", + }); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("keeps assignment-less registry rows display-only until an assigned row is persisted", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "prime-agent-legacy-assignment-row-")); + try { + const daemon = new AgentDaemon(join(tempDir, "daemon.sock"), { + defaultSessionConfig: { agentDir: tempDir, cwd: tempDir }, + createRuntime: vi.fn(), + }); + const registryPath = join(tempDir, "rlm-subagents.jsonl"); + writeFileSync( + registryPath, + `${JSON.stringify({ + type: "rlm_subagent", + childId: "legacy-child", + sessionName: "legacy", + sessionDir: tempDir, + sessionFile: join(tempDir, "legacy.jsonl"), + status: "completed", + })}\n`, + ); + const read = ( + daemon as unknown as { + readLatestRlmSubagentRegistryPath(path: string): Promise>; + } + ).readLatestRlmSubagentRegistryPath.bind(daemon); + const entries = await read(registryPath); + expect(entries).toHaveLength(1); + expect(entries[0]).not.toHaveProperty("assignmentId"); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("selects only B's latest durable reuse for legacy delete and passive catalog traversal", async () => { + const tempDir = mkdtempSync(join(tmpdir(), "prime-agent-reused-durable-registry-")); + try { + const fixture = makePersistedRlmDaemonFixture(tempDir); + const registryPath = join(fixture.parentArtifactDir, "rlm-subagents.jsonl"); + const rows = readFileSync(registryPath, "utf8") + .trim() + .split(/\r?\n/) + .map((line) => JSON.parse(line) as Record); + const a = rows[0]!; + const childBSessionDir = join(fixture.parentArtifactDir, "sub-reused-b"); + const childBManager = SessionManager.create(tempDir, childBSessionDir); + childBManager.newSession({ parentSession: fixture.parentSessionFile }); + childBManager.appendSessionInfo("reused-B"); + childBManager.appendMessage({ role: "user", content: "B's distinct session file", timestamp: 3 }); + childBManager.flushNow(); + const childBSessionFile = childBManager.getSessionFile(); + if (!childBSessionFile) throw new Error("Missing B session file"); + const assignmentB = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + writeFileSync( + registryPath, + `${JSON.stringify(a)}\n${JSON.stringify({ + ...a, + assignmentId: assignmentB, + sessionName: "reused-B", + sessionDir: childBSessionDir, + sessionFile: childBSessionFile, + parentSessionId: childBManager.getSessionId(), + status: "completed", + createdAt: 3, + updatedAt: "2026-01-01T00:00:03.000Z", + })}\n`, + ); + const internals = fixture.daemon as unknown as { + createRuntime(command: Extract): Promise; + createSubagentRuntimeHost(parent: ActiveSessionState): SubagentRuntimeHost; + listPassiveRlmSubagents(): Promise< + Array<{ entry: { childId: string; sessionFile: string; assignmentId?: string } }> + >; + findPassiveRlmSubagent( + target: string, + ): Promise<{ entry: { sessionFile: string; assignmentId?: string } } | undefined>; + buildSessionListWithPassiveRlmSubagents( + active: ActiveSessionState[], + saved: Awaited>, + jobs: AgentCronJob[], + ): Promise>; + }; + const parent = await internals.createRuntime({ type: "create", sessionPath: fixture.parentSessionFile }); + + // All passive selectors traverse the one current B incarnation, not A. + const passive = await internals.listPassiveRlmSubagents(); + expect(passive.filter(({ entry }) => entry.childId === fixture.childId)).toEqual([ + expect.objectContaining({ + entry: expect.objectContaining({ sessionFile: childBSessionFile, assignmentId: assignmentB }), + }), + ]); + expect(await internals.findPassiveRlmSubagent(fixture.childId)).toEqual( + expect.objectContaining({ entry: expect.objectContaining({ sessionFile: childBSessionFile }) }), + ); + const listed = await internals.buildSessionListWithPassiveRlmSubagents( + [], + await SessionManager.listAll(undefined, join(tempDir, "sessions")), + [], + ); + expect( + listed.filter((entry) => entry.rlmChildId === fixture.childId).map((entry) => entry.sessionFile), + ).toEqual([childBSessionFile]); + + const host = internals.createSubagentRuntimeHost(parent); + // An explicit old assignment remains exact, and does not select B. + await host.deleteRlmSubagentRuntime(fixture.childId, undefined, a.assignmentId as string); + let afterDelete = readFileSync(registryPath, "utf8") + .trim() + .split(/\r?\n/) + .map((line) => JSON.parse(line) as { assignmentId?: string; status: string }); + expect(afterDelete.at(-1)).toMatchObject({ assignmentId: a.assignmentId, status: "deleted" }); + + // The ABI's assignment-less delete resolves current B, never the old A. + await host.deleteRlmSubagentRuntime(fixture.childId); + afterDelete = readFileSync(registryPath, "utf8") + .trim() + .split(/\r?\n/) + .map((line) => JSON.parse(line) as { assignmentId?: string; status: string }); + expect(afterDelete.at(-1)).toMatchObject({ assignmentId: assignmentB, status: "deleted" }); + } finally { + rmSync(tempDir, { recursive: true, force: true }); + } + }); + + it("keeps exact stale A callbacks fenced after durable B reuse", async () => { + const daemon = new AgentDaemon("/tmp/prime-agent-durable-reuse-stale-callback.sock", { + defaultSessionConfig: { agentDir: "/tmp", cwd: "/tmp" }, + createRuntime: vi.fn(), + }); + const parent = makeState("parent"); + const childB = makeState("child-b", parent.activeSessionId); + const assignmentA = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa"; + const assignmentB = "bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb"; + Object.assign(childB.runtime.metadata, { + kind: "subagent", + parentActiveSessionId: parent.activeSessionId, + rlmChildId: "reused-child", + assignmentId: assignmentB, + }); + const internals = daemon as unknown as { + sessions: Map; + closeSession: ReturnType; + readLatestRlmSubagentRegistry: ReturnType; + }; + internals.readLatestRlmSubagentRegistry = vi.fn(async () => []); + internals.sessions.set(parent.activeSessionId, parent); + internals.sessions.set(childB.activeSessionId, childB); + internals.closeSession = vi.fn(async () => undefined); + const host = ( + daemon as unknown as { createSubagentRuntimeHost(parent: ActiveSessionState): SubagentRuntimeHost } + ).createSubagentRuntimeHost(parent); + expect(host.completeRlmSubagentRuntime?.("reused-child", childB.runtime.session, assignmentA)).toBe(false); + await host.deleteRlmSubagentRuntime?.("reused-child", childB.runtime.session, assignmentA); + expect(internals.closeSession).not.toHaveBeenCalled(); + expect(internals.sessions.get(childB.activeSessionId)).toBe(childB); + }); + it("persists a real child completion for passive discovery, roster, and listing", async () => { const tempDir = mkdtempSync(join(tmpdir(), "prime-agent-daemon-real-completion-")); try { @@ -796,7 +1698,9 @@ describe("daemon mode helpers", () => { ); if (!childState?.runtime.session.sessionFile) throw new Error("Missing child state"); const host = internals.createSubagentRuntimeHost(parentState); - expect(host.completeRlmSubagentRuntime?.("child-1", childRuntime.session)).toBe(true); + expect( + host.completeRlmSubagentRuntime?.("child-1", childRuntime.session, childRuntime.metadata.assignmentId!), + ).toBe(true); await ( daemon as unknown as { closeSession(state: ActiveSessionState, reason: "shutdown"): Promise } ).closeSession(childState, "shutdown"); @@ -6125,6 +7029,7 @@ describe("daemon mode helpers", () => { expect(parentSession.releaseRlmChildSession).toHaveBeenCalledWith( fixture.childId, childState!.runtime.session, + "11111111-1111-4111-8111-111111111111", ); } finally { releaseHydration(); @@ -6206,6 +7111,8 @@ describe("daemon mode helpers", () => { expect(parentSession.registerRlmChildSession).toHaveBeenCalledWith( fixture.childId, attachedState.runtime.session, + undefined, + "11111111-1111-4111-8111-111111111111", ); await expect( @@ -6376,7 +7283,12 @@ describe("daemon mode helpers", () => { expect( (parentState.runtime.session as unknown as { registerRlmChildSession: ReturnType }) .registerRlmChildSession, - ).toHaveBeenCalledWith(fixture.grandchildId, grandchildState.runtime.session); + ).toHaveBeenCalledWith( + fixture.grandchildId, + grandchildState.runtime.session, + undefined, + "22222222-2222-4222-8222-222222222222", + ); } finally { releaseHydration(); rmSync(tempDir, { recursive: true, force: true }); @@ -6530,7 +7442,15 @@ describe("daemon mode helpers", () => { const internals = fixture.daemon as unknown as { sessions: Map; closingSessions: Map; reason: "shutdown"; killedEffects?: Promise }>; - passivatingSessions: Map>; + passivatingSessions: Map< + string, + { + promise: Promise; + state: ActiveSessionState; + generation: string; + operation: { operationId: string; generation: string }; + } + >; createRuntime(command: Extract): Promise; findPassiveRlmSubagent(id: string): Promise; hydratePassiveRlmSubagent(passive: unknown): Promise; @@ -6560,7 +7480,12 @@ describe("daemon mode helpers", () => { promise: passivation, reason: "shutdown", }); - internals.passivatingSessions.set(resolve(fixture.childSessionFile), passivation); + internals.passivatingSessions.set(resolve(fixture.childSessionFile), { + promise: passivation, + state: closingChild, + generation: closingChild.eventGeneration, + operation: { operationId: "passivation", generation: closingChild.eventGeneration }, + }); markRaceStarted(); return internals.waitForBoundSession(closingChild); } @@ -6790,6 +7715,7 @@ describe("daemon mode helpers", () => { fixture.childId, childState.runtime.session, unsubscribeForwarder, + "11111111-1111-4111-8111-111111111111", ); expect(unsubscribeForwarder).not.toHaveBeenCalled(); emitChildUpdate("recap after failed close"); @@ -6878,7 +7804,11 @@ describe("daemon mode helpers", () => { expect(await internals.passivateIdleChildren(90, Date.parse("2036-08-01T12:00:00Z"), 2)).toBe(1); expect(internals.sessions.has(firstChild.activeSessionId)).toBe(false); - expect(parentSession.releaseRlmChildSession).toHaveBeenCalledWith(fixture.childId, firstChild.runtime.session); + expect(parentSession.releaseRlmChildSession).toHaveBeenCalledWith( + fixture.childId, + firstChild.runtime.session, + "11111111-1111-4111-8111-111111111111", + ); expect(fixture.runtimeSessions[1]?.disposeAsync).toHaveBeenCalledOnce(); const listed = (await internals.handleCommand(makeClient("list-client", parentState.activeSessionId), { @@ -7829,6 +8759,9 @@ describe("daemon mode helpers", () => { } as never; const existingClose = { promise: failedClose, + state, + generation: state.eventGeneration, + operation: { operationId: "closing", generation: state.eventGeneration }, reason: "shutdown" as const, descendants: new Set(), }; @@ -7890,6 +8823,9 @@ describe("daemon mode helpers", () => { } const existingClose = { promise: Promise.resolve(), + state: parent, + generation: parent.eventGeneration, + operation: { operationId: "closing", generation: parent.eventGeneration }, reason: "shutdown" as "shutdown" | "killed", descendants: new Set([child]), }; @@ -8551,11 +9487,9 @@ describe("daemon mode helpers", () => { state.runtime = { ...state.runtime, session: { isStreaming: false, steer, followUp } } as never; const internals = daemon as unknown as { sessions: Map; - recordWorkerRecoveryState: ReturnType; handleCommand(client: DaemonSocketClient, command: DaemonCommand): Promise; }; internals.sessions.set(state.activeSessionId, state); - internals.recordWorkerRecoveryState = vi.fn(); await expect( internals.handleCommand(makeClient("client-1", state.activeSessionId), { @@ -8576,9 +9510,6 @@ describe("daemon mode helpers", () => { agentMessageId: undefined, resumeIfIdle: true, }); - if (type === "follow_up") { - expect(internals.recordWorkerRecoveryState).toHaveBeenCalledWith(state, "follow_up_queued", true); - } }, ); @@ -9689,6 +10620,7 @@ function makePersistedRlmDaemonFixture( `${JSON.stringify({ type: "rlm_subagent", childId: grandchildId, + assignmentId: "22222222-2222-4222-8222-222222222222", sessionName: "nested-worker", sessionDir: grandchildSessionDir, sessionFile: grandchildSessionFile, @@ -9708,6 +10640,7 @@ function makePersistedRlmDaemonFixture( `${JSON.stringify({ type: "rlm_subagent", childId, + assignmentId: "11111111-1111-4111-8111-111111111111", sessionName: "spawn-worker", sessionDir: childSessionDir, sessionFile: childSessionFile, diff --git a/packages/coding-agent/test/daemon-session-summarizer-lifecycle.test.ts b/packages/coding-agent/test/daemon-session-summarizer-lifecycle.test.ts index e7f7ae044..0cb0da11c 100644 --- a/packages/coding-agent/test/daemon-session-summarizer-lifecycle.test.ts +++ b/packages/coding-agent/test/daemon-session-summarizer-lifecycle.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, test, vi } from "vitest"; +import { AGENT_TASK_STATES, type AgentTaskState } from "../src/core/session-manager.js"; import type { ActiveSessionState } from "../src/modes/daemon/active-session-state.js"; import { DaemonSessionSummarizer } from "../src/modes/daemon/daemon-session-summarizer.js"; @@ -39,9 +40,9 @@ describe("DaemonSessionSummarizer lifecycle", () => { vi.useRealTimers(); }); - test("runs the model call after the settle debounce and records the verdict", async () => { + test.each(AGENT_TASK_STATES)("persists the %s idle verdict after the settle debounce", async (taskState) => { vi.useFakeTimers(); - const generate = vi.fn().mockResolvedValue({ summary: "Added the health endpoint", taskState: "completed" }); + const generate = vi.fn().mockResolvedValue({ summary: `Fixture ${taskState} verdict`, taskState }); const onStatusChanged = vi.fn(); const summarizer = new DaemonSessionSummarizer(() => [], onStatusChanged, generate); const state = makeState({ working: false }); @@ -53,7 +54,11 @@ describe("DaemonSessionSummarizer lifecycle", () => { await vi.advanceTimersByTimeAsync(600); expect(generate).toHaveBeenCalledOnce(); - expect(state.summaryState).toMatchObject({ summary: "Added the health endpoint", taskState: "completed" }); + const expected = { summary: `Fixture ${taskState} verdict`, taskState, basedOnMessageCount: 2 }; + expect(state.summaryState).toEqual(expected); + // The summarizer passes the closed codec through unchanged; SessionManager owns + // durable JSON encoding and the SDK compatibility test verifies the JSON row. + expect((state as unknown as { appendedStatuses: unknown[] }).appendedStatuses).toEqual([expected]); expect(onStatusChanged).toHaveBeenCalled(); }); @@ -166,9 +171,13 @@ describe("DaemonSessionSummarizer lifecycle", () => { expect((state as unknown as { appendedStatuses: unknown[] }).appendedStatuses).toHaveLength(1); }); - test("seeds a subagent's persisted recap into memory", () => { + test.each(AGENT_TASK_STATES)("seeds a subagent's persisted %s recap into memory", (taskState) => { const summarizer = new DaemonSessionSummarizer(() => [], undefined, vi.fn()); - const persisted = { summary: "Reviewing the diff", taskState: "needs_input", basedOnMessageCount: 3 }; + const persisted: { summary: string; taskState: AgentTaskState; basedOnMessageCount: number } = { + summary: `Persisted ${taskState} fixture`, + taskState, + basedOnMessageCount: 3, + }; const state = makeState({ kind: "subagent", persisted }); summarizer.seed(state); diff --git a/packages/coding-agent/test/daemon-supervisor-identity.test.ts b/packages/coding-agent/test/daemon-supervisor-identity.test.ts new file mode 100644 index 000000000..edd2c073f --- /dev/null +++ b/packages/coding-agent/test/daemon-supervisor-identity.test.ts @@ -0,0 +1,231 @@ +import { randomUUID } from "node:crypto"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +const processState = vi.hoisted(() => ({ + startId: "start-a" as string | undefined, + unreadableAfterSignal: false, +})); +const signals = vi.hoisted(() => [] as Array<{ pid: number; signal: NodeJS.Signals }>); + +vi.mock("../src/core/session-lease.js", async (importOriginal) => { + const actual = await importOriginal(); + return { ...actual, getProcessStartId: () => processState.startId }; +}); +vi.mock("../src/utils/child-process.js", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + signalProcessGroupOrProcess: (pid: number, signal: NodeJS.Signals) => { + signals.push({ pid, signal }); + if (processState.unreadableAfterSignal) processState.startId = undefined; + }, + }; +}); + +import { DaemonSupervisor } from "../src/modes/daemon/daemon-supervisor.js"; + +const generationA = "11111111-1111-4111-8111-111111111111"; +const generationB = "22222222-2222-4222-8222-222222222222"; + +afterEach(() => { + processState.startId = "start-a"; + processState.unreadableAfterSignal = false; + signals.splice(0); +}); + +function worker(generation = generationA, process = { pid: 4321, processStartId: "start-a" }) { + return { + descriptor: { workerId: "worker", generation, process }, + } as unknown as any; +} + +function supervisorFor(target: ReturnType) { + return Object.assign(Object.create(DaemonSupervisor.prototype), { + workers: new Map([["worker", target]]), + log: vi.fn(), + }) as DaemonSupervisor; +} + +function stopWorkerFixture() { + const target = { + descriptor: { + workerId: "worker", + generation: generationA, + lifecycle: "ready", + process: { pid: 4321, processStartId: "start-a" }, + }, + descriptorPath: "/descriptor/worker.json", + summaries: new Map(), + snapshotCache: new Map(), + transcriptCaches: new Map(), + snapshotGenerations: new Map(), + intentionalStop: false, + stopRevision: 0, + } as any; + const persistWorker = vi.fn(); + const deleteWorkerDescriptor = vi.fn(); + const supervisor = Object.assign(supervisorFor(target), { + persistWorker, + persistWorkerStopTombstone: vi.fn((resident: typeof target, archiveSession = false) => { + resident.intentionalStop = true; + resident.descriptor.stopRequestedAt ??= new Date().toISOString(); + resident.descriptor.archiveOnStop ||= archiveSession; + persistWorker(resident); + }), + deleteWorkerDescriptor, + syncAgentPeers: vi.fn(async () => undefined), + broadcastHeartbeatsChanged: vi.fn(), + }) as any; + return { supervisor, target, persistWorker, deleteWorkerDescriptor }; +} + +describe("daemon supervisor C01 process identity fencing", () => { + it("signals only a matching current PID/start identity", () => { + const target = worker(); + const supervisor = supervisorFor(target) as any; + expect(supervisor.signalTrackedWorker(target, generationA, "SIGTERM")).toBe(true); + expect(signals).toEqual([{ pid: 4321, signal: "SIGTERM" }]); + }); + + it("never signals a mismatched or unreadable start identity", () => { + const target = worker(); + const supervisor = supervisorFor(target) as any; + processState.startId = "reused"; + expect(supervisor.signalTrackedWorker(target, generationA, "SIGKILL")).toBe(false); + processState.startId = undefined; + expect(supervisor.signalTrackedWorker(target, generationA, "SIGTERM")).toBe(false); + expect(signals).toEqual([]); + }); + + it("rejects a stale generation or an unpublished replacement object", () => { + const old = worker(generationA); + const replacement = worker(generationB); + const supervisor = supervisorFor(replacement) as any; + expect(supervisor.signalTrackedWorker(old, generationA, "SIGTERM")).toBe(false); + expect(supervisor.signalTrackedWorker(replacement, generationA, "SIGTERM")).toBe(false); + expect(signals).toEqual([]); + }); + + it("keeps callback fencing enabled by default and limits the rollback gate to callbacks", () => { + const target = worker(); + const client = {} as any; + (target as any).client = client; + const supervisor = supervisorFor(target) as any; + expect(supervisor.acceptsWorkerCallback(target, generationA, client)).toBe(true); + (supervisor as any).c01IdentityFencingEnabled = false; + // =0 may accept an old callback, but signalTrackedWorker remains hard fenced. + expect(supervisor.acceptsWorkerCallback(target, generationB, client)).toBe(true); + processState.startId = "reused"; + expect(supervisor.signalTrackedWorker(target, generationA, "SIGTERM")).toBe(false); + expect(signals).toEqual([]); + }); + + it("treats passivated descriptors as physically processless", () => { + const target = { descriptor: { workerId: "worker", generation: generationA, lifecycle: "passivated" } } as any; + const supervisor = supervisorFor(target) as any; + expect(supervisor.signalTrackedWorker(target, generationA, "SIGTERM")).toBe(false); + expect(signals).toEqual([]); + }); + + it("refuses signals when a current descriptor has no process identity", () => { + const target = { descriptor: { workerId: "worker", generation: generationA, lifecycle: "recovering" } } as any; + const supervisor = supervisorFor(target) as any; + expect(supervisor.signalTrackedWorker(target, generationA, "SIGKILL")).toBe(false); + expect(signals).toEqual([]); + }); + + it("retains a stop tombstone when a live worker becomes transiently unreadable after signaling", async () => { + const { supervisor, target, deleteWorkerDescriptor } = stopWorkerFixture(); + const kill = vi.spyOn(process, "kill").mockImplementation(() => true); + try { + // The immediate pre-signal reread is exact; the next reread loses + // start-id visibility, modeling a transient process-start lookup failure. + processState.unreadableAfterSignal = true; + await expect(supervisor.stopWorker(target, true, true)).rejects.toThrow("process identity is unreadable"); + expect(target.descriptor.stopRequestedAt).toEqual(expect.any(String)); + expect(supervisor.workers.get("worker")).toBe(target); + expect(target.stopFinalized).toBeUndefined(); + expect(deleteWorkerDescriptor).not.toHaveBeenCalled(); + expect(signals).toEqual([{ pid: 4321, signal: "SIGTERM" }]); + + // A later exact observation may complete cleanup. This simulates the + // original PID having exited and then being observably recycled. + processState.unreadableAfterSignal = false; + processState.startId = "reused"; + await expect(supervisor.stopWorker(target, true, true)).resolves.toBeUndefined(); + expect(target.stopFinalized).toBe(true); + expect(deleteWorkerDescriptor).toHaveBeenCalledWith(target); + } finally { + kill.mockRestore(); + } + }); + + it("finalizes a verified recycled PID without signaling it", async () => { + const { supervisor, target, deleteWorkerDescriptor } = stopWorkerFixture(); + processState.startId = "reused"; + await expect(supervisor.stopWorker(target, true, true)).resolves.toBeUndefined(); + expect(signals).toEqual([]); + expect(target.stopFinalized).toBe(true); + expect(supervisor.workers.has("worker")).toBe(false); + expect(deleteWorkerDescriptor).toHaveBeenCalledWith(target); + }); + + it("uses a new generation for a later launch incarnation", () => { + const first = randomUUID(); + const second = randomUUID(); + expect(first).not.toBe(second); + }); + describe("recovery join finalizers", () => { + it("does not let deferred recovery A clear an installed B join", async () => { + vi.useFakeTimers(); + try { + const target = worker(); + const supervisor = supervisorFor(target) as any; + supervisor.isWorkerRecoveryCandidate = vi.fn(() => false); + supervisor.deferWorkerRecovery(target, new Error("A")); + const deferredA = target.deferredRecovery; + const deferredB = Promise.resolve(); + target.deferredRecovery = deferredB; + await vi.advanceTimersByTimeAsync(10_000); + await deferredA; + expect(target.deferredRecovery).toBe(deferredB); + } finally { + vi.useRealTimers(); + } + }); + + it("clears its settled deferred join after the same worker publishes a new generation", async () => { + vi.useFakeTimers(); + try { + const target = worker(); + const supervisor = supervisorFor(target) as any; + supervisor.isWorkerRecoveryCandidate = vi.fn(() => false); + supervisor.deferWorkerRecovery(target, new Error("A")); + const deferred = target.deferredRecovery; + target.descriptor.generation = "published-after-A"; + await vi.advanceTimersByTimeAsync(10_000); + await deferred; + expect(target.deferredRecovery).toBeUndefined(); + } finally { + vi.useRealTimers(); + } + }); + + it("does not let recovery A clear an installed B join", async () => { + vi.useFakeTimers(); + try { + const target = worker(); + const supervisor = supervisorFor(target) as any; + supervisor.isWorkerRecoveryCancelled = vi.fn(() => true); + const recoveryA = supervisor.recoverWorker(target); + const recoveryB = Promise.resolve(); + target.recovery = recoveryB; + await vi.advanceTimersByTimeAsync(10_000); + await recoveryA; + expect(target.recovery).toBe(recoveryB); + } finally { + vi.useRealTimers(); + } + }); + }); +}); diff --git a/packages/coding-agent/test/daemon-supervisor-lazy-subagents.test.ts b/packages/coding-agent/test/daemon-supervisor-lazy-subagents.test.ts index 497c7e017..19cb97742 100644 --- a/packages/coding-agent/test/daemon-supervisor-lazy-subagents.test.ts +++ b/packages/coding-agent/test/daemon-supervisor-lazy-subagents.test.ts @@ -16,6 +16,8 @@ import { DaemonSupervisor } from "../src/modes/daemon/daemon-supervisor.js"; interface SupervisorInternals { workers: Map; refreshWorkerSummaries(worker: WorkerFixture): Promise; + wakePassivatedWorker(worker: WorkerFixture): Promise; + forwardToWorker(worker: WorkerFixture, command: Record): Promise; syncAgentPeers(): Promise; findSummaryInWorker(worker: WorkerFixture, selector: string): SessionSummary | undefined; createOrReuseWorker( @@ -35,10 +37,12 @@ interface SupervisorInternals { interface WorkerFixture { descriptor: { workerId: string; + generation: string; lifecycle: "ready"; rootActiveSessionId: string; rootSessionId: string; - pid: number; + pid?: number; + process?: { pid: number; processStartId: string }; authenticationToken: string; ownerClientId?: string; createCommand: { config: { cwd: string } }; @@ -71,10 +75,15 @@ function summary(overrides: Partial & Pick { expect(supervisor.findSummaryInWorker(resident, "88889999cccc")).toBe(child); }); + it("fences stale assignment A lookup, hydration, and forwarding after B reuses its child selector", async () => { + const directory = mkdtempSync(join(tmpdir(), "prime-supervisor-child-assignment-fence-")); + tempDirs.push(directory); + const supervisor = new DaemonSupervisor(join(directory, "daemon.sock"), { + defaultSessionConfig: { agentDir: directory, cwd: directory }, + descriptorDir: join(directory, "workers"), + }) as unknown as SupervisorInternals; + const childSelector = "reused-child"; + const staleRoot = summary({ + id: "a-root-active", + activeSessionId: "a-root-active", + sessionId: "a-root-session", + }); + const staleChild = summary({ + id: "a-child-active", + activeSessionId: "a-child-active", + sessionId: "a-child-session", + sessionName: childSelector, + runtimeKind: "subagent", + rlmChildId: childSelector, + }); + const replacementRoot = summary({ + id: "b-root-active", + activeSessionId: "b-root-active", + sessionId: "b-root-session", + }); + const replacementChild = summary({ + id: "b-child-active", + activeSessionId: "b-child-active", + sessionId: "b-child-session", + sessionName: childSelector, + runtimeKind: "subagent", + rlmChildId: childSelector, + }); + const stale = worker("shared-worker", [staleRoot, staleChild], "assignment-A"); + const replacement = worker("shared-worker", [replacementRoot, replacementChild], "assignment-B"); + let releaseStaleList!: () => void; + const staleList = new Promise((resolve) => { + releaseStaleList = resolve; + }); + let markStaleLookupStarted!: () => void; + const staleLookupStarted = new Promise((resolve) => { + markStaleLookupStarted = resolve; + }); + let releaseStaleForward!: () => void; + const staleForwardGate = new Promise((resolve) => { + releaseStaleForward = resolve; + }); + let markStaleForwardStarted!: () => void; + const staleForwardStarted = new Promise((resolve) => { + markStaleForwardStarted = resolve; + }); + stale.client.request.mockImplementation(async (command: { type: string }) => { + if (command.type === "list") { + markStaleLookupStarted(); + await staleList; + return success(undefined, "list", { sessions: [staleRoot, staleChild] }); + } + return success(undefined, "prompt"); + }); + replacement.client.request.mockResolvedValue(success(undefined, "prompt")); + supervisor.workers.set("shared-worker", stale); + const wake = vi.spyOn(supervisor, "wakePassivatedWorker").mockImplementation(async (candidate) => { + if (candidate === stale) { + markStaleForwardStarted(); + await staleForwardGate; + } + }); + + // A has selected this public child selector while both its list/hydration + // callback and its explicit wake-to-forward callback are held. + expect(supervisor.findSummaryInWorker(stale, childSelector)).toBe(staleChild); + const staleHydration = supervisor.refreshWorkerSummaries(stale); + const staleForward = supervisor.forwardToWorker(stale, { + type: "prompt", + activeSessionId: staleChild.activeSessionId!, + message: "obsolete A", + }); + await Promise.all([staleLookupStarted, staleForwardStarted]); + + // B has the same public child selector but a distinct assignment/generation. + // Release both A continuations only after B is the registry resident. + supervisor.workers.set("shared-worker", replacement); + releaseStaleList(); + releaseStaleForward(); + await staleHydration; + await expect(staleForward).rejects.toThrow("superseded"); + + // Assignment A cannot join B, wake or forward through B, overwrite B's + // descriptor/child registry, or clear B's replacement child. + expect(wake).toHaveBeenCalledExactlyOnceWith(stale); + expect(stale.client.request).toHaveBeenCalledTimes(1); + expect(stale.client.request).toHaveBeenCalledWith({ type: "list" }, 5000); + expect(replacement.client.request).not.toHaveBeenCalled(); + expect(supervisor.workers.get("shared-worker")).toBe(replacement); + expect(replacement.descriptor).toMatchObject({ + generation: "assignment-B", + rootActiveSessionId: "shared-worker-root-active", + rootSessionId: "shared-worker-root-session", + }); + expect(replacement.summaries.get("b-child-active")).toBe(replacementChild); + expect(replacement.summaries.has("a-child-active")).toBe(false); + + // The matching B assignment is still allowed to wake and forward normally. + await expect( + supervisor.forwardToWorker(replacement, { + type: "prompt", + activeSessionId: replacementChild.activeSessionId!, + message: "continue B", + }), + ).resolves.toMatchObject({ success: true, command: "prompt" }); + expect(wake).toHaveBeenLastCalledWith(replacement); + expect(replacement.client.request).toHaveBeenCalledWith( + expect.objectContaining({ type: "prompt", activeSessionId: "b-child-active" }), + expect.any(Number), + ); + }); + + it("rejects a request result when its assignment is replaced while the request is pending", async () => { + const directory = mkdtempSync(join(tmpdir(), "prime-supervisor-request-assignment-fence-")); + tempDirs.push(directory); + const supervisor = new DaemonSupervisor(join(directory, "daemon.sock"), { + defaultSessionConfig: { agentDir: directory, cwd: directory }, + descriptorDir: join(directory, "workers"), + }) as unknown as SupervisorInternals; + const stale = worker("shared-worker", [], "assignment-A"); + const replacement = worker("shared-worker", [], "assignment-B"); + let releaseRequest!: () => void; + const requestGate = new Promise((resolve) => { + releaseRequest = resolve; + }); + let markRequestStarted!: () => void; + const requestStarted = new Promise((resolve) => { + markRequestStarted = resolve; + }); + stale.client.request.mockImplementation(async (command: { type: string }) => { + if (command.type === "prompt") { + markRequestStarted(); + await requestGate; + } + return success(undefined, "prompt"); + }); + replacement.client.request.mockResolvedValue(success(undefined, "prompt")); + const wake = vi.spyOn(supervisor, "wakePassivatedWorker").mockResolvedValue(); + supervisor.workers.set("shared-worker", stale); + + const staleForward = supervisor.forwardToWorker(stale, { + type: "prompt", + activeSessionId: "a-child-active", + message: "obsolete A", + }); + await requestStarted; + supervisor.workers.set("shared-worker", replacement); + releaseRequest(); + + await expect(staleForward).rejects.toThrow("superseded"); + expect(wake).toHaveBeenCalledExactlyOnceWith(stale); + expect(stale.client.request).toHaveBeenCalledWith( + expect.objectContaining({ type: "prompt", activeSessionId: "a-child-active" }), + expect.any(Number), + ); + expect(replacement.client.request).not.toHaveBeenCalled(); + expect(supervisor.workers.get("shared-worker")).toBe(replacement); + + await expect( + supervisor.forwardToWorker(replacement, { + type: "prompt", + activeSessionId: "b-child-active", + message: "continue B", + }), + ).resolves.toMatchObject({ success: true, command: "prompt" }); + }); + + it("rejects a rename when its refresh is replaced while list hydration is pending", async () => { + const directory = mkdtempSync(join(tmpdir(), "prime-supervisor-rename-assignment-fence-")); + tempDirs.push(directory); + const supervisor = new DaemonSupervisor(join(directory, "daemon.sock"), { + defaultSessionConfig: { agentDir: directory, cwd: directory }, + descriptorDir: join(directory, "workers"), + }) as unknown as SupervisorInternals; + const staleRoot = summary({ + id: "a-root-active", + activeSessionId: "a-root-active", + sessionId: "a-root-session", + sessionName: "A rename", + }); + const replacementRoot = summary({ + id: "b-root-active", + activeSessionId: "b-root-active", + sessionId: "b-root-session", + sessionName: "B original", + }); + const stale = Object.assign(worker("shared-worker", [staleRoot], "assignment-A"), { + descriptorPath: join(directory, "assignment-A.json"), + }); + stale.descriptor.rootActiveSessionId = "a-root-active"; + stale.descriptor.rootSessionId = "a-root-session"; + const replacement = Object.assign( + worker("shared-worker", [replacementRoot], "22222222-2222-4222-8222-222222222222"), + { + descriptorPath: join(directory, "assignment-B.json"), + }, + ); + replacement.descriptor.process = { pid: process.pid, processStartId: "test-process-start" }; + delete replacement.descriptor.pid; + replacement.descriptor.rootActiveSessionId = "b-root-active"; + replacement.descriptor.rootSessionId = "b-root-session"; + let releaseList!: () => void; + const listGate = new Promise((resolve) => { + releaseList = resolve; + }); + let markListStarted!: () => void; + const listStarted = new Promise((resolve) => { + markListStarted = resolve; + }); + stale.client.request.mockImplementation(async (command: { type: string }) => { + if (command.type === "rename") return success(undefined, "rename", staleRoot); + if (command.type === "list") { + markListStarted(); + await listGate; + return success(undefined, "list", { sessions: [staleRoot] }); + } + throw new Error(`Unexpected stale request: ${command.type}`); + }); + replacement.client.request.mockImplementation(async (command: { type: string }) => { + if (command.type === "rename") return success(undefined, "rename", replacementRoot); + if (command.type === "list") return success(undefined, "list", { sessions: [replacementRoot] }); + throw new Error(`Unexpected replacement request: ${command.type}`); + }); + vi.spyOn(supervisor, "wakePassivatedWorker").mockResolvedValue(); + supervisor.workers.set("shared-worker", stale); + + const staleRename = supervisor.forwardToWorker(stale, { + type: "rename", + activeSessionId: "a-root-active", + name: "obsolete A rename", + }); + await listStarted; + supervisor.workers.set("shared-worker", replacement); + releaseList(); + + await expect(staleRename).rejects.toThrow("superseded"); + expect(stale.client.request).toHaveBeenCalledWith( + expect.objectContaining({ type: "rename" }), + expect.any(Number), + ); + expect(stale.client.request).toHaveBeenCalledWith({ type: "list" }, 5000); + expect(replacement.client.request).not.toHaveBeenCalled(); + expect(replacement.summaries.get("b-root-active")).toBe(replacementRoot); + expect(replacement.summaries.has("a-root-active")).toBe(false); + expect(replacement.descriptor).toMatchObject({ + generation: "22222222-2222-4222-8222-222222222222", + rootSessionId: "b-root-session", + }); + + await expect( + supervisor.forwardToWorker(replacement, { + type: "rename", + activeSessionId: "b-root-active", + name: "B rename", + }), + ).resolves.toMatchObject({ + success: true, + command: "rename", + data: expect.objectContaining({ id: "b-root-active" }), + }); + }); + it("rejects an explicit root name that collides with a saved root", async () => { const directory = mkdtempSync(join(tmpdir(), "prime-supervisor-root-name-")); tempDirs.push(directory); diff --git a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts b/packages/coding-agent/test/daemon-supervisor-monitor.test.ts index 1814acbf7..3fcf282d3 100644 --- a/packages/coding-agent/test/daemon-supervisor-monitor.test.ts +++ b/packages/coding-agent/test/daemon-supervisor-monitor.test.ts @@ -1,4 +1,5 @@ import type { ChildProcess, SpawnOptions } from "node:child_process"; +import { randomUUID } from "node:crypto"; import { EventEmitter } from "node:events"; import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import type { Socket } from "node:net"; @@ -6,6 +7,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import type { AgentMessage } from "@earendil-works/pi-agent-core"; import { afterEach, describe, expect, it, vi } from "vitest"; +import { getProcessStartId } from "../src/core/session-lease.js"; import type { DaemonSocketClient } from "../src/modes/daemon/active-session-state.js"; import { CommandRecoveryJournal } from "../src/modes/daemon/command-recovery-journal.js"; import { DaemonCatalogClient } from "../src/modes/daemon/daemon-catalog-process.js"; @@ -180,7 +182,31 @@ async function waitForFile(path: string): Promise { } } -function createExistingLaunchWorker(root: string, descriptorDir: string) { +type ExistingLaunchWorker = { + descriptor: { + workerId: string; + generation?: string; + lifecycle: string; + process?: { pid: number; processStartId: string }; + pid?: number; + lastError?: string; + [key: string]: unknown; + }; + descriptorPath: string; + summaries: Map; + snapshotCache: Map; + snapshotGenerations: Map>; + transcriptCaches: Map; + incomingTranscriptActiveSessionIds: Set; + duplicateIncomingTranscriptChunkIndexes: Map; + snapshotTransferFrames: Map; + snapshotLoads: Map>; + intentionalStop: boolean; + stopRevision: number; + [key: string]: unknown; +}; + +function createExistingLaunchWorker(root: string, descriptorDir: string): ExistingLaunchWorker { const workerId = "existing-worker"; const now = new Date().toISOString(); return { @@ -204,6 +230,7 @@ function createExistingLaunchWorker(root: string, descriptorDir: string) { descriptorPath: join(descriptorDir, `${workerId}.json`), summaries: new Map(), snapshotCache: new Map(), + snapshotGenerations: new Map>(), transcriptCaches: new Map(), incomingTranscriptActiveSessionIds: new Set(), duplicateIncomingTranscriptChunkIndexes: new Map(), @@ -520,7 +547,7 @@ describe("daemon worker supervisor monitoring", () => { { name: "descriptor persistence", persistFailure: true, error: new Error("descriptor persistence failed") }, ] as const)("keeps unidentifiable workers gated after $name fails", async (scenario) => { workerLaunchTestState.capture = true; - workerLaunchTestState.forceMissingProcessStartId = true; + workerLaunchTestState.forceMissingProcessStartId = false; workerLaunchTestState.fixtureMode = "rollback-gate"; const root = mkdtempSync(join(tmpdir(), "prime-supervisor-launch-gate-test-")); const gateMarkerPath = join(root, "committed-gate"); @@ -577,13 +604,44 @@ describe("daemon worker supervisor monitoring", () => { expect(workers.size).toBe(0); }); + it("fails launch before descriptor publication when process start identity is unreadable", async () => { + workerLaunchTestState.capture = true; + workerLaunchTestState.forceMissingProcessStartId = true; + workerLaunchTestState.fixtureMode = "rollback-gate"; + const root = mkdtempSync(join(tmpdir(), "prime-supervisor-missing-start-id-test-")); + const descriptorDir = join(root, "descriptors"); + mkdirSync(descriptorDir, { recursive: true }); + supervisorRegistryDirs.add(root); + const workers = new Map(); + const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), { + ...createSupervisorSnapshotState(), + defaultSessionConfig: { cwd: root, agentDir: root }, + descriptorDir, + socketPath: join(root, "supervisor.sock"), + workers, + assertRecoveryAllowed: vi.fn(async () => undefined), + log: vi.fn(), + }) as { + launchWorker(command: { type: "create"; config: { cwd: string; agentDir: string } }): Promise; + }; + + await expect(supervisor.launchWorker({ type: "create", config: { cwd: root, agentDir: root } })).rejects.toThrow( + "without a process start identity", + ); + expect(workers.size).toBe(0); + expect(readdirSync(descriptorDir).filter((name) => name.endsWith(".json"))).toEqual([]); + expect(workerLaunchTestState.spawned).toHaveLength(1); + const child = workerLaunchTestState.spawned[0]!.child; + expect(child.exitCode !== null || child.signalCode !== null).toBe(true); + }); + it("rolls back promptly when the child closes its startup gate before commit", async () => { const root = mkdtempSync(join(tmpdir(), "prime-supervisor-closed-gate-test-")); const descriptorDir = join(root, "descriptors"); mkdirSync(descriptorDir, { recursive: true }); supervisorRegistryDirs.add(root); workerLaunchTestState.capture = true; - workerLaunchTestState.forceMissingProcessStartId = true; + workerLaunchTestState.forceMissingProcessStartId = false; workerLaunchTestState.fixtureMode = "close-gate"; let assertionCount = 0; const workers = new Map(); @@ -640,7 +698,7 @@ describe("daemon worker supervisor monitoring", () => { mkdirSync(descriptorDir, { recursive: true }); supervisorRegistryDirs.add(root); workerLaunchTestState.capture = true; - workerLaunchTestState.forceMissingProcessStartId = true; + workerLaunchTestState.forceMissingProcessStartId = false; workerLaunchTestState.fixtureMode = "successful-gate"; workerLaunchTestState.gateMarkerPath = markerPath; const workers = new Map(); @@ -680,7 +738,7 @@ describe("daemon worker supervisor monitoring", () => { const worker = await supervisor.launchWorker({ type: "create", config: { cwd: root, agentDir: root } }); - expect(readFileSync(markerPath, "utf8")).toBe("start\n"); + expect(readFileSync(markerPath, "utf8")).toMatch(/^start\n[0-9a-f-]+\n$/); expect(connectWorker).toHaveBeenCalledOnce(); expect(worker.descriptor.lifecycle).toBe("ready"); expect(workers.size).toBe(1); @@ -694,6 +752,240 @@ describe("daemon worker supervisor monitoring", () => { await closed; }); + it("tags its own published replacement generation when real launch cleanup succeeds", async () => { + const root = mkdtempSync(join(tmpdir(), "prime-supervisor-real-launch-failure-test-")); + const descriptorDir = join(root, "descriptors"); + const markerPath = join(root, "startup-marker"); + mkdirSync(descriptorDir, { recursive: true }); + supervisorRegistryDirs.add(root); + workerLaunchTestState.capture = true; + workerLaunchTestState.forceMissingProcessStartId = false; + workerLaunchTestState.fixtureMode = "successful-gate"; + workerLaunchTestState.gateMarkerPath = markerPath; + const existing = createExistingLaunchWorker(root, descriptorDir); + const workers = new Map([[existing.descriptor.workerId, existing]]); + const connectFailure = new Error("fresh launch handshake failed"); + const connectWorker = vi.fn(async () => { + await waitForFile(markerPath); + throw connectFailure; + }); + const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), { + ...createSupervisorSnapshotState(), + defaultSessionConfig: { cwd: root, agentDir: root }, + descriptorDir, + socketPath: join(root, "supervisor.sock"), + workers, + shuttingDown: false, + assertRecoveryAllowed: vi.fn(async () => undefined), + connectWorker, + syncAgentPeers: vi.fn(async () => undefined), + broadcastHeartbeatsChanged: vi.fn(), + log: vi.fn(), + }) as { + launchWorker( + command: { type: "create"; config: { cwd: string; agentDir: string } }, + existing: ExistingLaunchWorker, + ): Promise; + workerLaunchFailureAttempt( + error: unknown, + worker: ExistingLaunchWorker, + ): { generation: string; cleanupVerified: boolean } | undefined; + }; + + const failure = await supervisor + .launchWorker({ type: "create", config: { cwd: root, agentDir: root } }, existing) + .then( + () => undefined, + (error: unknown) => error, + ); + + expect(failure).toBe(connectFailure); + expect(connectWorker).toHaveBeenCalledOnce(); + expect(supervisor.workerLaunchFailureAttempt(failure, existing)).toMatchObject({ + generation: existing.descriptor.generation, + cleanupVerified: true, + }); + expect(existing.descriptor).toMatchObject({ lifecycle: "recovering" }); + expect(existing.descriptor).not.toHaveProperty("process"); + expect(existing.descriptor).not.toHaveProperty("pid"); + expect(existing.descriptor).not.toHaveProperty("processStartId"); + expect(workers.get(existing.descriptor.workerId)).toBe(existing); + }); + + it("retries after real launchWorker catch tags a verified cleanup", async () => { + vi.useFakeTimers(); + const root = mkdtempSync(join(tmpdir(), "prime-supervisor-real-launch-retry-test-")); + const descriptorDir = join(root, "descriptors"); + mkdirSync(descriptorDir, { recursive: true }); + supervisorRegistryDirs.add(root); + workerLaunchTestState.capture = true; + workerLaunchTestState.forceMissingProcessStartId = false; + workerLaunchTestState.fixtureMode = "successful-gate"; + const worker = createExistingLaunchWorker(root, descriptorDir); + worker.descriptor.generation = randomUUID(); + delete worker.descriptor.pid; + worker.descriptor.lifecycle = "passivated"; + const workers = new Map([[worker.descriptor.workerId, worker]]); + const firstFailure = new Error("first launch handshake failed"); + let connectionCount = 0; + const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), { + ...createSupervisorSnapshotState(), + defaultSessionConfig: { cwd: root, agentDir: root }, + descriptorDir, + socketPath: join(root, "supervisor.sock"), + workers, + shuttingDown: false, + assertRecoveryAllowed: vi.fn(async () => undefined), + connectWorker: vi.fn(async (launched: ExistingLaunchWorker) => { + connectionCount++; + if (connectionCount === 1) throw firstFailure; + return { + request: vi.fn(async () => ({ + success: true, + data: { + id: launched.descriptor.rootActiveSessionId, + activeSessionId: launched.descriptor.rootActiveSessionId, + sessionId: "session-real-launch-retry", + cwd: root, + }, + })), + }; + }), + subscribeWorker: vi.fn(async () => undefined), + refreshWorkerSummaries: vi.fn(async () => undefined), + recoverUncertainWorkerOperations: vi.fn(async () => undefined), + syncAgentPeers: vi.fn(async () => undefined), + broadcastHeartbeatsChanged: vi.fn(), + log: vi.fn(), + }) as { + recoverWorker(worker: ExistingLaunchWorker): Promise; + isWorkerRecoveryEligible(worker: ExistingLaunchWorker): boolean; + }; + + const recovery = supervisor.recoverWorker(worker); + await vi.runAllTimersAsync(); + await recovery; + + expect(connectionCount).toBe(2); + expect(workerLaunchTestState.spawned).toHaveLength(2); + expect(worker.descriptor.lifecycle).toBe("ready"); + expect(worker.descriptor.process).toMatchObject({ pid: workerLaunchTestState.spawned[1]!.child.pid }); + // The first failed launch adopted its generation, then the retry published + // another one. Its completed join handle must still be released. + expect(worker.recovery).toBeUndefined(); + expect(supervisor.isWorkerRecoveryEligible(worker)).toBe(true); + + // A later safe recovery is consequently admitted rather than being stranded + // behind the resolved first recovery promise. + worker.descriptor.lifecycle = "recovering"; + const subsequentRecovery = supervisor.recoverWorker(worker); + await vi.runAllTimersAsync(); + await subsequentRecovery; + expect(connectionCount).toBe(3); + expect(workerLaunchTestState.spawned).toHaveLength(2); + expect(worker.descriptor.lifecycle).toBe("ready"); + expect(worker.recovery).toBeUndefined(); + }); + + it("does not let a settled recovery erase a newer recovery join handle", async () => { + vi.useFakeTimers(); + const root = mkdtempSync(join(tmpdir(), "prime-supervisor-recovery-join-test-")); + const descriptorDir = join(root, "descriptors"); + mkdirSync(descriptorDir, { recursive: true }); + supervisorRegistryDirs.add(root); + const worker = createExistingLaunchWorker(root, descriptorDir); + worker.descriptor.generation = randomUUID(); + delete worker.descriptor.pid; + worker.descriptor.lifecycle = "recovering"; + const workers = new Map([[worker.descriptor.workerId, worker]]); + const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), { + ...createSupervisorSnapshotState(), + workers, + shuttingDown: false, + assertRecoveryAllowed: vi.fn(async () => undefined), + }) as { recoverWorker(worker: ExistingLaunchWorker): Promise }; + + const oldRecovery = supervisor.recoverWorker(worker); + const newerRecovery = Promise.resolve(); + worker.recovery = newerRecovery; + // Finish the old cycle after it has been superseded. Exact promise equality + // is required so its finalizer cannot clear the newer join handle. + worker.intentionalStop = true; + await vi.runAllTimersAsync(); + await oldRecovery; + expect(worker.recovery).toBe(newerRecovery); + }); + + it("keeps a failed fresh launch process and ends recovery when cleanup cannot be verified", async () => { + const root = mkdtempSync(join(tmpdir(), "prime-supervisor-failed-launch-cleanup-test-")); + const descriptorDir = join(root, "descriptors"); + const markerPath = join(root, "startup-marker"); + mkdirSync(descriptorDir, { recursive: true }); + supervisorRegistryDirs.add(root); + workerLaunchTestState.capture = true; + workerLaunchTestState.forceMissingProcessStartId = false; + workerLaunchTestState.fixtureMode = "successful-gate"; + workerLaunchTestState.gateMarkerPath = markerPath; + const worker = createExistingLaunchWorker(root, descriptorDir); + worker.descriptor.generation = randomUUID(); + delete worker.descriptor.pid; + worker.descriptor.lifecycle = "passivated"; + const workers = new Map([[worker.descriptor.workerId, worker]]); + const connectFailure = new Error("fresh launch handshake failed"); + const cleanupFailure = new Error("could not verify child exit"); + const persistProcesslessRecoveryFailure = vi.fn(); + const signalTrackedWorker = vi.fn(); + const stopWorker = vi.fn(async () => { + throw cleanupFailure; + }); + const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), { + ...createSupervisorSnapshotState(), + defaultSessionConfig: { cwd: root, agentDir: root }, + descriptorDir, + socketPath: join(root, "supervisor.sock"), + workers, + shuttingDown: false, + assertRecoveryAllowed: vi.fn(async () => undefined), + connectWorker: vi.fn(async () => { + await waitForFile(markerPath); + throw connectFailure; + }), + stopWorker, + persistWorker: vi.fn(), + persistProcesslessRecoveryFailure, + signalTrackedWorker, + recoverUncertainWorkerOperations: vi.fn(async () => undefined), + log: vi.fn(), + }) as { + recoverWorker(worker: ExistingLaunchWorker): Promise; + workerLaunchFailureAttempt( + error: unknown, + worker: ExistingLaunchWorker, + ): { cleanupVerified: boolean; generation: string } | undefined; + }; + + await supervisor.recoverWorker(worker); + + // Adopting the failed generation must release even the no-retry path. + expect(worker.recovery).toBeUndefined(); + expect(stopWorker).toHaveBeenCalledOnce(); + expect(workerLaunchTestState.spawned).toHaveLength(1); + expect(worker.descriptor.lifecycle).toBe("failed"); + expect(worker.descriptor.process).toEqual({ + pid: workerLaunchTestState.spawned[0]!.child.pid, + processStartId: expect.any(String), + }); + expect(worker.descriptor.lastError).toContain(cleanupFailure.message); + expect(persistProcesslessRecoveryFailure).not.toHaveBeenCalled(); + expect(signalTrackedWorker).not.toHaveBeenCalled(); + expect(supervisor.workerLaunchFailureAttempt(connectFailure, worker)).toMatchObject({ + cleanupVerified: false, + generation: worker.descriptor.generation, + }); + // No second launch or recovery cleanup may signal/replace the retained process. + expect(workerLaunchTestState.spawned).toHaveLength(1); + }); + it("rolls back a published worker when shutdown admission and rollback persistence fail", async () => { const root = mkdtempSync(join(tmpdir(), "prime-supervisor-cancelled-launch-test-")); const descriptorDir = join(root, "descriptors"); @@ -701,7 +993,7 @@ describe("daemon worker supervisor monitoring", () => { mkdirSync(descriptorDir, { recursive: true }); supervisorRegistryDirs.add(root); workerLaunchTestState.capture = true; - workerLaunchTestState.forceMissingProcessStartId = true; + workerLaunchTestState.forceMissingProcessStartId = false; workerLaunchTestState.fixtureMode = "successful-gate"; workerLaunchTestState.gateMarkerPath = markerPath; const cancellation = recoveryDeniedError("supervisor_recovery_cancelled"); @@ -742,7 +1034,7 @@ describe("daemon worker supervisor monitoring", () => { cancellation, ); - expect(readFileSync(markerPath, "utf8")).toBe("start\n"); + expect(readFileSync(markerPath, "utf8")).toMatch(/^start\n[0-9a-f-]+\n$/); expect(connectWorker).toHaveBeenCalledOnce(); expect(persistenceCalls).toBe(2); expect(workers.size).toBe(0); @@ -758,7 +1050,7 @@ describe("daemon worker supervisor monitoring", () => { mkdirSync(descriptorDir, { recursive: true }); supervisorRegistryDirs.add(root); workerLaunchTestState.capture = true; - workerLaunchTestState.forceMissingProcessStartId = true; + workerLaunchTestState.forceMissingProcessStartId = false; workerLaunchTestState.fixtureMode = "successful-gate"; workerLaunchTestState.gateMarkerPath = markerPath; const cancellation = recoveryDeniedError("supervisor_recovery_cancelled"); @@ -822,7 +1114,7 @@ describe("daemon worker supervisor monitoring", () => { mkdirSync(descriptorDir, { recursive: true }); supervisorRegistryDirs.add(root); workerLaunchTestState.capture = true; - workerLaunchTestState.forceMissingProcessStartId = true; + workerLaunchTestState.forceMissingProcessStartId = false; workerLaunchTestState.fixtureMode = "successful-gate"; workerLaunchTestState.gateMarkerPath = markerPath; const cancellation = recoveryDeniedError("supervisor_recovery_cancelled"); @@ -955,6 +1247,51 @@ describe("daemon worker supervisor monitoring", () => { } }); + it("continues global shutdown after skipping quarantined evidence", async () => { + const stopWorker = vi.fn(async (worker: { descriptor: { workerId: string } }) => { + if (worker.descriptor.workerId === "failing") { + throw new Error("ordinary worker cleanup failed"); + } + }); + const log = vi.fn(); + const exit = vi.spyOn(process, "exit").mockImplementation(((code?: string | number | null) => { + throw new Error(`exit ${code}`); + }) as typeof process.exit); + const quarantined = { descriptor: { workerId: "quarantined" }, quarantined: true }; + const failing = { descriptor: { workerId: "failing" } }; + const healthy = { descriptor: { workerId: "healthy" } }; + const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), { + shuttingDown: false, + signalCleanupHandlers: [], + workers: new Map([ + ["quarantined", quarantined], + ["failing", failing], + ["healthy", healthy], + ]), + clients: new Set(), + catalog: { stop: vi.fn(async () => undefined) }, + stopWorker, + hasPersistedWorkerDescriptors: vi.fn(() => true), + clearIdleEvictionTimer: vi.fn(), + cleanupSocket: vi.fn(), + snapshotCacheRoot: "\0", + log, + }) as any; + + try { + await expect(supervisor.shutdown(24, true)).rejects.toThrow("exit 24"); + expect(stopWorker).toHaveBeenCalledTimes(2); + expect(stopWorker).toHaveBeenCalledWith(failing, true, false, true); + expect(stopWorker).toHaveBeenCalledWith(healthy, true, false, true); + expect(supervisor.workers.get("quarantined")).toBe(quarantined); + expect(log).toHaveBeenCalledWith(expect.stringContaining("quarantined session worker quarantined")); + expect(log).toHaveBeenCalledWith(expect.stringContaining("session worker failing")); + expect(exit).toHaveBeenCalledWith(24); + } finally { + exit.mockRestore(); + } + }); + it("does not poll a healthy supervisor after the startup check", async () => { vi.useFakeTimers(); let resolveProbe: () => void = () => undefined; @@ -1074,6 +1411,53 @@ describe("daemon worker supervisor monitoring", () => { expect(recoverWorker).toHaveBeenCalledOnce(); }); + it("recovers after a second disconnect callback clears the active client", async () => { + const client = {}; + const worker: DeferredRecoveryWorker = { + descriptor: { + workerId: "worker-second-close", + pid: process.pid, + rootActiveSessionId: "active-second-close", + lifecycle: "ready", + }, + client, + snapshotCache: new Map(), + incomingTranscriptActiveSessionIds: new Set(), + transcriptCaches: new Map(), + duplicateIncomingTranscriptChunkIndexes: new Map(), + snapshotTransferFrames: new Map(), + intentionalStop: false, + stopRevision: 0, + }; + let releaseAdmission!: () => void; + const admission = new Promise((resolve) => { + releaseAdmission = resolve; + }); + const assertRecoveryAllowed = vi.fn(async () => admission); + const recoverWorker = vi.fn(async () => { + worker.recovery = Promise.resolve(); + }); + const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), { + ...createSupervisorSnapshotState(), + workers: new Map([[worker.descriptor.workerId, worker]]), + shuttingDown: false, + assertRecoveryAllowed, + persistWorker: vi.fn(), + syncAgentPeers: vi.fn(async () => undefined), + recoverWorker, + }) as DeferredRecoveryHarness; + + const first = supervisor.handleWorkerClose(worker, client, new Error("first disconnect")); + await Promise.resolve(); + const second = supervisor.handleWorkerClose(worker, client, new Error("second disconnect")); + releaseAdmission(); + await Promise.all([first, second]); + + expect(worker.client).toBeUndefined(); + expect(worker.descriptor.lifecycle).toBe("recovering"); + expect(recoverWorker).toHaveBeenCalledOnce(); + }); + it("resumes deferred recovery after a concurrent recovery is denied", async () => { vi.useFakeTimers(); const client = {}; @@ -1264,7 +1648,6 @@ describe("daemon worker supervisor monitoring", () => { }; type RetryHarness = { workers: Map; - persistWorker: ReturnType; recoverWorker: ReturnType; handleCommand( client: DaemonSocketClient, @@ -1284,19 +1667,11 @@ describe("daemon worker supervisor monitoring", () => { intentionalStop: true, summaries: new Map(), }; - const persistWorker = vi.fn(() => { - expect(worker.intentionalStop).toBe(false); - expect(worker.descriptor.stopRequestedAt).toBeUndefined(); - expect(worker.descriptor.archiveOnStop).toBeUndefined(); - expect(worker.descriptor.lifecycle).toBe("recovering"); - expect(worker.descriptor.consecutiveFailures).toBe(0); - }); const recoverWorker = vi.fn(async () => { worker.descriptor.lifecycle = "ready"; }); const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), { workers: new Map([[worker.descriptor.workerId, worker]]), - persistWorker, recoverWorker, assertWorkerAccessibleToClient: vi.fn(), }) as RetryHarness; @@ -1306,9 +1681,12 @@ describe("daemon worker supervisor monitoring", () => { activeSessionId: worker.descriptor.rootSessionId, }); - expect(persistWorker).toHaveBeenCalledOnce(); + expect(worker.intentionalStop).toBe(false); + expect(worker.descriptor.stopRequestedAt).toBeUndefined(); + expect(worker.descriptor.archiveOnStop).toBeUndefined(); + expect(worker.descriptor.lifecycle).toBe("ready"); + expect(worker.descriptor.consecutiveFailures).toBe(0); expect(recoverWorker).toHaveBeenCalledWith(worker); - expect(persistWorker.mock.invocationCallOrder[0]).toBeLessThan(recoverWorker.mock.invocationCallOrder[0]!); }); it("cancels an in-flight recovery after an intentional stop tombstone", async () => { @@ -1351,8 +1729,8 @@ describe("daemon worker supervisor monitoring", () => { type RecoveryWorker = { descriptor: { workerId: string; - pid: number; - processStartId: string; + process: { pid: number; processStartId: string }; + generation: string; rootActiveSessionId: string; createCommand: { type: "create" }; }; @@ -1372,8 +1750,8 @@ describe("daemon worker supervisor monitoring", () => { const worker: RecoveryWorker = { descriptor: { workerId: "worker-reused-pid", - pid: process.pid, - processStartId: "different-process-start", + process: { pid: process.pid, processStartId: "different-process-start" }, + generation: randomUUID(), rootActiveSessionId: "active-1", createCommand: { type: "create" }, }, @@ -1398,6 +1776,155 @@ describe("daemon worker supervisor monitoring", () => { expect(supervisor.launchWorker).toHaveBeenCalledWith(worker.descriptor.createCommand, worker); }); + it("retries an existing worker when its freshly-published launch generation fails", async () => { + vi.useFakeTimers(); + type RecoveryWorker = { + descriptor: { + workerId: string; + generation: string; + rootActiveSessionId: string; + createCommand: { type: "create" }; + lifecycle: "recovering" | "ready" | "passivated"; + consecutiveFailures: number; + lastFailureAt?: string; + lastError?: string; + process?: { pid: number; processStartId: string }; + }; + intentionalStop: boolean; + stopRevision: number; + recovery?: Promise; + client?: { close(): void }; + }; + type RecoveryHarness = { + workers: Map; + shuttingDown: boolean; + persistWorker: ReturnType; + recoverUncertainWorkerOperations: ReturnType; + launchWorker: ReturnType; + assertRecoveryAllowed: ReturnType; + recordWorkerLaunchFailure(error: unknown, worker: RecoveryWorker, generation: string): void; + markWorkerLaunchFailureCleanupVerified(error: unknown, worker: RecoveryWorker, generation: string): void; + recoverWorker(worker: RecoveryWorker): Promise; + }; + const worker: RecoveryWorker = { + descriptor: { + workerId: "worker-fresh-launch-failure", + generation: "old-generation", + rootActiveSessionId: "active-1", + createCommand: { type: "create" }, + lifecycle: "recovering", + consecutiveFailures: 0, + }, + intentionalStop: false, + stopRevision: 0, + }; + let supervisor!: RecoveryHarness; + const launchWorker = vi.fn(async () => { + if (launchWorker.mock.calls.length === 1) { + worker.descriptor.generation = "failed-fresh-generation"; + const error = new Error("fresh launch handshake failed"); + supervisor.recordWorkerLaunchFailure(error, worker, worker.descriptor.generation); + // The mocked launch represents the production path after cleanup has + // verified the freshly-published process stopped. + supervisor.markWorkerLaunchFailureCleanupVerified(error, worker, worker.descriptor.generation); + throw error; + } + worker.descriptor.generation = "retried-generation"; + worker.descriptor.lifecycle = "ready"; + return worker; + }); + const persistWorker = vi.fn(); + supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), { + workers: new Map([[worker.descriptor.workerId, worker]]), + shuttingDown: false, + persistWorker, + recoverUncertainWorkerOperations: vi.fn(async () => {}), + launchWorker, + assertRecoveryAllowed: vi.fn(async () => {}), + log: vi.fn(), + }) as RecoveryHarness; + + const recovery = supervisor.recoverWorker(worker); + await vi.runAllTimersAsync(); + await recovery; + + expect(launchWorker).toHaveBeenCalledTimes(2); + expect(worker.descriptor).toMatchObject({ generation: "retried-generation", lifecycle: "ready" }); + // Verified cleanup bypasses processless failure persistence and retries directly. + expect(persistWorker).not.toHaveBeenCalled(); + }); + + it("fences a fresh launch failure when a concurrent replacement publishes another generation", async () => { + vi.useFakeTimers(); + type RecoveryWorker = { + descriptor: { + workerId: string; + generation: string; + rootActiveSessionId: string; + createCommand: { type: "create" }; + lifecycle: "recovering" | "ready"; + consecutiveFailures: number; + }; + intentionalStop: boolean; + stopRevision: number; + recovery?: Promise; + }; + type RecoveryHarness = { + workers: Map; + shuttingDown: boolean; + persistWorker: ReturnType; + recoverUncertainWorkerOperations: ReturnType; + launchWorker: ReturnType; + assertRecoveryAllowed: ReturnType; + recordWorkerLaunchFailure(error: unknown, worker: RecoveryWorker, generation: string): void; + markWorkerLaunchFailureCleanupVerified(error: unknown, worker: RecoveryWorker, generation: string): void; + recoverWorker(worker: RecoveryWorker): Promise; + }; + const worker: RecoveryWorker = { + descriptor: { + workerId: "worker-replaced-during-launch-failure", + generation: "old-generation", + rootActiveSessionId: "active-1", + createCommand: { type: "create" }, + lifecycle: "recovering", + consecutiveFailures: 0, + }, + intentionalStop: false, + stopRevision: 0, + }; + const persistWorker = vi.fn(); + const supervisor = Object.assign(Object.create(DaemonSupervisor.prototype), { + workers: new Map([[worker.descriptor.workerId, worker]]), + shuttingDown: false, + persistWorker, + recoverUncertainWorkerOperations: vi.fn(async () => {}), + assertRecoveryAllowed: vi.fn(async () => {}), + log: vi.fn(), + }) as RecoveryHarness; + supervisor.launchWorker = vi.fn(async () => { + worker.descriptor.generation = "failed-fresh-generation"; + const error = new Error("fresh launch handshake failed"); + supervisor.recordWorkerLaunchFailure(error, worker, worker.descriptor.generation); + // A newer supervisor attempt won after the failure was tagged. Its state + // is not evidence that the old recovery can persist or retry against. + worker.descriptor.generation = "concurrent-replacement-generation"; + worker.descriptor.lifecycle = "ready"; + throw error; + }); + + const recovery = supervisor.recoverWorker(worker); + await vi.runAllTimersAsync(); + await recovery; + + expect(supervisor.launchWorker).toHaveBeenCalledOnce(); + expect(worker.descriptor).toMatchObject({ + generation: "concurrent-replacement-generation", + lifecycle: "ready", + consecutiveFailures: 0, + }); + expect(persistWorker).not.toHaveBeenCalled(); + }); + it("does not relaunch a live worker whose process identity is unknown", async () => { vi.useFakeTimers(); type RecoveryWorker = { @@ -1458,7 +1985,7 @@ describe("daemon worker supervisor monitoring", () => { await vi.runAllTimersAsync(); await recovery; - expect(supervisor.connectWorker).toHaveBeenCalledTimes(3); + expect(supervisor.connectWorker).not.toHaveBeenCalled(); expect(supervisor.recoverUncertainWorkerOperations).not.toHaveBeenCalled(); expect(supervisor.launchWorker).not.toHaveBeenCalled(); expect(worker.descriptor.lifecycle).toBe("failed"); @@ -1469,8 +1996,8 @@ describe("daemon worker supervisor monitoring", () => { type RecoveryWorker = { descriptor: { workerId: string; - pid: number; - processStartId?: string; + process: { pid: number; processStartId: string }; + generation: string; rootActiveSessionId: string; createCommand: { type: "create" }; lifecycle?: string; @@ -1497,7 +2024,8 @@ describe("daemon worker supervisor monitoring", () => { const worker: RecoveryWorker = { descriptor: { workerId: "worker-peer-sync-failure", - pid: process.pid, + process: { pid: process.pid, processStartId: getProcessStartId(process.pid)! }, + generation: randomUUID(), rootActiveSessionId: "active-1", createCommand: { type: "create" }, consecutiveFailures: 1, @@ -1880,14 +2408,18 @@ describe("daemon worker supervisor monitoring", () => { sessionId: "root-session", sessionFile: "/tmp/root.jsonl", busy: true, - operation: "model_stream", + operation: "agent_start", + operationId: randomUUID(), + generation: randomUUID(), }); journal.record({ activeSessionId: "child-active", sessionId: "child-session", sessionFile: "/tmp/child.jsonl", busy: true, - operation: "tool_execution", + operation: "tool_execution_start", + operationId: randomUUID(), + generation: randomUUID(), }); const worker: RecoveryWorker = { descriptor: { @@ -1912,8 +2444,8 @@ describe("daemon worker supervisor monitoring", () => { await supervisor.recoverUncertainWorkerOperations(worker, false); expect(kill).not.toHaveBeenCalled(); expect(markInterrupted).toHaveBeenCalledTimes(2); - expect(markInterrupted).toHaveBeenCalledWith("/tmp/root.jsonl", "root-active", ["model_stream"]); - expect(markInterrupted).toHaveBeenCalledWith("/tmp/child.jsonl", "child-active", ["tool_execution"]); + expect(markInterrupted).toHaveBeenCalledWith("/tmp/root.jsonl", "root-active", ["agent_start"]); + expect(markInterrupted).toHaveBeenCalledWith("/tmp/child.jsonl", "child-active", ["tool_execution_start"]); } finally { kill.mockRestore(); rmSync(root, { recursive: true, force: true }); diff --git a/packages/coding-agent/test/daemon-supervisor-restart-passivation.test.ts b/packages/coding-agent/test/daemon-supervisor-restart-passivation.test.ts index aefbbfdab..1d8333601 100644 --- a/packages/coding-agent/test/daemon-supervisor-restart-passivation.test.ts +++ b/packages/coding-agent/test/daemon-supervisor-restart-passivation.test.ts @@ -41,6 +41,7 @@ interface WorkerFixture { archiveFinalization?: Promise; stopFinalized?: boolean; stopFailure?: Error; + quarantined?: true; } interface PassivatedWorkerFixture extends WorkerFixture { @@ -55,6 +56,7 @@ interface PassivatedWorkerFixture extends WorkerFixture { interface SupervisorInternals { workers: Map; loadWorkerDescriptors(): Promise; + persistWorker(worker: WorkerFixture): void; wakePassivatedWorker(worker: WorkerFixture): Promise; forwardToWorker(worker: WorkerFixture, command: object): Promise; stopWorker( @@ -68,6 +70,7 @@ interface SupervisorInternals { stopWorkerOnce: ReturnType; recoverWorker: ReturnType; passivatedSummaryForDescriptor(descriptor: DaemonWorkerDescriptor): Promise; + assertRecoveryAllowed(): Promise; catalog: { archive(sessionFile: string, sessionId: string): Promise }; } @@ -106,6 +109,7 @@ function descriptor( version: 1, workerId, pid: 999_999_999, + generation: "11111111-1111-4111-8111-111111111111", socketPath: join(fixture.root, `${workerId}.sock`), recoveryJournalPath: join(fixture.descriptorDir, `${workerId}.recovery.jsonl`), supervisorSocketPath: fixture.socketPath, @@ -379,6 +383,63 @@ describe("daemon supervisor restart passivation", () => { expect(worker.descriptor.lifecycle).toBe("ready"); }); + it("passivates an explicit wake spawn failure without persisting recovery intent", async () => { + const fixture = fixtureRoot(); + const session = persistSession(fixture.sessionDir, fixture.root, "completed"); + const supervisor = new DaemonSupervisor(fixture.socketPath, { + defaultSessionConfig: { agentDir: fixture.agentDir, cwd: fixture.root, sessionDir: fixture.sessionDir }, + descriptorDir: fixture.descriptorDir, + }) as unknown as SupervisorInternals; + const { + pid: _pid, + processStartId: _processStartId, + ...passive + } = descriptor(fixture, "wake-spawn-failure", session); + const worker: WorkerFixture = { + descriptor: { ...passive, lifecycle: "passivated" }, + descriptorPath: join(fixture.descriptorDir, "wake-spawn-failure.json"), + summaries: new Map(), + }; + writeFileSync(worker.descriptorPath, `${JSON.stringify(worker.descriptor, null, 2)}\n`); + supervisor.workers.set(worker.descriptor.workerId, worker); + supervisor.assertRecoveryAllowed = vi.fn(async () => undefined); + workerSpawn.mockImplementation(() => { + throw new Error("deterministic spawn failure"); + }); + vi.useFakeTimers(); + vi.setSystemTime(new Date("2026-01-02T03:04:05.000Z")); + try { + // Register the rejection handler before advancing timers: wake intentionally + // reports its failed operation, but must not leak an unhandled rejection. + const wakeResult = supervisor.wakePassivatedWorker(worker).then( + () => undefined, + (error: unknown) => error, + ); + await vi.runAllTimersAsync(); + expect(await wakeResult).toBeInstanceOf(Error); + + const diskBytes = readFileSync(worker.descriptorPath, "utf8"); + const persisted = JSON.parse(diskBytes); + expect(persisted).toMatchObject({ + lifecycle: "passivated", + lastError: "deterministic spawn failure", + }); + expect(persisted).not.toHaveProperty("process"); + expect(persisted).not.toHaveProperty("pid"); + expect(persisted).not.toHaveProperty("processStartId"); + expect(diskBytes).not.toContain('"lifecycle": "recovering"'); + expect(diskBytes).not.toContain('"lifecycle": "failed"'); + // The writer output is exactly the canonical in-memory descriptor, rather + // than an older processless recovery/failed serialization. + expect(diskBytes).toBe(`${JSON.stringify(worker.descriptor, null, 2)}\n`); + } finally { + vi.useRealTimers(); + workerSpawn.mockImplementation(() => { + throw new Error("unexpected worker spawn"); + }); + } + }); + it("rejects an incompatible telemetry attach before waking a passivated root", async () => { const fixture = fixtureRoot(); const session = persistSession(fixture.sessionDir, fixture.root, "completed"); @@ -430,9 +491,13 @@ describe("daemon supervisor restart passivation", () => { descriptorPath: join(fixture.descriptorDir, "read.json"), summaries: new Map(), }; + supervisor.workers.set(worker.descriptor.workerId, worker); + const request = vi.fn().mockResolvedValue({ type: "response", command: "prompt", success: true }); supervisor.recoverWorker = vi.fn(async () => { + // Launch publishes a fresh incarnation on this same resident object. + worker.descriptor.generation = "woken-generation"; worker.descriptor.lifecycle = "ready"; - worker.client = { request: vi.fn() }; + worker.client = { request }; }); for (const command of [ @@ -450,6 +515,11 @@ describe("daemon supervisor restart passivation", () => { message: "resume", }); expect(supervisor.recoverWorker).toHaveBeenCalledTimes(1); + expect(request).toHaveBeenCalledOnce(); + expect(request).toHaveBeenCalledWith( + expect.objectContaining({ type: "prompt", activeSessionId: "active-read" }), + expect.any(Number), + ); }); it.each([ @@ -472,6 +542,7 @@ describe("daemon supervisor restart passivation", () => { descriptorPath: join(fixture.descriptorDir, "wake-command.json"), summaries: new Map(), }; + supervisor.workers.set(worker.descriptor.workerId, worker); supervisor.recoverWorker = vi.fn(async () => { worker.descriptor.lifecycle = "ready"; worker.client = { request: vi.fn() }; @@ -741,7 +812,7 @@ describe("daemon supervisor restart passivation", () => { descriptorDir: fixture.descriptorDir, }) as unknown as SupervisorInternals; const worker = { - descriptor: entry, + descriptor: { ...entry, generation: "11111111-1111-4111-8111-111111111111" }, descriptorPath: join(fixture.descriptorDir, "owner-no-env.json"), summaries: new Map(), snapshotCache: new Map(), @@ -1001,7 +1072,7 @@ describe("daemon supervisor restart passivation", () => { await supervisor.stopWorker(worker, false); expect(supervisor.workers.has("update-stop")).toBe(false); - expect(JSON.parse(readFileSync(worker.descriptorPath, "utf8"))).toMatchObject({ lifecycle: "recovering" }); + expect(JSON.parse(readFileSync(worker.descriptorPath, "utf8"))).toMatchObject({ lifecycle: "passivated" }); }); it("late archive deletion removes a retained recovering descriptor before restart", async () => { @@ -1035,7 +1106,7 @@ describe("daemon supervisor restart passivation", () => { supervisor.catalog.archive = vi.fn(async () => undefined); await supervisor.stopWorker(worker, false); - expect(JSON.parse(readFileSync(worker.descriptorPath, "utf8"))).toMatchObject({ lifecycle: "recovering" }); + expect(JSON.parse(readFileSync(worker.descriptorPath, "utf8"))).toMatchObject({ lifecycle: "passivated" }); await supervisor.stopWorker(worker, true, false, true); expect(supervisor.catalog.archive).toHaveBeenCalledTimes(1); @@ -1174,37 +1245,70 @@ describe("daemon supervisor restart passivation", () => { expect(supervisor.recoverWorker).toHaveBeenCalledTimes(1); }); - it("keeps normalized v1 lifecycle roots visible and passive across successive reloads", async () => { + it("writes one canonical passive migration for a safely classifiable stale nested legacy identity", async () => { + const fixture = fixtureRoot(); + const session = persistSession(fixture.sessionDir, fixture.root, "completed"); + const entry = descriptor(fixture, "nested-legacy", session); + delete entry.pid; + entry.process = { pid: 999_999_999, processStartId: "stale-start" }; + delete entry.generation; + writeFileSync(join(fixture.descriptorDir, `${entry.workerId}.json`), JSON.stringify(entry)); + + const supervisor = new DaemonSupervisor(fixture.socketPath, { + defaultSessionConfig: { agentDir: fixture.agentDir, cwd: fixture.root, sessionDir: fixture.sessionDir }, + descriptorDir: fixture.descriptorDir, + }) as unknown as SupervisorInternals; + const persist = vi.spyOn(supervisor, "persistWorker"); + await supervisor.loadWorkerDescriptors(); + + const loaded = supervisor.workers.get(entry.workerId); + expect(loaded?.descriptor.generation).toMatch(/^[0-9a-f-]+$/); + expect(loaded?.descriptor.lifecycle).toBe("passivated"); + const persisted = JSON.parse(readFileSync(join(fixture.descriptorDir, `${entry.workerId}.json`), "utf8")); + expect(persisted).toMatchObject({ generation: loaded?.descriptor.generation, lifecycle: "passivated" }); + expect(persisted).not.toHaveProperty("pid"); + expect(persisted).not.toHaveProperty("processStartId"); + expect(persist).toHaveBeenCalledOnce(); + }); + + it("keeps malformed lifecycle descriptors visible as raw quarantined evidence across successive reloads", async () => { const fixture = fixtureRoot(); const missing = persistSession(fixture.sessionDir, fixture.root, "completed"); const unknown = persistSession(fixture.sessionDir, fixture.root, "completed"); const missingDescriptor = descriptor(fixture, "missing-lifecycle", missing); const unknownDescriptor = { ...descriptor(fixture, "unknown-lifecycle", unknown), lifecycle: "future_state" }; delete (missingDescriptor as Partial).lifecycle; - writeFileSync(join(fixture.descriptorDir, "missing-lifecycle.json"), JSON.stringify(missingDescriptor)); - writeFileSync(join(fixture.descriptorDir, "unknown-lifecycle.json"), JSON.stringify(unknownDescriptor)); + delete (missingDescriptor as Partial).generation; + delete (unknownDescriptor as Partial).generation; + const raw = new Map([ + ["missing-lifecycle", JSON.stringify(missingDescriptor)], + ["unknown-lifecycle", JSON.stringify(unknownDescriptor)], + ]); + for (const [workerId, contents] of raw) { + writeFileSync(join(fixture.descriptorDir, `${workerId}.json`), contents); + } - // The first reload normalizes untrusted lifecycle values. The next one must - // accept those durable, processless `recovering` descriptors rather than - // dropping their roots before it can classify them as passive. + // Unknown lifecycle input remains available to diagnostics, but its disk + // bytes are not C01 state. It must never be normalized into a writable, + // processless recovering row on either reload. const first = new DaemonSupervisor(fixture.socketPath, { defaultSessionConfig: { agentDir: fixture.agentDir, cwd: fixture.root, sessionDir: fixture.sessionDir }, descriptorDir: fixture.descriptorDir, }) as unknown as SupervisorInternals; first.recoverWorker = vi.fn(); + const firstWrite = vi.spyOn(first, "persistWorker"); workerSpawn.mockClear(); const kill = vi.spyOn(process, "kill"); try { await first.loadWorkerDescriptors(); expect(first.workers).toHaveLength(2); - for (const workerId of ["missing-lifecycle", "unknown-lifecycle"]) { + for (const workerId of raw.keys()) { const worker = first.workers.get(workerId); - expect(worker?.descriptor.lifecycle).toBe("recovering"); + expect(worker).toMatchObject({ quarantined: true, descriptor: { lifecycle: "recovering" } }); + expect(worker?.descriptor.process).toBeUndefined(); expect(worker?.descriptor.pid).toBeUndefined(); - expect(worker?.descriptor.processStartId).toBeUndefined(); - const persisted = JSON.parse(readFileSync(join(fixture.descriptorDir, `${workerId}.json`), "utf8")); - expect(persisted.lifecycle).toBe("recovering"); - expect(persisted.pid).toBeUndefined(); + await expect(first.wakePassivatedWorker(worker!)).rejects.toThrow("quarantined"); + expect(readFileSync(join(fixture.descriptorDir, `${workerId}.json`), "utf8")).toBe(raw.get(workerId)); } const second = new DaemonSupervisor(fixture.socketPath, { @@ -1213,19 +1317,16 @@ describe("daemon supervisor restart passivation", () => { }) as unknown as SupervisorInternals; second.recoverWorker = vi.fn(); await second.loadWorkerDescriptors(); - - // Completed roots become metadata-only on the second reload, retaining - // their summaries for explicit wake without an automatic wake or signal. expect(second.workers).toHaveLength(2); - for (const workerId of ["missing-lifecycle", "unknown-lifecycle"]) { + for (const workerId of raw.keys()) { const worker = second.workers.get(workerId); - expect(worker?.descriptor.lifecycle).toBe("passivated"); - expect(worker?.descriptor.pid).toBeUndefined(); - expect(worker?.descriptor.processStartId).toBeUndefined(); + expect(worker).toMatchObject({ quarantined: true, descriptor: { lifecycle: "recovering" } }); expect(worker?.client).toBeUndefined(); - expect(worker?.summaries.has(worker?.descriptor.rootActiveSessionId ?? "")).toBe(true); + expect(worker?.summaries).toHaveLength(0); + expect(readFileSync(join(fixture.descriptorDir, `${workerId}.json`), "utf8")).toBe(raw.get(workerId)); } expect(first.recoverWorker).not.toHaveBeenCalled(); + expect(firstWrite).not.toHaveBeenCalled(); expect(second.recoverWorker).not.toHaveBeenCalled(); expect(workerSpawn).not.toHaveBeenCalled(); expect(kill).not.toHaveBeenCalled(); @@ -1310,7 +1411,95 @@ describe("daemon supervisor restart passivation", () => { } }); - it("recovers rather than passivating malformed or stale durable task verdicts", async () => { + it.each(["flat", "nested"] as const)( + "quarantines unknown-start legacy %s identity with recoverable work without rewriting raw evidence", + async (shape) => { + const fixture = fixtureRoot(); + const session = persistSession(fixture.sessionDir, fixture.root, "completed"); + const entry = descriptor(fixture, `unknown-start-busy-${shape}`, session); + delete entry.generation; + if (shape === "nested") { + delete entry.pid; + entry.process = { pid: 999_999_999, processStartId: "unobservable-start" }; + } + // The impossible PID makes either legacy identity unobservable; the busy + // journal independently rejects passive classification. This exact + // combination must retain raw evidence rather than write processless + // `recovering` C01 state that could later be restarted. + writeFileSync( + entry.recoveryJournalPath, + `${JSON.stringify({ + version: 1, + activeSessionId: entry.rootActiveSessionId, + sessionId: session.id, + sessionFile: session.sessionFile, + busy: true, + operation: "legacy-work", + recordedAt: new Date(0).toISOString(), + })}\n`, + ); + const descriptorPath = join(fixture.descriptorDir, `${entry.workerId}.json`); + const raw = JSON.stringify(entry); + writeFileSync(descriptorPath, raw); + const supervisor = new DaemonSupervisor(fixture.socketPath, { + defaultSessionConfig: { agentDir: fixture.agentDir, cwd: fixture.root, sessionDir: fixture.sessionDir }, + descriptorDir: fixture.descriptorDir, + }) as unknown as SupervisorInternals; + supervisor.recoverWorker = vi.fn(); + const persist = vi.spyOn(supervisor, "persistWorker"); + const kill = vi.spyOn(process, "kill"); + try { + await supervisor.loadWorkerDescriptors(); + const worker = supervisor.workers.get(entry.workerId); + expect(worker).toMatchObject({ quarantined: true, descriptor: { lifecycle: "recovering" } }); + expect(worker?.descriptor.process).toBeUndefined(); + expect(worker?.descriptor.pid).toBeUndefined(); + expect(worker?.summaries).toHaveLength(0); + expect(readFileSync(descriptorPath, "utf8")).toBe(raw); + expect(persist).not.toHaveBeenCalled(); + expect(supervisor.recoverWorker).not.toHaveBeenCalled(); + await expect(supervisor.wakePassivatedWorker(worker!)).rejects.toThrow("quarantined"); + expect(kill).not.toHaveBeenCalled(); + } finally { + kill.mockRestore(); + } + }, + ); + + it("writes only canonical C01 descriptor shapes", () => { + const fixture = fixtureRoot(); + const session = persistSession(fixture.sessionDir, fixture.root, "completed"); + const supervisor = new DaemonSupervisor(fixture.socketPath, { + defaultSessionConfig: { agentDir: fixture.agentDir, cwd: fixture.root, sessionDir: fixture.sessionDir }, + descriptorDir: fixture.descriptorDir, + }) as unknown as SupervisorInternals; + const { pid: _pid, processStartId: _startId, ...passive } = descriptor(fixture, "writer-shape", session); + const worker: WorkerFixture = { + descriptor: { ...passive, lifecycle: "passivated" }, + descriptorPath: join(fixture.descriptorDir, "writer-shape.json"), + summaries: new Map(), + }; + + supervisor.persistWorker(worker); + const persisted = JSON.parse(readFileSync(worker.descriptorPath, "utf8")); + expect(persisted).toMatchObject({ generation: "11111111-1111-4111-8111-111111111111", lifecycle: "passivated" }); + expect(persisted).not.toHaveProperty("pid"); + expect(persisted).not.toHaveProperty("processStartId"); + expect(persisted).not.toHaveProperty("process"); + + worker.descriptor.generation = "legacy-generation"; + expect(() => supervisor.persistWorker(worker)).toThrow("canonical generation"); + worker.descriptor.generation = "11111111-1111-4111-8111-111111111111"; + for (const lifecycle of ["starting", "recovering", "failed"] as const) { + worker.descriptor.lifecycle = lifecycle; + expect(() => supervisor.persistWorker(worker)).toThrow("without a process identity"); + } + worker.descriptor.lifecycle = "passivated"; + worker.descriptor.process = { pid: process.pid, processStartId: "not-authoritative" }; + expect(() => supervisor.persistWorker(worker)).toThrow("passivated worker"); + }); + + it("quarantines malformed or stale durable task verdicts without rewriting legacy selectors", async () => { const fixture = fixtureRoot(); const malformed = persistSession(fixture.sessionDir, fixture.root, "completed"); const stale = persistSession(fixture.sessionDir, fixture.root, "completed"); @@ -1326,13 +1515,8 @@ describe("daemon supervisor restart passivation", () => { ] as const) { writeFileSync( session.sessionFile, - `${JSON.stringify({ - type: "agent_status", - id: `${workerId}-status`, - parentId: "root", - timestamp: new Date().toISOString(), - status, - })}\n`, + `${JSON.stringify({ type: "agent_status", id: `${workerId}-status`, parentId: "root", timestamp: new Date().toISOString(), status })} +`, { flag: "a" }, ); writeFileSync( @@ -1342,13 +1526,8 @@ describe("daemon supervisor restart passivation", () => { } writeFileSync( invalidLifecycle.sessionFile, - `${JSON.stringify({ - type: "session_state", - id: "invalid-lifecycle", - parentId: "root", - timestamp: new Date().toISOString(), - state: { status: "untrusted_lifecycle" }, - })}\n`, + `${JSON.stringify({ type: "session_state", id: "invalid-lifecycle", parentId: "root", timestamp: new Date().toISOString(), state: { status: "untrusted_lifecycle" } })} +`, { flag: "a" }, ); writeFileSync( @@ -1358,10 +1537,10 @@ describe("daemon supervisor restart passivation", () => { writeFileSync(truncatedLifecycle.sessionFile, '{"type":"session_state"', { flag: "a" }); writeFileSync( oversizedVerdict.sessionFile, - `\n{"type":"agent_status","padding":"${"x".repeat(2 * 1024 * 1024)}"}\n`, - { - flag: "a", - }, + ` +{"type":"agent_status","padding":"${"x".repeat(2 * 1024 * 1024)}"} +`, + { flag: "a" }, ); for (const [workerId, session] of [ ["truncated-lifecycle", truncatedLifecycle], @@ -1373,29 +1552,28 @@ describe("daemon supervisor restart passivation", () => { JSON.stringify(descriptor(fixture, workerId, session)), ); } + const raw = new Map( + [ + "malformed", + "stale", + "invalid-lifecycle", + "truncated-lifecycle", + "oversized-verdict", + "corrupt-array-artifact", + ].map((workerId) => [workerId, readFileSync(join(fixture.descriptorDir, `${workerId}.json`), "utf8")]), + ); const supervisor = new DaemonSupervisor(fixture.socketPath, { defaultSessionConfig: { agentDir: fixture.agentDir, cwd: fixture.root, sessionDir: fixture.sessionDir }, descriptorDir: fixture.descriptorDir, }) as unknown as SupervisorInternals; + const persist = vi.spyOn(supervisor, "persistWorker"); await supervisor.loadWorkerDescriptors(); - expect(supervisor.workers.get("malformed")?.descriptor.lifecycle).toBe("recovering"); - expect(supervisor.workers.get("stale")?.descriptor.lifecycle).toBe("recovering"); - expect(supervisor.workers.get("invalid-lifecycle")?.descriptor.lifecycle).toBe("recovering"); - expect(supervisor.workers.get("truncated-lifecycle")?.descriptor.lifecycle).toBe("recovering"); - expect(supervisor.workers.get("oversized-verdict")?.descriptor.lifecycle).toBe("recovering"); - // A top-level array is corrupt artifact state, so fail closed rather than - // passivating a root whose scheduled work cannot be reconstructed. - expect(supervisor.workers.get("corrupt-array-artifact")?.descriptor.lifecycle).toBe("recovering"); - for (const workerId of [ - "malformed", - "stale", - "invalid-lifecycle", - "truncated-lifecycle", - "oversized-verdict", - "corrupt-array-artifact", - ]) { - const persisted = JSON.parse(readFileSync(join(fixture.descriptorDir, `${workerId}.json`), "utf8")); - expect(persisted.pid).toBe(999_999_999); + for (const [workerId, contents] of raw) { + const worker = supervisor.workers.get(workerId); + expect(worker).toMatchObject({ quarantined: true, descriptor: { lifecycle: "recovering" } }); + expect(worker?.descriptor.process).toBeUndefined(); + expect(readFileSync(join(fixture.descriptorDir, `${workerId}.json`), "utf8")).toBe(contents); } + expect(persist).not.toHaveBeenCalled(); }); }); diff --git a/packages/coding-agent/test/sdk-session-manager.test.ts b/packages/coding-agent/test/sdk-session-manager.test.ts index 005d2bde8..82fda9cec 100644 --- a/packages/coding-agent/test/sdk-session-manager.test.ts +++ b/packages/coding-agent/test/sdk-session-manager.test.ts @@ -1,11 +1,47 @@ -import { existsSync, mkdirSync, realpathSync, rmSync } from "node:fs"; +import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { getModel } from "@earendil-works/pi-ai"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { createAgentSession } from "../src/core/sdk.js"; -import { SessionManager } from "../src/core/session-manager.js"; - +import { + AGENT_TASK_STATES, + CURRENT_SESSION_VERSION, + readSessionInfo, + SessionManager, +} from "../src/core/session-manager.js"; + +const CODEC_SCHEMA_REVISION = 3; +const CODEC_TIMESTAMP = "2026-01-02T03:04:05.000Z"; +const CODEC_HEADER = { + type: "session", + version: CODEC_SCHEMA_REVISION, + id: "codec-fixture-session", + timestamp: CODEC_TIMESTAMP, + cwd: "/codec-fixture/project", + rlmDepth: 0, +}; + +function writeCodecFixture(filePath: string, entries: readonly Record[]): void { + writeFileSync(filePath, `${[CODEC_HEADER, ...entries].map((entry) => JSON.stringify(entry)).join("\n")}\n`); +} + +function assistantFixture(parentId: string | null = null): Record { + return { + type: "message", + id: "codec-assistant-message", + parentId, + timestamp: CODEC_TIMESTAMP, + message: { role: "assistant", content: "fixture", timestamp: 0 }, + }; +} + +function readCodecEntries(filePath: string): Record[] { + return readFileSync(filePath, "utf8") + .trim() + .split("\n") + .map((line) => JSON.parse(line) as Record); +} describe("createAgentSession session manager defaults", () => { let tempDir: string; let cwd: string; @@ -93,3 +129,108 @@ describe("createAgentSession session manager defaults", () => { session.dispose(); }); }); + +describe("SessionManager durable codec compatibility", () => { + let tempDir: string; + + beforeEach(() => { + tempDir = join(tmpdir(), "pi-sdk-session-codec-fixture"); + rmSync(tempDir, { recursive: true, force: true }); + mkdirSync(tempDir, { recursive: true }); + }); + + afterEach(() => { + rmSync(tempDir, { recursive: true, force: true }); + }); + + it.each(AGENT_TASK_STATES)("writes and rereads the %s task-state codec", async (taskState) => { + const sessionFile = join(tempDir, `${taskState}.jsonl`); + writeCodecFixture(sessionFile, [assistantFixture()]); + const manager = SessionManager.open(sessionFile, tempDir); + + manager.appendAgentStatus({ + summary: `Fixture ${taskState} verdict`, + taskState, + basedOnMessageCount: 1, + }); + + const entries = readCodecEntries(sessionFile); + const stored = entries.at(-1); + expect(entries[0]).toEqual(CODEC_HEADER); + expect(CURRENT_SESSION_VERSION).toBe(CODEC_SCHEMA_REVISION); + expect(stored).toMatchObject({ + type: "agent_status", + parentId: "codec-assistant-message", + status: { + summary: `Fixture ${taskState} verdict`, + taskState, + basedOnMessageCount: 1, + }, + }); + expect(Object.keys(stored ?? {}).sort()).toEqual(["id", "parentId", "status", "timestamp", "type"]); + const storedStatus = stored?.status as Record | undefined; + expect(storedStatus).toBeDefined(); + expect(Object.keys(storedStatus!).sort()).toEqual(["basedOnMessageCount", "summary", "taskState"]); + expect(manager.getLatestAgentStatus()).toEqual({ + summary: `Fixture ${taskState} verdict`, + taskState, + basedOnMessageCount: 1, + }); + expect((await readSessionInfo(sessionFile))?.agentStatus).toEqual({ + summary: `Fixture ${taskState} verdict`, + taskState, + basedOnMessageCount: 1, + }); + }); + + it.each([ + ["unknown task state", { summary: "Unknown verdict", taskState: "waiting", basedOnMessageCount: 1 }], + ["malformed task state", { summary: "Malformed verdict", taskState: 42, basedOnMessageCount: 1 }], + ])("marks a %s durable status as untrusted", async (_description, status) => { + const sessionFile = join(tempDir, "invalid-status.jsonl"); + writeCodecFixture(sessionFile, [ + assistantFixture(), + { + type: "agent_status", + id: "codec-invalid-status", + parentId: "codec-assistant-message", + timestamp: CODEC_TIMESTAMP, + status, + }, + ]); + + const info = await readSessionInfo(sessionFile); + expect(info).toMatchObject({ hasInvalidDurableState: true }); + expect(info?.agentStatus).toBeUndefined(); + }); + + it.each(["hidden", "sleep"] as const)( + "reads legacy %s lifecycle state as archived without rewriting its public JSON", + async (legacyStatus) => { + const sessionFile = join(tempDir, `${legacyStatus}.jsonl`); + writeCodecFixture(sessionFile, [ + assistantFixture(), + { + type: "session_state", + id: "codec-legacy-state", + parentId: "codec-assistant-message", + timestamp: CODEC_TIMESTAMP, + state: { status: legacyStatus }, + }, + ]); + + const manager = SessionManager.open(sessionFile, tempDir); + expect(manager.getSessionState()).toEqual({ status: "archived" }); + // A current write appends a current state rather than mutating the legacy row. + manager.appendSessionState({ status: "archived" }); + + const entries = readCodecEntries(sessionFile); + expect(entries[0]).toEqual(CODEC_HEADER); + expect(entries.filter((entry) => entry.type === "session_state").map((entry) => entry.state)).toEqual([ + { status: legacyStatus }, + { status: "archived" }, + ]); + expect((await readSessionInfo(sessionFile))?.state).toEqual({ status: "archived" }); + }, + ); +}); diff --git a/packages/coding-agent/test/worker-recovery-journal.test.ts b/packages/coding-agent/test/worker-recovery-journal.test.ts index 1f8ce4796..8efe4ec6f 100644 --- a/packages/coding-agent/test/worker-recovery-journal.test.ts +++ b/packages/coding-agent/test/worker-recovery-journal.test.ts @@ -1,69 +1,419 @@ -import { appendFileSync, mkdtempSync, rmSync } from "node:fs"; +import { + appendFileSync, + chmodSync, + closeSync, + fsyncSync, + mkdirSync, + mkdtempSync, + openSync, + readFileSync, + renameSync, + rmSync, + unlinkSync, + writeSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { WorkerRecoveryJournal } from "../src/modes/daemon/worker-recovery-journal.js"; +import { + WorkerRecoveryJournal, + type WorkerRecoveryJournalFileSystem, +} from "../src/modes/daemon/worker-recovery-journal.js"; -describe("WorkerRecoveryJournal", () => { - const roots: string[] = []; +const generationA = "11111111-1111-4111-8111-111111111111"; +const generationB = "22222222-2222-4222-8222-222222222222"; +const operationA = "33333333-3333-4333-8333-333333333333"; +const operationB = "44444444-4444-4444-8444-444444444444"; +const operationC = "55555555-5555-4555-8555-555555555555"; + +function recordingFileSystem( + events: string[], + { + failTempFsync = false, + maximumWriteLength, + zeroTempWrite = false, + }: { + failTempFsync?: boolean | number; + maximumWriteLength?: number; + zeroTempWrite?: boolean; + } = {}, +): WorkerRecoveryJournalFileSystem { + const descriptors = new Map(); + let remainingTempFsyncFailures = + typeof failTempFsync === "number" ? failTempFsync : failTempFsync ? Number.POSITIVE_INFINITY : 0; + return { + mkdirSync, + readFileSync, + openSync(path, flags, mode) { + const fd = mode === undefined ? openSync(path, flags) : openSync(path, flags, mode); + descriptors.set( + fd, + path.endsWith(".tmp") ? "temp" : path.includes("worker.recovery") ? "journal" : "directory", + ); + events.push(`open:${flags}:${descriptors.get(fd)}:${mode?.toString(8) ?? ""}`); + return fd; + }, + writeSync(fd, data, offset, length) { + events.push(`write:${descriptors.get(fd)}`); + if (zeroTempWrite && descriptors.get(fd) === "temp") return 0; + return writeSync( + fd, + data, + offset, + maximumWriteLength === undefined ? length : Math.min(length, maximumWriteLength), + ); + }, + fsyncSync(fd) { + events.push(`fsync:${descriptors.get(fd)}`); + if (remainingTempFsyncFailures > 0 && descriptors.get(fd) === "temp") { + remainingTempFsyncFailures--; + const error = new Error("injected temporary-file fsync failure") as NodeJS.ErrnoException; + error.code = "EIO"; + throw error; + } + fsyncSync(fd); + }, + closeSync(fd) { + events.push(`close:${descriptors.get(fd)}`); + closeSync(fd); + }, + chmodSync, + renameSync(oldPath, newPath) { + events.push("rename"); + renameSync(oldPath, newPath); + }, + unlinkSync(path) { + events.push("unlink"); + unlinkSync(path); + }, + }; +} +describe("WorkerRecoveryJournal C01 identities", () => { + const roots: string[] = []; afterEach(() => { - for (const root of roots.splice(0)) { - rmSync(root, { recursive: true, force: true }); - } + for (const root of roots.splice(0)) rmSync(root, { recursive: true, force: true }); }); - function createPath(): string { + function path(): string { const root = mkdtempSync(join(tmpdir(), "prime-agent-worker-recovery-")); roots.push(root); return join(root, "worker.recovery.jsonl"); } - it("restores the latest operation state per session", () => { - const path = createPath(); - const journal = new WorkerRecoveryJournal(path); + const base = { + activeSessionId: "active", + sessionId: "session", + operation: "prompt" as const, + generation: generationA, + operationId: operationA, + }; + + it("allows only the operation that began a v2 checkpoint to clear it", () => { + const journal = new WorkerRecoveryJournal(path()); + journal.record({ ...base, busy: true }); + journal.record({ ...base, busy: false }); + expect(journal.getLatest()).toEqual([]); + }); + + it("allows an exact completion after session materialization changes checkpoint metadata", () => { + const journal = new WorkerRecoveryJournal(path()); + // The operation begins before its in-memory session has a file. The daemon + // reconstructs session metadata at completion, after materialization changes it. + journal.record({ ...base, busy: true, sessionId: "draft" }); + + // Operation equality remains an authority fence even when the payload changes. journal.record({ - activeSessionId: "active-1", - sessionId: "session-1", - sessionFile: "/tmp/session-1.jsonl", - busy: true, - operation: "prompt_accepted", + ...base, + busy: false, + operation: "tool_execution", + sessionId: "materialized", + sessionFile: "/sessions/materialized.jsonl", }); + expect(journal.getLatest()).toEqual([expect.objectContaining({ busy: true, operation: "prompt" })]); + journal.record({ - activeSessionId: "active-2", - sessionId: "session-2", + ...base, busy: false, - operation: "ready", + sessionId: "materialized", + sessionFile: "/sessions/materialized.jsonl", }); + expect(journal.getLatest()).toEqual([]); + }); + + it("retains a busy operation across restart when a different allowed operation replays its terminal token", () => { + const file = path(); + const journal = new WorkerRecoveryJournal(file); + journal.record({ ...base, busy: true }); + + // The token values match exactly; only the allowed operation differs. + journal.record({ ...base, busy: false, operation: "tool_execution" }); + expect(journal.getLatest()).toEqual([ + expect.objectContaining({ + busy: true, + operation: "prompt", + activeSessionId: base.activeSessionId, + generation: generationA, + operationId: operationA, + }), + ]); + + // Replay after a crash/restart must remain unable to erase the original evidence. + const restarted = new WorkerRecoveryJournal(file); + restarted.record({ ...base, busy: false, operation: "tool_execution" }); + expect(restarted.getLatest()).toEqual([ + expect.objectContaining({ busy: true, operation: "prompt", operationId: operationA }), + ]); + restarted.record({ ...base, busy: false }); + expect(restarted.getLatest()).toEqual([]); + }); - expect(WorkerRecoveryJournal.readLatest(path)).toEqual( + it("does not let an unstarted v2 completion manufacture a clear", () => { + const journal = new WorkerRecoveryJournal(path()); + journal.record({ ...base, busy: false }); + expect(journal.getLatest()).toEqual([]); + }); + + it("does not let overlapping same-family A complete B", () => { + const journal = new WorkerRecoveryJournal(path()); + journal.record({ ...base, busy: true }); + journal.record({ ...base, busy: true, operationId: operationB }); + journal.record({ ...base, busy: false, operationId: operationA }); + expect(journal.getLatest()).toEqual([expect.objectContaining({ busy: true, operationId: operationB })]); + }); + + it("refuses a stale operation completion while retaining another generation", () => { + const journal = new WorkerRecoveryJournal(path()); + journal.record({ ...base, busy: true }); + journal.record({ ...base, busy: false, operationId: operationB }); + journal.record({ ...base, busy: true, generation: generationB }); + journal.record({ ...base, busy: false, generation: generationB }); + expect(journal.getLatest()).toEqual([ + expect.objectContaining({ busy: true, operationId: operationA, generation: generationA }), + ]); + }); + + it("keeps v1 uncertain and malformed tails recoverable without letting them replace v2", () => { + const file = path(); + appendFileSync( + file, + `${JSON.stringify({ version: 1, activeSessionId: "old", sessionId: "old", busy: true, operation: "unknown", recordedAt: new Date().toISOString() })}\n{truncated`, + ); + const journal = new WorkerRecoveryJournal(file); + journal.record({ ...base, busy: true }); + expect(journal.getLatest()).toEqual( expect.arrayContaining([ - expect.objectContaining({ activeSessionId: "active-1", busy: true, operation: "prompt_accepted" }), - expect.objectContaining({ activeSessionId: "active-2", busy: false, operation: "ready" }), + expect.objectContaining({ version: 1, activeSessionId: "old", busy: true }), + expect.objectContaining({ version: 2, operationId: operationA }), ]), ); + expect(journal.hasUnreadableRecords()).toBe(true); }); + it("retains B after A completes and clears only B's exact terminal token across restart", () => { + const file = path(); + const journal = new WorkerRecoveryJournal(file); + // Same session, generation and operation family: only the UUID separates + // overlapping queued turns. This is the daemon scheduler's A/B ordering. + journal.record({ ...base, busy: true, operationId: operationA }); + journal.record({ ...base, busy: true, operationId: operationB }); + journal.record({ ...base, busy: false, operationId: operationA }); + expect(journal.getLatest()).toEqual([expect.objectContaining({ busy: true, operationId: operationB })]); - it("compacts stable checkpoints while preserving a truncated final record as recoverable", () => { - const path = createPath(); - const journal = new WorkerRecoveryJournal(path); - journal.record({ - activeSessionId: "active-1", - sessionId: "session-1", - busy: true, - operation: "bash_start", + // A restart reads B as crash evidence. A's stale callback cannot clear B. + const restarted = new WorkerRecoveryJournal(file); + restarted.record({ ...base, busy: false, operationId: operationA }); + expect(restarted.getLatest()).toEqual([expect.objectContaining({ busy: true, operationId: operationB })]); + restarted.record({ ...base, busy: false, operationId: operationB }); + expect(restarted.getLatest()).toEqual([]); + }); + + it("prunes completed v2 history inherited from an older journal on restart", () => { + const file = path(); + for (let index = 0; index < 128; index++) { + const operationId = `77777777-7777-4777-8777-${String(index).padStart(12, "0")}`; + appendFileSync( + file, + `${JSON.stringify({ version: 2, ...base, busy: false, operationId, recordedAt: new Date().toISOString() })}\n`, + ); + } + const restarted = new WorkerRecoveryJournal(file); + expect(restarted.getLatest()).toEqual([]); + expect(WorkerRecoveryJournal.readLatest(file)).toEqual([]); + }); + + it("bounds completed v2 history while retaining all busy operations and legacy uncertainty", () => { + const file = path(); + const journal = new WorkerRecoveryJournal(file); + const busyOperation = "55555555-5555-4555-8555-555555555555"; + journal.record({ ...base, busy: true, operationId: busyOperation }); + for (let index = 0; index < 128; index++) { + const operationId = `66666666-6666-4666-8666-${String(index).padStart(12, "0")}`; + journal.record({ ...base, busy: true, operationId }); + journal.record({ ...base, busy: false, operationId }); + } + appendFileSync( + file, + `${JSON.stringify({ version: 1, activeSessionId: "legacy", sessionId: "old", busy: true, operation: "unknown", recordedAt: new Date().toISOString() })}\n`, + ); + const restarted = new WorkerRecoveryJournal(file); + expect(restarted.getLatest()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ version: 2, operationId: busyOperation, busy: true }), + expect.objectContaining({ version: 1, activeSessionId: "legacy", busy: true }), + ]), + ); + expect(restarted.getLatest()).toHaveLength(2); + }); + it("fsyncs a restrictive replacement before rename and its parent directory after rename", () => { + const file = path(); + const tempPath = `${file}.fixed.tmp`; + const events: string[] = []; + const journal = new WorkerRecoveryJournal(file, { + fileSystem: recordingFileSystem(events), + makeTempPath: () => tempPath, }); - journal.record({ - activeSessionId: "active-1", - sessionId: "session-1", - busy: false, - operation: "bash_end", + journal.record({ ...base, busy: true }); + events.splice(0); + journal.record({ ...base, busy: false }); + expect(events).toEqual([ + "open:a:journal:600", + "write:journal", + "fsync:journal", + "close:journal", + "open:wx:temp:600", + "fsync:temp", + "close:temp", + "rename", + "open:r:directory:", + "fsync:directory", + "close:directory", + ]); + }); + + it("keeps a durably appended completion after compaction fails, then never resurrects it", () => { + const file = path(); + const tempPath = `${file}.failed.tmp`; + const journal = new WorkerRecoveryJournal(file, { + fileSystem: recordingFileSystem([], { failTempFsync: 1 }), + makeTempPath: () => tempPath, }); - appendFileSync(path, "{truncated"); + journal.record({ ...base, busy: true, operationId: operationA }); + journal.record({ ...base, busy: true, operationId: operationB }); + + // The terminal append commits before replacement. A replacement failure is + // reported to the caller, but must not restore A's old busy checkpoint. + expect(() => journal.record({ ...base, busy: false, operationId: operationA })).toThrow( + "injected temporary-file fsync failure", + ); + expect(journal.getLatest()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ busy: false, operationId: operationA }), + expect.objectContaining({ busy: true, operationId: operationB }), + ]), + ); + + // A completion replay is a no-op because A remains terminal in memory. + journal.record({ ...base, busy: false, operationId: operationA }); + expect(journal.getLatest()).toEqual( + expect.arrayContaining([expect.objectContaining({ busy: false, operationId: operationA })]), + ); + + // A later healthy compaction retains B while omitting the durable terminals. + // It must never rebuild A from the stale busy record; the same holds on restart. + journal.record({ ...base, busy: true, operationId: operationC }); + journal.record({ ...base, busy: false, operationId: operationC }); + expect(journal.getLatest()).toEqual([expect.objectContaining({ busy: true, operationId: operationB })]); + const restarted = new WorkerRecoveryJournal(file); + expect(restarted.getLatest()).toEqual([expect.objectContaining({ busy: true, operationId: operationB })]); + }); - expect(WorkerRecoveryJournal.readLatest(path)).toEqual([ - expect.objectContaining({ activeSessionId: "active-1", busy: false, operation: "bash_end" }), + it("never compacts malformed raw history, including terminal records, across restarts", () => { + const file = path(); + const received = { version: 2, ...base, busy: true, recordedAt: new Date().toISOString() }; + const completed = { version: 2, ...base, busy: false, recordedAt: new Date().toISOString() }; + const malformed = "{malformed raw recovery evidence}"; + const contents = `${JSON.stringify(received)}\n${malformed}\n${JSON.stringify(completed)}\n`; + appendFileSync(file, contents); + + const first = new WorkerRecoveryJournal(file); + expect(first.hasUnreadableRecords()).toBe(true); + expect(first.getLatest()).toEqual([expect.objectContaining({ busy: false, operationId: operationA })]); + expect(readFileSync(file, "utf8")).toBe(contents); + + // Both constructor auto-compaction and a completion-triggered compaction + // must fail closed by retaining the exact raw corruption for future recovery. + first.record({ ...base, busy: true, operationId: operationB }); + first.record({ ...base, busy: false, operationId: operationB }); + expect(readFileSync(file, "utf8")).toContain(malformed); + const second = new WorkerRecoveryJournal(file); + expect(second.hasUnreadableRecords()).toBe(true); + expect(second.getLatest()).toEqual( + expect.arrayContaining([expect.objectContaining({ busy: false, operationId: operationA })]), + ); + expect(readFileSync(file, "utf8")).toContain(malformed); + const third = new WorkerRecoveryJournal(file); + expect(third.hasUnreadableRecords()).toBe(true); + expect(readFileSync(file, "utf8")).toContain(malformed); + }); + + it("fails closed on replacement sync failure without removing a busy sibling journal record", () => { + const file = path(); + const tempPath = `${file}.fixed.tmp`; + const events: string[] = []; + const journal = new WorkerRecoveryJournal(file, { + fileSystem: recordingFileSystem(events, { failTempFsync: true }), + makeTempPath: () => tempPath, + }); + journal.record({ ...base, busy: true }); + journal.record({ ...base, busy: true, operationId: operationB }); + expect(() => journal.record({ ...base, busy: false })).toThrow("injected temporary-file fsync failure"); + expect(events).toContain("unlink"); + expect(events).not.toContain("rename"); + expect(() => readFileSync(tempPath, "utf8")).toThrow(); + expect(WorkerRecoveryJournal.readLatest(file)).toEqual( + expect.arrayContaining([expect.objectContaining({ busy: true, operationId: operationB })]), + ); + // The failed compaction also keeps the live instance conservative. + expect(journal.getLatest()).toEqual( + expect.arrayContaining([expect.objectContaining({ busy: true, operationId: operationB })]), + ); + }); + it("writes every byte when the filesystem reports partial writes", () => { + const file = path(); + const events: string[] = []; + const journal = new WorkerRecoveryJournal(file, { + fileSystem: recordingFileSystem(events, { maximumWriteLength: 1 }), + makeTempPath: (journalPath) => `${journalPath}.partial.tmp`, + }); + journal.record({ ...base, busy: true, operationId: operationA }); + journal.record({ ...base, busy: true, operationId: operationB, sessionFile: "é".repeat(4096) }); + journal.record({ ...base, busy: false, operationId: operationA }); + expect(events.filter((event) => event === "write:temp")).toHaveLength( + Buffer.byteLength(readFileSync(file, "utf8")), + ); + expect(WorkerRecoveryJournal.readLatest(file)).toEqual([ + expect.objectContaining({ busy: true, operationId: operationB }), ]); - expect(new WorkerRecoveryJournal(path).hasUnreadableRecords()).toBe(true); + }); + + it("fails before replacement when a write makes zero progress", () => { + const file = path(); + const tempPath = `${file}.zero.tmp`; + const events: string[] = []; + const journal = new WorkerRecoveryJournal(file, { + fileSystem: recordingFileSystem(events, { zeroTempWrite: true }), + makeTempPath: () => tempPath, + }); + journal.record({ ...base, busy: true, operationId: operationA }); + journal.record({ ...base, busy: true, operationId: operationB }); + expect(() => journal.record({ ...base, busy: false, operationId: operationA })).toThrow( + "Recovery journal write made no forward progress", + ); + expect(events).toContain("unlink"); + expect(events).not.toContain("rename"); + expect(() => readFileSync(tempPath, "utf8")).toThrow(); + expect(WorkerRecoveryJournal.readLatest(file)).toEqual( + expect.arrayContaining([expect.objectContaining({ busy: true, operationId: operationB })]), + ); }); });