diff --git a/docs/conversation-provenance.md b/docs/conversation-provenance.md index ba7a09eff..9fb1c322d 100644 --- a/docs/conversation-provenance.md +++ b/docs/conversation-provenance.md @@ -791,3 +791,19 @@ shapes should point back here rather than restating the rules differently. | `src/workspaces/issues/board.ts` | Issue/run/Inbox projections | | `src/services/uta-client/` | Alice -> UTA decision-correlation boundary | | `services/uta/src/domain/trading/` | Broker operation and execution authority | + + +### Conversation failure diagnostics + +`conversation read`, `await`, `collect`, and `ask --await` return a concise +`error` for failed/interrupted tasks, plus recorded `exitCode`, `signal`, +`killed`, and `processStarted` fields when available. Terminal structured +errors take priority over stderr; launch failures and watchdog termination +retain their explicit causes. Successful turns do not promote warnings or +recovered errors into a failure. + +`conversation read --task-id --mode detailed` also exposes the last 16 KiB +of stderr for failed/interrupted tasks, with `stderrTruncated` indicating a +clipped log. Existing tasks can recover this diagnostic from their log file; +missing logs still leave an exit/signal or generic failure explanation. Logs +remain diagnostics, never assistant replies. diff --git a/src/core/workspace-tool-center.ts b/src/core/workspace-tool-center.ts index 163a75a56..1dd1a4b9e 100644 --- a/src/core/workspace-tool-center.ts +++ b/src/core/workspace-tool-center.ts @@ -119,6 +119,12 @@ export interface WorkspaceConversationTask { readonly finishedAt?: number readonly durationMs?: number readonly error?: string + readonly exitCode?: number | null + readonly signal?: string | null + readonly killed?: boolean + readonly processStarted?: boolean + readonly stderrTail?: string + readonly stderrTruncated?: boolean readonly structured: HeadlessStructuredOutput | null } diff --git a/src/tool/conversation.spec.ts b/src/tool/conversation.spec.ts index 80237ec3f..9c74036f1 100644 --- a/src/tool/conversation.spec.ts +++ b/src/tool/conversation.spec.ts @@ -8,6 +8,7 @@ import { conversationAwaitFactory, conversationCollectFactory, conversationReadFactory, + taskProjection, } from './conversation.js' async function run(tool: Tool, args: Record) { @@ -363,3 +364,17 @@ describe('conversation_read', () => { }) }) }) + + +describe('conversation diagnostics projection', () => { + it('keeps stderr out of summaries but includes it in detailed failed output', () => { + const task = { ...completedTask, status: 'failed' as const, exitCode: 1, + stderrTail: 'No API key found', stderrTruncated: false } + expect(taskProjection(task, 'summary')).toMatchObject({ error: 'No API key found', exitCode: 1 }) + expect(taskProjection(task, 'summary')).not.toHaveProperty('stderrTail') + expect(taskProjection(task, 'detailed')).toMatchObject({ stderrTail: 'No API key found', stderrTruncated: false }) + }) + it('does not label a recovered successful turn as an error', () => { + expect(taskProjection({ ...completedTask, error: 'warning' }, 'summary')).not.toHaveProperty('error') + }) +}) diff --git a/src/tool/conversation.ts b/src/tool/conversation.ts index e57fae3c0..250d9cdd9 100644 --- a/src/tool/conversation.ts +++ b/src/tool/conversation.ts @@ -1,3 +1,4 @@ +import { headlessFailureSummary } from '../workspaces/headless-failure.js' import { tool } from 'ai' import { z } from 'zod' @@ -37,7 +38,7 @@ export function taskProjection(task: WorkspaceConversationTask, mode: 'summary' const errors = structured?.blocks .filter((block): block is Extract => block.type === 'error') .map((block) => block.message) ?? [] - const compactError = task.error ?? errors.at(-1) + const compactError = headlessFailureSummary(task) return { taskId: task.taskId, resumeId: task.resumeId, @@ -48,7 +49,12 @@ export function taskProjection(task: WorkspaceConversationTask, mode: 'summary' ...(task.parentTaskId ? { parentTaskId: task.parentTaskId } : {}), ...(task.durationMs !== undefined ? { durationMs: task.durationMs } : {}), ...(compactError ? { error: compactError } : {}), + ...(task.exitCode !== undefined ? { exitCode: task.exitCode } : {}), + ...(task.signal !== undefined ? { signal: task.signal } : {}), + ...(task.killed !== undefined ? { killed: task.killed } : {}), + ...(task.processStarted !== undefined ? { processStarted: task.processStarted } : {}), ...(mode === 'detailed' ? { + ...(task.stderrTail !== undefined ? { stderrTail: task.stderrTail, stderrTruncated: task.stderrTruncated ?? false } : {}), tools, errors, blocks: structured?.blocks ?? [], diff --git a/src/workspaces/conversation-control.spec.ts b/src/workspaces/conversation-control.spec.ts index f48d6ac71..0e354e6fe 100644 --- a/src/workspaces/conversation-control.spec.ts +++ b/src/workspaces/conversation-control.spec.ts @@ -408,6 +408,39 @@ describe('Workspace conversation control', () => { expect(dispatchHeadlessTask).not.toHaveBeenCalled() }) + it('recovers bounded stderr for a historical failure with no recorded error', async () => { + const logsDir = await mkdtemp(join(tmpdir(), 'conversation-failure-')) + dirs.push(logsDir) + const task: HeadlessTaskRecord = { + taskId: 'task-failed', resumeId: 'resume-1', wsId: 'ws-peer', agent: 'pi', + prompt: 'test', status: 'failed', startedAt: 1, exitCode: 1, processStarted: true, + } + await writeFile(headlessLogPaths(logsDir, task.taskId).stderr, + 'x'.repeat(100_000) + '\nNo API key found for the selected model\n') + const { svc } = fakeService({ task, logsDir }) + const result = await createWorkspaceConversationControl(svc).read(task.taskId) + expect(result).toMatchObject({ status: 'failed', exitCode: 1, processStarted: true, stderrTruncated: true }) + expect(result?.error).toContain('No API key found') + expect(Buffer.byteLength(result?.stderrTail ?? '')).toBeLessThanOrEqual(16 * 1024) + task.status = 'done' + const successful = await createWorkspaceConversationControl(svc).read(task.taskId) + expect(successful).not.toHaveProperty('error') + expect(successful).not.toHaveProperty('stderrTail') + }) + + it('returns the exit reason when a failed task has no log file', async () => { + const logsDir = await mkdtemp(join(tmpdir(), 'conversation-no-log-')) + dirs.push(logsDir) + const task: HeadlessTaskRecord = { + taskId: 'task-failed', resumeId: 'resume-1', wsId: 'ws-peer', agent: 'pi', + prompt: 'test', status: 'failed', startedAt: 1, signal: 'SIGKILL', exitCode: null, + } + const { svc } = fakeService({ task, logsDir }) + expect(await createWorkspaceConversationControl(svc).read(task.taskId)).toMatchObject({ + signal: 'SIGKILL', error: 'Agent process terminated by signal SIGKILL.', + }) + }) + it('reads normalized output without exposing the native runtime session id', async () => { const logsDir = await mkdtemp(join(tmpdir(), 'conversation-control-')) dirs.push(logsDir) diff --git a/src/workspaces/conversation-control.ts b/src/workspaces/conversation-control.ts index ba4ac00a6..0e79fcd13 100644 --- a/src/workspaces/conversation-control.ts +++ b/src/workspaces/conversation-control.ts @@ -1,4 +1,5 @@ import { readFile } from 'node:fs/promises' +import { headlessFailureSummary, readHeadlessStderr } from './headless-failure.js' import { readAutoPredictionPreferences, @@ -412,6 +413,10 @@ export function createWorkspaceConversationControl( const structured = await readStructuredSnapshot( headlessLogPaths(svc.headlessLogsDir, taskId).structured, ) + const stderr = task.status === 'failed' || task.status === 'interrupted' + ? await readHeadlessStderr(headlessLogPaths(svc.headlessLogsDir, taskId).stderr) + : undefined + const error = headlessFailureSummary({ ...task, structured, ...stderr }) const result: WorkspaceConversationTask = { taskId: task.taskId, resumeId: task.resumeId, @@ -424,7 +429,12 @@ export function createWorkspaceConversationControl( ...(task.trigger?.kind === 'issue' ? { issueId: task.trigger.issueId } : {}), ...(task.finishedAt !== undefined ? { finishedAt: task.finishedAt } : {}), ...(task.durationMs !== undefined ? { durationMs: task.durationMs } : {}), - ...(task.error ? { error: task.error } : {}), + ...(error ? { error } : {}), + ...(task.exitCode !== undefined ? { exitCode: task.exitCode } : {}), + ...(task.signal !== undefined ? { signal: task.signal } : {}), + ...(task.killed !== undefined ? { killed: task.killed } : {}), + ...(task.processStarted !== undefined ? { processStarted: task.processStarted } : {}), + ...stderr, } return result }, diff --git a/src/workspaces/headless-failure.spec.ts b/src/workspaces/headless-failure.spec.ts new file mode 100644 index 000000000..3c2695887 --- /dev/null +++ b/src/workspaces/headless-failure.spec.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from 'vitest' +import { headlessFailureSummary } from './headless-failure.js' +import type { HeadlessStructuredOutput } from './headless-output.js' + +const output = (blocks: HeadlessStructuredOutput['blocks']): HeadlessStructuredOutput => ({ + schemaVersion: 1, blocks, assistantText: null, + metrics: { textBlocks: 0, toolCalls: 0, toolFailures: 0 }, truncated: false, +}) + +describe('headless failure summary', () => { + it('prefers the terminal structured error to stderr warnings', () => { + expect(headlessFailureSummary({ status: 'failed', exitCode: 1, + structured: output([{ type: 'error', message: 'No API key found for the selected model' }]), + stderrTail: 'Experimental feature enabled', + })).toBe('No API key found for the selected model') + }) + it('does not report warnings or recovered errors as a successful turn failure', () => { + expect(headlessFailureSummary({ status: 'done', error: 'old error', + stderrTail: 'Experimental feature enabled', + structured: output([{ type: 'error', message: 'retrying' }, { type: 'text', text: 'Recovered' }]), + })).toBeUndefined() + }) + it('uses stderr after an earlier structured error has been recovered', () => { + expect(headlessFailureSummary({ status: 'failed', exitCode: 1, stderrTail: '\x1b[31mFatal startup error\x1b[0m\n', + structured: output([{ type: 'error', message: 'retrying' }, { type: 'text', text: 'Recovered' }]), + })).toBe('Fatal startup error') + }) + it('reports termination causes even when no diagnostic exists', () => { + expect(headlessFailureSummary({ status: 'failed', killed: true, stderrTail: 'Warning' })).toContain('timeout watchdog') + expect(headlessFailureSummary({ status: 'failed', signal: 'SIGTERM' })).toContain('SIGTERM') + expect(headlessFailureSummary({ status: 'failed', exitCode: 7 })).toContain('code 7') + expect(headlessFailureSummary({ status: 'interrupted' })).toContain('interrupted') + }) +}) diff --git a/src/workspaces/headless-failure.ts b/src/workspaces/headless-failure.ts new file mode 100644 index 000000000..6c01ec089 --- /dev/null +++ b/src/workspaces/headless-failure.ts @@ -0,0 +1,50 @@ +import { open } from 'node:fs/promises' +import { stripVTControlCharacters } from 'node:util' +import type { HeadlessStructuredOutput } from './headless-output.js' + +export interface HeadlessFailureInput { + readonly status: string + readonly structured?: HeadlessStructuredOutput | null + readonly error?: string + readonly stderrTail?: string + readonly exitCode?: number | null + readonly signal?: string | null + readonly killed?: boolean + readonly processStarted?: boolean +} + +/** Successful/recovered turns may contain stderr warnings or earlier errors. */ +export function headlessFailureSummary(input: HeadlessFailureInput): string | undefined { + if (input.status !== 'failed' && input.status !== 'interrupted') return undefined + const blocks = input.structured?.blocks ?? [] + const lastError = blocks.findLastIndex((block) => block.type === 'error') + const lastText = blocks.findLastIndex((block) => block.type === 'text') + const block = lastError > lastText ? blocks[lastError] : undefined + const structuredError = block?.type === 'error' ? block.message : undefined + const message = input.processStarted === false && input.error + ? input.error + : input.killed + ? 'Agent run was terminated by the timeout watchdog.' + : structuredError || input.error || input.stderrTail?.trim() + || (input.signal ? `Agent process terminated by signal ${input.signal}.` : undefined) + || (input.exitCode != null ? `Agent process exited with code ${input.exitCode}.` : undefined) + || (input.status === 'interrupted' ? 'Agent run was interrupted.' : 'Agent run failed without a diagnostic message.') + return stripVTControlCharacters(message).trim().slice(-2000) +} + +/** Read a bounded tail even for historical failures whose task record has no error. */ +export async function readHeadlessStderr(path: string): Promise<{ stderrTail: string; stderrTruncated: boolean } | undefined> { + const file = await open(path, 'r').catch(() => null) + if (!file) return undefined + try { + const size = (await file.stat()).size + const start = Math.max(0, size - 16 * 1024) + const buffer = Buffer.alloc(size - start) + const { bytesRead } = await file.read(buffer, 0, buffer.length, start) + return { stderrTail: stripVTControlCharacters(buffer.subarray(0, bytesRead).toString('utf8')), stderrTruncated: start > 0 } + } catch { + return undefined + } finally { + await file.close() + } +} diff --git a/src/workspaces/service.ts b/src/workspaces/service.ts index cc2a40e46..1d549a8d4 100644 --- a/src/workspaces/service.ts +++ b/src/workspaces/service.ts @@ -1,3 +1,4 @@ +import { headlessFailureSummary } from './headless-failure.js'; import { userDataHome } from '../core/paths.js'; import { resolveAliceProjectIdentity } from '@traderalice/guardian-runtime'; /** @@ -2109,6 +2110,7 @@ export async function createWorkspaceService(opts: CreateWorkspaceServiceOptions progressPublisher.offer(projectTurnProgress(r.structured)); await Promise.all([turnJournal.flush(), progressPublisher.flush()]); const status = headlessTaskStatus(r); + const failure = headlessFailureSummary({ ...r, status }); await headlessTasks.complete(rec.taskId, { status, finishedAt: Date.now(), @@ -2118,7 +2120,7 @@ export async function createWorkspaceService(opts: CreateWorkspaceServiceOptions exitCode: r.exitCode, signal: r.signal, killed: r.killed, - ...(r.error ? { error: r.error } : {}), + ...(failure ? { error: failure } : {}), output: { hasAssistantReply: r.structured.assistantText !== null, ...(r.structured.assistantText @@ -2136,21 +2138,21 @@ export async function createWorkspaceService(opts: CreateWorkspaceServiceOptions finishedAt: rec.finishedAt ?? Date.now(), assistantText: r.structured.assistantText, durationMs: r.durationMs, - ...(status !== 'done' && r.stderrTail ? { error: r.stderrTail.slice(-1000) } : {}), + ...(failure ? { error: failure } : {}), }); } if (r.processStarted === false) { await agentRuntimeLog.record('runtime.spawn_failed', { ...occupancySubject, ...(r.launchErrorCode ? { launchErrorCode: r.launchErrorCode } : {}), - ...(r.error ? { error: r.error } : {}), + ...(failure ? { error: failure } : {}), }); } else { await agentRuntimeLog.record('runtime.stopped', { ...occupancySubject, status, exitCode: r.exitCode, - ...(r.error ? { error: r.error } : {}), + ...(failure ? { error: failure } : {}), ...headlessCompletionAssets(r.structured), }); } @@ -2158,7 +2160,7 @@ export async function createWorkspaceService(opts: CreateWorkspaceServiceOptions task: rec, status, assistantText: r.structured.assistantText, - ...(status !== 'done' && r.stderrTail ? { error: r.stderrTail.slice(-1000) } : {}), + ...(failure ? { error: failure } : {}), }); await stampTelegramDeskFire(rec, r.structured.assistantText); // Scheduled one-shot issues are the only board items whose lifecycle can