diff --git a/packages/runtime/src/__tests__/computer-use-tools.test.ts b/packages/runtime/src/__tests__/computer-use-tools.test.ts index 09bb2989ef..badb671c63 100644 --- a/packages/runtime/src/__tests__/computer-use-tools.test.ts +++ b/packages/runtime/src/__tests__/computer-use-tools.test.ts @@ -19,11 +19,12 @@ import assert from 'node:assert/strict'; import { describe, test } from 'node:test'; -import type { CuAction } from '@maka/core/computer-use'; +import { CU_TOOL_ACTION_TYPES, type CuAction } from '@maka/core/computer-use'; import { zodSchema } from 'ai'; import { adaptToCuAction, buildComputerUseTools, + COMPUTER_USE_MODEL_SCREENSHOT_POLICY, snapshotComputerParams, type CuDispatchBackend, type CuObservation, @@ -2609,6 +2610,59 @@ describe('buildComputerUseTools — the `maka_computer` MakaTool', () => { assert.doesNotMatch(output.value[0]?.text ?? '', /super-secret-value/); }); + test('keeps PiP-only screenshots out of semantic action model output', () => { + const [tool] = buildComputerUseTools({ backend: fakeBackend() }); + const output = (includeScreenshotInModelOutput: boolean) => ({ + text: 'persisted result', + modelText: 'fresh semantic observation', + screenshot: { base64: 'AA==', mimeType: 'image/png' }, + includeScreenshotInModelOutput, + }); + const project = (includeScreenshotInModelOutput: boolean) => + tool.toModelOutput?.({ + toolCallId: 'tool-1', + input: {}, + output: output(includeScreenshotInModelOutput), + }) as { value: Array<{ type: string }> }; + + assert.deepEqual(project(false).value, [{ type: 'text', text: 'fresh semantic observation' }]); + assert.equal(project(true).value[1]?.type, 'file'); + }); + + test('classifies every canonical action for model-visible screenshots', () => { + assert.deepEqual(Object.keys(COMPUTER_USE_MODEL_SCREENSHOT_POLICY), [...CU_TOOL_ACTION_TYPES]); + }); + + test('binds model-visible screenshots to the immutable executed invocation', async () => { + const projectAfterMutation = async ( + includeScreenshot: boolean, + mutatedIncludeScreenshot: boolean, + ) => { + const backend = fakeBackend() as CuDispatchBackend & { + observeApp: NonNullable; + }; + backend.observeApp = async () => observation(); + const [tool] = buildComputerUseTools({ backend }); + const input = { + action: 'observe' as const, + app: 'Fixture', + include_screenshot: includeScreenshot, + }; + const output = await tool.impl(input, ctx()); + input.include_screenshot = mutatedIncludeScreenshot; + return tool.toModelOutput?.({ + toolCallId: 'tool-1', + input, + output, + }) as { value: Array<{ type: string }> }; + }; + + assert.equal((await projectAfterMutation(true, false)).value[1]?.type, 'file'); + const nonVisual = await projectAfterMutation(false, true); + assert.equal(nonVisual.value.length, 1); + assert.equal(nonVisual.value[0]?.type, 'text'); + }); + test('S18: an already-aborted signal short-circuits before any dispatch', async () => { const ac = new AbortController(); ac.abort(); diff --git a/packages/runtime/src/computer-use-tools.ts b/packages/runtime/src/computer-use-tools.ts index aad8286e0f..6784476e1e 100644 --- a/packages/runtime/src/computer-use-tools.ts +++ b/packages/runtime/src/computer-use-tools.ts @@ -35,6 +35,7 @@ import { isCuObservingAction, type CuAction, type CuPoint, + type CuToolActionType, type ComputerUseErrorCode, type ComputerUseWindowIdentity, } from '@maka/core/computer-use'; @@ -330,8 +331,9 @@ export const computerWireParams = z * Raw result of the `computer` tool. `text` is the S16-safe summary the runtime * records to session history (via coerceResultContent's text-only projection: * this object has no `kind`, so only `text` survives). `screenshot`, when - * present, rides along ONLY to feed `toModelOutput` — it never enters `text`, so - * the bounded frame base64 stays out of session history. + * present, feeds the local presentation layer and, only for explicitly visual + * actions, `toModelOutput`. It never enters `text`, so the bounded frame base64 + * stays out of session history. */ interface ComputerToolResult { text: string; @@ -339,6 +341,46 @@ interface ComputerToolResult { error?: ComputerUseErrorCode; failureClass?: 'ambiguous_target'; screenshot?: { base64: string; mimeType: string }; + includeScreenshotInModelOutput?: boolean; +} + +export const COMPUTER_USE_MODEL_SCREENSHOT_POLICY = { + list_apps: 'never', + launch_app: 'never', + observe: 'explicit', + click_element: 'never', + set_value: 'never', + select_text: 'never', + secondary_action: 'never', + scroll_element: 'never', + window_action: 'never', + element_sequence: 'never', + press_key: 'never', + screenshot: 'always', + cursor_position: 'never', + mouse_move: 'always', + left_click: 'always', + right_click: 'always', + middle_click: 'always', + double_click: 'always', + triple_click: 'always', + left_mouse_down: 'always', + left_mouse_up: 'always', + left_click_drag: 'always', + type: 'always', + key: 'always', + hold_key: 'always', + scroll: 'always', + wait: 'never', + zoom: 'always', +} as const satisfies Record; + +function shouldSendScreenshotToModel(input: ComputerParams): boolean { + const policy = COMPUTER_USE_MODEL_SCREENSHOT_POLICY[input.action]; + return ( + policy === 'always' || + (policy === 'explicit' && input.action === 'observe' && input.include_screenshot === true) + ); } export interface ComputerUseToolSet extends Array { @@ -984,6 +1026,7 @@ export function buildComputerUseTools(deps: { function deliveredWithoutFreshObservation( action: ComputerSummaryAction, result: CuRunResult, + includeScreenshotInModelOutput = false, ): ComputerToolResult { const evidence = summarizeEvidence(result.outcome.evidence); const hostEvidence = summarizeEvidence(result.outcome.evidence, 'host'); @@ -1008,6 +1051,7 @@ export function buildComputerUseTools(deps: { base64: screenshot.base64, mimeType: screenshot.mimeType, }, + includeScreenshotInModelOutput, } : {}), }; @@ -1596,6 +1640,7 @@ export function buildComputerUseTools(deps: { ): Promise => { if (abortSignal.aborted) return { text: 'computer aborted before start' }; const input = snapshotComputerParams(computerParams.parse(args)); + const includeScreenshotInModelOutput = shouldSendScreenshotToModel(input); // Before anything is claimed against a frame or dispatched: an argument // holding one of this host's own withheld-value placeholders is a replay // of the record, not a value, and every path below would have typed it. @@ -1604,7 +1649,7 @@ export function buildComputerUseTools(deps: { const invocationGeneration = presentationGenerations.get(sessionId) ?? 0; const releasePendingInvocation = trackPendingInvocation(sessionId, turnId); try { - return await withInvocationQueue(sessionId, abortSignal, async () => { + return await withInvocationQueue(sessionId, abortSignal, async () => { if ((presentationGenerations.get(sessionId) ?? 0) !== invocationGeneration) { return sessionFailure('user_stopped'); } @@ -2275,6 +2320,7 @@ export function buildComputerUseTools(deps: { text: persistedObservationText(observation), modelText: observationText({ ...observation, screenshot }), screenshot: { base64: screenshot.base64, mimeType: screenshot.mimeType }, + includeScreenshotInModelOutput, } : { text: persistedObservationText(observation), @@ -2340,6 +2386,7 @@ export function buildComputerUseTools(deps: { base64: screenshotObservation.screenshot.base64, mimeType: screenshotObservation.screenshot.mimeType, }, + includeScreenshotInModelOutput, }; } if ( @@ -2711,11 +2758,11 @@ export function buildComputerUseTools(deps: { state.reobserveRequired(); } } - // Carry the screenshot base64 on the raw result (which becomes the ai-sdk - // tool `output`) so `toModelOutput` below can hand the vision model an image - // block. Kept OFF `text`: coerceResultContent projects this object to a - // text-only session-log entry (no `kind` ⇒ only `text` survives), so the - // bounded frame never bloats history. + // Carry the screenshot base64 on the raw result for the local mirror. + // `toModelOutput` below sends it to the provider only for actions that + // explicitly need pixels. Kept OFF `text`: coerceResultContent projects + // this object to a text-only session-log entry (no `kind` => only `text` + // survives), so the bounded frame never bloats durable history. let bindingResult: BindingFailureReason | undefined; if (boundAction) bindingResult = consumeBoundAction(record, boundAction); if (bindingResult && !hasUncertainDeliveredOutcome(result)) { @@ -2741,11 +2788,19 @@ export function buildComputerUseTools(deps: { : undefined; } catch { presentation?.finish(result); - return deliveredWithoutFreshObservation(modelAction, result); + return deliveredWithoutFreshObservation( + modelAction, + result, + includeScreenshotInModelOutput, + ); } if (actionLease && result.outcome.ok && !freshObservation) { presentation?.finish(result); - return deliveredWithoutFreshObservation(modelAction, result); + return deliveredWithoutFreshObservation( + modelAction, + result, + includeScreenshotInModelOutput, + ); } presentation?.finish(withMirrorFrame(result, freshObservation)); const modelRefresh = freshObservation @@ -2772,6 +2827,7 @@ export function buildComputerUseTools(deps: { ...(!result.outcome.ok ? { error: result.outcome.error } : {}), ...(failureClass ? { failureClass } : {}), screenshot: { base64: screenshot.base64, mimeType: screenshot.mimeType }, + includeScreenshotInModelOutput, } : { text, @@ -2785,10 +2841,10 @@ export function buildComputerUseTools(deps: { releasePendingInvocation(); } }, - // Map the raw result into model-visible content: the summary as text, plus the - // screenshot as a native file block when present. Robust to the runtime's synthetic - // failure return shape ({ error }) from permission/loop-gate blocks, which - // reaches here as `output` too. + // Map the raw result into model-visible content. Semantic actions already + // return a fresh accessibility observation, so their automatically captured + // PiP frame stays local. Explicit visual requests and legacy coordinate + // actions still receive the native image block. toModelOutput: ({ output }) => { const o = (output ?? {}) as Partial & { error?: unknown }; const text = @@ -2803,7 +2859,7 @@ export function buildComputerUseTools(deps: { type: 'content', value: [ { type: 'text', text }, - ...(o.screenshot + ...(o.screenshot && o.includeScreenshotInModelOutput === true ? [ { type: 'file' as const,