From 6b68048bfc320493f9b6d5832504955e5f577de1 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Thu, 13 Aug 2026 14:03:07 +0200 Subject: [PATCH 1/6] Sequence tools after same-message patches; report tool failures to the room MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A message can carry both code patches and tool requests, and the tools routinely target the cards those patches create — a show-card for the instance a patch writes. The host ran both concurrently, so the tool could execute against a card that was still being written or indexed, and a hung execute left the spinner and the waiting ai-bot stuck forever with no trace in the room. Three layers: - The tool-processing drain requeues a message's tools while that message still has code patches pending auto-apply, bounded so stuck patches eventually fall through. - Tool execution is bounded by a timeout so a hang becomes an error. - Execution errors now post a 'failed' tool result event (the wire schema and prompt builder already understood the status) so the spinner clears everywhere and the bot can react; the UI renders the failure with a Retry action. Co-Authored-By: Claude Fable 5 --- .../components/matrix/room-message-tool.gts | 20 ++ .../app/lib/matrix-classes/message-tool.ts | 2 +- packages/host/app/services/tool-service.ts | 128 +++++++++- .../tests/acceptance/code-patches-test.gts | 227 ++++++++++++++++++ 4 files changed, 367 insertions(+), 10 deletions(-) diff --git a/packages/host/app/components/matrix/room-message-tool.gts b/packages/host/app/components/matrix/room-message-tool.gts index 15d8727476f..c7f5ee41f3c 100644 --- a/packages/host/app/components/matrix/room-message-tool.gts +++ b/packages/host/app/components/matrix/room-message-tool.gts @@ -225,6 +225,13 @@ export default class RoomMessageTool extends Component { return this.matrixService.failedToolState.get(toolRequest.id); } + // Execution failure reported through a room event (as opposed to + // failedToolState, which is this tab's in-memory state for a failure it + // produced itself). + private get failedToolCallState() { + return this.args.messageTool.status === 'failed' && !this.failedToolState; + } + private get invalidToolCallState() { return ( this.args.messageTool.status === 'invalid' && @@ -301,6 +308,19 @@ export default class RoomMessageTool extends Component { + {{else if this.failedToolCallState}} + + + + {{else if this.invalidToolCallState}} diff --git a/packages/host/app/lib/matrix-classes/message-tool.ts b/packages/host/app/lib/matrix-classes/message-tool.ts index 9540075a577..bc408b12be4 100644 --- a/packages/host/app/lib/matrix-classes/message-tool.ts +++ b/packages/host/app/lib/matrix-classes/message-tool.ts @@ -18,7 +18,7 @@ import type { Message } from './message'; import type { CardDef } from '@cardstack/base/card-api'; import type { SerializedFile } from '@cardstack/base/file-api'; -type ToolCallStatus = 'applied' | 'ready' | 'applying' | 'invalid'; +type ToolCallStatus = 'applied' | 'ready' | 'applying' | 'invalid' | 'failed'; export default class MessageTool { @tracked toolRequest: Partial; diff --git a/packages/host/app/services/tool-service.ts b/packages/host/app/services/tool-service.ts index 5c32288b622..3c33c884c17 100644 --- a/packages/host/app/services/tool-service.ts +++ b/packages/host/app/services/tool-service.ts @@ -65,6 +65,39 @@ const STUCK_PROCESSING_TIMEOUT_MS = isTesting() ? 1000 : 60_000; // result either way). Requeues are ~100ms apart (the drain debounce), so // this allows well over the normal sub-second catch-up. const MAX_TOOL_FINALIZATION_RETRIES = isTesting() ? 10 : 100; +// How many times drainToolProcessingQueue requeues a message's tools while +// that same message still has code patches pending auto-apply. Tools +// routinely target the very cards those patches create (a show-card for the +// instance a patch writes), so running them concurrently races the realm +// write/index. Requeues are ~100ms apart; on exhaustion the tools run +// anyway and the execute timeout below is the backstop. +const MAX_TOOL_PATCH_WAIT_RETRIES = isTesting() ? 20 : 600; +// Upper bound on a single tool execution. A tool awaiting a card that never +// becomes loadable would otherwise hang forever, and the result event — +// which is what un-sticks both the UI spinner and the waiting ai-bot — is +// only sent once execute settles. +const TOOL_EXECUTE_TIMEOUT_MS = isTesting() ? 3_000 : 120_000; + +// Promise.race with a cleared timer: the losing execute keeps running (we +// cannot cancel it), but the run task settles and reports. +async function withTimeout( + promise: Promise, + ms: number, + label: string, +): Promise { + let timer: ReturnType; + let timedOut = new Promise((_, reject) => { + timer = setTimeout( + () => reject(new Error(`${label} did not complete within ${ms}ms`)), + ms, + ); + }); + try { + return await Promise.race([promise, timedOut]); + } finally { + clearTimeout(timer!); + } +} type GenericCommand = Command< typeof CardDef | undefined, @@ -115,6 +148,9 @@ export default class ToolService extends Service { // How many times each queued event has been requeued waiting for the room // resource to fold the event's finalized content into its Message. private toolFinalizationRetries = new Map(); + // How many times each queued event's tools have been requeued waiting for + // that message's own code patches to finish auto-applying. + private toolPatchWaitRetries = new Map(); private codePatchProcessingEventQueue: string[] = []; private flushToolProcessingQueue: Promise | undefined; private flushCodePatchProcessingQueue: Promise | undefined; @@ -139,6 +175,7 @@ export default class ToolService extends Service { } this.toolProcessingEventQueue = []; this.toolFinalizationRetries.clear(); + this.toolPatchWaitRetries.clear(); this.codePatchProcessingEventQueue = []; this.flushToolProcessingQueue = undefined; this.flushCodePatchProcessingQueue = undefined; @@ -424,6 +461,34 @@ export default class ToolService extends Service { continue; } + // A message can carry both code patches and tool requests, and the + // tools routinely target the very cards those patches create (a + // show-card for the instance a patch writes). Running them while the + // patches are still applying races the realm write/index, so when + // this message still has patches the host is going to auto-apply + // ('act' mode), requeue the tools until those patches settle. + // Bounded: on exhaustion the tools run anyway and the execute + // timeout is the backstop. + if ( + roomResource.getActiveLLMModeForMessage(message.eventId) === 'act' && + this.messageHasUnsettledCodePatches(message) + ) { + let compoundKey = `${roomId}|${eventId}`; + let retries = this.toolPatchWaitRetries.get(compoundKey) ?? 0; + if (retries < MAX_TOOL_PATCH_WAIT_RETRIES) { + this.toolPatchWaitRetries.set(compoundKey, retries + 1); + if (!this.toolProcessingEventQueue.includes(compoundKey)) { + this.toolProcessingEventQueue.push(compoundKey); + } + debounce(this, this.drainToolProcessingQueue, 100); + continue; + } + console.error( + `Tools on event ${eventId} in room ${roomId} ran before its code patches settled (waited ${MAX_TOOL_PATCH_WAIT_RETRIES} rounds)`, + ); + } + this.toolPatchWaitRetries.delete(`${roomId}|${eventId}`); + // Collect all ready commands for this message let readyTools: any[] = []; for (let messageTool of message.tools) { @@ -795,8 +860,12 @@ export default class ToolService extends Service { ); [resultCard] = await all([ - await toolToRun.execute(typedInput as any), - await timeout(DELAY_FOR_APPLYING_UI), // leave a beat for the "applying" state of the UI to be shown + withTimeout( + toolToRun.execute(typedInput as any), + TOOL_EXECUTE_TIMEOUT_MS, + `Tool "${command.name}"`, + ), + timeout(DELAY_FOR_APPLYING_UI), // leave a beat for the "applying" state of the UI to be shown ]); } else if (command.name === 'patchCardInstance') { if (!hasPatchData(payload)) { @@ -812,13 +881,17 @@ export default class ToolService extends Service { fileUrl: `${cardId}.json`, }); - await this.store.patch( - cardId, - { - attributes: payload?.attributes?.patch?.attributes, - relationships: payload?.attributes?.patch?.relationships, - }, - { doNotWaitForPersist: true, clientRequestId }, + await withTimeout( + this.store.patch( + cardId, + { + attributes: payload?.attributes?.patch?.attributes, + relationships: payload?.attributes?.patch?.relationships, + }, + { doNotWaitForPersist: true, clientRequestId }, + ), + TOOL_EXECUTE_TIMEOUT_MS, + `Tool "${command.name}"`, ); } else { // Unrecognized tool. This can happen if a programmatically-provided @@ -852,6 +925,24 @@ export default class ToolService extends Service { console.error(error); await timeout(DELAY_FOR_APPLYING_UI); // leave a beat for the "applying" state of the UI to be shown this.matrixService.failedToolState.set(commandRequestId!, error); + // Report the failure to the room: the result event is what clears the + // UI spinner in other sessions and lets ai-bot react to the failure + // instead of waiting forever. The local failedToolState above still + // drives this tab's immediate Retry affordance. + try { + await this.matrixService.sendToolResultEvent({ + roomId: command.message.roomId, + invokedToolFromEventId: eventId, + toolCallId: commandRequestId!, + status: 'failed', + failureReason: error.message, + }); + } catch (sendError) { + console.error( + 'could not send failed tool result event to the room', + sendError, + ); + } } finally { this.currentlyExecutingToolRequestIds.delete(commandRequestId!); } @@ -1078,6 +1169,25 @@ export default class ToolService extends Service { } }; + // True while any code patch in the message has not reached a terminal + // state ('applied' or 'failed') — i.e. it is still 'ready' (queued for + // auto-apply) or 'applying'. + private messageHasUnsettledCodePatches(message: { + htmlParts?: Array<{ codeData: CodeData | null }> | null; + }): boolean { + if (!message.htmlParts) { + return false; + } + return message.htmlParts.some((part) => { + let codeData = part.codeData; + if (!codeData?.searchReplaceBlock) { + return false; + } + let status = this.getCodePatchStatus(codeData); + return status === 'ready' || status === 'applying'; + }); + } + getReadyCodePatches = ( htmlParts: Array<{ codeData: CodeData | null }>, ): CodeData[] => { diff --git a/packages/host/tests/acceptance/code-patches-test.gts b/packages/host/tests/acceptance/code-patches-test.gts index 47761810d93..4c97e5cdbfa 100644 --- a/packages/host/tests/acceptance/code-patches-test.gts +++ b/packages/host/tests/acceptance/code-patches-test.gts @@ -25,6 +25,7 @@ import { APP_BOXEL_MESSAGE_MSGTYPE, APP_BOXEL_DEBUG_MESSAGE_EVENT_TYPE, APP_BOXEL_TOOL_REQUESTS_KEY, + APP_BOXEL_TOOL_RESULT_EVENT_TYPE, APP_BOXEL_LLM_MODE, APP_BOXEL_CONTINUATION_OF_CONTENT_KEY, APP_BOXEL_HAS_CONTINUATION_CONTENT_KEY, @@ -2051,4 +2052,230 @@ ${REPLACE_MARKER}\n\`\`\``; 'code patch result event is dispatched', ); }); + + test("tools on a message auto-run only after that message's code patches settle", async function (assert) { + await visitOperatorMode({ + submode: 'code', + codePath: `${testRealmURL}hello.txt`, + }); + await click('[data-test-open-ai-assistant]'); + let roomId = getRoomIds().pop()!; + + await click('[data-test-llm-mode-option="act"]'); + + // One message carrying both a code patch and a tool request — the + // shape that used to race: the tool executed while the patch was + // still being applied. The spy on store.patch records the patch's + // status at the exact moment the tool executes. + let store = getService('store'); + let toolService = getService('tool-service'); + let statusWhenToolRan: string | undefined; + let originalPatch = store.patch.bind(store); + let eventId: string; + store.patch = (...args: Parameters) => { + statusWhenToolRan = toolService.getCodePatchStatus({ + roomId, + eventId, + codeBlockIndex: 0, + }); + return originalPatch(...args); + }; + + let codeBlock = `\`\`\` +http://test-realm/test/hello.txt +${SEARCH_MARKER} +Hello, world! +${SEPARATOR_MARKER} +Sequenced, world! +${REPLACE_MARKER} +\`\`\``; + + eventId = simulateRemoteMessage(roomId, '@aibot:localhost', { + body: codeBlock, + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + isStreamingFinished: true, + [APP_BOXEL_TOOL_REQUESTS_KEY]: [ + { + id: 'tool-after-patches', + name: 'patchCardInstance', + arguments: JSON.stringify({ + attributes: { + cardId: `${testRealmURL}index`, + patch: { attributes: {} }, + }, + }), + }, + ], + data: { + context: { + agentId: getService('matrix-service').agentId, + }, + }, + }); + + await waitFor( + '[data-test-message-idx="0"] [data-test-apply-state="applied"]', + ); + await waitUntil( + () => + getRoomEvents(roomId).some( + (event) => + event.type === APP_BOXEL_TOOL_RESULT_EVENT_TYPE && + event.content['m.relates_to']?.key === 'applied' && + event.content.commandRequestId === 'tool-after-patches', + ), + { timeout: 5000 }, + ); + + let events = getRoomEvents(roomId); + let patchResultIndex = events.findIndex( + (event) => + event.type === APP_BOXEL_CODE_PATCH_RESULT_EVENT_TYPE && + event.content['m.relates_to']?.key === 'applied', + ); + let toolResultIndex = events.findIndex( + (event) => + event.type === APP_BOXEL_TOOL_RESULT_EVENT_TYPE && + event.content['m.relates_to']?.key === 'applied' && + event.content.commandRequestId === 'tool-after-patches', + ); + assert.true(patchResultIndex >= 0, 'code patch result event exists'); + assert.true(toolResultIndex >= 0, 'tool result event exists'); + assert.true( + patchResultIndex < toolResultIndex, + `code patch settles before the tool runs (patch result at ${patchResultIndex}, tool result at ${toolResultIndex})`, + ); + assert.strictEqual( + statusWhenToolRan, + 'applied', + 'at the moment the tool executed, the patch had already settled', + ); + }); + + test('a tool whose execution fails posts a failed tool result event', async function (assert) { + await visitOperatorMode({ + submode: 'code', + codePath: `${testRealmURL}hello.txt`, + }); + await click('[data-test-open-ai-assistant]'); + let roomId = getRoomIds().pop()!; + + // Deterministic execution failure, independent of how the store treats + // an unknown card id. + let store = getService('store'); + store.patch = () => Promise.reject(new Error('patch exploded')); + + simulateRemoteMessage(roomId, '@aibot:localhost', { + body: 'Patching a card via a store that rejects', + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + isStreamingFinished: true, + [APP_BOXEL_TOOL_REQUESTS_KEY]: [ + { + id: 'tool-that-fails', + name: 'patchCardInstance', + arguments: JSON.stringify({ + attributes: { + cardId: `${testRealmURL}index`, + patch: { attributes: { name: 'x' } }, + }, + }), + }, + ], + data: { + context: { + agentId: getService('matrix-service').agentId, + }, + }, + }); + + await waitFor('[data-test-message-idx="0"] [data-test-tool-call-apply]'); + await click('[data-test-message-idx="0"] [data-test-tool-call-apply]'); + + await waitUntil( + () => + getRoomEvents(roomId).some( + (event) => + event.type === APP_BOXEL_TOOL_RESULT_EVENT_TYPE && + event.content['m.relates_to']?.key === 'failed' && + event.content.commandRequestId === 'tool-that-fails', + ), + { timeout: 5000 }, + ); + + let failedEvent = getRoomEvents(roomId).find( + (event) => + event.type === APP_BOXEL_TOOL_RESULT_EVENT_TYPE && + event.content['m.relates_to']?.key === 'failed', + ); + assert.ok( + failedEvent?.content.failureReason, + 'failed tool result event carries the failure reason', + ); + }); + + test('a tool whose execution hangs times out and posts a failed tool result event', async function (assert) { + await visitOperatorMode({ + submode: 'code', + codePath: `${testRealmURL}hello.txt`, + }); + await click('[data-test-open-ai-assistant]'); + let roomId = getRoomIds().pop()!; + + // Simulate the hang observed in the field: an execute that never + // settles (e.g. loading a card that never becomes available). + let store = getService('store'); + store.patch = () => new Promise(() => {}); + + simulateRemoteMessage(roomId, '@aibot:localhost', { + body: 'Patching via a store that never answers', + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + isStreamingFinished: true, + [APP_BOXEL_TOOL_REQUESTS_KEY]: [ + { + id: 'tool-that-hangs', + name: 'patchCardInstance', + arguments: JSON.stringify({ + attributes: { + cardId: `${testRealmURL}index`, + patch: { attributes: {} }, + }, + }), + }, + ], + data: { + context: { + agentId: getService('matrix-service').agentId, + }, + }, + }); + + await waitFor('[data-test-message-idx="0"] [data-test-tool-call-apply]'); + await click('[data-test-message-idx="0"] [data-test-tool-call-apply]'); + + // The test timeout for tool execution is 3s; the failed result must + // land on its own once it elapses. + await waitUntil( + () => + getRoomEvents(roomId).some( + (event) => + event.type === APP_BOXEL_TOOL_RESULT_EVENT_TYPE && + event.content['m.relates_to']?.key === 'failed' && + event.content.commandRequestId === 'tool-that-hangs', + ), + { timeout: 10_000 }, + ); + + let failedEvent = getRoomEvents(roomId).find( + (event) => + event.type === APP_BOXEL_TOOL_RESULT_EVENT_TYPE && + event.content['m.relates_to']?.key === 'failed', + ); + assert.ok( + failedEvent?.content.failureReason?.includes('did not complete'), + 'failure reason reports the timeout', + ); + }); }); From 1354baaaf70c87fac6aad7b5a3903501f4adc5df Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Thu, 13 Aug 2026 14:18:34 +0200 Subject: [PATCH 2/6] Type the store.patch spy against the bound original Co-Authored-By: Claude Fable 5 --- packages/host/tests/acceptance/code-patches-test.gts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/host/tests/acceptance/code-patches-test.gts b/packages/host/tests/acceptance/code-patches-test.gts index 4c97e5cdbfa..5f221b45c81 100644 --- a/packages/host/tests/acceptance/code-patches-test.gts +++ b/packages/host/tests/acceptance/code-patches-test.gts @@ -2072,14 +2072,14 @@ ${REPLACE_MARKER}\n\`\`\``; let statusWhenToolRan: string | undefined; let originalPatch = store.patch.bind(store); let eventId: string; - store.patch = (...args: Parameters) => { + store.patch = ((...args: Parameters) => { statusWhenToolRan = toolService.getCodePatchStatus({ roomId, eventId, codeBlockIndex: 0, }); return originalPatch(...args); - }; + }) as typeof store.patch; let codeBlock = `\`\`\` http://test-realm/test/hello.txt From 9b389e76267a573377352d43907636c51b8974d1 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Thu, 13 Aug 2026 14:43:40 +0200 Subject: [PATCH 3/6] Catch per-tool drain failures and report them to the room MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A throw inside validate() — module or input-schema loads against a busy realm — killed the whole drain pass silently: the request stayed claimed forever, its spinner never cleared, and the bot waited forever. Each tool's validation now fails alone, posting a failed result event with the reason. The errors-allow-retry test moves to the new contract: an execution failure dispatches a failed result event instead of nothing. Co-Authored-By: Claude Fable 5 --- packages/host/app/services/tool-service.ts | 36 ++++++++++- .../tests/acceptance/code-patches-test.gts | 63 +++++++++++++++++++ packages/host/tests/acceptance/tools-test.gts | 32 +++++++--- 3 files changed, 122 insertions(+), 9 deletions(-) diff --git a/packages/host/app/services/tool-service.ts b/packages/host/app/services/tool-service.ts index 3c33c884c17..9fb0cfe35f5 100644 --- a/packages/host/app/services/tool-service.ts +++ b/packages/host/app/services/tool-service.ts @@ -525,7 +525,41 @@ export default class ToolService extends Service { this.claimedToolRequestIds.add(messageTool.id); } - let isValid = await this.validate(messageTool); + // validate() loads the tool's module and input schema over the + // loader; against a realm that is busy (e.g. indexing files this + // same message just created) that can throw. Without this catch a + // single throw killed the whole drain pass silently: the request + // stayed claimed forever, its spinner never cleared, and the bot + // waited forever. Report it as a failed result instead. + let isValid = false; + try { + isValid = await this.validate(messageTool); + } catch (e) { + let reason = e instanceof Error ? e.message : String(e); + console.error( + `Tool processing failed for "${messageTool.name}" (${messageTool.id}):`, + e, + ); + try { + await this.matrixService.sendToolResultEvent({ + roomId: roomId!, + invokedToolFromEventId: + this.getCurrentEventIdForCommandRequest( + roomId!, + messageTool.id, + ) ?? messageTool.eventId, + toolCallId: messageTool.id!, + status: 'failed', + failureReason: reason, + }); + } catch (sendError) { + console.error( + 'could not send failed tool result event to the room', + sendError, + ); + } + continue; + } if (!isValid) { continue; } diff --git a/packages/host/tests/acceptance/code-patches-test.gts b/packages/host/tests/acceptance/code-patches-test.gts index 5f221b45c81..3920bff4469 100644 --- a/packages/host/tests/acceptance/code-patches-test.gts +++ b/packages/host/tests/acceptance/code-patches-test.gts @@ -2215,6 +2215,69 @@ ${REPLACE_MARKER} ); }); + test('a tool whose validation throws posts a failed tool result event instead of killing the drain', async function (assert) { + await visitOperatorMode({ + submode: 'code', + codePath: `${testRealmURL}hello.txt`, + }); + await click('[data-test-open-ai-assistant]'); + let roomId = getRoomIds().pop()!; + + await click('[data-test-llm-mode-option="act"]'); + + // Simulate the field failure: validate() loads the tool's module and + // input schema over the loader, which can throw against a busy realm. + let toolService = getService('tool-service'); + toolService.validate = () => { + throw new Error('loader exploded'); + }; + + simulateRemoteMessage(roomId, '@aibot:localhost', { + body: 'Running a tool whose validation throws', + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + isStreamingFinished: true, + [APP_BOXEL_TOOL_REQUESTS_KEY]: [ + { + id: 'tool-validate-throws', + name: 'patchCardInstance', + arguments: JSON.stringify({ + attributes: { + cardId: `${testRealmURL}index`, + patch: { attributes: {} }, + }, + }), + }, + ], + data: { + context: { + agentId: getService('matrix-service').agentId, + }, + }, + }); + + await waitUntil( + () => + getRoomEvents(roomId).some( + (event) => + event.type === APP_BOXEL_TOOL_RESULT_EVENT_TYPE && + event.content['m.relates_to']?.key === 'failed' && + event.content.commandRequestId === 'tool-validate-throws', + ), + { timeout: 5000 }, + ); + + let failedEvent = getRoomEvents(roomId).find( + (event) => + event.type === APP_BOXEL_TOOL_RESULT_EVENT_TYPE && + event.content['m.relates_to']?.key === 'failed', + ); + assert.ok( + failedEvent?.content.failureReason?.includes('loader exploded'), + 'failure reason carries the thrown error', + ); + }); + test('a tool whose execution hangs times out and posts a failed tool result event', async function (assert) { await visitOperatorMode({ submode: 'code', diff --git a/packages/host/tests/acceptance/tools-test.gts b/packages/host/tests/acceptance/tools-test.gts index c4cd411c7bf..d18c80322e3 100644 --- a/packages/host/tests/acceptance/tools-test.gts +++ b/packages/host/tests/acceptance/tools-test.gts @@ -1957,23 +1957,39 @@ module('Acceptance | Tools tests', function (hooks) { }); await settled(); - let commandResultEvents = await getRoomEvents(roomId).filter( - (event) => event.type === APP_BOXEL_TOOL_RESULT_EVENT_TYPE, + // The failure is reported to the room so the bot can react and other + // sessions' spinners clear. + let failedResultEvents = await getRoomEvents(roomId).filter( + (event) => + event.type === APP_BOXEL_TOOL_RESULT_EVENT_TYPE && + event.content['m.relates_to']?.key === 'failed', ); assert.strictEqual( - commandResultEvents.length, + failedResultEvents.length, + 1, + 'failed command result event dispatched', + ); + let appliedResultEvents = await getRoomEvents(roomId).filter( + (event) => + event.type === APP_BOXEL_TOOL_RESULT_EVENT_TYPE && + event.content['m.relates_to']?.key === 'applied', + ); + assert.strictEqual( + appliedResultEvents.length, 0, - 'No command result event dispatched', + 'no applied command result event dispatched', ); maybeBoomShouldBoom = false; await click('[data-test-alert-action-button="Retry"]'); - commandResultEvents = await getRoomEvents(roomId).filter( - (event) => event.type === APP_BOXEL_TOOL_RESULT_EVENT_TYPE, + appliedResultEvents = await getRoomEvents(roomId).filter( + (event) => + event.type === APP_BOXEL_TOOL_RESULT_EVENT_TYPE && + event.content['m.relates_to']?.key === 'applied', ); assert.strictEqual( - commandResultEvents.length, + appliedResultEvents.length, 1, - 'Command result event was dispatched', + 'applied command result event was dispatched after retry', ); assert.dom('[data-test-apply-state="applied"]').exists(); }); From 6172636af6af749f11b02cfb09a27764926fefc1 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Thu, 13 Aug 2026 16:18:09 +0200 Subject: [PATCH 4/6] Wait for patched files' index invalidations before running same-message tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An applied patch means the write landed, not that the index has caught up: show-card on a just-created card still failed with not-found after the patches settled. The drain now awaits the tracked incremental-index invalidations of the message's patched files — the same milestone checkCorrectness waits on — before dispatching the tools. Co-Authored-By: Claude Fable 5 --- packages/host/app/services/tool-service.ts | 29 ++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/packages/host/app/services/tool-service.ts b/packages/host/app/services/tool-service.ts index 9fb0cfe35f5..04a4dac5538 100644 --- a/packages/host/app/services/tool-service.ts +++ b/packages/host/app/services/tool-service.ts @@ -28,6 +28,7 @@ import { AI_BOT_EXECUTOR } from '@cardstack/runtime-common/commands'; import { basicMappings } from '@cardstack/runtime-common/helpers/ai'; import { getToolRequests } from '@cardstack/runtime-common/matrix-constants'; +import ENV from '@cardstack/host/config/environment'; import type MatrixService from '@cardstack/host/services/matrix-service'; import type Realm from '@cardstack/host/services/realm'; import CheckCorrectnessTool from '@cardstack/host/tools/check-correctness'; @@ -489,6 +490,20 @@ export default class ToolService extends Service { } this.toolPatchWaitRetries.delete(`${roomId}|${eventId}`); + // Applied patches mean the write landed, not that the index has + // caught up — a tool loading a just-created card would still miss + // it. Wait for the tracked index invalidations of this message's + // patched files, the same milestone checkCorrectness waits on. A + // no-op when nothing was tracked (e.g. the patches were applied by + // another session). + for (let fileUrl of this.patchedFileUrls(message)) { + await this.waitForInvalidationAfterAIAssistantRequest( + roomId!, + fileUrl, + ENV.cardRenderTimeout, + ); + } + // Collect all ready commands for this message let readyTools: any[] = []; for (let messageTool of message.tools) { @@ -1203,6 +1218,20 @@ export default class ToolService extends Service { } }; + // The distinct file URLs this message's code patches target. + private patchedFileUrls(message: { + htmlParts?: Array<{ codeData: CodeData | null }> | null; + }): string[] { + let urls = new Set(); + for (let part of message.htmlParts ?? []) { + let codeData = part.codeData; + if (codeData?.searchReplaceBlock && codeData.fileUrl) { + urls.add(codeData.fileUrl); + } + } + return [...urls]; + } + // True while any code patch in the message has not reached a terminal // state ('applied' or 'failed') — i.e. it is still 'ready' (queued for // auto-apply) or 'applying'. From 4a9c7951069f231fc624a40cfacebe76ccec340d Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Fri, 14 Aug 2026 13:10:48 +0200 Subject: [PATCH 5/6] Harden failed-tool handling around retries, reloads, and index waits MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A tool's 'failed' result event is terminal for auto-execution but not for the user: a Retry can succeed afterward, leaving two result events for one request. Result consumers (prompt assembly, message building) now take the latest result instead of the first, so a successful retry supersedes the stale failure. The drain and the stuck-processing invalidator treat 'failed' as terminal so a reload no longer re-runs a failed tool unattended, and failed result events now carry the operator-mode context the bot's agent routing reads. The drain's index-invalidation wait no longer consumes the one-shot waiter on timeout (checkCorrectness still needs it), resolves collision-renamed files through a redirect map, runs per-file waits in parallel, and is skipped when the message has no runnable tools. Code patch blocks with no resolvable file URL no longer count as unsettled — they are never applied, so they stalled the message's tools for the whole retry budget. The execute timeout now brackets module resolution and input construction too (both hang the same way a slow execute does), checkCorrectness gets headroom for its two legitimate index-wait windows, and a failed result-send in the validate path records the local failed state so the Retry affordance survives. A room-reported failure now also gets the is-failed styling. Co-Authored-By: Claude Fable 5 --- .../ai-bot/tests/prompt-construction-test.ts | 109 ++++++++ .../components/matrix/room-message-tool.gts | 6 +- .../app/lib/matrix-classes/message-builder.ts | 38 ++- packages/host/app/services/tool-service.ts | 256 ++++++++++++------ packages/runtime-common/ai/prompt.ts | 11 +- 5 files changed, 325 insertions(+), 95 deletions(-) diff --git a/packages/ai-bot/tests/prompt-construction-test.ts b/packages/ai-bot/tests/prompt-construction-test.ts index 12a81443fb2..5cc83406b63 100644 --- a/packages/ai-bot/tests/prompt-construction-test.ts +++ b/packages/ai-bot/tests/prompt-construction-test.ts @@ -2763,6 +2763,115 @@ Attached Files (files with newer versions don't show their content): assert.equal(messageText(result[5]).trim(), expected.trim()); }); + test('a later applied result supersedes an earlier failed result for the same request', async () => { + // A tool can fail and then succeed on a user Retry; both result events + // stay in the room forever. The prompt must reflect the latest one, or + // the model is permanently told the call failed and may re-issue it. + const history: DiscreteMatrixEvent[] = [ + { + type: 'm.room.message', + room_id: 'room-id-1', + sender: '@user:localhost', + content: { + body: 'set the title', + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + data: { context: { tools: [], functions: [] } }, + }, + origin_server_ts: 1722242847000, + unsigned: { age: 1000, transaction_id: 't0' }, + event_id: 'user-event-id-1', + status: EventStatus.SENT, + }, + { + type: 'm.room.message', + room_id: 'room-id-1', + sender: '@aibot:localhost', + content: { + body: 'Setting the title', + msgtype: APP_BOXEL_MESSAGE_MSGTYPE, + format: 'org.matrix.custom.html', + data: { context: { functions: [] } }, + [APP_BOXEL_TOOL_REQUESTS_KEY]: [ + { + id: 'retried-tool-call-id-1', + name: 'patchCardInstance', + arguments: JSON.stringify({ + attributes: { description: 'Set the title' }, + }), + }, + ], + }, + origin_server_ts: 1722242849000, + unsigned: { age: 900, transaction_id: 't1' }, + event_id: 'retried-command-event-id-1', + status: EventStatus.SENT, + }, + { + type: APP_BOXEL_TOOL_RESULT_EVENT_TYPE, + room_id: 'room-id-1', + sender: '@user:localhost', + content: { + 'm.relates_to': { + event_id: 'retried-command-event-id-1', + rel_type: APP_BOXEL_TOOL_RESULT_REL_TYPE, + key: 'failed', + }, + msgtype: APP_BOXEL_TOOL_RESULT_WITH_NO_OUTPUT_MSGTYPE, + commandRequestId: 'retried-tool-call-id-1', + failureReason: 'store exploded', + data: { context: { tools: [], functions: [] } }, + }, + origin_server_ts: 1722242853000, + unsigned: { age: 800, transaction_id: 't2' }, + event_id: 'failed-result-id-1', + status: EventStatus.SENT, + }, + { + type: APP_BOXEL_TOOL_RESULT_EVENT_TYPE, + room_id: 'room-id-1', + sender: '@user:localhost', + content: { + 'm.relates_to': { + event_id: 'retried-command-event-id-1', + rel_type: APP_BOXEL_TOOL_RESULT_REL_TYPE, + key: 'applied', + }, + msgtype: APP_BOXEL_TOOL_RESULT_WITH_NO_OUTPUT_MSGTYPE, + commandRequestId: 'retried-tool-call-id-1', + data: { context: { tools: [], functions: [] } }, + }, + origin_server_ts: 1722242857000, + unsigned: { age: 700, transaction_id: 't3' }, + event_id: 'applied-result-id-1', + status: EventStatus.SENT, + }, + ]; + const result = await buildPromptForModel( + history, + '@aibot:localhost', + [], + [], + [], + fakeMatrixClient, + ); + let toolMessages = result.filter((m) => m.role === 'tool'); + assert.equal(toolMessages.length, 1, 'one tool message per request'); + assert.equal( + (toolMessages[0] as { tool_call_id?: string }).tool_call_id, + 'retried-tool-call-id-1', + ); + let content = messageText(toolMessages[0]); + assert.true( + content.includes('executed'), + `the retried call reads as executed, got: ${content}`, + ); + assert.false( + content.includes('failed'), + `no stale failure text survives the retry, got: ${content}`, + ); + }); + test('pairs a pre-rename request/result (legacy wire keys) with the same tool_call_id', async () => { // A room whose history predates the command → tool rename replays events // with the legacy spellings forever; prompt assembly must pair them diff --git a/packages/host/app/components/matrix/room-message-tool.gts b/packages/host/app/components/matrix/room-message-tool.gts index c7f5ee41f3c..98c011438ad 100644 --- a/packages/host/app/components/matrix/room-message-tool.gts +++ b/packages/host/app/components/matrix/room-message-tool.gts @@ -244,7 +244,11 @@ export default class RoomMessageTool extends Component { } private get hasFailedState() { - return !!(this.failedToolState || this.didFailCorrectnessCheck); + return !!( + this.failedToolState || + this.failedToolCallState || + this.didFailCorrectnessCheck + ); }