From 0ae799e2cf8f11bf095de3497db864698c075a96 Mon Sep 17 00:00:00 2001 From: Matic Jurglic Date: Fri, 14 Aug 2026 14:57:35 +0200 Subject: [PATCH] Keep prompt history byte-stable so the cache prefix survives MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Prompt caching is an exact prefix match, but attachment rendering rewrote already-sent history: a message's attached card/file content was dropped retroactively once a newer version was attached later, and read-file tool results lost their content the same way. Every rewrite re-billed the whole prompt after the change point at full input price on every later turn — in observed sessions 40-60% of the total cost. A message's attachments now render from its own snapshot alone, with the attachment headers telling the model that later attachments of the same card/file supersede earlier ones. Carrying the superseded content forward costs cached-read tokens, a fraction of the re-bill. Two supporting changes in the ai-bot: Anthropic-model requests are biased to Anthropic itself (caches live per provider, so spreading a room's requests across providers turns a warm prefix into a full-price miss), and the usage recorded on each turn now carries the serving provider and generation id so cache misses are attributable from the room timeline. Co-Authored-By: Claude Fable 5 --- packages/ai-bot/lib/responder.ts | 8 + packages/ai-bot/main.ts | 11 + .../ai-bot/tests/prompt-construction-test.ts | 58 ++-- packages/base/matrix-event.gts | 7 + packages/runtime-common/ai/prompt.ts | 265 +++++++----------- 5 files changed, 174 insertions(+), 175 deletions(-) diff --git a/packages/ai-bot/lib/responder.ts b/packages/ai-bot/lib/responder.ts index 9144dee0636..82b4b882dbe 100644 --- a/packages/ai-bot/lib/responder.ts +++ b/packages/ai-bot/lib/responder.ts @@ -309,6 +309,12 @@ export class Responder { let cachedTokens = (chunk.usage as any).prompt_tokens_details ?.cached_tokens; let costUsd = (chunk.usage as any).cost; + // OpenRouter also stamps each chunk with the provider that served the + // request and the generation id. Recording them alongside the counts + // makes a cache miss attributable: a cachedTokens collapse with a + // provider change is a routing miss, not a prompt-shape bug. + let provider = (chunk as any).provider; + let generationId = chunk.id; // Hand the counts to the publisher so they ride on the final room // event. When the final edit has already gone out (the usage chunk // trails the finish chunk), finalize() sends one more edit to carry @@ -318,6 +324,8 @@ export class Responder { completionTokens: chunk.usage.completion_tokens, ...(typeof cachedTokens === 'number' ? { cachedTokens } : {}), ...(typeof costUsd === 'number' ? { costUsd } : {}), + ...(typeof provider === 'string' ? { provider } : {}), + ...(typeof generationId === 'string' ? { generationId } : {}), }; log.info( `Request used ${chunk.usage.prompt_tokens} prompt tokens (${ diff --git a/packages/ai-bot/main.ts b/packages/ai-bot/main.ts index 6f5def90ee6..ad6e7898f91 100644 --- a/packages/ai-bot/main.ts +++ b/packages/ai-bot/main.ts @@ -142,6 +142,17 @@ class Assistant { // the cast. (request as Record).usage = { include: true }; + // Prompt caches live per provider, and the router is otherwise free to + // spread a room's requests across providers — which turns a warm cache + // prefix into a full-price miss mid-conversation. Bias Anthropic-model + // requests to Anthropic itself, keeping fallbacks for availability. + if (this.getModel(prompt).startsWith('anthropic/')) { + (request as Record).provider = { + order: ['anthropic'], + allow_fallbacks: true, + }; + } + if (prompt.reasoningEffort !== undefined) { request.reasoning_effort = prompt.reasoningEffort; } diff --git a/packages/ai-bot/tests/prompt-construction-test.ts b/packages/ai-bot/tests/prompt-construction-test.ts index 12a81443fb2..0644e1d4c1c 100644 --- a/packages/ai-bot/tests/prompt-construction-test.ts +++ b/packages/ai-bot/tests/prompt-construction-test.ts @@ -860,7 +860,7 @@ Current date and time: 2025-06-11T11:43:00.533Z assert.equal(attachedCards.length, 0); }); - test('downloads and includes most recent version of attached files', async () => { + test('each message keeps its own attached-file snapshot content', async () => { const history: DiscreteMatrixEvent[] = [ { type: 'm.room.message', @@ -1042,7 +1042,24 @@ Current date and time: 2025-06-11T11:43:00.533Z }, ]; - // Set up mock responses for file downloads + // Set up mock responses for file downloads — every message's snapshot + // is downloaded now, so each version needs a response. + mockResponses.set('http://test.com/spaghetti-recipe-a.gts', { + ok: true, + text: 'spaghetti content version a', + }); + mockResponses.set('http://test.com/best-friends-a.txt', { + ok: true, + text: 'best friends version a', + }); + mockResponses.set('http://test.com/spaghetti-recipe-b.gts', { + ok: true, + text: 'spaghetti content version b', + }); + mockResponses.set('http://test.com/best-friends-b.txt', { + ok: true, + text: 'best friends version b', + }); mockResponses.set('http://test.com/spaghetti-recipe-c.gts', { ok: true, text: 'this is the content of the spaghetti-recipe.gts file', @@ -1071,33 +1088,35 @@ Current date and time: 2025-06-11T11:43:00.533Z assert.ok( messageText(userMessages[0]).includes( ` -Attached Files (files with newer versions don't show their content): -[spaghetti-recipe.gts](http://test-realm-server/my-realm/spaghetti-recipe.gts) -[best-friends.txt](http://test-realm-server/my-realm/best-friends.txt) +[spaghetti-recipe.gts](http://test-realm-server/my-realm/spaghetti-recipe.gts): + 1: spaghetti content version a +[best-friends.txt](http://test-realm-server/my-realm/best-friends.txt): + 1: best friends version a `.trim(), ), + 'first message keeps its own snapshot content', ); assert.ok( messageText(userMessages[1]).includes( ` -Attached Files (files with newer versions don't show their content): -[spaghetti-recipe.gts](http://test-realm-server/my-realm/spaghetti-recipe.gts) -[best-friends.txt](http://test-realm-server/my-realm/best-friends.txt) -[file-that-does-not-exist.txt](http://test.com/my-realm/file-that-does-not-exist.txt) -[example.pdf](http://test.com/my-realm/example.pdf): [application/pdf] +[spaghetti-recipe.gts](http://test-realm-server/my-realm/spaghetti-recipe.gts): + 1: spaghetti content version b +[best-friends.txt](http://test-realm-server/my-realm/best-friends.txt): + 1: best friends version b `.trim(), ), + 'second message keeps its own snapshot content', ); assert.ok( messageText(userMessages[2]).includes( ` -Attached Files (files with newer versions don't show their content): [spaghetti-recipe.gts](http://test-realm-server/my-realm/spaghetti-recipe.gts): 1: this is the content of the spaghetti-recipe.gts file [best-friends.txt](http://test-realm-server/my-realm/best-friends.txt): 1: this is the content of the best-friends.txt file `.trim(), ), + 'latest message includes its content with line numbers', ); assert.ok( @@ -1408,9 +1427,9 @@ Attached Files (files with newer versions don't show their content): 'http://localhost:4201/experiments/Author/1', ), ); - assert.false( + assert.true( messageText(userMessages[0]).includes('"firstName": "Terry"'), - 'should not include the contents of the first version of the card in the first user message', + 'each message keeps its own snapshot of the card content', ); assert.true( messageText(userMessages[1]).includes( @@ -4145,7 +4164,7 @@ Current date and time: 2025-06-11T11:43:00.533Z assert.true( messageText(toolCallMessage!).includes( ` -Attached Files (files with newer versions don't show their content): +Attached Files (each shows its content as of this message; a later attachment of the same file supersedes it): [postcard.gts](http://test-realm-server/user/test-realm/postcard.gts): 1: export default Postcard extends CardDef {} `.trim(), @@ -4183,7 +4202,7 @@ Attached Files (files with newer versions don't show their content): assert.true( messageText(toolCallMessage!).includes( ` -Attached Cards (cards with newer versions don't show their content): +Attached Cards (each shows its content as of this message; a later attachment of the same card supersedes it): [ { "url": "mxc://mock-server/nashville", @@ -5941,7 +5960,7 @@ new 'the surviving user message keeps its body', ); }); - test('only the most recent message attachments include file content in the prompt', async () => { + test('every message keeps its attached file content in the prompt', async () => { // Policy: files attached to older messages should show metadata only, // even if they are NOT re-attached in later messages. // Only the most recent user message's attachments should include content. @@ -6041,14 +6060,15 @@ new let userMessages = prompt.filter((m) => m.role === 'user'); - // Older message's unique file (config.json) should show metadata only, not content + // The older message keeps its own snapshot's content — re-rendering it + // later would change already-sent history bytes and break prompt caching. assert.ok( messageText(userMessages[0]).includes('[config.json]'), 'First message mentions config.json', ); - assert.notOk( + assert.ok( messageText(userMessages[0]).includes('"key": "value"'), - 'First message should NOT include config.json content (not the current message)', + 'First message keeps its config.json snapshot content', ); // Most recent message's file (utils.ts) should include content diff --git a/packages/base/matrix-event.gts b/packages/base/matrix-event.gts index a3b73e5d96d..98c35e6f6dd 100644 --- a/packages/base/matrix-event.gts +++ b/packages/base/matrix-event.gts @@ -283,6 +283,13 @@ export interface TokenUsage { // What the provider charged for the whole request, in USD. Absent when // the provider reports no inline cost. costUsd?: number; + // Which upstream provider served the request (the router's routing + // target). Prompt caches live per provider, so a surprising cache miss is + // attributable when this changes between turns. Absent when unreported. + provider?: string; + // The router-side generation id for the request, for post-hoc lookup of + // routing and cache detail. Absent when unreported. + generationId?: string; } export interface SkillsConfigEvent extends RoomStateEvent { diff --git a/packages/runtime-common/ai/prompt.ts b/packages/runtime-common/ai/prompt.ts index 4b3267acf8f..7cf0ba05d1f 100644 --- a/packages/runtime-common/ai/prompt.ts +++ b/packages/runtime-common/ai/prompt.ts @@ -99,14 +99,14 @@ function getLog() { * When building prompts for the model, file attachments are handled * according to the following rules: * - * 1. **Content inclusion**: Only the most recent user message's attached - * files have their content downloaded and included in the prompt. - * Older messages show file metadata (name, type) only. This keeps - * the prompt focused on fresh data and avoids redundant downloads. - * - * 2. **Supersession**: Within the current message, if a file (by - * sourceUrl) is re-attached in a later event, the earlier version - * is shown as metadata only (the later version wins). + * 1. **Byte-stable rendering**: A message's attachments are rendered from + * that message's own snapshot alone — content included at every age, + * never re-rendered because a newer version was attached later. Prompt + * caching is an exact prefix match, so rewriting an already-sent + * message re-bills the whole rest of the prompt at full input price on + * every later turn; carrying the superseded content forward costs only + * cached-read tokens. The model is told that a later attachment of the + * same card/file supersedes earlier ones. * * 3. **MIME type handling** (see `modality.ts` for classification): * - Text-based types (text/*, application/vnd.card+json, @@ -727,43 +727,35 @@ export function hasSomeAttachedCards( return false; } +// A message's rendering must depend only on the message itself, never on +// later history. Prompt caching is an exact prefix match: re-rendering an +// already-sent message (e.g. dropping an attachment's content because a +// newer version was attached later) changes bytes mid-history and re-bills +// everything after them at the full input price on every subsequent turn. +// Carrying the superseded content forward instead costs only cached-read +// tokens — a fraction of that. export async function getAttachedCards( client: MatrixClient, matrixEvent: MatrixEventWithBoxelContext, - history: DiscreteMatrixEvent[], ) { let attachedCards = matrixEvent.content?.data?.attachedCards ?? []; let results = await Promise.all( attachedCards.map(async (attachedCard: SerializedFileDef) => { - // If the file is attached later in the history, we should not include the content here - let shouldIncludeContent = !history - .slice(history.indexOf(matrixEvent) + 1) - .some((event) => { - // event is not always MatrixEventWithBoxelContext but casting lets us safely check attachedCards - return ( - event as MatrixEventWithBoxelContext - ).content?.data?.attachedCards?.some( - (cardAttachment: SerializedFileDef) => - cardAttachment.sourceUrl === attachedCard.sourceUrl, - ); - }); let result: SerializedFileDef = { url: attachedCard.url, sourceUrl: attachedCard.sourceUrl ?? '', name: attachedCard.name, contentType: attachedCard.contentType, }; - if (shouldIncludeContent) { - if (attachedCard.content) { - result.content = JSON.parse(attachedCard.content); - } else { - try { - result.content = await downloadFile(client, attachedCard); - } catch (error) { - getLog().error(`Failed to fetch file ${attachedCard.url}:`, error); - result.error = `Error loading attached card: ${(error as Error).message}`; - result.content = undefined; - } + if (attachedCard.content) { + result.content = JSON.parse(attachedCard.content); + } else { + try { + result.content = await downloadFile(client, attachedCard); + } catch (error) { + getLog().error(`Failed to fetch file ${attachedCard.url}:`, error); + result.error = `Error loading attached card: ${(error as Error).message}`; + result.content = undefined; } } return result; @@ -776,30 +768,17 @@ export async function getAttachedCards( return results; } +// Byte-stable for the same reason as getAttachedCards above: each message +// keeps its own snapshot's content forever, so history bytes never change +// after they are first sent. export async function getAttachedFiles( client: MatrixClient, matrixEvent: MatrixEventWithBoxelContext, - history: DiscreteMatrixEvent[], - isCurrentMessage: boolean = false, ): Promise { let attachedFiles = matrixEvent.content?.data?.attachedFiles ?? []; return Promise.all( attachedFiles.map(async (file: SerializedFileDef) => { - let isSuperseded = history - .slice(history.indexOf(matrixEvent) + 1) - .some((event) => - ( - event as MatrixEventWithBoxelContext - ).content?.data?.attachedFiles?.some( - (f: SerializedFileDef) => f.sourceUrl === file.sourceUrl, - ), - ); - - if ( - isCurrentMessage && - !isSuperseded && - isTextBasedContentType(file.contentType) - ) { + if (isTextBasedContentType(file.contentType)) { return downloadTextContent(client, file); } return toFileDefMetadata(file); @@ -808,7 +787,6 @@ export async function getAttachedFiles( } export async function loadCurrentlySerializedFileDefs( - client: MatrixClient, history: DiscreteMatrixEvent[], aiBotUserId: string, ): Promise { @@ -832,14 +810,9 @@ export async function loadCurrentlySerializedFileDefs( return []; } - // Reuse getAttachedFiles with isCurrentMessage=true — this is always the - // most recent user event, so supersession can't apply (no later events). - return getAttachedFiles( - client, - lastMessageEventByUser as MatrixEventWithBoxelContext, - history, - true, - ); + // The only consumer checks presence, so metadata is enough — downloading + // content here would be a per-turn fetch whose bytes are never sent. + return attachedFiles.map(toFileDefMetadata); } export function attachedFilesToMessage( @@ -1198,8 +1171,6 @@ async function toResultMessages( let attachmentResult = await buildAttachmentsMessagePart( client, toolResult, - history, - true, ); content = [content, attachmentResult.text].filter(Boolean).join('\n\n'); let toolMessage: OpenAIPromptMessage = { @@ -1578,13 +1549,6 @@ export async function buildPromptForModel( throw new Error("Username must be a full id, e.g. '@aibot:localhost'"); } let historicalMessages: OpenAIPromptMessage[] = []; - let lastUserMessageEvent = findLast( - history, - (event) => - event.sender !== aiBotUserId && - event.type === 'm.room.message' && - !isToolOrCodePatchResult(event), - ); for (let event of history) { if (event.type !== 'm.room.message') { continue; @@ -1628,12 +1592,9 @@ export async function buildPromptForModel( ).forEach((message) => historicalMessages.push(message)); } if (event.sender !== aiBotUserId) { - let isCurrentMessage = event === lastUserMessageEvent; let attachmentResult = await buildAttachmentsMessagePart( client, event as CardMessageEvent, - history, - isCurrentMessage, inputModalities, ); if (attachmentResult.mediaParts.length > 0) { @@ -1695,7 +1656,6 @@ export async function buildPromptForModel( ]; messages = messages.concat(historicalMessages); let contextContent = await buildContextMessage( - client, history, aiBotUserId, tools, @@ -2206,102 +2166,97 @@ function hasAppliedChanges( ); } +// Renders a message's attachments from that message's own snapshot alone. +// Nothing here may depend on later history or on whether the message is the +// current one: the rendering becomes part of the cached prompt prefix, and +// any retroactive change re-bills everything after it (see getAttachedCards). export const buildAttachmentsMessagePart = async ( client: MatrixClient, matrixEvent: MatrixEventWithBoxelContext, - history: DiscreteMatrixEvent[], - isCurrentMessage: boolean = false, inputModalities?: string[], ): Promise<{ text: string; mediaParts: ContentPart[] }> => { - let attachedCards = await getAttachedCards(client, matrixEvent, history); + let attachedCards = await getAttachedCards(client, matrixEvent); let text = ''; if (attachedCards.length > 0) { - text += `Attached Cards (cards with newer versions don't show their content):\n${JSON.stringify(attachedCards, null, 2)}\n`; + text += `Attached Cards (each shows its content as of this message; a later attachment of the same card supersedes it):\n${JSON.stringify(attachedCards, null, 2)}\n`; } - let attachedFiles = await getAttachedFiles( - client, - matrixEvent, - history, - isCurrentMessage, - ); + let attachedFiles = await getAttachedFiles(client, matrixEvent); let mediaParts: ContentPart[] = []; let mediaSourceUrls = new Set(); let unsupportedFiles: { name: string; contentType: string }[] = []; - if (isCurrentMessage) { - for (let f of attachedFiles) { - if (!f.url) { - continue; - } - // Check model capability before downloading - let modality = requiredModality(f.contentType); - if (!modality) { - continue; // not a multimodal type — handled as text metadata below - } - if (inputModalities && !inputModalities.includes(modality)) { - unsupportedFiles.push({ - name: f.name ?? 'unknown', - contentType: f.contentType ?? 'unknown', + for (let f of attachedFiles) { + if (!f.url) { + continue; + } + // Check model capability before downloading + let modality = requiredModality(f.contentType); + if (!modality) { + continue; // not a multimodal type — handled as text metadata below + } + if (inputModalities && !inputModalities.includes(modality)) { + unsupportedFiles.push({ + name: f.name ?? 'unknown', + contentType: f.contentType ?? 'unknown', + }); + continue; + } + try { + if (isImageContentType(f.contentType)) { + let dataUrl = await downloadFileAsBase64DataUrl( + client, + f.url, + f.contentType!, + ); + mediaParts.push({ + type: 'image_url', + image_url: { url: dataUrl }, }); - continue; - } - try { - if (isImageContentType(f.contentType)) { - let dataUrl = await downloadFileAsBase64DataUrl( - client, - f.url, - f.contentType!, - ); - mediaParts.push({ - type: 'image_url', - image_url: { url: dataUrl }, - }); - } else if (isPdfContentType(f.contentType)) { - let dataUrl = await downloadFileAsBase64DataUrl( - client, - f.url, - f.contentType!, - ); - mediaParts.push({ - type: 'file', - file: { - filename: f.name ?? 'document.pdf', - file_data: dataUrl, - }, - }); - } else if (isAudioContentType(f.contentType)) { - let format = audioFormatFromMime(f.contentType!); - if (!format) { - getLog().error(`Unsupported audio format: ${f.contentType}`); - continue; - } - let dataUrl = await downloadFileAsBase64DataUrl( - client, - f.url, - f.contentType!, - ); - // Strip data URL prefix — OpenRouter expects raw base64 for audio - let base64 = dataUrl.replace(/^data:[^;]+;base64,/, ''); - mediaParts.push({ - type: 'input_audio', - input_audio: { data: base64, format }, - }); - } else if (isVideoContentType(f.contentType)) { - let dataUrl = await downloadFileAsBase64DataUrl( - client, - f.url, - f.contentType!, - ); - mediaParts.push({ - type: 'video_url', - video_url: { url: dataUrl }, - }); - } - if (f.sourceUrl) { - mediaSourceUrls.add(f.sourceUrl); + } else if (isPdfContentType(f.contentType)) { + let dataUrl = await downloadFileAsBase64DataUrl( + client, + f.url, + f.contentType!, + ); + mediaParts.push({ + type: 'file', + file: { + filename: f.name ?? 'document.pdf', + file_data: dataUrl, + }, + }); + } else if (isAudioContentType(f.contentType)) { + let format = audioFormatFromMime(f.contentType!); + if (!format) { + getLog().error(`Unsupported audio format: ${f.contentType}`); + continue; } - } catch (e) { - getLog().error(`Failed to download media file ${f.url}:`, e); + let dataUrl = await downloadFileAsBase64DataUrl( + client, + f.url, + f.contentType!, + ); + // Strip data URL prefix — OpenRouter expects raw base64 for audio + let base64 = dataUrl.replace(/^data:[^;]+;base64,/, ''); + mediaParts.push({ + type: 'input_audio', + input_audio: { data: base64, format }, + }); + } else if (isVideoContentType(f.contentType)) { + let dataUrl = await downloadFileAsBase64DataUrl( + client, + f.url, + f.contentType!, + ); + mediaParts.push({ + type: 'video_url', + video_url: { url: dataUrl }, + }); + } + if (f.sourceUrl) { + mediaSourceUrls.add(f.sourceUrl); } + } catch (e) { + getLog().error(`Failed to download media file ${f.url}:`, e); } } if (unsupportedFiles.length > 0) { @@ -2311,7 +2266,7 @@ export const buildAttachmentsMessagePart = async ( text += `Note: The following files were not sent to the model because it does not support their input type: ${fileList}\n`; } if (attachedFiles.length > 0) { - text += `Attached Files (files with newer versions don't show their content):\n${attachedFilesToMessage( + text += `Attached Files (each shows its content as of this message; a later attachment of the same file supersedes it):\n${attachedFilesToMessage( attachedFiles, { omitSourceUrls: mediaSourceUrls, @@ -2322,7 +2277,6 @@ export const buildAttachmentsMessagePart = async ( }; export const buildContextMessage = async ( - client: MatrixClient, history: DiscreteMatrixEvent[], aiBotUserId: string, tools: Tool[], @@ -2331,7 +2285,6 @@ export const buildContextMessage = async ( let result = ''; let attachedFiles = await loadCurrentlySerializedFileDefs( - client, history, aiBotUserId, );