From 16e99b580da97c267a2c9225ee52ffa0fade6295 Mon Sep 17 00:00:00 2001 From: tianrking <10758833+tianrking@users.noreply.github.com> Date: Thu, 20 Aug 2026 09:02:49 +0800 Subject: [PATCH] feat(runtime): materialize verified native PDF inputs Gate PDF byte materialization on explicit first-party provider and wire contracts, then share the durable attachment path across current turns, replay, steering, and compaction. Add PDF and combined binary budgets plus exact provider-wire and fail-closed regression coverage. Generated-by: OpenAI Codex --- packages/core/src/attachments.ts | 32 +- .../execution-model-composition.test.ts | 103 +++- .../src/server/execution-model-composition.ts | 3 + .../src/__tests__/ai-sdk-backend.test.ts | 465 +++++++++++++++++- .../src/__tests__/pdf-input-contract.test.ts | 196 ++++++++ .../src/__tests__/request-shape.test.ts | 35 ++ packages/runtime/src/ai-sdk-backend.ts | 258 +++++++--- packages/runtime/src/ai-sdk-compaction.ts | 18 +- packages/runtime/src/model-runtime.ts | 31 +- packages/runtime/src/request-shape.ts | 13 +- .../__tests__/artifact-attachments.test.ts | 1 + packages/storage/src/artifact-attachments.ts | 2 +- 12 files changed, 1044 insertions(+), 113 deletions(-) create mode 100644 packages/runtime/src/__tests__/pdf-input-contract.test.ts diff --git a/packages/core/src/attachments.ts b/packages/core/src/attachments.ts index 6fe82ed2bd..8d959349f9 100644 --- a/packages/core/src/attachments.ts +++ b/packages/core/src/attachments.ts @@ -21,9 +21,14 @@ import type { AttachmentRef, StorageRef } from './events.js'; import { isCanonicalArtifactEntityId } from './artifacts.js'; /** Lives in core so @maka/runtime and @maka/storage share one type without a package cycle. */ -export type AttachmentByteReader = ( - ref: StorageRef, -) => Promise<{ ok: true; bytes: Uint8Array } | { ok: false; reason: string }>; +export type AttachmentByteReader = (ref: StorageRef) => Promise< + | { + ok: true; + bytes: Uint8Array; + mimeType?: string; // MIME sniffed from the loaded bytes when available. + } + | { ok: false; reason: string } +>; export const ATTACHMENT_RESOURCE_PREFIX = 'maka://runtime/attachments'; @@ -77,6 +82,20 @@ export const READ_IMAGE_TOO_LARGE_MESSAGE = `Image exceeds the ${MAX_READ_IMAGE_ export const MAX_PROVIDER_IMAGE_REQUEST_BYTES = 12 * 1024 * 1024; export const PROVIDER_IMAGE_BUDGET_EXCEEDED_MESSAGE = `Image was read, but the per-request image budget (${MAX_PROVIDER_IMAGE_REQUEST_BYTES / 1024 / 1024}MB across all images this turn) was exceeded; earlier images were sent and this one was omitted. Read fewer or smaller images.`; +/** + * Native PDF requests are Base64 encoded. Sixteen raw MiB expands to roughly + * 21.4 MiB, leaving headroom under Anthropic's 32 MiB whole-request limit for + * text, tool schemas, JSON framing, and other content. + */ +export const MAX_PROVIDER_PDF_REQUEST_BYTES = 16 * 1024 * 1024; + +/** + * Shared raw-byte ceiling across image and PDF inputs. Eighteen raw MiB + * expands to 24 MiB in Base64, preserving 8 MiB of whole-request headroom on + * the strictest verified native PDF route. + */ +export const MAX_PROVIDER_BINARY_REQUEST_BYTES = 18 * 1024 * 1024; + const MIME_BY_EXTENSION: Readonly> = { png: 'image/png', jpg: 'image/jpeg', @@ -107,10 +126,9 @@ export function guessMimeFromName(fileName: string): string { /** * Route a MIME type to an {@link AttachmentRef} kind. The runtime - * consumption split is image vs. everything-else (images become provider - * image parts; other kinds are read on demand by the model via Read), so this - * only needs to single out the kinds that change - * consumption or display. Unknown / unmapped MIME falls back to `other`. + * consumption split singles out images and PDFs (authorized routes can send + * them as provider file parts); text-like kinds are read on demand by the + * model via Read. Unknown / unmapped MIME falls back to `other`. * * `fileName` is consulted for kinds whose MIME is unreliable across OSes * (Office documents arrive as `application/octet-stream` or a long diff --git a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts index af100b503c..73fcc26da7 100644 --- a/packages/runtime-host/src/__tests__/execution-model-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-model-composition.test.ts @@ -268,6 +268,106 @@ test('backend creation admits an enabled model a snapshot never listed', async ( await backend.dispose(); }); +test('Host composition carries verified native PDF input through the provider wire', async () => { + const modelId = 'gpt-4o'; + const provider = await startProvider(); + let attachmentReads = 0; + let backend: Awaited> | undefined; + try { + backend = await createHostAiSdkBackend( + backendCreationFixture({ + abortSignal: new AbortController().signal, + modelId, + resolveExecutionConnection: async () => ({ + kind: 'ready', + connection: { + slug: 'backend-creation-connection', + providerType: 'openai', + baseUrl: provider.baseUrl, + enabledModelIds: [modelId], + models: [ + { + id: modelId, + capabilities: { chat: true, functionCalling: true }, + modalities: { input: ['text', 'image', 'pdf'], output: ['text'] }, + contextWindow: 8_192, + maxOutputTokens: 1_024, + }, + ], + }, + networkProxy: { enabled: false }, + secretMaterial: { connection: { secret: API_KEY } }, + }), + readPricing: async () => ({ revision: 0, overrides: [] }), + artifacts: { + readDurableAttachmentBinary: async ({ + artifactId, + sessionId, + }: { + artifactId: string; + sessionId: string; + }) => { + attachmentReads += 1; + assert.equal(artifactId, 'brief'); + assert.equal(sessionId, 'backend-creation-session'); + return { ok: true, base64: 'JVBERi0=', mimeType: 'application/pdf' }; + }, + } as unknown as HostAiSdkBackendInput['artifacts'], + }), + ); + + const events = []; + for await (const event of backend.send({ + invocationId: 'pdf-composition-invocation', + runId: 'pdf-composition-run', + turnId: 'pdf-composition-turn', + text: 'Read the attached PDF.', + attachments: [ + { + kind: 'pdf', + name: 'brief.pdf', + mimeType: 'application/pdf', + bytes: 8, + ref: { + kind: 'session_file', + sessionId: 'backend-creation-session', + relativePath: 'brief', + }, + }, + ], + context: [], + runtimeContext: [], + })) { + events.push(event); + } + + assert.equal( + events.find((event) => event.type === 'complete')?.stopReason, + 'end_turn', + JSON.stringify({ events, providerRequests: provider.requests }), + ); + assert.equal(attachmentReads, 1); + assert.equal(provider.requests.length, 1); + const messages = provider.requests[0]?.body.messages; + assert.ok(Array.isArray(messages)); + const filePart = messages + .flatMap((message: { content?: unknown }) => + Array.isArray(message.content) ? message.content : [], + ) + .find((part: { type?: unknown }) => part.type === 'file'); + assert.deepEqual(filePart, { + type: 'file', + file: { + filename: 'brief.pdf', + file_data: 'data:application/pdf;base64,JVBERi0=', + }, + }); + } finally { + await backend?.dispose(); + await provider.close(); + } +}); + test('provider dispatch fails closed when the Run Composition commit fails', async () => { const provider = await startProvider(); let commits = 0; @@ -3196,6 +3296,7 @@ function backendCreationFixture(input: { recordModelCallAttempt?: BackendFactoryContext['recordModelCallAttempt']; createFetchTransport?: HostAiSdkBackendInput['createFetchTransport']; createRunComposer?: HostAiSdkBackendInput['createRunComposer']; + artifacts?: HostAiSdkBackendInput['artifacts']; }): HostAiSdkBackendInput { const runtimePolicy = input.runtimePolicy ?? @@ -3267,7 +3368,7 @@ function backendCreationFixture(input: { runtimePolicy, ...(input.oauthCredentials ? { oauthCredentials: input.oauthCredentials } : {}), createRunComposer, - artifacts: {}, + artifacts: input.artifacts ?? {}, executionArtifacts: { recordToolArtifacts: async () => undefined, toolResultArchive: createToolResultArchiveCapability({ diff --git a/packages/runtime-host/src/server/execution-model-composition.ts b/packages/runtime-host/src/server/execution-model-composition.ts index 015e9b6442..eec8a5e63b 100644 --- a/packages/runtime-host/src/server/execution-model-composition.ts +++ b/packages/runtime-host/src/server/execution-model-composition.ts @@ -33,6 +33,7 @@ import { buildLlmHistorySummarizer } from '@maka/runtime/history-compact-summari import { buildOpenAiCodexHistoryCompactor } from '@maka/runtime/openai-codex-history-compactor'; import { buildPricingLookup, recordToolInvocation } from '@maka/runtime/telemetry'; import { buildProviderOptions, getAIModel } from '@maka/runtime/model-factory'; +import { resolveModelNativePdfInputSupport } from '@maka/runtime/model-runtime'; import { createProviderRequestCaptureRecorder } from '@maka/runtime/provider-request-telemetry'; import { createProxiedFetchTransport, @@ -149,6 +150,7 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom input.context.header.thinkingLevel, ); const contextWindow = resolveSelectedModelContextWindow(target.connection, target.model); + const supportsNativePdfInput = resolveModelNativePdfInputSupport(target.connection, target.model); let modelComposition: HostRunComposer; try { modelComposition = await readDuringBackendCreation( @@ -370,6 +372,7 @@ export async function createHostAiSdkBackend(input: HostAiSdkBackendInput): Prom target.model, relayModelProfile(target.connection, target.model)?.vision, ), + ...(supportsNativePdfInput ? { supportsNativePdfInput } : {}), readAttachmentBytes: createAttachmentByteReader({ artifactStore: input.artifacts, sessionId: input.context.sessionId, diff --git a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts index 98844d2eef..fa57af2665 100644 --- a/packages/runtime/src/__tests__/ai-sdk-backend.test.ts +++ b/packages/runtime/src/__tests__/ai-sdk-backend.test.ts @@ -29,7 +29,7 @@ import type { AttachmentByteReader } from '@maka/core/attachments'; import type { BackendSendInput } from '@maka/core/backend-types'; import type { LlmConnection } from '@maka/core/llm-connections'; import type { SessionHeader } from '@maka/core/session'; -import type { StorageRef } from '@maka/core/events'; +import type { AttachmentRef, StorageRef } from '@maka/core/events'; import { encodeCanonicalRuntimeEvent } from '@maka/core/canonical-runtime-event'; import type { SessionEvent } from '@maka/core/events'; import type { RuntimeEvent } from '@maka/core/runtime-event'; @@ -2219,6 +2219,389 @@ describe('AiSdkBackend model history', () => { ); }); + test('materializes PDF attachments consistently for current, RuntimeEvent, and stored replay', async () => { + const pdfBytes = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d, 1, 2, 3]); + const pdf = { + kind: 'pdf' as const, + name: 'brief.pdf', + mimeType: 'application/pdf', + bytes: pdfBytes.length, + ref: { kind: 'session_file' as const, sessionId: 'session-1', relativePath: 'brief' }, + }; + const cases: Array<{ name: string; input: BackendSendInput }> = [ + { + name: 'current turn', + input: { + turnId: 'turn-current', + text: 'read the current PDF', + attachments: [pdf], + context: [], + runtimeContext: [], + }, + }, + { + name: 'RuntimeEvent replay', + input: { + turnId: 'turn-current', + text: 'continue', + context: [], + runtimeContext: [ + runtimeEvent({ + id: 'rt-pdf', + turnId: 'turn-prev', + role: 'user', + author: 'user', + content: { kind: 'text', text: 'read the replayed PDF', attachments: [pdf] }, + }), + runtimeTextEvent({ + id: 'rt-answer', + turnId: 'turn-prev', + role: 'model', + author: 'agent', + text: 'noted', + }), + ], + }, + }, + { + name: 'stored-message fallback', + input: { + turnId: 'turn-current', + text: 'continue', + context: [ + { + type: 'user', + id: 'stored-user', + turnId: 'turn-prev', + ts: 1, + text: 'read the stored PDF', + attachments: [pdf], + }, + { + type: 'assistant', + id: 'stored-assistant', + turnId: 'turn-prev', + ts: 2, + text: 'noted', + modelId: 'm', + }, + ], + runtimeContext: [ + { + id: 'rt-terminal', + invocationId: 'inv-prev', + runId: 'run-prev', + sessionId: 'session-1', + turnId: 'turn-prev', + ts: 1, + partial: false, + role: 'model', + author: 'agent', + status: 'completed', + actions: { endInvocation: true }, + }, + ], + }, + }, + ]; + + for (const scenario of cases) { + const model = completionModel(); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => model, + tools: [], + newId: idGenerator(), + now: monotonicClock(), + supportsNativePdfInput: true, + readAttachmentBytes: async () => ({ ok: true, bytes: pdfBytes }), + }); + + await drain(backend.send(scenario.input)); + + const prompt = compactPrompt(model) as Array<{ content: unknown }>; + const pdfParts = prompt + .flatMap((message) => (Array.isArray(message.content) ? message.content : [])) + .filter( + (part: any) => + part.type === 'file' && + part.mediaType === 'application/pdf' && + part.filename === 'brief.pdf', + ); + assert.equal(pdfParts.length, 1, `${scenario.name}: ${JSON.stringify(prompt)}`); + assert.deepEqual(pdfParts[0]?.data, { type: 'data', data: pdfBytes }); + } + }); + + test('never reads or sends PDF bytes without a verified input contract', async () => { + let reads = 0; + const model = completionModel(); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => model, + tools: [], + newId: idGenerator(), + now: monotonicClock(), + readAttachmentBytes: async () => { + reads += 1; + return { ok: true, bytes: new Uint8Array([0xde, 0xad, 0xbe, 0xef]) }; + }, + }); + + await drain( + backend.send({ + turnId: 'turn-current', + text: 'inspect this report', + attachments: [pdfAttachment('report', 4)], + context: [], + runtimeContext: [], + }), + ); + + const prompt = compactPrompt(model) as Array<{ content: unknown }>; + const parts = prompt.flatMap((message) => + Array.isArray(message.content) ? message.content : [], + ); + assert.equal(reads, 0); + assert.equal( + parts.some((part: any) => part.type === 'file' && part.mediaType === 'application/pdf'), + false, + ); + assert.match(JSON.stringify(prompt), /report\.pdf.*application\/pdf/); + }); + + test('locally omits a declared PDF when its loaded bytes are another MIME type', async () => { + let reads = 0; + const model = completionModel(); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => model, + tools: [], + newId: idGenerator(), + now: monotonicClock(), + supportsNativePdfInput: true, + readAttachmentBytes: async () => { + reads += 1; + return { + ok: true, + bytes: new Uint8Array([0xde, 0xad, 0xbe, 0xef]), + mimeType: 'image/png', + }; + }, + }); + + await drain( + backend.send({ + turnId: 'turn-current', + text: 'inspect this report', + attachments: [pdfAttachment('mislabeled', 4)], + context: [], + runtimeContext: [], + }), + ); + + const prompt = compactPrompt(model) as Array<{ content: unknown }>; + assert.equal(reads, 1); + assert.equal(JSON.stringify(prompt).includes('3q2+7w=='), false); + assert.match( + JSON.stringify(prompt), + /mislabeled\.pdf.*loaded bytes are image\/png, not application\/pdf/, + ); + }); + + test('does not charge an unavailable PDF against the PDF subtype budget', async () => { + const model = completionModel(); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => model, + tools: [], + newId: idGenerator(), + now: monotonicClock(), + supportsNativePdfInput: true, + maxProviderPdfRequestBytes: 10, + maxProviderBinaryRequestBytes: 10, + readAttachmentBytes: async (ref: StorageRef) => + ref.kind === 'session_file' && ref.relativePath === 'missing' + ? { ok: false, reason: 'not_found' } + : { ok: true, bytes: new Uint8Array(10) }, + }); + + await drain( + backend.send({ + turnId: 'turn-current', + text: 'inspect both reports', + attachments: [pdfAttachment('missing', 10), pdfAttachment('available', 10)], + context: [], + runtimeContext: [], + }), + ); + + const prompt = compactPrompt(model) as Array<{ content: unknown }>; + const parts = prompt.flatMap((message) => + Array.isArray(message.content) ? message.content : [], + ); + assert.equal( + parts.filter((part: any) => part.type === 'file' && part.mediaType === 'application/pdf') + .length, + 1, + ); + assert.match(parts.map((part: any) => part.text ?? '').join('\n'), /missing\.pdf.*not_found/); + }); + + test('enforces the PDF subtype budget from bytes read, not attachment metadata', async () => { + const model = completionModel(); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => model, + tools: [], + newId: idGenerator(), + now: monotonicClock(), + supportsNativePdfInput: true, + maxProviderPdfRequestBytes: 15, + maxProviderBinaryRequestBytes: 30, + readAttachmentBytes: async () => ({ ok: true, bytes: new Uint8Array(10) }), + }); + + await drain( + backend.send({ + turnId: 'turn-current', + text: 'inspect these reports', + attachments: [pdfAttachment('first', 1), pdfAttachment('second', 1)], + context: [], + runtimeContext: [], + }), + ); + + const prompt = compactPrompt(model) as Array<{ content: unknown }>; + const parts = prompt.flatMap((message) => + Array.isArray(message.content) ? message.content : [], + ); + assert.equal( + parts.filter((part: any) => part.type === 'file' && part.mediaType === 'application/pdf') + .length, + 1, + ); + assert.match( + parts.map((part: any) => part.text ?? '').join('\n'), + /1 PDF attachment.*PDF budget/, + ); + }); + + test('does not re-read a budget-omitted durable PDF on later provider steps', async () => { + const bytes = new Uint8Array(10); + const attachment = pdfAttachment('too-large', bytes.length); + const durable = durableTurnHarness('turn-pdf-read-cache', 'inspect the attached PDF'); + durable.anchor.content = { + kind: 'text', + text: 'inspect the attached PDF', + attachments: [attachment], + }; + const loop = countingToolLoopModel(1); + let reads = 0; + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => loop.model, + tools: [testTool('Read', z.object({ path: z.string() }))], + loadTurnRuntimeEvents: durable.loadTurnRuntimeEvents, + newId: idGenerator(), + now: monotonicClock(), + supportsNativePdfInput: true, + maxProviderPdfRequestBytes: 5, + maxProviderBinaryRequestBytes: 5, + readAttachmentBytes: async () => { + reads += 1; + return { ok: true, bytes, mimeType: 'application/pdf' }; + }, + }); + + await drainDurably(backend.send(durable.input({ attachments: [attachment] })), durable); + + assert.equal(loop.callCount(), 2); + assert.equal(reads, 1, 'the cached omission must prevent a repeated durable byte read'); + }); + + test('enforces one combined budget across image and PDF inputs', async () => { + const model = completionModel(); + const backend = createTestAiSdkBackend({ + sessionId: 'session-1', + header: header(), + appendMessage: async () => {}, + connection: connection(), + apiKey: 'sk-test', + modelId: 'mock-model-id', + modelFactory: () => model, + tools: [], + newId: idGenerator(), + now: monotonicClock(), + supportsVision: true, + supportsNativePdfInput: true, + maxProviderImageRequestBytes: 20, + maxProviderPdfRequestBytes: 20, + maxProviderBinaryRequestBytes: 15, + readAttachmentBytes: async () => ({ ok: true, bytes: new Uint8Array(10) }), + }); + + await drain( + backend.send({ + turnId: 'turn-current', + text: 'inspect both inputs', + attachments: [ + { + kind: 'image', + name: 'chart.png', + mimeType: 'image/png', + bytes: 10, + ref: { kind: 'session_file', sessionId: 'session-1', relativePath: 'chart' }, + }, + pdfAttachment('report', 10), + ], + context: [], + runtimeContext: [], + }), + ); + + const prompt = compactPrompt(model) as Array<{ content: unknown }>; + const parts = prompt.flatMap((message) => + Array.isArray(message.content) ? message.content : [], + ); + assert.equal(parts.filter((part: any) => part.type === 'file').length, 1); + assert.equal((parts.find((part: any) => part.type === 'file') as any)?.mediaType, 'image/png'); + assert.match( + parts.map((part: any) => part.text ?? '').join('\n'), + /1 binary attachment.*combined image\/PDF request budget/, + ); + }); + test('reports unavailable attachment reads without consuming image budget', async () => { const model = completionModel(); const backend = createTestAiSdkBackend({ @@ -12293,7 +12676,13 @@ describe('AiSdkBackend steering durability and identity', () => { const steeringBackend = ( model: MockLanguageModelV4, options: Partial< - Pick + Pick< + AiSdkBackendInput, + | 'supportsVision' + | 'supportsNativePdfInput' + | 'readAttachmentBytes' + | 'loadTurnRuntimeEvents' + > > = {}, ): AiSdkBackend => createTestAiSdkBackend({ @@ -12493,6 +12882,7 @@ describe('AiSdkBackend steering durability and identity', () => { test('persists canonical steering content and materializes attachments for the model', async () => { const model = textCompletionModel('done'); const pngBytes = new Uint8Array([137, 80, 78, 71]); + const pdfBytes = new Uint8Array([37, 80, 68, 70, 45]); const image = { kind: 'image' as const, name: 'first.png', @@ -12504,14 +12894,18 @@ describe('AiSdkBackend steering durability and identity', () => { kind: 'pdf' as const, name: 'second.pdf', mimeType: 'application/pdf', - bytes: 12, + bytes: pdfBytes.length, ref: { kind: 'session_file' as const, sessionId: 'session-1', relativePath: 'second.pdf' }, }; const backend = steeringBackend(model, { supportsVision: true, + supportsNativePdfInput: true, readAttachmentBytes: async (ref) => { - assert.deepEqual(ref, image.ref); - return { ok: true, bytes: pngBytes }; + if (ref.kind === 'session_file' && ref.relativePath === image.ref.relativePath) { + return { ok: true, bytes: pngBytes }; + } + assert.deepEqual(ref, document.ref); + return { ok: true, bytes: pdfBytes }; }, }); const content = { @@ -12553,7 +12947,7 @@ describe('AiSdkBackend steering durability and identity', () => { }>; assert.deepEqual( parts.map((part) => part.type), - ['text', 'file'], + ['text', 'file', 'file'], ); assert.equal( parts[0]?.text, @@ -12563,6 +12957,9 @@ describe('AiSdkBackend steering durability and identity', () => { ); assert.equal(parts[1]?.mediaType, 'image/png'); assert.notEqual(parts[1]?.data, undefined); + assert.equal(parts[2]?.mediaType, 'application/pdf'); + assert.equal((parts[2] as { filename?: string } | undefined)?.filename, 'second.pdf'); + assert.notEqual(parts[2]?.data, undefined); assert.equal(JSON.stringify(prompt).includes('human-only command'), false); }); @@ -12870,7 +13267,16 @@ describe('AiSdkBackend steering durability and identity', () => { // projection wraps it. A future turn's history must show the model the // same form the original request used — one canonical provider projection. const model = textCompletionModel('done'); - const backend = steeringBackend(model); + const pdfBytes = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d]); + const attachment = pdfAttachment('steered', pdfBytes.length); + const backend = steeringBackend(model, { + supportsNativePdfInput: true, + readAttachmentBytes: async () => ({ + ok: true, + bytes: pdfBytes, + mimeType: 'application/pdf', + }), + }); const steeredEvent = runtimeTextEvent({ id: 'rt-steer', turnId: 'turn-prev', @@ -12878,7 +13284,10 @@ describe('AiSdkBackend steering durability and identity', () => { author: 'user', text: 'steered earlier', }); - (steeredEvent.content as { steering?: true }).steering = true; + (steeredEvent.content as { steering?: true; attachments?: AttachmentRef[] }).steering = true; + (steeredEvent.content as { steering?: true; attachments?: AttachmentRef[] }).attachments = [ + attachment, + ]; await drain( backend.send({ turnId: 'turn-current', @@ -12904,9 +13313,33 @@ describe('AiSdkBackend steering durability and identity', () => { }), ); - assert.deepEqual(compactPrompt(model), [ - { role: 'user', content: [{ type: 'text', text: 'original ask' }] }, - { role: 'user', content: [{ type: 'text', text: buildSteeringEnvelope('steered earlier') }] }, + const prompt = compactPrompt(model) as Array<{ role: string; content: unknown }>; + assert.deepEqual(prompt[0], { + role: 'user', + content: [{ type: 'text', text: 'original ask' }], + }); + assert.equal(prompt[1]?.role, 'user'); + const steeringParts = prompt[1]?.content as Array<{ + type: string; + text?: string; + data?: unknown; + mediaType?: string; + filename?: string; + }>; + assert.equal( + steeringParts[0]?.text, + buildSteeringEnvelope( + 'steered earlier\n\n\nRead argument: {"ref":"maka://runtime/attachments/steered"}\nThis is a Session resource, not a workspace file. Use the ref above; never use the display name as a path.\nname: "steered.pdf"\nmime_type: "application/pdf"\n', + ), + ); + assert.equal(steeringParts[1]?.type, 'file'); + assert.deepEqual(steeringParts[1]?.data, { type: 'data', data: pdfBytes }); + assert.equal(steeringParts[1]?.mediaType, 'application/pdf'); + assert.equal(steeringParts[1]?.filename, 'steered.pdf'); + assert.deepEqual(model.doStreamCalls[0]?.prompt[1]?.providerOptions, { + maka: { steeringEventId: 'rt-steer' }, + }); + assert.deepEqual(prompt.slice(2), [ { role: 'assistant', content: [{ type: 'text', text: 'ok' }] }, { role: 'user', content: [{ type: 'text', text: 'continue' }] }, ]); @@ -13956,6 +14389,16 @@ function sandboxSnapshot(): SandboxDiagnosticsSnapshot { }; } +function pdfAttachment(relativePath: string, bytes: number): AttachmentRef { + return { + kind: 'pdf', + name: `${relativePath}.pdf`, + mimeType: 'application/pdf', + bytes, + ref: { kind: 'session_file', sessionId: 'session-1', relativePath }, + }; +} + function connection(): LlmConnection { return { slug: 'anthropic-main', diff --git a/packages/runtime/src/__tests__/pdf-input-contract.test.ts b/packages/runtime/src/__tests__/pdf-input-contract.test.ts new file mode 100644 index 0000000000..d02df1d393 --- /dev/null +++ b/packages/runtime/src/__tests__/pdf-input-contract.test.ts @@ -0,0 +1,196 @@ +import assert from 'node:assert/strict'; +import { describe, test } from 'node:test'; +import type { LlmConnection } from '@maka/core/llm-connections'; +import { getAIModel } from '../model-factory.js'; +import { resolveModelNativePdfInputSupport } from '../model-runtime.js'; + +function connection(providerType: LlmConnection['providerType'], modelId: string): LlmConnection { + return { + slug: `${providerType}-pdf-test`, + name: `${providerType} PDF test`, + providerType, + baseUrl: 'https://provider.invalid/v1', + defaultModel: modelId, + enabled: true, + createdAt: 0, + updatedAt: 0, + }; +} + +describe('native PDF input gate', () => { + test('authorizes only PDF-capable models on supported provider identities and wires', () => { + assert.equal( + resolveModelNativePdfInputSupport( + connection('anthropic', 'claude-opus-4-8'), + 'claude-opus-4-8', + ), + true, + ); + assert.equal(resolveModelNativePdfInputSupport(connection('openai', 'gpt-4o'), 'gpt-4o'), true); + assert.equal( + resolveModelNativePdfInputSupport(connection('openai', 'gpt-5.4'), 'gpt-5.4'), + true, + ); + }); + + test('does not infer PDF support for relays, subscriptions, or unknown models', () => { + const declaredPdfModel = { + id: 'relay-pdf-model', + modalities: { input: ['text', 'pdf'] as const, output: ['text'] as const }, + }; + for (const providerType of [ + 'openai-compatible', + 'openai-codex', + 'anthropic-compatible', + 'claude-subscription', + ] as const) { + const relay = { + ...connection(providerType, declaredPdfModel.id), + models: [ + { + ...declaredPdfModel, + modalities: { + input: [...declaredPdfModel.modalities.input], + output: [...declaredPdfModel.modalities.output], + }, + }, + ], + }; + assert.equal( + resolveModelNativePdfInputSupport(relay, declaredPdfModel.id), + false, + providerType, + ); + } + assert.equal( + resolveModelNativePdfInputSupport(connection('openai', 'unknown-model'), 'unknown-model'), + false, + ); + assert.equal( + resolveModelNativePdfInputSupport( + { + ...connection('openai', 'gpt-4o'), + models: [ + { + id: 'gpt-4o', + modalities: { input: ['text'], output: ['text'] }, + }, + ], + }, + 'gpt-4o', + ), + false, + 'an explicit provider inventory must outrank generated PDF metadata', + ); + }); +}); + +describe('AI SDK PDF wire lowering', () => { + test('lowers one generic PDF file part to each verified native request shape', async () => { + const chat = await captureRequestBody('openai', 'gpt-4o'); + assert.deepEqual(requestContentPart(chat, 'messages', 1), { + type: 'file', + file: { + filename: 'brief.pdf', + file_data: 'data:application/pdf;base64,JVBERi0=', + }, + }); + + const responses = await captureRequestBody('openai', 'gpt-5.4'); + assert.deepEqual(requestContentPart(responses, 'input', 1), { + type: 'input_file', + filename: 'brief.pdf', + file_data: 'data:application/pdf;base64,JVBERi0=', + }); + + const anthropic = await captureRequestBody('anthropic', 'claude-opus-4-8'); + assert.deepEqual(requestContentPart(anthropic, 'messages', 1), { + type: 'document', + source: { type: 'base64', media_type: 'application/pdf', data: 'JVBERi0=' }, + title: 'brief.pdf', + }); + }); +}); + +function requestContentPart( + body: Record, + field: 'messages' | 'input', + partIndex: number, +): unknown { + const messages = body[field]; + assert.ok(Array.isArray(messages), `${field} must be an array`); + const firstMessage = messages[0] as { content?: unknown } | undefined; + assert.ok( + firstMessage && Array.isArray(firstMessage.content), + `${field}[0].content must be an array`, + ); + assert.ok(partIndex in firstMessage.content, `${field}[0].content[${partIndex}] must exist`); + return firstMessage.content[partIndex]; +} + +async function captureRequestBody( + providerType: 'openai' | 'anthropic', + modelId: 'gpt-4o' | 'gpt-5.4' | 'claude-opus-4-8', +): Promise> { + let requestBody: Record | undefined; + const fetch = (async (_url: string | URL | Request, init?: RequestInit) => { + requestBody = JSON.parse(String(init?.body)) as Record; + if (providerType === 'anthropic') { + return Response.json({ + id: 'msg_pdf', + type: 'message', + role: 'assistant', + model: modelId, + content: [{ type: 'text', text: 'ok' }], + stop_reason: 'end_turn', + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1 }, + }); + } + if (modelId === 'gpt-4o') { + return Response.json({ + id: 'chat_pdf', + object: 'chat.completion', + created: 0, + model: modelId, + choices: [ + { index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }); + } + return Response.json({ + id: 'resp_pdf', + object: 'response', + status: 'completed', + output: [], + usage: { input_tokens: 1, output_tokens: 1 }, + }); + }) as unknown as typeof globalThis.fetch; + const model = getAIModel({ + connection: connection(providerType, modelId), + apiKey: 'test-key', + modelId, + fetch, + }); + + await model.doGenerate({ + prompt: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Read the PDF.' }, + { + type: 'file', + data: { type: 'data', data: new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d]) }, + mediaType: 'application/pdf', + filename: 'brief.pdf', + }, + ], + }, + ], + }); + + assert.ok(requestBody); + return requestBody; +} diff --git a/packages/runtime/src/__tests__/request-shape.test.ts b/packages/runtime/src/__tests__/request-shape.test.ts index 55954f65ff..40217dccc5 100644 --- a/packages/runtime/src/__tests__/request-shape.test.ts +++ b/packages/runtime/src/__tests__/request-shape.test.ts @@ -19,6 +19,7 @@ import { describe, test } from 'node:test'; import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; import { canonicalizeToolSet, @@ -184,6 +185,40 @@ describe('prepared provider request capture', () => { assert.ok(result.segments.every((segment) => /^sha256:[a-f0-9]{64}$/.test(segment.hash))); }); + test('summarizes file bytes before request telemetry serializes them', () => { + const bytes = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d]); + const result = requestShape.capturePreparedProviderRequest({ + providerId: 'openai', + modelId: 'gpt-test', + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'read this file' }, + { + type: 'file', + data: { type: 'data', data: bytes }, + mediaType: 'application/pdf', + filename: 'brief.pdf', + }, + ], + }, + ], + }); + + const serialized = JSON.parse(result.serializedRequest) as { + messages: Array<{ + content: Array<{ data?: { data?: unknown } }>; + }>; + }; + assert.deepEqual(serialized.messages[0]?.content[1]?.data?.data, { + byteLength: bytes.byteLength, + sha256: `sha256:${createHash('sha256').update(bytes).digest('hex')}`, + }); + assert.equal(result.serializedRequest.includes('"0":37'), false); + assert.equal(result.requestBytes, Buffer.byteLength(result.serializedRequest, 'utf8')); + }); + test('names a tool schema from the payload, and only that segment kind', () => { // A size nobody can attribute is not actionable: "tool definitions are 40%" // names no tool to remove (#2323). diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index 48bb09af68..0e35ec44d2 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -95,10 +95,12 @@ import { YIELD_AGENT_GRAPH_TOOL_NAME, type YieldAgentGraphToolResult, } from './stream-graph-supervisor-tools.js'; -import type { AttachmentByteReader } from '@maka/core/attachments'; import { + MAX_PROVIDER_BINARY_REQUEST_BYTES, MAX_PROVIDER_IMAGE_REQUEST_BYTES, + MAX_PROVIDER_PDF_REQUEST_BYTES, PROVIDER_IMAGE_BUDGET_EXCEEDED_MESSAGE, + type AttachmentByteReader, } from '@maka/core/attachments'; import { stripUndefinedDeep } from '@maka/core/tool-args-identity'; import { pricingModelKey } from '@maka/core/usage-stats/pricing'; @@ -178,7 +180,7 @@ import { compactionDecisionDiagnosticPatch } from './compaction-boundary.js'; import type { AutomaticMemoryCompactionDecision, AutomaticMemoryCompactionDispatch, - ProviderImageBudget, + ProviderAttachmentBudget, } from './ai-sdk-compaction.js'; import { contextDiagnosticsCompactionOf, @@ -207,7 +209,6 @@ import { formatTextWithInlineRefs, steeringMessagesMissingFromBase, steeringModelMessage, - steeringProviderOptions, stripSteeringMessages, type RuntimeEventModelReplayItem, type RuntimeEventModelReplayPlan, @@ -835,9 +836,10 @@ export interface AiSdkBackendInput extends AiSdkCompactionCapabilities { */ recordToolArtifacts?: ToolArtifactRecorder; /** - * Optional attachment byte reader. When set, image attachments on the current - * user turn may be rendered as provider image parts instead of placeholder text. - * Caller wires this to the session ArtifactStore; runtime never imports storage. + * Optional attachment byte reader. When set, authorized image and PDF + * attachments may be rendered as provider file parts instead of placeholder + * text. Caller wires this to the session ArtifactStore; runtime never imports + * storage. */ readAttachmentBytes?: AttachmentByteReader; /** @@ -845,7 +847,11 @@ export interface AiSdkBackendInput extends AiSdkCompactionCapabilities { * image parts; false/unknown stay as text refs with a fallback note. */ supportsVision?: boolean; + /** True only when Runtime Host verified the provider identity and resolved wire for native PDFs. */ + supportsNativePdfInput?: boolean; maxProviderImageRequestBytes?: number; + maxProviderPdfRequestBytes?: number; + maxProviderBinaryRequestBytes?: number; /** Host-owned bounded long-term-memory extraction. Source tools are Runtime-reserved. */ memoryExtraction?: MemoryExtractionSourceCapabilities; } @@ -989,11 +995,15 @@ class TurnScope { watchdog: StreamWatchdog | null = null; runTrace: RunTrace | null = null; /** - * Image allowance for this turn, accumulated across its provider steps. Owned - * by the scope so an overlapping turn cannot spend it, and non-null for the - * scope's whole life so no path has to decide what "no budget" means. + * Binary attachment allowance for this turn, accumulated across its provider + * steps. Owned by the scope so an overlapping turn cannot spend it, and + * non-null for the scope's whole life so no path has to decide what "no + * budget" means. */ - readonly imageBudget: ProviderImageBudget = { used: 0, decisions: new Map() }; + readonly attachmentBudget: ProviderAttachmentBudget = { + used: { image: 0, pdf: 0, total: 0 }, + decisions: new Map(), + }; /** * User messages steered into this turn, drained from the caller's queue at * step boundaries. Each entry is the canonical envelope-wrapped user @@ -1118,8 +1128,8 @@ export class AiSdkBackend implements AgentBackend { modelAdapter: this.modelAdapter, createProviderRequestTracker: (trackerInput) => this.createProviderRequestTracker(trackerInput), - materializeRuntimeReplayPlan: (plan, imageBudget, checkpoint) => - this.materializeRuntimeReplayPlan(plan, imageBudget, undefined, checkpoint), + materializeRuntimeReplayPlan: (plan, attachmentBudget, checkpoint) => + this.materializeRuntimeReplayPlan(plan, attachmentBudget, undefined, checkpoint), canReplayProviderNative: (plan) => this.canReplayProviderNative(plan), appendTurnTailPrompt: (content, turnTailPrompt) => this.appendTurnTailPrompt(content, turnTailPrompt), @@ -1339,7 +1349,12 @@ export class AiSdkBackend implements AgentBackend { orchestrationMode: identity.orchestrationMode, ...(identity.invocationId ? { invocationId: identity.invocationId } : {}), materializeDefaultToolResultOutput: ({ toolCallId, output }) => - this.materializeToolResultOutput(identity.scope().imageBudget, output, false, toolCallId), + this.materializeToolResultOutput( + identity.scope().attachmentBudget, + output, + false, + toolCallId, + ), spawnChildAgent: input.spawnChildAgent, spawnChildSession: input.spawnChildSession, prepareChildAgentResume: input.prepareChildAgentResume, @@ -1867,7 +1882,7 @@ export class AiSdkBackend implements AgentBackend { const currentUserContent = input.continuation ? undefined : await this.buildCurrentUserContent( - scope.imageBudget, + scope.attachmentBudget, input.text, input.attachments, input.quotes, @@ -1951,7 +1966,7 @@ export class AiSdkBackend implements AgentBackend { }); const currentTurnMessages = await this.materializeRuntimeReplayPlan( { ...replayPlan, items: replayItems }, - scope.imageBudget, + scope.attachmentBudget, settledModelOutputs, projectionCheckpoint, ); @@ -3421,7 +3436,7 @@ export class AiSdkBackend implements AgentBackend { if (!input.runtimeContext) { return { status: 'ready', - messages: await this.materializePriorMessages(scope.imageBudget, priorStored), + messages: await this.materializePriorMessages(scope.attachmentBudget, priorStored), gate: 'stored_message_projection', diagnostics: [], }; @@ -3430,7 +3445,7 @@ export class AiSdkBackend implements AgentBackend { (event) => event.turnId !== input.turnId, ); const projectedMessages = await this.materializePriorMessages( - scope.imageBudget, + scope.attachmentBudget, priorStored, buildSteeringSidecar(priorRuntimeContext), ); @@ -3560,7 +3575,7 @@ export class AiSdkBackend implements AgentBackend { const materializeReplayFallback = (): Promise => fallbackUsesRuntimeReplay ? this.materializeRuntimeReplayTextOnly( - scope.imageBudget, + scope.attachmentBudget, plan, projectedHistoryCompactCheckpoint, ) @@ -3596,7 +3611,7 @@ export class AiSdkBackend implements AgentBackend { status: 'ready', messages: await this.materializeRuntimeReplayPlan( plan, - scope.imageBudget, + scope.attachmentBudget, undefined, projectedHistoryCompactCheckpoint, ), @@ -3620,7 +3635,7 @@ export class AiSdkBackend implements AgentBackend { degradedPlan.items.length > 0 || hasProviderHistoryCompactCheckpoint ? await this.materializeRuntimeReplayPlan( degradedPlan, - scope.imageBudget, + scope.attachmentBudget, undefined, projectedHistoryCompactCheckpoint, ) @@ -3639,7 +3654,7 @@ export class AiSdkBackend implements AgentBackend { status: 'ready', messages: await this.materializeRuntimeReplayPlan( plan, - scope.imageBudget, + scope.attachmentBudget, undefined, projectedHistoryCompactCheckpoint, ), @@ -3711,7 +3726,7 @@ export class AiSdkBackend implements AgentBackend { */ private async materializeRuntimeReplayPlan( plan: RuntimeEventModelReplayPlan, - budget: ProviderImageBudget, + budget: ProviderAttachmentBudget, settledModelOutputs?: ReadonlyMap, historyCompactCheckpoint?: HistoryCompactCheckpoint, ): Promise { @@ -4110,7 +4125,7 @@ export class AiSdkBackend implements AgentBackend { } private async materializeRuntimeReplayTextOnly( - budget: ProviderImageBudget, + budget: ProviderAttachmentBudget, plan: RuntimeEventModelReplayPlan, historyCompactCheckpoint?: HistoryCompactCheckpoint, ): Promise { @@ -4160,22 +4175,27 @@ export class AiSdkBackend implements AgentBackend { } private async materializeRuntimeReplayItem( - budget: ProviderImageBudget, + budget: ProviderAttachmentBudget, item: Extract, ): Promise { if (item.role === 'user') { if (item.steering) { - // Already envelope-wrapped by the plan; carry the structured identity - // so injection dedupe recognizes the replayed message. - return { - role: 'user', - content: item.content, - providerOptions: steeringProviderOptions(item.steering.eventId), - }; + // The plan has already wrapped the envelope. Re-materialize its durable + // attachments with the same key as live steering, then restore the + // structured identity so injection dedupe recognizes the replayed message. + return steeringModelMessage( + item.steering.eventId, + await this.appendAttachmentParts( + budget, + item.content, + item.attachments, + `steering:${item.steering.eventId}`, + ), + ); } return { role: 'user', - content: await this.appendImageParts( + content: await this.appendAttachmentParts( budget, item.content, item.attachments, @@ -4191,7 +4211,7 @@ export class AiSdkBackend implements AgentBackend { } private async materializePriorMessages( - budget: ProviderImageBudget, + budget: ProviderAttachmentBudget, stored: readonly StoredMessage[], steeringSidecar?: ReadonlyMap, ): Promise { @@ -4207,7 +4227,7 @@ export class AiSdkBackend implements AgentBackend { out.push( steeringModelMessage( sidecar.eventId, - await this.appendImageParts( + await this.appendAttachmentParts( budget, buildSteeringEnvelope(formatTextWithInlineRefs(m.text, m)), m.attachments, @@ -4219,7 +4239,7 @@ export class AiSdkBackend implements AgentBackend { } out.push({ role: 'user', - content: await this.appendImageParts( + content: await this.appendAttachmentParts( budget, formatTextWithInlineRefs(m.text, m), m.attachments, @@ -4258,87 +4278,153 @@ export class AiSdkBackend implements AgentBackend { } /** A decision key deduplicates re-materialization; no key charges each occurrence. */ - private chargeImageBudget( - budget: ProviderImageBudget, + private chargeAttachmentBudget( + budget: ProviderAttachmentBudget, + kind: 'image' | 'pdf', bytes: number, decisionKey?: string, - ): boolean { + ): 'keep' | 'image_limit' | 'pdf_limit' | 'combined_limit' { if (decisionKey !== undefined) { const cached = budget.decisions.get(decisionKey); if (cached !== undefined) return cached; } - const keep = - budget.used + bytes <= - (this.input.maxProviderImageRequestBytes ?? MAX_PROVIDER_IMAGE_REQUEST_BYTES); - if (keep) budget.used += bytes; - if (decisionKey !== undefined) budget.decisions.set(decisionKey, keep); - return keep; + const subtypeLimit = + kind === 'image' + ? (this.input.maxProviderImageRequestBytes ?? MAX_PROVIDER_IMAGE_REQUEST_BYTES) + : (this.input.maxProviderPdfRequestBytes ?? MAX_PROVIDER_PDF_REQUEST_BYTES); + const decision = + budget.used[kind] + bytes > subtypeLimit + ? kind === 'image' + ? 'image_limit' + : 'pdf_limit' + : budget.used.total + bytes > + (this.input.maxProviderBinaryRequestBytes ?? MAX_PROVIDER_BINARY_REQUEST_BYTES) + ? 'combined_limit' + : 'keep'; + if (decision === 'keep') { + budget.used[kind] += bytes; + budget.used.total += bytes; + } + if (decisionKey !== undefined) budget.decisions.set(decisionKey, decision); + return decision; } /** * Render provider-visible content for a user message: keep the given - * (already-formatted) text, and append image attachments as provider image - * parts only for explicitly vision-capable models. Non-image attachments stay - * as placeholder refs in the text. Shared by the current turn, RuntimeEvent - * replay, and the stored-message fallback so all paths present images identically. + * (already-formatted) text, then append explicitly authorized image and PDF + * file parts. The PDF gate is a verified provider-identity/wire decision, + * not a model-name or SDK-encoding guess. Shared by the current turn, + * RuntimeEvent replay, stored-message fallback, steering, and compaction so + * every path presents one durable attachment occurrence identically. */ - private async appendImageParts( - budget: ProviderImageBudget, + private async appendAttachmentParts( + budget: ProviderAttachmentBudget, textContent: string, attachments?: AttachmentRef[], decisionKeyPrefix?: string, ): Promise { - const images = attachments?.filter((a) => a.kind === 'image') ?? []; - if (images.length === 0) { - return textContent; - } - if (this.input.supportsVision !== true) { - return appendNonVisionImageFallbackNotice(textContent); - } - if (!this.input.readAttachmentBytes) { - return textContent; - } + const binaryAttachments = + attachments?.filter( + (attachment): attachment is AttachmentRef & { kind: 'image' | 'pdf' } => + attachment.kind === 'image' || attachment.kind === 'pdf', + ) ?? []; + if (binaryAttachments.length === 0) return textContent; + const hasUnsupportedImages = + this.input.supportsVision !== true && + binaryAttachments.some((attachment) => attachment.kind === 'image'); + const fallbackText = hasUnsupportedImages + ? appendNonVisionImageFallbackNotice(textContent) + : textContent; + const eligibleAttachments = binaryAttachments + .map((attachment, index) => ({ attachment, index })) + .filter( + ({ attachment }) => + (attachment.kind === 'image' && this.input.supportsVision === true) || + (attachment.kind === 'pdf' && this.input.supportsNativePdfInput === true), + ); + if (eligibleAttachments.length === 0 || !this.input.readAttachmentBytes) return fallbackText; const parts: Array< | { type: 'text'; text: string } | { type: 'file'; data: { type: 'data'; data: Uint8Array }; mediaType: string; + filename?: string; } - > = [{ type: 'text', text: textContent }]; - let omittedByBudget = 0; - for (const [index, image] of images.entries()) { - const read = await this.input.readAttachmentBytes(image.ref); + > = [{ type: 'text', text: fallbackText }]; + const omitted = { image_limit: 0, pdf_limit: 0, combined_limit: 0 }; + for (const { attachment, index } of eligibleAttachments) { + const decisionKey = + decisionKeyPrefix === undefined + ? undefined + : `${decisionKeyPrefix}:${attachment.kind}:${index}`; + const cachedDecision = + decisionKey === undefined ? undefined : budget.decisions.get(decisionKey); + if (cachedDecision !== undefined && cachedDecision !== 'keep') { + omitted[cachedDecision] += 1; + continue; + } + let read: Awaited>; + try { + read = await this.input.readAttachmentBytes(attachment.ref); + } catch { + read = { ok: false, reason: 'read_failed' }; + } if (!read.ok) { parts.push({ type: 'text', - text: `Image attachment "${image.name}" could not be loaded: ${read.reason}.`, + text: `${attachment.kind === 'pdf' ? 'PDF' : 'Image'} attachment "${attachment.name}" could not be loaded: ${read.reason}.`, }); continue; } - const decisionKey = - decisionKeyPrefix === undefined ? undefined : `${decisionKeyPrefix}:image:${index}`; - if (!this.chargeImageBudget(budget, read.bytes.length, decisionKey)) { - omittedByBudget += 1; + const mediaType = read.mimeType ?? attachment.mimeType; + if (attachment.kind === 'pdf' && mediaType !== 'application/pdf') { + parts.push({ + type: 'text', + text: `PDF attachment "${attachment.name}" was omitted because its loaded bytes are ${mediaType}, not application/pdf.`, + }); + continue; + } + const decision = this.chargeAttachmentBudget( + budget, + attachment.kind, + read.bytes.length, + decisionKey, + ); + if (decision !== 'keep') { + omitted[decision] += 1; continue; } parts.push({ type: 'file', data: { type: 'data', data: read.bytes }, - mediaType: image.mimeType, + mediaType, + ...(attachment.kind === 'pdf' ? { filename: attachment.name } : {}), }); } - if (omittedByBudget > 0) { + if (omitted.image_limit > 0) { parts.push({ type: 'text', - text: `[${omittedByBudget} image attachment(s) omitted: the per-request image budget was exceeded. Earlier images were sent; ask the user to send fewer or smaller images.]`, + text: `[${omitted.image_limit} image attachment(s) omitted: the per-request image budget was exceeded. Earlier images were sent; ask the user to send fewer or smaller images.]`, + }); + } + if (omitted.pdf_limit > 0) { + parts.push({ + type: 'text', + text: `[${omitted.pdf_limit} PDF attachment(s) omitted: the per-request PDF budget was exceeded. Earlier PDFs were sent; ask the user to send fewer or smaller PDFs.]`, + }); + } + if (omitted.combined_limit > 0) { + parts.push({ + type: 'text', + text: `[${omitted.combined_limit} binary attachment(s) omitted: the combined image/PDF request budget was exceeded. Earlier attachments were sent; ask the user to send fewer or smaller attachments.]`, }); } return parts; } private async materializeToolResultOutput( - budget: ProviderImageBudget, + budget: ProviderAttachmentBudget, output: unknown, isError: boolean, decisionKey: string, @@ -4350,8 +4436,9 @@ export class AiSdkBackend implements AgentBackend { if (!this.input.readAttachmentBytes) { return toolResultText('Image was read, but its stored bytes are unavailable.'); } - if (budget && budget.decisions.get(decisionKey) === false) { - return toolResultText(PROVIDER_IMAGE_BUDGET_EXCEEDED_MESSAGE); + const cachedDecision = budget.decisions.get(decisionKey); + if (cachedDecision !== undefined && cachedDecision !== 'keep') { + return toolResultText(this.imageBudgetFailureMessage(cachedDecision)); } let read: Awaited>; try { @@ -4362,8 +4449,9 @@ export class AiSdkBackend implements AgentBackend { if (!read.ok) { return toolResultText(`Image could not be loaded from artifact storage: ${read.reason}.`); } - if (!this.chargeImageBudget(budget, read.bytes.length, decisionKey)) { - return toolResultText(PROVIDER_IMAGE_BUDGET_EXCEEDED_MESSAGE); + const decision = this.chargeAttachmentBudget(budget, 'image', read.bytes.length, decisionKey); + if (decision !== 'keep') { + return toolResultText(this.imageBudgetFailureMessage(decision)); } return { type: 'content', @@ -4381,14 +4469,22 @@ export class AiSdkBackend implements AgentBackend { }; } + private imageBudgetFailureMessage( + decision: 'image_limit' | 'pdf_limit' | 'combined_limit', + ): string { + return decision === 'combined_limit' + ? `Image was read, but the combined image/PDF request budget (${MAX_PROVIDER_BINARY_REQUEST_BYTES / 1024 / 1024}MB across all binary attachments this turn) was exceeded; earlier attachments were sent and this one was omitted. Read fewer or smaller attachments.` + : PROVIDER_IMAGE_BUDGET_EXCEEDED_MESSAGE; + } + private async buildCurrentUserContent( - budget: ProviderImageBudget, + budget: ProviderAttachmentBudget, text: string, attachments?: AttachmentRef[], quotes?: QuoteRef[], runtimeEventId?: string, ): Promise { - return await this.appendImageParts( + return await this.appendAttachmentParts( budget, formatTextWithInlineRefs(text, { ...(attachments !== undefined ? { attachments } : {}), @@ -4515,8 +4611,8 @@ export class AiSdkBackend implements AgentBackend { // Materialize provider content before publishing the durable event. // After consumption there must be no fallible gap before ack/injection. const eventId = this.newId(); - const providerContent = await this.appendImageParts( - scope.imageBudget, + const providerContent = await this.appendAttachmentParts( + scope.attachmentBudget, buildSteeringEnvelope(formatTextWithInlineRefs(lease.content.text, lease.content)), lease.content.attachments, `steering:${eventId}`, diff --git a/packages/runtime/src/ai-sdk-compaction.ts b/packages/runtime/src/ai-sdk-compaction.ts index 1e96937f15..0697481040 100644 --- a/packages/runtime/src/ai-sdk-compaction.ts +++ b/packages/runtime/src/ai-sdk-compaction.ts @@ -102,14 +102,16 @@ import { } from './provider-image-overflow-recovery.js'; /** - * Image byte allowance for one turn, accumulated across its provider steps. + * Binary attachment allowance for one turn, accumulated across its provider + * steps. Decisions are cached by durable occurrence so rebuilding a request + * neither spends the allowance twice nor changes which attachments are sent. * * Charged while a request's content is materialized, so it belongs to the turn * issuing that request — never to the backend, which serves several turns. */ -export interface ProviderImageBudget { - used: number; - decisions: Map; +export interface ProviderAttachmentBudget { + used: { image: number; pdf: number; total: number }; + decisions: Map; } /** @@ -124,7 +126,7 @@ export interface ProviderImageBudget { */ export interface ProviderRequestOrigin { runId: string | undefined; - imageBudget: ProviderImageBudget; + attachmentBudget: ProviderAttachmentBudget; } export interface AutomaticMemoryCompactionDispatch { @@ -164,7 +166,7 @@ export interface AiSdkCompactionDeps { */ materializeRuntimeReplayPlan: ( plan: RuntimeEventModelReplayPlan, - imageBudget: ProviderImageBudget, + attachmentBudget: ProviderAttachmentBudget, checkpoint?: HistoryCompactCheckpoint, ) => Promise; canReplayProviderNative: (plan: RuntimeEventModelReplayPlan) => boolean; @@ -188,7 +190,7 @@ export class AiSdkCompaction { }) => ProviderRequestTracker | undefined; private readonly materializeRuntimeReplayPlan: ( plan: RuntimeEventModelReplayPlan, - imageBudget: ProviderImageBudget, + attachmentBudget: ProviderAttachmentBudget, checkpoint?: HistoryCompactCheckpoint, ) => Promise; private readonly canReplayProviderNative: (plan: RuntimeEventModelReplayPlan) => boolean; @@ -997,7 +999,7 @@ export class AiSdkCompaction { ); const replacementMessages = await this.materializeRuntimeReplayPlan( { ...replayPlan, items: replayItemsWithAnchorTail }, - input.origin.imageBudget, + input.origin.attachmentBudget, plan.checkpoint, ); // Apply the shape only when it actually shrinks the request versus the diff --git a/packages/runtime/src/model-runtime.ts b/packages/runtime/src/model-runtime.ts index a44ffaf805..505e26fb5e 100644 --- a/packages/runtime/src/model-runtime.ts +++ b/packages/runtime/src/model-runtime.ts @@ -25,7 +25,11 @@ import { type ProviderRuntimeAdapter, type ProviderType, } from '@maka/core/llm-connections'; -import { lookupModelProviderOverride, openAiAdapterApiProtocol } from '@maka/core/model-metadata'; +import { + lookupModelProviderOverride, + openAiAdapterApiProtocol, + resolveModelPdfSupport, +} from '@maka/core/model-metadata'; import { isRetiredProvider } from '@maka/core/provider-registry'; import { resolveApplyPatchProfile, type ApplyPatchProfile } from './apply-patch-profile.js'; @@ -130,6 +134,31 @@ export function resolveModelRuntime( }; } +/** + * Allow native PDF materialization only when both model metadata and the + * provider identity plus resolved wire authorize it. This intentionally does + * not verify a configured endpoint: relays and subscriptions fail closed even + * when they use an SDK-compatible wire. + */ +export function resolveModelNativePdfInputSupport( + connection: ModelRuntimeConnection, + modelId: string, +): boolean { + // Do not even resolve a relay/subscription runtime. Those providers may use + // an SDK-compatible adapter, but they are not an explicit first-party PDF + // integration and must fail closed (including retired provider records). + if (connection.providerType !== 'anthropic' && connection.providerType !== 'openai') { + return false; + } + if (!resolveModelPdfSupport(connection.providerType, connection.models, modelId)) return false; + const { wire } = resolveModelRuntime(connection, modelId); + return ( + (connection.providerType === 'anthropic' && wire === 'anthropic-messages') || + (connection.providerType === 'openai' && + (wire === 'openai-chat' || wire === 'openai-responses')) + ); +} + export function modelUsesAnthropicMessages( connection: ModelRuntimeConnection, modelId: string, diff --git a/packages/runtime/src/request-shape.ts b/packages/runtime/src/request-shape.ts index db20f23d2c..f301b01e99 100644 --- a/packages/runtime/src/request-shape.ts +++ b/packages/runtime/src/request-shape.ts @@ -222,9 +222,10 @@ export function capturePreparedProviderRequest( tools: input.tools ?? [], providerOptions: input.providerOptions ?? {}, }; - // This is the evidence body, not the hash canonicalizer: preserve the exact - // JSON ordering and values presented at the model-call seam. - const serializedRequest = JSON.stringify(payload); + // Persist diagnostic evidence, never a raw binary request body. `stableStringify` + // preserves the complete JSON-safe shape while summarizing typed bytes by + // length and digest before either telemetry or an artifact can observe them. + const serializedRequest = stableStringify(payload); const segments: PreparedRequestSegment[] = []; for (const [index, tool] of (input.tools ?? []).entries()) { @@ -696,6 +697,12 @@ function canonicalize(value: unknown, parentKey?: string): unknown { : items; } if (value instanceof Date) return value.toISOString(); + if (value instanceof Uint8Array) { + return { + byteLength: value.byteLength, + sha256: `sha256:${createHash('sha256').update(value).digest('hex')}`, + }; + } if (!isObjectLike(value)) return String(value); const out: Record = {}; diff --git a/packages/storage/src/__tests__/artifact-attachments.test.ts b/packages/storage/src/__tests__/artifact-attachments.test.ts index 7a23eadf46..d70a848022 100644 --- a/packages/storage/src/__tests__/artifact-attachments.test.ts +++ b/packages/storage/src/__tests__/artifact-attachments.test.ts @@ -84,6 +84,7 @@ describe('artifact attachment authority', () => { assert.deepEqual(await reader(sessionFileRef('image-1')), { ok: true, bytes: Buffer.from(png), + mimeType: 'image/png', }); await store.delete('image-1'); assert.deepEqual(await reader(sessionFileRef('image-1')), { diff --git a/packages/storage/src/artifact-attachments.ts b/packages/storage/src/artifact-attachments.ts index 075ba28dac..bf5e128f32 100644 --- a/packages/storage/src/artifact-attachments.ts +++ b/packages/storage/src/artifact-attachments.ts @@ -98,7 +98,7 @@ export function createAttachmentByteReader(input: { maxBytes, }); return result.ok - ? { ok: true, bytes: Buffer.from(result.base64, 'base64') } + ? { ok: true, bytes: Buffer.from(result.base64, 'base64'), mimeType: result.mimeType } : { ok: false, reason: result.reason }; }; }