diff --git a/lib/delegate.ts b/lib/delegate.ts index 8d6d85a..84322f3 100644 --- a/lib/delegate.ts +++ b/lib/delegate.ts @@ -59,7 +59,9 @@ export interface DelegateFailure { export type DelegateResult = DelegateSuccess | DelegateFailure; -/** Build provider-specific reasoning params for the request body, if any. */ +/** Build provider-specific reasoning params for the request body, if any. + * OpenAI-compatible APIs only — Anthropic Messages has no + * `reasoning_effort` field (see `buildAnthropicBody`). */ function buildReasoningParams( visionModel: Model, level: ReasoningLevel, @@ -68,6 +70,59 @@ function buildReasoningParams( return { reasoning_effort: level }; } +/** True when the model speaks Anthropic Messages (POST {baseUrl}/v1/messages). + * Models registered with `api: "anthropic-messages"` (MiniMax, Claude, + * Qwen via Anthropic-compat endpoints) would 404 if we blindly used the + * OpenAI `/chat/completions` route. */ +function isAnthropicMessagesApi(visionModel: Model): boolean { + return visionModel.api === "anthropic-messages"; +} + +/** Anthropic Messages request body: images as `source.base64` blocks, the + * system prompt as a top-level `system` field (NOT a messages entry), and + * no `temperature`/`reasoning_effort` (not part of the Messages schema). */ +function buildAnthropicBody( + visionModel: Model, + image: LoadedImage, + prompt: string, + systemPrompt: string | undefined, +): Record { + const body: Record = { + model: visionModel.id, + max_tokens: 4096, + messages: [ + { + role: "user", + content: [ + { + type: "image", + source: { + type: "base64", + media_type: image.mimeType, + data: image.data, + }, + }, + { type: "text", text: prompt }, + ], + }, + ], + }; + if (systemPrompt && systemPrompt.length > 0) body.system = systemPrompt; + return body; +} + +/** Extract the assistant text from an Anthropic Messages response: the first + * `content` block of type `text`; if the model returned only a `thinking` + * block (reasoning models), fall back to its `thinking` text. */ +function extractAnthropicText(json: unknown): string | undefined { + const blocks = (json as { content?: Array<{ type?: string; text?: string; thinking?: string }> })?.content; + if (!Array.isArray(blocks)) return undefined; + const textBlock = blocks.find((b) => b.type === "text" && typeof b.text === "string" && b.text.length > 0); + if (textBlock?.text) return textBlock.text; + const thinkingBlock = blocks.find((b) => b.type === "thinking" && typeof b.thinking === "string" && b.thinking.length > 0); + return thinkingBlock?.thinking; +} + /** * Call the vision model's OpenAI-compat chat/completions endpoint with the * image as a data URL + the user's prompt (and an optional system prompt). @@ -85,6 +140,57 @@ export async function callVisionModel( systemPrompt?: string, ): Promise { const baseUrl = visionModel.baseUrl.replace(/\/+$/, ""); + const anthropic = isAnthropicMessagesApi(visionModel); + + // API-aware routing: Anthropic Messages models POST to {baseUrl}/v1/messages + // with the Messages schema; everything else keeps the OpenAI-compatible + // /chat/completions shape. Previously every model was forced onto the + // OpenAI route, so `api: "anthropic-messages"` models (e.g. MiniMax M3 at + // api.minimaxi.com/anthropic) got a hard 404 from a nonexistent + // `/chat/completions` route. + const url = anthropic ? `${baseUrl}/v1/messages` : `${baseUrl}/chat/completions`; + const body: Record = anthropic + ? buildAnthropicBody(visionModel, image, prompt, systemPrompt) + : buildOpenAIBody(visionModel, image, prompt, systemPrompt, reasoning); + + const headers: Record = { "Content-Type": "application/json" }; + if (apiKey) headers.Authorization = `Bearer ${apiKey}`; + if (providerHeaders) Object.assign(headers, providerHeaders); + + const response = await fetch(url, { + method: "POST", + headers, + body: JSON.stringify(body), + signal, + }); + + if (!response.ok) { + const errBody = await response.text().catch(() => ""); + throw new Error( + `Vision model returned ${response.status}: ${errBody.slice(0, 500)}`, + ); + } + + const json = await response.json(); + const text = anthropic + ? extractAnthropicText(json) + : extractOpenAIText(json); + if (!text) { + throw new Error("Vision model returned no content in the response"); + } + return text; +} + +/** OpenAI-compatible request body (the historical behavior of this tool): + * images as `image_url` data URLs, system prompt as a leading system + * message, optional `reasoning_effort` for reasoning models. */ +function buildOpenAIBody( + visionModel: Model, + image: LoadedImage, + prompt: string, + systemPrompt: string | undefined, + reasoning: ReasoningLevel, +): Record { const messages: unknown[] = []; if (systemPrompt && systemPrompt.length > 0) { messages.push({ role: "system", content: systemPrompt }); @@ -107,34 +213,14 @@ export async function callVisionModel( }; const reasoningParams = buildReasoningParams(visionModel, reasoning); if (reasoningParams) Object.assign(body, reasoningParams); + return body; +} - const headers: Record = { "Content-Type": "application/json" }; - if (apiKey) headers.Authorization = `Bearer ${apiKey}`; - if (providerHeaders) Object.assign(headers, providerHeaders); - - const response = await fetch(`${baseUrl}/chat/completions`, { - method: "POST", - headers, - body: JSON.stringify(body), - signal, - }); - - if (!response.ok) { - const errBody = await response.text().catch(() => ""); - throw new Error( - `Vision model returned ${response.status}: ${errBody.slice(0, 500)}`, - ); - } - - const json = (await response.json()) as { - choices?: Array<{ message?: { content?: string; reasoning_content?: string } }>; - }; - const msg = json.choices?.[0]?.message; - const text = msg?.content || msg?.reasoning_content; - if (!text) { - throw new Error("Vision model returned no content in the response"); - } - return text; +/** Extract the assistant text from an OpenAI-compatible response. */ +function extractOpenAIText(json: unknown): string | undefined { + const choices = (json as { choices?: Array<{ message?: { content?: string; reasoning_content?: string } }> })?.choices; + const msg = choices?.[0]?.message; + return msg?.content || msg?.reasoning_content; } function formatImageError(error: { code: string; path?: string; message?: string }, inputPath: string): string { diff --git a/tests/delegate.test.ts b/tests/delegate.test.ts index 415f25d..a388a05 100644 --- a/tests/delegate.test.ts +++ b/tests/delegate.test.ts @@ -143,6 +143,157 @@ test("callVisionModel: sends chat/completions POST with image data URL + prompt" } }); +// ── v0.6.0: API-shape-aware routing (anthropic-messages) ──────────────── + +test("callVisionModel: anthropic-messages api → POST {baseUrl}/v1/messages with Anthropic body", async () => { + const m = mockFetch({ + status: 200, + body: { content: [{ type: "text", text: "a red square" }] }, + }); + try { + const text = await callVisionModel( + makeVisionModel({ api: "anthropic-messages" as Api, baseUrl: "https://api.example.com/anthropic" }), + "key-123", + undefined, + { data: PNG_1x1_B64, mimeType: "image/png" }, + "describe this", + undefined, + "off", + ); + assert.equal(text, "a red square"); + assert.equal(m.calls.length, 1); + assert.equal(m.calls[0]!.url, "https://api.example.com/anthropic/v1/messages"); + const init = m.calls[0]!.init; + assert.equal(init.method, "POST"); + const headers = init.headers as Record; + assert.equal(headers.Authorization, "Bearer key-123"); + assert.equal(headers["Content-Type"], "application/json"); + const body = JSON.parse(init.body as string); + assert.equal(body.model, "minimax-m3:cloud"); + assert.equal(body.max_tokens, 4096); + assert.ok(!("temperature" in body), "Anthropic Messages has no temperature field"); + assert.equal(body.messages.length, 1); + assert.equal(body.messages[0].role, "user"); + const content = body.messages[0].content; + assert.equal(content[0].type, "image"); + assert.equal(content[0].source.type, "base64"); + assert.equal(content[0].source.media_type, "image/png"); + assert.equal(content[0].source.data, PNG_1x1_B64); + assert.equal(content[1].type, "text"); + assert.equal(content[1].text, "describe this"); + } finally { + m.restore(); + } +}); + +test("callVisionModel: anthropic-messages api → system prompt goes to top-level system field", async () => { + const m = mockFetch({ status: 200, body: { content: [{ type: "text", text: "ok" }] } }); + try { + await callVisionModel( + makeVisionModel({ api: "anthropic-messages" as Api, baseUrl: "https://api.example.com/anthropic" }), + "k", + undefined, + { data: PNG_1x1_B64, mimeType: "image/png" }, + "p", + undefined, + "off", + "You are a forensic analyst.", + ); + const body = JSON.parse(m.calls[0]!.init.body as string); + assert.equal(body.system, "You are a forensic analyst."); + assert.equal(body.messages.length, 1, "system is top-level, not a messages entry"); + assert.equal(body.messages[0].role, "user"); + } finally { + m.restore(); + } +}); + +test("callVisionModel: anthropic-messages api → parses text block from content array", async () => { + const m = mockFetch({ + status: 200, + body: { + content: [ + { type: "thinking", thinking: "it has four sides" }, + { type: "text", text: "a red square" }, + ], + }, + }); + try { + const text = await callVisionModel( + makeVisionModel({ api: "anthropic-messages" as Api, baseUrl: "https://api.example.com/anthropic" }), + "k", + undefined, + { data: PNG_1x1_B64, mimeType: "image/png" }, + "what shape", + undefined, + "off", + ); + assert.equal(text, "a red square", "text block wins over thinking block"); + } finally { + m.restore(); + } +}); + +test("callVisionModel: anthropic-messages api → falls back to thinking block when no text block", async () => { + const m = mockFetch({ + status: 200, + body: { content: [{ type: "thinking", thinking: "thought-only response" }] }, + }); + try { + const text = await callVisionModel( + makeVisionModel({ api: "anthropic-messages" as Api, baseUrl: "https://api.example.com/anthropic" }), + "k", + undefined, + { data: PNG_1x1_B64, mimeType: "image/png" }, + "p", + undefined, + "off", + ); + assert.equal(text, "thought-only response"); + } finally { + m.restore(); + } +}); + +test("callVisionModel: anthropic-messages api → error path surfaces status + body excerpt", async () => { + const m = mockFetchError(404, "404 page not found"); + try { + await assert.rejects( + callVisionModel( + makeVisionModel({ api: "anthropic-messages" as Api, baseUrl: "https://api.example.com/anthropic" }), + "k", + undefined, + { data: PNG_1x1_B64, mimeType: "image/png" }, + "p", + undefined, + "off", + ), + /404: 404 page not found/, + ); + } finally { + m.restore(); + } +}); + +test("callVisionModel: anthropic-messages api → reasoning_effort NOT sent (Anthropic has no such field)", async () => { + const m = mockFetch({ status: 200, body: { content: [{ type: "text", text: "ok" }] } }); + try { + await callVisionModel( + makeVisionModel({ api: "anthropic-messages" as Api, baseUrl: "https://api.example.com/anthropic", reasoning: true }), + "k", + undefined, + { data: PNG_1x1_B64, mimeType: "image/png" }, + "p", + undefined, + "high", + ); + const body = JSON.parse(m.calls[0]!.init.body as string); + assert.equal(body.reasoning_effort, undefined); + } finally { + m.restore(); + } +}); + test("callVisionModel: falls back to reasoning_content when content is empty", async () => { const m = mockFetch({ status: 200,