diff --git a/CHANGELOG.md b/CHANGELOG.md index 00645d5..3bc6a8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,12 @@ ## Unreleased +## 7.0.9 + +### Changed + +- Updated `@tangle-network/agent-eval` to `0.144.1` so Knowledge and Runtime install the same official-optimizer callback contract. + ## 7.0.8 ### Changed diff --git a/README.md b/README.md index 2c443af..aa18047 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Supply application callbacks for those decisions, or use `@tangle-network/agent- ## Install ```bash -pnpm add @tangle-network/agent-knowledge@7.0.8 @tangle-network/agent-eval@0.143.0 +pnpm add @tangle-network/agent-knowledge@7.0.9 @tangle-network/agent-eval@0.144.1 ``` Requires Node.js 20.19 or later. diff --git a/package.json b/package.json index 6fe8e35..da63b98 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-knowledge", - "version": "7.0.8", + "version": "7.0.9", "description": "Build, search, evaluate, and improve source-backed knowledge bases.", "homepage": "https://github.com/tangle-network/agent-knowledge#readme", "repository": { @@ -75,7 +75,7 @@ "verify:official-optimizers": "node scripts/verify-official-optimizers.mjs" }, "dependencies": { - "@tangle-network/agent-eval": "0.143.0", + "@tangle-network/agent-eval": "0.144.1", "@tangle-network/agent-interface": "0.43.0", "proper-lockfile": "4.1.2", "zod": "^4.4.3" diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7f2e453..b87bc4c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -16,8 +16,8 @@ importers: .: dependencies: '@tangle-network/agent-eval': - specifier: 0.143.0 - version: 0.143.0 + specifier: 0.144.1 + version: 0.144.1 '@tangle-network/agent-interface': specifier: 0.43.0 version: 0.43.0 @@ -627,8 +627,8 @@ packages: '@tangle-network/agent-core@0.4.33': resolution: {integrity: sha512-if3DeIo4e2c9wELJjrWOu4SsKv3WnWdDrp+kezP6JrpiQ1DXValRRlN/aTppicFHepw67OD5JQI9i351Z/hvDQ==} - '@tangle-network/agent-eval@0.143.0': - resolution: {integrity: sha512-Vx26rz0+qhb1QYd8YkSfoTgvdcoWtqAkR+4H7de4iu4mXKjLlefdWm5iRd4Nwqxgrd4R3Zrgm17N/b282mMCLQ==} + '@tangle-network/agent-eval@0.144.1': + resolution: {integrity: sha512-OGwF96faPiiGzcb5QQnCvdzYdTCybJ4JBPLIjgTs21pBkJZkjb0Jqylcto4P31b3e3YWjryts0CALBSDr+0oLA==} engines: {node: '>=20'} hasBin: true @@ -2785,7 +2785,7 @@ snapshots: '@tangle-network/agent-interface': 0.43.0 zod: 4.4.3 - '@tangle-network/agent-eval@0.143.0': + '@tangle-network/agent-eval@0.144.1': dependencies: '@asteasolutions/zod-to-openapi': 9.1.0(zod@4.4.3) '@hono/node-server': 2.0.12(hono@4.12.32) diff --git a/tests/contracts/agent-stack-cohort.ts b/tests/contracts/agent-stack-cohort.ts index 3fa65cd..52d4bd5 100644 --- a/tests/contracts/agent-stack-cohort.ts +++ b/tests/contracts/agent-stack-cohort.ts @@ -1,3 +1,4 @@ +import type { ExternalOptimizerModelCall } from '@tangle-network/agent-eval/campaign' import type { AgentCandidateKnowledgeRef } from '@tangle-network/agent-interface' import { fromAgentCandidateKnowledgeRef, toAgentCandidateKnowledgeRef } from '../../src/index' @@ -8,3 +9,10 @@ export function roundTripCanonicalKnowledgeCandidate( ): AgentCandidateKnowledgeRef { return toAgentCandidateKnowledgeRef(fromAgentCandidateKnowledgeRef(candidate)) } + +/** The installed Eval cohort exposes the caller-owned official optimizer model callback. */ +export function acceptExternalOptimizerModelCall( + call: ExternalOptimizerModelCall, +): ExternalOptimizerModelCall { + return call +} diff --git a/tests/kb-improvement/optimization.test.ts b/tests/kb-improvement/optimization.test.ts index 326853c..352767b 100644 --- a/tests/kb-improvement/optimization.test.ts +++ b/tests/kb-improvement/optimization.test.ts @@ -50,7 +50,12 @@ describe('optimizeKnowledgeBasePolicy', () => { expect('testScenarios' in input).toBe(false) return { winnerSurface: '{"mode":"grounded"}', - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, } @@ -156,7 +161,12 @@ describe('optimizeKnowledgeBasePolicy', () => { methodCalls += 1 return { winnerSurface: '{"mode":"candidate"}', - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, } @@ -250,7 +260,12 @@ describe('optimizeKnowledgeBasePolicy', () => { expect('testScenarios' in input).toBe(false) return { winnerSurface: '{"evidence":"required","maxSources":4}', - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, } @@ -337,7 +352,12 @@ describe('optimizeKnowledgeBasePolicy', () => { await writeFile(join(root, 'knowledge', 'concurrent-change.md'), '# Concurrent change\n') return { winnerSurface: '{"evidence":"required","maxSources":2}', - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, } diff --git a/tests/memory/improvement.test.ts b/tests/memory/improvement.test.ts index 5b44187..2134228 100644 --- a/tests/memory/improvement.test.ts +++ b/tests/memory/improvement.test.ts @@ -188,7 +188,12 @@ describe('agent memory improvement', () => { await duplicateRuns return { winnerSurface: surface, - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, } @@ -386,6 +391,7 @@ describe('agent memory improvement', () => { runDir: '/runs/incomplete-method-cost', method: selectingMethod([{ visibility: 'private' }, { visibility: 'team' }], undefined, { totalCostUsd: 0, + costProvenance: { kind: 'uncaptured', usd: null }, accountingComplete: false, incompleteReasons: ['external optimizer usage unavailable'], }), @@ -439,7 +445,12 @@ function baseOptions( function selectingMethod( configs: readonly TConfig[], inputs?: string[][], - cost = { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost = { + totalCostUsd: 0, + costProvenance: { kind: 'observed' as const, usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, ): OptimizationMethod { return { name: 'fixture-selection', diff --git a/tests/official-optimization.integration.test.ts b/tests/official-optimization.integration.test.ts index 6bfc3dc..567a798 100644 --- a/tests/official-optimization.integration.test.ts +++ b/tests/official-optimization.integration.test.ts @@ -1,10 +1,13 @@ import { spawnSync } from 'node:child_process' import { mkdtemp, rm } from 'node:fs/promises' -import { createServer, type Server } from 'node:http' import { tmpdir } from 'node:os' import { join } from 'node:path' -import type { JudgeConfig, Scenario } from '@tangle-network/agent-eval/campaign' -import { afterEach, describe, expect, it } from 'vitest' +import type { + ExternalOptimizerModelCall, + JudgeConfig, + Scenario, +} from '@tangle-network/agent-eval/campaign' +import { describe, expect, it } from 'vitest' import type { RetrievalEvalArtifact, RetrievalEvalScenario } from '../src' import { testExecutionRef } from './support/optimization' @@ -19,25 +22,11 @@ const { runRetrievalImprovementLoop, runSerializedKnowledgeOptimization } = know const python = process.env.AGENT_EVAL_TEST_PYTHON const describeWithOfficialEngines = python ? describe : describe.skip -const openServers: Server[] = [] - -afterEach(async () => { - await Promise.all( - openServers.splice(0).map( - (server) => - new Promise((resolve, reject) => { - server.closeAllConnections?.() - server.close((error) => (error ? reject(error) : resolve())) - }), - ), - ) -}) describeWithOfficialEngines('official optimizer integration', () => { it('runs official GEPA through retrieval optimization and final scoring', async () => { assertPythonModules(python!, ['agent_eval_rpc.gepa_bridge', 'gepa.optimize_anything']) const root = await mkdtemp(join(tmpdir(), 'agent-knowledge-gepa-')) - const baseUrl = await startModelServer('```\n{"k":2}\n```') try { const method = gepaOptimizationMethod({ name: 'official-gepa-retrieval', @@ -67,7 +56,7 @@ describeWithOfficialEngines('official optimizer integration', () => { }, }, }, - optimizer: optimizerModel(baseUrl), + optimizer: optimizerModel('```\n{"k":2}\n```', 'agent-knowledge-test:gepa-fixed-response'), describeScenario: (scenario) => ({ query: scenario.query, expected: scenario.expected, @@ -144,7 +133,6 @@ describeWithOfficialEngines('official optimizer integration', () => { reasoning: 'Add the missing response rule.', }, }) - const baseUrl = await startModelServer(modelResponse) try { const method = skillOptOptimizationMethod({ name: 'official-skillopt-policy', @@ -161,7 +149,7 @@ describeWithOfficialEngines('official optimizer integration', () => { maxAnalystRounds: 1, evaluationWorkers: 1, }, - optimizer: optimizerModel(baseUrl), + optimizer: optimizerModel(modelResponse, 'agent-knowledge-test:skillopt-fixed-response'), maxEvaluations: 3, describeScenario: (scenario) => ({ prompt: scenario.prompt }), describeArtifact: (artifact) => ({ candidate: artifact.candidate }), @@ -253,25 +241,61 @@ function pythonRunner(command: string, module: string) { return { command, args: ['-m', module] } } -function optimizerModel(baseUrl: string) { +function optimizerModel(content: string, callRef: string) { + const pricing = { + inputUsdPerMillion: 1, + outputUsdPerMillion: 2, + } + const call: ExternalOptimizerModelCall = async ({ callId, request }) => + optimizerModelCall({ callId, request, content, pricing }) return { model: 'local-optimizer', - baseUrl, - apiKey: 'local-test-key', + callRef, + call, budget: { maxCostUsd: 1, - maxRequests: 10, + maxRequests: 100, maxRequestBytes: 100_000, maxResponseBytes: 100_000, maxOutputTokensPerRequest: 2_000, - pricing: { - inputUsdPerMillion: 1, - outputUsdPerMillion: 2, - }, + pricing, }, } } +async function optimizerModelCall({ + callId, + request, + content, + pricing, +}: { + callId: string + request: { model: string } + content: string + pricing: { inputUsdPerMillion: number; outputUsdPerMillion: number } +}) { + return { + succeeded: true as const, + response: { + content, + usage: { promptTokens: 11, completionTokens: 13, totalTokens: 24 }, + costUsd: 0.000037, + model: request.model, + durationMs: 0, + finishReason: 'stop', + contentEmpty: false, + raw: { id: callId, fixture: true }, + }, + receipt: { + model: request.model, + inputTokens: 11, + outputTokens: 13, + customTokenPricing: pricing, + }, + execution: { kind: 'fixed-test-response', callId, model: request.model }, + } +} + function assertPythonModules(command: string, modules: readonly string[]): void { const imports = modules.map((module) => `import ${module}`).join('; ') const result = spawnSync(command, ['-c', imports], { encoding: 'utf8' }) @@ -279,41 +303,3 @@ function assertPythonModules(command: string, modules: readonly string[]): void throw new Error(`official optimizer Python environment is unavailable: ${result.stderr.trim()}`) } } - -async function startModelServer(content: string): Promise { - const server = createServer(async (request, response) => { - for await (const _chunk of request) { - // Drain the request before replying. - } - response.writeHead(200, { 'content-type': 'application/json' }) - response.end( - JSON.stringify({ - id: 'local-completion', - choices: [ - { - finish_reason: 'stop', - index: 0, - message: { role: 'assistant', content }, - }, - ], - model: 'local-optimizer', - usage: { - prompt_tokens: 20, - completion_tokens: 20, - total_tokens: 40, - }, - }), - ) - }) - openServers.push(server) - await new Promise((resolve, reject) => { - server.once('error', reject) - server.listen(0, '127.0.0.1', () => { - server.off('error', reject) - resolve() - }) - }) - const address = server.address() - if (!address || typeof address === 'string') throw new Error('model server did not bind') - return `http://127.0.0.1:${address.port}/v1` -} diff --git a/tests/official-optimizer-resume.contract.test.ts b/tests/official-optimizer-resume.contract.test.ts index eba3c51..cdc06b5 100644 --- a/tests/official-optimizer-resume.contract.test.ts +++ b/tests/official-optimizer-resume.contract.test.ts @@ -66,8 +66,19 @@ describe('official optimizer resume identity', () => { trainer: { epochs: 1, batchSize: 1 }, optimizer: { model: 'unused-test-model', - baseUrl: 'http://127.0.0.1:1/v1', - apiKey: 'unused-test-key', + callRef: 'knowledge-test:never-invoked', + call: async ({ request }) => ({ + succeeded: false, + error: 'the fake optimizer issued an unexpected model call', + receipt: { + model: request.model, + inputTokens: 0, + outputTokens: 0, + costUnknown: true, + usageUnknown: true, + }, + execution: { kind: 'unexpected-test-call' }, + }), budget: modelBudget, }, maxEvaluations: 1, diff --git a/tests/rag-improvement-loop.test.ts b/tests/rag-improvement-loop.test.ts index e99d408..d592202 100644 --- a/tests/rag-improvement-loop.test.ts +++ b/tests/rag-improvement-loop.test.ts @@ -35,7 +35,12 @@ describe('RAG knowledge improvement loop', () => { expect('testScenarios' in input).toBe(false) return { winnerSurface: '{"answerMode":"grounded","k":2}', - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, } @@ -198,7 +203,11 @@ describe('RAG knowledge improvement loop', () => { finalScenarioIds: ['answer-final-a', 'answer-final-b'], datasetRef: testExecutionRef('answer-final-dataset'), evaluatorRef: testExecutionRef('answer-final-evaluator'), - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + accountingComplete: true, + incompleteReasons: [], + }, } }, answerQualityCostCeiling: 0, @@ -356,7 +365,12 @@ describe('RAG knowledge improvement loop', () => { methodCalled = true return { winnerSurface: input.baselineSurface, - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, } diff --git a/tests/retrieval-eval.test.ts b/tests/retrieval-eval.test.ts index 568f413..88f95da 100644 --- a/tests/retrieval-eval.test.ts +++ b/tests/retrieval-eval.test.ts @@ -198,7 +198,12 @@ describe('retrieval eval', () => { ]) return { winnerSurface: '{\n "k": 2\n}', - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, } @@ -257,7 +262,12 @@ describe('retrieval eval', () => { methodCalled = true return { winnerSurface: input.baselineSurface, - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, } @@ -292,7 +302,12 @@ describe('retrieval eval', () => { methodCalled = true return { winnerSurface: input.baselineSurface, - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, } @@ -329,7 +344,12 @@ describe('retrieval eval', () => { async optimize() { return { winnerSurface: 'null', - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, } diff --git a/tests/support/optimization.ts b/tests/support/optimization.ts index c35f170..238cc4e 100644 --- a/tests/support/optimization.ts +++ b/tests/support/optimization.ts @@ -14,7 +14,12 @@ export function fixedOptimizationMethod( async optimize() { return { winnerSurface, - cost: { totalCostUsd: 0, accountingComplete: true, incompleteReasons: [] }, + cost: { + totalCostUsd: 0, + costProvenance: { kind: 'observed', usd: 0 }, + accountingComplete: true, + incompleteReasons: [], + }, } }, }