diff --git a/.evolve/skill-runs.jsonl b/.evolve/skill-runs.jsonl index 8de93464..c0b3f321 100644 --- a/.evolve/skill-runs.jsonl +++ b/.evolve/skill-runs.jsonl @@ -49,3 +49,5 @@ {"skill":"/agent-eval","ts":"2026-07-30T11:43:12Z","project":"agent-eval-profile-axis","target":"expandProfileAxes canonical harness propagation","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/verify","ts":"2026-07-30T11:43:12Z","project":"agent-eval-profile-axis","target":"agent-eval expandProfileAxes harness source fix","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/stop","operatorOverride":null,"transcriptPath":null,"traceDir":null} {"skill":"/verify","ts":"2026-08-03T22:53:14Z","project":"agent-eval-official-optimizer-observations-20260803","target":"Agent Eval optimizer execution-owner contract","operatorPrompt":"","durationMin":null,"verdict":"passed","dispatchedTo":"/ship","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/agent-eval","ts":"2026-08-04T04:23:29Z","project":"agent-eval-profile-matrix-hardening-20260804","target":"profile matrix model evidence, cancellation, and cache identity","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/verify","operatorOverride":null,"transcriptPath":null,"traceDir":null} +{"skill":"/verify","ts":"2026-08-04T04:23:29Z","project":"agent-eval-profile-matrix-hardening-20260804","target":"agent-eval profile matrix hardening 0.144.3","operatorPrompt":"","durationMin":null,"verdict":"PASS","dispatchedTo":"/ship","operatorOverride":null,"transcriptPath":null,"traceDir":null} diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c5092e0..526f6b3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes to `@tangle-network/agent-eval` and its sibling `agent-eval- --- +## [0.144.3] - 2026-08-03 - exact profile matrix evidence + +### Changed + +- Campaign cells expose every distinct agent receipt model in `resolvedModels` and expose `resolvedModel` only when all agent receipts agree. +- Snapshot validation accepts Router `-MMDD` model snapshots while rejecting routing selectors such as `@preset/name`. + +### Fixed + +- `runProfileMatrix` rejects mismatched receipt models, multiple resolved snapshots within one profile, and duplicate profile identities before they can corrupt comparisons. +- A failed profile campaign now cancels active sibling campaigns through their existing abort signals instead of allowing additional paid work to continue. +- Profile campaign cache identity now includes the caller commit, optional `dispatchRef`, profile identity, and comparison config, so changed execution cannot reuse and relabel an old cell. + ## [0.144.2] - 2026-08-03 - concurrent exact profile comparison ### Changed diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 45cb818a..5a5bbcba 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "agent-eval-rpc" -version = "0.144.2" +version = "0.144.3" description = "Python RPC client, official optimizer bridge, and DSPy metric adapter for @tangle-network/agent-eval." readme = "README.md" requires-python = ">=3.10" diff --git a/clients/python/src/agent_eval_rpc/__init__.py b/clients/python/src/agent_eval_rpc/__init__.py index a8994eaa..9f5a48a0 100644 --- a/clients/python/src/agent_eval_rpc/__init__.py +++ b/clients/python/src/agent_eval_rpc/__init__.py @@ -53,7 +53,7 @@ try: __version__ = version("agent-eval-rpc") except PackageNotFoundError: - __version__ = "0.144.2" + __version__ = "0.144.3" __all__ = [ "Client", diff --git a/clients/python/uv.lock b/clients/python/uv.lock index 1a824111..5efb7723 100644 --- a/clients/python/uv.lock +++ b/clients/python/uv.lock @@ -34,7 +34,7 @@ conflicts = [[ [[package]] name = "agent-eval-rpc" -version = "0.144.2" +version = "0.144.3" source = { editable = "." } dependencies = [ { name = "filelock" }, diff --git a/docs/eval-surface-map.md b/docs/eval-surface-map.md index 7de16f20..e49bb1b3 100644 --- a/docs/eval-surface-map.md +++ b/docs/eval-surface-map.md @@ -19,7 +19,9 @@ composition point already exists (see Produced-state grading below).** When variants of the same task run inside one `runCampaign`, give those scenarios the same `seedGroup` so each repetition uses common randomness. Use `runProfileMatrix` instead when profiles are separate campaign axes. Set `maxConcurrency` for cases within one profile and `maxProfileConcurrency` for independent profile columns; results retain caller order regardless of completion order. -An execution profile may carry the provider-facing moving model alias only when each paid-call receipt supplies the related snapshot-bearing model stored in the resulting `RunRecord`. +Every paid-call receipt must match the profile, and a moving alias must resolve to one snapshot across the entire profile column. +The caller commit and profile config are always part of cache identity; set `dispatchRef` when execution behavior can change without a new commit. +A failed profile cancels active sibling columns before the matrix rejects. | `runAgentMatrix` | The bare N-axis cartesian scheduler with concurrency control. The layer beneath the eval surface: reach for it only when you need raw scheduling, not eval semantics. | cell results | Mental model: **measure** (`runCampaign`/`runEval`) → **factor** (`runProfileMatrix`) → diff --git a/package.json b/package.json index 00ee9c62..566eb7d7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-eval", - "version": "0.144.2", + "version": "0.144.3", "description": "Evaluate and improve AI agents from runs, traces, judges, and feedback. Compare candidates, cluster failures, measure lift, and gate releases.", "homepage": "https://github.com/tangle-network/agent-eval#readme", "repository": { diff --git a/src/analyst/benchmark-implementation.ts b/src/analyst/benchmark-implementation.ts index c74d9b1e..3fdd408c 100644 --- a/src/analyst/benchmark-implementation.ts +++ b/src/analyst/benchmark-implementation.ts @@ -10,7 +10,7 @@ export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_DEPENDENCY_LOCK_SHA256 = - 'bc15c7ea5cc3e78c081a8a8ec94e5588a344a77d0410fc6746bdce5de886b4cf' + 'f0d788f72ea83b6bd485f2ae61e58dda7e1d87ec79284ce245b3a7f0cc7ca812' /** The published benchmark evidence was produced at this package version, by * the retired one-shot direct runner, before trace analysts moved to the @@ -117,7 +117,7 @@ export const ANALYST_BENCHMARK_IMPLEMENTATION_FILES = Object.freeze([ ]) export const ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 = - 'd317d9e7593ddc13f22a4a1e3d62d2fa1e4591395ef3984a0c7585a75794cf8d' + '67cd83aaaa158d44a9fa32e172cbe68fa97f01ae673ccb117469688b628bb9b1' export function analystBenchmarkImplementationDigest() { return ANALYST_BENCHMARK_IMPLEMENTATION_SHA256 diff --git a/src/campaign/presets/run-profile-matrix.ts b/src/campaign/presets/run-profile-matrix.ts index 1d1ef621..fcbee538 100644 --- a/src/campaign/presets/run-profile-matrix.ts +++ b/src/campaign/presets/run-profile-matrix.ts @@ -104,6 +104,9 @@ export interface RunProfileMatrixOptions /** Git SHA the harness ran from — stamped onto every RunRecord (mandatory * for paper-grade records). */ commitSha: string + /** Additional stable identity for dispatch behavior that can change without + * changing `commitSha`, such as a caller-owned executable or remote config. */ + dispatchRef?: string /** Logical experiment id shared across the whole matrix so the promotion * gate can pair profiles on matched scenarios. Default: a hash of the * profile + scenario ids. */ @@ -229,35 +232,52 @@ interface BuildRecordArgs { ) => { prompt: string; completion: string } | undefined } -/** - * Resolve the concrete, snapshot-bearing model for a cell whose profile - * declared a moving alias or `HARNESS_NATIVE_MODEL`. The dispatch must report it via - * the model in a paid-call receipt — surfaced as `cell.resolvedModel`. Throws when it is - * missing or lacks a snapshot, so a provenance-broken row can never be - * recorded as the bare sentinel. - */ -function requireResolvedModel( +function receiptModels(cell: CampaignCellResult): string[] { + const reported = cell.resolvedModels ?? (cell.resolvedModel ? [cell.resolvedModel] : []) + return [...new Set(reported.map((model) => model.trim()).filter(Boolean))] +} + +/** Resolve and validate every paid-call model used by one cell. */ +function recordModel( cell: CampaignCellResult, profileId: string, declaredModel: string, ): string { - const resolved = cell.resolvedModel?.trim() - if (!resolved) { + const reported = receiptModels(cell) + if (modelHasSnapshot(declaredModel)) { + for (const model of reported) { + if (model !== declaredModel) { + throw new ProfileMatrixError( + `profile '${profileId}' paid-call model '${model}' for cell '${cell.cellId}' does not match its declared exact model '${declaredModel}'`, + ) + } + } + return declaredModel + } + + if (reported.length === 0) { throw new ProfileMatrixError( `profile '${profileId}' declared a moving model but its dispatch reported no resolved model for cell '${cell.cellId}' — return a snapshot-bearing model in the ctx.cost.runPaidCall receipt`, ) } - if (!modelHasSnapshot(resolved)) { - throw new ProfileMatrixError( - `profile '${profileId}' resolved to model '${resolved}' for cell '${cell.cellId}', which lacks a snapshot version — pin it (name@YYYY-MM-DD or name-YYYYMMDD) in the paid-call receipt`, - ) + for (const model of reported) { + if (!modelHasSnapshot(model)) { + throw new ProfileMatrixError( + `profile '${profileId}' resolved to model '${model}' for cell '${cell.cellId}', which lacks a snapshot version — pin it in the paid-call receipt`, + ) + } + if (declaredModel !== HARNESS_NATIVE_MODEL && !sameMovingModel(declaredModel, model)) { + throw new ProfileMatrixError( + `profile '${profileId}' declared model '${declaredModel}' but cell '${cell.cellId}' reported unrelated snapshot '${model}'`, + ) + } } - if (declaredModel !== HARNESS_NATIVE_MODEL && !sameMovingModel(declaredModel, resolved)) { + if (reported.length !== 1) { throw new ProfileMatrixError( - `profile '${profileId}' declared model '${declaredModel}' but cell '${cell.cellId}' reported unrelated snapshot '${resolved}'`, + `profile '${profileId}' cell '${cell.cellId}' reported multiple paid-call model snapshots: ${reported.join(', ')}`, ) } - return resolved + return reported[0]! } function sameMovingModel(declared: string, resolved: string): boolean { @@ -265,6 +285,7 @@ function sameMovingModel(declared: string, resolved: string): boolean { .replace(/@[^/]+$/u, '') .replace(/-\d{8}$/u, '') .replace(/-\d{4}-\d{2}-\d{2}$/u, '') + .replace(/-\d{4}$/u, '') .replace(/:date-[^/]+$/u, '') return ( resolvedBase === declared || @@ -273,19 +294,6 @@ function sameMovingModel(declared: string, resolved: string): boolean { ) } -/** Resolve the immutable model identity written to one record. A moving provider alias may - * execute, but it is never acceptable evidence unless the paid-call receipt names the pinned - * snapshot that actually served the request. */ -function recordModel( - cell: CampaignCellResult, - profileId: string, - declaredModel: string, -): string { - return modelHasSnapshot(declaredModel) - ? declaredModel - : requireResolvedModel(cell, profileId, declaredModel) -} - function buildRunRecord( args: BuildRecordArgs, ): RunRecord { @@ -340,10 +348,26 @@ export async function runProfileMatrix( const integrityMode = opts.integrity ?? 'assert' const maxProfileConcurrency = opts.maxProfileConcurrency ?? 1 const profileIds = opts.profiles.map(agentProfileId) + if (opts.dispatchRef !== undefined && opts.dispatchRef.trim().length === 0) { + throw new ProfileMatrixError('dispatchRef must be a non-empty string when provided') + } + const duplicateProfileIds = profileIds.filter((id, index) => profileIds.indexOf(id) !== index) + if (duplicateProfileIds.length > 0) { + throw new ProfileMatrixError( + `duplicate agentProfileId values are not allowed: ${[...new Set(duplicateProfileIds)].join(', ')}`, + ) + } const experimentId = opts.experimentId ?? `pm_${sha({ profileIds, scenarios: opts.scenarios.map((s) => s.id) }).slice(0, 16)}` - const matrixId = `mtx_${sha({ experimentId, profileIds, seed, splitTag }).slice(0, 16)}` + const matrixId = `mtx_${sha({ + experimentId, + profileIds, + seed, + splitTag, + commitSha: opts.commitSha, + dispatchRef: opts.dispatchRef ?? null, + }).slice(0, 16)}` // Scenario lookup for the corpus-text extractor (records carry trajectory text). const scenarioById = new Map(opts.scenarios.map((s) => [s.id, s])) @@ -389,103 +413,123 @@ export async function runProfileMatrix( const campaigns: Record> = {} const byProfile: Record = {} - const columns = await mapConcurrent(opts.profiles, maxProfileConcurrency, async (profile) => { - const profileHash = agentProfileHash(profile) - const profileId = agentProfileId(profile) - const declaredModel = agentProfileModelId(profile) - const configHash = sha({ - profile: profileHash, - judges: (opts.judges ?? []).map((j) => j.name), - seed, - splitTag, - }) - - // Bind the profile into a campaign dispatch. Name it so the campaign's - // manifest hash is stable + distinct per profile. - const dispatch = (scenario: TScenario, ctx: DispatchContext): Promise => - opts.dispatch(profile, scenario, ctx) - Object.defineProperty(dispatch, 'name', { value: `profile_${sanitize(profileId)}` }) - - const campaign = await runCampaign({ - scenarios: opts.scenarios, - dispatch, - judges: opts.judges, - seed, - reps: opts.reps, - maxConcurrency: opts.maxConcurrency, - costCeiling: opts.costCeiling, - labeledStore: opts.labeledStore, - captureSource: opts.captureSource, - storage: opts.storage, - now: opts.now, - runDir: join(opts.runDir, sanitize(profileId)), - }) - - // The canonical (profile, harness, model) identity for every record in this - // column, so results group by `groupRunsByAgentProfileCell` (harness/model - // aware). Harness comes from the axis stamp `expandProfileAxes` left on the - // profile; a profile that wasn't axis-expanded simply has no harness in its - // cell (unchanged grouping). A moving model alias means the cell identity - // must carry the resolved snapshot per cell (surfaced via - // `cell.resolvedModel`), so the pivot groups by the model that actually ran and - // the cell identity matches the RunRecord's pinned model. - const axis = harnessAxisOf(profile) - const buildCellIdentity = (cellModel: string): Promise => - buildAgentProfileCell({ - profileId, - sourceProfile: { kind: 'agent-interface-profile', hash: profileHash }, - model: cellModel, - ...(axis ? { harness: { id: axis.harness } } : {}), + const columns = await mapConcurrent( + opts.profiles, + maxProfileConcurrency, + async (profile, _index, signal) => { + const profileHash = agentProfileHash(profile) + const profileId = agentProfileId(profile) + const declaredModel = agentProfileModelId(profile) + const configHash = sha({ + profile: profileHash, + judges: (opts.judges ?? []).map((j) => j.name), + seed, + splitTag, + dispatchRef: opts.dispatchRef ?? null, }) - // A profile with a pinned declared model builds its cell identity once and - // shares it; a moving alias builds one per cell after resolution. - const sharedCellIdentity = modelHasSnapshot(declaredModel) - ? await buildCellIdentity(declaredModel) - : undefined - - const profileRecords: RunRecord[] = [] - for (const cell of campaign.cells) { - const agentProfileCell = - sharedCellIdentity ?? (await buildCellIdentity(recordModel(cell, profileId, declaredModel))) - const record = buildRunRecord({ - cell, - profile, + const dispatchRef = `profile-matrix:${sha({ + commitSha: opts.commitSha, + caller: opts.dispatchRef ?? null, + profileId, profileHash, configHash, - experimentId, - splitTag, - commitSha: opts.commitSha, - matrixId, - agentProfileCell, - scenario: scenarioById.get(cell.scenarioId), - corpusText: opts.corpusText, + })}` + + // Bind the profile into a campaign dispatch. Name it so the campaign's + // manifest hash is stable + distinct per profile. + const dispatch = (scenario: TScenario, ctx: DispatchContext): Promise => + opts.dispatch(profile, scenario, ctx) + Object.defineProperty(dispatch, 'name', { value: `profile_${sanitize(profileId)}` }) + + const campaign = await runCampaign({ + scenarios: opts.scenarios, + dispatch, + dispatchRef, + signal, + judges: opts.judges, + seed, + reps: opts.reps, + maxConcurrency: opts.maxConcurrency, + costCeiling: opts.costCeiling, + labeledStore: opts.labeledStore, + captureSource: opts.captureSource, + storage: opts.storage, + now: opts.now, + runDir: join(opts.runDir, sanitize(profileId)), }) - if (validate) validateRunRecord(record) - profileRecords.push(record) - } - const costProvenance = campaign.aggregates.cost.costProvenance - const totalCostUsd = costProvenance.kind === 'uncaptured' ? null : costProvenance.usd - return { - profileId, - campaign, - records: profileRecords, - summary: { + // The canonical (profile, harness, model) identity for every record in this + // column, so results group by `groupRunsByAgentProfileCell` (harness/model + // aware). Harness comes from the axis stamp `expandProfileAxes` left on the + // profile; a profile that wasn't axis-expanded simply has no harness in its + // cell (unchanged grouping). A moving model alias means the cell identity + // must carry the resolved snapshot per cell (surfaced via + // `cell.resolvedModels`), so the pivot groups by the model that actually ran and + // the cell identity matches the RunRecord's pinned model. + const axis = harnessAxisOf(profile) + const buildCellIdentity = (cellModel: string): Promise => + buildAgentProfileCell({ + profileId, + sourceProfile: { kind: 'agent-interface-profile', hash: profileHash }, + model: cellModel, + ...(axis ? { harness: { id: axis.harness } } : {}), + }) + // A profile with a pinned declared model builds its cell identity once and + // shares it; a moving alias builds one per cell after resolution. + const sharedCellIdentity = modelHasSnapshot(declaredModel) + ? await buildCellIdentity(declaredModel) + : undefined + + const profileRecords: RunRecord[] = [] + for (const cell of campaign.cells) { + const agentProfileCell = + sharedCellIdentity ?? + (await buildCellIdentity(recordModel(cell, profileId, declaredModel))) + const record = buildRunRecord({ + cell, + profile, + profileHash, + configHash, + experimentId, + splitTag, + commitSha: opts.commitSha, + matrixId, + agentProfileCell, + scenario: scenarioById.get(cell.scenarioId), + corpusText: opts.corpusText, + }) + if (validate) validateRunRecord(record) + profileRecords.push(record) + } + + const recordedModels = [...new Set(profileRecords.map((record) => record.model))] + if (recordedModels.length > 1) { + throw new ProfileMatrixError( + `profile '${profileId}' resolved to multiple model snapshots: ${recordedModels.join(', ')}`, + ) + } + + const costProvenance = campaign.aggregates.cost.costProvenance + const totalCostUsd = costProvenance.kind === 'uncaptured' ? null : costProvenance.usd + return { profileId, - profileHash, - model: modelHasSnapshot(declaredModel) - ? declaredModel - : (profileRecords[0]?.model ?? declaredModel), - records: profileRecords.length, - meanComposite: meanOrNull( - profileRecords.map(scoreOf).filter((score): score is number => score !== undefined), - ), - totalCostUsd, - costProvenance, - integrity: summarizeBackendIntegrity(profileRecords), - }, - } - }) + campaign, + records: profileRecords, + summary: { + profileId, + profileHash, + model: recordedModels[0] ?? declaredModel, + records: profileRecords.length, + meanComposite: meanOrNull( + profileRecords.map(scoreOf).filter((score): score is number => score !== undefined), + ), + totalCostUsd, + costProvenance, + integrity: summarizeBackendIntegrity(profileRecords), + }, + } + }, + ) // Merge in caller order so concurrency cannot change output or report ordering. for (const column of columns) { diff --git a/src/campaign/provenance.ts b/src/campaign/provenance.ts index 261509da..e41bd05e 100644 --- a/src/campaign/provenance.ts +++ b/src/campaign/provenance.ts @@ -577,6 +577,7 @@ export function campaignMeasurementDigest costProvenance: cell.costProvenance, costCallIds: [...(cell.costCallIds ?? [])].sort(), tokenUsage: cell.tokenUsage, + resolvedModels: [...(cell.resolvedModels ?? [])].sort(), resolvedModel: cell.resolvedModel ?? null, durationMs: cell.durationMs, seed: cell.seed, diff --git a/src/campaign/run-campaign.ts b/src/campaign/run-campaign.ts index b019b44a..e0868a48 100644 --- a/src/campaign/run-campaign.ts +++ b/src/campaign/run-campaign.ts @@ -14,6 +14,7 @@ import { CostAccountingIncompleteError, type CostLedgerHandle, type CostLedgerSummary, + type CostReceipt, } from '../cost-ledger' import { BackendIntegrityError, type BackendIntegrityReport } from '../integrity/backend-integrity' import { confidenceInterval } from '../statistics' @@ -419,8 +420,9 @@ async function executeCell( if (cached.status === 'hit') { const receiptProblem = cachedCellReceiptProblem(cached.cell, args.costLedger, stableCostTags) if (receiptProblem === undefined) { - enforceDispatchUsage(cached.cell, args.opts.expectUsage ?? 'warn') - return { cell: { ...cached.cell, cached: true }, artifactsByPath: {} } + const cell = withCurrentAgentModelEvidence(cached.cell, args.costLedger, stableCostTags) + enforceDispatchUsage(cell, args.opts.expectUsage ?? 'warn') + return { cell: { ...cell, cached: true }, artifactsByPath: {} } } if (!args.opts.rerunInvalidCachedCells) { throw invalidCachedCellsError([ @@ -588,7 +590,7 @@ async function executeCell( ? { cacheWrite: agentCost.cacheWriteTokens } : {}), } - const resolvedModel = agentReceipts.at(-1)?.model + const agentModelEvidence = modelEvidenceFromReceipts(agentReceipts) const dispatchResult = { cellId: args.slot.cellId, artifact, @@ -654,7 +656,7 @@ async function executeCell( costProvenance: agentCost.costProvenance, costCallIds, tokenUsage, - ...(resolvedModel ? { resolvedModel } : {}), + ...agentModelEvidence, durationMs: Date.now() - startMs, seed: args.slot.cellSeed, cached: false, @@ -1096,6 +1098,32 @@ function assertScheduleCachesReusable(args: { if (blocked.length > 0) throw invalidCachedCellsError(blocked) } +function modelEvidenceFromReceipts( + receipts: ReadonlyArray>, +): Pick, 'resolvedModels' | 'resolvedModel'> { + const models = [...new Set(receipts.map((receipt) => receipt.model))] + if (models.length === 0) return {} + return { + resolvedModels: models, + ...(models.length === 1 ? { resolvedModel: models[0] } : {}), + } +} + +function withCurrentAgentModelEvidence( + cached: CampaignCellResult, + costLedger: CostLedgerHandle, + stableCostTags: Record, +): CampaignCellResult { + const cachedCallIds = new Set(cached.costCallIds ?? []) + const receipts = costLedger + .list({ channel: 'agent', tags: stableCostTags }) + .filter((receipt) => cachedCallIds.has(receipt.callId)) + const cell = { ...cached } + delete cell.resolvedModel + delete cell.resolvedModels + return Object.assign(cell, modelEvidenceFromReceipts(receipts)) +} + function cachedCellReceiptProblem( cached: CampaignCellResult, costLedger: CostLedgerHandle, diff --git a/src/campaign/types.ts b/src/campaign/types.ts index deb55348..bbd8fd33 100644 --- a/src/campaign/types.ts +++ b/src/campaign/types.ts @@ -584,9 +584,11 @@ export interface CampaignCellResult { /** Agent-call token usage committed by `ctx.cost.runPaidCall`. * `{ input: 0, output: 0 }` when no paid agent call was recorded. */ tokenUsage: CampaignTokenUsage - /** Concrete model from the latest committed agent receipt. Consumed by - * `buildRunRecord` to pin the model when the declared profile uses a - * runtime-resolved sentinel. */ + /** Every distinct concrete model reported by committed agent receipts, in + * first-seen order. Multiple entries mean the cell used mixed models. */ + resolvedModels?: string[] + /** Concrete model when every committed agent receipt agrees. Omitted when + * the cell used multiple models. */ resolvedModel?: string durationMs: number seed: number diff --git a/src/concurrency.test.ts b/src/concurrency.test.ts index 3b468c5e..6b06ff67 100644 --- a/src/concurrency.test.ts +++ b/src/concurrency.test.ts @@ -30,6 +30,7 @@ describe('mapConcurrent', () => { let releaseFailure: (() => void) | undefined let releaseInFlight: (() => void) | undefined let releaseStarted: (() => void) | undefined + let inFlightSignal: AbortSignal | undefined const failNow = new Promise((resolve) => { releaseFailure = resolve }) @@ -41,7 +42,7 @@ describe('mapConcurrent', () => { }) let settled = false - const run = mapConcurrent([0, 1, 2, 3], 2, async (_value, index) => { + const run = mapConcurrent([0, 1, 2, 3], 2, async (_value, index, signal) => { started.push(index) if (started.length === 2) releaseStarted?.() await bothStarted @@ -49,6 +50,7 @@ describe('mapConcurrent', () => { await failNow throw firstError } + inFlightSignal = signal await finishInFlight return index }) @@ -70,6 +72,7 @@ describe('mapConcurrent', () => { releaseInFlight?.() const outcome = await run expect(outcome).toEqual({ status: 'rejected', error: firstError }) + expect(inFlightSignal?.aborted).toBe(true) expect(started).toEqual([0, 1]) }) diff --git a/src/concurrency.ts b/src/concurrency.ts index 940936a1..6baa1d21 100644 --- a/src/concurrency.ts +++ b/src/concurrency.ts @@ -121,14 +121,14 @@ export async function mapConcurrentRange(options: MapConcurrentRangeOptions( items: readonly T[], concurrency: number, - map: (item: T, index: number) => Promise, + map: (item: T, index: number, signal: AbortSignal) => Promise, ): Promise { return mapConcurrentRange({ count: items.length, maxConcurrency: concurrency, label: 'mapConcurrent', - map(index) { - return map(items[index]!, index) + map(index, signal) { + return map(items[index]!, index, signal) }, }) } diff --git a/src/contract/self-improve.ts b/src/contract/self-improve.ts index 92b072d0..d3a9497f 100644 --- a/src/contract/self-improve.ts +++ b/src/contract/self-improve.ts @@ -1024,7 +1024,13 @@ function cellsToRunRecords( const promptHash = surfaceContentHash(surface) const configHash = surfaceContentHash(candidateId) return cells.map((cell) => { - const model = cell.resolvedModel ?? fallbackModel + const receiptModels = cell.resolvedModels ?? (cell.resolvedModel ? [cell.resolvedModel] : []) + if (receiptModels.length > 1) { + throw new ValidationError( + `selfImprove cell ${cell.cellId} used multiple agent models: ${receiptModels.join(', ')}`, + ) + } + const model = receiptModels[0] ?? fallbackModel if (!model) { throw new ValidationError( `selfImprove.model is required when cell ${cell.cellId} has no paid-call model receipt`, diff --git a/src/run-record.ts b/src/run-record.ts index 307e43e7..8a670a51 100644 --- a/src/run-record.ts +++ b/src/run-record.ts @@ -614,19 +614,46 @@ function validateJudgeScores(value: unknown, path: string): void { } /** - * Heuristic snapshot check. Accepts: - * - `name@YYYY-MM-DD` (Anthropic style: `claude-sonnet-4-6@2025-04-15`) - * - `name-YYYYMMDD` (OpenAI style: `gpt-4o-2024-11-20`) - * - `name@` (allow opaque snapshots like `@v3`) - * - explicit `:date-...` Vertex-style tags - * - * Rejects bare aliases like `claude-sonnet-4` or `gpt-4o` that remap - * silently as providers ship new snapshots. + * Snapshot check for provider model identifiers. Accepts ISO and compact + * dates, Router's `-MMDD` snapshots, one opaque `@token`, and Vertex-style + * `:date-token` suffixes. Routing selectors such as `@preset/name` are not + * immutable model identities. */ export function modelHasSnapshot(model: string): boolean { - if (model.includes('@')) return true - if (/-\d{8}$/.test(model)) return true - if (/-\d{4}-\d{2}-\d{2}$/.test(model)) return true - if (/:date-/.test(model)) return true - return false + if (model.length === 0 || model.trim() !== model) return false + + const opaqueAt = model.lastIndexOf('@') + if (opaqueAt > 0) { + const base = model.slice(0, opaqueAt) + const token = model.slice(opaqueAt + 1) + if (!base.includes('@') && /^[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$/u.test(token)) { + return true + } + } + + const isoDate = model.match(/-(\d{4})-(\d{2})-(\d{2})$/u) + if (isoDate && validSnapshotDate(isoDate[1]!, isoDate[2]!, isoDate[3]!)) return true + + const compactDate = model.match(/-(\d{4})(\d{2})(\d{2})$/u) + if (compactDate && validSnapshotDate(compactDate[1]!, compactDate[2]!, compactDate[3]!)) { + return true + } + + const routerDate = model.match(/-(\d{2})(\d{2})$/u) + if (routerDate && validSnapshotDate(undefined, routerDate[1]!, routerDate[2]!)) return true + + return /:date-[A-Za-z0-9](?:[A-Za-z0-9._-]*[A-Za-z0-9])?$/u.test(model) +} + +function validSnapshotDate(year: string | undefined, month: string, day: string): boolean { + const monthNumber = Number(month) + const dayNumber = Number(day) + if (!Number.isInteger(monthNumber) || monthNumber < 1 || monthNumber > 12) return false + + const yearNumber = year === undefined ? undefined : Number(year) + const leapYear = + yearNumber === undefined || + (yearNumber % 4 === 0 && (yearNumber % 100 !== 0 || yearNumber % 400 === 0)) + const daysInMonth = [31, leapYear ? 29 : 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31] + return Number.isInteger(dayNumber) && dayNumber >= 1 && dayNumber <= daysInMonth[monthNumber - 1]! } diff --git a/src/trace-analyst/otlp-to-run-records.ts b/src/trace-analyst/otlp-to-run-records.ts index 214a3e17..9c037b55 100644 --- a/src/trace-analyst/otlp-to-run-records.ts +++ b/src/trace-analyst/otlp-to-run-records.ts @@ -40,6 +40,7 @@ */ import { + modelHasSnapshot, type RunCostProvenance, type RunRecord, type RunSplitTag, @@ -634,11 +635,3 @@ function ensureSnapshot(model: string, fallbackModel: string): string { : '@otlp' return `${model}${fallbackTag}` } - -function modelHasSnapshot(model: string): boolean { - if (model.includes('@')) return true - if (/-\d{8}$/.test(model)) return true - if (/-\d{4}-\d{2}-\d{2}$/.test(model)) return true - if (/:date-/.test(model)) return true - return false -} diff --git a/tests/campaign/run-campaign.test.ts b/tests/campaign/run-campaign.test.ts index dc110ce8..2c8ef02d 100644 --- a/tests/campaign/run-campaign.test.ts +++ b/tests/campaign/run-campaign.test.ts @@ -129,6 +129,42 @@ describe('runCampaign — core primitive', () => { ) }) + it('records every distinct agent receipt model instead of labeling mixed calls as the last one', async () => { + const models = ['model-a@2026-08-01', 'model-b@2026-08-02'] + const storage = inMemoryCampaignStorage() + const dispatch: DispatchFn = async (scenario, ctx) => { + for (const model of models) { + const paid = await ctx.cost.runPaidCall({ + actor: 'worker', + model, + execute: async () => ({ text: model, intent: scenario.intent }), + receipt: () => ({ + model, + inputTokens: 10, + outputTokens: 2, + actualCostUsd: 0.001, + }), + }) + if (!paid.succeeded) throw paid.error + } + return { text: 'done', intent: scenario.intent } + } + const options = { + scenarios: SCENARIOS.slice(0, 1), + dispatch, + runDir, + storage, + } + const result = await runCampaign(options) + const resumed = await runCampaign(options) + + expect(result.cells[0]!.resolvedModels).toEqual(models) + expect(result.cells[0]!.resolvedModel).toBeUndefined() + expect(resumed.cells[0]!.cached).toBe(true) + expect(resumed.cells[0]!.resolvedModels).toEqual(models) + expect(resumed.cells[0]!.resolvedModel).toBeUndefined() + }) + it('uses common random seeds for scenario variants in the same seed group', async () => { const observed = new Map() const scenarios: FakeScenario[] = [ diff --git a/tests/campaign/run-profile-matrix.test.ts b/tests/campaign/run-profile-matrix.test.ts index 2b3b8334..e3425d53 100644 --- a/tests/campaign/run-profile-matrix.test.ts +++ b/tests/campaign/run-profile-matrix.test.ts @@ -482,6 +482,194 @@ describe('runProfileMatrix', () => { ).rejects.toThrow(/reported unrelated snapshot/u) }) + it('rejects every mismatched paid-call model even when the last receipt matches a pinned profile', async () => { + await expect( + runProfileMatrix({ + ...baseOpts(), + profiles: [PROFILES[0]!], + scenarios: [SCENARIOS[0]!], + reps: 1, + dispatch: async (profile, scenario, ctx) => { + await paidArtifact(ctx, undefined, { + model: 'other-model@2026-08-01', + inputTokens: 5, + outputTokens: 1, + actualCostUsd: 0.001, + }) + return paidArtifact( + ctx, + { text: `${profile.name}:${scenario.id}` }, + { + model: 'test-model@2025-01-01', + inputTokens: 5, + outputTokens: 1, + actualCostUsd: 0.001, + }, + ) + }, + }), + ).rejects.toThrow(/paid-call model 'other-model@2026-08-01'.*does not match/u) + }) + + it('rejects multiple resolved snapshots across cells in one moving profile', async () => { + const moving: AgentProfile = { + name: 'moving', + model: { default: 'deepseek-v4-flash' }, + } + await expect( + runProfileMatrix({ + ...baseOpts(), + profiles: [moving], + scenarios: SCENARIOS.slice(0, 2), + reps: 1, + dispatch: async (profile, scenario, ctx) => + paidArtifact( + ctx, + { text: `${profile.name}:${scenario.id}` }, + { + model: + scenario.id === 's1' + ? 'deepseek/deepseek-v4-flash-0731' + : 'deepseek/deepseek-v4-flash-0801', + inputTokens: 12, + outputTokens: 3, + actualCostUsd: 0.001, + }, + ), + }), + ).rejects.toThrow(/resolved to multiple model snapshots/u) + }) + + it('rejects duplicate profile ids before dispatch', async () => { + let calls = 0 + await expect( + runProfileMatrix({ + ...baseOpts(), + profiles: [PROFILES[0]!, PROFILES[0]!], + dispatch: async (profile, scenario, ctx) => { + calls += 1 + return realDispatch(profile, scenario, ctx) + }, + }), + ).rejects.toThrow(/duplicate agentProfileId/u) + expect(calls).toBe(0) + }) + + it('cancels a paid sibling profile campaign when another profile fails model validation', async () => { + const profiles: AgentProfile[] = [ + { name: 'bad', model: { default: 'gpt-4o' } }, + { name: 'slow', model: { default: 'test-model@2025-01-01' } }, + { name: 'not-started', model: { default: 'test-model@2025-01-01' } }, + ] + const started: string[] = [] + let markSlowStarted: (() => void) | undefined + const slowStarted = new Promise((resolve) => { + markSlowStarted = resolve + }) + let slowCancelled = false + + await expect( + runProfileMatrix({ + ...baseOpts(), + profiles, + scenarios: [SCENARIOS[0]!], + reps: 1, + maxProfileConcurrency: 2, + dispatch: async (profile, scenario, ctx) => { + started.push(profile.name!) + if (profile.name === 'bad') { + await slowStarted + return paidArtifact( + ctx, + { text: `${profile.name}:${scenario.id}` }, + { + model: 'unrelated-model@2026-08-01', + inputTokens: 12, + outputTokens: 3, + actualCostUsd: 0.001, + }, + ) + } + const paid = await ctx.cost.runPaidCall({ + actor: 'slow-worker', + model: 'test-model@2025-01-01', + execute: (signal) => + new Promise((_resolve, reject) => { + markSlowStarted?.() + const stop = () => { + slowCancelled = true + reject(signal.reason instanceof Error ? signal.reason : new Error('cancelled')) + } + if (signal.aborted) stop() + else signal.addEventListener('abort', stop, { once: true }) + }), + receipt: () => ({ + model: 'test-model@2025-01-01', + inputTokens: 0, + outputTokens: 0, + actualCostUsd: 0, + }), + }) + if (!paid.succeeded) throw paid.error + return paid.value + }, + }), + ).rejects.toThrow(/reported unrelated snapshot/u) + + expect(slowCancelled).toBe(true) + expect(started).toEqual(['bad', 'slow']) + }) + + it('does not reuse cached cells across different caller commits', async () => { + const storage = inMemoryCampaignStorage() + const profile = PROFILES[0]! + let calls = 0 + const dispatch: ProfileDispatchFn = async ( + candidate, + scenario, + ctx, + ) => { + calls += 1 + return paidArtifact( + ctx, + { text: `${candidate.name}:${scenario.id}:${calls}` }, + { + model: 'test-model@2025-01-01', + inputTokens: 12, + outputTokens: 3, + actualCostUsd: 0.001, + }, + ) + } + const common = { + ...baseOpts(), + profiles: [profile], + scenarios: [SCENARIOS[0]!], + reps: 1, + storage, + dispatch, + } + + const first = await runProfileMatrix({ ...common, commitSha: 'commit-one' }) + const second = await runProfileMatrix({ ...common, commitSha: 'commit-two' }) + const externalChange = await runProfileMatrix({ + ...common, + commitSha: 'commit-two', + dispatchRef: 'remote-executor-v2', + }) + + expect(calls).toBe(3) + expect(first.campaigns[agentProfileId(profile)]!.manifestHash).not.toBe( + second.campaigns[agentProfileId(profile)]!.manifestHash, + ) + expect(second.campaigns[agentProfileId(profile)]!.manifestHash).not.toBe( + externalChange.campaigns[agentProfileId(profile)]!.manifestHash, + ) + expect(second.campaigns[agentProfileId(profile)]!.cells[0]!.cached).toBe(false) + expect(second.records[0]!.commitSha).toBe('commit-two') + expect(second.records[0]!.configHash).not.toBe(externalChange.records[0]!.configHash) + }) + it('runs independent profile campaigns at caller-controlled concurrency and preserves order', async () => { const profiles: AgentProfile[] = ['one', 'two', 'three'].map((name) => ({ name, diff --git a/tests/run-record.test.ts b/tests/run-record.test.ts index 4c3c05a5..e38b5c6c 100644 --- a/tests/run-record.test.ts +++ b/tests/run-record.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest' import { buildAgentProfileCell } from '../src/agent-profile-cell' import { isRunRecord, + modelHasSnapshot, parseRunRecordSafe, type RunRecord, RunRecordValidationError, @@ -194,6 +195,13 @@ describe('validateRunRecord — mandatory field enforcement', () => { expect(() => validateRunRecord(makeRecord({ model: 'claude-x-20250415' }))).not.toThrow() }) + it('accepts Router MMDD snapshots without mistaking routing presets for snapshots', () => { + expect(modelHasSnapshot('deepseek-v4-flash-0731')).toBe(true) + expect(modelHasSnapshot('moonshotai/kimi-k2-0905')).toBe(true) + expect(modelHasSnapshot('deepseek-v4-flash@preset/default')).toBe(false) + expect(modelHasSnapshot('deepseek-v4-flash-1332')).toBe(false) + }) + it('throws on non-numeric raw entry', () => { const r = makeRecord({ outcome: {