From a4ac4d90740c914bbd7e4ed95eef17b718073efc Mon Sep 17 00:00:00 2001 From: me2seeks Date: Wed, 29 Jul 2026 11:43:22 +0800 Subject: [PATCH 1/2] fix(headless): score structured verifier failures Treat valid structured pass and failure outcomes as authoritative when the agent cell exits unsuccessfully. Validate reward and final-attempt agreement, and project stored failures as scored so they remain locked on resume. Refs #1585 --- .../__tests__/fixed-prompt-controller.test.ts | 96 ++++++++++++++++++- .../src/__tests__/kimi-protocol-ab.test.ts | 4 - .../src/__tests__/pier-task-runner.test.ts | 18 ++-- .../headless/src/fixed-prompt-controller.ts | 88 ++++++++++++++--- 4 files changed, 177 insertions(+), 29 deletions(-) diff --git a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts index 8ab1932866..9cd09775a3 100644 --- a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts +++ b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts @@ -2274,7 +2274,7 @@ describe('fixed prompt controller', () => { }); }); - test('keeps Harbor verifier setup failures out of prompt scoring', async () => { + test('keeps verifier infrastructure outcomes out of prompt scoring', async () => { await withDir(async (dir) => { const systemPromptPath = join(dir, 'system_prompt.md'); await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); @@ -2291,7 +2291,19 @@ describe('fixed prompt controller', () => { harborOutput({ taskId: 'task-a', reward: 0, + status: 'failed', errorClass: 'infra_failed', + verifier: { + outcome: 'failed', + attempts: [ + { + attempt: 1, + classification: 'infra_failed', + durationMs: 20, + reward: 0, + }, + ], + }, }), now: () => 100, newId: idFactory(), @@ -2746,6 +2758,88 @@ describe('fixed prompt controller', () => { }); }); + test('keeps a structured verifier failure authoritative after an agent failure', async () => { + await withDir(async (dir) => { + const systemPromptPath = join(dir, 'system_prompt.md'); + await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); + + for (const errorClass of ['runtime_error', 'infra_failed', 'network']) { + const result = await runFixedPromptController({ + runId: `run-${errorClass}`, + roundId: 'round-1', + config, + systemPromptPath, + resultsJsonlPath: join(dir, `results-${errorClass}.jsonl`), + tasks: [{ id: 'task-a', path: '/bench/task-a' }], + taskRunner: async () => + harborOutput({ + taskId: 'task-a', + reward: 0, + status: 'failed', + errorClass, + verifier: { + outcome: 'failed', + attempts: [{ attempt: 1, classification: 'failed', durationMs: 20, reward: 0 }], + }, + }), + }); + + assert.equal(result.events[0]?.type, 'task_completed'); + assert.equal(result.events[0]?.passed, false); + assert.equal(result.events[0]?.scored, true); + assert.equal(result.events[0]?.eligible, true); + assert.equal(result.events[0]?.errorClass, errorClass); + } + }); + }); + + test('projects a stored structured verifier failure without resampling Harbor', async () => { + await withDir(async (dir) => { + const systemPromptPath = join(dir, 'system_prompt.md'); + const resultsJsonlPath = join(dir, 'results.jsonl'); + await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); + const stored = taskCompletedEvent({ taskId: 'task-a' }); + assert.equal(stored.type, 'task_completed'); + if (stored.type !== 'task_completed') throw new Error('expected completed fixture'); + await appendFixedPromptWalEvent(resultsJsonlPath, { + ...stored, + status: 'failed', + passed: false, + scored: false, + eligible: false, + errorClass: 'runtime_error', + harbor: { + reward: 0, + verifier: { + outcome: 'failed', + attempts: [{ attempt: 1, classification: 'failed', durationMs: 20, reward: 0 }], + }, + }, + }); + let harborCalls = 0; + + const result = await runFixedPromptController({ + runId: 'run-1', + roundId: 'round-1', + config, + systemPromptPath, + resultsJsonlPath, + tasks: [{ id: 'task-a', path: '/bench/task-a' }], + taskRunner: async () => { + harborCalls += 1; + return harborOutput({ taskId: 'task-a' }); + }, + }); + + assert.equal(harborCalls, 0); + assert.equal(result.events[0]?.type, 'task_completed'); + assert.equal(result.events[0]?.passed, false); + assert.equal(result.events[0]?.scored, true); + assert.equal(result.events[0]?.eligible, true); + assert.equal(result.events[0]?.errorClass, 'runtime_error'); + }); + }); + test('projects a stored structured verifier pass without resampling Harbor', async () => { await withDir(async (dir) => { const systemPromptPath = join(dir, 'system_prompt.md'); diff --git a/packages/headless/src/__tests__/kimi-protocol-ab.test.ts b/packages/headless/src/__tests__/kimi-protocol-ab.test.ts index d1092d7e37..73e60f8720 100644 --- a/packages/headless/src/__tests__/kimi-protocol-ab.test.ts +++ b/packages/headless/src/__tests__/kimi-protocol-ab.test.ts @@ -298,10 +298,6 @@ describe('Kimi protocol A/B', () => { taskRunner: async (input) => { const terminal = output(input.task.id, join(dir, 'unused-trace.jsonl')); terminal.harbor.reward = 0; - terminal.harbor.verifier = { - outcome: 'failed', - attempts: [{ attempt: 1, classification: 'failed', durationMs: 1, reward: 0 }], - }; terminal.cell.status = 'failed'; terminal.cell.errorClass = 'provider_billing'; delete terminal.cell.traceEventsPath; diff --git a/packages/headless/src/__tests__/pier-task-runner.test.ts b/packages/headless/src/__tests__/pier-task-runner.test.ts index 5ae7a65683..9ff9190af9 100644 --- a/packages/headless/src/__tests__/pier-task-runner.test.ts +++ b/packages/headless/src/__tests__/pier-task-runner.test.ts @@ -751,14 +751,9 @@ test('pier-graded failed cells stay scored through the fixed-prompt controller', }); test('pier and harbor outputs drive identical controller events for an infra-failed graded cell', async () => { - // Cross-runner parity lock for the scoring semantics INHERITED from the - // fixed-prompt controller (predating this PR): a CLI-crash cell - // (errorClass=infra_failed) with pier grade reward=0 is excluded via - // isProviderInfraFailure (scored=false), while reward=1 scores through - // structuredVerifierPassed. Whether that asymmetry is desirable is a - // controller question out of this PR's scope; the runner invariant is that - // Pier and Harbor produce controller-identical events for the same trial - // shape, so neither side can drift unilaterally. + // Cross-runner parity lock: once either harness produces a valid structured + // pass/fail grade, the verifier is authoritative even if the agent cell + // exited with an infrastructure error. await withDirs(async ({ jobsDir, repo }) => { const dir = await mkdtemp(join(tmpdir(), 'maka-pier-parity-')); try { @@ -767,6 +762,7 @@ test('pier and harbor outputs drive identical controller events for an infra-fai await writeFile(systemPromptPath, systemPrompt, 'utf8'); const promptHash = hashSystemPrompt(systemPrompt); for (const reward of [0, 1]) { + const normalizedEvents: Array> = []; const cell = cellOutput({ status: 'failed', errorClass: 'infra_failed', @@ -807,7 +803,6 @@ test('pier and harbor outputs drive identical controller events for an infra-fai }, cell: pierOutput.cell, }; - const normalizedEvents: Array> = []; for (const [flavor, output] of [ ['pier', pierOutput], ['harbor', harborOutput], @@ -828,6 +823,11 @@ test('pier and harbor outputs drive identical controller events for an infra-fai normalizedEvents.push(event); } assert.deepEqual(normalizedEvents[0], normalizedEvents[1]); + assert.equal(normalizedEvents[0]?.type, 'task_completed'); + assert.equal(normalizedEvents[0]?.passed, reward > 0); + assert.equal(normalizedEvents[0]?.scored, true); + assert.equal(normalizedEvents[0]?.eligible, true); + assert.equal(normalizedEvents[0]?.errorClass, reward > 0 ? undefined : 'infra_failed'); } } finally { await rm(dir, { recursive: true, force: true }); diff --git a/packages/headless/src/fixed-prompt-controller.ts b/packages/headless/src/fixed-prompt-controller.ts index c900badaa9..9663661bf1 100644 --- a/packages/headless/src/fixed-prompt-controller.ts +++ b/packages/headless/src/fixed-prompt-controller.ts @@ -358,7 +358,7 @@ export async function readFixedPromptWal(path: string): Promise 0 && input.output.harbor.verifier?.outcome === 'passed'; + const verifierGrade = structuredVerifierGrade(input.output.harbor); const identityMismatch = classifyExplicitIdentityMismatch( input.output.cell.executionIdentity, input.expectedPromptHash, @@ -595,7 +594,7 @@ function taskEventFromOutput(input: { error: identityMismatch.error, }); } - if (isProviderInfraFailure(input.output.cell.errorClass) && !structuredVerifierPassed) { + if (isProviderInfraFailure(input.output.cell.errorClass) && verifierGrade === undefined) { return taskInfraFailedEvent({ ...input, errorClass: input.output.cell.errorClass, @@ -633,12 +632,11 @@ function taskCompletedEvent(input: { const { output } = input; const promptHash = output.cell.promptHash ?? output.cell.executionIdentity?.systemPromptHash; const deadlineSettled = output.cell.deadlineSettlement?.source === 'benchmark.deadline'; - const structuredVerifierPassed = - output.harbor.reward > 0 && output.harbor.verifier?.outcome === 'passed'; + const verifierGrade = structuredVerifierGrade(output.harbor); const verifierGraded = output.cell.status === 'completed' || deadlineSettled || - structuredVerifierPassed || + verifierGrade !== undefined || ((output.cell.errorClass === 'max_tokens' || output.cell.errorClass === 'tool_step_cap_reached' || output.cell.errorClass === 'policy_denied') && @@ -649,7 +647,8 @@ function taskCompletedEvent(input: { : deadlineSettled ? 'budget_exhausted' : (output.cell.errorClass ?? 'verification_failed'); - const scored = verifierGraded && !isUnscoredCellFailure(errorClass); + const scored = + verifierGraded && (verifierGrade !== undefined || !isUnscoredCellFailure(errorClass)); const agentFailure = output.cell.status === 'failed' && errorClass === 'tool_step_cap_reached'; return { schemaVersion: FIXED_PROMPT_WAL_SCHEMA_VERSION, @@ -1101,13 +1100,18 @@ function projectLegacyTimeoutOutcome(event: FixedPromptWalEvent): FixedPromptWal }; } -function projectStructuredVerifierPassOutcome(event: FixedPromptWalEvent): FixedPromptWalEvent { - if ( - event.type !== 'task_completed' || - event.harbor.reward <= 0 || - event.harbor.verifier?.outcome !== 'passed' - ) - return event; +function projectStructuredVerifierOutcome(event: FixedPromptWalEvent): FixedPromptWalEvent { + if (event.type !== 'task_completed') return event; + const verifierGrade = structuredVerifierGrade(event.harbor); + if (verifierGrade === undefined) return event; + if (verifierGrade === 'failed') { + return { + ...event, + passed: false, + scored: true, + eligible: true, + }; + } const { errorClass: _legacyFailureClass, ...rest } = event; return { ...rest, @@ -1117,6 +1121,60 @@ function projectStructuredVerifierPassOutcome(event: FixedPromptWalEvent): Fixed }; } +/** + * Grants scoring authority only when the structured outcome, reward, and final + * verifier attempt agree. Harbor validates this contract while reading its + * artifact; this boundary check also protects alternate runners and stored WAL + * events from treating malformed or infrastructure-only attempts as grades. + */ +function structuredVerifierGrade(harbor: TaskRunOutput['harbor']): 'passed' | 'failed' | undefined { + const verifier = harbor.verifier; + if ( + !Number.isFinite(harbor.reward) || + !verifier || + verifier.attempts.length < 1 || + verifier.attempts.length > 2 + ) + return undefined; + if ( + verifier.attempts.some( + (attempt, index) => + attempt.attempt !== index + 1 || + !Number.isFinite(attempt.durationMs) || + attempt.durationMs < 0 || + (attempt.reward !== undefined && !Number.isFinite(attempt.reward)), + ) + ) + return undefined; + if ( + verifier.attempts + .slice(0, -1) + .some( + (attempt) => + attempt.classification !== 'infra_setup_failed' && + attempt.classification !== 'infra_failed', + ) + ) + return undefined; + + const finalAttempt = verifier.attempts.at(-1)!; + if ( + verifier.outcome === 'passed' && + harbor.reward > 0 && + finalAttempt.classification === 'passed' && + (finalAttempt.reward ?? 0) > 0 + ) + return 'passed'; + if ( + verifier.outcome === 'failed' && + harbor.reward === 0 && + finalAttempt.classification === 'failed' && + finalAttempt.reward === 0 + ) + return 'failed'; + return undefined; +} + function budgetExhaustedArtifactRefs(error: unknown): FixedPromptBudgetExhaustedArtifactRefs { if (isBudgetExhaustedError(error)) { const refs = (error as { artifactRefs?: FixedPromptBudgetExhaustedArtifactRefs }).artifactRefs; From 3a8026f5de260c7c58a32d4c9f282ffb889982cc Mon Sep 17 00:00:00 2001 From: me2seeks Date: Thu, 30 Jul 2026 09:46:16 +0800 Subject: [PATCH 2/2] fix(headless): validate stored verifier attempts Treat verifier data from runners and persisted WAL as untrusted runtime input. Missing or non-array attempts now stay on the existing ungraded path instead of throwing during resume. --- .../__tests__/fixed-prompt-controller.test.ts | 50 +++++++++++++++++++ .../headless/src/fixed-prompt-controller.ts | 24 ++++++--- 2 files changed, 66 insertions(+), 8 deletions(-) diff --git a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts index 9cd09775a3..7b584b05d8 100644 --- a/packages/headless/src/__tests__/fixed-prompt-controller.test.ts +++ b/packages/headless/src/__tests__/fixed-prompt-controller.test.ts @@ -2840,6 +2840,56 @@ describe('fixed prompt controller', () => { }); }); + test('keeps malformed stored verifier attempts on the ungraded path', async () => { + await withDir(async (dir) => { + const systemPromptPath = join(dir, 'system_prompt.md'); + await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8'); + + for (const [label, verifier] of [ + ['missing', { outcome: 'failed' }], + ['non-array', { outcome: 'failed', attempts: 'not-an-array' }], + ] as const) { + const resultsJsonlPath = join(dir, `results-${label}.jsonl`); + const stored = taskCompletedEvent({ taskId: 'task-a' }); + assert.equal(stored.type, 'task_completed'); + if (stored.type !== 'task_completed') throw new Error('expected completed fixture'); + await writeFile( + resultsJsonlPath, + `${JSON.stringify({ + ...stored, + status: 'failed', + passed: false, + scored: false, + eligible: false, + errorClass: 'infra_failed', + harbor: { reward: 0, verifier }, + })}\n`, + 'utf8', + ); + let runnerCalls = 0; + + const result = await runFixedPromptController({ + runId: 'run-1', + roundId: 'round-1', + config, + systemPromptPath, + resultsJsonlPath, + tasks: [{ id: 'task-a', path: '/bench/task-a' }], + taskRunner: async () => { + runnerCalls += 1; + return harborOutput({ taskId: 'task-a' }); + }, + }); + + assert.equal(runnerCalls, 0); + assert.equal(result.events[0]?.type, 'task_completed'); + assert.equal(result.events[0]?.scored, false); + assert.equal(result.events[0]?.eligible, false); + assert.equal(result.events[0]?.errorClass, 'infra_failed'); + } + }); + }); + test('projects a stored structured verifier pass without resampling Harbor', async () => { await withDir(async (dir) => { const systemPromptPath = join(dir, 'system_prompt.md'); diff --git a/packages/headless/src/fixed-prompt-controller.ts b/packages/headless/src/fixed-prompt-controller.ts index 9663661bf1..7f6897fb7c 100644 --- a/packages/headless/src/fixed-prompt-controller.ts +++ b/packages/headless/src/fixed-prompt-controller.ts @@ -1127,11 +1127,14 @@ function projectStructuredVerifierOutcome(event: FixedPromptWalEvent): FixedProm * artifact; this boundary check also protects alternate runners and stored WAL * events from treating malformed or infrastructure-only attempts as grades. */ -function structuredVerifierGrade(harbor: TaskRunOutput['harbor']): 'passed' | 'failed' | undefined { +function structuredVerifierGrade(harbor: unknown): 'passed' | 'failed' | undefined { + if (!isRecord(harbor) || typeof harbor.reward !== 'number' || !Number.isFinite(harbor.reward)) + return undefined; + const reward = harbor.reward; const verifier = harbor.verifier; if ( - !Number.isFinite(harbor.reward) || - !verifier || + !isRecord(verifier) || + !Array.isArray(verifier.attempts) || verifier.attempts.length < 1 || verifier.attempts.length > 2 ) @@ -1139,10 +1142,13 @@ function structuredVerifierGrade(harbor: TaskRunOutput['harbor']): 'passed' | 'f if ( verifier.attempts.some( (attempt, index) => + !isRecord(attempt) || attempt.attempt !== index + 1 || + typeof attempt.durationMs !== 'number' || !Number.isFinite(attempt.durationMs) || attempt.durationMs < 0 || - (attempt.reward !== undefined && !Number.isFinite(attempt.reward)), + (attempt.reward !== undefined && + (typeof attempt.reward !== 'number' || !Number.isFinite(attempt.reward))), ) ) return undefined; @@ -1158,18 +1164,20 @@ function structuredVerifierGrade(harbor: TaskRunOutput['harbor']): 'passed' | 'f return undefined; const finalAttempt = verifier.attempts.at(-1)!; + if (!isRecord(finalAttempt)) return undefined; + const finalReward = typeof finalAttempt.reward === 'number' ? finalAttempt.reward : undefined; if ( verifier.outcome === 'passed' && - harbor.reward > 0 && + reward > 0 && finalAttempt.classification === 'passed' && - (finalAttempt.reward ?? 0) > 0 + (finalReward ?? 0) > 0 ) return 'passed'; if ( verifier.outcome === 'failed' && - harbor.reward === 0 && + reward === 0 && finalAttempt.classification === 'failed' && - finalAttempt.reward === 0 + finalReward === 0 ) return 'failed'; return undefined;