fix(headless): lock valid structured verifier failures as scored results

packages/headless/src/__tests__/fixed-prompt-controller.test.ts

@@ -2274,7 +2274,7 @@ describe('fixed prompt controller', () => {
     });
   });
 
-  test('keeps Harbor verifier setup failures out of prompt scoring', async () => {
+  test('keeps verifier infrastructure outcomes out of prompt scoring', async () => {
     await withDir(async (dir) => {
       const systemPromptPath = join(dir, 'system_prompt.md');
       await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8');
@@ -2291,7 +2291,19 @@ describe('fixed prompt controller', () => {
           harborOutput({
             taskId: 'task-a',
             reward: 0,
+            status: 'failed',
             errorClass: 'infra_failed',
+            verifier: {
+              outcome: 'failed',
+              attempts: [
+                {
+                  attempt: 1,
+                  classification: 'infra_failed',
+                  durationMs: 20,
+                  reward: 0,
+                },
+              ],
+            },
           }),
         now: () => 100,
         newId: idFactory(),
@@ -2746,6 +2758,138 @@ describe('fixed prompt controller', () => {
     });
   });
 
+  test('keeps a structured verifier failure authoritative after an agent failure', async () => {
+    await withDir(async (dir) => {
+      const systemPromptPath = join(dir, 'system_prompt.md');
+      await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8');
+
+      for (const errorClass of ['runtime_error', 'infra_failed', 'network']) {
+        const result = await runFixedPromptController({
+          runId: `run-${errorClass}`,
+          roundId: 'round-1',
+          config,
+          systemPromptPath,
+          resultsJsonlPath: join(dir, `results-${errorClass}.jsonl`),
+          tasks: [{ id: 'task-a', path: '/bench/task-a' }],
+          taskRunner: async () =>
+            harborOutput({
+              taskId: 'task-a',
+              reward: 0,
+              status: 'failed',
+              errorClass,
+              verifier: {
+                outcome: 'failed',
+                attempts: [{ attempt: 1, classification: 'failed', durationMs: 20, reward: 0 }],
+              },
+            }),
+        });
+
+        assert.equal(result.events[0]?.type, 'task_completed');
+        assert.equal(result.events[0]?.passed, false);
+        assert.equal(result.events[0]?.scored, true);
+        assert.equal(result.events[0]?.eligible, true);
+        assert.equal(result.events[0]?.errorClass, errorClass);
+      }
+    });
+  });
+
+  test('projects a stored structured verifier failure without resampling Harbor', async () => {
+    await withDir(async (dir) => {
+      const systemPromptPath = join(dir, 'system_prompt.md');
+      const resultsJsonlPath = join(dir, 'results.jsonl');
+      await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8');
+      const stored = taskCompletedEvent({ taskId: 'task-a' });
+      assert.equal(stored.type, 'task_completed');
+      if (stored.type !== 'task_completed') throw new Error('expected completed fixture');
+      await appendFixedPromptWalEvent(resultsJsonlPath, {
+        ...stored,
+        status: 'failed',
+        passed: false,
+        scored: false,
+        eligible: false,
+        errorClass: 'runtime_error',
+        harbor: {
+          reward: 0,
+          verifier: {
+            outcome: 'failed',
+            attempts: [{ attempt: 1, classification: 'failed', durationMs: 20, reward: 0 }],
+          },
+        },
+      });
+      let harborCalls = 0;
+
+      const result = await runFixedPromptController({
+        runId: 'run-1',
+        roundId: 'round-1',
+        config,
+        systemPromptPath,
+        resultsJsonlPath,
+        tasks: [{ id: 'task-a', path: '/bench/task-a' }],
+        taskRunner: async () => {
+          harborCalls += 1;
+          return harborOutput({ taskId: 'task-a' });
+        },
+      });
+
+      assert.equal(harborCalls, 0);
+      assert.equal(result.events[0]?.type, 'task_completed');
+      assert.equal(result.events[0]?.passed, false);
+      assert.equal(result.events[0]?.scored, true);
+      assert.equal(result.events[0]?.eligible, true);
+      assert.equal(result.events[0]?.errorClass, 'runtime_error');
+    });
+  });
+
+  test('keeps malformed stored verifier attempts on the ungraded path', async () => {
+    await withDir(async (dir) => {
+      const systemPromptPath = join(dir, 'system_prompt.md');
+      await writeFile(systemPromptPath, 'fixed prompt\n', 'utf8');
+
+      for (const [label, verifier] of [
+        ['missing', { outcome: 'failed' }],
+        ['non-array', { outcome: 'failed', attempts: 'not-an-array' }],
+      ] as const) {
+        const resultsJsonlPath = join(dir, `results-${label}.jsonl`);
+        const stored = taskCompletedEvent({ taskId: 'task-a' });
+        assert.equal(stored.type, 'task_completed');
+        if (stored.type !== 'task_completed') throw new Error('expected completed fixture');
+        await writeFile(
+          resultsJsonlPath,
+          `${JSON.stringify({
+            ...stored,
+            status: 'failed',
+            passed: false,
+            scored: false,
+            eligible: false,
+            errorClass: 'infra_failed',
+            harbor: { reward: 0, verifier },
+          })}\n`,
+          'utf8',
+        );
+        let runnerCalls = 0;
+
+        const result = await runFixedPromptController({
+          runId: 'run-1',
+          roundId: 'round-1',
+          config,
+          systemPromptPath,
+          resultsJsonlPath,
+          tasks: [{ id: 'task-a', path: '/bench/task-a' }],
+          taskRunner: async () => {
+            runnerCalls += 1;
+            return harborOutput({ taskId: 'task-a' });
+          },
+        });
+
+        assert.equal(runnerCalls, 0);
+        assert.equal(result.events[0]?.type, 'task_completed');
+        assert.equal(result.events[0]?.scored, false);
+        assert.equal(result.events[0]?.eligible, false);
+        assert.equal(result.events[0]?.errorClass, 'infra_failed');
+      }
+    });
+  });
+
   test('projects a stored structured verifier pass without resampling Harbor', async () => {
     await withDir(async (dir) => {
       const systemPromptPath = join(dir, 'system_prompt.md');

packages/headless/src/__tests__/kimi-protocol-ab.test.ts

@@ -298,10 +298,6 @@ describe('Kimi protocol A/B', () => {
       taskRunner: async (input) => {
         const terminal = output(input.task.id, join(dir, 'unused-trace.jsonl'));
         terminal.harbor.reward = 0;
-        terminal.harbor.verifier = {
-          outcome: 'failed',
-          attempts: [{ attempt: 1, classification: 'failed', durationMs: 1, reward: 0 }],
-        };
         terminal.cell.status = 'failed';
         terminal.cell.errorClass = 'provider_billing';
         delete terminal.cell.traceEventsPath;

packages/headless/src/__tests__/pier-task-runner.test.ts

@@ -751,14 +751,9 @@ test('pier-graded failed cells stay scored through the fixed-prompt controller',
 });
 
 test('pier and harbor outputs drive identical controller events for an infra-failed graded cell', async () => {
-  // Cross-runner parity lock for the scoring semantics INHERITED from the
-  // fixed-prompt controller (predating this PR): a CLI-crash cell
-  // (errorClass=infra_failed) with pier grade reward=0 is excluded via
-  // isProviderInfraFailure (scored=false), while reward=1 scores through
-  // structuredVerifierPassed. Whether that asymmetry is desirable is a
-  // controller question out of this PR's scope; the runner invariant is that
-  // Pier and Harbor produce controller-identical events for the same trial
-  // shape, so neither side can drift unilaterally.
+  // Cross-runner parity lock: once either harness produces a valid structured
+  // pass/fail grade, the verifier is authoritative even if the agent cell
+  // exited with an infrastructure error.
   await withDirs(async ({ jobsDir, repo }) => {
     const dir = await mkdtemp(join(tmpdir(), 'maka-pier-parity-'));
     try {
@@ -767,6 +762,7 @@ test('pier and harbor outputs drive identical controller events for an infra-fai
       await writeFile(systemPromptPath, systemPrompt, 'utf8');
       const promptHash = hashSystemPrompt(systemPrompt);
       for (const reward of [0, 1]) {
+        const normalizedEvents: Array<Record<string, unknown>> = [];
         const cell = cellOutput({
           status: 'failed',
           errorClass: 'infra_failed',
@@ -807,7 +803,6 @@ test('pier and harbor outputs drive identical controller events for an infra-fai
           },
           cell: pierOutput.cell,
         };
-        const normalizedEvents: Array<Record<string, unknown>> = [];
         for (const [flavor, output] of [
           ['pier', pierOutput],
           ['harbor', harborOutput],
@@ -828,6 +823,11 @@ test('pier and harbor outputs drive identical controller events for an infra-fai
           normalizedEvents.push(event);
         }
         assert.deepEqual(normalizedEvents[0], normalizedEvents[1]);
+        assert.equal(normalizedEvents[0]?.type, 'task_completed');
+        assert.equal(normalizedEvents[0]?.passed, reward > 0);
+        assert.equal(normalizedEvents[0]?.scored, true);
+        assert.equal(normalizedEvents[0]?.eligible, true);
+        assert.equal(normalizedEvents[0]?.errorClass, reward > 0 ? undefined : 'infra_failed');
       }
     } finally {
       await rm(dir, { recursive: true, force: true });

packages/headless/src/fixed-prompt-controller.ts

@@ -358,7 +358,7 @@ export async function readFixedPromptWal(path: string): Promise<FixedPromptWalEv
       throw error;
     }
   }
-  return events.map(projectLegacyTimeoutOutcome).map(projectStructuredVerifierPassOutcome);
+  return events.map(projectLegacyTimeoutOutcome).map(projectStructuredVerifierOutcome);
 }
 
 export async function readHarborTaskRunOutput(
@@ -580,8 +580,7 @@ function taskEventFromOutput(input: {
   | FixedPromptTaskCompletedEvent
   | FixedPromptTaskPlumbingFailedEvent
   | FixedPromptTaskInfraFailedEvent {
-  const structuredVerifierPassed =
-    input.output.harbor.reward > 0 && input.output.harbor.verifier?.outcome === 'passed';
+  const verifierGrade = structuredVerifierGrade(input.output.harbor);
   const identityMismatch = classifyExplicitIdentityMismatch(
     input.output.cell.executionIdentity,
     input.expectedPromptHash,
@@ -595,7 +594,7 @@ function taskEventFromOutput(input: {
       error: identityMismatch.error,
     });
   }
-  if (isProviderInfraFailure(input.output.cell.errorClass) && !structuredVerifierPassed) {
+  if (isProviderInfraFailure(input.output.cell.errorClass) && verifierGrade === undefined) {
     return taskInfraFailedEvent({
       ...input,
       errorClass: input.output.cell.errorClass,
@@ -633,12 +632,11 @@ function taskCompletedEvent(input: {
   const { output } = input;
   const promptHash = output.cell.promptHash ?? output.cell.executionIdentity?.systemPromptHash;
   const deadlineSettled = output.cell.deadlineSettlement?.source === 'benchmark.deadline';
-  const structuredVerifierPassed =
-    output.harbor.reward > 0 && output.harbor.verifier?.outcome === 'passed';
+  const verifierGrade = structuredVerifierGrade(output.harbor);
   const verifierGraded =
     output.cell.status === 'completed' ||
     deadlineSettled ||
-    structuredVerifierPassed ||
+    verifierGrade !== undefined ||
     ((output.cell.errorClass === 'max_tokens' ||
       output.cell.errorClass === 'tool_step_cap_reached' ||
       output.cell.errorClass === 'policy_denied') &&
@@ -649,7 +647,8 @@ function taskCompletedEvent(input: {
     : deadlineSettled
       ? 'budget_exhausted'
       : (output.cell.errorClass ?? 'verification_failed');
-  const scored = verifierGraded && !isUnscoredCellFailure(errorClass);
+  const scored =
+    verifierGraded && (verifierGrade !== undefined || !isUnscoredCellFailure(errorClass));
   const agentFailure = output.cell.status === 'failed' && errorClass === 'tool_step_cap_reached';
   return {
     schemaVersion: FIXED_PROMPT_WAL_SCHEMA_VERSION,
@@ -1101,13 +1100,18 @@ function projectLegacyTimeoutOutcome(event: FixedPromptWalEvent): FixedPromptWal
   };
 }
 
-function projectStructuredVerifierPassOutcome(event: FixedPromptWalEvent): FixedPromptWalEvent {
-  if (
-    event.type !== 'task_completed' ||
-    event.harbor.reward <= 0 ||
-    event.harbor.verifier?.outcome !== 'passed'
-  )
-    return event;
+function projectStructuredVerifierOutcome(event: FixedPromptWalEvent): FixedPromptWalEvent {
+  if (event.type !== 'task_completed') return event;
+  const verifierGrade = structuredVerifierGrade(event.harbor);
+  if (verifierGrade === undefined) return event;
+  if (verifierGrade === 'failed') {
+    return {
+      ...event,
+      passed: false,
+      scored: true,
+      eligible: true,
+    };
+  }
   const { errorClass: _legacyFailureClass, ...rest } = event;
   return {
     ...rest,
@@ -1117,6 +1121,68 @@ function projectStructuredVerifierPassOutcome(event: FixedPromptWalEvent): Fixed
   };
 }
 
+/**
+ * Grants scoring authority only when the structured outcome, reward, and final
+ * verifier attempt agree. Harbor validates this contract while reading its
+ * artifact; this boundary check also protects alternate runners and stored WAL
+ * events from treating malformed or infrastructure-only attempts as grades.
+ */
+function structuredVerifierGrade(harbor: unknown): 'passed' | 'failed' | undefined {
+  if (!isRecord(harbor) || typeof harbor.reward !== 'number' || !Number.isFinite(harbor.reward))
+    return undefined;
+  const reward = harbor.reward;
+  const verifier = harbor.verifier;
+  if (
+    !isRecord(verifier) ||
+    !Array.isArray(verifier.attempts) ||
+    verifier.attempts.length < 1 ||
+    verifier.attempts.length > 2
+  )
+    return undefined;
+  if (
+    verifier.attempts.some(
+      (attempt, index) =>
+        !isRecord(attempt) ||
+        attempt.attempt !== index + 1 ||
+        typeof attempt.durationMs !== 'number' ||
+        !Number.isFinite(attempt.durationMs) ||
+        attempt.durationMs < 0 ||
+        (attempt.reward !== undefined &&
+          (typeof attempt.reward !== 'number' || !Number.isFinite(attempt.reward))),
+    )
+  )
+    return undefined;
+  if (
+    verifier.attempts
+      .slice(0, -1)
+      .some(
+        (attempt) =>
+          attempt.classification !== 'infra_setup_failed' &&
+          attempt.classification !== 'infra_failed',
+      )
+  )
+    return undefined;
+
+  const finalAttempt = verifier.attempts.at(-1)!;
+  if (!isRecord(finalAttempt)) return undefined;
+  const finalReward = typeof finalAttempt.reward === 'number' ? finalAttempt.reward : undefined;
+  if (
+    verifier.outcome === 'passed' &&
+    reward > 0 &&
+    finalAttempt.classification === 'passed' &&
+    (finalReward ?? 0) > 0
+  )
+    return 'passed';
+  if (
+    verifier.outcome === 'failed' &&
+    reward === 0 &&
+    finalAttempt.classification === 'failed' &&
+    finalReward === 0
+  )
+    return 'failed';
+  return undefined;
+}
+
 function budgetExhaustedArtifactRefs(error: unknown): FixedPromptBudgetExhaustedArtifactRefs {
   if (isBudgetExhaustedError(error)) {
     const refs = (error as { artifactRefs?: FixedPromptBudgetExhaustedArtifactRefs }).artifactRefs;