From 198af97a4c10dc70f1a479b8cba20a2645132b81 Mon Sep 17 00:00:00 2001 From: me2seeks Date: Mon, 20 Jul 2026 12:26:18 +0800 Subject: [PATCH] refactor(headless): extract fixed prompt WAL types Move fixed-prompt WAL schema constants and event types into a package-local leaf, then migrate production schema consumers to depend on it directly. Keep controller imports compatible through explicit re-exports and add a source contract for ownership and dependency direction. The JSONL schema and runtime behavior are unchanged. --- ...ompt-wal-types-extraction-contract.test.ts | 84 +++++ packages/headless/src/ab-run.ts | 3 +- packages/headless/src/ab-summary.ts | 6 +- packages/headless/src/ab-types.ts | 3 +- .../headless/src/fixed-prompt-controller.ts | 317 ++---------------- .../headless/src/fixed-prompt-wal-types.ts | 287 ++++++++++++++++ packages/headless/src/harbor-task-runner.ts | 3 +- packages/headless/src/prompt-ab-run.ts | 7 +- packages/headless/src/prompt-ab-types.ts | 7 +- .../headless/src/prompt-acceptance-policy.ts | 4 +- .../headless/src/prompt-candidate-loop.ts | 5 +- .../headless/src/prompt-optimization-loop.ts | 8 +- .../prompt-optimization-replay-evidence.ts | 4 +- .../src/prompt-optimization-replay-state.ts | 2 +- .../src/prompt-optimization-replay-sweeps.ts | 7 +- .../headless/src/prompt-structural-smoke.ts | 2 +- .../src/rsi-controller-attribution.ts | 4 +- packages/headless/src/rsi-round-analysis.ts | 2 +- .../src/runtime-policy-ab-lifecycle.ts | 2 +- 19 files changed, 438 insertions(+), 319 deletions(-) create mode 100644 packages/headless/src/__tests__/fixed-prompt-wal-types-extraction-contract.test.ts create mode 100644 packages/headless/src/fixed-prompt-wal-types.ts diff --git a/packages/headless/src/__tests__/fixed-prompt-wal-types-extraction-contract.test.ts b/packages/headless/src/__tests__/fixed-prompt-wal-types-extraction-contract.test.ts new file mode 100644 index 0000000000..36acaa789a --- /dev/null +++ b/packages/headless/src/__tests__/fixed-prompt-wal-types-extraction-contract.test.ts @@ -0,0 +1,84 @@ +import assert from 'node:assert/strict'; +import { existsSync } from 'node:fs'; +import { readFile } from 'node:fs/promises'; +import { join, resolve } from 'node:path'; +import { describe, test } from 'node:test'; +import { + FIXED_PROMPT_WAL_SCHEMA_VERSION as CONTROLLER_SCHEMA_VERSION, + PROMPT_CANDIDATE_FAILURE_PATTERNS as CONTROLLER_FAILURE_PATTERNS, + type FixedPromptWalEvent as ControllerWalEvent, +} from '../fixed-prompt-controller.js'; +import { + FIXED_PROMPT_WAL_SCHEMA_VERSION, + PROMPT_CANDIDATE_FAILURE_PATTERNS, + type FixedPromptWalEvent, +} from '../fixed-prompt-wal-types.js'; + +const REPO_ROOT = resolveRepoRoot(); + +async function readRepo(path: string): Promise { + return readFile(join(REPO_ROOT, path), 'utf8'); +} + +function resolveRepoRoot(): string { + const cwd = resolve(process.cwd()); + if (existsSync(join(cwd, 'packages', 'headless', 'src', 'fixed-prompt-controller.ts'))) + return cwd; + const fromWorkspace = resolve(cwd, '..', '..'); + if (existsSync(join(fromWorkspace, 'packages', 'headless', 'src', 'fixed-prompt-controller.ts'))) + return fromWorkspace; + return cwd; +} + +function acceptsWalEvent(_event: FixedPromptWalEvent): void {} + +describe('Fixed prompt WAL types extraction contract', () => { + test('the controller preserves its existing WAL exports', () => { + assert.equal(CONTROLLER_SCHEMA_VERSION, FIXED_PROMPT_WAL_SCHEMA_VERSION); + assert.strictEqual(CONTROLLER_FAILURE_PATTERNS, PROMPT_CANDIDATE_FAILURE_PATTERNS); + + const acceptsControllerEvent: (event: ControllerWalEvent) => void = acceptsWalEvent; + assert.equal(typeof acceptsControllerEvent, 'function'); + }); + + test('the WAL schema lives in a leaf with no controller or Node dependency', async () => { + const schema = await readRepo('packages/headless/src/fixed-prompt-wal-types.ts'); + + assert.match(schema, /export const FIXED_PROMPT_WAL_SCHEMA_VERSION = 1;/); + assert.match(schema, /export interface FixedPromptTaskCompletedEvent/); + assert.match(schema, /export interface FixedPromptTaskBudgetExhaustedEvent/); + assert.match(schema, /export interface PromptCandidateDecisionEvent/); + assert.match(schema, /export interface RsiControllerAttributionEvent/); + assert.match(schema, /export type FixedPromptWalEvent =/); + assert.match(schema, /export type FixedPromptTaskWalEvent =/); + assert.doesNotMatch(schema, /from '\.\/fixed-prompt-controller\.js'/); + assert.doesNotMatch(schema, /from 'node:/); + }); + + test('the controller imports the schema instead of redeclaring it', async () => { + const controller = await readRepo('packages/headless/src/fixed-prompt-controller.ts'); + + assert.match(controller, /from '\.\/fixed-prompt-wal-types\.js'/); + assert.doesNotMatch(controller, /export const FIXED_PROMPT_WAL_SCHEMA_VERSION =/); + assert.doesNotMatch(controller, /export interface FixedPromptTaskCompletedEvent/); + assert.doesNotMatch(controller, /export interface PromptCandidateDecisionEvent/); + assert.doesNotMatch(controller, /export interface RsiControllerAttributionEvent/); + assert.doesNotMatch(controller, /export type FixedPromptWalEvent =/); + assert.doesNotMatch(controller, /export type FixedPromptTaskWalEvent =/); + }); + + test('package-local schema consumers depend directly on the WAL leaf', async () => { + const consumers = await Promise.all( + [ + 'packages/headless/src/harbor-task-runner.ts', + 'packages/headless/src/prompt-candidate-loop.ts', + 'packages/headless/src/prompt-structural-smoke.ts', + 'packages/headless/src/rsi-controller-attribution.ts', + ].map((path) => readRepo(path)), + ); + + for (const consumer of consumers) { + assert.match(consumer, /from '\.\/fixed-prompt-wal-types\.js'/); + } + }); +}); diff --git a/packages/headless/src/ab-run.ts b/packages/headless/src/ab-run.ts index 3aaeeb4578..266b1f7f1d 100644 --- a/packages/headless/src/ab-run.ts +++ b/packages/headless/src/ab-run.ts @@ -1,5 +1,6 @@ import type { AbArmSpec, AbComparisonSummary, RunAbComparisonInput } from './ab-types.js'; -import type { FixedPromptTask, FixedPromptTaskWalEvent } from './fixed-prompt-controller.js'; +import type { FixedPromptTask } from './fixed-prompt-controller.js'; +import type { FixedPromptTaskWalEvent } from './fixed-prompt-wal-types.js'; import { assertFinitePositive, assertPositiveInt } from './numeric-guards.js'; import { summarizeAbComparison } from './ab-summary.js'; diff --git a/packages/headless/src/ab-summary.ts b/packages/headless/src/ab-summary.ts index 45eccdb7ac..42ba9072f4 100644 --- a/packages/headless/src/ab-summary.ts +++ b/packages/headless/src/ab-summary.ts @@ -1,7 +1,5 @@ -import { - BUDGET_EXHAUSTED_RUNTIME_UNAVAILABLE_REASON, - type FixedPromptTaskWalEvent, -} from './fixed-prompt-controller.js'; +import { BUDGET_EXHAUSTED_RUNTIME_UNAVAILABLE_REASON } from './fixed-prompt-controller.js'; +import type { FixedPromptTaskWalEvent } from './fixed-prompt-wal-types.js'; import type { HarborCellTokenSummary } from './cell-output.js'; import { assertRatio } from './numeric-guards.js'; import type { diff --git a/packages/headless/src/ab-types.ts b/packages/headless/src/ab-types.ts index 6fae17d3f6..1f67bfe7ac 100644 --- a/packages/headless/src/ab-types.ts +++ b/packages/headless/src/ab-types.ts @@ -1,4 +1,5 @@ -import type { FixedPromptTask, FixedPromptTaskWalEvent } from './fixed-prompt-controller.js'; +import type { FixedPromptTask } from './fixed-prompt-controller.js'; +import type { FixedPromptTaskWalEvent } from './fixed-prompt-wal-types.js'; import type { HarborCellContextBudgetPolicySnapshot } from './cell-output.js'; export type AbExperimentKind = 'prompt' | 'tools' | 'provider' | 'runtime' | 'harness'; diff --git a/packages/headless/src/fixed-prompt-controller.ts b/packages/headless/src/fixed-prompt-controller.ts index 01d38cbbb5..c900badaa9 100644 --- a/packages/headless/src/fixed-prompt-controller.ts +++ b/packages/headless/src/fixed-prompt-controller.ts @@ -3,28 +3,56 @@ import { appendFile, mkdir, readFile, truncate, writeFile } from 'node:fs/promis import { dirname, resolve } from 'node:path'; import { validateHarborCellOutput, - type HarborCellContextBudgetPolicySnapshot, - type HarborCellContextBudgetSummary, - type HarborCellContinuationSummary, - type HarborCellDeadlineSettlement, type HarborCellExecutionIdentity, type HarborCellOutput, - type HarborCellRuntimeRefs, - type HarborCellTaskToolSummary, type HarborCellTokenSummary, } from './cell-output.js'; import type { Config } from './contracts.js'; +import { + FIXED_PROMPT_WAL_SCHEMA_VERSION, + type FixedPromptTaskAttemptStartedEvent, + type FixedPromptTaskBudgetExhaustedEvent, + type FixedPromptTaskCompletedEvent, + type FixedPromptTaskInfraFailedEvent, + type FixedPromptTaskPlumbingFailedEvent, + type FixedPromptTaskWalEvent, + type FixedPromptWalEvent, + type HarborVerifierOutcome, + type UnscoredCellFailureClass, +} from './fixed-prompt-wal-types.js'; import { syncParentDirectory } from './immutable-file.js'; -import type { MakaChangeAuditRecord } from './change-audit.js'; import type { HarborBillingMode } from './harbor-task-runner.js'; import { assertFinitePositive, assertPositiveInt, assertRatio } from './numeric-guards.js'; import { hashHeadlessSystemPrompt } from './system-prompts.js'; -export const FIXED_PROMPT_WAL_SCHEMA_VERSION = 1; +export { + FIXED_PROMPT_WAL_SCHEMA_VERSION, + PROMPT_CANDIDATE_FAILURE_PATTERNS, +} from './fixed-prompt-wal-types.js'; +export type { + FixedPromptTaskAttemptStartedEvent, + FixedPromptTaskBudgetExhaustedEvent, + FixedPromptTaskCompletedEvent, + FixedPromptTaskInfraFailedEvent, + FixedPromptTaskPlumbingFailedEvent, + FixedPromptTaskWalEvent, + FixedPromptWalEvent, + HarborVerifierAttempt, + HarborVerifierOutcome, + PromptCandidateCommittedEvent, + PromptCandidateDecisionEvent, + PromptCandidateFailurePattern, + PromptCandidateRationale, + PromptCandidateRewardHackScan, + RsiControllerAttributionEvent, + RsiPredictedFixOutcome, + RsiRiskTaskOutcome, + RsiRootCauseSignalMatch, +} from './fixed-prompt-wal-types.js'; + export const BUDGET_EXHAUSTED_RUNTIME_UNAVAILABLE_REASON = 'budget_exhausted_before_cell_output'; const LEGACY_TIMEOUT_MISSING_EXECUTION_IDENTITY_ERROR = 'Timed-out Harbor attempt did not produce execution identity attestation'; -type UnscoredCellFailureClass = 'infra_failed' | 'setup_failed' | 'verification_error'; const walWriteTails = new Map>(); export interface FixedPromptTask { @@ -48,18 +76,6 @@ export type HarborTaskRunCellOutput = HarborCellOutput & { providerTelemetryPath?: string; }; -export interface HarborVerifierAttempt { - attempt: number; - classification: 'passed' | 'failed' | 'timeout' | 'infra_setup_failed' | 'infra_failed'; - durationMs: number; - reward?: number; -} - -export interface HarborVerifierOutcome { - outcome: 'passed' | 'failed' | 'candidate_timeout'; - attempts: HarborVerifierAttempt[]; -} - /** * The provider-neutral seam the fixed-prompt controller and A/B schedulers * consume: run one task attempt and return its reward plus cell artifacts. Two @@ -116,265 +132,6 @@ export interface ReadHarborTaskRunOutputInput { cellOutputPath: string; } -export interface FixedPromptTaskCompletedEvent { - schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; - type: 'task_completed'; - id: string; - ts: number; - runId: string; - roundId: string; - resumeFingerprint?: string; - taskId: string; - status: HarborCellOutput['status']; - passed: boolean; - scored: boolean; - eligible: boolean; - errorClass?: string; - promptHash?: string; - executionIdentity?: HarborCellExecutionIdentity; - runtimeRefs?: HarborCellRuntimeRefs; - deadlineSettlement?: HarborCellDeadlineSettlement; - tokenSummary?: HarborCellTokenSummary; - contextBudgetPolicy?: HarborCellContextBudgetPolicySnapshot; - contextBudgetSummary?: HarborCellContextBudgetSummary; - continuationSummary?: HarborCellContinuationSummary; - taskToolSummary?: HarborCellTaskToolSummary; - steps: number; - durationMs: number; - runtimeEventsPath: string; - traceEventsPath?: string; - providerTelemetryPath?: string; - harbor: { - reward: number; - verifierFailureSummary?: string; - verifier?: HarborVerifierOutcome; - }; -} - -export interface FixedPromptTaskAttemptStartedEvent { - schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; - type: 'task_attempt_started'; - id: string; - ts: number; - runId: string; - roundId: string; - resumeFingerprint?: string; - taskId: string; - promptHash: string; -} - -export interface FixedPromptTaskInfraFailedEvent { - schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; - type: 'task_infra_failed'; - id: string; - ts: number; - runId: string; - roundId: string; - resumeFingerprint?: string; - taskId: string; - status: 'infra_failed'; - passed: false; - scored: false; - eligible: false; - errorClass: - | 'infra_error' - | 'provider_billing' - | 'auth' - | 'rate_limit' - | 'provider_unavailable' - | 'network'; - error: string; - providerTelemetryPath?: string; -} - -export interface FixedPromptTaskBudgetExhaustedEvent { - schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; - type: 'task_budget_exhausted'; - id: string; - ts: number; - runId: string; - roundId: string; - resumeFingerprint?: string; - taskId: string; - status: 'budget_exhausted'; - passed: false; - scored: false; - eligible: boolean; - errorClass: 'budget_exhausted'; - error: string; - evidenceErrorClass?: - | FixedPromptTaskPlumbingFailedEvent['errorClass'] - | UnscoredCellFailureClass - | FixedPromptTaskInfraFailedEvent['errorClass']; - evidenceError?: string; - expectedPromptHash: string; - runtimeEventsPath?: string; - traceEventsPath?: string; - providerTelemetryPath?: string; - runtimeEventsUnavailableReason?: string; - tokenSummary?: HarborCellTokenSummary; - tokenSummarySource?: 'final' | 'checkpoint'; - executionIdentity?: HarborCellExecutionIdentity; - runtimeRefs?: HarborCellRuntimeRefs; - contextBudgetPolicy?: HarborCellContextBudgetPolicySnapshot; - contextBudgetSummary?: HarborCellContextBudgetSummary; - continuationSummary?: HarborCellContinuationSummary; - taskToolSummary?: HarborCellTaskToolSummary; - steps?: number; - durationMs?: number; -} - -export interface FixedPromptTaskPlumbingFailedEvent { - schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; - type: 'task_plumbing_failed'; - id: string; - ts: number; - runId: string; - roundId: string; - resumeFingerprint?: string; - taskId: string; - status: 'plumbing_failed'; - passed: false; - scored: false; - eligible: false; - errorClass: - | 'missing_token_usage' - | 'zero_cost_with_tokens' - | 'prompt_hash_mismatch' - | 'missing_prompt_hash' - | 'missing_execution_identity' - | 'execution_identity_mismatch' - | 'missing_provider_request_trace' - | 'invalid_provider_request_trace' - | 'orphaned_sampled_attempt'; - error: string; - promptHash?: string; - expectedPromptHash?: string; - runtimeRefs?: HarborCellRuntimeRefs; - tokenSummary?: HarborCellTokenSummary; - contextBudgetPolicy?: HarborCellContextBudgetPolicySnapshot; - contextBudgetSummary?: HarborCellContextBudgetSummary; - continuationSummary?: HarborCellContinuationSummary; - taskToolSummary?: HarborCellTaskToolSummary; - steps?: number; - durationMs?: number; - runtimeEventsPath?: string; - traceEventsPath?: string; - providerTelemetryPath?: string; - harbor?: { - reward: number; - }; -} - -export interface PromptCandidateCommittedEvent { - schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; - type: 'prompt_candidate_committed'; - id: string; - ts: number; - runId: string; - roundId: string; - commitSha: string; - summary: string; - promptHash: string; - heldInTaskSetHash: string; - heldInTaskIds: readonly string[]; - candidateRationaleHash: string; - candidateRationale: PromptCandidateRationale; -} - -export const PROMPT_CANDIDATE_FAILURE_PATTERNS = [ - 'coverage_regression', - 'tool_failed', - 'max_tokens', - 'runtime_error', - 'verification_failed', - 'other', -] as const; - -export type PromptCandidateFailurePattern = (typeof PROMPT_CANDIDATE_FAILURE_PATTERNS)[number]; - -export interface PromptCandidateRationale - extends MakaChangeAuditRecord< - 'system_prompt', - string, - string, - string, - PromptCandidateFailurePattern - > {} - -export type PromptCandidateRewardHackScan = - | { decision: 'clean' } - | { decision: 'quarantine'; reason: string; matchedPatterns?: readonly string[] }; - -export interface PromptCandidateDecisionEvent { - schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; - type: 'prompt_candidate_decided'; - id: string; - ts: number; - runId: string; - roundId: string; - decision: 'keep' | 'discard'; - reason: string; - candidateCommitSha: string; - previousLastKeptCommitSha: string; - lastKeptCommitSha: string; - previousHeldInReferencePassEligibleRate: number | null; - heldInReferencePassEligibleRate: number | null; - originalCommitSha: string; - originalHeldOutPassEligibleRate: number | null; - heldInPassRateNoiseBand: number; - heldOutPassRateNoiseBand: number; - rewardHackScan?: PromptCandidateRewardHackScan; - samplingPromptHash?: string; - metrics: unknown; -} - -export type RsiPredictedFixOutcome = - | 'improved' - | 'unchanged' - | 'regressed' - | 'unscored' - | 'missing'; -export type RsiRiskTaskOutcome = 'safe' | 'regressed' | 'unscored' | 'missing'; -export type RsiRootCauseSignalMatch = 'matched' | 'contradicted' | 'unknown'; - -export interface RsiControllerAttributionEvent { - schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; - type: 'rsi_controller_attribution'; - id: string; - ts: number; - runId: string; - roundId: string; - candidateCommitSha: string; - heldInTaskSetHash: string; - candidateRationaleHash: string; - evidenceRefs: readonly string[]; - predictedFixes: Array<{ taskId: string; outcome: RsiPredictedFixOutcome }>; - riskTasks: Array<{ taskId: string; outcome: RsiRiskTaskOutcome }>; - unexpectedHeldInFlips: Array<{ taskId: string; from: string; to: string }>; - decision: { - decision: 'keep' | 'discard'; - reason: string; - }; - rootCauseSignalMatch: RsiRootCauseSignalMatch; -} - -export type FixedPromptWalEvent = - | FixedPromptTaskAttemptStartedEvent - | FixedPromptTaskCompletedEvent - | FixedPromptTaskInfraFailedEvent - | FixedPromptTaskBudgetExhaustedEvent - | FixedPromptTaskPlumbingFailedEvent - | PromptCandidateCommittedEvent - | PromptCandidateDecisionEvent - | RsiControllerAttributionEvent; - -export type FixedPromptTaskWalEvent = - | FixedPromptTaskCompletedEvent - | FixedPromptTaskInfraFailedEvent - | FixedPromptTaskBudgetExhaustedEvent - | FixedPromptTaskPlumbingFailedEvent; - export interface RunFixedPromptControllerInput { runId: string; roundId: string; diff --git a/packages/headless/src/fixed-prompt-wal-types.ts b/packages/headless/src/fixed-prompt-wal-types.ts new file mode 100644 index 0000000000..6461495fc6 --- /dev/null +++ b/packages/headless/src/fixed-prompt-wal-types.ts @@ -0,0 +1,287 @@ +import type { + HarborCellContextBudgetPolicySnapshot, + HarborCellContextBudgetSummary, + HarborCellContinuationSummary, + HarborCellDeadlineSettlement, + HarborCellExecutionIdentity, + HarborCellOutput, + HarborCellRuntimeRefs, + HarborCellTaskToolSummary, + HarborCellTokenSummary, +} from './cell-output.js'; +import type { MakaChangeAuditRecord } from './change-audit.js'; + +export const FIXED_PROMPT_WAL_SCHEMA_VERSION = 1; + +export type UnscoredCellFailureClass = 'infra_failed' | 'setup_failed' | 'verification_error'; + +export interface HarborVerifierAttempt { + attempt: number; + classification: 'passed' | 'failed' | 'timeout' | 'infra_setup_failed' | 'infra_failed'; + durationMs: number; + reward?: number; +} + +export interface HarborVerifierOutcome { + outcome: 'passed' | 'failed' | 'candidate_timeout'; + attempts: HarborVerifierAttempt[]; +} + +export interface FixedPromptTaskCompletedEvent { + schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; + type: 'task_completed'; + id: string; + ts: number; + runId: string; + roundId: string; + resumeFingerprint?: string; + taskId: string; + status: HarborCellOutput['status']; + passed: boolean; + scored: boolean; + eligible: boolean; + errorClass?: string; + promptHash?: string; + executionIdentity?: HarborCellExecutionIdentity; + runtimeRefs?: HarborCellRuntimeRefs; + deadlineSettlement?: HarborCellDeadlineSettlement; + tokenSummary?: HarborCellTokenSummary; + contextBudgetPolicy?: HarborCellContextBudgetPolicySnapshot; + contextBudgetSummary?: HarborCellContextBudgetSummary; + continuationSummary?: HarborCellContinuationSummary; + taskToolSummary?: HarborCellTaskToolSummary; + steps: number; + durationMs: number; + runtimeEventsPath: string; + traceEventsPath?: string; + providerTelemetryPath?: string; + harbor: { + reward: number; + verifierFailureSummary?: string; + verifier?: HarborVerifierOutcome; + }; +} + +export interface FixedPromptTaskAttemptStartedEvent { + schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; + type: 'task_attempt_started'; + id: string; + ts: number; + runId: string; + roundId: string; + resumeFingerprint?: string; + taskId: string; + promptHash: string; +} + +export interface FixedPromptTaskInfraFailedEvent { + schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; + type: 'task_infra_failed'; + id: string; + ts: number; + runId: string; + roundId: string; + resumeFingerprint?: string; + taskId: string; + status: 'infra_failed'; + passed: false; + scored: false; + eligible: false; + errorClass: + | 'infra_error' + | 'provider_billing' + | 'auth' + | 'rate_limit' + | 'provider_unavailable' + | 'network'; + error: string; + providerTelemetryPath?: string; +} + +export interface FixedPromptTaskBudgetExhaustedEvent { + schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; + type: 'task_budget_exhausted'; + id: string; + ts: number; + runId: string; + roundId: string; + resumeFingerprint?: string; + taskId: string; + status: 'budget_exhausted'; + passed: false; + scored: false; + eligible: boolean; + errorClass: 'budget_exhausted'; + error: string; + evidenceErrorClass?: + | FixedPromptTaskPlumbingFailedEvent['errorClass'] + | UnscoredCellFailureClass + | FixedPromptTaskInfraFailedEvent['errorClass']; + evidenceError?: string; + expectedPromptHash: string; + runtimeEventsPath?: string; + traceEventsPath?: string; + providerTelemetryPath?: string; + runtimeEventsUnavailableReason?: string; + tokenSummary?: HarborCellTokenSummary; + tokenSummarySource?: 'final' | 'checkpoint'; + executionIdentity?: HarborCellExecutionIdentity; + runtimeRefs?: HarborCellRuntimeRefs; + contextBudgetPolicy?: HarborCellContextBudgetPolicySnapshot; + contextBudgetSummary?: HarborCellContextBudgetSummary; + continuationSummary?: HarborCellContinuationSummary; + taskToolSummary?: HarborCellTaskToolSummary; + steps?: number; + durationMs?: number; +} + +export interface FixedPromptTaskPlumbingFailedEvent { + schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; + type: 'task_plumbing_failed'; + id: string; + ts: number; + runId: string; + roundId: string; + resumeFingerprint?: string; + taskId: string; + status: 'plumbing_failed'; + passed: false; + scored: false; + eligible: false; + errorClass: + | 'missing_token_usage' + | 'zero_cost_with_tokens' + | 'prompt_hash_mismatch' + | 'missing_prompt_hash' + | 'missing_execution_identity' + | 'execution_identity_mismatch' + | 'missing_provider_request_trace' + | 'invalid_provider_request_trace' + | 'orphaned_sampled_attempt'; + error: string; + promptHash?: string; + expectedPromptHash?: string; + runtimeRefs?: HarborCellRuntimeRefs; + tokenSummary?: HarborCellTokenSummary; + contextBudgetPolicy?: HarborCellContextBudgetPolicySnapshot; + contextBudgetSummary?: HarborCellContextBudgetSummary; + continuationSummary?: HarborCellContinuationSummary; + taskToolSummary?: HarborCellTaskToolSummary; + steps?: number; + durationMs?: number; + runtimeEventsPath?: string; + traceEventsPath?: string; + providerTelemetryPath?: string; + harbor?: { + reward: number; + }; +} + +export interface PromptCandidateCommittedEvent { + schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; + type: 'prompt_candidate_committed'; + id: string; + ts: number; + runId: string; + roundId: string; + commitSha: string; + summary: string; + promptHash: string; + heldInTaskSetHash: string; + heldInTaskIds: readonly string[]; + candidateRationaleHash: string; + candidateRationale: PromptCandidateRationale; +} + +export const PROMPT_CANDIDATE_FAILURE_PATTERNS = [ + 'coverage_regression', + 'tool_failed', + 'max_tokens', + 'runtime_error', + 'verification_failed', + 'other', +] as const; + +export type PromptCandidateFailurePattern = (typeof PROMPT_CANDIDATE_FAILURE_PATTERNS)[number]; + +export interface PromptCandidateRationale + extends MakaChangeAuditRecord< + 'system_prompt', + string, + string, + string, + PromptCandidateFailurePattern + > {} + +export type PromptCandidateRewardHackScan = + | { decision: 'clean' } + | { decision: 'quarantine'; reason: string; matchedPatterns?: readonly string[] }; + +export interface PromptCandidateDecisionEvent { + schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; + type: 'prompt_candidate_decided'; + id: string; + ts: number; + runId: string; + roundId: string; + decision: 'keep' | 'discard'; + reason: string; + candidateCommitSha: string; + previousLastKeptCommitSha: string; + lastKeptCommitSha: string; + previousHeldInReferencePassEligibleRate: number | null; + heldInReferencePassEligibleRate: number | null; + originalCommitSha: string; + originalHeldOutPassEligibleRate: number | null; + heldInPassRateNoiseBand: number; + heldOutPassRateNoiseBand: number; + rewardHackScan?: PromptCandidateRewardHackScan; + samplingPromptHash?: string; + metrics: unknown; +} + +export type RsiPredictedFixOutcome = + | 'improved' + | 'unchanged' + | 'regressed' + | 'unscored' + | 'missing'; +export type RsiRiskTaskOutcome = 'safe' | 'regressed' | 'unscored' | 'missing'; +export type RsiRootCauseSignalMatch = 'matched' | 'contradicted' | 'unknown'; + +export interface RsiControllerAttributionEvent { + schemaVersion: typeof FIXED_PROMPT_WAL_SCHEMA_VERSION; + type: 'rsi_controller_attribution'; + id: string; + ts: number; + runId: string; + roundId: string; + candidateCommitSha: string; + heldInTaskSetHash: string; + candidateRationaleHash: string; + evidenceRefs: readonly string[]; + predictedFixes: Array<{ taskId: string; outcome: RsiPredictedFixOutcome }>; + riskTasks: Array<{ taskId: string; outcome: RsiRiskTaskOutcome }>; + unexpectedHeldInFlips: Array<{ taskId: string; from: string; to: string }>; + decision: { + decision: 'keep' | 'discard'; + reason: string; + }; + rootCauseSignalMatch: RsiRootCauseSignalMatch; +} + +export type FixedPromptWalEvent = + | FixedPromptTaskAttemptStartedEvent + | FixedPromptTaskCompletedEvent + | FixedPromptTaskInfraFailedEvent + | FixedPromptTaskBudgetExhaustedEvent + | FixedPromptTaskPlumbingFailedEvent + | PromptCandidateCommittedEvent + | PromptCandidateDecisionEvent + | RsiControllerAttributionEvent; + +export type FixedPromptTaskWalEvent = + | FixedPromptTaskCompletedEvent + | FixedPromptTaskInfraFailedEvent + | FixedPromptTaskBudgetExhaustedEvent + | FixedPromptTaskPlumbingFailedEvent; diff --git a/packages/headless/src/harbor-task-runner.ts b/packages/headless/src/harbor-task-runner.ts index 2394be5e49..4aa4b62635 100644 --- a/packages/headless/src/harbor-task-runner.ts +++ b/packages/headless/src/harbor-task-runner.ts @@ -25,9 +25,8 @@ import { type TaskRunInput, type TaskRunOutput, type TaskRunner, - type HarborVerifierAttempt, - type HarborVerifierOutcome, } from './fixed-prompt-controller.js'; +import type { HarborVerifierAttempt, HarborVerifierOutcome } from './fixed-prompt-wal-types.js'; import { HARBOR_ORACLE_EXECUTION_POLICY, HARBOR_ORACLE_MAX_ATTEMPTS, diff --git a/packages/headless/src/prompt-ab-run.ts b/packages/headless/src/prompt-ab-run.ts index b384ef7e58..d73e277341 100644 --- a/packages/headless/src/prompt-ab-run.ts +++ b/packages/headless/src/prompt-ab-run.ts @@ -1,8 +1,5 @@ -import { - runFixedPromptController, - type FixedPromptTask, - type FixedPromptTaskWalEvent, -} from './fixed-prompt-controller.js'; +import { runFixedPromptController, type FixedPromptTask } from './fixed-prompt-controller.js'; +import type { FixedPromptTaskWalEvent } from './fixed-prompt-wal-types.js'; import { renderAbComparisonMarkdown } from './ab-render.js'; import { runAbComparison } from './ab-run.js'; import { summarizeAbComparison } from './ab-summary.js'; diff --git a/packages/headless/src/prompt-ab-types.ts b/packages/headless/src/prompt-ab-types.ts index 82a1cd2f07..f764d023b0 100644 --- a/packages/headless/src/prompt-ab-types.ts +++ b/packages/headless/src/prompt-ab-types.ts @@ -10,11 +10,8 @@ import type { AbTaskComparison, AbTaskLevelSummary, } from './ab-types.js'; -import type { - FixedPromptTask, - FixedPromptTaskWalEvent, - TaskRunner, -} from './fixed-prompt-controller.js'; +import type { FixedPromptTask, TaskRunner } from './fixed-prompt-controller.js'; +import type { FixedPromptTaskWalEvent } from './fixed-prompt-wal-types.js'; export interface SummarizePromptAbComparisonInput { runId: string; diff --git a/packages/headless/src/prompt-acceptance-policy.ts b/packages/headless/src/prompt-acceptance-policy.ts index 3cd719547a..0d249c1ca7 100644 --- a/packages/headless/src/prompt-acceptance-policy.ts +++ b/packages/headless/src/prompt-acceptance-policy.ts @@ -1,12 +1,12 @@ +import { appendFixedPromptWalEvent } from './fixed-prompt-controller.js'; import { - appendFixedPromptWalEvent, FIXED_PROMPT_WAL_SCHEMA_VERSION, type FixedPromptTaskCompletedEvent, type FixedPromptWalEvent, type FixedPromptTaskWalEvent, type PromptCandidateDecisionEvent, type PromptCandidateRewardHackScan, -} from './fixed-prompt-controller.js'; +} from './fixed-prompt-wal-types.js'; export type PromptAcceptanceDecision = 'keep' | 'discard'; diff --git a/packages/headless/src/prompt-candidate-loop.ts b/packages/headless/src/prompt-candidate-loop.ts index b21a239661..548f868c28 100644 --- a/packages/headless/src/prompt-candidate-loop.ts +++ b/packages/headless/src/prompt-candidate-loop.ts @@ -4,15 +4,14 @@ import { realpathSync } from 'node:fs'; import { lstat, readdir, readFile, readlink, realpath, writeFile } from 'node:fs/promises'; import { basename, dirname, isAbsolute, relative, resolve } from 'node:path'; import { promisify } from 'node:util'; +import { appendFixedPromptWalEvent, hashSystemPrompt } from './fixed-prompt-controller.js'; import { - appendFixedPromptWalEvent, FIXED_PROMPT_WAL_SCHEMA_VERSION, - hashSystemPrompt, PROMPT_CANDIDATE_FAILURE_PATTERNS, type PromptCandidateCommittedEvent, type PromptCandidateFailurePattern, type PromptCandidateRationale, -} from './fixed-prompt-controller.js'; +} from './fixed-prompt-wal-types.js'; import { projectRsiPromptAttribution, type ProjectRsiPromptAttributionInput, diff --git a/packages/headless/src/prompt-optimization-loop.ts b/packages/headless/src/prompt-optimization-loop.ts index 3db7ef56cb..41d444905c 100644 --- a/packages/headless/src/prompt-optimization-loop.ts +++ b/packages/headless/src/prompt-optimization-loop.ts @@ -4,19 +4,21 @@ import { isDeepStrictEqual } from 'node:util'; import type { Config } from './contracts.js'; import { appendFixedPromptWalEvent, - FIXED_PROMPT_WAL_SCHEMA_VERSION, runFixedPromptController, hashSystemPrompt, readFixedPromptWal, writeFixedPromptResultsTsv, type FixedPromptControllerResult, type FixedPromptTask, + type TaskRunner, +} from './fixed-prompt-controller.js'; +import { + FIXED_PROMPT_WAL_SCHEMA_VERSION, type FixedPromptTaskCompletedEvent, type FixedPromptTaskWalEvent, - type TaskRunner, type PromptCandidateRewardHackScan, type RsiControllerAttributionEvent, -} from './fixed-prompt-controller.js'; +} from './fixed-prompt-wal-types.js'; import { extractTrajectoryDigest, runPromptCandidateRound, diff --git a/packages/headless/src/prompt-optimization-replay-evidence.ts b/packages/headless/src/prompt-optimization-replay-evidence.ts index 9beaf35b10..098a0abcee 100644 --- a/packages/headless/src/prompt-optimization-replay-evidence.ts +++ b/packages/headless/src/prompt-optimization-replay-evidence.ts @@ -1,11 +1,11 @@ import { isDeepStrictEqual } from 'node:util'; +import type { FixedPromptControllerResult } from './fixed-prompt-controller.js'; import type { - FixedPromptControllerResult, FixedPromptWalEvent, PromptCandidateCommittedEvent, PromptCandidateDecisionEvent, RsiControllerAttributionEvent, -} from './fixed-prompt-controller.js'; +} from './fixed-prompt-wal-types.js'; import type { PromptAcceptanceResult } from './prompt-acceptance-policy.js'; import { assertCandidateMatchesStableTaskSet, diff --git a/packages/headless/src/prompt-optimization-replay-state.ts b/packages/headless/src/prompt-optimization-replay-state.ts index 78a22e4fa6..eb7b662474 100644 --- a/packages/headless/src/prompt-optimization-replay-state.ts +++ b/packages/headless/src/prompt-optimization-replay-state.ts @@ -8,7 +8,7 @@ import type { FixedPromptWalEvent, PromptCandidateCommittedEvent, PromptCandidateDecisionEvent, -} from './fixed-prompt-controller.js'; +} from './fixed-prompt-wal-types.js'; import { hashCandidateRationale, hashHeldInTaskSet } from './prompt-candidate-loop.js'; const execFileAsync = promisify(execFile); diff --git a/packages/headless/src/prompt-optimization-replay-sweeps.ts b/packages/headless/src/prompt-optimization-replay-sweeps.ts index 98e2f3ec1d..5ab8d084f9 100644 --- a/packages/headless/src/prompt-optimization-replay-sweeps.ts +++ b/packages/headless/src/prompt-optimization-replay-sweeps.ts @@ -1,8 +1,5 @@ -import type { - FixedPromptControllerResult, - FixedPromptTaskWalEvent, - FixedPromptWalEvent, -} from './fixed-prompt-controller.js'; +import type { FixedPromptControllerResult } from './fixed-prompt-controller.js'; +import type { FixedPromptTaskWalEvent, FixedPromptWalEvent } from './fixed-prompt-wal-types.js'; import { isTaskEvent, taskEventMatchesPromptIdentity } from './prompt-optimization-replay-state.js'; export function replayControllerSweep(input: { diff --git a/packages/headless/src/prompt-structural-smoke.ts b/packages/headless/src/prompt-structural-smoke.ts index 40fe2f08b3..552c268a39 100644 --- a/packages/headless/src/prompt-structural-smoke.ts +++ b/packages/headless/src/prompt-structural-smoke.ts @@ -1,4 +1,4 @@ -import type { FixedPromptTaskWalEvent, FixedPromptWalEvent } from './fixed-prompt-controller.js'; +import type { FixedPromptTaskWalEvent, FixedPromptWalEvent } from './fixed-prompt-wal-types.js'; import { PROMPT_REWARD_HACK_QUARANTINE_REASON } from './prompt-acceptance-policy.js'; import { validateRsiControllerAttribution } from './rsi-controller-attribution.js'; diff --git a/packages/headless/src/rsi-controller-attribution.ts b/packages/headless/src/rsi-controller-attribution.ts index 0fb255fedc..f9794d6bb7 100644 --- a/packages/headless/src/rsi-controller-attribution.ts +++ b/packages/headless/src/rsi-controller-attribution.ts @@ -5,7 +5,7 @@ import type { RsiPredictedFixOutcome, RsiRiskTaskOutcome, RsiRootCauseSignalMatch, -} from './fixed-prompt-controller.js'; +} from './fixed-prompt-wal-types.js'; import type { PromptAcceptanceResult } from './prompt-acceptance-policy.js'; import type { RsiRoundAnalysis, RsiTaskOutcome, RsiTaskTransition } from './rsi-round-analysis.js'; @@ -13,7 +13,7 @@ export type { RsiPredictedFixOutcome, RsiRiskTaskOutcome, RsiRootCauseSignalMatch, -} from './fixed-prompt-controller.js'; +} from './fixed-prompt-wal-types.js'; export interface RsiControllerAttribution { runId: string; diff --git a/packages/headless/src/rsi-round-analysis.ts b/packages/headless/src/rsi-round-analysis.ts index 74d35d5bce..3d2446575b 100644 --- a/packages/headless/src/rsi-round-analysis.ts +++ b/packages/headless/src/rsi-round-analysis.ts @@ -2,7 +2,7 @@ import { createHash } from 'node:crypto'; import { createReadStream } from 'node:fs'; import { readFile, stat } from 'node:fs/promises'; import { createInterface } from 'node:readline'; -import type { FixedPromptTaskWalEvent } from './fixed-prompt-controller.js'; +import type { FixedPromptTaskWalEvent } from './fixed-prompt-wal-types.js'; const DEFAULT_MAX_TOOL_FAILURE_CLUSTERS = 10; const DEFAULT_MAX_TOOL_FAILURE_JSONL_BYTES = 1_000_000; diff --git a/packages/headless/src/runtime-policy-ab-lifecycle.ts b/packages/headless/src/runtime-policy-ab-lifecycle.ts index 654a34e1c7..e376561594 100644 --- a/packages/headless/src/runtime-policy-ab-lifecycle.ts +++ b/packages/headless/src/runtime-policy-ab-lifecycle.ts @@ -7,8 +7,8 @@ import { hashSystemPrompt, readFixedPromptWal, selectFixedPromptRoundTaskEvents, - type FixedPromptTaskWalEvent, } from './fixed-prompt-controller.js'; +import type { FixedPromptTaskWalEvent } from './fixed-prompt-wal-types.js'; import { runtimePolicyArmResumeFingerprint, runRuntimePolicyAbComparisonUnlocked,