diff --git a/.gitignore b/.gitignore index d9c889fe..a514b142 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ corpus/ test_repo/ .sic-run-*/ .swe-run-*/ +.gen2-runs/ +.gen3-runs/ diff --git a/bench/src/agent-graphs-gen2.mts b/bench/src/agent-graphs-gen2.mts index 81b8466c..6f1840aa 100644 --- a/bench/src/agent-graphs-gen2.mts +++ b/bench/src/agent-graphs-gen2.mts @@ -24,7 +24,7 @@ * Smoke: GEN2_SMOKE=1 pnpm tsx src/agent-graphs-gen2.mts — stubs both LLM calls, * exercises the full loop wiring + gate + report at zero cost. * - * Writes skills/agent-graphs/gen2.json; on ship, replaces SKILL.md with v2. + * Writes skills/agent-graphs/generations/gen2.json; on ship, replaces SKILL.md with v2. */ import { createHash } from 'node:crypto' @@ -56,7 +56,7 @@ import { const HERE = dirname(fileURLToPath(import.meta.url)) const REPO = join(HERE, '..', '..') const SKILL_PATH = join(REPO, 'skills', 'agent-graphs', 'SKILL.md') -const OUT_PATH = join(REPO, 'skills', 'agent-graphs', 'gen2.json') +const OUT_PATH = join(REPO, 'skills', 'agent-graphs', 'generations', 'gen2.json') const RUNS_ROOT = join(REPO, '.gen2-runs') const SMOKE = process.env.GEN2_SMOKE === '1' diff --git a/bench/src/agent-graphs-gen3.mts b/bench/src/agent-graphs-gen3.mts new file mode 100644 index 00000000..60dfa39e --- /dev/null +++ b/bench/src/agent-graphs-gen3.mts @@ -0,0 +1,660 @@ +/** + * Generation v3 of the agent-graphs skill improvement loop — same composition as + * agent-graphs-gen2.mts (agent-eval's `runImprovementLoop` + the two caller closures + * from agent-graphs-improve.mts), with the gen3 protocol deltas: + * + * • baseline surface is the PROMOTED v2 SKILL.md (sha asserted at startup); + * • TRAIN grew to 7 cases (the 5 prior + artifact-mission-release-notes and + * audited-single-writer, both targeting the residual mission-in-deliverable + * under-graphing cluster); HOLDOUT is unchanged and never enters the prompt; + * • k=5 reps per case per surface, sequential, author temp 0.2; + * • transient router 5xx: bounded per-cell retry (up to 3, receipts kept) BEFORE a + * cell is declared failed — the #723 workaround for the gen2 503 cell loss; + * • the revision prompt carries ONLY the measured k=5 v2 TRAIN failures plus a + * mechanical per-check failure tally — no inherited failure-cluster narrative. + * + * Gate: ship iff v3 holdout mean > v2 holdout mean + * and v3 train mean >= v2 train mean - 0.05 + * and neither anti-over-graphing case regresses + * (single-agent-suffices, runtime-discovered-fanout). + * + * Run: pnpm tsx src/agent-graphs-gen3.mts (from bench/) + * Smoke: GEN3_SMOKE=1 pnpm tsx src/agent-graphs-gen3.mts — stubs both LLM calls; + * writes into .gen3-runs/ only, never the tracked artifact. + * + * Writes skills/agent-graphs/generations/gen3.json; on ship, replaces SKILL.md with v3. + */ + +import { createHash } from 'node:crypto' +import { mkdirSync, writeFileSync } from 'node:fs' +import { join, dirname } from 'node:path' +import { fileURLToPath } from 'node:url' +import { setTimeout as sleep } from 'node:timers/promises' +import { + runEval, + runImprovementLoop, + type CampaignResult, + type DispatchContext, + type Gate, + type GateContext, + type JudgeConfig, + type MutableSurface, + type ProposeContext, + type ProposedCandidate, + type SurfaceProposer, +} from '@tangle-network/agent-eval/campaign' +import { + type AuthoredArtifact, + type CaseSpec, + callAuthor, + dispatchWithSurface, + judgeArtifact, + loadInputs, +} from './agent-graphs-improve.mts' + +const HERE = dirname(fileURLToPath(import.meta.url)) +const REPO = join(HERE, '..', '..') +const SKILL_PATH = join(REPO, 'skills', 'agent-graphs', 'SKILL.md') +const OUT_PATH = join(REPO, 'skills', 'agent-graphs', 'generations', 'gen3.json') +const RUNS_ROOT = join(REPO, '.gen3-runs') +const SMOKE = process.env.GEN3_SMOKE === '1' +// Smoke runs must never clobber the tracked generation record. +const EFFECTIVE_OUT = SMOKE ? join(RUNS_ROOT, 'gen3-smoke.json') : OUT_PATH + +// The promoted v2 surface this generation improves on (generations/gen2.json surfaces.v2Sha256). +const EXPECTED_V2_SHA = '4c6615b6164f6c5a86efb2596556bdf325d33f08a4e1715cae9d71cb28b6255e' + +const K = 5 +const SEED = 42 +const TRAIN_IDS = [ + 'floor-trap-pi', + 'review-pipeline', + 'single-agent-suffices', + 'cap-as-stop-mistake', + 'runtime-discovered-fanout', + 'artifact-mission-release-notes', + 'audited-single-writer', +] as const +const HOLDOUT_IDS = ['mission-in-deliverable', 'steer-heavy-drafting', 'unmeasured-harness'] as const +// The anti-over-graphing cases an "always graph" hack would regress on. +const DEGENERATE_IDS = ['single-agent-suffices', 'runtime-discovered-fanout'] as const +// Holdout-brief phrases that must never reach the reviser (belt over the whole-brief check). +const BANNED_PROMPT_STRINGS = ['CHANGELOG', 'redirect it up to five times', 'three probes on claude-code'] as const + +type GraphScenario = CaseSpec & { kind: 'agent-graph-case' } +type CellArtifact = AuthoredArtifact & { repIndex: number; surfaceSha: string } + +function sha256(text: string): string { + return createHash('sha256').update(text).digest('hex') +} + +// ── Captured evidence (fed to the proposer; TRAIN-filtered at prompt build) ──── + +interface JudgedRecord { + surfaceSha: string + scenarioId: string + rep: number + decision: string + score: number + failures: string[] + validationError?: string +} + +const judged: JudgedRecord[] = [] + +function makeJudge(): JudgeConfig { + return { + name: 'deterministic-expect', + judgeVersion: 'gen3-1', + dimensions: [{ key: 'expect', description: 'fraction of case expectations satisfied' }], + score({ artifact, scenario }) { + const { score, reasons } = judgeArtifact(artifact, scenario) + judged.push({ + surfaceSha: artifact.surfaceSha, + scenarioId: scenario.id, + rep: artifact.repIndex, + decision: artifact.decision, + score, + failures: reasons.filter((r) => !r.startsWith('PASS')), + ...(artifact.validationError !== undefined ? { validationError: artifact.validationError } : {}), + }) + return { dimensions: { expect: score }, composite: score, notes: reasons.join('\n') } + }, + } +} + +// ── Dispatch: closure A + the bounded transient-retry policy (#723 workaround) ─ + +const MAX_TRANSIENT_RETRIES = 3 +const TRANSIENT_PATTERN = + /HTTP 5\d\d|platform_unreachable|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|fetch failed|socket|TimeoutError|aborted|empty content/i + +interface RetryReceipt { + surfaceSha12: string + scenarioId: string + rep: number + attempt: number + error: string + at: string +} + +interface CellFailure { + surfaceSha12: string + scenarioId: string + rep: number + attempts: number + error: string +} + +const retryReceipts: RetryReceipt[] = [] +const cellFailures: CellFailure[] = [] + +function smokeArtifact(scenario: GraphScenario): AuthoredArtifact { + return { decision: 'single-agent', reason: `smoke stub for ${scenario.id}`, raw: '{}' } +} + +async function dispatchCell( + surface: MutableSurface, + scenario: GraphScenario, + ctx: DispatchContext, +): Promise { + if (typeof surface !== 'string') throw new Error('gen3 surfaces are strings') + const surfaceSha = sha256(surface) + let lastErr: unknown + for (let attempt = 1; attempt <= 1 + MAX_TRANSIENT_RETRIES; attempt += 1) { + try { + const artifact = SMOKE ? smokeArtifact(scenario) : await dispatchWithSurface(surface, scenario) + return { ...artifact, repIndex: ctx.rep, surfaceSha } + } catch (err) { + lastErr = err + const message = err instanceof Error ? err.message : String(err) + const transient = TRANSIENT_PATTERN.test(message) + if (!transient || attempt > MAX_TRANSIENT_RETRIES) break + retryReceipts.push({ + surfaceSha12: surfaceSha.slice(0, 12), + scenarioId: scenario.id, + rep: ctx.rep, + attempt, + error: message.slice(0, 300), + at: new Date().toISOString(), + }) + await sleep(5_000 * attempt) + } + } + const message = lastErr instanceof Error ? lastErr.message : String(lastErr) + cellFailures.push({ + surfaceSha12: surfaceSha.slice(0, 12), + scenarioId: scenario.id, + rep: ctx.rep, + attempts: Math.min(1 + MAX_TRANSIENT_RETRIES, retryReceipts.filter((r) => r.scenarioId === scenario.id && r.rep === ctx.rep && r.surfaceSha12 === surfaceSha.slice(0, 12)).length + 1), + error: message.slice(0, 300), + }) + throw lastErr +} + +// ── The reviser proposer (prompt = v2 text + measured k=5 TRAIN failures ONLY) ─ + +let revisionPrompt = '' +let revisionPromptSha256 = '' + +function failKey(line: string): string { + return line.match(/^FAIL ([^:]+):/)?.[1] ?? line.split(':')[0] ?? line.slice(0, 40) +} + +/** Mechanical per-case tally of failing checks in the baseline's TRAIN measurements. */ +function tallyTrainFailures(baselineSha: string): Record> { + const tally: Record> = {} + for (const r of judged) { + if (r.surfaceSha !== baselineSha) continue + if (!(TRAIN_IDS as readonly string[]).includes(r.scenarioId)) continue + for (const f of r.failures) { + const key = failKey(f) + tally[r.scenarioId] = tally[r.scenarioId] ?? {} + tally[r.scenarioId][key] = (tally[r.scenarioId][key] ?? 0) + 1 + } + } + return tally +} + +function buildRevisionPrompt(v2Surface: string, trainCases: GraphScenario[]): string { + const v2Sha = sha256(v2Surface) + const caseBlocks = trainCases.map((kase) => { + const rows = judged + .filter((r) => r.surfaceSha === v2Sha && r.scenarioId === kase.id) + .sort((a, b) => a.rep - b.rep) + .map((r) => { + const fails = r.failures.length > 0 ? r.failures.join('\n ') : '(all checks passed)' + return ` rep ${r.rep}: decision=${r.decision} score=${r.score.toFixed(2)}\n ${fails}` + }) + return [``, `brief: ${kase.brief}`, `measured (k=${K}):`, ...rows, ''].join('\n') + }) + const tally = tallyTrainFailures(v2Sha) + const tallyLines = trainCases.map((kase) => { + const byKey = tally[kase.id] + if (!byKey || Object.keys(byKey).length === 0) return ` ${kase.id}: clean (no failing checks)` + const parts = Object.entries(byKey) + .sort((a, b) => b[1] - a[1]) + .map(([key, count]) => `${key} failed ${count}/${K} reps`) + return ` ${kase.id}: ${parts.join(', ')}` + }) + return [ + 'You are revising an agent-skill document. The skill below ("v2") instructs a model to author', + `agent graphs (or decline to) from loose case briefs. It was measured k=${K} per case against a`, + 'deterministic scorer; the per-rep results for the training cases are listed after the text.', + '', + '', + v2Surface, + '', + '', + 'Measured training results:', + '', + ...caseBlocks, + '', + 'Mechanical failure tally (check -> failed reps, from the measurements above; this tally is the', + 'ONLY ground truth about what is failing — do not assume any earlier generation\'s failure', + 'clusters still hold):', + ...tallyLines, + '', + 'Rewrite the skill into v3 targeting exactly the failing checks in the tally. For each failing', + 'check, find the doctrine gap that lets the author fail it and close that gap. Leave the clean', + 'cases\' behavior alone.', + '', + 'Hard constraints:', + '- Keep the YAML frontmatter: `name: agent-graphs` unchanged; `description:` must be a single', + ' line of at most 96 characters.', + '- Total file must stay under 20000 bytes.', + '- Keep the decision honest: "single-agent" and "dynamic-workflow" remain the CORRECT answers', + ' when one profile suffices or when topology is discovered mid-run. Do not teach "always', + ' graph" — fixing under-graphing must not create over-graphing.', + '- Keep the existing correct doctrine (traversal caps, analyzes-cap-is-not-a-stop, budget', + ' floors, deliverable-carries-mission, offline proving) — sharpen it, do not delete it.', + '- The skill is consumed by a model that must output a strict JSON graph spec; keep the text', + ' operational, not narrative.', + '', + 'Reply with the COMPLETE revised SKILL.md between the markers, nothing else:', + '<<>>', + ].join('\n') +} + +function extractSkill(reply: string): string { + const m = reply.match(/<<>>/) + if (!m?.[1]) throw new Error('proposer reply carries no <<>> block') + return `${m[1].trim()}\n` +} + +function validateSkillGate(text: string): string[] { + const problems: string[] = [] + const fm = text.match(/^---\n([\s\S]*?)\n---(?:\n|$)/)?.[1] + if (!fm) problems.push('missing YAML frontmatter') + const name = fm?.match(/^name:\s*(.+)$/m)?.[1]?.trim() + if (name !== 'agent-graphs') problems.push(`frontmatter name is ${JSON.stringify(name)}, expected agent-graphs`) + const description = fm?.match(/^description:\s*(.+)$/m)?.[1]?.trim().replace(/^["']|["']$/g, '') + if (!description) problems.push('frontmatter description missing') + else if (description.length > 96) problems.push(`description is ${description.length} chars (max 96)`) + if (Buffer.byteLength(text) > 20_000) problems.push(`file is ${Buffer.byteLength(text)} bytes (max 20000)`) + return problems +} + +function assertNoHoldoutLeak(prompt: string): void { + for (const id of HOLDOUT_IDS) { + if (prompt.includes(id)) throw new Error(`holdout id '${id}' leaked into the revision prompt`) + } + const allCases = loadInputs().cases + for (const id of HOLDOUT_IDS) { + const brief = allCases.find((c) => c.id === id)?.brief + if (brief && prompt.includes(brief)) { + throw new Error(`holdout brief for '${id}' leaked into the revision prompt`) + } + } + for (const phrase of BANNED_PROMPT_STRINGS) { + if (prompt.includes(phrase)) { + throw new Error(`banned holdout phrase '${phrase}' leaked into the revision prompt`) + } + } +} + +function makeProposer(v2Surface: string, trainCases: GraphScenario[]): SurfaceProposer { + return { + kind: 'agent-graphs-skill-reviser', + async propose(_ctx: ProposeContext): Promise { + revisionPrompt = buildRevisionPrompt(v2Surface, trainCases) + assertNoHoldoutLeak(revisionPrompt) + revisionPromptSha256 = sha256(revisionPrompt) + if (SMOKE) { + return [ + { + surface: v2Surface.replace( + '# Agent graphs', + '# Agent graphs\n\n(smoke marker: candidate differs from baseline)', + ), + label: 'smoke-candidate', + rationale: 'zero-cost wiring check', + }, + ] + } + let prompt = revisionPrompt + let lastProblems: string[] = [] + for (let attempt = 0; attempt < 2; attempt += 1) { + const reply = await callAuthor(prompt, 0.7, 12_000) + const skill = extractSkill(reply) + lastProblems = validateSkillGate(skill) + if (lastProblems.length === 0) { + return [ + { + surface: skill, + label: 'gen3-revision', + rationale: 'glm-5.2 rewrite targeting the k=5-measured failing checks on the 7 train cases', + }, + ] + } + prompt = `${revisionPrompt}\n\nYour previous attempt violated: ${lastProblems.join('; ')}. Fix these and reply again with the full file between the markers.` + } + throw new Error(`proposer surface failed the skills gate after retry: ${lastProblems.join('; ')}`) + }, + } +} + +// ── The protocol gate ────────────────────────────────────────────────────────── + +const trainMeanBySurfaceSha = new Map() +const trainCaseMeansBySurfaceSha = new Map>() + +function campaignPerRep(campaign: CampaignResult) { + const perCase = new Map>() + for (const cell of campaign.cells) { + const s = cell.judgeScores['deterministic-expect'] + if (!s || s.failed) continue + const rows = perCase.get(cell.scenarioId) ?? [] + rows.push({ rep: cell.rep, score: s.composite, decision: cell.artifact?.decision ?? 'unknown' }) + perCase.set(cell.scenarioId, rows) + } + for (const rows of perCase.values()) rows.sort((a, b) => a.rep - b.rep) + return perCase +} + +/** Split mean per protocol: mean over cases of the per-case rep means. */ +function splitMean(perCase: Map>, ids: readonly string[]): number { + const caseMeans = ids.map((id) => { + const rows = perCase.get(id) ?? [] + return rows.length === 0 ? 0 : rows.reduce((s, r) => s + r.score, 0) / rows.length + }) + return caseMeans.reduce((s, x) => s + x, 0) / Math.max(caseMeans.length, 1) +} + +/** Worst-case split mean: every missing rep of every case scored as 0 (denominator K). */ +function splitMeanImputedZero(perCase: Map>, ids: readonly string[]): number { + const caseMeans = ids.map((id) => { + const rows = perCase.get(id) ?? [] + return rows.reduce((s, r) => s + r.score, 0) / K + }) + return caseMeans.reduce((s, x) => s + x, 0) / Math.max(caseMeans.length, 1) +} + +function holdoutMeanFromScores( + scores: Map>, +): number { + const values: number[] = [] + for (const byJudge of scores.values()) { + const s = byJudge['deterministic-expect'] + if (s && !s.failed) values.push(s.composite) + } + return values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length +} + +function makeGate(v2Sha: string): Gate { + return { + name: 'gen3-protocol-gate', + async decide(ctx: GateContext) { + const winnerHoldout = holdoutMeanFromScores(ctx.judgeScores) + const baselineHoldout = ctx.baselineJudgeScores ? holdoutMeanFromScores(ctx.baselineJudgeScores) : 0 + const v2Train = trainMeanBySurfaceSha.get(v2Sha) + const candidateShas = [...trainMeanBySurfaceSha.keys()].filter((k) => k !== v2Sha) + const v3Train = candidateShas.length === 1 ? trainMeanBySurfaceSha.get(candidateShas[0] ?? '') : undefined + const v2Cases = trainCaseMeansBySurfaceSha.get(v2Sha) + const v3Cases = candidateShas.length === 1 ? trainCaseMeansBySurfaceSha.get(candidateShas[0] ?? '') : undefined + const holdoutOk = winnerHoldout > baselineHoldout + const trainOk = v2Train !== undefined && v3Train !== undefined && v3Train >= v2Train - 0.05 + const degenerateOk = + v2Cases !== undefined && + v3Cases !== undefined && + DEGENERATE_IDS.every((id) => (v3Cases.get(id) ?? 0) >= (v2Cases.get(id) ?? 0)) + const ship = holdoutOk && trainOk && degenerateOk + return { + decision: ship ? ('ship' as const) : ('hold' as const), + delta: winnerHoldout - baselineHoldout, + reasons: [ + `holdout: winner ${winnerHoldout.toFixed(3)} vs baseline ${baselineHoldout.toFixed(3)} → ${holdoutOk ? 'pass' : 'fail'}`, + `train: v3 ${v3Train?.toFixed(3) ?? 'unmeasured'} vs v2 ${v2Train?.toFixed(3) ?? 'unmeasured'} - 0.05 → ${trainOk ? 'pass' : 'fail'}`, + `degenerate cases non-regression → ${degenerateOk ? 'pass' : 'fail'}`, + ], + contributingGates: [ + { name: 'holdout-mean-strictly-better', status: holdoutOk ? 'pass' : 'fail', detail: { winnerHoldout, baselineHoldout } }, + { name: 'train-mean-within-0.05', status: trainOk ? 'pass' : 'fail', detail: { v3Train, v2Train } }, + { name: 'anti-over-graphing-non-regression', status: degenerateOk ? 'pass' : 'fail', detail: { degenerateIds: [...DEGENERATE_IDS] } }, + ], + } + }, + } +} + +// ── The run ──────────────────────────────────────────────────────────────────── + +interface RepRow { + rep: number + score: number + decision: string +} + +function tableFor(perCase: Map, ids: readonly string[]): Record { + return Object.fromEntries(ids.map((id) => [id, perCase.get(id) ?? []])) +} + +function printSplit(label: string, perCase: Map, ids: readonly string[]): void { + console.log(` ${label}:`) + for (const id of ids) { + const rows = perCase.get(id) ?? [] + const reps = rows.map((r) => r.score.toFixed(2)).join(' ') + const mean = rows.length === 0 ? 0 : rows.reduce((s, r) => s + r.score, 0) / rows.length + console.log(` ${id.padEnd(32)} reps=[${reps}] mean=${mean.toFixed(3)}`) + } + console.log(` split mean = ${splitMean(perCase, ids).toFixed(4)}`) +} + +function firstCellSurfaceSha(campaign: CampaignResult): string | undefined { + for (const cell of campaign.cells) { + const sha = cell.artifact?.surfaceSha + if (typeof sha === 'string') return sha + } + return undefined +} + +async function main(): Promise { + const inputs = loadInputs() + const v2Surface = inputs.surface + const v2Sha = sha256(v2Surface) + if (!SMOKE && v2Sha !== EXPECTED_V2_SHA) { + throw new Error( + `working-tree SKILL.md sha ${v2Sha.slice(0, 12)} != promoted v2 ${EXPECTED_V2_SHA.slice(0, 12)}; gen3 must start from the promoted v2 surface`, + ) + } + const byId = new Map(inputs.cases.map((c) => [c.id, c])) + const missing = [...TRAIN_IDS, ...HOLDOUT_IDS].filter((id) => !byId.has(id)) + if (missing.length > 0) throw new Error(`cases missing from skills/agent-graphs/cases: ${missing.join(', ')}`) + if (TRAIN_IDS.length + HOLDOUT_IDS.length !== inputs.cases.length) { + throw new Error(`split covers ${TRAIN_IDS.length + HOLDOUT_IDS.length} of ${inputs.cases.length} cases`) + } + const toScenario = (id: string): GraphScenario => ({ ...(byId.get(id) as CaseSpec), kind: 'agent-graph-case' }) + const trainScenarios = TRAIN_IDS.map(toScenario) + const holdoutScenarios = HOLDOUT_IDS.map(toScenario) + + console.log( + `gen3 ${SMOKE ? '(SMOKE) ' : ''}v2=${v2Sha.slice(0, 12)} (${v2Surface.length} chars, ${inputs.source}); train=${TRAIN_IDS.length} holdout=${HOLDOUT_IDS.length} k=${K}`, + ) + + const runDir = join(RUNS_ROOT, SMOKE ? 'smoke-loop' : 'loop') + mkdirSync(runDir, { recursive: true }) + + const result = await runImprovementLoop({ + scenarios: trainScenarios, + holdoutScenarios, + reps: K, + seed: SEED, + maxConcurrency: 1, + candidateConcurrency: 1, + populationSize: 1, + maxGenerations: 1, + baselineSurface: v2Surface, + dispatchRef: SMOKE ? 'gen3-smoke-stub' : 'agent-graphs-author/glm-5.2/temp-0.2', + dispatchWithSurface: dispatchCell, + // Room for the worst retry ladder: 4 author attempts x (2x240s) + backoffs. + dispatchTimeoutMs: 2_400_000, + expectUsage: 'off', + judges: [makeJudge()], + proposer: makeProposer(v2Surface, trainScenarios), + analyzeGeneration: async ({ candidates }) => { + for (const c of candidates) { + const perCase = campaignPerRep(c.campaign) + const sha = firstCellSurfaceSha(c.campaign) + if (sha !== undefined) { + trainMeanBySurfaceSha.set(sha, splitMean(perCase, TRAIN_IDS)) + trainCaseMeansBySurfaceSha.set( + sha, + new Map( + TRAIN_IDS.map((id) => { + const rows = perCase.get(id) ?? [] + return [id, rows.length === 0 ? 0 : rows.reduce((s, r) => s + r.score, 0) / rows.length] + }), + ), + ) + } + } + return [] + }, + gate: makeGate(v2Sha), + autoOnPromote: 'none', + runDir, + }) + + // ── Assemble the four arms ── + const v2Train = campaignPerRep(result.baselineCampaign) + const candidateGen = result.generations[0]?.surfaces[0] + if (!candidateGen) throw new Error('loop produced no generation-0 candidate campaign') + const v3Surface = candidateGen.surface + if (typeof v3Surface !== 'string') throw new Error('candidate surface is not a string') + const v3Sha = sha256(v3Surface) + const v3Train = campaignPerRep(candidateGen.campaign) + const v2Holdout = campaignPerRep(result.baselineOnHoldout) + + // When upstream winner-selection kept the baseline (e.g. the coverage/no-op guard), + // the protocol still requires v3 measured on holdout: same judge, reps, seed. + const winnerIsCandidate = result.winnerSurfaceHash !== undefined && result.winnerSurface === v3Surface + let v3HoldoutCampaign: CampaignResult + if (winnerIsCandidate) { + v3HoldoutCampaign = result.winnerOnHoldout + } else { + console.log('upstream winner = baseline; measuring v3 on holdout via runEval for the protocol gate') + v3HoldoutCampaign = await runEval({ + scenarios: holdoutScenarios, + dispatch: (scenario, ctx) => dispatchCell(v3Surface, scenario, ctx), + dispatchRef: SMOKE ? 'gen3-smoke-stub-v3' : 'agent-graphs-author/glm-5.2/temp-0.2/v3', + judges: [makeJudge()], + reps: K, + seed: SEED, + maxConcurrency: 1, + dispatchTimeoutMs: 2_400_000, + expectUsage: 'off', + runDir: join(RUNS_ROOT, SMOKE ? 'smoke-v3-holdout' : 'v3-holdout'), + }) + } + const v3Holdout = campaignPerRep(v3HoldoutCampaign) + + // ── Protocol gate, applied to the assembled arms ── + const v2TrainMean = splitMean(v2Train, TRAIN_IDS) + const v3TrainMean = splitMean(v3Train, TRAIN_IDS) + const v2HoldoutMean = splitMean(v2Holdout, HOLDOUT_IDS) + const v3HoldoutMean = splitMean(v3Holdout, HOLDOUT_IDS) + + const caseMean = (perCase: Map, id: string): number => { + const rows = perCase.get(id) ?? [] + return rows.length === 0 ? 0 : rows.reduce((s, r) => s + r.score, 0) / rows.length + } + const degenerate = Object.fromEntries( + DEGENERATE_IDS.map((id) => [id, { v2: caseMean(v2Train, id), v3: caseMean(v3Train, id) }]), + ) + const degenerateOk = DEGENERATE_IDS.every((id) => caseMean(v3Train, id) >= caseMean(v2Train, id)) + const promoted = v3HoldoutMean > v2HoldoutMean && v3TrainMean >= v2TrainMean - 0.05 && degenerateOk + const gateVerdict = promoted ? 'ship' : 'hold' + + // Robustness: the same gate with every failed/missing cell imputed as 0. + const imputed = { + v2TrainMean: splitMeanImputedZero(v2Train, TRAIN_IDS), + v3TrainMean: splitMeanImputedZero(v3Train, TRAIN_IDS), + v2HoldoutMean: splitMeanImputedZero(v2Holdout, HOLDOUT_IDS), + v3HoldoutMean: splitMeanImputedZero(v3Holdout, HOLDOUT_IDS), + } + const promotedUnderImputation = + imputed.v3HoldoutMean > imputed.v2HoldoutMean && + imputed.v3TrainMean >= imputed.v2TrainMean - 0.05 && + degenerateOk + + console.log('\nv2 (baseline surface):') + printSplit('train', v2Train, TRAIN_IDS) + printSplit('holdout', v2Holdout, HOLDOUT_IDS) + console.log('v3 (revised surface):') + printSplit('train', v3Train, TRAIN_IDS) + printSplit('holdout', v3Holdout, HOLDOUT_IDS) + console.log(`\ndegenerate check (anti-over-graphing cases, v2 → v3): ${JSON.stringify(degenerate)}`) + console.log(`retries: ${retryReceipts.length} receipt(s); unrecovered cell failures: ${cellFailures.length}`) + console.log(`upstream gate: ${result.gateResult.decision} [${result.gateResult.reasons.join(' | ')}]`) + console.log( + `protocol gate: ${gateVerdict} (holdout ${v2HoldoutMean.toFixed(3)} → ${v3HoldoutMean.toFixed(3)}, train ${v2TrainMean.toFixed(3)} → ${v3TrainMean.toFixed(3)}, degenerate ${degenerateOk ? 'ok' : 'REGRESSED'})`, + ) + + const out = { + generation: 3, + date: new Date().toISOString(), + smoke: SMOKE, + authorModel: 'glm-5.2', + authorTemperature: 0.2, + proposerModel: 'glm-5.2', + proposerTemperature: 0.7, + split: { train: TRAIN_IDS, holdout: HOLDOUT_IDS }, + k: K, + seed: SEED, + surfaces: { v2Sha256: v2Sha, v3Sha256: v3Sha, v3Label: result.generations[0]?.record.candidates[0]?.label }, + perCase: { + v2: { train: tableFor(v2Train, TRAIN_IDS), holdout: tableFor(v2Holdout, HOLDOUT_IDS) }, + v3: { train: tableFor(v3Train, TRAIN_IDS), holdout: tableFor(v3Holdout, HOLDOUT_IDS) }, + }, + aggregates: { + v2: { trainMean: v2TrainMean, holdoutMean: v2HoldoutMean }, + v3: { trainMean: v3TrainMean, holdoutMean: v3HoldoutMean }, + }, + trainFailureTallyV2: tallyTrainFailures(v2Sha), + degenerateCheck: degenerate, + retryReceipts, + cellFailures, + worstCaseImputation: { ...imputed, promotedUnderImputation, note: 'every failed or missing rep scored 0 with denominator k' }, + upstreamGate: result.gateResult, + upstreamWinnerWasCandidate: winnerIsCandidate, + gateVerdict, + promoted, + revisionPromptSha256, + v3Surface, + } + mkdirSync(dirname(EFFECTIVE_OUT), { recursive: true }) + writeFileSync(EFFECTIVE_OUT, `${JSON.stringify(out, null, 2)}\n`) + console.log(`written: ${EFFECTIVE_OUT}`) + + if (promoted && !SMOKE) { + writeFileSync(SKILL_PATH, v3Surface) + console.log(`promoted: ${SKILL_PATH} replaced with v3 (${v3Sha.slice(0, 12)})`) + } +} + +main().catch((err) => { + console.error(err instanceof Error ? (err.stack ?? err.message) : String(err)) + process.exit(1) +}) diff --git a/bench/src/agent-graphs-improve.mts b/bench/src/agent-graphs-improve.mts index 2205cbdb..3d42a560 100644 --- a/bench/src/agent-graphs-improve.mts +++ b/bench/src/agent-graphs-improve.mts @@ -13,7 +13,7 @@ * per satisfied expectation, equal weights. * * Baseline run: pnpm tsx src/agent-graphs-improve.mts (from bench/) - * Writes skills/agent-graphs/baseline-v1.json and prints the per-case table. + * Writes skills/agent-graphs/generations/gen1-baseline.json and prints the per-case table. * * Author model: tangle-router glm-5.2, temperature 0.2, one retry on unparseable JSON. */ @@ -42,7 +42,7 @@ const REPO = join(HERE, '..', '..') // The skill was re-homed skills/codemode → skills/agent-graphs (925460fe); these paths follow it. const SKILL_PATH = join(REPO, 'skills', 'agent-graphs', 'SKILL.md') const CASES_DIR = join(REPO, 'skills', 'agent-graphs', 'cases') -const OUT_PATH = join(REPO, 'skills', 'agent-graphs', 'baseline-v1.json') +const OUT_PATH = join(REPO, 'skills', 'agent-graphs', 'generations', 'gen1-baseline.json') // The v1 surface + cases were removed from the working tree by 5b8d4da5 ("replace codemode plan // with current graph guide"); the baseline still measures the v1 text, pinned in git history. // Override with SKILL_REF to measure another committed version. @@ -89,6 +89,7 @@ const GENEROUS_PER_CHILD_TOKENS = 50_000 export interface CaseExpect { correctAnswerIsNoGraph?: boolean correctAnswerIsDynamicWorkflow?: boolean + correctAnswerIsGraph?: boolean nodes?: number analyzesWarranted?: boolean floorTrap?: boolean @@ -439,6 +440,15 @@ export function judgeArtifact(artifact: AuthoredArtifact, kase: CaseSpec): { sco note: `decision=${artifact.decision}`, }) } + if (e.correctAnswerIsGraph !== undefined) { + // Explicit dialect check for cases whose whole point is that a cheap-sounding brief + // still warrants a graph; requires an authored graph, not just the word "graph". + checks.push({ + key: 'correctAnswerIsGraph', + pass: graphOk, + note: `decision=${artifact.decision}${artifact.decision === 'graph' && g === undefined ? ' (no graph payload)' : ''}`, + }) + } if (e.mustBudgetAtLeast !== undefined) { const perChild = graphOk ? perChildTokens(g) : 0 checks.push({ diff --git a/skills/agent-graphs/IMPROVE.md b/skills/agent-graphs/IMPROVE.md index 5d345613..92269073 100644 --- a/skills/agent-graphs/IMPROVE.md +++ b/skills/agent-graphs/IMPROVE.md @@ -35,6 +35,16 @@ Holdout discipline: at least 3 of the 8 held out, never trained on; `runImprovem - No new optimizer, campaign runner, judge plumbing, or ledger — all named above. - No live-backend scoring in the loop. Live runs are pursuit work, not skill-improvement work; the loop stays offline and free. +## Version history + +The live tree carries only the current `SKILL.md`; every prior surface text is recoverable from git history via the pinned sha256s below, and each generation's full measurement record lives in `generations/`. + +| gen | date | surface sha256 (short) | holdout mean | verdict | +| --- | --- | --- | --- | --- | +| 1 | 2026-08-03 | `582429a1` | 0.444 (k=3 re-measure in `generations/gen2.json`, n=9 holdout cells) | baseline | +| 2 | 2026-08-03 | `4c6615b6` | 0.611 (k=3, n=9 holdout cells); 0.600 at the gen3 k=5 re-measure | SHIP (#722) | +| 3 | 2026-08-03 | `54e7b38b` (not promoted; v2 stays live) | 0.900 — invalidated | **HOLD**: verifier found case-design contamination and scorer leniency; reasons + gen4 requirements in `generations/gen3.json` `verifierHold` | + ## Known upstream gap this loop will hit `OptimizationMethodResult` returns `winnerSurface` only — full candidate history is an owed upstream extension (recorded in discovery docs 22/25). Workaround needing no code: `runOptimization` already retains every generation's surfaces. diff --git a/skills/agent-graphs/cases/artifact-mission-release-notes.json b/skills/agent-graphs/cases/artifact-mission-release-notes.json new file mode 100644 index 00000000..6de72737 --- /dev/null +++ b/skills/agent-graphs/cases/artifact-mission-release-notes.json @@ -0,0 +1,10 @@ +{ + "id": "artifact-mission-release-notes", + "brief": "Produce a release-notes file for version 2.0 that passes our repo format checker.", + "expect": { + "correctAnswerIsGraph": true, + "nodes": 1, + "deliverableDescribeCarriesMission": true, + "checkIsMechanical": true + } +} diff --git a/skills/agent-graphs/cases/audited-single-writer.json b/skills/agent-graphs/cases/audited-single-writer.json new file mode 100644 index 00000000..cc4c466a --- /dev/null +++ b/skills/agent-graphs/cases/audited-single-writer.json @@ -0,0 +1,9 @@ +{ + "id": "audited-single-writer", + "brief": "Have a separate worker write the summary document so I can audit exactly what it did afterwards.", + "expect": { + "correctAnswerIsGraph": true, + "nodes": 1, + "reason": "auditability warrants delegation despite a cheap-sounding task" + } +} diff --git a/skills/agent-graphs/baseline-v1.json b/skills/agent-graphs/generations/gen1-baseline.json similarity index 100% rename from skills/agent-graphs/baseline-v1.json rename to skills/agent-graphs/generations/gen1-baseline.json diff --git a/skills/agent-graphs/gen2.json b/skills/agent-graphs/generations/gen2.json similarity index 100% rename from skills/agent-graphs/gen2.json rename to skills/agent-graphs/generations/gen2.json diff --git a/skills/agent-graphs/generations/gen3.json b/skills/agent-graphs/generations/gen3.json new file mode 100644 index 00000000..47329ddb --- /dev/null +++ b/skills/agent-graphs/generations/gen3.json @@ -0,0 +1,701 @@ +{ + "generation": 3, + "date": "2026-08-03T19:49:18.796Z", + "smoke": false, + "authorModel": "glm-5.2", + "authorTemperature": 0.2, + "proposerModel": "glm-5.2", + "proposerTemperature": 0.7, + "split": { + "train": [ + "floor-trap-pi", + "review-pipeline", + "single-agent-suffices", + "cap-as-stop-mistake", + "runtime-discovered-fanout", + "artifact-mission-release-notes", + "audited-single-writer" + ], + "holdout": [ + "mission-in-deliverable", + "steer-heavy-drafting", + "unmeasured-harness" + ] + }, + "k": 5, + "seed": 42, + "surfaces": { + "v2Sha256": "4c6615b6164f6c5a86efb2596556bdf325d33f08a4e1715cae9d71cb28b6255e", + "v3Sha256": "54e7b38bc4b4ad22890d8330ac4dc3d39e4e759a384c0afbfe8cc8ea67f0f560", + "v3Label": "gen3-revision" + }, + "perCase": { + "v2": { + "train": { + "floor-trap-pi": [ + { + "rep": 0, + "score": 1, + "decision": "graph" + }, + { + "rep": 1, + "score": 1, + "decision": "graph" + }, + { + "rep": 2, + "score": 1, + "decision": "graph" + }, + { + "rep": 3, + "score": 1, + "decision": "graph" + }, + { + "rep": 4, + "score": 1, + "decision": "graph" + } + ], + "review-pipeline": [ + { + "rep": 0, + "score": 0.8, + "decision": "graph" + }, + { + "rep": 1, + "score": 0.8, + "decision": "graph" + }, + { + "rep": 2, + "score": 0.4, + "decision": "graph" + }, + { + "rep": 3, + "score": 0.8, + "decision": "graph" + }, + { + "rep": 4, + "score": 0.8, + "decision": "graph" + } + ], + "single-agent-suffices": [ + { + "rep": 0, + "score": 1, + "decision": "single-agent" + }, + { + "rep": 1, + "score": 1, + "decision": "single-agent" + }, + { + "rep": 2, + "score": 1, + "decision": "single-agent" + }, + { + "rep": 3, + "score": 1, + "decision": "single-agent" + }, + { + "rep": 4, + "score": 1, + "decision": "single-agent" + } + ], + "cap-as-stop-mistake": [ + { + "rep": 0, + "score": 1, + "decision": "graph" + }, + { + "rep": 1, + "score": 1, + "decision": "graph" + }, + { + "rep": 2, + "score": 1, + "decision": "graph" + }, + { + "rep": 3, + "score": 1, + "decision": "graph" + }, + { + "rep": 4, + "score": 1, + "decision": "graph" + } + ], + "runtime-discovered-fanout": [ + { + "rep": 0, + "score": 1, + "decision": "dynamic-workflow" + }, + { + "rep": 1, + "score": 1, + "decision": "dynamic-workflow" + }, + { + "rep": 2, + "score": 1, + "decision": "dynamic-workflow" + }, + { + "rep": 3, + "score": 1, + "decision": "dynamic-workflow" + }, + { + "rep": 4, + "score": 1, + "decision": "dynamic-workflow" + } + ], + "artifact-mission-release-notes": [ + { + "rep": 0, + "score": 0, + "decision": "single-agent" + }, + { + "rep": 1, + "score": 0, + "decision": "single-agent" + }, + { + "rep": 2, + "score": 0, + "decision": "single-agent" + }, + { + "rep": 3, + "score": 0, + "decision": "single-agent" + }, + { + "rep": 4, + "score": 0, + "decision": "single-agent" + } + ], + "audited-single-writer": [ + { + "rep": 0, + "score": 1, + "decision": "graph" + }, + { + "rep": 1, + "score": 1, + "decision": "graph" + }, + { + "rep": 2, + "score": 1, + "decision": "graph" + }, + { + "rep": 3, + "score": 1, + "decision": "graph" + }, + { + "rep": 4, + "score": 1, + "decision": "graph" + } + ] + }, + "holdout": { + "mission-in-deliverable": [ + { + "rep": 0, + "score": 0, + "decision": "single-agent" + }, + { + "rep": 1, + "score": 0, + "decision": "single-agent" + }, + { + "rep": 2, + "score": 0, + "decision": "single-agent" + }, + { + "rep": 3, + "score": 0, + "decision": "single-agent" + }, + { + "rep": 4, + "score": 0, + "decision": "single-agent" + } + ], + "steer-heavy-drafting": [ + { + "rep": 0, + "score": 1, + "decision": "graph" + }, + { + "rep": 1, + "score": 1, + "decision": "graph" + }, + { + "rep": 2, + "score": 1, + "decision": "graph" + }, + { + "rep": 3, + "score": 1, + "decision": "graph" + }, + { + "rep": 4, + "score": 1, + "decision": "graph" + } + ], + "unmeasured-harness": [ + { + "rep": 0, + "score": 1, + "decision": "graph" + }, + { + "rep": 1, + "score": 1, + "decision": "graph" + }, + { + "rep": 2, + "score": 1, + "decision": "graph" + }, + { + "rep": 3, + "score": 0.5, + "decision": "graph" + }, + { + "rep": 4, + "score": 0.5, + "decision": "graph" + } + ] + } + }, + "v3": { + "train": { + "floor-trap-pi": [ + { + "rep": 0, + "score": 1, + "decision": "graph" + }, + { + "rep": 1, + "score": 1, + "decision": "graph" + }, + { + "rep": 2, + "score": 1, + "decision": "graph" + }, + { + "rep": 3, + "score": 1, + "decision": "graph" + }, + { + "rep": 4, + "score": 1, + "decision": "graph" + } + ], + "review-pipeline": [ + { + "rep": 0, + "score": 0.6, + "decision": "graph" + }, + { + "rep": 1, + "score": 0.4, + "decision": "graph" + }, + { + "rep": 2, + "score": 0.6, + "decision": "graph" + }, + { + "rep": 3, + "score": 0.6, + "decision": "graph" + }, + { + "rep": 4, + "score": 0.6, + "decision": "graph" + } + ], + "single-agent-suffices": [ + { + "rep": 0, + "score": 1, + "decision": "single-agent" + }, + { + "rep": 1, + "score": 1, + "decision": "single-agent" + }, + { + "rep": 2, + "score": 1, + "decision": "single-agent" + }, + { + "rep": 3, + "score": 1, + "decision": "single-agent" + }, + { + "rep": 4, + "score": 1, + "decision": "single-agent" + } + ], + "cap-as-stop-mistake": [ + { + "rep": 0, + "score": 1, + "decision": "graph" + }, + { + "rep": 1, + "score": 1, + "decision": "graph" + }, + { + "rep": 2, + "score": 1, + "decision": "graph" + }, + { + "rep": 3, + "score": 1, + "decision": "graph" + }, + { + "rep": 4, + "score": 1, + "decision": "graph" + } + ], + "runtime-discovered-fanout": [ + { + "rep": 0, + "score": 1, + "decision": "dynamic-workflow" + }, + { + "rep": 1, + "score": 1, + "decision": "dynamic-workflow" + }, + { + "rep": 2, + "score": 1, + "decision": "dynamic-workflow" + }, + { + "rep": 3, + "score": 1, + "decision": "dynamic-workflow" + }, + { + "rep": 4, + "score": 1, + "decision": "dynamic-workflow" + } + ], + "artifact-mission-release-notes": [ + { + "rep": 0, + "score": 1, + "decision": "graph" + }, + { + "rep": 1, + "score": 1, + "decision": "graph" + }, + { + "rep": 2, + "score": 1, + "decision": "graph" + }, + { + "rep": 3, + "score": 1, + "decision": "graph" + }, + { + "rep": 4, + "score": 1, + "decision": "graph" + } + ], + "audited-single-writer": [ + { + "rep": 0, + "score": 1, + "decision": "graph" + }, + { + "rep": 1, + "score": 1, + "decision": "graph" + }, + { + "rep": 2, + "score": 1, + "decision": "graph" + }, + { + "rep": 3, + "score": 1, + "decision": "graph" + }, + { + "rep": 4, + "score": 1, + "decision": "graph" + } + ] + }, + "holdout": { + "mission-in-deliverable": [ + { + "rep": 0, + "score": 1, + "decision": "graph" + }, + { + "rep": 1, + "score": 1, + "decision": "graph" + }, + { + "rep": 2, + "score": 1, + "decision": "graph" + }, + { + "rep": 3, + "score": 1, + "decision": "graph" + }, + { + "rep": 4, + "score": 1, + "decision": "graph" + } + ], + "steer-heavy-drafting": [ + { + "rep": 0, + "score": 1, + "decision": "graph" + }, + { + "rep": 1, + "score": 1, + "decision": "graph" + }, + { + "rep": 2, + "score": 1, + "decision": "graph" + }, + { + "rep": 3, + "score": 1, + "decision": "graph" + }, + { + "rep": 4, + "score": 1, + "decision": "graph" + } + ], + "unmeasured-harness": [ + { + "rep": 0, + "score": 1, + "decision": "graph" + }, + { + "rep": 1, + "score": 0.5, + "decision": "graph" + }, + { + "rep": 2, + "score": 0.5, + "decision": "graph" + }, + { + "rep": 3, + "score": 1, + "decision": "graph" + }, + { + "rep": 4, + "score": 0.5, + "decision": "graph" + } + ] + } + } + }, + "aggregates": { + "v2": { + "trainMean": 0.8171428571428571, + "holdoutMean": 0.6 + }, + "v3": { + "trainMean": 0.9371428571428572, + "holdoutMean": 0.9 + } + }, + "trainFailureTallyV2": { + "floor-trap-pi": { + "validationError": 4 + }, + "review-pipeline": { + "nodes": 4, + "edge": 3, + "validationError": 1 + }, + "cap-as-stop-mistake": { + "validationError": 2 + }, + "artifact-mission-release-notes": { + "correctAnswerIsGraph": 5, + "nodes": 5, + "deliverableDescribeCarriesMission": 5 + } + }, + "degenerateCheck": { + "single-agent-suffices": { + "v2": 1, + "v3": 1 + }, + "runtime-discovered-fanout": { + "v2": 1, + "v3": 1 + } + }, + "retryReceipts": [ + { + "surfaceSha12": "4c6615b6164f", + "scenarioId": "runtime-discovered-fanout", + "rep": 0, + "attempt": 1, + "error": "author failed after retry: router HTTP 503: {\"error\":{\"message\":\"Inference temporarily unavailable due to upstream capacity. Please retry shortly.\",\"type\":\"server_error\",\"code\":\"upstream_unavailable\",\"generationId\":\"gen_01KZ4H32JMWZ3ZY2WSQNQ79S", + "at": "2026-08-03T19:19:59.377Z" + } + ], + "cellFailures": [], + "worstCaseImputation": { + "v2TrainMean": 0.8171428571428571, + "v3TrainMean": 0.9371428571428572, + "v2HoldoutMean": 0.6, + "v3HoldoutMean": 0.9, + "promotedUnderImputation": true, + "note": "every failed or missing rep scored 0 with denominator k" + }, + "upstreamGate": { + "decision": "hold", + "delta": 0.30000000000000004, + "reasons": [ + "holdout: winner 0.900 vs baseline 0.600 \u2192 pass", + "train: v3 unmeasured vs v2 0.817 - 0.05 \u2192 fail", + "degenerate cases non-regression \u2192 fail" + ], + "contributingGates": [ + { + "name": "holdout-mean-strictly-better", + "status": "pass", + "detail": { + "winnerHoldout": 0.9, + "baselineHoldout": 0.6 + } + }, + { + "name": "train-mean-within-0.05", + "status": "fail", + "detail": { + "v2Train": 0.8171428571428571 + } + }, + { + "name": "anti-over-graphing-non-regression", + "status": "fail", + "detail": { + "degenerateIds": [ + "single-agent-suffices", + "runtime-discovered-fanout" + ] + } + } + ] + }, + "upstreamWinnerWasCandidate": true, + "gateVerdict": "HOLD \u2014 script gate said ship, adversarial verification invalidated it", + "promoted": false, + "revisionPromptSha256": "0f05d5c08831e3b5207464aeeaeb38ef9ce400f80898021a667eaf8f077bc608", + "v3Surface": "---\nname: agent-graphs\ndescription: Author runGraph programs from AgentProfiles and versioned prompt directives.\n---\n\n# Agent graphs\n\nUse this skill when every role is known before execution and the relationship between roles must be reviewable as data.\nThe output is an `AgentGraph` executed by `runGraph`, not a new coordinator or workflow framework.\n\n## Choose the existing entry point\n\n| Need | Use |\n| --- | --- |\n| Known roles with versioned work and analysis instructions | `runGraph` |\n| A standard fixed shape such as parallel attempts, a chain, or a review panel | `fanout`, `pipeline`, `verify`, or `panel` |\n| A model decides which workers to create while it works | `supervise` |\n| One profile can complete the task directly | Run that profile without composition |\n\nDo not force a dynamic task into a static graph.\nDo not use a graph when a smaller shipped primitive already expresses the work.\n\n### Strict authoring decisions (Do not under-graph)\n\n- **Cheapness is not the dialect test:** Do not bail to `single-agent` just because a brief sounds trivial (e.g., \"write a one-line file\"). If the brief implies roles, observers, or a specific tight budget, author the graph.\n- **Independent Checkers:** If a brief requires an artifact to pass an independent check, format checker, or validator (e.g., \"passes our repo format checker\"), you MUST author a graph with an analyst node equipped to perform the check. Do not bail to `single-agent` by assuming it is a trivial file write.\n- **Budget Floor Traps:** If a brief demands an impossibly \"tight\" budget (e.g., a few thousand tokens), do not dodge it by dropping to `single-agent`. Author the graph and explicitly set `budget` to the valid measured executor floor.\n- **Distinct Perspectives and Parallelism:** If a brief requests N parallel instances of the same role, you MUST create N distinct worker nodes and N `delegates` edges. If a brief requests different perspectives (e.g., \"reviewed by two different perspectives\") or a neutral decider, create a distinct node for EACH requested perspective. Do not collapse multiple requested reviewers or distinct roles into a single node.\n- **Mandatory Analysts:** If a brief requires independent observation, review, or post-settle findings (e.g., \"neutral decider\", \"watch the worker\", \"passes format checker\"), you MUST author `analyzes` edges. Do not omit analysts and attempt to merge their logic into the root's prompt.\n- **Caps are not stops:** Do not use an analysis edge `maxTraversals` cap as a global stop condition. To stop after N findings, use `deliverable.check` or `maxTraversals` on a `delegates` edge.\n\n## Author the complete contract\n\nAn `AgentGraph` has four required fields: `nodes`, `edges`, `deliverable`, and `budget`.\n`runGraph(graph, options)` validates graph structure and prompt references before it spends compute.\n\n### Nodes\n\nEach node is `{ id, profile }`, where `profile` is a complete canonical `AgentProfile`.\nSet `profile.name` equal to `id` because Runtime uses that value to select and route the node.\nPut the standing role in `profile.prompt.systemPrompt` and capabilities in the profile's tools, MCP, resources, hooks, and subagents.\nDo not rebuild profile materialization in graph code.\n\n### Delegation edges\n\nA delegation edge is `{ kind: 'delegates', from, to, directive, maxTraversals? }`.\nThe directive is a registered, versioned `PromptHandle`, such as `promptHandle('delegates/research-brief/v1')`.\nEach spawn and each later steer over the same edge consumes one traversal.\nThe default cap is `defaultEdgeTraversalCap`; exhaustion refuses further delegation.\n\nThe current graph form has one root and a static set of worker nodes.\nEvery delegation edge starts at the root, and each worker has exactly one incoming delegation edge.\nUse a new directive version to change a brief instead of adding a second edge to the same worker.\n\n### Analysis edges\n\nAn analysis edge is `{ kind: 'analyzes', analyst, over, to, directive, maxTraversals? }`.\nIt runs after a listed worker settles and routes findings to one node.\n\n`analyst` has two supported forms:\n\n- A lens id from `options.analysts` runs a caller-supplied analysis function.\n- A graph node id runs that node's pinned `AgentProfile` as a tool-equipped analyst.\n\nAn analyst node has no incoming delegation edge, so the root cannot hand it ordinary work.\nAn id cannot be both a registered lens and an analyst node.\n`over` lists delegated worker nodes only; Runtime refuses the root and analyst nodes because neither settles as an ordinary worker.\nAn analysis traversal cap records excess findings as `unpropagated`; it does not stop the run.\n\n### Completion and budget\n\n`deliverable.check(output)` is the independent completion test.\nIt must accept a genuinely complete result and reject junk.\nPut the concrete mission in `deliverable.describe`; Runtime uses that text as the root's task.\n\n`budget` is one conserved pool for the full graph.\nSet `options.perWorker` explicitly from the actual executor cost.\n**Do not mix `runGraph` options with other entry points.** Never emit `supervise`-specific fields like `perWorker.maxIterations` in a `runGraph` graph spec; `runGraph` strictly expects `budget`, `perWorker` token allocations, and traversal caps.\nFor Pi, `WORKER_TOKEN_FLOOR.pi` is 31,211 input tokens before useful work, so a worker allocation below that value is refused. If a brief asks for a budget lower than the floor, do not switch to `single-agent`; output the graph with the floor allocation.\nTreat an unmeasured executor floor as unknown rather than zero.\nAnalyst nodes spend from the same pool and need the same honest accounting as ordinary workers.\n\n## Authoring procedure\n\n1. **Classify correctly:** Verify if this needs `single-agent`, `dynamic-workflow`, or a static `runGraph`. If independent review, parallel workers, or independent validation (e.g. format checkers) are requested, use `runGraph`.\n2. **Define completion first:** Write the completion test and its description.\n3. **Select entry point:** Choose the smallest shipped entry point from the table above.\n4. **Define Roles:** Give every distinct role one complete `AgentProfile`. If N parallel instances or distinct perspectives (e.g., two different reviewers) are requested, create N nodes. Merge roles only if their standing prompts and capabilities are identical.\n5. **Register directives:** Register a versioned directive for every edge.\n6. **Delegate work:** Add one delegation edge per ordinary worker from the root.\n7. **Attach analysts:** Add `analyzes` edges only when findings must be produced independently after a worker settles. Do not skip this if the brief asked for a watcher/reviewer, a neutral decider, or an independent format checker.\n8. **Size the pool:** Set budget, per-worker allocation, traversal caps, time, and concurrency from measured executor behavior. Ensure budgets meet the executor floor. Do not mix `runGraph` options with `supervise` options.\n9. **Prove and inspect:** Run the structure offline, then run the real backend and inspect its result.\n\n## Prove the graph before spending\n\nUse an injected `brain` plus `makeWorkerAgent` to exercise graph structure without a network call.\nCover invalid profiles, unknown directives, impossible analysis routes, traversal exhaustion, successful completion, and rejected junk.\nStart from the runnable programs in `examples/graphs/` rather than creating a second graph runner.\n\nOffline execution proves control flow only.\nA real task must still use the intended backend, profiles, tools, completion test, and budget before claiming the graph solves that task.\n\n## Read the complete result\n\n| Field | Meaning |\n| --- | --- |\n| `result.result.kind` and `reason` | Whether a result won and why execution ended |\n| `result.result.spentTotal` | Tokens and money, including whether each total is known |\n| `result.ledger` | Every delivered, stripped, empty, or unpropagated edge traversal with byte counts |\n| `result.exhaustedEdges` | Every edge whose cap was reached, including normal lifecycle endings |\n| Journal `edge` events | Durable copies of traversal evidence |\n\nZero traversals on an expected edge means the graph did not exercise that relationship.\n`usdKnown: false` means cost is missing, not free.\nA passing completion test proves only what that test checks.\n\n## Common mistakes\n\n- Bailing to `single-agent` because a brief sounds trivial or assumes a file write is simple, instead of respecting requested independent checkers (e.g., \"passes format checker\"), roles, or budget floors.\n- Mixing `runGraph` options with `supervise` options (e.g., emitting `perWorker.maxIterations` in a `runGraph` spec).\n- Collapsing multiple requested reviewers or distinct perspectives into a single worker node.\n- Skipping `analyzes` edges when an observer, reviewer, independent check, or neutral decider is explicitly requested.\n- Putting the task only in a spawn prompt instead of `deliverable.describe`.\n- Giving a node a `profile.name` different from its id.\n- Delegating ordinary work to an analyst node.\n- Listing the root or an analyst node in `analyzes.over`.\n- Using an analysis cap as a stop condition.\n- Allowing a driver-authored spawn profile to add capabilities instead of defining them on the pinned node profile.\n- Reading only thrown cap errors and missing `result.exhaustedEdges` on budget or cancellation endings.\n- Treating unknown spend as zero.\n- Claiming recursive or runtime-discovered structure when the current graph is a static root with workers and analysts.\n\n## Improve only after measurement\n\nRuntime already optimizes one inline skill through `improve(profile, { surface: 'skills', skills: { resourceName }, ... })`.\nPut the exact skill bytes in `profile.resources.skills`, set `profile.resources.failOnError: true`, supply disjoint development and final-test tasks, and pass a complete Agent Eval optimization method.\nDo not create a graph-specific optimizer, campaign runner, candidate store, or promotion path.\n\n## Then consider\n\n- `loop-writer` when the required dynamic structure still cannot be expressed by `supervise` or another shipped primitive; pass the exact missing behavior and the completion test.\n- `verify` before publishing a graph consumer; pass the real backend command, expected result fields, and failure cases.\n", + "verifierHold": { + "verdict": "HOLD", + "reasons": [ + "case-design contamination: train case artifact-mission-release-notes is a template-level paraphrase of holdout mission-in-deliverable, designed from its measured failure; the +0.300 holdout gain rests entirely on that case, so it evidences targeting, not generalization", + "scorer leniency: a v3-authored graph that runGraph REFUSED offline (invalid perWorker.maxIterations) scored 1.00 because the case scores only deliverableDescribeCarriesMission; validationError must zero a graph decision", + "gate-record discrepancy: upstreamGate recorded hold (v3 train mean never reached it) while the report claimed ship without disclosing the divergence", + "revision prompt sha not independently reproducible: per-rep failure inputs not persisted" + ], + "standing": "the k=5 measurements remain valid evidence; the promotion claim does not. v2 stays live.", + "gen4Requirements": [ + "scorer: validationError => score 0 for any graph decision", + "author JSON contract aligned with the skill text (remove perWorker.maxIterations or lower it correctly)", + "holdout refresh: cases authored blind by an agent given only the case schema, never failure history; contaminated comparison retired", + "persist revision-prompt inputs alongside the sha", + "fix in-loop gate wiring so the upstream record is authoritative" + ] + } +}