diff --git a/.gitignore b/.gitignore index d9c889fe..e9e3436d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ bench/data/ bench/experiments/ bench/runs/ bench/ab-runs/ +.gen2-runs/ **/__pycache__/ .claude/ bench/scripts/__pycache__/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 797a76d5..96aa41b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,26 @@ ## Unreleased +## 0.127.0 + +### One canonical loop API + +BREAKING. The superseded `runLoop`, `RunLoopOptions`, and `StdioToolDescriptor` exports are removed. +Callers use `runAgentRounds`, `RunAgentRoundsOptions`, and `McpToolDescriptor`, which were already the canonical generalized names for the same behavior. +The benchmark arm formerly named `loop` is now named `multishot`, which describes the compared method without referring to the deleted alias. + +### Agent graphs fail before spend and reuse the existing Eval path + +- Graph validation now refuses an `analyzes` edge over the root because that edge can never fire. +- The `agent-graphs` skill turns a loose task into either a single-agent run, a fixed graph, or a dynamic `supervise()` workflow according to what the task actually requires. +- The accompanying benchmark executable supplies Agent Eval's caller-owned authoring and deterministic scoring functions and records a baseline without adding a second optimization system. +- The author is one canonical, overridable `AgentProfile`, executed through Runtime and Pi; it defaults to Tangle Router's DeepSeek V4 Flash and carries the skill as an inline profile resource. +- The first complete skill-improvement generation ran through Agent Eval's existing `runImprovementLoop`: 5 development cases and 3 final-test cases at 3 repetitions each. + The revised skill improved the development mean from 0.507 to 0.960 and the final-test mean from 0.444 to 0.611, so the declared rule promoted it. + One of 33 requested cells was lost to an HTTP 503; the checked-in record includes that asymmetry and the supplementary final-test measurement. +- Future baseline and improvement runs author through canonical, overridable `AgentProfile` values executed by Runtime and Pi. + The default is Tangle Router's DeepSeek V4 Flash; the historical promoted generation used GLM-5.2. + ### Python bridge install hints match the required Eval substrate The documented `agent-eval-rpc` install commands — the `OfficialOptimizerUnavailableError` hint, the README's `officialGepa`/`officialSkillOpt` sections, and the bench GEPA seat hint — now pin `0.143.0`, the Python client published in lockstep with the `@tangle-network/agent-eval` range this package requires. diff --git a/bench/CHANGELOG.md b/bench/CHANGELOG.md index 8595750d..184ad522 100644 --- a/bench/CHANGELOG.md +++ b/bench/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## 0.8.0 + +- Add the executable `agent-graphs` baseline run with eight authoring cases and deterministic scoring over the graph edge record. +- Add offline execution inputs so the authoring and scoring path can be tested without model spend. +- Define the author as an overridable `AgentProfile` and run it through Runtime and Pi, defaulting to Tangle Router's DeepSeek V4 Flash; remove the old direct HTTP call, fixed token cap, fixed timeout, and deleted `codemode` file fallback. +- Record one result per case and keep that small-sample limit explicit rather than treating it as a stable comparison. +- Consume Runtime 0.127.0 and its canonical `runAgentRounds` and `McpToolDescriptor` names. + ## 0.7.1 - Consume Runtime 0.126.0 with Eval 0.143.0 and Knowledge 7.0.8, so campaign cost remains observed, estimated, or explicitly uncaptured across the complete benchmark dependency tree. diff --git a/bench/package.json b/bench/package.json index 167a1b7f..7a74ed32 100644 --- a/bench/package.json +++ b/bench/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-bench", - "version": "0.7.1", + "version": "0.8.0", "type": "module", "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.", "repository": { diff --git a/bench/scripts/run-package-tests.mjs b/bench/scripts/run-package-tests.mjs index f215d241..7aebc022 100644 --- a/bench/scripts/run-package-tests.mjs +++ b/bench/scripts/run-package-tests.mjs @@ -7,6 +7,7 @@ import { promisify } from 'node:util' const execFileAsync = promisify(execFile) const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') const sourceDir = path.join(benchDir, 'src') +const fixtureEnv = { ...process.env, GIT_ALLOW_TEST_IDENTITY: '1' } async function collectTests(dir) { const files = [] @@ -61,17 +62,25 @@ if (nodeTests.length > 0) { process.execPath, ['--test', '--import', 'tsx', ...nodeTests.map((file) => path.relative(benchDir, file))], { - ...process.env, + ...fixtureEnv, TSX_TSCONFIG_PATH: 'tsconfig.public.json', }, ) } if (vitestTests.length > 0) { - await run('npx', ['vitest', 'run', ...vitestTests.map((file) => path.relative(benchDir, file))]) + await run( + 'npx', + ['vitest', 'run', ...vitestTests.map((file) => path.relative(benchDir, file))], + fixtureEnv, + ) } -await run(python, ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py']) +await run( + python, + ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py'], + fixtureEnv, +) console.log( `package tests passed: ${tests.length}/${tests.length} TypeScript files (${nodeTests.length} node:test + ${vitestTests.length} vitest) + Pier bridge`, diff --git a/bench/src/agent-graphs-gen2.mts b/bench/src/agent-graphs-gen2.mts index 81b8466c..33c1e017 100644 --- a/bench/src/agent-graphs-gen2.mts +++ b/bench/src/agent-graphs-gen2.mts @@ -6,8 +6,8 @@ * measurement (v2 on TRAIN, reps=3), the enforced-disjoint holdout scoring of both * arms, winner selection, and the gate invocation. What this file owns: the same two * closures the baseline run owned (author dispatch + deterministic scorer, imported - * from agent-graphs-improve.mts), the reviser proposer (glm-5.2, temp 0.7, TRAIN - * failures only), and the protocol gate: + * from agent-graphs-improve.mts), the AgentProfile-driven reviser (TRAIN failures + * only), and the protocol gate: * * ship iff v2 holdout mean > v1 holdout mean * and v2 train mean >= v1 train mean - 0.05 @@ -21,8 +21,9 @@ * run asserts the holdout ids and briefs are absent from the final prompt string. * * Run: pnpm tsx src/agent-graphs-gen2.mts (from bench/) - * Smoke: GEN2_SMOKE=1 pnpm tsx src/agent-graphs-gen2.mts — stubs both LLM calls, - * exercises the full loop wiring + gate + report at zero cost. + * Smoke: GEN2_SMOKE=1 pnpm tsx src/agent-graphs-gen2.mts — stubs both model calls, + * exercises the full loop wiring + gate + report at zero cost and writes only + * under the ignored .gen2-runs directory. * * Writes skills/agent-graphs/gen2.json; on ship, replaces SKILL.md with v2. */ @@ -47,6 +48,8 @@ import { import { type AuthoredArtifact, type CaseSpec, + authorProfileLabel, + buildAgentGraphsAuthorProfile, callAuthor, dispatchWithSurface, judgeArtifact, @@ -56,9 +59,63 @@ import { const HERE = dirname(fileURLToPath(import.meta.url)) const REPO = join(HERE, '..', '..') const SKILL_PATH = join(REPO, 'skills', 'agent-graphs', 'SKILL.md') -const OUT_PATH = join(REPO, 'skills', 'agent-graphs', 'gen2.json') const RUNS_ROOT = join(REPO, '.gen2-runs') const SMOKE = process.env.GEN2_SMOKE === '1' +const OUT_PATH = SMOKE + ? join(RUNS_ROOT, 'smoke-result.json') + : join(REPO, 'skills', 'agent-graphs', 'gen2.json') + +function positiveInteger(name: string, raw: string | undefined, fallback: number): number { + const value = raw === undefined ? fallback : Number(raw) + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`) + } + return value +} + +function optionalPositiveInteger(name: string, raw: string | undefined): number | undefined { + return raw === undefined ? undefined : positiveInteger(name, raw, 1) +} + +function executionEnv(role: 'AUTHOR' | 'PROPOSER'): NodeJS.ProcessEnv { + const specific = (name: string): string | undefined => + process.env[`AGENT_GRAPHS_GEN2_${role}_${name}`] + return { + ...process.env, + AGENT_GRAPHS_AUTHOR_PROFILE_NAME: + specific('PROFILE_NAME') ?? + process.env.AGENT_GRAPHS_AUTHOR_PROFILE_NAME ?? + `agent-graphs-gen2-${role.toLowerCase()}`, + AGENT_GRAPHS_AUTHOR_HARNESS: + specific('HARNESS') ?? process.env.AGENT_GRAPHS_AUTHOR_HARNESS ?? 'pi', + AGENT_GRAPHS_AUTHOR_PROVIDER: + specific('PROVIDER') ?? process.env.AGENT_GRAPHS_AUTHOR_PROVIDER ?? 'tangle-router', + AGENT_GRAPHS_AUTHOR_MODEL: + specific('MODEL') ?? process.env.AGENT_GRAPHS_AUTHOR_MODEL ?? 'glm-5.2', + AGENT_GRAPHS_AUTHOR_REASONING_EFFORT: + specific('REASONING_EFFORT') ?? + process.env.AGENT_GRAPHS_AUTHOR_REASONING_EFFORT ?? + 'ultracode', + AGENT_GRAPHS_AUTHOR_ATTEMPTS: + specific('ATTEMPTS') ?? process.env.AGENT_GRAPHS_AUTHOR_ATTEMPTS, + AGENT_GRAPHS_AUTHOR_TIMEOUT_MS: + specific('TIMEOUT_MS') ?? process.env.AGENT_GRAPHS_AUTHOR_TIMEOUT_MS, + ...(role === 'PROPOSER' + ? { + AGENT_GRAPHS_AUTHOR_SYSTEM_PROMPT: + specific('SYSTEM_PROMPT') ?? + 'You revise an agent skill from measured development-case failures. Follow the requested output format exactly and do not use final-test cases.', + } + : {}), + } +} + +const AUTHOR_ENV = executionEnv('AUTHOR') +const PROPOSER_ENV = executionEnv('PROPOSER') +const DISPATCH_TIMEOUT_MS = optionalPositiveInteger( + 'AGENT_GRAPHS_GEN2_DISPATCH_TIMEOUT_MS', + process.env.AGENT_GRAPHS_GEN2_DISPATCH_TIMEOUT_MS, +) const K = 3 const SEED = 42 @@ -96,6 +153,15 @@ interface JudgedRecord { const judged: JudgedRecord[] = [] +interface RevisionAttemptEvidence { + status: string + usage?: { input: number; output: number; costUsd?: number; model?: string } + validationProblems?: string[] + error?: string +} + +const revisionAttempts: RevisionAttemptEvidence[] = [] + function makeJudge(): JudgeConfig { return { name: 'deterministic-expect', @@ -122,7 +188,12 @@ function makeJudge(): JudgeConfig { function smokeArtifact(scenario: GraphScenario): AuthoredArtifact { // Deterministic offline stand-in: always "single-agent" — wrong on graph cases, // right on the no-graph case; enough to exercise scoring + gate arithmetic. - return { decision: 'single-agent', reason: `smoke stub for ${scenario.id}`, raw: '{}' } + return { + decision: 'single-agent', + reason: `smoke stub for ${scenario.id}`, + raw: '{}', + authorAttempts: [], + } } async function dispatchCell( @@ -131,7 +202,9 @@ async function dispatchCell( ctx: DispatchContext, ): Promise { if (typeof surface !== 'string') throw new Error('gen2 surfaces are strings') - const artifact = SMOKE ? smokeArtifact(scenario) : await dispatchWithSurface(surface, scenario) + const artifact = SMOKE + ? smokeArtifact(scenario) + : await dispatchWithSurface(surface, scenario, AUTHOR_ENV) return { ...artifact, repIndex: ctx.rep, surfaceSha: sha256(surface) } } @@ -215,6 +288,12 @@ function validateSkillGate(text: string): string[] { } function makeProposer(v1Surface: string, trainCases: GraphScenario[]): SurfaceProposer { + const proposerProfile = buildAgentGraphsAuthorProfile(v1Surface, PROPOSER_ENV) + const attemptLimit = positiveInteger( + 'AGENT_GRAPHS_GEN2_PROPOSER_ATTEMPTS', + process.env.AGENT_GRAPHS_GEN2_PROPOSER_ATTEMPTS, + 2, + ) return { kind: 'agent-graphs-skill-reviser', async propose(_ctx: ProposeContext): Promise { @@ -245,23 +324,43 @@ function makeProposer(v1Surface: string, trainCases: GraphScenario[]): SurfacePr } let prompt = revisionPrompt let lastProblems: string[] = [] - for (let attempt = 0; attempt < 2; attempt += 1) { - const reply = await callAuthor(prompt, 0.7, 12_000) - const skill = extractSkill(reply) - lastProblems = validateSkillGate(skill) - if (lastProblems.length === 0) { - return [ - { - surface: skill, - label: 'gen2-revision', - rationale: - 'glm-5.2 rewrite targeting under-graphing on cheap briefs, missing analyzes edges, and collapsed identical-role parallelism', - }, - ] + for (let attempt = 0; attempt < attemptLimit; attempt += 1) { + let evidence: RevisionAttemptEvidence | undefined + try { + const turn = await callAuthor(proposerProfile, prompt, PROPOSER_ENV) + evidence = { status: turn.status, usage: turn.usage } + revisionAttempts.push(evidence) + if (turn.status !== 'completed') { + throw new Error(turn.error?.message ?? `proposer turn ended with status ${turn.status}`) + } + const skill = extractSkill(turn.finalText) + lastProblems = validateSkillGate(skill) + evidence.validationProblems = lastProblems + if (lastProblems.length === 0) { + return [ + { + surface: skill, + label: 'gen2-revision', + rationale: + 'profile-authored rewrite targeting measured under-graphing, missing analysis, and collapsed parallel roles', + }, + ] + } + prompt = `${revisionPrompt}\n\nYour previous attempt violated: ${lastProblems.join('; ')}. Fix these and reply again with the full file between the markers.` + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + if (evidence === undefined) revisionAttempts.push({ status: 'failed', error: message }) + else evidence.error = message + if (attempt + 1 < attemptLimit) { + prompt = `${revisionPrompt}\n\nYour previous attempt failed: ${message}. Reply again with the full file between the markers.` + } } - prompt = `${revisionPrompt}\n\nYour previous attempt violated: ${lastProblems.join('; ')}. Fix these and reply again with the full file between the markers.` } - throw new Error(`proposer surface failed the skills gate after retry: ${lastProblems.join('; ')}`) + const detail = + lastProblems.length > 0 + ? lastProblems.join('; ') + : revisionAttempts.at(-1)?.error ?? 'unknown proposer failure' + throw new Error(`proposer surface failed after ${attemptLimit} attempts: ${detail}`) }, } } @@ -358,6 +457,10 @@ async function main(): Promise { const inputs = loadInputs() const v1Surface = inputs.surface const v1Sha = sha256(v1Surface) + const authorProfile = buildAgentGraphsAuthorProfile(v1Surface, AUTHOR_ENV) + const proposerProfile = buildAgentGraphsAuthorProfile(v1Surface, PROPOSER_ENV) + const authorRef = authorProfileLabel(authorProfile) + const proposerRef = authorProfileLabel(proposerProfile) const byId = new Map(inputs.cases.map((c) => [c.id, c])) const missing = [...TRAIN_IDS, ...HOLDOUT_IDS].filter((id) => !byId.has(id)) if (missing.length > 0) throw new Error(`cases missing from skills/agent-graphs/cases: ${missing.join(', ')}`) @@ -380,14 +483,22 @@ async function main(): Promise { holdoutScenarios, reps: K, seed: SEED, - maxConcurrency: 1, - candidateConcurrency: 1, + maxConcurrency: positiveInteger( + 'AGENT_GRAPHS_GEN2_MAX_CONCURRENCY', + process.env.AGENT_GRAPHS_GEN2_MAX_CONCURRENCY, + 1, + ), + candidateConcurrency: positiveInteger( + 'AGENT_GRAPHS_GEN2_CANDIDATE_CONCURRENCY', + process.env.AGENT_GRAPHS_GEN2_CANDIDATE_CONCURRENCY, + 1, + ), populationSize: 1, maxGenerations: 1, baselineSurface: v1Surface, - dispatchRef: SMOKE ? 'gen2-smoke-stub' : 'agent-graphs-author/glm-5.2/temp-0.2', + dispatchRef: SMOKE ? 'gen2-smoke-stub' : authorRef, dispatchWithSurface: dispatchCell, - dispatchTimeoutMs: 600_000, + ...(DISPATCH_TIMEOUT_MS === undefined ? {} : { dispatchTimeoutMs: DISPATCH_TIMEOUT_MS }), expectUsage: 'off', judges: [makeJudge()], proposer: makeProposer(v1Surface, trainScenarios), @@ -428,12 +539,12 @@ async function main(): Promise { v2HoldoutCampaign = await runEval({ scenarios: holdoutScenarios, dispatch: (scenario, ctx) => dispatchCell(v2Surface, scenario, ctx), - dispatchRef: SMOKE ? 'gen2-smoke-stub-v2' : 'agent-graphs-author/glm-5.2/temp-0.2/v2', + dispatchRef: SMOKE ? 'gen2-smoke-stub-v2' : `${authorRef}/candidate`, judges: [makeJudge()], reps: K, seed: SEED, maxConcurrency: 1, - dispatchTimeoutMs: 600_000, + ...(DISPATCH_TIMEOUT_MS === undefined ? {} : { dispatchTimeoutMs: DISPATCH_TIMEOUT_MS }), expectUsage: 'off', runDir: join(RUNS_ROOT, SMOKE ? 'smoke-v2-holdout' : 'v2-holdout'), }) @@ -470,10 +581,10 @@ async function main(): Promise { generation: 2, date: new Date().toISOString(), smoke: SMOKE, - authorModel: 'glm-5.2', - authorTemperature: 0.2, - proposerModel: 'glm-5.2', - proposerTemperature: 0.7, + authorProfile: authorRef, + proposerProfile: proposerRef, + authorTemperature: null, + proposerTemperature: null, split: { train: TRAIN_IDS, holdout: HOLDOUT_IDS }, k: K, seed: SEED, @@ -492,6 +603,7 @@ async function main(): Promise { gateVerdict, promoted, revisionPromptSha256, + revisionAttempts, v2Surface, } writeFileSync(OUT_PATH, `${JSON.stringify(out, null, 2)}\n`) diff --git a/bench/src/agent-graphs-improve.mts b/bench/src/agent-graphs-improve.mts index 2205cbdb..6ddd7b84 100644 --- a/bench/src/agent-graphs-improve.mts +++ b/bench/src/agent-graphs-improve.mts @@ -1,5 +1,5 @@ /** - * codemode-skill improvement harness — the BASELINE half of skills/agent-graphs/IMPROVE.md. + * Agent-graphs skill improvement runner — the baseline half of skills/agent-graphs/IMPROVE.md. * * The improving artifact is the skill TEXT (`skills/agent-graphs/SKILL.md`), a `MutableSurface` * string. This file owns exactly the two slots the agent-eval machinery leaves to the caller: @@ -15,18 +15,25 @@ * Baseline run: pnpm tsx src/agent-graphs-improve.mts (from bench/) * Writes skills/agent-graphs/baseline-v1.json and prints the per-case table. * - * Author model: tangle-router glm-5.2, temperature 0.2, one retry on unparseable JSON. + * Author execution: one canonical AgentProfile through the Runtime bridge executor. + * Defaults to pi + Tangle Router + DeepSeek V4 Flash; every choice and retry count is overridable. */ import { execFileSync } from 'node:child_process' -import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs' -import { homedir } from 'node:os' -import { join, dirname, resolve as resolvePath } from 'node:path' +import { mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs' +import { dirname, join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { + defineInlineResource, + harnessTypeSchema, + reasoningEffortSchema, + type AgentProfile, +} from '@tangle-network/agent-interface' import { type AgentGraph, type AnalystRegistry, + collectAgentTurn, + createExecutor, defaultEdgeTraversalCap, type EdgeTraversal, GraphEdgeCapError, @@ -34,19 +41,18 @@ import { promptHandle, type RunGraphOptions, runGraph, + streamAgentTurn, } from '../../src/runtime/index.ts' import { leafSeam, scriptedBrain, type ScriptedTurn } from './agent-graphs-improve/offline-seams.mts' const HERE = dirname(fileURLToPath(import.meta.url)) const REPO = join(HERE, '..', '..') -// The skill was re-homed skills/codemode → skills/agent-graphs (925460fe); these paths follow it. const SKILL_PATH = join(REPO, 'skills', 'agent-graphs', 'SKILL.md') const CASES_DIR = join(REPO, 'skills', 'agent-graphs', 'cases') const OUT_PATH = join(REPO, 'skills', 'agent-graphs', 'baseline-v1.json') -// The v1 surface + cases were removed from the working tree by 5b8d4da5 ("replace codemode plan -// with current graph guide"); the baseline still measures the v1 text, pinned in git history. -// Override with SKILL_REF to measure another committed version. -const SKILL_REF = process.env.SKILL_REF ?? 'afb40bc1' +// Set SKILL_REF to measure an exact committed agent-graphs surface and case set. +// With no ref, read the working tree and fail if its canonical files are absent. +const SKILL_REF = process.env.SKILL_REF function gitShow(ref: string, path: string): string { return execFileSync('git', ['show', `${ref}:${path}`], { cwd: REPO, encoding: 'utf8' }) @@ -61,48 +67,40 @@ function gitLs(ref: string, path: string): string[] { .filter((l) => l.endsWith('.json')) } -/** The surface + cases: from the working tree when present, else pinned from git history. */ +/** Load either one explicit commit or the canonical working-tree files, never a legacy alias. */ export function loadInputs(): { surface: string; cases: CaseSpec[]; source: string } { - if (existsSync(SKILL_PATH)) { - const surface = readFileSync(SKILL_PATH, 'utf8') - const cases = readdirSync(CASES_DIR) - .filter((f) => f.endsWith('.json')) + if (SKILL_REF !== undefined) { + const surface = gitShow(SKILL_REF, 'skills/agent-graphs/SKILL.md') + const cases = gitLs(SKILL_REF, 'skills/agent-graphs/cases') .sort() - .map((f) => JSON.parse(readFileSync(join(CASES_DIR, f), 'utf8')) as CaseSpec) - return { surface, cases, source: 'working-tree' } + .map((p) => parseCase(gitShow(SKILL_REF, p), `${SKILL_REF}:${p}`)) + return { surface, cases, source: `git:${SKILL_REF}` } } - const surface = gitShow(SKILL_REF, 'skills/agent-graphs/SKILL.md') - const cases = gitLs(SKILL_REF, 'skills/agent-graphs/cases') + + const surface = readFileSync(SKILL_PATH, 'utf8') + const cases = readdirSync(CASES_DIR) + .filter((f) => f.endsWith('.json')) .sort() - .map((p) => JSON.parse(gitShow(SKILL_REF, p)) as CaseSpec) - return { surface, cases, source: `git:${SKILL_REF}` } + .map((f) => parseCase(readFileSync(join(CASES_DIR, f), 'utf8'), join(CASES_DIR, f))) + return { surface, cases, source: 'working-tree' } } // The measured pi floor the floor-trap case scores against (src/runtime/supervise/budget-floor.ts). const PI_TOKEN_FLOOR = 31_211 -// "Budget generously" proxy for unmeasured harnesses: floor-unknown means per-child headroom -// well above the one measured floor; 50k is the scorer's line, documented not tuned. -const GENEROUS_PER_CHILD_TOKENS = 50_000 - // ── Case + artifact shapes ───────────────────────────────────────────────────── export interface CaseExpect { correctAnswerIsNoGraph?: boolean correctAnswerIsDynamicWorkflow?: boolean - nodes?: number + delegatedWorkers?: number analyzesWarranted?: boolean - floorTrap?: boolean mustBudgetAtLeast?: number - correctAuthorOverridesBrief?: boolean maxTraversalsAtLeast?: number deliverableDescribeCarriesMission?: boolean - checkIsMechanical?: boolean trapIsAnalyzesCapAsStop?: boolean correctStopIsDelegatesCapOrDeliverable?: boolean - wrongIfAnalystIsNode?: boolean - generousBudgetsBecauseFloorUnknown?: boolean + reasonMentionsUnknownBudget?: boolean edges?: string[] - reason?: string } export interface CaseSpec { @@ -111,6 +109,37 @@ export interface CaseSpec { expect: CaseExpect } +const SCORED_EXPECTATION_KEYS = new Set([ + 'correctAnswerIsNoGraph', + 'correctAnswerIsDynamicWorkflow', + 'delegatedWorkers', + 'analyzesWarranted', + 'mustBudgetAtLeast', + 'maxTraversalsAtLeast', + 'deliverableDescribeCarriesMission', + 'trapIsAnalyzesCapAsStop', + 'correctStopIsDelegatesCapOrDeliverable', + 'reasonMentionsUnknownBudget', + 'edges', +]) + +function parseCase(text: string, source: string): CaseSpec { + const value = JSON.parse(text) as Partial + if (!value.expect || typeof value.expect !== 'object' || Array.isArray(value.expect)) { + throw new Error(`${source}: expect must be an object`) + } + const unknown = Object.keys(value.expect).filter( + (key) => !SCORED_EXPECTATION_KEYS.has(key as keyof CaseExpect), + ) + if (unknown.length > 0) { + throw new Error(`${source}: unscored expectation keys: ${unknown.join(', ')}`) + } + if (typeof value.id !== 'string' || typeof value.brief !== 'string') { + throw new Error(`${source}: id and brief must be strings`) + } + return value as CaseSpec +} + export type AuthoredEdge = | { kind: 'delegates'; from: string; to: string; maxTraversals?: number } | { kind: 'analyzes'; analyst: string; over: string[]; to: string; maxTraversals?: number } @@ -141,43 +170,81 @@ export interface AuthoredArtifact { validationError?: string /** The author model's raw reply, retained for audit. */ raw: string + /** Every paid attempt, including parse failures and retries. */ + authorAttempts: AuthorAttemptEvidence[] } // ── The author model call ────────────────────────────────────────────────────── -const ROUTER_URL = 'https://router.tangle.tools/v1/chat/completions' -const AUTHOR_MODEL = 'glm-5.2' +const DEFAULT_AUTHOR_SYSTEM_PROMPT = [ + 'You are an agent-graph author.', + 'Follow the attached agent-graphs skill exactly; it is your only workflow doctrine.', + 'Decide whether the task needs one agent, a fixed AgentGraph, or a dynamic supervise workflow.', + 'Reply with JSON only: no markdown fences and no prose outside the JSON.', + '{"decision":"graph"|"single-agent"|"dynamic-workflow","reason":string,"graph"?:{...}}', + 'Include "graph" if and only if decision is "graph", with this exact shape:', + '{"nodes":[{"id":string,"systemPrompt":string}, ...],', + ' "edges":[{"kind":"delegates","from":string,"to":string,"maxTraversals"?:number}', + ' | {"kind":"analyzes","analyst":string,"over":[string,...],"to":string,"maxTraversals"?:number}, ...],', + ' "budget":{"maxIterations":number,"maxTokens":number},', + ' "perWorker"?:{"maxIterations"?:number,"maxTokens"?:number},', + ' "deliverableDescribe":string}', + 'The root node must be first; every delegates edge starts at the root.', + 'An analyst is either a registered lens id or a graph node with no incoming delegation edge.', + 'The deliverable description is the driver mission, not a generic completion phrase.', +].join('\n') + +function positiveInteger(name: string, raw: string | undefined, fallback: number): number { + const value = raw === undefined ? fallback : Number(raw) + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`) + } + return value +} -function routerToken(): string { - const raw = readFileSync(join(homedir(), '.config', 'tangle', 'router-token.json'), 'utf8') - return (JSON.parse(raw) as { token: string }).token +function optionalPositiveInteger(name: string, raw: string | undefined): number | undefined { + return raw === undefined ? undefined : positiveInteger(name, raw, 1) } -function authorPrompt(surface: string, kase: CaseSpec): string { +/** The exact portable author definition; the skill under test remains a first-class resource. */ +export function buildAgentGraphsAuthorProfile( + surface: string, + env: NodeJS.ProcessEnv = process.env, +): AgentProfile { + const harness = harnessTypeSchema.parse(env.AGENT_GRAPHS_AUTHOR_HARNESS ?? 'pi') + const reasoningEffort = reasoningEffortSchema.parse( + env.AGENT_GRAPHS_AUTHOR_REASONING_EFFORT ?? 'ultracode', + ) + return { + name: env.AGENT_GRAPHS_AUTHOR_PROFILE_NAME ?? 'agent-graphs-author', + harness, + model: { + provider: env.AGENT_GRAPHS_AUTHOR_PROVIDER ?? 'tangle-router', + default: env.AGENT_GRAPHS_AUTHOR_MODEL ?? 'deepseek-v4-flash', + reasoningEffort, + }, + prompt: { + systemPrompt: env.AGENT_GRAPHS_AUTHOR_SYSTEM_PROMPT ?? DEFAULT_AUTHOR_SYSTEM_PROMPT, + }, + resources: { + failOnError: true, + skills: [defineInlineResource('agent-graphs', surface)], + }, + } +} + +export function authorProfileLabel(profile: AgentProfile): string { + return [profile.harness, profile.model?.provider, profile.model?.default] + .filter((part): part is string => typeof part === 'string' && part.length > 0) + .join('/') +} + +function authorPrompt(kase: CaseSpec): string { return [ - 'You are an agent-graph author. Follow the skill below EXACTLY — it is your only doctrine.', - '', - '', - surface, - '', - '', ``, kase.brief, '', - '', - 'First decide the dialect per the skill. Then reply with JSON ONLY — no markdown fences, no prose outside the JSON:', - '{"decision":"graph"|"single-agent"|"dynamic-workflow","reason":string,"graph"?:{...}}', - '', - 'Include "graph" if and only if decision is "graph", with this exact shape:', - '{"nodes":[{"id":string,"systemPrompt":string}, ...],', - ' "edges":[{"kind":"delegates","from":string,"to":string,"maxTraversals"?:number}', - ' | {"kind":"analyzes","analyst":string,"over":[string,...],"to":string,"maxTraversals"?:number}, ...],', - ' "budget":{"maxIterations":number,"maxTokens":number},', - ' "perWorker"?:{"maxIterations"?:number,"maxTokens"?:number},', - ' "deliverableDescribe":string}', - '', - 'Rules: the root node must be listed first in "nodes"; every delegates edge originates at the root;', - 'analysts are registry lens ids, never node ids; "deliverableDescribe" is the driver\'s real mission text.', + 'Apply the attached skill to this case and return the required JSON object.', ].join('\n') } @@ -189,25 +256,61 @@ function extractJson(text: string): string { return stripped.slice(start, end + 1) } -export async function callAuthor(prompt: string, temperature = 0.2, maxTokens = 6000): Promise { - const res = await fetch(ROUTER_URL, { - method: 'POST', - headers: { Authorization: `Bearer ${routerToken()}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ - model: AUTHOR_MODEL, - temperature, - max_tokens: maxTokens, - messages: [{ role: 'user', content: prompt }], - }), - signal: AbortSignal.timeout(240_000), +export interface AuthorAttemptEvidence { + status: string + usage?: { input: number; output: number; costUsd?: number; model?: string } + raw?: string + error?: string +} + +class AuthoringFailedError extends Error { + constructor( + message: string, + readonly attempts: AuthorAttemptEvidence[], + ) { + super(message) + this.name = 'AuthoringFailedError' + } +} + +export async function callAuthor( + profile: AgentProfile, + prompt: string, + env: NodeJS.ProcessEnv = process.env, +): Promise<{ + finalText: string + status: string + usage: { input: number; output: number; costUsd?: number; model?: string } + error?: { message: string } +}> { + const bridgeBearer = env.AGENT_GRAPHS_BRIDGE_BEARER ?? env.BRIDGE_BEARER + if (!bridgeBearer) { + throw new Error('AGENT_GRAPHS_BRIDGE_BEARER or BRIDGE_BEARER is required') + } + const bridgeUrl = env.AGENT_GRAPHS_BRIDGE_URL ?? env.BRIDGE_URL ?? 'http://127.0.0.1:3355' + const timeoutMs = optionalPositiveInteger( + 'AGENT_GRAPHS_AUTHOR_TIMEOUT_MS', + env.AGENT_GRAPHS_AUTHOR_TIMEOUT_MS, + ) + const factory = createExecutor({ + backend: 'bridge', + bridgeUrl, + bridgeBearer, + agentProfile: profile, }) - if (!res.ok) throw new Error(`router HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`) - const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - const content = data.choices?.[0]?.message?.content - if (typeof content !== 'string' || content.trim().length === 0) { - throw new Error('router returned empty content') + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, agentRunName: profile.name ?? 'agent-graphs-author' }, + prompt, + timeoutMs === undefined ? {} : { timeoutMs }, + ), + ) + return { + finalText: turn.finalText, + status: turn.status, + usage: turn.usage, + ...(turn.error ? { error: { message: turn.error.message } } : {}), } - return content } interface AuthoredReply { @@ -215,15 +318,39 @@ interface AuthoredReply { reason: string graph?: AuthoredGraphSpec raw: string + authorAttempts: AuthorAttemptEvidence[] } -/** Prompt the author; one retry on unparseable JSON (or a transport fault). */ -async function authorOnce(surface: string, kase: CaseSpec): Promise { - const prompt = authorPrompt(surface, kase) +/** Prompt the profile; every attempt is retained and the caller controls the retry count. */ +async function authorOnce( + surface: string, + kase: CaseSpec, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const prompt = authorPrompt(kase) + const profile = buildAgentGraphsAuthorProfile(surface, env) + const attemptLimit = positiveInteger( + 'AGENT_GRAPHS_AUTHOR_ATTEMPTS', + env.AGENT_GRAPHS_AUTHOR_ATTEMPTS, + 2, + ) + const attempts: AuthorAttemptEvidence[] = [] let lastErr: unknown - for (let attempt = 0; attempt < 2; attempt += 1) { + for (let attempt = 0; attempt < attemptLimit; attempt += 1) { + let evidence: AuthorAttemptEvidence | undefined try { - const raw = await callAuthor(prompt) + const turn = await callAuthor(profile, prompt, env) + evidence = { + status: turn.status, + usage: turn.usage, + raw: turn.finalText, + ...(turn.error ? { error: turn.error.message } : {}), + } + attempts.push(evidence) + if (turn.status !== 'completed') { + throw new Error(turn.error?.message ?? `author turn ended with status ${turn.status}`) + } + const raw = turn.finalText const parsed = JSON.parse(extractJson(raw)) as { decision?: string reason?: string @@ -238,12 +365,24 @@ async function authorOnce(surface: string, kase: CaseSpec): Promise { +export async function runAuthoredOffline( + spec: AuthoredGraphSpec, + runId: string, +): Promise { const rootId = findRootId(spec) - const workerIds = spec.nodes.map((n) => n.id).filter((id) => id !== rootId) + const nodeIds = new Set(spec.nodes.map((node) => node.id)) + const workerIds = [ + ...new Set( + spec.edges + .filter((edge): edge is Extract => + edge.kind === 'delegates') + .map((edge) => edge.to), + ), + ] + const analystNodeIds = [ + ...new Set( + spec.edges + .filter((edge): edge is Extract => + edge.kind === 'analyzes' && nodeIds.has(edge.analyst)) + .map((edge) => edge.analyst), + ), + ] const graph: AgentGraph = { nodes: spec.nodes.map((n) => ({ @@ -295,10 +453,14 @@ async function runAuthoredOffline(spec: AuthoredGraphSpec, runId: string): Promi budget: spec.budget, } - // Lenses for whatever analyst ids the author named: ENVIRONMENT, never nodes. + // Only non-node analyst ids are registry lenses. A node id runs through the same + // makeWorkerAgent path as every other graph node and must not also enter the registry. const analystIds = [ ...new Set( - spec.edges.filter((e) => e.kind === 'analyzes').map((e) => (e as { analyst: string }).analyst), + spec.edges + .filter((edge): edge is Extract => + edge.kind === 'analyzes' && !nodeIds.has(edge.analyst)) + .map((edge) => edge.analyst), ), ] const analysts: AnalystRegistry | undefined = @@ -314,6 +476,10 @@ async function runAuthoredOffline(spec: AuthoredGraphSpec, runId: string): Promi : undefined const received: AgentProfile[] = [] + const analystNodeRuns = spec.edges + .filter((edge): edge is Extract => + edge.kind === 'analyzes' && nodeIds.has(edge.analyst)) + .reduce((sum, edge) => sum + edge.over.length, 0) const turns: ScriptedTurn[] = [ { toolCalls: workerIds.map((id) => ({ @@ -321,7 +487,9 @@ async function runAuthoredOffline(spec: AuthoredGraphSpec, runId: string): Promi arguments: { profile: { name: id }, task: `work the '${id}' role` }, })), }, - ...workerIds.map(() => ({ toolCalls: [{ name: 'await_event', arguments: {} }] })), + ...Array.from({ length: workerIds.length + analystNodeRuns }, () => ({ + toolCalls: [{ name: 'await_event', arguments: {} }], + })), { content: 'done' }, ] @@ -330,10 +498,10 @@ async function runAuthoredOffline(spec: AuthoredGraphSpec, runId: string): Promi maxLiveWorkers: Math.max(workerIds.length, 1), ...(spec.perWorker !== undefined ? { perWorker: spec.perWorker } : {}), ...(analysts !== undefined ? { analysts } : {}), - makeWorkerAgent: leafSeam( - received, - Object.fromEntries(workerIds.map((id) => [id, { withTrace: true }])), - ), + makeWorkerAgent: leafSeam(received, { + ...Object.fromEntries(workerIds.map((id) => [id, { withTrace: true }])), + ...Object.fromEntries(analystNodeIds.map((id) => [id, {}])), + }), brain: scriptedBrain(turns), } @@ -347,17 +515,22 @@ async function runAuthoredOffline(spec: AuthoredGraphSpec, runId: string): Promi /** CLOSURE A — `dispatchWithSurface(surface, scenario)`: author from the skill text, lower, * execute offline. A refusal is data (`validationError`), never a crash. */ -export async function dispatchWithSurface(surface: string, scenario: CaseSpec): Promise { - const reply = await authorOnce(surface, scenario) +export async function dispatchWithSurface( + surface: string, + scenario: CaseSpec, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const reply = await authorOnce(surface, scenario, env) const artifact: AuthoredArtifact = { decision: reply.decision, reason: reply.reason, ...(reply.graph !== undefined ? { graph: reply.graph } : {}), raw: reply.raw, + authorAttempts: reply.authorAttempts, } if (reply.decision !== 'graph' || reply.graph === undefined) return artifact try { - artifact.run = await runAuthoredOffline(reply.graph, `codemode-${scenario.id}`) + artifact.run = await runAuthoredOffline(reply.graph, `agent-graphs-${scenario.id}`) } catch (err) { if (err instanceof GraphEdgeCapError) { // Cap exhaustion still carries the full ledger — keep the evidence AND the refusal. @@ -449,14 +622,14 @@ export function judgeArtifact(artifact: AuthoredArtifact, kase: CaseSpec): { sco : `no graph authored (decision=${artifact.decision})`, }) } - if (e.nodes !== undefined) { - const total = graphOk ? g.nodes.length : 0 - const workers = graphOk ? Math.max(total - 1, 0) : 0 + if (e.delegatedWorkers !== undefined) { + const workers = graphOk ? delegatesEdges(g).length : 0 checks.push({ - key: 'nodes', - // The case files don't say whether the count includes the root; accept either reading. - pass: graphOk && (workers === e.nodes || total === e.nodes), - note: graphOk ? `workers=${workers} total=${total} expected=${e.nodes}` : 'no graph authored', + key: 'delegatedWorkers', + pass: graphOk && workers === e.delegatedWorkers, + note: graphOk + ? `delegated workers=${workers} expected=${e.delegatedWorkers}` + : 'no graph authored', }) } if (e.maxTraversalsAtLeast !== undefined) { @@ -516,34 +689,21 @@ export function judgeArtifact(artifact: AuthoredArtifact, kase: CaseSpec): { sco note: graphOk ? `analyzes edges=${analyzesEdges(g).length} warranted=${e.analyzesWarranted}` : 'no graph authored', }) } - if (e.wrongIfAnalystIsNode !== undefined) { - const nodeIds = graphOk ? new Set(g.nodes.map((n) => n.id)) : new Set() - const offenders = graphOk ? analyzesEdges(g).filter((a) => nodeIds.has(a.analyst)) : [] - checks.push({ - key: 'wrongIfAnalystIsNode', - pass: graphOk && offenders.length === 0, - note: graphOk - ? offenders.length === 0 - ? 'no analyst id collides with a node id' - : `analyst ids that are nodes: ${offenders.map((o) => o.analyst).join(', ')}` - : 'no graph authored', - }) - } - if (e.generousBudgetsBecauseFloorUnknown !== undefined) { - const perChild = graphOk ? perChildTokens(g) : 0 + if (e.reasonMentionsUnknownBudget !== undefined) { + const namesUncertainty = /unknown|unmeasured|measure|not (?:yet )?(?:known|established)/i.test( + artifact.reason, + ) checks.push({ - key: 'generousBudgetsBecauseFloorUnknown', - pass: graphOk && perChild >= GENEROUS_PER_CHILD_TOKENS, - note: graphOk - ? `per-child tokens ${perChild} vs generous line ${GENEROUS_PER_CHILD_TOKENS}` - : 'no graph authored', + key: 'reasonMentionsUnknownBudget', + pass: namesUncertainty === e.reasonMentionsUnknownBudget, + note: `reason names unmeasured budget=${namesUncertainty}`, }) } if (e.edges !== undefined) { for (const want of e.edges) { if (/delegates/i.test(want)) { const rootId = graphOk ? findRootId(g) : '' - const workers = graphOk ? g.nodes.map((n) => n.id).filter((id) => id !== rootId) : [] + const workers = graphOk ? delegatesEdges(g).map((edge) => edge.to) : [] const covered = graphOk ? workers.every( (id) => @@ -598,6 +758,15 @@ interface CaseResult { runResultKind?: string ledgerRows?: number exhaustedEdges?: ReadonlyArray + authorAttempts: AuthorAttemptEvidence[] + authorFailed?: true +} + +export function baselineIsPublishable( + only: string | undefined, + results: ReadonlyArray<{ authorFailed?: true }>, +): boolean { + return only === undefined && results.length > 0 && results.every((result) => !result.authorFailed) } async function main(): Promise { @@ -606,9 +775,10 @@ async function main(): Promise { // CASE= runs a subset — the smoke lever; the baseline artifact is only written on a full run. const only = process.env.CASE const cases: CaseSpec[] = inputs.cases.filter((c) => only === undefined || c.id === only) + const authorProfile = buildAgentGraphsAuthorProfile(surface) console.log( - `codemode baseline: skill v1 (${surface.length} chars, source=${inputs.source}), ${cases.length} cases, author=${AUTHOR_MODEL}`, + `agent-graphs baseline: skill v1 (${surface.length} chars, source=${inputs.source}), ${cases.length} cases, author=${authorProfileLabel(authorProfile)}`, ) const results: CaseResult[] = [] @@ -633,6 +803,7 @@ async function main(): Promise { exhaustedEdges: artifact.run.exhaustedEdges, } : {}), + authorAttempts: artifact.authorAttempts, }) console.log(`${artifact.decision} score=${score.toFixed(2)} (${Math.round((Date.now() - t0) / 1000)}s)`) for (const line of reasons.filter((x) => !x.startsWith('PASS'))) console.log(` ${line}`) @@ -646,6 +817,8 @@ async function main(): Promise { reasons: [`AUTHOR-FAILED: ${message}`], validationError: message, reason: '', + authorAttempts: err instanceof AuthoringFailedError ? err.attempts : [], + authorFailed: true, }) console.log(`AUTHOR-FAILED (${message.slice(0, 80)})`) } @@ -663,17 +836,37 @@ async function main(): Promise { : scores.length % 2 === 1 ? (scores[mid] ?? 0) : ((scores[mid - 1] ?? 0) + (scores[mid] ?? 0)) / 2 + const authorAttempts = results.flatMap((result) => result.authorAttempts) + const usageReceipts = authorAttempts.flatMap((attempt) => + attempt.usage === undefined ? [] : [attempt.usage], + ) + const costAccountingComplete = + authorAttempts.length > 0 && + authorAttempts.every((attempt) => attempt.usage?.costUsd !== undefined) + const authorUsage = { + attempts: authorAttempts.length, + usageReceipts: usageReceipts.length, + input: usageReceipts.reduce((sum, usage) => sum + usage.input, 0), + output: usageReceipts.reduce((sum, usage) => sum + usage.output, 0), + costUsd: costAccountingComplete + ? usageReceipts.reduce((sum, usage) => sum + (usage.costUsd ?? 0), 0) + : null, + costAccountingComplete, + } + const out = { skillVersion: 'v1', surfaceSource: inputs.source, - authorModel: AUTHOR_MODEL, - temperature: 0.2, + authorProfile, + authorModel: authorProfileLabel(authorProfile), + temperature: null, date: new Date().toISOString(), n: results.length, aggregate: { mean, median, min: scores[0] ?? 0, max: scores[scores.length - 1] ?? 0 }, + authorUsage, cases: results, } - const wrote = only === undefined + const wrote = baselineIsPublishable(only, results) if (wrote) { mkdirSync(dirname(OUT_PATH), { recursive: true }) writeFileSync(OUT_PATH, `${JSON.stringify(out, null, 2)}\n`) @@ -690,13 +883,20 @@ async function main(): Promise { } console.log('─'.repeat(96)) console.log(`mean=${mean.toFixed(3)} median=${median.toFixed(3)} min=${(scores[0] ?? 0).toFixed(2)} max=${(scores[scores.length - 1] ?? 0).toFixed(2)} n=${results.length}`) - console.log(wrote ? `written: ${OUT_PATH}` : 'subset run (CASE set) — baseline artifact NOT written') + if (wrote) { + console.log(`written: ${OUT_PATH}`) + } else if (only !== undefined) { + console.log('subset run (CASE set) — baseline artifact NOT written') + } else { + throw new Error( + `baseline artifact NOT written: ${results.filter((result) => result.authorFailed).length}/${results.length} cases lacked a scorable author result`, + ) + } } -// Run the baseline only when executed directly; gen2 imports this module for its closures. -const invokedDirectly = - process.argv[1] !== undefined && fileURLToPath(import.meta.url) === resolvePath(process.argv[1]) -if (invokedDirectly) { +// Run the baseline only when executed directly; the improvement runner imports its closures. +const invokedPath = process.argv[1] +if (invokedPath !== undefined && resolve(invokedPath) === fileURLToPath(import.meta.url)) { main().catch((err) => { console.error(err instanceof Error ? (err.stack ?? err.message) : String(err)) process.exit(1) diff --git a/bench/src/agent-graphs-improve.test.mts b/bench/src/agent-graphs-improve.test.mts new file mode 100644 index 00000000..72fd374d --- /dev/null +++ b/bench/src/agent-graphs-improve.test.mts @@ -0,0 +1,113 @@ +import assert from 'node:assert/strict' +import { execFileSync } from 'node:child_process' +import { readFileSync } from 'node:fs' +import { dirname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import test from 'node:test' +import { + baselineIsPublishable, + buildAgentGraphsAuthorProfile, + runAuthoredOffline, +} from './agent-graphs-improve.mts' + +const here = dirname(fileURLToPath(import.meta.url)) +const runner = join(here, 'agent-graphs-improve.mts') +const improvementRunner = join(here, 'agent-graphs-gen2.mts') + +test('agent-graphs runner loads only its canonical working-tree inputs', () => { + const source = readFileSync(runner, 'utf8') + assert.doesNotMatch(source, /skills['"], ['"]codemode|codemode-/) + for (const path of [runner, improvementRunner]) { + const modelSource = readFileSync(path, 'utf8') + assert.doesNotMatch(modelSource, /\bfetch\(|chat\/completions|max_tokens|AbortSignal\.timeout/) + } + const improvementSource = readFileSync(improvementRunner, 'utf8') + assert.match(improvementSource, /buildAgentGraphsAuthorProfile/) + assert.match(improvementSource, /dispatchWithSurface\(surface, scenario, AUTHOR_ENV\)/) + assert.match(improvementSource, /callAuthor\(proposerProfile, prompt, PROPOSER_ENV\)/) + + const stdout = execFileSync(process.execPath, ['--import', 'tsx', runner], { + cwd: join(here, '..'), + encoding: 'utf8', + env: { ...process.env, CASE: '__input-path-smoke__', SKILL_REF: undefined }, + }) + + assert.match(stdout, /agent-graphs baseline: skill v1 \(.+source=working-tree\), 0 cases/) + assert.match(stdout, /subset run \(CASE set\) — baseline artifact NOT written/) +}) + +test('agent-graphs author is a canonical, overridable Pi profile', () => { + const baseline = buildAgentGraphsAuthorProfile('# Agent graphs\nUse the smallest correct form.\n', {}) + assert.equal(baseline.harness, 'pi') + assert.deepEqual(baseline.model, { + provider: 'tangle-router', + default: 'deepseek-v4-flash', + reasoningEffort: 'ultracode', + }) + assert.match(baseline.prompt?.systemPrompt ?? '', /attached agent-graphs skill/) + assert.match(baseline.prompt?.systemPrompt ?? '', /registered lens id or a graph node/) + assert.doesNotMatch(baseline.prompt?.systemPrompt ?? '', /never node ids/) + assert.deepEqual(baseline.resources?.skills, [ + { + kind: 'inline', + name: 'agent-graphs', + content: '# Agent graphs\nUse the smallest correct form.\n', + }, + ]) + + const overridden = buildAgentGraphsAuthorProfile('custom', { + AGENT_GRAPHS_AUTHOR_HARNESS: 'opencode', + AGENT_GRAPHS_AUTHOR_PROVIDER: 'custom-provider', + AGENT_GRAPHS_AUTHOR_MODEL: 'custom-model', + AGENT_GRAPHS_AUTHOR_REASONING_EFFORT: 'low', + AGENT_GRAPHS_AUTHOR_SYSTEM_PROMPT: 'custom system', + }) + assert.equal(overridden.harness, 'opencode') + assert.deepEqual(overridden.model, { + provider: 'custom-provider', + default: 'custom-model', + reasoningEffort: 'low', + }) + assert.equal(overridden.prompt?.systemPrompt, 'custom system') +}) + +test('offline scoring executes an analyst graph node through the real graph path', async () => { + const result = await runAuthoredOffline( + { + nodes: [ + { id: 'driver', systemPrompt: 'Drive the work.' }, + { id: 'implementer', systemPrompt: 'Implement the change.' }, + { id: 'reviewer', systemPrompt: 'Review the trace.' }, + ], + edges: [ + { kind: 'delegates', from: 'driver', to: 'implementer' }, + { + kind: 'analyzes', + analyst: 'reviewer', + over: ['implementer'], + to: 'driver', + }, + ], + budget: { maxIterations: 20, maxTokens: 200_000 }, + perWorker: { maxIterations: 4, maxTokens: 40_000 }, + deliverableDescribe: 'Implement the requested change and return the completed artifact.', + }, + 'agent-graphs-test-analyst-node', + ) + + assert.equal(result.resultKind, 'winner') + assert.deepEqual( + result.ledger.map((row) => [row.kind, row.outcome]), + [ + ['delegates', 'delivered'], + ['analyzes', 'delivered'], + ], + ) +}) + +test('an incomplete full run cannot replace the canonical baseline', () => { + assert.equal(baselineIsPublishable(undefined, [{}, {}]), true) + assert.equal(baselineIsPublishable(undefined, [{}, { authorFailed: true }]), false) + assert.equal(baselineIsPublishable('one-case', [{}]), false) + assert.equal(baselineIsPublishable(undefined, []), false) +}) diff --git a/bench/src/agent-graphs-improve/offline-seams.mts b/bench/src/agent-graphs-improve/offline-seams.mts index a0b73585..e1201bf1 100644 --- a/bench/src/agent-graphs-improve/offline-seams.mts +++ b/bench/src/agent-graphs-improve/offline-seams.mts @@ -1,5 +1,5 @@ /** - * OFFLINE seams for the codemode-skill improvement harness — copied from + * Offline execution inputs for the agent-graphs skill improvement runner, copied from * `examples/graphs/shared.ts` (the same two seams the kernel's own graph tests use), with the * imports rewritten to the worktree's own src so the bench runs against the in-repo kernel * without a dist build: diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index b1e3ac3e..37a3226e 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -7,7 +7,7 @@ # Primitive catalog — the never-stale anti-reinvention inventory -> **GENERATED** from `@tangle-network/agent-runtime@0.126.0` and `@tangle-network/agent-eval@0.143.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. +> **GENERATED** from `@tangle-network/agent-runtime@0.127.0` and `@tangle-network/agent-eval@0.143.0` by `scripts/gen-primitive-catalog.mjs`. Do NOT hand-edit — run `pnpm run docs:api`. This is the mechanical companion to the JUDGMENT in `canonical-api.md` (§2 decision table + §1.5 AgentProfile law): that doc says WHICH primitive to reach for and what NOT to build; this catalog proves WHAT exists. Per-symbol signatures + `file:line` live in the per-module pages under `docs/api/`. ## 1. agent-runtime — own public surface diff --git a/docs/canonical-api.md b/docs/canonical-api.md index c1c56424..1c9686f5 100644 --- a/docs/canonical-api.md +++ b/docs/canonical-api.md @@ -4,7 +4,7 @@ Generated signatures and the complete export list live in docs/api/. Run pnpm docs:freshness after editing this file. --> -> **Version 0.126.0.** +> **Version 0.127.0.** > [`docs/api/primitive-catalog.md`](./api/primitive-catalog.md) lists every export and import path. > `agent-eval` must satisfy `>=0.143.0 <0.144.0`. > `sandbox` must satisfy `>=0.17.2 <0.18.0`. diff --git a/package.json b/package.json index e1d069fe..ba228406 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@tangle-network/agent-runtime", - "version": "0.126.0", + "version": "0.127.0", "description": "Shared task-lifecycle skeleton for agents: a recursive loop kernel for chat turns, one-shot tasks, and multi-attempt loops, with trace capture and eval-gated self-improvement. Domain behavior lives in adapters; scoring and ship-gates in @tangle-network/agent-eval.", "homepage": "https://github.com/tangle-network/agent-runtime#readme", "repository": { diff --git a/skills/agent-graphs/baseline-v1.json b/skills/agent-graphs/baseline-v1.json index f09cd748..7bae315c 100644 --- a/skills/agent-graphs/baseline-v1.json +++ b/skills/agent-graphs/baseline-v1.json @@ -1,146 +1,229 @@ { "skillVersion": "v1", "surfaceSource": "working-tree", - "authorModel": "glm-5.2", - "temperature": 0.2, - "date": "2026-08-03T17:42:45.828Z", + "authorProfile": { + "name": "agent-graphs-author", + "harness": "pi", + "model": { + "provider": "tangle-router", + "default": "deepseek-v4-flash", + "reasoningEffort": "ultracode" + }, + "prompt": { + "systemPrompt": "You are an agent-graph author.\nFollow the attached agent-graphs skill exactly; it is your only workflow doctrine.\nDecide whether the task needs one agent, a fixed AgentGraph, or a dynamic supervise workflow.\nReply with JSON only: no markdown fences and no prose outside the JSON.\n{\"decision\":\"graph\"|\"single-agent\"|\"dynamic-workflow\",\"reason\":string,\"graph\"?:{...}}\nInclude \"graph\" if and only if decision is \"graph\", with this exact shape:\n{\"nodes\":[{\"id\":string,\"systemPrompt\":string}, ...],\n \"edges\":[{\"kind\":\"delegates\",\"from\":string,\"to\":string,\"maxTraversals\"?:number}\n | {\"kind\":\"analyzes\",\"analyst\":string,\"over\":[string,...],\"to\":string,\"maxTraversals\"?:number}, ...],\n \"budget\":{\"maxIterations\":number,\"maxTokens\":number},\n \"perWorker\"?:{\"maxIterations\"?:number,\"maxTokens\"?:number},\n \"deliverableDescribe\":string}\nThe root node must be first; every delegates edge starts at the root.\nAn analyst is either a registered lens id or a graph node with no incoming delegation edge.\nThe deliverable description is the driver mission, not a generic completion phrase." + }, + "resources": { + "failOnError": true, + "skills": [ + { + "kind": "inline", + "name": "agent-graphs", + "content": "---\nname: agent-graphs\ndescription: Author runGraph programs from AgentProfiles and versioned prompt directives.\n---\n\n# Agent graphs\n\nUse this skill when every role is known before execution and the relationship between roles must be reviewable as data.\nThe output is an `AgentGraph` executed by `runGraph`, not a new coordinator or workflow framework.\n\n## Choose the existing entry point\n\n| Need | Use |\n| --- | --- |\n| Known roles with versioned work and analysis instructions | `runGraph` |\n| A standard fixed shape such as parallel attempts, a chain, or a review panel | `fanout`, `pipeline`, `verify`, or `panel` |\n| A model decides which workers to create while it works | `supervise` |\n| One profile can complete the task directly | Run that profile without composition |\n\nDo not force a dynamic task into a static graph.\nDo not use a graph when a smaller shipped primitive already expresses the work.\n\n## Author the complete contract\n\nAn `AgentGraph` has four required fields: `nodes`, `edges`, `deliverable`, and `budget`.\n`runGraph(graph, options)` validates graph structure and prompt references before it spends compute.\n\n### Nodes\n\nEach node is `{ id, profile }`, where `profile` is a complete canonical `AgentProfile`.\nSet `profile.name` equal to `id` because Runtime uses that value to select and route the node.\nPut the standing role in `profile.prompt.systemPrompt` and capabilities in the profile's tools, MCP, resources, hooks, and subagents.\nDo not rebuild profile materialization in graph code.\n\n### Delegation edges\n\nA delegation edge is `{ kind: 'delegates', from, to, directive, maxTraversals? }`.\nThe directive is a registered, versioned `PromptHandle`, such as `promptHandle('delegates/research-brief/v1')`.\nEach spawn and each later steer over the same edge consumes one traversal.\nThe default cap is `defaultEdgeTraversalCap`; exhaustion refuses further delegation.\n\nThe current graph form has one root and a static set of worker nodes.\nEvery delegation edge starts at the root, and each worker has exactly one incoming delegation edge.\nUse a new directive version to change a brief instead of adding a second edge to the same worker.\n\n### Analysis edges\n\nAn analysis edge is `{ kind: 'analyzes', analyst, over, to, directive, maxTraversals? }`.\nIt runs after a listed worker settles and routes findings to one node.\n\n`analyst` has two supported forms:\n\n- A lens id from `options.analysts` runs a caller-supplied analysis function.\n- A graph node id runs that node's pinned `AgentProfile` as a tool-equipped analyst.\n\nAn analyst node has no incoming delegation edge, so the root cannot hand it ordinary work.\nAn id cannot be both a registered lens and an analyst node.\n`over` lists delegated worker nodes only; Runtime refuses the root and analyst nodes because neither settles as an ordinary worker.\nAn analysis traversal cap records excess findings as `unpropagated`; it does not stop the run.\n\n### Completion and budget\n\n`deliverable.check(output)` is the independent completion test.\nIt must accept a genuinely complete result and reject junk.\nPut the concrete mission in `deliverable.describe`; Runtime uses that text as the root's task.\n\n`budget` is one conserved pool for the full graph.\nSet `options.perWorker` explicitly from the actual executor cost.\nFor Pi, `WORKER_TOKEN_FLOOR.pi` is 31,211 input tokens before useful work, so a worker allocation below that value is refused.\nTreat an unmeasured executor floor as unknown rather than zero.\nAnalyst nodes spend from the same pool and need the same honest accounting as ordinary workers.\n\n## Authoring procedure\n\n1. Write the completion test and its description first.\n2. Choose the smallest shipped entry point from the table above.\n3. Give every distinct role one complete `AgentProfile`; merge roles whose standing prompts and capabilities are identical.\n4. Register a versioned directive for every edge.\n5. Add one delegation edge per ordinary worker.\n6. Add analysis only when findings must be produced independently after a worker settles.\n7. Size the shared pool, per-worker allocation, traversal caps, time, and concurrency from measured executor behavior.\n8. Run the structure offline, then run the real backend and inspect its result.\n\n## Prove the graph before spending\n\nUse an injected `brain` plus `makeWorkerAgent` to exercise graph structure without a network call.\nCover invalid profiles, unknown directives, impossible analysis routes, traversal exhaustion, successful completion, and rejected junk.\nStart from the runnable programs in `examples/graphs/` rather than creating a second graph runner.\n\nOffline execution proves control flow only.\nA real task must still use the intended backend, profiles, tools, completion test, and budget before claiming the graph solves that task.\n\n## Read the complete result\n\n| Field | Meaning |\n| --- | --- |\n| `result.result.kind` and `reason` | Whether a result won and why execution ended |\n| `result.result.spentTotal` | Tokens and money, including whether each total is known |\n| `result.ledger` | Every delivered, stripped, empty, or unpropagated edge traversal with byte counts |\n| `result.exhaustedEdges` | Every edge whose cap was reached, including normal lifecycle endings |\n| Journal `edge` events | Durable copies of traversal evidence |\n\nZero traversals on an expected edge means the graph did not exercise that relationship.\n`usdKnown: false` means cost is missing, not free.\nA passing completion test proves only what that test checks.\n\n## Common mistakes\n\n- Putting the task only in a spawn prompt instead of `deliverable.describe`.\n- Giving a node a `profile.name` different from its id.\n- Delegating ordinary work to an analyst node.\n- Listing the root or an analyst node in `analyzes.over`.\n- Using an analysis cap as a stop condition.\n- Allowing a driver-authored spawn profile to add capabilities instead of defining them on the pinned node profile.\n- Reading only thrown cap errors and missing `result.exhaustedEdges` on budget or cancellation endings.\n- Treating unknown spend as zero.\n- Claiming recursive or runtime-discovered structure when the current graph is a static root with workers and analysts.\n\n## Improve only after measurement\n\nRuntime already optimizes one inline skill through `improve(profile, { surface: 'skills', skills: { resourceName }, ... })`.\nPut the exact skill bytes in `profile.resources.skills`, set `profile.resources.failOnError: true`, supply disjoint development and final-test tasks, and pass a complete Agent Eval optimization method.\nDo not create a graph-specific optimizer, campaign runner, candidate store, or promotion path.\n\n## Then consider\n\n- `loop-writer` when the required dynamic structure still cannot be expressed by `supervise` or another shipped primitive; pass the exact missing behavior and the completion test.\n- `verify` before publishing a graph consumer; pass the real backend command, expected result fields, and failure cases.\n" + } + ] + } + }, + "authorModel": "pi/tangle-router/deepseek-v4-flash", + "temperature": null, + "date": "2026-08-03T19:11:53.577Z", "n": 8, "aggregate": { - "mean": 0.6375, - "median": 0.8, + "mean": 0.6875, + "median": 1, "min": 0, - "max": 1, - "n": 8, - "correction": "first publication reported median 1.0 from an upper-middle indexing bug; corrected to the interpolated median", - "samplePerCase": 1, - "varianceNote": "decision-level variance observed at temp 0.2 (cap-as-stop-mistake flipped graph->single-agent on rerun); treat per-case scores as n=1 draws, not stable properties. Reps belong to the v2 gate round." + "max": 1 + }, + "authorUsage": { + "attempts": 8, + "usageReceipts": 8, + "input": 159370, + "output": 32135, + "costUsd": null, + "costAccountingComplete": false }, "cases": [ { "id": "cap-as-stop-mistake", + "decision": "dynamic-workflow", + "score": 0, + "reasons": [ + "FAIL trapIsAnalyzesCapAsStop: no graph authored", + "FAIL correctStopIsDelegatesCapOrDeliverable: no graph authored" + ], + "reason": "This case is the 'cap-as-stop mistake' from the agent-graphs skill: the requested behavior is to stop the whole run after three findings, but an analysis edge's maxTraversals is not a stop condition — the skill explicitly lists 'using an analysis cap as a stop condition' as a common mistake and states that an analysis traversal cap records excess findings as unpropagated and does not stop the run. A static graph with an analyst watching a worker therefore cannot express 'halt everything at three findings.' Stopping after accumulating three findings is a runtime supervision decision about whether work should continue, which matches the `supervise` entry point (a supervising agent watches the worker and decides when to end the run), not a fixed root-with-workers-and-analyst graph. The cap-as-stop mistake is avoided by delegating this to supervise rather than encoding the limit as an analysis cap.", + "authorAttempts": [ + { + "status": "completed", + "usage": { + "input": 8029, + "output": 1151 + }, + "raw": "{\"decision\":\"dynamic-workflow\",\"reason\":\"This case is the 'cap-as-stop mistake' from the agent-graphs skill: the requested behavior is to stop the whole run after three findings, but an analysis edge's maxTraversals is not a stop condition — the skill explicitly lists 'using an analysis cap as a stop condition' as a common mistake and states that an analysis traversal cap records excess findings as unpropagated and does not stop the run. A static graph with an analyst watching a worker therefore cannot express 'halt everything at three findings.' Stopping after accumulating three findings is a runtime supervision decision about whether work should continue, which matches the `supervise` entry point (a supervising agent watches the worker and decides when to end the run), not a fixed root-with-workers-and-analyst graph. The cap-as-stop mistake is avoided by delegating this to supervise rather than encoding the limit as an analysis cap.\"}" + } + ] + }, + { + "id": "floor-trap-pi", + "decision": "single-agent", + "score": 0, + "reasons": [ + "FAIL mustBudgetAtLeast: no graph authored (decision=single-agent)", + "FAIL delegatedWorkers: no graph authored", + "FAIL analyzesWarranted: no graph authored" + ], + "reason": "Writing a one-line file is a direct task one profile can complete without composition; the skill's entry-point table says to run that profile without a graph when a single profile can complete the task directly. A static graph would also be wasteful here: Pi workers carry a WORKER_TOKEN_FLOOR of ~31k input tokens before useful work, so a 'few thousand token' budget cannot be expressed as a graph worker allocation anyway. No analysis, no supervision, no multi-role structure is needed.", + "authorAttempts": [ + { + "status": "completed", + "usage": { + "input": 9212, + "output": 503 + }, + "raw": "{\"decision\":\"single-agent\",\"reason\":\"Writing a one-line file is a direct task one profile can complete without composition; the skill's entry-point table says to run that profile without a graph when a single profile can complete the task directly. A static graph would also be wasteful here: Pi workers carry a WORKER_TOKEN_FLOOR of ~31k input tokens before useful work, so a 'few thousand token' budget cannot be expressed as a graph worker allocation anyway. No analysis, no supervision, no multi-role structure is needed.\"}" + } + ] + }, + { + "id": "mission-in-deliverable", "decision": "graph", "score": 1, "reasons": [ - "PASS trapIsAnalyzesCapAsStop: analyzes caps=true delegates caps=true", - "PASS correctStopIsDelegatesCapOrDeliverable: delegates caps=true, reason mentions deliverable=true" + "PASS deliverableDescribeCarriesMission: describe 69 chars, domain words hit: [changelog, format] of [changelog, format]" ], - "reason": "The brief asks for an analyst to watch a worker \u2014 a legitimate unaddressable observation lens \u2014 so a static graph is the right dialect. But the brief's stated stop mechanism is the textbook pothole: an analyzes edge's maxTraversals is observability-only; on exhaustion the traversal is ledgered 'unpropagated' and it never refuses, so it cannot terminate a run. Only a delegates cap or the deliverable check can. I therefore encode 'three findings' in the deliverable (the real termination oracle) and let the worker settle once three findings exist; the findings-lens analyst observes and routes to the root with maxTraversals=3 as the intended observation budget, but it is NOT load-bearing for termination. If the brief truly needs hard termination at exactly three analyst findings regardless of worker output, that is not expressible in the current P0 topology and must be re-scoped before running.", + "reason": "This is a known-shape task with two reviewable roles: a writer that produces the CHANGELOG entry and an independent format-check analyst that verifies it after the writer settles, with findings routed back to the root so the root can steer fixes until the check passes. That matches the runGraph entry point (static root + workers + analysts), not supervise or a single profile. Critically, the concrete mission (\"Produce a CHANGELOG entry for last week that passes our format check\") is placed in deliverableDescribe — the driver mission — rather than only in a spawn directive, per the skill's completion/describe guidance.", "authoredGraph": { "nodes": [ { "id": "root", - "systemPrompt": "You are the root orchestrator. Hold the mission: produce at least three distinct findings on the delegated subject. Delegate to the worker, accept the findings-lens observations routed back to you, and stop spawning the moment the worker's settled output contains three findings. Do not delegate further once three are present." + "systemPrompt": "Driver for the changelog task. You receive the mission from the graph's driver task (deliverable.describe). Delegate the changelog writing to the writer, then review the format-check analyst's findings about the writer's entry and decide whether the deliverable is complete. If the format check reports violations, steer the writer to fix them and iterate until the entry passes. Your standing role is orchestration and final accept/reject of the deliverable." }, { - "id": "worker", - "systemPrompt": "You are the findings worker. Investigate the delegated brief and emit discrete, numbered findings, one paragraph each. Continue until you have produced at least three distinct findings, then settle. Do not pad or repeat." + "id": "writer", + "systemPrompt": "Changelog writer. Produce a CHANGELOG entry covering last week's changes in the project's required changelog format. Follow any steering directives from the root exactly, revising only the entry, and settle once the entry is written." + }, + { + "id": "format-checker", + "systemPrompt": "Format-check analyst. Inspect the writer's CHANGELOG entry and verify it against the project's changelog format check. Report concrete violations with line-level detail and the exact rule broken; do not edit the entry yourself. Confirm explicitly when the entry passes the format check." } ], "edges": [ { "kind": "delegates", "from": "root", - "to": "worker", - "maxTraversals": 6 + "to": "writer", + "maxTraversals": 4 }, { "kind": "analyzes", - "analyst": "findings-lens", + "analyst": "format-checker", "over": [ - "worker" + "writer" ], "to": "root", - "maxTraversals": 3 + "maxTraversals": 4 } ], "budget": { - "maxIterations": 8, - "maxTokens": 240000 + "maxIterations": 6, + "maxTokens": 200000 }, "perWorker": { - "maxIterations": 4, - "maxTokens": 80000 + "maxIterations": 2, + "maxTokens": 50000 }, - "deliverableDescribe": "Investigate the delegated subject and report at least three distinct, numbered findings. The run terminates when the worker's settled output contains three findings; the findings-lens analyst observes the worker and routes its observations to the root (observation only \u2014 it does not stop the run)." + "deliverableDescribe": "Produce a CHANGELOG entry for last week that passes our format check." }, "runResultKind": "winner", "ledgerRows": 2, - "exhaustedEdges": [] - }, - { - "id": "floor-trap-pi", - "decision": "single-agent", - "score": 0, - "reasons": [ - "FAIL mustBudgetAtLeast: no graph authored (decision=single-agent)", - "FAIL nodes: no graph authored", - "FAIL analyzesWarranted: no graph authored" - ], - "reason": "The task is trivially one-shot-able by a single pi agent (writing a one-line file) without requiring parallel fan-out or independent verification. Furthermore, the brief's tight budget constraint of a few thousand tokens falls drastically below the measured 31,211 input token floor (`WORKER_TOKEN_FLOOR`) required for a `runGraph` spawn, meaning any graph attempt would fail with a `below-runtime-floor` refusal." - }, - { - "id": "mission-in-deliverable", - "decision": "single-agent", - "score": 0, - "reasons": [ - "FAIL deliverableDescribeCarriesMission: no graph authored" - ], - "reason": "A single agent can author a CHANGELOG entry that satisfies a format check in a single pass, requiring no topology, parallelism, or independent verification." + "exhaustedEdges": [], + "authorAttempts": [ + { + "status": "completed", + "usage": { + "input": 9806, + "output": 2241 + }, + "raw": "{\"decision\":\"graph\",\"reason\":\"This is a known-shape task with two reviewable roles: a writer that produces the CHANGELOG entry and an independent format-check analyst that verifies it after the writer settles, with findings routed back to the root so the root can steer fixes until the check passes. That matches the runGraph entry point (static root + workers + analysts), not supervise or a single profile. Critically, the concrete mission (\\\"Produce a CHANGELOG entry for last week that passes our format check\\\") is placed in deliverableDescribe — the driver mission — rather than only in a spawn directive, per the skill's completion/describe guidance.\",\"graph\":{\"nodes\":[{\"id\":\"root\",\"systemPrompt\":\"Driver for the changelog task. You receive the mission from the graph's driver task (deliverable.describe). Delegate the changelog writing to the writer, then review the format-check analyst's findings about the writer's entry and decide whether the deliverable is complete. If the format check reports violations, steer the writer to fix them and iterate until the entry passes. Your standing role is orchestration and final accept/reject of the deliverable.\"},{\"id\":\"writer\",\"systemPrompt\":\"Changelog writer. Produce a CHANGELOG entry covering last week's changes in the project's required changelog format. Follow any steering directives from the root exactly, revising only the entry, and settle once the entry is written.\"},{\"id\":\"format-checker\",\"systemPrompt\":\"Format-check analyst. Inspect the writer's CHANGELOG entry and verify it against the project's changelog format check. Report concrete violations with line-level detail and the exact rule broken; do not edit the entry yourself. Confirm explicitly when the entry passes the format check.\"}],\"edges\":[{\"kind\":\"delegates\",\"from\":\"root\",\"to\":\"writer\",\"maxTraversals\":4},{\"kind\":\"analyzes\",\"analyst\":\"format-checker\",\"over\":[\"writer\"],\"to\":\"root\",\"maxTraversals\":4}],\"budget\":{\"maxIterations\":6,\"maxTokens\":200000},\"perWorker\":{\"maxIterations\":2,\"maxTokens\":50000},\"deliverableDescribe\":\"Produce a CHANGELOG entry for last week that passes our format check.\"}}" + } + ] }, { "id": "review-pipeline", "decision": "graph", - "score": 0.6, + "score": 1, "reasons": [ - "PASS nodes: workers=2 total=3 expected=3", - "FAIL analyzesWarranted: analyzes edges=0 warranted=true", - "PASS wrongIfAnalystIsNode: no analyst id collides with a node id", - "PASS edge:delegates to each reviewer: workers [security-reviewer, maintainability-reviewer] each delegated-to with >0 ledger traversals: true", - "FAIL edge:analyzes routing findings to root: analyzes\u2192root edges=0, fired with >0 traversals=false" + "PASS delegatedWorkers: delegated workers=2 expected=2", + "PASS analyzesWarranted: analyzes edges=1 warranted=true", + "PASS edge:delegates to each reviewer: workers [reviewer-correctness, reviewer-performance] each delegated-to with >0 ledger traversals: true", + "PASS edge:analyzes routing findings to root: analyzes→root edges=1, fired with >0 traversals=true" ], - "reason": "The case calls for two genuinely distinct review perspectives (security/correctness vs maintainability) that must settle independently, then a neutral arbiter synthesizing both \u2014 a fixed topology with independent verification that earns composition. One agent cannot maintain two addressable, independently-settled roles plus a neutral third.", + "reason": "The case is a merge-gate review with all roles known before execution: two reviewers with distinct standing perspectives plus a neutral decider who must rule only after both reviews settle. That is a fixed, known-role shape whose relationships need to be reviewable as data, so it fits runGraph (per the skill's entry table, known roles with versioned work/analysis instructions), not single-agent (three distinct roles) and not supervise (no runtime worker discovery — nothing decides which workers to create while working). The two reviewers are ordinary delegated workers (one delegation edge each from the root, each with a different standing prompt), and the neutral decider is an analyst node with no incoming delegation edge that analyzes over the two reviewers and routes its verdict to the root; the root drives the pipeline and emits the final merge decision. The mission (produce a reviewable approve-or-request-changes verdict from two independent perspectives before anything merges) lives in deliverableDescribe, and perWorker.maxTokens is sized above the Pi worker token floor (~31,211) so the allocation is not refused.", "authoredGraph": { "nodes": [ { - "id": "merge-arbiter", - "systemPrompt": "You are the neutral merge arbiter. You delegate code changes to two independent reviewers \u2014 one for security/correctness, one for maintainability \u2014 then collect both reviews and render a final merge decision: APPROVE, REQUEST-CHANGES, or REJECT. You must cite specific findings from each reviewer in your decision rationale. You do not review code yourself; you synthesize and decide." + "id": "root", + "systemPrompt": "You are the merge-gate driver for this code change. Delegate the same diff to two independent reviewers, each with a distinct standing perspective; collect both settled reviews plus the neutral decider's verdict; then produce the final deliverable: a single merge decision (approve or request-changes) with a concise rationale that accounts for both perspectives, plus a pointer to each reviewer's findings. Do not let either review's conclusions bias the other before they are produced — they must stay independent." + }, + { + "id": "reviewer-correctness", + "systemPrompt": "You are the correctness and security reviewer. Review the code change strictly from the correctness/security perspective: logic errors, race conditions, unsafe patterns, input validation, and vulnerability exposure. Produce concrete findings with file/line references and severity. Do not editorialize on style, performance, or architecture beyond what affects correctness or security." }, { - "id": "security-reviewer", - "systemPrompt": "You are a security and correctness reviewer. Examine code changes for vulnerabilities, input-validation gaps, race conditions, data-integrity risks, logic errors, and unhandled edge cases. Report each finding with a severity rating (critical, high, medium, low) and a concrete location in the diff. Do not comment on style or naming unless it introduces a correctness risk." + "id": "reviewer-performance", + "systemPrompt": "You are the performance and maintainability reviewer. Review the code change strictly from the performance/maintainability perspective: algorithmic complexity, hot-path cost, resource usage, API stability, and readability debt. Produce concrete findings with file/line references and severity. Do not editorialize on correctness or security beyond what affects performance or maintainability." }, { - "id": "maintainability-reviewer", - "systemPrompt": "You are a maintainability and clarity reviewer. Examine code changes for naming quality, structural clarity, test coverage, documentation gaps, convention adherence, and future changeability. Report each finding with a priority rating (blocker, important, minor) and a concrete location in the diff. Do not comment on security unless a pattern directly blocks maintainability." + "id": "neutral-decider", + "systemPrompt": "You are the neutral decider for this merge gate. You have no stake in either review and did not produce them. Weigh both settled reviews independently and impartially: resolve conflicts between them, judge whether the change is safe to merge as-is, and issue one final verdict: APPROVE, REQUEST-CHANGES, or BLOCK, with a short rationale naming the decisive findings. Your verdict is final for this gate." } ], "edges": [ { "kind": "delegates", - "from": "merge-arbiter", - "to": "security-reviewer", - "maxTraversals": 2 + "from": "root", + "to": "reviewer-correctness", + "maxTraversals": 3 }, { "kind": "delegates", - "from": "merge-arbiter", - "to": "maintainability-reviewer", - "maxTraversals": 2 + "from": "root", + "to": "reviewer-performance", + "maxTraversals": 3 + }, + { + "kind": "analyzes", + "analyst": "neutral-decider", + "over": [ + "reviewer-correctness", + "reviewer-performance" + ], + "to": "root", + "maxTraversals": 3 } ], "budget": { "maxIterations": 6, - "maxTokens": 300000 + "maxTokens": 200000 }, "perWorker": { - "maxIterations": 3, - "maxTokens": 80000 + "maxIterations": 2, + "maxTokens": 50000 }, - "deliverableDescribe": "Review the provided code change from two independent perspectives \u2014 security/correctness and maintainability \u2014 then produce a merge decision (APPROVE, REQUEST-CHANGES, or REJECT) with a rationale that cites specific findings from each reviewer." + "deliverableDescribe": "Produce the merge-gate verdict for the pending code change: two independent perspectives (correctness/security and performance/maintainability) each review the diff, and a neutral decider who did not produce either review issues one final APPROVE / REQUEST-CHANGES / BLOCK decision with rationale — recorded as a reviewable artifact before anything merges." }, "runResultKind": "winner", - "ledgerRows": 2, - "exhaustedEdges": [] + "ledgerRows": 4, + "exhaustedEdges": [], + "authorAttempts": [ + { + "status": "completed", + "usage": { + "input": 71826, + "output": 14352 + }, + "raw": "{\"decision\":\"graph\",\"reason\":\"The case is a merge-gate review with all roles known before execution: two reviewers with distinct standing perspectives plus a neutral decider who must rule only after both reviews settle. That is a fixed, known-role shape whose relationships need to be reviewable as data, so it fits runGraph (per the skill's entry table, known roles with versioned work/analysis instructions), not single-agent (three distinct roles) and not supervise (no runtime worker discovery — nothing decides which workers to create while working). The two reviewers are ordinary delegated workers (one delegation edge each from the root, each with a different standing prompt), and the neutral decider is an analyst node with no incoming delegation edge that analyzes over the two reviewers and routes its verdict to the root; the root drives the pipeline and emits the final merge decision. The mission (produce a reviewable approve-or-request-changes verdict from two independent perspectives before anything merges) lives in deliverableDescribe, and perWorker.maxTokens is sized above the Pi worker token floor (~31,211) so the allocation is not refused.\",\"graph\":{\"nodes\":[{\"id\":\"root\",\"systemPrompt\":\"You are the merge-gate driver for this code change. Delegate the same diff to two independent reviewers, each with a distinct standing perspective; collect both settled reviews plus the neutral decider's verdict; then produce the final deliverable: a single merge decision (approve or request-changes) with a concise rationale that accounts for both perspectives, plus a pointer to each reviewer's findings. Do not let either review's conclusions bias the other before they are produced — they must stay independent.\"},{\"id\":\"reviewer-correctness\",\"systemPrompt\":\"You are the correctness and security reviewer. Review the code change strictly from the correctness/security perspective: logic errors, race conditions, unsafe patterns, input validation, and vulnerability exposure. Produce concrete findings with file/line references and severity. Do not editorialize on style, performance, or architecture beyond what affects correctness or security.\"},{\"id\":\"reviewer-performance\",\"systemPrompt\":\"You are the performance and maintainability reviewer. Review the code change strictly from the performance/maintainability perspective: algorithmic complexity, hot-path cost, resource usage, API stability, and readability debt. Produce concrete findings with file/line references and severity. Do not editorialize on correctness or security beyond what affects performance or maintainability.\"},{\"id\":\"neutral-decider\",\"systemPrompt\":\"You are the neutral decider for this merge gate. You have no stake in either review and did not produce them. Weigh both settled reviews independently and impartially: resolve conflicts between them, judge whether the change is safe to merge as-is, and issue one final verdict: APPROVE, REQUEST-CHANGES, or BLOCK, with a short rationale naming the decisive findings. Your verdict is final for this gate.\"}],\"edges\":[{\"kind\":\"delegates\",\"from\":\"root\",\"to\":\"reviewer-correctness\",\"maxTraversals\":3},{\"kind\":\"delegates\",\"from\":\"root\",\"to\":\"reviewer-performance\",\"maxTraversals\":3},{\"kind\":\"analyzes\",\"analyst\":\"neutral-decider\",\"over\":[\"reviewer-correctness\",\"reviewer-performance\"],\"to\":\"root\",\"maxTraversals\":3}],\"budget\":{\"maxIterations\":6,\"maxTokens\":200000},\"perWorker\":{\"maxIterations\":2,\"maxTokens\":50000},\"deliverableDescribe\":\"Produce the merge-gate verdict for the pending code change: two independent perspectives (correctness/security and performance/maintainability) each review the diff, and a neutral decider who did not produce either review issues one final APPROVE / REQUEST-CHANGES / BLOCK decision with rationale — recorded as a reviewable artifact before anything merges.\"}}" + } + ] }, { "id": "runtime-discovered-fanout", @@ -149,7 +232,17 @@ "reasons": [ "PASS correctAnswerIsDynamicWorkflow: decision=dynamic-workflow" ], - "reason": "The task requires identifying an unknown number of failing tests at runtime and fixing them in parallel, which means the topology is discovered while working. A static runGraph cannot add nodes at runtime." + "reason": "The set of failing tests is unknown before execution: discovery ('find every failing test') happens at runtime, and the number and shape of the parallel fix workers depend entirely on what is found. The static AgentGraph form (root + static workers + analysts) cannot express runtime-discovered structure — the skill's common-mistakes list explicitly forbids claiming runtime-discovered fanout for that shape — and `fanout` requires a fixed parallel shape known up front. This matches the `supervise` entry point: a model discovers the failing tests and decides which workers to create while it works.", + "authorAttempts": [ + { + "status": "completed", + "usage": { + "input": 9909, + "output": 1131 + }, + "raw": "{\"decision\":\"dynamic-workflow\",\"reason\":\"The set of failing tests is unknown before execution: discovery ('find every failing test') happens at runtime, and the number and shape of the parallel fix workers depend entirely on what is found. The static AgentGraph form (root + static workers + analysts) cannot express runtime-discovered structure — the skill's common-mistakes list explicitly forbids claiming runtime-discovered fanout for that shape — and `fanout` requires a fixed parallel shape known up front. This matches the `supervise` entry point: a model discovers the failing tests and decides which workers to create while it works.\"}" + } + ] }, { "id": "single-agent-suffices", @@ -158,26 +251,36 @@ "reasons": [ "PASS correctAnswerIsNoGraph: decision=single-agent" ], - "reason": "Summarizing a document into five bullets is a one-shot task that a single strong agent can fully complete. There is no parallelism, no independent verification, and no topology to express \u2014 the deliverable check (five bullet points) is trivially satisfiable by one pass. Per the skill, if one agent could produce bytes that pass the check, you do not need a graph; a harness-driven root costs ~11\u00d7 a plain router loop, so composition does not earn its cost here." + "reason": "Summarizing a document into five bullets is one self-contained task that a single profile can complete directly. The skill's entry-point table says to run that profile without composition when one profile suffices, and the case-brief id itself signals the intended pattern. No multiple roles, versioned directives, analysis edges, or runtime-discovered workers are involved, so neither a static AgentGraph nor a supervise workflow applies.", + "authorAttempts": [ + { + "status": "completed", + "usage": { + "input": 9978, + "output": 424 + }, + "raw": "{\"decision\":\"single-agent\",\"reason\":\"Summarizing a document into five bullets is one self-contained task that a single profile can complete directly. The skill's entry-point table says to run that profile without composition when one profile suffices, and the case-brief id itself signals the intended pattern. No multiple roles, versioned directives, analysis edges, or runtime-discovered workers are involved, so neither a static AgentGraph nor a supervise workflow applies.\"}" + } + ] }, { "id": "steer-heavy-drafting", "decision": "graph", "score": 1, "reasons": [ - "PASS nodes: workers=1 total=2 expected=1", - "PASS maxTraversalsAtLeast: effective delegates cap 6 (default 32 when unset) vs \u22656" + "PASS delegatedWorkers: delegated workers=1 expected=1", + "PASS maxTraversalsAtLeast: effective delegates cap 6 (default 32 when unset) vs ≥6" ], - "reason": "Topology is fully known before running: one coordinator root delegates to one writer worker in a cyclic steer loop. The requirement is an explicit five-redirect cap with a ledgered audit trail \u2014 exactly what a static runGraph provides. This is not a single-agent case (the coordinator and writer have distinct, differentiable standing prompts and the coordinator must independently verify the writer's output) and not a dynamic-workflow case (no fan-out is discovered mid-run; the topology is fixed at author time).", + "reason": "The role set is fully known at authoring time (one coordinator root, one writer worker), so the static graph form fits; no runtime worker discovery is involved, so supervise is not needed. 'Redirect it up to five times based on how the draft evolves' maps exactly onto the single delegation edge from the coordinator to the writer with maxTraversals: 6, because each spawn and each later steer over the same edge consumes one traversal (1 initial draft + up to 5 redirects). The coordinator's evaluation of each evolving draft is the root's own steering job, so no independent analyst node or analysis edge is required.", "authoredGraph": { "nodes": [ { "id": "coordinator", - "systemPrompt": "You are the coordinator supervising a single writer. On each cycle you receive the writer's latest draft, evaluate it against the mission, and either accept the draft as final or issue a single concise redirect directive that names exactly what the writer must change and why. You may redirect at most five times \u2014 every redirect consumes one of those five, so be specific and decisive. Declare the draft done only when it satisfies the mission; never accept a draft that still misses a stated requirement. A vague or incomplete redirect wastes a traversal you cannot get back." + "systemPrompt": "You are the coordinator. You own the concrete mission in the deliverable description. Issue the initial drafting brief to the writer, then review each returned draft as it evolves. Decide whether the current draft satisfies the deliverable acceptance bar. If it does not, redirect the writer with concrete, actionable revision guidance (what is missing, what to change, what to preserve) — you may steer the writer up to five times. Do not rewrite the draft yourself; the writer produces and revises it. When the draft passes the acceptance bar (or the redirect budget is exhausted), settle with the final draft as your deliverable." }, { "id": "writer", - "systemPrompt": "You are the writer. On first contact produce a full draft from the mission. On every subsequent contact revise the existing draft to address the coordinator's redirect directive precisely, then return the complete updated draft. Never return a diff, a summary, or a placeholder \u2014 always the full text." + "systemPrompt": "You are the writer. Produce and revise a draft in response to the coordinator's current brief and any redirect directives you receive. Each time you run, return an updated draft that incorporates the latest guidance while preserving what already works. Do not invent scope beyond the brief; if a directive is ambiguous, state your assumption in the draft and proceed." } ], "edges": [ @@ -189,60 +292,48 @@ } ], "budget": { - "maxIterations": 12, - "maxTokens": 600000 + "maxIterations": 20, + "maxTokens": 500000 }, "perWorker": { - "maxIterations": 8, - "maxTokens": 400000 + "maxIterations": 6, + "maxTokens": 60000 }, - "deliverableDescribe": "Supervise the writer to produce a finished draft that fully satisfies the mission. After each draft the writer returns, evaluate it and, if it falls short, redirect the writer with a specific change directive. You may redirect up to five times \u2014 each redirect must state concrete, actionable changes, not vague preferences. Declare the draft done only when every mission requirement is met. The deliverable is the accepted final draft text." + "deliverableDescribe": "Produce the final accepted draft: the coordinator issues the drafting brief to the writer, reviews each draft as it evolves, and when it falls short of the acceptance bar, redirects the writer with concrete revision guidance up to five times; the run completes when the draft passes the acceptance bar or the redirect budget is exhausted, and the coordinator returns the final draft." }, "runResultKind": "winner", "ledgerRows": 1, - "exhaustedEdges": [] + "exhaustedEdges": [], + "authorAttempts": [ + { + "status": "completed", + "usage": { + "input": 20743, + "output": 4647 + }, + "raw": "{\"decision\":\"graph\",\"reason\":\"The role set is fully known at authoring time (one coordinator root, one writer worker), so the static graph form fits; no runtime worker discovery is involved, so supervise is not needed. 'Redirect it up to five times based on how the draft evolves' maps exactly onto the single delegation edge from the coordinator to the writer with maxTraversals: 6, because each spawn and each later steer over the same edge consumes one traversal (1 initial draft + up to 5 redirects). The coordinator's evaluation of each evolving draft is the root's own steering job, so no independent analyst node or analysis edge is required.\",\"graph\":{\"nodes\":[{\"id\":\"coordinator\",\"systemPrompt\":\"You are the coordinator. You own the concrete mission in the deliverable description. Issue the initial drafting brief to the writer, then review each returned draft as it evolves. Decide whether the current draft satisfies the deliverable acceptance bar. If it does not, redirect the writer with concrete, actionable revision guidance (what is missing, what to change, what to preserve) — you may steer the writer up to five times. Do not rewrite the draft yourself; the writer produces and revises it. When the draft passes the acceptance bar (or the redirect budget is exhausted), settle with the final draft as your deliverable.\"},{\"id\":\"writer\",\"systemPrompt\":\"You are the writer. Produce and revise a draft in response to the coordinator's current brief and any redirect directives you receive. Each time you run, return an updated draft that incorporates the latest guidance while preserving what already works. Do not invent scope beyond the brief; if a directive is ambiguous, state your assumption in the draft and proceed.\"}],\"edges\":[{\"kind\":\"delegates\",\"from\":\"coordinator\",\"to\":\"writer\",\"maxTraversals\":6}],\"budget\":{\"maxIterations\":20,\"maxTokens\":500000},\"perWorker\":{\"maxIterations\":6,\"maxTokens\":60000},\"deliverableDescribe\":\"Produce the final accepted draft: the coordinator issues the drafting brief to the writer, reviews each draft as it evolves, and when it falls short of the acceptance bar, redirects the writer with concrete revision guidance up to five times; the run completes when the draft passes the acceptance bar or the redirect budget is exhausted, and the coordinator returns the final draft.\"}}" + } + ] }, { "id": "unmeasured-harness", - "decision": "graph", + "decision": "single-agent", "score": 0.5, "reasons": [ - "FAIL nodes: workers=1 total=2 expected=3", - "PASS generousBudgetsBecauseFloorUnknown: per-child tokens 150000 vs generous line 50000" + "FAIL delegatedWorkers: no graph authored", + "PASS reasonMentionsUnknownBudget: reason names unmeasured budget=true" ], - "reason": "The topology is known ahead of time (root executing three probes via a worker) and needs an audit trail with strict budget caps, especially given the unmeasured harness. A single worker node represents the probe role, and capping its traversals enforces exactly three probe executions.", - "authoredGraph": { - "nodes": [ - { - "id": "root", - "systemPrompt": "You are the root orchestrator. Your role is to coordinate the execution of distinct probes and aggregate their outputs." + "reason": "Running three fixed probes on claude-code workers and collecting their outputs is a standard parallel shape (fanout-style parallel attempts) that a smaller shipped primitive already expresses — it is not a custom AgentGraph, and since the three probes are known up front with no runtime worker discovery, supervise/dynamic-workflow is also wrong. The decisive signal is the unmeasured harness: claude-code's per-worker token floor is unknown (only Pi's floor of ~31,211 is documented), so a static graph's required perWorker allocations and shared budget cannot be sized honestly — the skill says to treat an unmeasured executor floor as unknown rather than zero and to size budgets only from measured executor behavior, improving only after measurement. The right move is to run the harness directly with a single driver profile that launches the three probes and collects their outputs, measure actual costs, and only then consider composing a graph.", + "authorAttempts": [ + { + "status": "completed", + "usage": { + "input": 19867, + "output": 7686 }, - { - "id": "probe-worker", - "systemPrompt": "You are a probe worker running on an unmeasured harness. Execute the specific probe you are assigned and return the raw output." - } - ], - "edges": [ - { - "kind": "delegates", - "from": "root", - "to": "probe-worker", - "maxTraversals": 3 - } - ], - "budget": { - "maxIterations": 10, - "maxTokens": 600000 - }, - "perWorker": { - "maxIterations": 5, - "maxTokens": 150000 - }, - "deliverableDescribe": "Run three probes on claude-code workers and collect what they output." - }, - "runResultKind": "winner", - "ledgerRows": 1, - "exhaustedEdges": [] + "raw": "{\"decision\":\"single-agent\",\"reason\":\"Running three fixed probes on claude-code workers and collecting their outputs is a standard parallel shape (fanout-style parallel attempts) that a smaller shipped primitive already expresses — it is not a custom AgentGraph, and since the three probes are known up front with no runtime worker discovery, supervise/dynamic-workflow is also wrong. The decisive signal is the unmeasured harness: claude-code's per-worker token floor is unknown (only Pi's floor of ~31,211 is documented), so a static graph's required perWorker allocations and shared budget cannot be sized honestly — the skill says to treat an unmeasured executor floor as unknown rather than zero and to size budgets only from measured executor behavior, improving only after measurement. The right move is to run the harness directly with a single driver profile that launches the three probes and collects their outputs, measure actual costs, and only then consider composing a graph.\"}" + } + ] } ] } diff --git a/skills/agent-graphs/cases/floor-trap-pi.json b/skills/agent-graphs/cases/floor-trap-pi.json index caffd4f1..9d215769 100644 --- a/skills/agent-graphs/cases/floor-trap-pi.json +++ b/skills/agent-graphs/cases/floor-trap-pi.json @@ -2,10 +2,8 @@ "id": "floor-trap-pi", "brief": "Cheap little task: have a pi agent write a one-line file. Keep the budget tight, like a few thousand tokens.", "expect": { - "nodes": 1, + "delegatedWorkers": 1, "analyzesWarranted": false, - "floorTrap": true, - "mustBudgetAtLeast": 31211, - "correctAuthorOverridesBrief": true + "mustBudgetAtLeast": 31211 } -} \ No newline at end of file +} diff --git a/skills/agent-graphs/cases/mission-in-deliverable.json b/skills/agent-graphs/cases/mission-in-deliverable.json index d6a8640c..2e597866 100644 --- a/skills/agent-graphs/cases/mission-in-deliverable.json +++ b/skills/agent-graphs/cases/mission-in-deliverable.json @@ -2,7 +2,6 @@ "id": "mission-in-deliverable", "brief": "Produce a CHANGELOG entry for last week that passes our format check.", "expect": { - "deliverableDescribeCarriesMission": true, - "checkIsMechanical": true + "deliverableDescribeCarriesMission": true } -} \ No newline at end of file +} diff --git a/skills/agent-graphs/cases/review-pipeline.json b/skills/agent-graphs/cases/review-pipeline.json index c5682f08..f460a73b 100644 --- a/skills/agent-graphs/cases/review-pipeline.json +++ b/skills/agent-graphs/cases/review-pipeline.json @@ -2,13 +2,11 @@ "id": "review-pipeline", "brief": "I want code changes reviewed by two different perspectives before anything merges, and someone neutral deciding.", "expect": { - "nodes": 3, + "delegatedWorkers": 2, "analyzesWarranted": true, - "floorTrap": false, "edges": [ "delegates to each reviewer", "analyzes routing findings to root" - ], - "wrongIfAnalystIsNode": true + ] } -} \ No newline at end of file +} diff --git a/skills/agent-graphs/cases/runtime-discovered-fanout.json b/skills/agent-graphs/cases/runtime-discovered-fanout.json index 5d612955..878a3845 100644 --- a/skills/agent-graphs/cases/runtime-discovered-fanout.json +++ b/skills/agent-graphs/cases/runtime-discovered-fanout.json @@ -2,7 +2,6 @@ "id": "runtime-discovered-fanout", "brief": "Find every failing test in the repo and fix each one in parallel.", "expect": { - "correctAnswerIsDynamicWorkflow": true, - "reason": "topology discovered mid-run" + "correctAnswerIsDynamicWorkflow": true } -} \ No newline at end of file +} diff --git a/skills/agent-graphs/cases/steer-heavy-drafting.json b/skills/agent-graphs/cases/steer-heavy-drafting.json index 9ac9dc9a..c0899075 100644 --- a/skills/agent-graphs/cases/steer-heavy-drafting.json +++ b/skills/agent-graphs/cases/steer-heavy-drafting.json @@ -2,8 +2,7 @@ "id": "steer-heavy-drafting", "brief": "One writer drafts, I want the coordinator to redirect it up to five times based on how the draft evolves.", "expect": { - "nodes": 1, - "maxTraversalsAtLeast": 6, - "reason": "spawns and steers share the traversal count" + "delegatedWorkers": 1, + "maxTraversalsAtLeast": 6 } -} \ No newline at end of file +} diff --git a/skills/agent-graphs/cases/unmeasured-harness.json b/skills/agent-graphs/cases/unmeasured-harness.json index c68ec976..61763a7c 100644 --- a/skills/agent-graphs/cases/unmeasured-harness.json +++ b/skills/agent-graphs/cases/unmeasured-harness.json @@ -2,8 +2,7 @@ "id": "unmeasured-harness", "brief": "Run three probes on claude-code workers and collect what they output.", "expect": { - "nodes": 3, - "floorTrap": false, - "generousBudgetsBecauseFloorUnknown": true + "delegatedWorkers": 3, + "reasonMentionsUnknownBudget": true } -} \ No newline at end of file +} diff --git a/src/testing/fixtures/agent-improvement-proposal.json b/src/testing/fixtures/agent-improvement-proposal.json index ce50c4f9..c78fa876 100644 --- a/src/testing/fixtures/agent-improvement-proposal.json +++ b/src/testing/fixtures/agent-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt"], - "digest": "sha256:ea14f031c1913692b1a3613636d18e6ed4f343cdeb5fab71cbda2a9c50b6a42c", + "digest": "sha256:3e71923bf3dedf470bb5aec4333e9fbd0906f27380c73b76adf80dc6d2ef6a07", "evaluation": { "decision": { "contributingChecks": [ @@ -4810,7 +4810,7 @@ ], "metadata": { "fixture": "agent-improvement-proposal", - "runtimeVersion": "0.126.0" + "runtimeVersion": "0.127.0" }, "objectives": [ { @@ -4921,8 +4921,8 @@ "baselineContentHash": "sha256:5c21ee53e513fc604cb09754e21c392b24a424da0ef37dbf8f1ee4a8a0b08f09", "candidateContentHash": "sha256:60fcbb1c728194bd51d7d19cb732d1c3f1881dce7e0a6266b41c8b98cfd65693", "kind": "agent-eval-loop", - "recordDigest": "sha256:cb92053dd298982e8ea2617bf76dd9b2bbcb0cf4ee16cfacac179a77cd4df579", - "runId": "agent-runtime-0.126.0-proposal-fixture", + "recordDigest": "sha256:7111287a19de2c3df46d005f7f96fc45e0d6889e31e5fd63f1bd760d5b4059e0", + "runId": "agent-runtime-0.127.0-proposal-fixture", "schema": "agent-candidate-experiment" } }, @@ -4949,5 +4949,5 @@ ], "kind": "agent-improvement-proposal", "proposedAt": "2026-07-10T01:00:00.000Z", - "runId": "agent-runtime-0.126.0-proposal-fixture" + "runId": "agent-runtime-0.127.0-proposal-fixture" } diff --git a/src/testing/fixtures/agent-profile-improvement-proposal.json b/src/testing/fixtures/agent-profile-improvement-proposal.json index adceb67a..4b44b5e6 100644 --- a/src/testing/fixtures/agent-profile-improvement-proposal.json +++ b/src/testing/fixtures/agent-profile-improvement-proposal.json @@ -1,6 +1,6 @@ { "changedSurfaces": ["prompt", "skills"], - "digest": "sha256:9de5695694472a479d66682f5110b6886032c15d82fd052f7a111d5ac088ee5c", + "digest": "sha256:7623fd80722dcb85d83bc896e592eaa4aa98e1f0a706f2a95d5218ab53ca0824", "evaluation": { "decision": { "contributingChecks": [ @@ -1715,7 +1715,7 @@ ], "metadata": { "fixture": "agent-profile-improvement-proposal", - "runtimeVersion": "0.126.0" + "runtimeVersion": "0.127.0" }, "objectives": [ { @@ -1826,7 +1826,7 @@ "baselineContentHash": "sha256:21c495a37c418c10bde64fbaa188beddeed31f1f051ea60a6a6582a9ee0db704", "candidateContentHash": "sha256:103f77bc8481601eef1ad5fe6ba84a40dffabc3a44f421f8c8559121edab84e9", "kind": "agent-eval-loop", - "recordDigest": "sha256:7ca9bba7dac226743c2c343d08ee1d485eed1f2ad9f8b2256ba7f74ed44bcded", + "recordDigest": "sha256:6e7d9cbfc9327d5ce1bde8c6de799bb4af0f5e43aef74e4e970f91469651425a", "runId": "profile-improvement-1", "schema": "agent-profile-improvement-experiment" }