chore(release): prepare Runtime 0.127.0 and Bench 0.8.0

.gitignore

@@ -9,6 +9,7 @@ bench/data/
 bench/experiments/
 bench/runs/
 bench/ab-runs/
+.gen2-runs/
 **/__pycache__/
 .claude/
 bench/scripts/__pycache__/

CHANGELOG.md

@@ -2,6 +2,26 @@
 
 ## Unreleased
 
+## 0.127.0
+
+### One canonical loop API
+
+BREAKING. The superseded `runLoop`, `RunLoopOptions`, and `StdioToolDescriptor` exports are removed.
+Callers use `runAgentRounds`, `RunAgentRoundsOptions`, and `McpToolDescriptor`, which were already the canonical generalized names for the same behavior.
+The benchmark arm formerly named `loop` is now named `multishot`, which describes the compared method without referring to the deleted alias.
+
+### Agent graphs fail before spend and reuse the existing Eval path
+
+- Graph validation now refuses an `analyzes` edge over the root because that edge can never fire.
+- The `agent-graphs` skill turns a loose task into either a single-agent run, a fixed graph, or a dynamic `supervise()` workflow according to what the task actually requires.
+- The accompanying benchmark executable supplies Agent Eval's caller-owned authoring and deterministic scoring functions and records a baseline without adding a second optimization system.
+- The author is one canonical, overridable `AgentProfile`, executed through Runtime and Pi; it defaults to Tangle Router's DeepSeek V4 Flash and carries the skill as an inline profile resource.
+- The first complete skill-improvement generation ran through Agent Eval's existing `runImprovementLoop`: 5 development cases and 3 final-test cases at 3 repetitions each.
+  The revised skill improved the development mean from 0.507 to 0.960 and the final-test mean from 0.444 to 0.611, so the declared rule promoted it.
+  One of 33 requested cells was lost to an HTTP 503; the checked-in record includes that asymmetry and the supplementary final-test measurement.
+- Future baseline and improvement runs author through canonical, overridable `AgentProfile` values executed by Runtime and Pi.
+  The default is Tangle Router's DeepSeek V4 Flash; the historical promoted generation used GLM-5.2.
+
 ### Python bridge install hints match the required Eval substrate
 
 The documented `agent-eval-rpc` install commands — the `OfficialOptimizerUnavailableError` hint, the README's `officialGepa`/`officialSkillOpt` sections, and the bench GEPA seat hint — now pin `0.143.0`, the Python client published in lockstep with the `@tangle-network/agent-eval` range this package requires.

bench/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## 0.8.0
+
+- Add the executable `agent-graphs` baseline run with eight authoring cases and deterministic scoring over the graph edge record.
+- Add offline execution inputs so the authoring and scoring path can be tested without model spend.
+- Define the author as an overridable `AgentProfile` and run it through Runtime and Pi, defaulting to Tangle Router's DeepSeek V4 Flash; remove the old direct HTTP call, fixed token cap, fixed timeout, and deleted `codemode` file fallback.
+- Record one result per case and keep that small-sample limit explicit rather than treating it as a stable comparison.
+- Consume Runtime 0.127.0 and its canonical `runAgentRounds` and `McpToolDescriptor` names.
+
 ## 0.7.1
 
 - Consume Runtime 0.126.0 with Eval 0.143.0 and Knowledge 7.0.8, so campaign cost remains observed, estimated, or explicitly uncaptured across the complete benchmark dependency tree.

bench/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@tangle-network/agent-bench",
-  "version": "0.7.1",
+  "version": "0.8.0",
   "type": "module",
   "description": "Benchmark adapters and execution for agent-runtime across coding, tool-use, RAG, memory, browser, and terminal tasks.",
   "repository": {

bench/scripts/run-package-tests.mjs

@@ -7,6 +7,7 @@ import { promisify } from 'node:util'
 const execFileAsync = promisify(execFile)
 const benchDir = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..')
 const sourceDir = path.join(benchDir, 'src')
+const fixtureEnv = { ...process.env, GIT_ALLOW_TEST_IDENTITY: '1' }
 
 async function collectTests(dir) {
   const files = []
@@ -61,17 +62,25 @@ if (nodeTests.length > 0) {
     process.execPath,
     ['--test', '--import', 'tsx', ...nodeTests.map((file) => path.relative(benchDir, file))],
     {
-      ...process.env,
+      ...fixtureEnv,
       TSX_TSCONFIG_PATH: 'tsconfig.public.json',
     },
   )
 }
 
 if (vitestTests.length > 0) {
-  await run('npx', ['vitest', 'run', ...vitestTests.map((file) => path.relative(benchDir, file))])
+  await run(
+    'npx',
+    ['vitest', 'run', ...vitestTests.map((file) => path.relative(benchDir, file))],
+    fixtureEnv,
+  )
 }
 
-await run(python, ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py'])
+await run(
+  python,
+  ['-m', 'unittest', 'discover', '-s', 'pier_agents', '-p', '*_test.py'],
+  fixtureEnv,
+)
 
 console.log(
   `package tests passed: ${tests.length}/${tests.length} TypeScript files (${nodeTests.length} node:test + ${vitestTests.length} vitest) + Pier bridge`,

bench/src/agent-graphs-gen2.mts

@@ -6,8 +6,8 @@
  * measurement (v2 on TRAIN, reps=3), the enforced-disjoint holdout scoring of both
  * arms, winner selection, and the gate invocation. What this file owns: the same two
  * closures the baseline run owned (author dispatch + deterministic scorer, imported
- * from agent-graphs-improve.mts), the reviser proposer (glm-5.2, temp 0.7, TRAIN
- * failures only), and the protocol gate:
+ * from agent-graphs-improve.mts), the AgentProfile-driven reviser (TRAIN failures
+ * only), and the protocol gate:
  *
  *     ship  iff  v2 holdout mean > v1 holdout mean
  *           and  v2 train mean >= v1 train mean - 0.05
@@ -21,8 +21,9 @@
  * run asserts the holdout ids and briefs are absent from the final prompt string.
  *
  * Run:   pnpm tsx src/agent-graphs-gen2.mts             (from bench/)
- * Smoke: GEN2_SMOKE=1 pnpm tsx src/agent-graphs-gen2.mts   — stubs both LLM calls,
- *        exercises the full loop wiring + gate + report at zero cost.
+ * Smoke: GEN2_SMOKE=1 pnpm tsx src/agent-graphs-gen2.mts   — stubs both model calls,
+ *        exercises the full loop wiring + gate + report at zero cost and writes only
+ *        under the ignored .gen2-runs directory.
  *
  * Writes skills/agent-graphs/gen2.json; on ship, replaces SKILL.md with v2.
  */
@@ -47,6 +48,8 @@ import {
 import {
   type AuthoredArtifact,
   type CaseSpec,
+  authorProfileLabel,
+  buildAgentGraphsAuthorProfile,
   callAuthor,
   dispatchWithSurface,
   judgeArtifact,
@@ -56,9 +59,63 @@ import {
 const HERE = dirname(fileURLToPath(import.meta.url))
 const REPO = join(HERE, '..', '..')
 const SKILL_PATH = join(REPO, 'skills', 'agent-graphs', 'SKILL.md')
-const OUT_PATH = join(REPO, 'skills', 'agent-graphs', 'gen2.json')
 const RUNS_ROOT = join(REPO, '.gen2-runs')
 const SMOKE = process.env.GEN2_SMOKE === '1'
+const OUT_PATH = SMOKE
+  ? join(RUNS_ROOT, 'smoke-result.json')
+  : join(REPO, 'skills', 'agent-graphs', 'gen2.json')
+
+function positiveInteger(name: string, raw: string | undefined, fallback: number): number {
+  const value = raw === undefined ? fallback : Number(raw)
+  if (!Number.isSafeInteger(value) || value <= 0) {
+    throw new Error(`${name} must be a positive integer`)
+  }
+  return value
+}
+
+function optionalPositiveInteger(name: string, raw: string | undefined): number | undefined {
+  return raw === undefined ? undefined : positiveInteger(name, raw, 1)
+}
+
+function executionEnv(role: 'AUTHOR' | 'PROPOSER'): NodeJS.ProcessEnv {
+  const specific = (name: string): string | undefined =>
+    process.env[`AGENT_GRAPHS_GEN2_${role}_${name}`]
+  return {
+    ...process.env,
+    AGENT_GRAPHS_AUTHOR_PROFILE_NAME:
+      specific('PROFILE_NAME') ??
+      process.env.AGENT_GRAPHS_AUTHOR_PROFILE_NAME ??
+      `agent-graphs-gen2-${role.toLowerCase()}`,
+    AGENT_GRAPHS_AUTHOR_HARNESS:
+      specific('HARNESS') ?? process.env.AGENT_GRAPHS_AUTHOR_HARNESS ?? 'pi',
+    AGENT_GRAPHS_AUTHOR_PROVIDER:
+      specific('PROVIDER') ?? process.env.AGENT_GRAPHS_AUTHOR_PROVIDER ?? 'tangle-router',
+    AGENT_GRAPHS_AUTHOR_MODEL:
+      specific('MODEL') ?? process.env.AGENT_GRAPHS_AUTHOR_MODEL ?? 'glm-5.2',
+    AGENT_GRAPHS_AUTHOR_REASONING_EFFORT:
+      specific('REASONING_EFFORT') ??
+      process.env.AGENT_GRAPHS_AUTHOR_REASONING_EFFORT ??
+      'ultracode',
+    AGENT_GRAPHS_AUTHOR_ATTEMPTS:
+      specific('ATTEMPTS') ?? process.env.AGENT_GRAPHS_AUTHOR_ATTEMPTS,
+    AGENT_GRAPHS_AUTHOR_TIMEOUT_MS:
+      specific('TIMEOUT_MS') ?? process.env.AGENT_GRAPHS_AUTHOR_TIMEOUT_MS,
+    ...(role === 'PROPOSER'
+      ? {
+          AGENT_GRAPHS_AUTHOR_SYSTEM_PROMPT:
+            specific('SYSTEM_PROMPT') ??
+            'You revise an agent skill from measured development-case failures. Follow the requested output format exactly and do not use final-test cases.',
+        }
+      : {}),
+  }
+}
+
+const AUTHOR_ENV = executionEnv('AUTHOR')
+const PROPOSER_ENV = executionEnv('PROPOSER')
+const DISPATCH_TIMEOUT_MS = optionalPositiveInteger(
+  'AGENT_GRAPHS_GEN2_DISPATCH_TIMEOUT_MS',
+  process.env.AGENT_GRAPHS_GEN2_DISPATCH_TIMEOUT_MS,
+)
 
 const K = 3
 const SEED = 42
@@ -96,6 +153,15 @@ interface JudgedRecord {
 
 const judged: JudgedRecord[] = []
 
+interface RevisionAttemptEvidence {
+  status: string
+  usage?: { input: number; output: number; costUsd?: number; model?: string }
+  validationProblems?: string[]
+  error?: string
+}
+
+const revisionAttempts: RevisionAttemptEvidence[] = []
+
 function makeJudge(): JudgeConfig<CellArtifact, GraphScenario> {
   return {
     name: 'deterministic-expect',
@@ -122,7 +188,12 @@ function makeJudge(): JudgeConfig<CellArtifact, GraphScenario> {
 function smokeArtifact(scenario: GraphScenario): AuthoredArtifact {
   // Deterministic offline stand-in: always "single-agent" — wrong on graph cases,
   // right on the no-graph case; enough to exercise scoring + gate arithmetic.
-  return { decision: 'single-agent', reason: `smoke stub for ${scenario.id}`, raw: '{}' }
+  return {
+    decision: 'single-agent',
+    reason: `smoke stub for ${scenario.id}`,
+    raw: '{}',
+    authorAttempts: [],
+  }
 }
 
 async function dispatchCell(
@@ -131,7 +202,9 @@ async function dispatchCell(
   ctx: DispatchContext,
 ): Promise<CellArtifact> {
   if (typeof surface !== 'string') throw new Error('gen2 surfaces are strings')
-  const artifact = SMOKE ? smokeArtifact(scenario) : await dispatchWithSurface(surface, scenario)
+  const artifact = SMOKE
+    ? smokeArtifact(scenario)
+    : await dispatchWithSurface(surface, scenario, AUTHOR_ENV)
   return { ...artifact, repIndex: ctx.rep, surfaceSha: sha256(surface) }
 }
 
@@ -215,6 +288,12 @@ function validateSkillGate(text: string): string[] {
 }
 
 function makeProposer(v1Surface: string, trainCases: GraphScenario[]): SurfaceProposer {
+  const proposerProfile = buildAgentGraphsAuthorProfile(v1Surface, PROPOSER_ENV)
+  const attemptLimit = positiveInteger(
+    'AGENT_GRAPHS_GEN2_PROPOSER_ATTEMPTS',
+    process.env.AGENT_GRAPHS_GEN2_PROPOSER_ATTEMPTS,
+    2,
+  )
   return {
     kind: 'agent-graphs-skill-reviser',
     async propose(_ctx: ProposeContext): Promise<ProposedCandidate[]> {
@@ -245,23 +324,43 @@ function makeProposer(v1Surface: string, trainCases: GraphScenario[]): SurfacePr
       }
       let prompt = revisionPrompt
       let lastProblems: string[] = []
-      for (let attempt = 0; attempt < 2; attempt += 1) {
-        const reply = await callAuthor(prompt, 0.7, 12_000)
-        const skill = extractSkill(reply)
-        lastProblems = validateSkillGate(skill)
-        if (lastProblems.length === 0) {
-          return [
-            {
-              surface: skill,
-              label: 'gen2-revision',
-              rationale:
-                'glm-5.2 rewrite targeting under-graphing on cheap briefs, missing analyzes edges, and collapsed identical-role parallelism',
-            },
-          ]
+      for (let attempt = 0; attempt < attemptLimit; attempt += 1) {
+        let evidence: RevisionAttemptEvidence | undefined
+        try {
+          const turn = await callAuthor(proposerProfile, prompt, PROPOSER_ENV)
+          evidence = { status: turn.status, usage: turn.usage }
+          revisionAttempts.push(evidence)
+          if (turn.status !== 'completed') {
+            throw new Error(turn.error?.message ?? `proposer turn ended with status ${turn.status}`)
+          }
+          const skill = extractSkill(turn.finalText)
+          lastProblems = validateSkillGate(skill)
+          evidence.validationProblems = lastProblems
+          if (lastProblems.length === 0) {
+            return [
+              {
+                surface: skill,
+                label: 'gen2-revision',
+                rationale:
+                  'profile-authored rewrite targeting measured under-graphing, missing analysis, and collapsed parallel roles',
+              },
+            ]
+          }
+          prompt = `${revisionPrompt}\n\nYour previous attempt violated: ${lastProblems.join('; ')}. Fix these and reply again with the full file between the markers.`
+        } catch (error) {
+          const message = error instanceof Error ? error.message : String(error)
+          if (evidence === undefined) revisionAttempts.push({ status: 'failed', error: message })
+          else evidence.error = message
+          if (attempt + 1 < attemptLimit) {
+            prompt = `${revisionPrompt}\n\nYour previous attempt failed: ${message}. Reply again with the full file between the markers.`
+          }
         }
-        prompt = `${revisionPrompt}\n\nYour previous attempt violated: ${lastProblems.join('; ')}. Fix these and reply again with the full file between the markers.`
       }
-      throw new Error(`proposer surface failed the skills gate after retry: ${lastProblems.join('; ')}`)
+      const detail =
+        lastProblems.length > 0
+          ? lastProblems.join('; ')
+          : revisionAttempts.at(-1)?.error ?? 'unknown proposer failure'
+      throw new Error(`proposer surface failed after ${attemptLimit} attempts: ${detail}`)
     },
   }
 }
@@ -358,6 +457,10 @@ async function main(): Promise<void> {
   const inputs = loadInputs()
   const v1Surface = inputs.surface
   const v1Sha = sha256(v1Surface)
+  const authorProfile = buildAgentGraphsAuthorProfile(v1Surface, AUTHOR_ENV)
+  const proposerProfile = buildAgentGraphsAuthorProfile(v1Surface, PROPOSER_ENV)
+  const authorRef = authorProfileLabel(authorProfile)
+  const proposerRef = authorProfileLabel(proposerProfile)
   const byId = new Map(inputs.cases.map((c) => [c.id, c]))
   const missing = [...TRAIN_IDS, ...HOLDOUT_IDS].filter((id) => !byId.has(id))
   if (missing.length > 0) throw new Error(`cases missing from skills/agent-graphs/cases: ${missing.join(', ')}`)
@@ -380,14 +483,22 @@ async function main(): Promise<void> {
     holdoutScenarios,
     reps: K,
     seed: SEED,
-    maxConcurrency: 1,
-    candidateConcurrency: 1,
+    maxConcurrency: positiveInteger(
+      'AGENT_GRAPHS_GEN2_MAX_CONCURRENCY',
+      process.env.AGENT_GRAPHS_GEN2_MAX_CONCURRENCY,
+      1,
+    ),
+    candidateConcurrency: positiveInteger(
+      'AGENT_GRAPHS_GEN2_CANDIDATE_CONCURRENCY',
+      process.env.AGENT_GRAPHS_GEN2_CANDIDATE_CONCURRENCY,
+      1,
+    ),
     populationSize: 1,
     maxGenerations: 1,
     baselineSurface: v1Surface,
-    dispatchRef: SMOKE ? 'gen2-smoke-stub' : 'agent-graphs-author/glm-5.2/temp-0.2',
+    dispatchRef: SMOKE ? 'gen2-smoke-stub' : authorRef,
     dispatchWithSurface: dispatchCell,
-    dispatchTimeoutMs: 600_000,
+    ...(DISPATCH_TIMEOUT_MS === undefined ? {} : { dispatchTimeoutMs: DISPATCH_TIMEOUT_MS }),
     expectUsage: 'off',
     judges: [makeJudge()],
     proposer: makeProposer(v1Surface, trainScenarios),
@@ -428,12 +539,12 @@ async function main(): Promise<void> {
     v2HoldoutCampaign = await runEval<GraphScenario, CellArtifact>({
       scenarios: holdoutScenarios,
       dispatch: (scenario, ctx) => dispatchCell(v2Surface, scenario, ctx),
-      dispatchRef: SMOKE ? 'gen2-smoke-stub-v2' : 'agent-graphs-author/glm-5.2/temp-0.2/v2',
+      dispatchRef: SMOKE ? 'gen2-smoke-stub-v2' : `${authorRef}/candidate`,
       judges: [makeJudge()],
       reps: K,
       seed: SEED,
       maxConcurrency: 1,
-      dispatchTimeoutMs: 600_000,
+      ...(DISPATCH_TIMEOUT_MS === undefined ? {} : { dispatchTimeoutMs: DISPATCH_TIMEOUT_MS }),
       expectUsage: 'off',
       runDir: join(RUNS_ROOT, SMOKE ? 'smoke-v2-holdout' : 'v2-holdout'),
     })
@@ -470,10 +581,10 @@ async function main(): Promise<void> {
     generation: 2,
     date: new Date().toISOString(),
     smoke: SMOKE,
-    authorModel: 'glm-5.2',
-    authorTemperature: 0.2,
-    proposerModel: 'glm-5.2',
-    proposerTemperature: 0.7,
+    authorProfile: authorRef,
+    proposerProfile: proposerRef,
+    authorTemperature: null,
+    proposerTemperature: null,
     split: { train: TRAIN_IDS, holdout: HOLDOUT_IDS },
     k: K,
     seed: SEED,
@@ -492,6 +603,7 @@ async function main(): Promise<void> {
     gateVerdict,
     promoted,
     revisionPromptSha256,
+    revisionAttempts,
     v2Surface,
   }
   writeFileSync(OUT_PATH, `${JSON.stringify(out, null, 2)}\n`)