From 3a1406da95a162a419d3271c5c310571357eea22 Mon Sep 17 00:00:00 2001 From: Drew Stone Date: Mon, 3 Aug 2026 15:01:30 -0600 Subject: [PATCH] refactor(runtime): enforce canonical model execution boundary --- bench/scripts/appworld_driver.py | 169 ------ bench/scripts/trata-hedge/README.md | 6 +- bench/scripts/trata-hedge/run.sh | 8 +- bench/scripts/trata-hedge/solve.py | 83 --- bench/src/aec-gate.mts | 23 +- bench/src/agent-graphs-gen2.mts | 15 +- bench/src/agent-graphs-improve.mts | 151 +++-- bench/src/atom-humaneval.mts | 95 +-- bench/src/atom-mcp-e2e.mts | 38 +- bench/src/benchmarks/appworld.ts | 59 +- bench/src/benchmarks/cadbench.ts | 23 +- bench/src/benchmarks/finresearchbench.ts | 30 +- bench/src/benchmarks/finsearchcomp.ts | 34 +- bench/src/benchmarks/frames.ts | 30 +- bench/src/benchmarks/simpleqa.ts | 30 +- bench/src/benchmarks/trata-hedge.ts | 38 +- bench/src/clbench-context-gate.mts | 35 +- bench/src/david-attribution.mts | 34 +- bench/src/david-goliath.mts | 40 +- bench/src/gate.ts | 87 +-- bench/src/generate-eval/certify.ts | 18 +- bench/src/hev-eval.mts | 46 +- bench/src/hev-improve.mts | 45 +- bench/src/hev-structural.mts | 45 +- bench/src/humaneval-gate.mts | 20 +- bench/src/humaneval-object-ablation.mts | 56 +- bench/src/humaneval-repair-gate.mts | 54 +- bench/src/mbpp-structural.mts | 45 +- bench/src/mcp-mount-probe.mts | 34 +- bench/src/research-shot.ts | 26 +- bench/src/router-turn.ts | 64 +++ bench/src/sandbox-run.ts | 25 +- bench/src/search-bench/parametric-check.mts | 23 +- bench/src/supervisor-arena.mts | 45 +- bench/src/swe-jail.test.ts | 96 ++++ bench/src/swe-jail.ts | 107 +++- bench/src/swe-repro-calibrate.mts | 5 + bench/src/swe-stream.mts | 24 + bench/src/swe-structural.mts | 20 +- bench/src/trata-gate.mts | 44 +- bench/src/trata-gepa.mts | 38 +- bench/src/trata-hedge-solve.mts | 77 +++ bench/src/worker-blender.ts | 28 +- bench/src/worker-browser.ts | 23 +- bench/src/worker-build123d.ts | 25 +- bench/src/worker-cad.ts | 43 +- docs/api/primitive-catalog.md | 5 +- examples/p1-parity/run-parity.ts | 83 ++- .../supervisor-loop/run-supervisor-mcp.ts | 57 +- examples/supervisor-loop/run.ts | 14 +- examples/supervisor-loop/shared.ts | 17 +- package.json | 3 +- scripts/check-model-execution-boundary.mjs | 233 ++++++++ .../check-model-execution-boundary.test.mjs | 51 ++ src/runtime/index.ts | 7 +- src/runtime/inline-sandbox-client.ts | 26 +- src/runtime/router-client.complete.test.ts | 104 ++++ src/runtime/router-client.ts | 77 ++- src/runtime/stream-agent-turn.test.ts | 144 +++-- src/runtime/stream-agent-turn.ts | 541 ++++++++++++++++-- src/runtime/supervise/completion-gate.ts | 62 ++ src/runtime/supervise/materialization.ts | 4 +- src/runtime/supervise/runtime.ts | 384 ++++++++++++- src/types.ts | 4 + 64 files changed, 2898 insertions(+), 1022 deletions(-) delete mode 100644 bench/scripts/trata-hedge/solve.py create mode 100644 bench/src/router-turn.ts create mode 100644 bench/src/swe-jail.test.ts create mode 100644 bench/src/trata-hedge-solve.mts create mode 100644 scripts/check-model-execution-boundary.mjs create mode 100644 scripts/check-model-execution-boundary.test.mjs diff --git a/bench/scripts/appworld_driver.py b/bench/scripts/appworld_driver.py index 2ca8eb38..739da1c4 100644 --- a/bench/scripts/appworld_driver.py +++ b/bench/scripts/appworld_driver.py @@ -9,10 +9,7 @@ import argparse import json -import os -import re import sys -import time def fail(msg: str) -> None: @@ -20,166 +17,6 @@ def fail(msg: str) -> None: sys.exit(1) -_CODE_RE = re.compile(r"```(?:python|py)?\s*\n(.*?)```", re.DOTALL) - - -def _extract_code(text: str) -> str: - blocks = _CODE_RE.findall(text or "") - return (blocks[-1] if blocks else "").strip() - - -def _router_chat(base: str, key: str, model: str, messages: list, timeout: float = 180.0): - """One router chat-completion with retry on transient/429/5xx. Returns - (content, input_tokens, output_tokens). Raises on exhausted retries.""" - import httpx - - url = base.rstrip("/") + "/chat/completions" - last = None - for attempt in range(4): - try: - r = httpx.post( - url, - headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"}, - json={"model": model, "messages": messages}, - timeout=timeout, - ) - if r.status_code in (429, 500, 502, 503, 504): - last = f"{r.status_code}: {r.text[:160]}" - time.sleep(2**attempt) - continue - r.raise_for_status() - d = r.json() - content = (d["choices"][0]["message"].get("content") or "") - usage = d.get("usage") or {} - return content, int(usage.get("prompt_tokens", 0) or 0), int(usage.get("completion_tokens", 0) or 0) - except Exception as e: # noqa: BLE001 - last = str(e) - if attempt < 3: - time.sleep(2**attempt) - continue - raise RuntimeError(f"router_chat failed after retries: {last}") - raise RuntimeError(f"router_chat exhausted: {last}") - - -def _build_system(directive: str, world) -> str: - sup = world.task.supervisor - apps = list(getattr(world.task, "allowed_apps", []) or []) - descs = getattr(world.task, "app_descriptions", "") - desc_str = json.dumps(descs) if isinstance(descs, (dict, list)) else str(descs) - return ( - f"You are an AI agent completing a digital task for your supervisor " - f"{getattr(sup, 'first_name', '')} {getattr(sup, 'last_name', '')} " - f"(email {getattr(sup, 'email', '')}, phone {getattr(sup, 'phone_number', '')}) " - "by WRITING PYTHON that calls app APIs (the apis..(...) surface).\n\n" - f"Available apps: {', '.join(apps)}.\n" - f"App descriptions: {desc_str[:1500]}\n\n" - "How to work, one step per turn:\n" - "- Discover APIs with apis.api_docs.show_api_descriptions(app_name='') and " - "apis.api_docs.show_api_doc(app_name='', api_name='') BEFORE calling them.\n" - "- Get the supervisor's app passwords with apis.supervisor.show_account_passwords(), then log in " - "to each app you use to obtain its access_token.\n" - "- Write ONE short Python code block per turn. After it runs you SEE its OUTPUT (or error " - "traceback) — use that to decide the next step. Print intermediate values you need.\n" - "- Iterate: inspect -> authenticate -> act -> verify. Do not guess API names or arguments.\n" - "- When the task is fully done call apis.supervisor.complete_task(answer=) (include the " - "answer if the task asks a question, otherwise apis.supervisor.complete_task()).\n" - "- Reply with EXACTLY ONE fenced ```python block per turn and nothing else.\n\n" - f"{directive}" - ) - - -def cmd_react(args) -> None: - """Multi-turn REPL agent: the model writes a python block, the engine executes - it in the PERSISTENT world, the output is fed back, and it iterates until it - completes the task or hits max-turns. Then AppWorld's own evaluator scores it. - Config (directive, model, router creds, max_turns) arrives as JSON on stdin so - the candidate directive can be arbitrarily long. The directive is the optimized - surface; the loop + contract are fixed.""" - cfg = {} - raw = sys.stdin.read() - if raw.strip(): - try: - cfg = json.loads(raw) - except Exception as e: # noqa: BLE001 - fail(f"react config JSON parse failed: {e}") - directive = str(cfg.get("directive", "")) - model = str(cfg.get("model", "gpt-4o")) - max_turns = int(cfg.get("max_turns", 8)) - router_base = str(cfg.get("router_base", "https://router.tangle.tools/v1")) - router_key = str(cfg.get("router_key") or os.environ.get("TANGLE_API_KEY", "")) - if not router_key: - fail("react: router_key/TANGLE_API_KEY required") - - try: - from appworld import AppWorld - except Exception as e: # noqa: BLE001 - fail(f"appworld import failed: {e}") - - in_tok = 0 - out_tok = 0 - turns = 0 - turns_log: list = [] - try: - with AppWorld( - task_id=args.task_id, - experiment_name="bench-react", - raise_on_failure=False, - ) as world: - messages = [ - {"role": "system", "content": _build_system(directive, world)}, - {"role": "user", "content": f"Task: {world.task.instruction}"}, - ] - for turn in range(max_turns): - turns = turn + 1 - content, ui, uo = _router_chat(router_base, router_key, model, messages) - in_tok += ui - out_tok += uo - code = _extract_code(content) - messages.append({"role": "assistant", "content": content}) - if not code: - messages.append({ - "role": "user", - "content": "Reply with exactly one ```python block that makes progress, " - "or call apis.supervisor.complete_task().", - }) - continue - output = world.execute(code) - turns_log.append({"code": code[:600], "output": str(output)[:600]}) - messages.append({"role": "user", "content": "OUTPUT:\n" + str(output)[:4000]}) - if world.task_completed(): - break - evaluation = world.evaluate().to_dict() - except Exception as e: # noqa: BLE001 - fail(f"react of {args.task_id} failed: {e}") - - if "success" not in evaluation or "num_tests" not in evaluation: - fail(f"evaluation dict missing success/num_tests keys: {sorted(evaluation.keys())}") - passes = evaluation.get("passes", []) - failures = evaluation.get("failures", []) - n_pass = len(passes) if isinstance(passes, list) else int(passes or 0) - n_fail = len(failures) if isinstance(failures, list) else int(failures or 0) - print( - json.dumps( - { - "success": bool(evaluation["success"]), - "passes": n_pass, - "fails": n_fail, - "num_tests": int(evaluation["num_tests"]), - # Failed sub-test names — the evidence a trace analyst steers on. - "failure_names": [str(f)[:160] for f in failures][:8] - if isinstance(failures, list) - else [], - "turns": turns, - "input_tokens": in_tok, - "output_tokens": out_tok, - "transcript": "\n---\n".join( - f"CODE:\n{t['code']}\nOUTPUT:\n{t['output']}" for t in turns_log[-3:] - )[:1600], - } - ) - ) - - def cmd_session(args) -> None: """Dumb world shim: a persistent AppWorld session driven over stdin JSONL. NO LLM calls here — the agent loop lives in the runtime (routerToolLoop); @@ -336,10 +173,6 @@ def main() -> None: p_eval.add_argument("--task-id", required=True) p_eval.add_argument("--split", required=True) - p_react = sub.add_parser("react") - p_react.add_argument("--task-id", required=True) - p_react.add_argument("--split", required=True) - p_session = sub.add_parser("session") p_session.add_argument("--task-id", required=True) p_session.add_argument("--split", required=True) @@ -349,8 +182,6 @@ def main() -> None: cmd_load(args) elif args.cmd == "evaluate": cmd_evaluate(args) - elif args.cmd == "react": - cmd_react(args) elif args.cmd == "session": cmd_session(args) diff --git a/bench/scripts/trata-hedge/README.md b/bench/scripts/trata-hedge/README.md index 9eb25e30..8c2794a0 100644 --- a/bench/scripts/trata-hedge/README.md +++ b/bench/scripts/trata-hedge/README.md @@ -21,8 +21,8 @@ sparse (1 iff all themes). No deployable ground-truth checker — it's an **orac ## Status (2026-06-06): pipeline PROVEN end-to-end -Our solver → their **real** Gemini-3.1-pro judge → a genuine graded result. Every link -works. The naive **single-shot** baseline (gpt-4o, ~3 of N corpus files in one context +Our Runtime-backed solver → their **real** Gemini-3.1-pro judge → a genuine graded result. Every link +works. The naive **single-shot** baseline (DeepSeek V4 Flash, ~3 of N corpus files in one context window) scores **0/4** — a floor: it hit only 1/3 moves on a few themes with hallucinations flagged, because it could not explore the full corpus. The bench is built for **agentic** exploration; a fair baseline needs our sandbox runtime as the solver @@ -35,7 +35,7 @@ for **agentic** exploration; a fair baseline needs our sandbox runtime as the so git clone https://github.com/Trata-Inc/trata-hedge-bench /tmp/thb dotenvx run -f ~/company/devops/secrets/.env.keys -f ~/company/devops/secrets/agent-state.env -- \ - bash bench/scripts/trata-hedge/run.sh /tmp/thb/environments/ gpt-4o + bash bench/scripts/trata-hedge/run.sh /tmp/thb/environments/ deepseek-v4-flash ``` ## Gotchas (each cost a debugging cycle) diff --git a/bench/scripts/trata-hedge/run.sh b/bench/scripts/trata-hedge/run.sh index 7fa5bc05..cbb60be9 100755 --- a/bench/scripts/trata-hedge/run.sh +++ b/bench/scripts/trata-hedge/run.sh @@ -17,14 +17,18 @@ set -euo pipefail ENV="${1:?usage: run.sh [model]}" -MODEL="${2:-${WORKER_MODEL:-gpt-4o}}" +MODEL="${2:-${WORKER_MODEL:-}}" +if [[ -z "$MODEL" ]]; then + echo "WORKER_MODEL is required (or pass it as argument 2)" >&2 + exit 2 +fi HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ANS=/tmp/thb-answer.txt REWARD=/tmp/thb-reward.txt DETAILS=/tmp/thb-details.json echo "[trata] solve: $ENV (model=$MODEL)" -WORKER_MODEL="$MODEL" python3 "$HERE/solve.py" "$ENV" "$ANS" +WORKER_MODEL="$MODEL" pnpm --dir "$HERE/../../.." exec tsx "$HERE/../../src/trata-hedge-solve.mts" "$ENV" "$ANS" sudo -n mkdir -p /app && sudo -n cp "$ANS" /app/answer.txt && sudo -n chmod 644 /app/answer.txt diff --git a/bench/scripts/trata-hedge/solve.py b/bench/scripts/trata-hedge/solve.py deleted file mode 100644 index 71454ec0..00000000 --- a/bench/scripts/trata-hedge/solve.py +++ /dev/null @@ -1,83 +0,0 @@ -"""trata-hedge-bench solver — our system's analyst, graded by THEIR Gemini judge. - -Reads a task's instruction.md + data corpus, writes a cited analysis to answer.txt. -The corpus is large (~5MB/task), so a single-shot router call is a LOWER-BOUND -baseline (it sees only what fits in context) — the bench is built for agentic -exploration, which our sandbox runtime provides (see README; pending sandbox health). - -Notes learned the hard way: -- router.tangle.tools is behind Cloudflare bot-fight: a default Python-urllib - User-Agent gets 403 (CF 1010) on large bodies → send a browser UA. -- gpt-4.1 is NOT keyed for direct router calls (403); gpt-4o / claude-sonnet-4-6 / - gpt-4o-mini are. Very large bodies (~350KB) can 503 — keep DATA_BUDGET in range. - - WORKER_MODEL=gpt-4o DATA_BUDGET=160000 python3 solve.py -""" -import json -import os -import sys -import urllib.request -from pathlib import Path - -env = Path(sys.argv[1]) -out = sys.argv[2] if len(sys.argv) > 2 else "/tmp/thb-answer.txt" -model = os.environ.get("WORKER_MODEL", "gpt-4o") -budget = int(os.environ.get("DATA_BUDGET", "160000")) -instruction = (env / "instruction.md").read_text() -data_dir = env / "environment" / "data" - - -def rank(p: Path) -> int: - s = str(p) - return (0 if "earnings_call" in s else 1 if "financials" in s else 2 if "company_profiles" in s - else 3 if "press_releases" in s else 4) - - -files = sorted((p for p in data_dir.rglob("*") if p.is_file()), key=rank) -blocks, used, cited = [], 0, [] -for p in files: - rel = p.relative_to(data_dir) - try: - c = p.read_text(errors="replace") - except Exception: - continue - if used + len(c) > budget: - continue - used += len(c) - cited.append(str(rel)) - blocks.append(f"\n=== FILE: data/{rel} ===\n{c}") -print(f"[solve] {len(cited)}/{len(files)} files in context ({used} chars)", file=sys.stderr) - -prompt = ( - instruction - + "\n\n--- AVAILABLE DATA (cite files by their `data/` name inline) ---\n" - + "".join(blocks) - + "\n\n--- END DATA ---\nWrite ONLY the full analysis (no preamble). Inline-cite every claim with its `data/`." -) - -key = os.environ["TANGLE_API_KEY"] -base = os.environ.get("ROUTER_BASE", "https://router.tangle.tools/v1") -body = json.dumps({ - "model": model, - "messages": [{"role": "user", "content": prompt}], - "temperature": float(os.environ.get("TEMPERATURE", "0.5")), - "max_tokens": int(os.environ.get("MAX_TOKENS", "6000")), -}).encode() -req = urllib.request.Request( - f"{base}/chat/completions", data=body, - headers={ - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - # browser UA: router is behind Cloudflare bot-fight, default urllib UA → 403 on big bodies - "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36", - }, -) -try: - with urllib.request.urlopen(req, timeout=300) as r: - resp = json.load(r) -except urllib.error.HTTPError as e: - print(f"[solve] HTTP {e.code}: {e.read().decode(errors='replace')[:400]}", file=sys.stderr) - raise -answer = resp["choices"][0]["message"]["content"] -Path(out).write_text(answer) -print(f"[solve] wrote {len(answer)} chars -> {out} (model={model})", file=sys.stderr) diff --git a/bench/src/aec-gate.mts b/bench/src/aec-gate.mts index c5f0952c..3b7b068c 100644 --- a/bench/src/aec-gate.mts +++ b/bench/src/aec-gate.mts @@ -22,7 +22,8 @@ import { resolveAdapter } from './adapters' import type { BenchmarkAdapter, BenchTask } from './benchmarks/types' import { type AttemptRecord, appendRunRecord, buildRunRecordFromAttempts } from './corpus' import { composeStrategies } from './directives' -import { type RouterConfig, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import type { RouterConfig } from '@tangle-network/agent-runtime/kernel' +import { runBenchRouterTurn } from './router-turn' import { pool } from './stats.mts' function must(name: string): string { @@ -65,8 +66,18 @@ async function runAttempt( let lastErr: unknown for (let attempt = 0; attempt < 3; attempt += 1) { try { - const res = await routerChatWithUsage(cfg, [{ role: 'user', content: prompt }]) - const content = typeof res.content === 'string' ? res.content : '' + const res = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'aec-worker', + model: { provider: 'tangle-router', default: cfg.model }, + }, + }, + prompt, + ) + const content = res.finalText const verdict = await adapter.judge(task, content) return { prompt, @@ -74,8 +85,10 @@ async function runAttempt( score: verdict.score, resolved: verdict.resolved, wallMs: Date.now() - startedAt, - ...(res.costUsd !== undefined ? { costUsd: res.costUsd } : {}), - ...(res.usage ? { tokensIn: res.usage.input, tokensOut: res.usage.output } : {}), + ...(res.usage.costUsd !== undefined ? { costUsd: res.usage.costUsd } : {}), + ...(res.usage.tokensKnown === false + ? {} + : { tokensIn: res.usage.input, tokensOut: res.usage.output }), } } catch (err) { lastErr = err diff --git a/bench/src/agent-graphs-gen2.mts b/bench/src/agent-graphs-gen2.mts index 81b8466c..c85180dd 100644 --- a/bench/src/agent-graphs-gen2.mts +++ b/bench/src/agent-graphs-gen2.mts @@ -47,6 +47,7 @@ import { import { type AuthoredArtifact, type CaseSpec, + buildAgentGraphsAuthorProfile, callAuthor, dispatchWithSurface, judgeArtifact, @@ -215,6 +216,16 @@ function validateSkillGate(text: string): string[] { } function makeProposer(v1Surface: string, trainCases: GraphScenario[]): SurfaceProposer { + const proposerProfile = buildAgentGraphsAuthorProfile(v1Surface, { + ...process.env, + AGENT_GRAPHS_AUTHOR_PROFILE_NAME: 'agent-graphs-skill-reviser', + AGENT_GRAPHS_AUTHOR_SYSTEM_PROMPT: + 'Revise an agent skill from measured development-case failures. Return only the requested artifact.', + }) + const attemptLimit = Number(process.env.AGENT_GRAPHS_GEN2_PROPOSER_ATTEMPTS ?? 2) + if (!Number.isSafeInteger(attemptLimit) || attemptLimit <= 0) { + throw new Error('AGENT_GRAPHS_GEN2_PROPOSER_ATTEMPTS must be a positive integer') + } return { kind: 'agent-graphs-skill-reviser', async propose(_ctx: ProposeContext): Promise { @@ -245,8 +256,8 @@ function makeProposer(v1Surface: string, trainCases: GraphScenario[]): SurfacePr } let prompt = revisionPrompt let lastProblems: string[] = [] - for (let attempt = 0; attempt < 2; attempt += 1) { - const reply = await callAuthor(prompt, 0.7, 12_000) + for (let attempt = 0; attempt < attemptLimit; attempt += 1) { + const reply = await callAuthor(proposerProfile, prompt) const skill = extractSkill(reply) lastProblems = validateSkillGate(skill) if (lastProblems.length === 0) { diff --git a/bench/src/agent-graphs-improve.mts b/bench/src/agent-graphs-improve.mts index 2205cbdb..90236164 100644 --- a/bench/src/agent-graphs-improve.mts +++ b/bench/src/agent-graphs-improve.mts @@ -20,13 +20,19 @@ import { execFileSync } from 'node:child_process' import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs' -import { homedir } from 'node:os' import { join, dirname, resolve as resolvePath } from 'node:path' import { fileURLToPath } from 'node:url' -import type { AgentProfile } from '@tangle-network/agent-interface' +import { + defineInlineResource, + harnessTypeSchema, + reasoningEffortSchema, + type AgentProfile, +} from '@tangle-network/agent-interface' import { type AgentGraph, type AnalystRegistry, + collectAgentTurn, + createExecutor, defaultEdgeTraversalCap, type EdgeTraversal, GraphEdgeCapError, @@ -34,6 +40,7 @@ import { promptHandle, type RunGraphOptions, runGraph, + streamAgentTurn, } from '../../src/runtime/index.ts' import { leafSeam, scriptedBrain, type ScriptedTurn } from './agent-graphs-improve/offline-seams.mts' @@ -145,22 +152,16 @@ export interface AuthoredArtifact { // ── The author model call ────────────────────────────────────────────────────── -const ROUTER_URL = 'https://router.tangle.tools/v1/chat/completions' -const AUTHOR_MODEL = 'glm-5.2' - -function routerToken(): string { - const raw = readFileSync(join(homedir(), '.config', 'tangle', 'router-token.json'), 'utf8') - return (JSON.parse(raw) as { token: string }).token +function positiveInteger(name: string, raw: string | undefined, fallback: number): number { + const value = raw === undefined ? fallback : Number(raw) + if (!Number.isSafeInteger(value) || value <= 0) { + throw new Error(`${name} must be a positive integer`) + } + return value } -function authorPrompt(surface: string, kase: CaseSpec): string { +function authorPrompt(kase: CaseSpec): string { return [ - 'You are an agent-graph author. Follow the skill below EXACTLY — it is your only doctrine.', - '', - '', - surface, - '', - '', ``, kase.brief, '', @@ -189,25 +190,61 @@ function extractJson(text: string): string { return stripped.slice(start, end + 1) } -export async function callAuthor(prompt: string, temperature = 0.2, maxTokens = 6000): Promise { - const res = await fetch(ROUTER_URL, { - method: 'POST', - headers: { Authorization: `Bearer ${routerToken()}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ - model: AUTHOR_MODEL, - temperature, - max_tokens: maxTokens, - messages: [{ role: 'user', content: prompt }], - }), - signal: AbortSignal.timeout(240_000), +export function buildAgentGraphsAuthorProfile( + surface: string, + env: NodeJS.ProcessEnv = process.env, +): AgentProfile { + return { + name: env.AGENT_GRAPHS_AUTHOR_PROFILE_NAME ?? 'agent-graphs-author', + harness: harnessTypeSchema.parse(env.AGENT_GRAPHS_AUTHOR_HARNESS ?? 'pi'), + model: { + provider: env.AGENT_GRAPHS_AUTHOR_PROVIDER ?? 'tangle-router', + default: env.AGENT_GRAPHS_AUTHOR_MODEL ?? 'deepseek-v4-flash', + reasoningEffort: reasoningEffortSchema.parse( + env.AGENT_GRAPHS_AUTHOR_REASONING_EFFORT ?? 'ultracode', + ), + }, + prompt: { + systemPrompt: + env.AGENT_GRAPHS_AUTHOR_SYSTEM_PROMPT ?? + 'Apply the attached agent-graphs skill exactly. Return only the requested artifact.', + }, + resources: { + failOnError: true, + skills: [defineInlineResource('agent-graphs', surface)], + }, + } +} + +export async function callAuthor( + profile: AgentProfile, + prompt: string, + env: NodeJS.ProcessEnv = process.env, +): Promise { + const bridgeBearer = env.AGENT_GRAPHS_BRIDGE_BEARER ?? env.BRIDGE_BEARER + if (!bridgeBearer) throw new Error('AGENT_GRAPHS_BRIDGE_BEARER or BRIDGE_BEARER is required') + const factory = createExecutor({ + backend: 'bridge', + bridgeUrl: env.AGENT_GRAPHS_BRIDGE_URL ?? env.BRIDGE_URL ?? 'http://127.0.0.1:3355', + bridgeBearer, }) - if (!res.ok) throw new Error(`router HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`) - const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - const content = data.choices?.[0]?.message?.content - if (typeof content !== 'string' || content.trim().length === 0) { - throw new Error('router returned empty content') + const timeoutRaw = env.AGENT_GRAPHS_AUTHOR_TIMEOUT_MS + const timeoutMs = + timeoutRaw === undefined + ? undefined + : positiveInteger('AGENT_GRAPHS_AUTHOR_TIMEOUT_MS', timeoutRaw, 1) + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile, agentRunName: profile.name ?? 'agent-graphs-author' }, + prompt, + timeoutMs === undefined ? {} : { timeoutMs }, + ), + ) + if (turn.status !== 'completed') { + throw new Error(turn.error?.message ?? `author turn ended with status ${turn.status}`) } - return content + if (!turn.finalText.trim()) throw new Error('author returned empty content') + return turn.finalText } interface AuthoredReply { @@ -219,11 +256,17 @@ interface AuthoredReply { /** Prompt the author; one retry on unparseable JSON (or a transport fault). */ async function authorOnce(surface: string, kase: CaseSpec): Promise { - const prompt = authorPrompt(surface, kase) + const prompt = authorPrompt(kase) + const profile = buildAgentGraphsAuthorProfile(surface) + const attempts = positiveInteger( + 'AGENT_GRAPHS_AUTHOR_ATTEMPTS', + process.env.AGENT_GRAPHS_AUTHOR_ATTEMPTS, + 2, + ) let lastErr: unknown - for (let attempt = 0; attempt < 2; attempt += 1) { + for (let attempt = 0; attempt < attempts; attempt += 1) { try { - const raw = await callAuthor(prompt) + const raw = await callAuthor(profile, prompt) const parsed = JSON.parse(extractJson(raw)) as { decision?: string reason?: string @@ -243,7 +286,9 @@ async function authorOnce(surface: string, kase: CaseSpec): Promise((_, reject) => { - const t = setTimeout(() => reject(new Error('offline run timed out (120s)')), 120_000) + const t = setTimeout( + () => reject(new Error(`offline run timed out (${timeoutMs}ms)`)), + timeoutMs, + ) t.unref?.() }) const res: GraphResult = await Promise.race([runGraph(graph, opts), timeout]) @@ -607,8 +669,16 @@ async function main(): Promise { const only = process.env.CASE const cases: CaseSpec[] = inputs.cases.filter((c) => only === undefined || c.id === only) + const authorProfile = buildAgentGraphsAuthorProfile(surface) + const authorLabel = [ + authorProfile.harness, + authorProfile.model?.provider, + authorProfile.model?.default, + ] + .filter(Boolean) + .join('/') console.log( - `codemode baseline: skill v1 (${surface.length} chars, source=${inputs.source}), ${cases.length} cases, author=${AUTHOR_MODEL}`, + `codemode baseline: skill v1 (${surface.length} chars, source=${inputs.source}), ${cases.length} cases, author=${authorLabel}`, ) const results: CaseResult[] = [] @@ -666,8 +736,7 @@ async function main(): Promise { const out = { skillVersion: 'v1', surfaceSource: inputs.source, - authorModel: AUTHOR_MODEL, - temperature: 0.2, + authorProfile, date: new Date().toISOString(), n: results.length, aggregate: { mean, median, min: scores[0] ?? 0, max: scores[scores.length - 1] ?? 0 }, diff --git a/bench/src/atom-humaneval.mts b/bench/src/atom-humaneval.mts index 36ec5719..fd61204e 100644 --- a/bench/src/atom-humaneval.mts +++ b/bench/src/atom-humaneval.mts @@ -21,20 +21,18 @@ import { type AgentProfile, type AgentSpec, contentAddress, - type DriverAgentOptions, - driverAgent, + createExecutor, createExecutorRegistry, createSupervisor, - type Executor, - type ExecutorResult, gateOnDeliverable, InMemoryResultBlobStore, InMemorySpawnJournal, + mapExecutorResult, type RouterConfig, - routerBrain, - routerChatWithUsage, + supervisorAgent, } from '../../src/runtime/index' import { basePrompt, extractCode, type HumanEvalTask, loadHumanEval, runChecker } from './benchmarks/humaneval' +import { runBenchRouterTurn } from './router-turn' function must(k: string): string { const v = process.env[k] @@ -54,38 +52,31 @@ const cfg: RouterConfig = { } const driverCfg: RouterConfig = { ...cfg, model: process.env.DRIVER_MODEL ?? cfg.model } -// The driver-LLM brain is the SHARED `routerBrain` (the canonical ToolLoopChat seam) — it forwards -// usage/costUsd, so this bench's driver arms meter their own inference into the conserved pool. - // ── A gated router worker: one router call → candidate code, settled valid ⟺ the tests pass ── function humanEvalWorker(task: HumanEvalTask, label: string): Agent { - let artifact: ExecutorResult | undefined - const inner: Executor = { - runtime: 'router', - async execute(_t, signal) { - const res = await routerChatWithUsage(cfg, [{ role: 'user', content: basePrompt(task) }], { - temperature: WORKER_TEMP, - ...(signal ? { signal } : {}), - }) - const code = extractCode(res.content) - artifact = { - outRef: contentAddress(code), - out: code, - spent: { iterations: 1, tokens: res.usage ?? { input: 0, output: 0 }, usd: res.costUsd ?? 0, ms: 0 }, - } - return artifact - }, - teardown: () => Promise.resolve({ destroyed: true }), - resultArtifact: () => { - if (!artifact) throw new Error('resultArtifact read before execute') - return artifact - }, + const profile: AgentProfile = { + name: label, + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt: basePrompt(task) }, } - const gated = gateOnDeliverable(inner, { - check: async (out) => (await runChecker(task, String(out))).pass === 1, - describe: `${task.taskId}: the provided test suite passes`, + const routerFactory = createExecutor({ + backend: 'router', + ...cfg, + temperature: WORKER_TEMP, }) - const spec: AgentSpec = { profile: { name: label } as AgentProfile, harness: null, executor: gated } + const executorFactory = (spec: AgentSpec, ctx: Parameters[1]) => { + const inner = routerFactory(spec, ctx) + const mapped = mapExecutorResult(inner, (result) => { + const raw = result.out as { content?: unknown } + const code = extractCode(typeof raw?.content === 'string' ? raw.content : '') + return { outRef: contentAddress(code), out: code } + }) + return gateOnDeliverable(mapped, { + check: async (out) => (await runChecker(task, String(out))).pass === 1, + describe: `${task.taskId}: the provided test suite passes`, + }) + } + const spec: AgentSpec = { profile, harness: null, executorFactory } return { name: label, act: async () => '', executorSpec: spec } as Agent & { executorSpec: AgentSpec } @@ -113,16 +104,20 @@ async function driveTask( spawns += 1 return w } - const opts: DriverAgentOptions = { - name: `drv-${task.taskId}`, - brain: routerBrain(driverCfg), + const root = supervisorAgent( + { + name: `drv-${task.taskId}`, + model: { provider: 'tangle-router', default: driverCfg.model }, + prompt: { systemPrompt: driverSystem }, + }, + { + router: driverCfg, blobs, makeWorkerAgent: makeWorker, perWorker: { maxIterations: 2, maxTokens: 4000 }, - systemPrompt: driverSystem, maxTurns: K + 4, - } - const root = driverAgent(opts) + }, + ) const runId = `he-${task.taskId.replace('/', '-')}` const result = await createSupervisor().run(root, basePrompt(task), { budget: { maxIterations: 100, maxTokens: 400_000 }, @@ -145,15 +140,25 @@ async function blindTask(task: HumanEvalTask): Promise { for (let i = 0; i < K; i += 1) { // A transient router error is a FAILED attempt, not a crash — the driver arm already types // an executor throw into a `down` settlement, so the blind arm must match (fair comparison). - let res: { content: string } + let content = '' try { - res = await routerChatWithUsage(cfg, [{ role: 'user', content: basePrompt(task) }], { - temperature: WORKER_TEMP, - }) + const res = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'humaneval-blind-atom-worker', + model: { provider: 'tangle-router', default: cfg.model }, + }, + temperature: WORKER_TEMP, + }, + basePrompt(task), + ) + content = res.finalText } catch { continue } - if ((await runChecker(task, extractCode(res.content))).pass === 1) return true + if ((await runChecker(task, extractCode(content))).pass === 1) return true } return false } diff --git a/bench/src/atom-mcp-e2e.mts b/bench/src/atom-mcp-e2e.mts index 0c588607..5ef29e0a 100644 --- a/bench/src/atom-mcp-e2e.mts +++ b/bench/src/atom-mcp-e2e.mts @@ -21,7 +21,9 @@ import { type Agent, type AgentProfile, type AgentSpec, + collectAgentTurn, contentAddress, + createExecutor, createExecutorRegistry, createSupervisor, type Executor, @@ -31,6 +33,7 @@ import { InMemorySpawnJournal, runInWorkspace, type Scope, + streamAgentTurn, type Workspace, } from '../../src/runtime/index' import { asAuthoredProfile } from '../../src/runtime/supervise/authoring' @@ -83,19 +86,30 @@ async function bridgeChat(opts: { cwd?: string mcpUrl?: string }): Promise { - const r = await fetch(`${BRIDGE}/chat/completions`, { - method: 'POST', - headers: { authorization: `Bearer ${BEARER}`, 'content-type': 'application/json' }, - body: JSON.stringify({ - model: MODEL, - messages: opts.messages, - ...(opts.cwd ? { cwd: opts.cwd } : {}), - ...(opts.mcpUrl ? { mcp: { mcpServers: { coordination: { type: 'http', url: opts.mcpUrl } } } } : {}), - }), + if (!BEARER) throw new Error('TANGLE_API_KEY is required') + const profile: AgentProfile = { + name: opts.mcpUrl ? 'atom-mcp-supervisor-turn' : 'atom-mcp-worker-turn', + model: { default: MODEL }, + ...(opts.mcpUrl + ? { mcp: { coordination: { transport: 'http', url: opts.mcpUrl } } } + : {}), + } + const factory = createExecutor({ + backend: 'bridge', + bridgeUrl: BRIDGE.replace(/\/v1$/u, ''), + bridgeBearer: BEARER, + ...(opts.cwd ? { cwd: opts.cwd } : {}), }) - if (!r.ok) return `(bridge HTTP ${r.status}: ${(await r.text()).slice(0, 200)})` - const j = (await r.json()) as { choices?: Array<{ message?: { content?: string } }> } - return j.choices?.[0]?.message?.content ?? '' + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile }, + opts.messages.map((message) => message.content).join('\n\n'), + ), + ) + if (turn.status !== 'completed') { + throw new Error(turn.error?.message ?? `bridge turn ended with ${turn.status}`) + } + return turn.finalText } const transcripts: Array<{ who: string; said: string; delivered?: boolean }> = [] diff --git a/bench/src/benchmarks/appworld.ts b/bench/src/benchmarks/appworld.ts index c30ead86..ffe74dc6 100644 --- a/bench/src/benchmarks/appworld.ts +++ b/bench/src/benchmarks/appworld.ts @@ -22,7 +22,13 @@ import { spawn } from 'node:child_process' import { join } from 'node:path' import { createInterface } from 'node:readline' -import { type OutputAdapter, routerToolLoop, type ToolSpec } from '@tangle-network/agent-runtime/kernel' +import { + collectAgentTurn, + createExecutor, + type OutputAdapter, + streamAgentTurn, + type ToolSpec, +} from '@tangle-network/agent-runtime/kernel' import { benchRoot, preflightVenvImports, runVenvScriptStdin, venvPython } from './_harness' import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' @@ -285,7 +291,7 @@ async function withWorldSession( } } -/** SandboxClient whose leaf is OUR routerToolLoop driving a persistent world session. */ +/** SandboxClient whose leaf is Runtime's profile-bound Router executor driving a world session. */ export function appworldToolLoopClient(cfg: { model: string routerBaseUrl: string @@ -310,30 +316,55 @@ export function appworldToolLoopClient(cfg: { const directive = prompt.replace(REACT_HEADER, '').trim() const out = await withWorldSession(taskId as string, split as string, async (call, instruction) => { const system = directive ? `${SESSION_SYSTEM}\n\n${directive}` : SESSION_SYSTEM - const loop = await routerToolLoop( - { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model }, - system, - `Task: ${instruction}`, - [EXECUTE_TOOL], - async (name, args) => { + const transcriptSteps: Array<{ args: string; result: string }> = [] + const profile = { + name: 'appworld-react-worker', + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt: system }, + tools: { execute_python: true }, + } + const factory = createExecutor({ + backend: 'router-tools', + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + model: cfg.model, + tools: [EXECUTE_TOOL], + maxTurns, + executeToolCall: async (name, args) => { if (name !== 'execute_python') return `error: unknown tool ${name}` const res = await call({ op: 'execute', code: String(args.code ?? '') }) const done = res.task_completed === true - return `${String(res.output ?? '')}${done ? '\n\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}` + const result = `${String(res.output ?? '')}${done ? '\n\n[TASK MARKED COMPLETE — reply with a final summary and do not call the tool again]' : ''}` + transcriptSteps.push({ args: JSON.stringify(args), result }) + return result }, - { maxTurns }, + }) + const loop = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile }, + `Task: ${instruction}`, + ), ) + if (loop.status !== 'completed') { + throw new Error(loop.error?.message ?? `AppWorld turn ended with ${loop.status}`) + } const verdict = (await call({ op: 'evaluate' })) as unknown as ReactResult - const transcript = loop.toolTrace + const transcript = transcriptSteps .slice(-3) .map((t) => `CODE:\n${t.args.slice(0, 600)}\nOUTPUT:\n${t.result.slice(0, 600)}`) .join('\n---\n') .slice(0, 1600) + const finalEvent = loop.events.at(-1) + const resultMetadata = + finalEvent?.type === 'final' && finalEvent.metadata?.result + ? (finalEvent.metadata.result as { spent?: { iterations?: number } }) + : undefined return { ...verdict, - turns: loop.turns, - input_tokens: loop.usage.input, - output_tokens: loop.usage.output, + turns: resultMetadata?.spent?.iterations, + ...(loop.usage.tokensKnown === false + ? {} + : { input_tokens: loop.usage.input, output_tokens: loop.usage.output }), transcript, } satisfies ReactResult }) diff --git a/bench/src/benchmarks/cadbench.ts b/bench/src/benchmarks/cadbench.ts index f6665623..b0e47312 100644 --- a/bench/src/benchmarks/cadbench.ts +++ b/bench/src/benchmarks/cadbench.ts @@ -14,6 +14,7 @@ import { readFile } from 'node:fs/promises' import type { BenchScore, BenchTask, BenchmarkAdapter, LoadOptions } from './types' import { renderBpy } from '../worker-blender' +import { runBenchRouterTurn } from '../router-turn' interface CadBenchMeta { name: string @@ -48,14 +49,20 @@ async function judgeCriteria( `Return ONLY a JSON array of exactly ${criteria.length} booleans (true=satisfied, false=not), in order, no prose.\n\nCRITERIA:\n${numbered}\n\nSCRIPT:\n\`\`\`python\n${script.slice(0, 6000)}\n\`\`\`` const content: unknown[] = [{ type: 'text', text }] for (const url of renders) content.push({ type: 'image_url', image_url: { url } }) - const res = await fetch(`${base}/chat/completions`, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` }, - body: JSON.stringify({ model, max_tokens: 1500, temperature: 0, messages: [{ role: 'user', content }] }), - }) - if (!res.ok) throw new Error(`judge ${model} ${res.status}: ${(await res.text()).slice(0, 200)}`) - const data = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - const raw = data.choices?.[0]?.message?.content ?? '' + const turn = await runBenchRouterTurn( + { + routerBaseUrl: base, + routerKey: key, + profile: { + name: 'cadbench-vision-judge', + model: { provider: 'tangle-router', default: model }, + }, + temperature: 0, + maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 1500), + }, + { messages: [{ role: 'user', content }] }, + ) + const raw = turn.finalText const m = /\[\s*(?:true|false)[\s\S]*?\]/i.exec(raw) if (!m) return { passed: criteria.map(() => false), note: `judge returned no parseable verdict: ${raw.slice(0, 80)}` } let arr: unknown diff --git a/bench/src/benchmarks/finresearchbench.ts b/bench/src/benchmarks/finresearchbench.ts index b739732d..11033925 100644 --- a/bench/src/benchmarks/finresearchbench.ts +++ b/bench/src/benchmarks/finresearchbench.ts @@ -9,6 +9,7 @@ import { readFile, stat } from 'node:fs/promises' import { join } from 'node:path' +import { runBenchRouterTurn } from '../router-turn' import { benchRoot } from './_harness' import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' @@ -180,22 +181,21 @@ function parseJudgeScore(content: string): { score: number; raw: unknown } { async function runOfficialJudge(meta: FinResearchMeta, response: string): Promise { if (!meta.judgeSystemPrompt) throw new Error(`FinResearchBench task ${meta.id} missing judge_system_prompt`) const router = routerConfig() - const res = await fetch(`${router.baseUrl}/chat/completions`, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` }, - body: JSON.stringify({ - model: router.model, + const turn = await runBenchRouterTurn( + { + routerBaseUrl: router.baseUrl, + routerKey: router.key, + profile: { + name: 'finresearchbench-judge', + model: { provider: 'tangle-router', default: router.model }, + prompt: { systemPrompt: meta.judgeSystemPrompt }, + }, temperature: 0, - messages: [ - { role: 'system', content: meta.judgeSystemPrompt }, - { role: 'user', content: fillTemplate(meta, response) }, - ], - }), - }) - if (!res.ok) throw new Error(`FinResearchBench judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`) - const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - const content = body.choices?.[0]?.message?.content - if (typeof content !== 'string') throw new Error(`FinResearchBench judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`) + }, + fillTemplate(meta, response), + ) + const content = turn.finalText + if (!content) throw new Error('FinResearchBench judge returned no message content') const { score, raw } = parseJudgeScore(content) return { resolved: score >= Number(process.env.FINRESEARCHBENCH_PASS_THRESHOLD ?? 0.8), diff --git a/bench/src/benchmarks/finsearchcomp.ts b/bench/src/benchmarks/finsearchcomp.ts index b926c2c3..6842e098 100644 --- a/bench/src/benchmarks/finsearchcomp.ts +++ b/bench/src/benchmarks/finsearchcomp.ts @@ -32,6 +32,7 @@ import { readFile } from 'node:fs/promises' import { join } from 'node:path' import { fileURLToPath } from 'node:url' +import { runBenchRouterTurn } from '../router-turn' import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' const BENCH_ROOT = fileURLToPath(new URL('../..', import.meta.url)) @@ -206,26 +207,21 @@ function parseJudgeOutput(content: string): { resolved: boolean; score: number; /** Run the record's own judge via the router. Fail loud on transport/parse errors. */ async function runRecordJudge(meta: FinSearchMeta, response: string, router: JudgeRouter): Promise { - const res = await fetch(`${router.baseUrl}/chat/completions`, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` }, - body: JSON.stringify({ - model: router.model, + const turn = await runBenchRouterTurn( + { + routerBaseUrl: router.baseUrl, + routerKey: router.key, + profile: { + name: 'finsearchcomp-judge', + model: { provider: 'tangle-router', default: router.model }, + prompt: { systemPrompt: meta.judgeSystemPrompt }, + }, temperature: 0, - messages: [ - { role: 'system', content: meta.judgeSystemPrompt }, - { role: 'user', content: fillJudgePrompt(meta, response) }, - ], - }), - }) - if (!res.ok) { - throw new Error(`FinSearchComp judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`) - } - const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - const content = body.choices?.[0]?.message?.content - if (typeof content !== 'string') { - throw new Error(`FinSearchComp judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`) - } + }, + fillJudgePrompt(meta, response), + ) + const content = turn.finalText + if (!content) throw new Error('FinSearchComp judge returned no message content') const { resolved, score, raw } = parseJudgeOutput(content) return { resolved, diff --git a/bench/src/benchmarks/frames.ts b/bench/src/benchmarks/frames.ts index 26d332f1..9842beea 100644 --- a/bench/src/benchmarks/frames.ts +++ b/bench/src/benchmarks/frames.ts @@ -25,6 +25,7 @@ import { readFile } from 'node:fs/promises' import { join } from 'node:path' import { fileURLToPath } from 'node:url' import { promisify } from 'node:util' +import { runBenchRouterTurn } from '../router-turn' import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' const execFileAsync = promisify(execFile) @@ -322,24 +323,21 @@ async function tier2Judge( candidate: string, router: JudgeRouter, ): Promise { - const res = await fetch(`${router.baseUrl}/chat/completions`, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` }, - body: JSON.stringify({ - model: router.model, + const turn = await runBenchRouterTurn( + { + routerBaseUrl: router.baseUrl, + routerKey: router.key, + profile: { + name: 'frames-equivalence-judge', + model: { provider: 'tangle-router', default: router.model }, + }, temperature: 0, seed: 0, - messages: [{ role: 'user', content: JUDGE_PROMPT(question, gold, candidate) }], - }), - }) - if (!res.ok) { - throw new Error(`FRAMES Tier-2 judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`) - } - const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - const content = body.choices?.[0]?.message?.content - if (typeof content !== 'string') { - throw new Error(`FRAMES Tier-2 judge returned no message content: ${JSON.stringify(body).slice(0, 300)}`) - } + }, + JUDGE_PROMPT(question, gold, candidate), + ) + const content = turn.finalText + if (!content) throw new Error('FRAMES Tier-2 judge returned no message content') const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/) const raw = (fenced ? fenced[1] : content)?.trim() ?? '' let parsed: { verdict?: unknown } diff --git a/bench/src/benchmarks/simpleqa.ts b/bench/src/benchmarks/simpleqa.ts index 5a0a197b..d6a458d8 100644 --- a/bench/src/benchmarks/simpleqa.ts +++ b/bench/src/benchmarks/simpleqa.ts @@ -30,6 +30,7 @@ import { readFile } from 'node:fs/promises' import { join } from 'node:path' import { fileURLToPath } from 'node:url' import { promisify } from 'node:util' +import { runBenchRouterTurn } from '../router-turn' import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' const execFileAsync = promisify(execFile) @@ -195,23 +196,20 @@ async function gradeAnswer( predicted: string, router: GraderRouter, ): Promise { - const res = await fetch(`${router.baseUrl}/chat/completions`, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` }, - body: JSON.stringify({ - model: router.model, + const turn = await runBenchRouterTurn( + { + routerBaseUrl: router.baseUrl, + routerKey: router.key, + profile: { + name: 'simpleqa-grader', + model: { provider: 'tangle-router', default: router.model }, + }, temperature: 0, - messages: [{ role: 'user', content: GRADER_PROMPT(question, gold, predicted) }], - }), - }) - if (!res.ok) { - throw new Error(`SimpleQA grader HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`) - } - const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - const content = body.choices?.[0]?.message?.content - if (typeof content !== 'string') { - throw new Error(`SimpleQA grader returned no message content: ${JSON.stringify(body).slice(0, 300)}`) - } + }, + GRADER_PROMPT(question, gold, predicted), + ) + const content = turn.finalText + if (!content) throw new Error('SimpleQA grader returned no message content') const fenced = content.match(/```(?:json)?\s*([\s\S]*?)```/) const raw = (fenced ? fenced[1] : content)?.trim() ?? '' let parsed: { grade?: unknown } diff --git a/bench/src/benchmarks/trata-hedge.ts b/bench/src/benchmarks/trata-hedge.ts index 5aff96bc..555f4d74 100644 --- a/bench/src/benchmarks/trata-hedge.ts +++ b/bench/src/benchmarks/trata-hedge.ts @@ -26,6 +26,7 @@ import { readdirSync, readFileSync, statSync } from 'node:fs' import { join } from 'node:path' +import { runBenchRouterTurn } from '../router-turn' import type { BenchmarkAdapter, BenchScore, BenchTask, LoadOptions } from './types' const DEFAULT_BENCH_ROOT = '/tmp/trata-hedge-bench' @@ -191,25 +192,26 @@ function parseJsonFallback(raw: string): unknown { async function callJudge(router: JudgeRouter, prompt: string, maxAttempts = 2): Promise { for (let i = 0; i < maxAttempts; i++) { - const res = await fetch(`${router.baseUrl}/chat/completions`, { - method: 'POST', - headers: { 'content-type': 'application/json', authorization: `Bearer ${router.key}` }, - body: JSON.stringify({ - model: router.model, - temperature: 0, - max_tokens: 16384, - messages: [{ role: 'user', content: prompt }], - }), - }) - if (!res.ok) { - if (i < maxAttempts - 1) continue - throw new Error(`Trata judge HTTP ${res.status}: ${(await res.text()).slice(0, 300)}`) + try { + const turn = await runBenchRouterTurn( + { + routerBaseUrl: router.baseUrl, + routerKey: router.key, + profile: { + name: 'trata-hedge-judge', + model: { provider: 'tangle-router', default: router.model }, + }, + temperature: 0, + maxTokens: Number(process.env.JUDGE_MAX_TOKENS ?? 16384), + }, + prompt, + ) + const content = turn.finalText + const parsed = parseJsonFallback(content) + if (parsed !== null) return parsed + } catch (error) { + if (i + 1 === maxAttempts) throw error } - const body = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - const content = body.choices?.[0]?.message?.content - if (typeof content !== 'string') continue - const parsed = parseJsonFallback(content) - if (parsed !== null) return parsed } return null } diff --git a/bench/src/clbench-context-gate.mts b/bench/src/clbench-context-gate.mts index ecf33784..9009f36c 100644 --- a/bench/src/clbench-context-gate.mts +++ b/bench/src/clbench-context-gate.mts @@ -36,7 +36,8 @@ import { execFileSync } from 'node:child_process' import { existsSync, readFileSync } from 'node:fs' import { composeStrategies } from './directives' import { type AttemptRecord, appendRunRecord, buildRunRecordFromAttempts } from './corpus' -import { type RouterConfig, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import type { RouterConfig } from '@tangle-network/agent-runtime/kernel' +import { runBenchRouterTurn } from './router-turn' import { selfConsistencySelect, verifierGroundedSelect } from './selector' import { type PairedLift, pairedLift, pool } from './stats.mts' @@ -162,8 +163,19 @@ async function judgeRubrics(cfg: RouterConfig, task: CtxTask, output: string): P // NOT throw — one bad grade would otherwise crash the whole N×K×2 run. graded=0 // marks it as judge-failed so it's distinguishable from a real 0/N rubric pass. try { - const res = await routerChatWithUsage(cfg, [{ role: 'user', content: judgePrompt(rubricsText, output) }], { temperature: 0 }) - return parseJudge(typeof res.content === 'string' ? res.content : '', task.rubrics.length) + const res = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'clbench-rubric-judge', + model: { provider: 'tangle-router', default: cfg.model }, + }, + temperature: 0, + }, + judgePrompt(rubricsText, output), + ) + return parseJudge(res.finalText, task.rubrics.length) } catch { return { fraction: 0, allPass: false, graded: 0 } } @@ -213,8 +225,21 @@ async function main(): Promise { } console.log(`\n▶ solving ${units.length} attempts (${tasks.length} tasks × ${k} shots × 2 arms) via router, conc=${solveConcurrency}`) const outputs = await pool(units, solveConcurrency, async (u) => { - const res = await routerChatWithUsage(workerCfg, u.messages, { temperature: Number(process.env.TEMPERATURE ?? '0.8') }) - return typeof res.content === 'string' ? res.content : '' + const system = u.messages.find((message) => message.role === 'system')?.content + const res = await runBenchRouterTurn( + { + routerBaseUrl: workerCfg.routerBaseUrl, + routerKey: workerCfg.routerKey, + profile: { + name: 'clbench-context-worker', + model: { provider: 'tangle-router', default: workerCfg.model }, + ...(system ? { prompt: { systemPrompt: system } } : {}), + }, + temperature: Number(process.env.TEMPERATURE ?? '0.8'), + }, + { messages: u.messages.filter((message) => message.role !== 'system') }, + ) + return res.finalText }) console.log(`▶ grading ${outputs.length} completions with the rubric judge (${judgeModel}), conc=${solveConcurrency}`) diff --git a/bench/src/david-attribution.mts b/bench/src/david-attribution.mts index 3911ade0..081ffb52 100644 --- a/bench/src/david-attribution.mts +++ b/bench/src/david-attribution.mts @@ -20,24 +20,40 @@ import { mkdtempSync, writeFileSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval' +import { runBenchRouterTurn } from './router-turn' -const KEY = process.env.TANGLE_API_KEY! +const KEY = process.env.TANGLE_API_KEY +if (!KEY) throw new Error('TANGLE_API_KEY required') const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant' const N = Number(process.env.N ?? 8) const T = Number(process.env.T ?? 5) const NTASKS = Number(process.env.NTASKS ?? 60) const CONC = Number(process.env.CONCURRENCY ?? 6) +const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 1000) +const LLM_TIMEOUT_MS = Number(process.env.LLM_TIMEOUT_MS ?? 60_000) -const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) async function chat(messages: { role: string; content: string }[], temp: number): Promise { - for (let a = 0; ; a++) { - try { - const r = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify({ model: DAVID, messages, temperature: temp, max_tokens: 1000 }), signal: AbortSignal.timeout(60_000) }) - if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(r.status)) { if (a >= 5) return ''; await sleep(700 * 2 ** a); continue } - if (!r.ok) return '' - return (((await r.json()) as { choices?: { message?: { content?: string } }[] }).choices?.[0]?.message?.content) ?? '' - } catch { if (a >= 5) return ''; await sleep(700 * 2 ** a) } + try { + const system = messages.find((message) => message.role === 'system')?.content + const turn = await runBenchRouterTurn( + { + routerBaseUrl: ROUTER, + routerKey: KEY, + profile: { + name: 'david-attribution-worker', + model: { provider: 'tangle-router', default: DAVID }, + ...(system ? { prompt: { systemPrompt: system } } : {}), + }, + temperature: temp, + maxTokens: MAX_TOKENS, + timeoutMs: LLM_TIMEOUT_MS, + }, + { messages: messages.filter((message) => message.role !== 'system') }, + ) + return turn.finalText + } catch { + return '' } } const exec = (f: string, a: string[], o: object) => new Promise((res) => execFile(f, a, { ...o, maxBuffer: 8e6 }, (e) => res((e as { code?: number } | null)?.code ?? (e ? 1 : 0)))) diff --git a/bench/src/david-goliath.mts b/bench/src/david-goliath.mts index 405d4206..95dbaac9 100644 --- a/bench/src/david-goliath.mts +++ b/bench/src/david-goliath.mts @@ -17,7 +17,7 @@ * generator punch above its solo weight — the standing "verification is live" claim * at its most dramatic. Paired McNemar on per-task discordant pairs for significance. * - * Run from cwd=bench: env DAVID=groq/llama-3.1-8b-instant GOLIATH=anthropic/claude-haiku-4-5-20251001 \ + * Run from cwd=bench: env DAVID=glm-5.2 GOLIATH=deepseek-v4-flash \ * N=8 T=5 NTASKS=164 REPS=2 node_modules/.bin/tsx src/david-goliath.mts */ import { execFile } from 'node:child_process' @@ -25,18 +25,22 @@ import { mkdtempSync, writeFileSync, rmSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { loadHumanEval, extractCode, type HumanEvalTask } from './benchmarks/humaneval' +import { runBenchRouterTurn } from './router-turn' const KEY = process.env.TANGLE_API_KEY if (!KEY) throw new Error('TANGLE_API_KEY required') +const ROUTER_KEY = KEY const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' const DAVID = process.env.DAVID ?? 'groq/llama-3.1-8b-instant' -const GOLIATH = process.env.GOLIATH ?? 'anthropic/claude-haiku-4-5-20251001' +const GOLIATH = process.env.GOLIATH ?? 'deepseek-v4-flash' const N = Number(process.env.N ?? 8) // David candidate solutions const T = Number(process.env.T ?? 5) // David generated tests const NTASKS = Number(process.env.NTASKS ?? 164) const REPS = Number(process.env.REPS ?? 2) const CONC = Number(process.env.CONCURRENCY ?? 6) const EXEC_TIMEOUT = Number(process.env.EXEC_TIMEOUT_MS ?? 6000) +const LLM_TIMEOUT = Number(process.env.LLM_TIMEOUT_MS ?? 60_000) +const MAX_TOKENS = Number(process.env.MAX_TOKENS ?? 1000) // Approx $/1M tokens (in,out) for cost accounting — the router does not price // every model inline, so use public rates; a cheap/frontier gap of ~20-30x is the @@ -56,18 +60,30 @@ const addU = (a: Usage, b: Usage) => { a.in += b.in; a.out += b.out } const usd = (m: string, u: Usage) => { const [pi, po] = priceOf(m); return (u.in * pi + u.out * po) / 1e6 } async function chat(model: string, messages: { role: string; content: string }[], temperature: number, usage: Usage): Promise { - for (let a = 0; ; a++) { - try { - const r = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify({ model, messages, temperature, max_tokens: 1000 }), signal: AbortSignal.timeout(60_000) }) - if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(r.status)) { if (a >= 5) return ''; await sleep(700 * 2 ** a); continue } - if (!r.ok) return '' - const j = (await r.json()) as { choices?: { message?: { content?: string } }[]; usage?: { prompt_tokens?: number; completion_tokens?: number } } - addU(usage, { in: j.usage?.prompt_tokens ?? 0, out: j.usage?.completion_tokens ?? 0 }) - return j.choices?.[0]?.message?.content ?? '' - } catch { if (a >= 5) return ''; await sleep(700 * 2 ** a) } + try { + const system = messages.find((message) => message.role === 'system')?.content + const result = await runBenchRouterTurn( + { + routerBaseUrl: ROUTER, + routerKey: ROUTER_KEY, + profile: { + name: 'david-goliath-worker', + model: { provider: 'tangle-router', default: model }, + ...(system ? { prompt: { systemPrompt: system } } : {}), + }, + temperature, + maxTokens: MAX_TOKENS, + timeoutMs: LLM_TIMEOUT, + }, + { messages: messages.filter((message) => message.role !== 'system') }, + ) + if (result.usage.tokensKnown === false) throw new Error('provider omitted token usage') + addU(usage, { in: result.usage.input, out: result.usage.output }) + return result.finalText + } catch { + return '' } } -const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) const exec = (file: string, args: string[], o: object) => new Promise<{ code: number; stdout: string }>((res) => execFile(file, args, { ...o, maxBuffer: 8 * 1024 * 1024 }, (e, stdout) => res({ code: (e as { code?: number } | null)?.code ?? (e ? 1 : 0), stdout: String(stdout) }))) async function runPy(program: string): Promise<{ ok: boolean }> { const d = mkdtempSync(join(tmpdir(), 'dg-')) diff --git a/bench/src/gate.ts b/bench/src/gate.ts index ad9afdd6..3ab31075 100644 --- a/bench/src/gate.ts +++ b/bench/src/gate.ts @@ -32,28 +32,26 @@ import type { AgentSpec, Budget, CombinatorShape, - DefaultVerdict, EqualKArm, EqualKVerdict, ExecutorContext, ExecutorRegistry, Executor, ExecutorFactory, - ExecutorResult, Outcome, Persona, - Runtime, Spend, SupervisedResult, TrajectoryReport, } from '@tangle-network/agent-runtime/kernel' import { definePersona, - routerChatWithUsage, + createExecutor, equalKOnCost, fanout, InMemoryResultBlobStore, InMemorySpawnJournal, + mapExecutorResult, runPersonified, trajectoryReport, } from '@tangle-network/agent-runtime/kernel' @@ -114,58 +112,29 @@ function extractArtifact(adapter: BenchmarkAdapter, content: string): string { * a judge throw rejects the leaf (the scope types it into a `down` settlement — never a silent 0). */ export function benchSolveLeaf(opts: BenchSolverOptions, spec: AgentSpec, ctx: ExecutorContext): Executor { - const controller = new AbortController() - const abortIfSignalled = () => { - if (ctx.signal.aborted) controller.abort() - } - abortIfSignalled() - if (!ctx.signal.aborted) ctx.signal.addEventListener('abort', abortIfSignalled, { once: true }) - - let artifact: ExecutorResult | undefined - - return { - runtime: 'bench-router' as Runtime, - async execute(task, signal): Promise> { + const inner = createExecutor({ + backend: 'router', + routerBaseUrl: opts.routerBaseUrl, + routerKey: opts.routerKey, + model: opts.model, + temperature: opts.temperature ?? 0.7, + })(spec, ctx) + return mapExecutorResult(inner, async (result, task) => { const t = task as SolveTask - const system = spec.profile.prompt?.systemPrompt - const messages = [ - ...(typeof system === 'string' && system.length > 0 - ? [{ role: 'system', content: system }] - : []), - { role: 'user', content: t.prompt }, - ] - const started = Date.now() - const linked = linkSignals(signal, controller.signal) - const chat = await routerChatWithUsage( - { routerBaseUrl: opts.routerBaseUrl, routerKey: opts.routerKey, model: opts.model }, - messages, - { temperature: opts.temperature ?? 0.7, ...(linked ? { signal: linked } : {}) }, - ) - const candidate = extractArtifact(opts.adapter, chat.content) + const raw = result.out as { content?: unknown } + const content = typeof raw?.content === 'string' ? raw.content : '' + const candidate = extractArtifact(opts.adapter, content) const score = await opts.adapter.judge(t.instance, candidate) - const verdict: DefaultVerdict = { - valid: score.resolved, - score: score.score, - ...(score.detail ? { notes: score.detail } : {}), - } - const spent: Spend = { - iterations: 1, - tokens: chat.usage ? { input: chat.usage.input, output: chat.usage.output } : { input: 0, output: 0 }, - usd: chat.costUsd ?? 0, - ms: Date.now() - started, + return { + outRef: fnv('bench', { id: t.instance.id, candidate }), + out: candidate, + verdict: { + valid: score.resolved, + score: score.score, + ...(score.detail ? { notes: score.detail } : {}), + }, } - artifact = { outRef: fnv('bench', { id: t.instance.id, candidate }), out: candidate, verdict, spent } - return artifact - }, - teardown(): Promise<{ destroyed: boolean }> { - controller.abort() - return Promise.resolve({ destroyed: true }) - }, - resultArtifact() { - if (!artifact) throw new Error('benchSolveLeaf: resultArtifact() read before execute()') - return artifact - }, - } + }) } /** @@ -447,15 +416,3 @@ export async function runGate(opts: RunGateOptions): Promise { } /** Link two abort signals into one that fires when either does; `undefined` when neither is set. */ -function linkSignals(a: AbortSignal, b: AbortSignal): AbortSignal | undefined { - if (a.aborted || b.aborted) { - const c = new AbortController() - c.abort() - return c.signal - } - const c = new AbortController() - const onAbort = () => c.abort() - a.addEventListener('abort', onAbort, { once: true }) - b.addEventListener('abort', onAbort, { once: true }) - return c.signal -} diff --git a/bench/src/generate-eval/certify.ts b/bench/src/generate-eval/certify.ts index e1186dfa..7dfbcd37 100644 --- a/bench/src/generate-eval/certify.ts +++ b/bench/src/generate-eval/certify.ts @@ -24,8 +24,8 @@ import { execSync } from 'node:child_process' import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { dirname, join, resolve } from 'node:path' -import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' import { scoreTask, taskToPrompt } from '../search-bench/tasks' +import { runBenchRouterTurn } from '../router-turn' import { type EvalCertification, type GeneratedEval, generatedEvalSchemaVersion, parseCandidate } from './schema' const certifierId = 'agent-runtime/generate-eval@1' @@ -121,10 +121,18 @@ export async function discriminationGate( const apiKey = opts.gateApiKey ?? process.env.EVAL_GATE_API_KEY ?? process.env.TANGLE_API_KEY const model = opts.gateModel ?? process.env.EVAL_GATE_MODEL ?? 'deepseek-v4-flash' if (!apiKey) throw new Error('discrimination gate needs EVAL_GATE_API_KEY (or TANGLE_API_KEY)') - const res = await routerChatWithUsage({ routerBaseUrl: baseUrl, routerKey: apiKey, model }, [ - { role: 'user', content: taskToPrompt(candidate) }, - ]) - const { score, reasons } = scoreTask(candidate, res.content) + const res = await runBenchRouterTurn( + { + routerBaseUrl: baseUrl, + routerKey: apiKey, + profile: { + name: 'generated-eval-parametric-check', + model: { provider: 'tangle-router', default: model }, + }, + }, + taskToPrompt(candidate), + ) + const { score, reasons } = scoreTask(candidate, res.finalText) return score === 0 ? { passed: true, detail: `parametric ${model} failed as required (${reasons.join('; ')})` } : { passed: false, detail: `parametric ${model} SOLVED the task from memory — not search-discriminating` } diff --git a/bench/src/hev-eval.mts b/bench/src/hev-eval.mts index 07d6d50d..dc018a89 100644 --- a/bench/src/hev-eval.mts +++ b/bench/src/hev-eval.mts @@ -9,19 +9,38 @@ */ import { readFileSync } from 'node:fs' import { extractCode, loadHumanEval, runChecker, type HumanEvalTask } from './benchmarks/humaneval' +import { runBenchRouterTurn } from './router-turn' const SEED_INSTRUCTION = 'Complete the following Python function. Output the COMPLETE function definition (signature, docstring optional, body) inside a single ```python code block. Include any imports the function needs. Do not write tests or example calls.' -async function complete(base: string, key: string, model: string, prompt: string, maxTokens: number): Promise { - const res = await fetch(`${base}/chat/completions`, { - method: 'POST', - headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ model, max_tokens: maxTokens, temperature: 0.2, messages: [{ role: 'user', content: prompt }] }), - }) - if (!res.ok) return '' - const d = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - return d.choices?.[0]?.message?.content ?? '' +async function complete( + base: string, + key: string, + model: string, + instruction: string, + prompt: string, + maxTokens: number, +): Promise { + try { + const turn = await runBenchRouterTurn( + { + routerBaseUrl: base, + routerKey: key, + profile: { + name: 'humaneval-worker', + model: { provider: 'tangle-router', default: model }, + prompt: { systemPrompt: instruction }, + }, + temperature: 0.2, + maxTokens, + }, + prompt, + ) + return turn.finalText + } catch { + return '' + } } async function main(): Promise { @@ -55,7 +74,14 @@ async function main(): Promise { const t = tasks[i] i += 1 if (!t) continue - const reply = await complete(base, apiKey, model, `${instruction}\n\n\`\`\`python\n${t.prompt}\`\`\``, maxTokens) + const reply = await complete( + base, + apiKey, + model, + instruction, + `\`\`\`python\n${t.prompt}\`\`\``, + maxTokens, + ) const { pass: p } = await runChecker(t, extractCode(reply)) if (p === 1) pass += 1 else fails.push(t.taskId) diff --git a/bench/src/hev-improve.mts b/bench/src/hev-improve.mts index 081c176b..14ce3b87 100644 --- a/bench/src/hev-improve.mts +++ b/bench/src/hev-improve.mts @@ -27,6 +27,7 @@ import { officialOptimizerModel, requiredTokenPricing, } from './official-optimizer-config.mjs' +import { runBenchRouterTurn } from './router-turn' // The SEED instruction GEPA evolves. Byte-identical to humaneval.ts basePrompt's // solveInstruction so the baseline arm reproduces the plain-prompt denominator. @@ -39,21 +40,28 @@ interface Completion { tokOut: number } -async function complete(base: string, key: string, model: string, prompt: string, maxTokens: number): Promise { - const res = await fetch(`${base}/chat/completions`, { - method: 'POST', - headers: { Authorization: `Bearer ${key}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ model, max_tokens: maxTokens, temperature: 0.2, messages: [{ role: 'user', content: prompt }] }), - }) - if (!res.ok) throw new Error(`completion HTTP ${res.status}: ${(await res.text()).slice(0, 200)}`) - const d = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }> - usage?: { prompt_tokens?: number; completion_tokens?: number } +async function complete( + base: string, + key: string, + profile: AgentProfile, + prompt: string, + maxTokens: number, +): Promise { + const result = await runBenchRouterTurn( + { + routerBaseUrl: base, + routerKey: key, + profile, + temperature: 0.2, + maxTokens, + }, + prompt, + ) + return { + text: result.finalText, + tokIn: result.usage.input, + tokOut: result.usage.output, } - const text = d.choices?.[0]?.message?.content ?? '' - const tokIn = d.usage?.prompt_tokens ?? 0 - const tokOut = d.usage?.completion_tokens ?? 0 - return { text, tokIn, tokOut } } async function main(): Promise { @@ -115,13 +123,18 @@ async function main(): Promise { if (instr === undefined) throw new Error('agent: candidate profile has no system prompt') const t = byId.get(scenario.id) if (!t) throw new Error(`agent: unknown scenario ${scenario.id}`) - const prompt = `${instr}\n\n\`\`\`python\n${t.prompt}\`\`\`` + const prompt = `\`\`\`python\n${t.prompt}\`\`\`` + const executionProfile: AgentProfile = { + ...candidate, + name: candidate.name ?? 'humaneval-improvement-worker', + model: { ...candidate.model, provider: 'tangle-router', default: workerModel }, + } const t0 = Date.now() const paid = await ctx.cost.runPaidCall({ channel: 'agent', actor: 'humaneval-worker', model: workerModel, - execute: () => complete(base, key, workerModel, prompt, workerMaxTokens), + execute: () => complete(base, key, executionProfile, prompt, workerMaxTokens), receipt: (result) => { const usageUnknown = result.tokIn === 0 && result.tokOut === 0 return { diff --git a/bench/src/hev-structural.mts b/bench/src/hev-structural.mts index ed2d1585..2f0fe743 100644 --- a/bench/src/hev-structural.mts +++ b/bench/src/hev-structural.mts @@ -46,6 +46,7 @@ import { tmpdir } from 'node:os' import { join } from 'node:path' import { type HumanEvalTask, extractCode, loadHumanEval } from './benchmarks/humaneval' import { composeStrategies } from './directives' +import { runBenchRouterTurn } from './router-turn' import { type PairedLift, pairedLift, pool } from './stats.mts' const dockerImage = 'python:3.12-slim' @@ -345,35 +346,39 @@ async function complete(cfg: ClientCfg, messages: Array<{ role: string; content: let lastErr = '' for (let attempt = 1; attempt <= 4; attempt += 1) { if (attempt > 1) await new Promise((r) => setTimeout(r, 2000 * 2 ** attempt)) - const ctl = new AbortController() - const timer = setTimeout(() => ctl.abort(), 240_000) try { - const res = await fetch(`${cfg.base}/chat/completions`, { - method: 'POST', - headers: { Authorization: `Bearer ${cfg.key}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ model: cfg.model, max_tokens: cfg.maxTokens, temperature: cfg.temperature, messages }), - signal: ctl.signal, - }) - if (!res.ok) { - lastErr = `HTTP ${res.status}: ${(await res.text()).slice(0, 200)}` - continue - } - const d = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }> - usage?: { prompt_tokens?: number; completion_tokens?: number } - } - const content = d.choices?.[0]?.message?.content ?? '' + const system = messages.find((message) => message.role === 'system')?.content + const result = await runBenchRouterTurn( + { + routerBaseUrl: cfg.base, + routerKey: cfg.key, + profile: { + name: 'humaneval-structural-worker', + model: { provider: 'tangle-router', default: cfg.model }, + ...(system ? { prompt: { systemPrompt: system } } : {}), + }, + temperature: cfg.temperature, + maxTokens: cfg.maxTokens, + timeoutMs: Number(process.env.LLM_TIMEOUT_MS ?? 240_000), + }, + { messages: messages.filter((message) => message.role !== 'system') }, + ) + if (result.usage.tokensKnown === false) throw new Error('provider omitted token usage') + const content = result.finalText // Reasoning models starve `content` when reasoning exhausts max_tokens — an // empty reply is a transient fault to retry, not a candidate to score. if (content.trim() === '') { lastErr = 'empty content' continue } - return { content, attempts: attempt, tokensIn: d.usage?.prompt_tokens ?? 0, tokensOut: d.usage?.completion_tokens ?? 0 } + return { + content, + attempts: attempt, + tokensIn: result.usage.input, + tokensOut: result.usage.output, + } } catch (e) { lastErr = e instanceof Error ? e.message : String(e) - } finally { - clearTimeout(timer) } } throw new Error(`completion failed after retries: ${lastErr}`) diff --git a/bench/src/humaneval-gate.mts b/bench/src/humaneval-gate.mts index 80b852c3..9222fafc 100644 --- a/bench/src/humaneval-gate.mts +++ b/bench/src/humaneval-gate.mts @@ -43,7 +43,8 @@ import { composeStrategies } from './directives' import { basePrompt, type CheckResult, extractCode, type HumanEvalTask, loadHumanEval, runChecker } from './benchmarks/humaneval' -import { type RouterConfig, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import type { RouterConfig } from '@tangle-network/agent-runtime/kernel' +import { runBenchRouterTurn } from './router-turn' import { selfConsistencySelect, verifierGroundedSelect } from './selector' import { type PairedLift, pairedLift, pool } from './stats.mts' @@ -110,10 +111,19 @@ async function main(): Promise { console.log(`\n▶ solving ${units.length} attempts (${tasks.length} tasks × ${k} shots × 2 arms) via router, conc=${solveConcurrency}`) const codes = await pool(units, solveConcurrency, async (u) => { - const res = await routerChatWithUsage(cfg, [{ role: 'user', content: u.prompt }], { - temperature: Number(process.env.TEMPERATURE ?? '0.8'), - }) - return extractCode(typeof res.content === 'string' ? res.content : '') + const res = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'humaneval-gate-worker', + model: { provider: 'tangle-router', default: cfg.model }, + }, + temperature: Number(process.env.TEMPERATURE ?? '0.8'), + }, + u.prompt, + ) + return extractCode(res.finalText) }) console.log(`▶ running ${codes.length} candidates through the Docker deployable checker, conc=${dockerConcurrency}`) diff --git a/bench/src/humaneval-object-ablation.mts b/bench/src/humaneval-object-ablation.mts index 76510256..03b47d6b 100644 --- a/bench/src/humaneval-object-ablation.mts +++ b/bench/src/humaneval-object-ablation.mts @@ -19,7 +19,7 @@ * container with a hard timeout — seconds per task. Paired * McNemar over the per-task pass/fail difference gives the significance. * - * Run from cwd=bench: env WORKER_MODEL=google/gemini-2.5-flash-lite N=60 K=3 \ + * Run from cwd=bench: env WORKER_MODEL=deepseek-v4-flash N=60 K=3 \ * REPS=2 node_modules/.bin/tsx src/humaneval-object-ablation.mts */ import { @@ -28,11 +28,13 @@ import { runPythonProgram, type HumanEvalTask, } from './benchmarks/humaneval' +import { runBenchRouterTurn } from './router-turn' const ROUTER = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' const KEY = process.env.TANGLE_API_KEY if (!KEY) throw new Error('TANGLE_API_KEY required') -const MODEL = process.env.WORKER_MODEL ?? 'google/gemini-2.5-flash-lite' +const ROUTER_KEY = KEY +const MODEL = process.env.WORKER_MODEL ?? 'deepseek-v4-flash' const N = Number(process.env.N ?? 60) const OFFSET = Number(process.env.OFFSET ?? 0) const K = Number(process.env.K ?? 3) // rounds/budget per task (equal for both arms) @@ -40,23 +42,49 @@ const REPS = Number(process.env.REPS ?? 2) const CONC = Number(process.env.CONCURRENCY ?? 6) const EXEC_TIMEOUT = Number(process.env.EXEC_TIMEOUT_MS ?? 8000) -interface ChatMsg { role: string; content: string; tool_calls?: unknown; tool_call_id?: string; name?: string } +interface ChatMsg extends Record { role: string; content: string; tool_calls?: unknown; tool_call_id?: string; name?: string } interface Tool { type: 'function'; function: { name: string; description: string; parameters: unknown } } async function router(messages: ChatMsg[], tools?: Tool[]): Promise<{ content: string; toolCalls: { id: string; name: string; args: Record }[] }> { - const body: Record = { model: MODEL, messages, temperature: 0.4 } - if (tools) { body.tools = tools; body.tool_choice = 'auto' } for (let attempt = 0; ; attempt++) { - let res: Response try { - res = await fetch(`${ROUTER}/chat/completions`, { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEY}` }, body: JSON.stringify(body), signal: AbortSignal.timeout(60_000) }) - } catch (e) { if (attempt >= 5) throw e; await sleep(800 * 2 ** attempt); continue } - if ([408, 429, 500, 502, 503, 504, 520, 522, 524].includes(res.status)) { if (attempt >= 5) throw new Error(`router ${res.status} exhausted`); await sleep(800 * 2 ** attempt); continue } - if (!res.ok) throw new Error(`router ${res.status}: ${(await res.text()).slice(0, 200)}`) - const j = (await res.json()) as { choices?: { message?: { content?: string; tool_calls?: { id: string; function: { name: string; arguments: string } }[] } }[] } - const m = j.choices?.[0]?.message - const toolCalls = (m?.tool_calls ?? []).map((t) => { let args: Record = {}; try { args = JSON.parse(t.function.arguments) } catch { /* keep {} */ } return { id: t.id, name: t.function.name, args } }) - return { content: m?.content ?? '', toolCalls } + const system = messages.find((message) => message.role === 'system')?.content + const result = await runBenchRouterTurn( + { + routerBaseUrl: ROUTER, + routerKey: ROUTER_KEY, + profile: { + name: 'humaneval-object-ablation-worker', + model: { provider: 'tangle-router', default: MODEL }, + ...(system ? { prompt: { systemPrompt: system } } : {}), + ...(tools + ? { tools: Object.fromEntries(tools.map((tool) => [tool.function.name, true])) } + : {}), + }, + temperature: 0.4, + ...(tools ? { tools, toolChoice: 'auto' as const } : {}), + timeoutMs: Number(process.env.LLM_TIMEOUT_MS ?? 60_000), + }, + { messages: messages.filter((message) => message.role !== 'system') }, + ) + const toolCalls = result.toolCalls.map((call) => { + let args: Record = {} + try { + args = JSON.parse(call.arguments) as Record + } catch { + // Keep the empty argument object; the tool returns a useful error. + } + return { id: call.id, name: call.name, args } + }) + return { content: result.finalText, toolCalls } + } catch (error) { + const message = error instanceof Error ? error.message : String(error) + const status = Number(/router (\d+)/.exec(message)?.[1]) + const transient = + !Number.isFinite(status) || [408, 429, 500, 502, 503, 504, 520, 522, 524].includes(status) + if (!transient || attempt >= 5) throw error + await sleep(800 * 2 ** attempt) + } } } const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms)) diff --git a/bench/src/humaneval-repair-gate.mts b/bench/src/humaneval-repair-gate.mts index 3ebbc10b..dd74c3eb 100644 --- a/bench/src/humaneval-repair-gate.mts +++ b/bench/src/humaneval-repair-gate.mts @@ -20,9 +20,16 @@ * tsx src/humaneval-repair-gate.mts */ import { type HumanEvalTask, basePrompt, extractCode, loadHumanEval, runChecker } from './benchmarks/humaneval' -import { type RouterConfig, type ToolSpec, routerChatWithUsage, routerToolLoop } from '@tangle-network/agent-runtime/kernel' +import { + collectAgentTurn, + createExecutor, + streamAgentTurn, + type RouterConfig, + type ToolSpec, +} from '@tangle-network/agent-runtime/kernel' import { verifierGroundedSelect } from './selector' import { type PairedLift, pairedLift, pool } from './stats.mts' +import { runBenchRouterTurn } from './router-turn' function must(name: string): string { const v = process.env[name] @@ -53,12 +60,19 @@ const repairSystem = [ /** repair@K: one worker, up to K inference turns, steering on real test failures. */ async function repairAttempt(cfg: RouterConfig, task: HumanEvalTask, k: number): Promise { let lastTested = '' - const r = await routerToolLoop( - cfg, - repairSystem, - basePrompt(task), - [runTestsTool], - async (name, args) => { + const profile = { + name: 'humaneval-repair-worker', + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt: repairSystem }, + tools: { run_tests: true }, + } + const factory = createExecutor({ + backend: 'router-tools', + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + model: cfg.model, + tools: [runTestsTool], + executeToolCall: async (name, args) => { if (name !== 'run_tests') return `error: unknown tool ${name}` const code = extractCode(String(args.code ?? '')) lastTested = code @@ -67,11 +81,18 @@ async function repairAttempt(cfg: RouterConfig, task: HumanEvalTask, k: number): ? 'ALL TESTS PASSED. Reply with the final function now; do not call run_tests again.' : `TESTS FAILED:\n${res.detail ?? 'no output'}\n\nFix the function and call run_tests again.` }, - { maxTurns: k, temperature: 0.3 }, + maxTurns: k, + temperature: 0.3, + }) + const r = await collectAgentTurn( + streamAgentTurn({ kind: 'executor', factory, profile }, basePrompt(task)), ) + if (r.status !== 'completed') { + throw new Error(r.error?.message ?? `repair turn ended with status ${r.status}`) + } // Judge the model's final answer; fall back to the last code it tested (it may // report "done" without re-pasting the passing function). - const finalCode = extractCode(r.final) || lastTested + const finalCode = extractCode(r.finalText) || lastTested if (!finalCode) return 0 return (await runChecker(task, finalCode)).pass } @@ -81,8 +102,19 @@ async function blindAttempts(cfg: RouterConfig, task: HumanEvalTask, k: number): const base = basePrompt(task) const passes: number[] = [] for (let i = 0; i < k; i += 1) { - const res = await routerChatWithUsage(cfg, [{ role: 'user', content: base }], { temperature: 0.8 }) - passes.push((await runChecker(task, extractCode(res.content))).pass) + const res = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'humaneval-blind-worker', + model: { provider: 'tangle-router', default: cfg.model }, + }, + temperature: 0.8, + }, + base, + ) + passes.push((await runChecker(task, extractCode(res.finalText))).pass) } return passes } diff --git a/bench/src/mbpp-structural.mts b/bench/src/mbpp-structural.mts index 576c945b..a09d0c72 100644 --- a/bench/src/mbpp-structural.mts +++ b/bench/src/mbpp-structural.mts @@ -32,6 +32,7 @@ import { tmpdir } from 'node:os' import { join, resolve } from 'node:path' import { fileURLToPath } from 'node:url' import { extractCode } from './benchmarks/humaneval' +import { runBenchRouterTurn } from './router-turn' import { type PairedLift, pairedLift, pool } from './stats.mts' const dockerImage = 'python:3.12-slim' @@ -326,33 +327,37 @@ async function complete(cfg: ClientCfg, messages: Array<{ role: string; content: let lastErr = '' for (let attempt = 1; attempt <= 4; attempt += 1) { if (attempt > 1) await new Promise((r) => setTimeout(r, 2000 * 2 ** attempt)) - const ctl = new AbortController() - const timer = setTimeout(() => ctl.abort(), 240_000) try { - const res = await fetch(`${cfg.base}/chat/completions`, { - method: 'POST', - headers: { Authorization: `Bearer ${cfg.key}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ model: cfg.model, max_tokens: cfg.maxTokens, temperature: cfg.temperature, messages }), - signal: ctl.signal, - }) - if (!res.ok) { - lastErr = `HTTP ${res.status}: ${(await res.text()).slice(0, 200)}` - continue - } - const d = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }> - usage?: { prompt_tokens?: number; completion_tokens?: number } - } - const content = d.choices?.[0]?.message?.content ?? '' + const system = messages.find((message) => message.role === 'system')?.content + const result = await runBenchRouterTurn( + { + routerBaseUrl: cfg.base, + routerKey: cfg.key, + profile: { + name: 'mbpp-structural-worker', + model: { provider: 'tangle-router', default: cfg.model }, + ...(system ? { prompt: { systemPrompt: system } } : {}), + }, + temperature: cfg.temperature, + maxTokens: cfg.maxTokens, + timeoutMs: Number(process.env.LLM_TIMEOUT_MS ?? 240_000), + }, + { messages: messages.filter((message) => message.role !== 'system') }, + ) + if (result.usage.tokensKnown === false) throw new Error('provider omitted token usage') + const content = result.finalText if (content.trim() === '') { lastErr = 'empty content' continue } - return { content, attempts: attempt, tokensIn: d.usage?.prompt_tokens ?? 0, tokensOut: d.usage?.completion_tokens ?? 0 } + return { + content, + attempts: attempt, + tokensIn: result.usage.input, + tokensOut: result.usage.output, + } } catch (e) { lastErr = e instanceof Error ? e.message : String(e) - } finally { - clearTimeout(timer) } } throw new Error(`completion failed after retries: ${lastErr}`) diff --git a/bench/src/mcp-mount-probe.mts b/bench/src/mcp-mount-probe.mts index 699e1408..81f7a148 100644 --- a/bench/src/mcp-mount-probe.mts +++ b/bench/src/mcp-mount-probe.mts @@ -14,6 +14,8 @@ import { type Agent, type AgentProfile, type AgentSpec, + collectAgentTurn, + createExecutor, createExecutorRegistry, createSupervisor, type Executor, @@ -21,6 +23,7 @@ import { InMemoryResultBlobStore, InMemorySpawnJournal, type Scope, + streamAgentTurn, type UsageEvent, } from '../../src/runtime/index' import { serveCoordinationMcp } from '../../src/runtime/supervise/coordination-mcp' @@ -53,18 +56,27 @@ function deliveringLeaf(name: string, out: unknown): Agent { } async function bridgeChat(messages: Array<{ role: string; content: string }>, mcpUrl: string): Promise { - const r = await fetch(`${BRIDGE.replace(/\/$/, '')}/chat/completions`, { - method: 'POST', - headers: { authorization: `Bearer ${BEARER}`, 'content-type': 'application/json' }, - body: JSON.stringify({ - model: MODEL, - messages, - mcp: { mcpServers: { coordination: { type: 'http', url: mcpUrl } } }, - }), + if (!BEARER) throw new Error('TANGLE_API_KEY is required') + const profile: AgentProfile = { + name: 'mcp-mount-probe-supervisor', + model: { default: MODEL }, + mcp: { coordination: { transport: 'http', url: mcpUrl } }, + } + const factory = createExecutor({ + backend: 'bridge', + bridgeUrl: BRIDGE.replace(/\/v1\/?$/u, ''), + bridgeBearer: BEARER, }) - if (!r.ok) return `(bridge HTTP ${r.status}: ${(await r.text()).slice(0, 200)})` - const j = (await r.json()) as { choices?: Array<{ message?: { content?: string } }> } - return j.choices?.[0]?.message?.content ?? '' + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile }, + messages.map((message) => message.content).join('\n\n'), + ), + ) + if (turn.status !== 'completed') { + throw new Error(turn.error?.message ?? `bridge turn ended with ${turn.status}`) + } + return turn.finalText } async function main(): Promise { diff --git a/bench/src/research-shot.ts b/bench/src/research-shot.ts index 88183376..f57dd708 100644 --- a/bench/src/research-shot.ts +++ b/bench/src/research-shot.ts @@ -1,7 +1,7 @@ /** * One research rollout as a reusable primitive: 2-step RAG — (1) provider-pinned web * search via the router's proven `/v1/search?provider=` + `web_fetch` of the top-K - * result pages, (2) answer with that evidence via `routerChatWithUsage` (no tools on the + * result pages, (2) answer with that evidence through Runtime's profile-bound turn (no tools on the * answer call → `content` always present, so a search arm differs from the parametric * control ONLY by the evidence). Pure router HTTP (bearer `TANGLE_API_KEY`). * @@ -10,7 +10,7 @@ * — the only difference is who drives the rounds (a flat best-of-k pool vs the real * `runAgentRounds` kernel with analyst steering). */ -import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import { runBenchRouterTurn } from './router-turn' export interface ShotCfg { model: string @@ -109,15 +109,21 @@ export async function runResearchShot(prompt: string, taskId: string, attempt: n : 'Answer from your own knowledge. ') + 'If you are not fully certain, still COMMIT to your single best estimate — never refuse, defer, or reply with a question.' const userContent = useSearch && context ? `${prompt}\n\n=== WEB SEARCH RESULTS (provider: ${cfg.search}) ===\n${context}` : prompt - const { content } = await routerChatWithUsage( - { routerBaseUrl: cfg.routerBaseUrl, routerKey: cfg.routerKey, model: cfg.model }, - [ - { role: 'system', content: commit }, - { role: 'user', content: userContent }, - ], - { temperature: cfg.temperature, ...(cfg.timeoutMs ? { signal: AbortSignal.timeout(cfg.timeoutMs) } : {}) }, + const turn = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'research-shot-answerer', + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt: commit }, + }, + temperature: cfg.temperature, + ...(cfg.timeoutMs ? { timeoutMs: cfg.timeoutMs } : {}), + }, + userContent, ) - const answer = content.trim() + const answer = turn.finalText.trim() const ok = answer.length > 0 return { taskId, attempt, answer, ok, searches, wallMs: Date.now() - startedAt, ...(ok ? {} : { detail: `empty answer (searches=${searches})` }) } } catch (err) { diff --git a/bench/src/router-turn.ts b/bench/src/router-turn.ts new file mode 100644 index 00000000..f31f5b4c --- /dev/null +++ b/bench/src/router-turn.ts @@ -0,0 +1,64 @@ +import type { AgentProfile, ReasoningEffort } from '@tangle-network/agent-interface' +import { + collectAgentTurn, + createExecutor, + streamAgentTurn, + type CollectedAgentTurn, + type ToolSpec, +} from '@tangle-network/agent-runtime/kernel' + +export interface BenchRouterTurnConfig { + routerBaseUrl: string + routerKey: string + profile: AgentProfile + temperature?: number + maxTokens?: number + seed?: number + reasoningEffort?: ReasoningEffort + extraBody?: Readonly> + tools?: ReadonlyArray + toolChoice?: 'auto' | 'required' | 'none' + timeoutMs?: number + signal?: AbortSignal +} + +/** + * The benchmark-side entry to Runtime's canonical one-turn path. + * It is only an ergonomic composition: Runtime still parses the exact profile, + * materializes the executor, records identity/usage/result events, and refuses + * profile axes the direct Router backend cannot carry. + */ +export async function runBenchRouterTurn( + config: BenchRouterTurnConfig, + input: string | { readonly messages: ReadonlyArray>> }, +): Promise { + if (!config.profile.model?.default) { + throw new Error('runBenchRouterTurn: profile.model.default is required') + } + const factory = createExecutor({ + backend: 'router', + routerBaseUrl: config.routerBaseUrl, + routerKey: config.routerKey, + ...(config.temperature !== undefined ? { temperature: config.temperature } : {}), + ...(config.maxTokens !== undefined ? { maxTokens: config.maxTokens } : {}), + ...(config.seed !== undefined ? { seed: config.seed } : {}), + ...(config.reasoningEffort ? { reasoningEffort: config.reasoningEffort } : {}), + ...(config.extraBody ? { extraBody: config.extraBody } : {}), + ...(config.tools ? { tools: config.tools } : {}), + ...(config.toolChoice ? { toolChoice: config.toolChoice } : {}), + }) + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile: config.profile }, + input, + { + ...(config.timeoutMs === undefined ? {} : { timeoutMs: config.timeoutMs }), + ...(config.signal ? { signal: config.signal } : {}), + }, + ), + ) + if (turn.status !== 'completed') { + throw new Error(turn.error?.message ?? `Router turn ended with status ${turn.status}`) + } + return turn +} diff --git a/bench/src/sandbox-run.ts b/bench/src/sandbox-run.ts index d665f6bc..7716a3b4 100644 --- a/bench/src/sandbox-run.ts +++ b/bench/src/sandbox-run.ts @@ -13,11 +13,11 @@ import { type AgentProfile, type AgentRunSpec, type OutputAdapter, - routerChatWithUsage, } from '@tangle-network/agent-runtime/kernel' // `BackendType` is the sandbox SDK's harness union and its canonical home. agent-runtime consumes // it from there too; it is not re-exported from the kernel barrel. import type { BackendType } from '@tangle-network/sandbox' +import { runBenchRouterTurn } from './router-turn' /** Parse the agent's final answer from the event stream (harness-agnostic). * The default deliverable; a benchmark whose artifact is a file overrides via @@ -63,18 +63,21 @@ export const llmAnalyst = (cfg: { routerBaseUrl: string; routerKey: string; mode .map((e) => (typeof e === 'string' ? e : JSON.stringify(e))) .join('\n') .slice(-2000) - const { content } = await routerChatWithUsage(cfg, [ + const systemPrompt = + "You review an AI agent's previous attempt at a task. From the task, the attempt's output, and its execution trace ALONE, judge whether it correctly and completely solved the task. If you find a specific fault — a wrong value, a guessed API signature, a missing step, a misread requirement — name it and give the concrete correction in 1-3 sentences. Reply exactly 'no change needed' if the attempt looks correct and complete." + const turn = await runBenchRouterTurn( { - role: 'system', - content: - "You review an AI agent's previous attempt at a task. From the task, the attempt's output, and its execution trace ALONE, judge whether it correctly and completely solved the task. If you find a specific fault — a wrong value, a guessed API signature, a missing step, a misread requirement — name it and give the concrete correction in 1-3 sentences. Reply exactly 'no change needed' if the attempt looks correct and complete.", + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'sandbox-run-analyst', + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt }, + }, }, - { - role: 'user', - content: `Task:\n${task ?? '(task unavailable)'}\n\nPrevious answer:\n${last?.output ?? '(none)'}\n\nTrace tail:\n${traceTail}`, - }, - ]) - return content + `Task:\n${task ?? '(task unavailable)'}\n\nPrevious answer:\n${last?.output ?? '(none)'}\n\nTrace tail:\n${traceTail}`, + ) + return turn.finalText } /** Cost-dial backend = the SDK's canonical `BackendType` (single source of truth; no local diff --git a/bench/src/search-bench/parametric-check.mts b/bench/src/search-bench/parametric-check.mts index 9780fb7e..5494dd7b 100644 --- a/bench/src/search-bench/parametric-check.mts +++ b/bench/src/search-bench/parametric-check.mts @@ -11,7 +11,7 @@ */ import { writeFileSync } from 'node:fs' import { runPool } from '../run-pool' -import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import { runBenchRouterTurn } from '../router-turn' import { freshTasks } from './tasks-fresh' import { scoreTask, taskToPrompt } from './tasks' @@ -28,9 +28,24 @@ async function main(): Promise { const outcomes = await runPool(freshTasks, conc, async (task) => { try { - const res = await routerChatWithUsage(cfg, [{ role: 'user', content: taskToPrompt(task) }]) - const { score } = scoreTask(task, res.content) - return { id: task.id, score: score as 0 | 1 | null, cost: res.costUsd, err: undefined as string | undefined } + const res = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'search-parametric-check', + model: { provider: 'tangle-router', default: model }, + }, + }, + taskToPrompt(task), + ) + const { score } = scoreTask(task, res.finalText) + return { + id: task.id, + score: score as 0 | 1 | null, + cost: res.usage.usdKnown === false ? undefined : res.usage.costUsd, + err: undefined as string | undefined, + } } catch (err) { return { id: task.id, score: null as 0 | 1 | null, cost: undefined, err: err instanceof Error ? err.message : String(err) } } diff --git a/bench/src/supervisor-arena.mts b/bench/src/supervisor-arena.mts index d292a647..6bfc622a 100644 --- a/bench/src/supervisor-arena.mts +++ b/bench/src/supervisor-arena.mts @@ -37,6 +37,7 @@ import { appendFileSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from import { tmpdir } from 'node:os' import { join } from 'node:path' import { type HumanEvalTask, basePrompt, extractCode, loadHumanEval } from './benchmarks/humaneval' +import { runBenchRouterTurn } from './router-turn' import { pool } from './stats.mts' // ---------- pre-registered task sets (verbatim from the prereg; DO NOT EDIT) ---------- @@ -294,33 +295,37 @@ async function complete(cfg: ClientCfg, messages: Array<{ role: string; content: let lastErr = '' for (let attempt = 1; attempt <= 4; attempt += 1) { if (attempt > 1) await new Promise((r) => setTimeout(r, 2000 * 2 ** attempt)) - const ctl = new AbortController() - const timer = setTimeout(() => ctl.abort(), Number(process.env.LLM_TIMEOUT_MS ?? 240_000)) try { - const res = await fetch(`${cfg.base}/chat/completions`, { - method: 'POST', - headers: { Authorization: `Bearer ${cfg.key}`, 'Content-Type': 'application/json' }, - body: JSON.stringify({ model: cfg.model, max_tokens: cfg.maxTokens, temperature: cfg.temperature, messages }), - signal: ctl.signal, - }) - if (!res.ok) { - lastErr = `HTTP ${res.status}: ${(await res.text()).slice(0, 200)}` - continue - } - const d = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }> - usage?: { prompt_tokens?: number; completion_tokens?: number } - } - const content = d.choices?.[0]?.message?.content ?? '' + const system = messages.find((message) => message.role === 'system')?.content + const result = await runBenchRouterTurn( + { + routerBaseUrl: cfg.base, + routerKey: cfg.key, + profile: { + name: 'supervisor-arena-agent', + model: { provider: 'tangle-router', default: cfg.model }, + ...(system ? { prompt: { systemPrompt: system } } : {}), + }, + temperature: cfg.temperature, + maxTokens: cfg.maxTokens, + timeoutMs: Number(process.env.LLM_TIMEOUT_MS ?? 240_000), + }, + { messages: messages.filter((message) => message.role !== 'system') }, + ) + if (result.usage.tokensKnown === false) throw new Error('provider omitted token usage') + const content = result.finalText if (content.trim() === '') { lastErr = 'empty content' continue } - return { content, attempts: attempt, tokensIn: d.usage?.prompt_tokens ?? 0, tokensOut: d.usage?.completion_tokens ?? 0 } + return { + content, + attempts: attempt, + tokensIn: result.usage.input, + tokensOut: result.usage.output, + } } catch (e) { lastErr = e instanceof Error ? e.message : String(e) - } finally { - clearTimeout(timer) } } throw new Error(`completion failed after retries: ${lastErr}`) diff --git a/bench/src/swe-jail.test.ts b/bench/src/swe-jail.test.ts new file mode 100644 index 00000000..1e15e077 --- /dev/null +++ b/bench/src/swe-jail.test.ts @@ -0,0 +1,96 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { zaiChatRaw } from './swe-jail' + +test('zaiChatRaw preserves the SWE transport shape through Runtime', async () => { + const originalFetch = globalThis.fetch + let requestBody: Record | undefined + let requestUrl = '' + globalThis.fetch = async (input, init) => { + requestUrl = String(input) + requestBody = JSON.parse(String(init?.body)) as Record + return new Response( + JSON.stringify({ + choices: [ + { + message: { + content: null, + tool_calls: [ + { + id: 'call-1', + type: 'function', + function: { name: 'run', arguments: '{"command":"pwd"}' }, + }, + ], + }, + }, + ], + usage: { prompt_tokens: 11, completion_tokens: 3 }, + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ) + } + + try { + const result = await zaiChatRaw( + { base: 'http://router.test/v1', key: 'secret', timeoutMs: 1_000 }, + { + model: 'deepseek-v4-flash', + messages: [{ role: 'user', content: 'inspect' }], + tools: [ + { + type: 'function', + function: { name: 'run', parameters: { type: 'object' } }, + }, + ], + tool_choice: 'required', + temperature: 0.1, + max_tokens: 32_768, + thinking: { type: 'enabled' }, + }, + { + name: 'swe-jail-test-worker', + model: { provider: 'tangle-router', default: 'deepseek-v4-flash' }, + tools: { run: true }, + }, + ) + + assert.equal(requestUrl, 'http://router.test/v1/chat/completions') + assert.deepEqual(requestBody, { + model: 'deepseek-v4-flash', + messages: [{ role: 'user', content: 'inspect' }], + tools: [ + { + type: 'function', + function: { name: 'run', parameters: { type: 'object' } }, + }, + ], + tool_choice: 'required', + temperature: 0.1, + max_tokens: 32_768, + thinking: { type: 'enabled' }, + }) + assert.equal(result.attempts, 1) + assert.deepEqual(result.json, { + choices: [ + { + message: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call-1', + type: 'function', + function: { name: 'run', arguments: '{"command":"pwd"}' }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + usage: { prompt_tokens: 11, completion_tokens: 3 }, + }) + } finally { + globalThis.fetch = originalFetch + } +}) diff --git a/bench/src/swe-jail.ts b/bench/src/swe-jail.ts index 57aeeb40..749b9da5 100644 --- a/bench/src/swe-jail.ts +++ b/bench/src/swe-jail.ts @@ -12,12 +12,15 @@ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' import { tmpdir } from 'node:os' import { join } from 'node:path' import { promisify } from 'node:util' +import type { AgentProfile } from '@tangle-network/agent-interface' +import type { ToolSpec } from '@tangle-network/agent-runtime/kernel' +import { runBenchRouterTurn } from './router-turn' const exec = promisify(execFile) export const tail = (s: string, n: number): string => (s.length > n ? `…${s.slice(s.length - n)}` : s) -// ---------- zai chat client (plain fetch; patient 429 ladder) ---------- +// ---------- zai chat client (Runtime transport; patient 429 ladder) ---------- export interface ZaiCfg { base: string @@ -30,9 +33,9 @@ export interface ZaiCfg { } export interface ZaiRaw { - /** The parsed /chat/completions JSON, verbatim. */ + /** The OpenAI-compatible message and usage fields expected by existing SWE callers. */ json: Record - /** HTTP attempts spent (retries included). */ + /** Runtime completion attempts spent (retries included). */ attempts: number } @@ -44,7 +47,29 @@ export interface ZaiRaw { * tool_calls — is the glm reasoning path starving `content` when reasoning eats max_tokens, and is * retried too (a tool_calls turn with empty content is a NORMAL tool-loop turn, not starvation). */ -export async function zaiChatRaw(cfg: ZaiCfg, body: Record): Promise { +export async function zaiChatRaw( + cfg: ZaiCfg, + body: Record, + profile: AgentProfile, +): Promise { + const { + model, + messages, + tools, + temperature, + max_tokens: maxTokens, + tool_choice: toolChoice, + ...extraBody + } = body + if (typeof model !== 'string' || model.length === 0) { + throw new Error('completion body.model must be a non-empty string') + } + if (!Array.isArray(messages)) throw new Error('completion body.messages must be an array') + const typedTools = Array.isArray(tools) ? (tools as ToolSpec[]) : [] + const typedToolChoice = + toolChoice === 'auto' || toolChoice === 'required' || toolChoice === 'none' + ? toolChoice + : undefined let lastErr = '' let delayBase = 2_000 for (let attempt = 1; attempt <= 7; attempt += 1) { @@ -63,30 +88,70 @@ export async function zaiChatRaw(cfg: ZaiCfg, body: Record): Pr const ctl = new AbortController() const timer = setTimeout(() => ctl.abort(), perCallTimeout) try { - const res = await fetch(`${cfg.base}/chat/completions`, { - method: 'POST', - headers: { Authorization: `Bearer ${cfg.key}`, 'Content-Type': 'application/json' }, - body: JSON.stringify(body), - signal: ctl.signal, - }) - if (!res.ok) { - lastErr = `HTTP ${res.status}: ${(await res.text()).slice(0, 200)}` - delayBase = res.status === 429 ? 60_000 : 2_000 - continue + const result = await runBenchRouterTurn( + { + routerBaseUrl: cfg.base, + routerKey: cfg.key, + profile, + ...(typeof temperature === 'number' ? { temperature } : {}), + ...(typeof maxTokens === 'number' ? { maxTokens } : {}), + ...(typedToolChoice ? { toolChoice: typedToolChoice } : {}), + tools: typedTools, + extraBody, + signal: ctl.signal, + }, + { messages: messages as Array> }, + ) + const toolCalls = result.toolCalls.map((call, index) => ({ + id: call.id ?? `call_${index}`, + name: call.name, + arguments: call.arguments, + })) + const message = { + role: 'assistant', + content: toolCalls.length > 0 && result.finalText === '' ? null : result.finalText, + ...(toolCalls.length > 0 + ? { + tool_calls: toolCalls.map((call) => ({ + id: call.id, + type: 'function', + function: { name: call.name, arguments: call.arguments }, + })), + } + : {}), } - const json = (await res.json()) as Record - const msg = ((json.choices as Array<{ message?: Record }> | undefined)?.[0]?.message ?? {}) as { - content?: string - tool_calls?: unknown[] - } - const hasToolCalls = Array.isArray(msg.tool_calls) && msg.tool_calls.length > 0 - if (!hasToolCalls && String(msg.content ?? '').trim() === '') { + if (toolCalls.length === 0 && result.finalText.trim() === '') { lastErr = 'empty content' continue } + const finalEvent = result.events.at(-1) + const finishReason = + finalEvent?.type === 'final' + ? finalEvent.reason + : toolCalls.length > 0 + ? 'tool_calls' + : 'stop' + const json: Record = { + choices: [ + { + message, + finish_reason: finishReason, + }, + ], + ...(result.usage.tokensKnown !== false + ? { + usage: { + prompt_tokens: result.usage.input, + completion_tokens: result.usage.output, + }, + } + : {}), + } return { json, attempts: attempt } } catch (e) { lastErr = e instanceof Error ? e.message : String(e) + const status = Number(/router (\d+)/.exec(lastErr)?.[1]) + delayBase = status === 429 ? 60_000 : 2_000 } finally { clearTimeout(timer) } diff --git a/bench/src/swe-repro-calibrate.mts b/bench/src/swe-repro-calibrate.mts index b06db532..2ea3a9f8 100644 --- a/bench/src/swe-repro-calibrate.mts +++ b/bench/src/swe-repro-calibrate.mts @@ -89,6 +89,11 @@ async function complete(messages: ChatMsg[]): Promise { const { json, attempts } = await zaiChatRaw( { base: ZAI_BASE, key: ZAI_KEY, timeoutMs: LLM_TIMEOUT_MS }, { model: MODEL, max_tokens: MAX_TOKENS, temperature: TEMP, messages }, + { + name: 'swe-reproduction-calibrator', + model: { provider: 'tangle-router', default: MODEL }, + prompt: { systemPrompt: AUTHOR_SYSTEM }, + }, ) const d = json as { choices?: Array<{ message?: { content?: string } }> diff --git a/bench/src/swe-stream.mts b/bench/src/swe-stream.mts index 722fd13c..4d8a30e2 100644 --- a/bench/src/swe-stream.mts +++ b/bench/src/swe-stream.mts @@ -274,9 +274,24 @@ const makeTransport = counter.guardedMsgs += assertNoHiddenLeak(marks, msgs) // Inject the honored reasoning-budget knob (thinking) here at the single shared worker // chokepoint: makeTransport is byte-identical across arms F and L, so the budget is symmetric. + const model = String(body.model ?? '') + const systemPrompt = msgs.find((message) => message.role === 'system')?.content + const toolNames = Array.isArray(body.tools) + ? (body.tools as Array<{ function?: { name?: unknown } }>).flatMap((tool) => + typeof tool.function?.name === 'string' ? [tool.function.name] : [], + ) + : [] const { json, attempts } = await zaiChatRaw( { base: ZAI_BASE, key: ZAI_KEY, timeoutMs: LLM_TIMEOUT_MS, deadlineAt: guard.deadlineAt }, { ...body, ...WORKER_REASONING }, + { + name: 'swe-stream-worker', + model: { provider: 'tangle-router', default: model }, + ...(typeof systemPrompt === 'string' ? { prompt: { systemPrompt } } : {}), + ...(toolNames.length > 0 + ? { tools: Object.fromEntries(toolNames.map((name) => [name, true])) } + : {}), + }, ) counter.calls += 1 counter.httpAttempts += attempts @@ -643,6 +658,11 @@ async function acquireRepro( const { json } = await zaiChatRaw( { base: ZAI_BASE, key: ZAI_KEY, timeoutMs: LLM_TIMEOUT_MS, deadlineAt }, { model: REPRO_MODEL, max_tokens: MAX_TOKENS, temperature: 0.2, messages }, + { + name: 'swe-reproduction-author', + model: { provider: 'tangle-router', default: REPRO_MODEL }, + prompt: { systemPrompt: reproAuthorSystem(REPRO_TIMEOUT_S) }, + }, ) const d = json as { choices?: Array<{ message?: { content?: string } }>; usage?: { prompt_tokens?: number; completion_tokens?: number } } out.authorCalls += 1 @@ -810,6 +830,10 @@ async function superviseRepair( const { json, attempts } = await zaiChatRaw( { base: ZAI_BASE, key: ZAI_KEY, timeoutMs: LLM_TIMEOUT_MS, deadlineAt }, { model: SUPERVISOR_MODEL, max_tokens: SUPERVISOR_MAX_TOKENS, temperature: 0.2, messages }, + { + name: 'swe-repair-supervisor', + model: { provider: 'tangle-router', default: SUPERVISOR_MODEL }, + }, ) const d = json as { choices?: Array<{ message?: { content?: string } }>; usage?: { prompt_tokens?: number; completion_tokens?: number } } const planRaw = d.choices?.[0]?.message?.content ?? '' diff --git a/bench/src/swe-structural.mts b/bench/src/swe-structural.mts index baeaa7f2..6656499b 100644 --- a/bench/src/swe-structural.mts +++ b/bench/src/swe-structural.mts @@ -138,7 +138,25 @@ const makeTransport = async (body: Record): Promise => { const msgs = (body.messages ?? []) as Array<{ role?: string; content?: unknown }> counter.guardedMsgs += assertNoHiddenLeak(marks, msgs) - const { json, attempts } = await zaiChatRaw({ base: ZAI_BASE, key: ZAI_KEY, timeoutMs: LLM_TIMEOUT_MS }, body) + const model = String(body.model ?? '') + const systemPrompt = msgs.find((message) => message.role === 'system')?.content + const toolNames = Array.isArray(body.tools) + ? (body.tools as Array<{ function?: { name?: unknown } }>).flatMap((tool) => + typeof tool.function?.name === 'string' ? [tool.function.name] : [], + ) + : [] + const { json, attempts } = await zaiChatRaw( + { base: ZAI_BASE, key: ZAI_KEY, timeoutMs: LLM_TIMEOUT_MS }, + body, + { + name: 'swe-structural-worker', + model: { provider: 'tangle-router', default: model }, + ...(typeof systemPrompt === 'string' ? { prompt: { systemPrompt } } : {}), + ...(toolNames.length > 0 + ? { tools: Object.fromEntries(toolNames.map((name) => [name, true])) } + : {}), + }, + ) counter.calls += 1 counter.httpAttempts += attempts const u = (json as { usage?: { prompt_tokens?: number; completion_tokens?: number } }).usage diff --git a/bench/src/trata-gate.mts b/bench/src/trata-gate.mts index 1c25eea4..5e731806 100644 --- a/bench/src/trata-gate.mts +++ b/bench/src/trata-gate.mts @@ -30,6 +30,7 @@ import { appendFileSync } from 'node:fs' import { resolveAdapter } from './adapters' import type { BenchScore, BenchTask } from './benchmarks/types' +import { runBenchRouterTurn } from './router-turn' import { runPool } from './run-pool' function must(name: string): string { @@ -61,33 +62,26 @@ async function workerComplete( cfg: { routerBaseUrl: string; routerKey: string; model: string; timeoutMs: number }, ): Promise<{ answer: string; inputTokens: number; outputTokens: number; durationMs: number }> { const startedAt = Date.now() - const res = await fetch(`${cfg.routerBaseUrl}/chat/completions`, { - method: 'POST', - signal: AbortSignal.timeout(cfg.timeoutMs), - headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.routerKey}` }, - body: JSON.stringify({ - model: cfg.model, + const result = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'trata-financial-analyst', + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt: ANALYST_SYSTEM }, + }, temperature: 0, - max_tokens: 4096, - messages: [ - { role: 'system', content: ANALYST_SYSTEM }, - { role: 'user', content: task.prompt }, - ], - }), - }) - if (!res.ok) { - const body = (await res.text()).slice(0, 300) - throw new Error(`router ${res.status} for ${task.id}: ${body}`) - } - const j = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }> - usage?: { prompt_tokens?: number; completion_tokens?: number } - } - const answer = j.choices?.[0]?.message?.content ?? '' + maxTokens: Number(process.env.WORKER_MAX_TOKENS ?? 4096), + timeoutMs: cfg.timeoutMs, + }, + task.prompt, + ) + if (result.usage.tokensKnown === false) throw new Error('worker provider omitted token usage') return { - answer, - inputTokens: j.usage?.prompt_tokens ?? 0, - outputTokens: j.usage?.completion_tokens ?? 0, + answer: result.finalText, + inputTokens: result.usage.input, + outputTokens: result.usage.output, durationMs: Date.now() - startedAt, } } diff --git a/bench/src/trata-gepa.mts b/bench/src/trata-gepa.mts index 056c8f15..c618fcda 100644 --- a/bench/src/trata-gepa.mts +++ b/bench/src/trata-gepa.mts @@ -60,6 +60,7 @@ import { officialOptimizerModel, requiredTokenPricing, } from './official-optimizer-config.mjs' +import { runBenchRouterTurn } from './router-turn' interface TrataScenario extends Scenario { task: BenchTask @@ -122,23 +123,28 @@ async function chatComplete( messages: Array<{ role: string; content: string }>, signal: AbortSignal, ): Promise<{ content: string; usage?: { input: number; output: number } }> { - const res = await fetch(`${baseUrl}/chat/completions`, { - method: 'POST', - signal: AbortSignal.any([signal, AbortSignal.timeout(180_000)]), - headers: { 'content-type': 'application/json', authorization: `Bearer ${key}` }, - body: JSON.stringify({ model, temperature: 0, max_tokens: maxTokens, messages }), - }) - if (!res.ok) throw new Error(`router ${res.status}: ${(await res.text()).slice(0, 300)}`) - const j = (await res.json()) as { - choices?: Array<{ message?: { content?: string } }> - usage?: { prompt_tokens?: number; completion_tokens?: number } + const system = messages.find((message) => message.role === 'system')?.content + const result = await runBenchRouterTurn( + { + routerBaseUrl: baseUrl, + routerKey: key, + profile: { + name: 'trata-gepa-worker', + model: { provider: 'tangle-router', default: model }, + ...(system ? { prompt: { systemPrompt: system } } : {}), + }, + temperature: 0, + maxTokens, + signal, + }, + { messages: messages.filter((message) => message.role !== 'system') }, + ) + return { + content: result.finalText, + ...(result.usage.tokensKnown === false + ? {} + : { usage: { input: result.usage.input, output: result.usage.output } }), } - const content = j.choices?.[0]?.message?.content ?? '' - const usage = - j.usage?.prompt_tokens != null - ? { input: j.usage.prompt_tokens, output: j.usage.completion_tokens ?? 0 } - : undefined - return { content, usage } } async function main(): Promise { diff --git a/bench/src/trata-hedge-solve.mts b/bench/src/trata-hedge-solve.mts new file mode 100644 index 00000000..7ba73e7f --- /dev/null +++ b/bench/src/trata-hedge-solve.mts @@ -0,0 +1,77 @@ +/** Single-shot lower-bound solver for trata-hedge-bench. */ +import { readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs' +import { join, relative, resolve } from 'node:path' +import { runBenchRouterTurn } from './router-turn' + +const environment = process.argv[2] +if (!environment) throw new Error('usage: solve.mts [out.txt]') +const output = process.argv[3] ?? '/tmp/thb-answer.txt' +const model = process.env.WORKER_MODEL +if (!model) throw new Error('WORKER_MODEL is required') +const dataBudget = Number(process.env.DATA_BUDGET ?? 160_000) +const maxTokens = Number(process.env.MAX_TOKENS ?? 6_000) +const temperature = Number(process.env.TEMPERATURE ?? 0.5) +const routerKey = process.env.TANGLE_API_KEY +if (!routerKey) throw new Error('TANGLE_API_KEY is required') +const routerBaseUrl = process.env.ROUTER_BASE ?? 'https://router.tangle.tools/v1' + +for (const [name, value] of [ + ['DATA_BUDGET', dataBudget], + ['MAX_TOKENS', maxTokens], +] as const) { + if (!Number.isSafeInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`) +} +if (!Number.isFinite(temperature)) throw new Error('TEMPERATURE must be finite') + +const root = resolve(environment) +const dataDir = join(root, 'environment', 'data') +const instruction = readFileSync(join(root, 'instruction.md'), 'utf8') + +function filesUnder(dir: string): string[] { + return readdirSync(dir) + .flatMap((name) => { + const path = join(dir, name) + return statSync(path).isDirectory() ? filesUnder(path) : [path] + }) +} + +function rank(path: string): number { + if (path.includes('earnings_call')) return 0 + if (path.includes('financials')) return 1 + if (path.includes('company_profiles')) return 2 + if (path.includes('press_releases')) return 3 + return 4 +} + +let used = 0 +const blocks: string[] = [] +const files = filesUnder(dataDir).sort((a, b) => rank(a) - rank(b) || a.localeCompare(b)) +for (const path of files) { + const content = readFileSync(path, 'utf8') + if (used + content.length > dataBudget) continue + used += content.length + blocks.push(`\n=== FILE: data/${relative(dataDir, path)} ===\n${content}`) +} +process.stderr.write(`[solve] ${blocks.length}/${files.length} files in context (${used} chars)\n`) + +const prompt = + '--- AVAILABLE DATA (cite files by their `data/` name inline) ---\n' + + blocks.join('') + + '\n\n--- END DATA ---\nWrite ONLY the full analysis (no preamble). Inline-cite every claim with its `data/`.' + +const result = await runBenchRouterTurn( + { + routerBaseUrl, + routerKey, + profile: { + name: 'trata-hedge-solver', + model: { provider: 'tangle-router', default: model }, + prompt: { systemPrompt: instruction }, + }, + temperature, + maxTokens, + }, + prompt, +) +writeFileSync(output, result.finalText) +process.stderr.write(`[solve] wrote ${result.finalText.length} chars -> ${output} (model=${model})\n`) diff --git a/bench/src/worker-blender.ts b/bench/src/worker-blender.ts index 6e03c6f8..35163fc3 100644 --- a/bench/src/worker-blender.ts +++ b/bench/src/worker-blender.ts @@ -18,7 +18,7 @@ import type { Span } from '@tangle-network/agent-eval' import type { BenchTask } from './benchmarks/types' import { DEFAULT_BLENDER_DIRECTIVE } from './directives' import { runRefineLoop } from './refine-loop' -import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import { runBenchRouterTurn } from './router-turn' export { DEFAULT_BLENDER_DIRECTIVE } from './directives' @@ -177,17 +177,23 @@ export async function solveBlenderLocal(task: BenchTask, cfg: BlenderLocalConfig runShot: async (user, round, dir) => { const runnerPath = join(dir, 'runner.py') const scriptPath = join(dir, 'model.py') - const { content, usage: u } = await routerChatWithUsage( - cfg, - [ - { role: 'system', content: directive }, - { role: 'user', content: user }, - ], - { temperature: 0.3 }, + const turn = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'blender-worker', + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt: directive }, + }, + temperature: 0.3, + }, + user, ) - if (u) { - usage.input += u.input - usage.output += u.output + const content = turn.finalText + if (turn.usage.tokensKnown !== false) { + usage.input += turn.usage.input + usage.output += turn.usage.output } const script = extractPy(content) trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span) diff --git a/bench/src/worker-browser.ts b/bench/src/worker-browser.ts index 013ec70a..96ca2b1f 100644 --- a/bench/src/worker-browser.ts +++ b/bench/src/worker-browser.ts @@ -18,7 +18,7 @@ import { readFile } from 'node:fs/promises' import type { Span } from '@tangle-network/agent-eval' import type { BenchTask } from './benchmarks/types' -import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import { runBenchRouterTurn } from './router-turn' export interface BrowserLocalConfig { routerBaseUrl: string @@ -67,10 +67,23 @@ export async function solveBrowserLocal(task: BenchTask, cfg: BrowserLocalConfig trace.push({ spanId: 's-task', runId, kind: 'llm', name: 'web task', model: cfg.model, messages: [{ role: 'user', content: goal }], startedAt: tick(), endedAt: tick(), status: 'ok' } as Span) - const { content, usage } = await routerChatWithUsage(cfg, [ - { role: 'system', content: directive }, - { role: 'user', content: task.prompt }, - ]) + const turn = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'browser-local-worker', + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt: directive }, + }, + }, + task.prompt, + ) + const content = turn.finalText + const usage = + turn.usage.tokensKnown === false + ? undefined + : { input: turn.usage.input, output: turn.usage.output } const artifact = content.trim() const elementId = /ELEMENT:\s*\[?(\d+)\]?/i.exec(artifact)?.[1] ?? '' diff --git a/bench/src/worker-build123d.ts b/bench/src/worker-build123d.ts index 179af399..f4eefc5b 100644 --- a/bench/src/worker-build123d.ts +++ b/bench/src/worker-build123d.ts @@ -21,7 +21,7 @@ import type { Span } from '@tangle-network/agent-eval' import type { BenchTask } from './benchmarks/types' import { DEFAULT_BUILD123D_DIRECTIVE } from './directives' import { runRefineLoop } from './refine-loop' -import { routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import { runBenchRouterTurn } from './router-turn' export { DEFAULT_BUILD123D_DIRECTIVE } from './directives' @@ -106,13 +106,22 @@ export async function solveBuild123dLocal(task: BenchTask, cfg: Build123dConfig) runShot: async (user, round, dir) => { const scriptPath = join(dir, 'build.py') const stepPath = join(dir, 'output.step') - const { content, usage: u } = await routerChatWithUsage(cfg, [ - { role: 'system', content: sys }, - { role: 'user', content: user }, - ]) - if (u) { - usage.input += u.input - usage.output += u.output + const turn = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'build123d-worker', + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt: sys }, + }, + }, + user, + ) + const content = turn.finalText + if (turn.usage.tokensKnown !== false) { + usage.input += turn.usage.input + usage.output += turn.usage.output } const source = extractPy(content) trace.push({ spanId: `s-author-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span) diff --git a/bench/src/worker-cad.ts b/bench/src/worker-cad.ts index 2799fd51..3757a6c9 100644 --- a/bench/src/worker-cad.ts +++ b/bench/src/worker-cad.ts @@ -17,12 +17,13 @@ import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises' import { tmpdir } from 'node:os' import { join } from 'node:path' import { promisify } from 'node:util' -import { acquireSandbox, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel' +import { acquireSandbox } from '@tangle-network/agent-runtime/kernel' import { Sandbox } from '@tangle-network/sandbox' import type { Span } from '@tangle-network/agent-eval' import type { BenchTask } from './benchmarks/types' import { DEFAULT_CAD_DIRECTIVE, DEFAULT_CAD_SANDBOX_DIRECTIVE } from './directives' import { runRefineLoop } from './refine-loop' +import { runBenchRouterTurn } from './router-turn' export { DEFAULT_CAD_DIRECTIVE } from './directives' @@ -138,13 +139,22 @@ export async function solveCadRefineLocal(task: BenchTask, cfg: CadLocalConfig): const scadPath = join(dir, 'model.scad') const stlPath = join(dir, 'model.stl') const pngPath = join(dir, 'model.png') - const { content, usage: u } = await routerChatWithUsage(cfg, [ - { role: 'system', content: directive }, - { role: 'user', content: user }, - ]) - if (u) { - usage.input += u.input - usage.output += u.output + const turn = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'cad-local-worker', + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt: directive }, + }, + }, + user, + ) + const content = turn.finalText + if (turn.usage.tokensKnown !== false) { + usage.input += turn.usage.input + usage.output += turn.usage.output } const scad = extractScad(content) trace.push({ spanId: `s-reply-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: content.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span) @@ -253,10 +263,19 @@ export async function solveCadRefine(task: BenchTask, cfg: CadRefineConfig): Pro ? task.prompt : `Your previous OpenSCAD had this problem:\n${lastErr}\n\nHere is the previous source:\n${history[history.length - 1]?.artifact ?? ''}\n\nFix it so it compiles AND better matches the brief:\n${task.prompt}`, runShot: async (user, round, box) => { - const { content: reply } = await routerChatWithUsage(cfg, [ - { role: 'system', content: sys }, - { role: 'user', content: user }, - ]) + const turn = await runBenchRouterTurn( + { + routerBaseUrl: cfg.routerBaseUrl, + routerKey: cfg.routerKey, + profile: { + name: 'cad-sandbox-worker', + model: { provider: 'tangle-router', default: cfg.model }, + prompt: { systemPrompt: sys }, + }, + }, + user, + ) + const reply = turn.finalText const scad = extractScad(reply) trace.push({ spanId: `s-reply-${round}`, runId, kind: 'llm', name: `author r${round}`, model: cfg.model, messages: [{ role: 'user', content: round === 1 ? task.prompt : 'refine' }], output: reply.slice(0, 600), startedAt: tick(), endedAt: tick(), status: 'ok' } as Span) trace.push({ spanId: `s-write-${round}`, runId, kind: 'tool', name: 'write_file', toolName: 'create_file', args: { path: 'model.scad', content: scad }, startedAt: tick(), endedAt: tick(), status: 'ok' } as Span) diff --git a/docs/api/primitive-catalog.md b/docs/api/primitive-catalog.md index b1e3ac3e..fda35b9d 100644 --- a/docs/api/primitive-catalog.md +++ b/docs/api/primitive-catalog.md @@ -510,7 +510,7 @@ Import from `@tangle-network/agent-runtime/intelligence` — 166 exports. ### Execution kernel — recursive atom, supervision, executors, round-synchronous loop -Import from `@tangle-network/agent-runtime/kernel` — 707 exports. +Import from `@tangle-network/agent-runtime/kernel` — 709 exports. | Symbol | Kind | Summary | |---|---|---| @@ -617,6 +617,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 707 exports. | `loopDispatch` | function | Adapter for `runProfileMatrix` (profile is an axis). Returns a | | `loopUntil` | function | `loopUntil(seed, spec)` — one `step` child per round; `fold` accumulates each settlement into | | `makeFinding` | function | Convenience factory: produce a fully-formed AnalystFinding with the | +| `mapExecutorResult` | function | Transform a Runtime executor's terminal artifact without losing its private | | `mapSandboxEvent` | function | Project one `SandboxEvent` onto the `RuntimeStreamEvent` chat-UX vocabulary, | | `mapSandboxToolEvent` | function | Project one `SandboxEvent` onto the `tool_call` / `tool_result` variants of | | `materializeLocalMcp` | function | Spawn every explicitly trusted stdio server in `profile.mcp` as a same-host | @@ -1068,7 +1069,7 @@ Import from `@tangle-network/agent-runtime/kernel` — 707 exports. | `WorktreeCheckRunner` | type | The single shell-command-in-worktree runner seam (replaces the per-executor copies). | | `WorktreePatchArtifact` | type | Terminal artifact of one worktree-CLI run — the canonical worktree-harness result (the captured | -**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcquireOptions`, `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgentGraph`, `AgenticOptions`, `AgenticRunResult`, `AgenticTask`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnInput`, `AgentTurnResult`, `AllWorkersStalledOptions`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `AnytimeTaskCurve`, `ArtifactHandle`, `AuditIntentInput`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `CheckpointRef`, `CheckpointRequest`, `CheckRunContext`, `CliWorktreeBridgeSeam`, `CoordinationMcpHandle`, `CopyOptions`, `CorpusReadbackOptions`, `CreateAgentEnvironmentInput`, `CreateTangleSandboxExactProcessProviderOptions`, `DefinedLeaderboard`, `DispatchReport`, `Driver`, `DriverAgentOptions`, `EventBus`, `EvolutionArchiveNode`, `EvolutionAuthor`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ForkRequest`, `GitWorkspaceOptions`, `GraphResult`, `HarvestCorpusOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LocalSandboxClientOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `MaterializeLocalMcpOptions`, `McpEnvironmentOptions`, `McpToolDescriptor`, `NodeSnapshot`, `NoProgressForOptions`, `Observation`, `ObserveInput`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PlateauOptions`, `ProgressTrackerOptions`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ReproductionCheck`, `ResolveSandboxClientOptions`, `ResourceRequest`, `RollingDispatchOptions`, `RouterChatResult`, `RouterChatToolsResult`, `RouterConfig`, `RouterToolLoopResult`, `RunAgenticOptions`, `RunAgentRoundsOptions`, `RunGraphOptions`, `SandboxRun`, `ShotSpec`, `SpawnOpts`, `StdioMcpConnection`, `StdioMcpServerSpec`, `SteerableSandboxArgs`, `Strategy`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `StructuralRolloutConfig`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SupervisorSpanOptions`, `SupervisorSpanRecorder`, `SurfaceScore`, `ToolSpec`, `ToolStepInput`, `TraceSource`, `TrajectoryAnalysis`, `UntrackedCopyStats`, `ValidationCtx`, `Validator`, `VerifierEnvironmentOptions`, `WatchTraceOptions`, `WaterfallCollector`, `WaterfallReport`, `WaterfallSpan`, `WorkerEvidenceInput`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `EdgeDeliveryOutcome`, `GraphEdge`, `LoopTraceEvent`, `MakeWorkerAgent`, `RepairStop`, `SandboxControlClient`, `WorkspaceCommit`. +**Undocumented supporting types** (add a TSDoc line at the declaration to earn a table row): `AcquireOptions`, `AgentEnvironment`, `AgentEnvironmentCapabilities`, `AgentEnvironmentEvent`, `AgentEnvironmentProvider`, `AgentEnvironmentQuery`, `AgentEnvironmentSummary`, `AgentGraph`, `AgenticOptions`, `AgenticRunResult`, `AgenticTask`, `AgenticTool`, `AgentSession`, `AgentSessionRef`, `AgentTurnInput`, `AgentTurnResult`, `AllWorkersStalledOptions`, `AnalystRegistry`, `AnytimeReport`, `AnytimeStrategySummary`, `AnytimeTaskCurve`, `ArtifactHandle`, `AuditIntentInput`, `AuditIntentOptions`, `AuthoredHarness`, `AuthoredStrategy`, `AuthorStrategyOptions`, `BenchmarkConfig`, `BenchmarkLift`, `BenchmarkStrategySummary`, `BenchmarkTaskRow`, `BudgetPool`, `BusStats`, `ChampionPick`, `CheckpointRef`, `CheckpointRequest`, `CheckRunContext`, `CliWorktreeBridgeSeam`, `CoordinationMcpHandle`, `CopyOptions`, `CorpusReadbackOptions`, `CreateAgentEnvironmentInput`, `CreateTangleSandboxExactProcessProviderOptions`, `DefinedLeaderboard`, `DispatchReport`, `Driver`, `DriverAgentOptions`, `EventBus`, `EvolutionArchiveNode`, `EvolutionAuthor`, `EvolutionBandInfo`, `EvolutionCandidate`, `EvolutionGeneration`, `EvolutionReport`, `ExecRequest`, `ExecResult`, `ExecutorResultMapping`, `ForkRequest`, `GitWorkspaceOptions`, `GraphResult`, `HarvestCorpusOptions`, `HarvestFailure`, `HarvestReport`, `Inbox`, `InProcessSandboxClientOptions`, `IntentAudit`, `Iteration`, `Leaderboard`, `LeaderboardOptions`, `LocalSandboxClientOptions`, `LoopDecisionPayload`, `LoopDispatchOptions`, `LoopEndedPayload`, `LoopIterationEndedPayload`, `LoopIterationStartedPayload`, `LoopPlanDescription`, `LoopResult`, `LoopSandboxPlacement`, `LoopStartedPayload`, `LoopTraceEmitter`, `LoopWinner`, `MaterializeLocalMcpOptions`, `McpEnvironmentOptions`, `McpToolDescriptor`, `NodeSnapshot`, `NoProgressForOptions`, `Observation`, `ObserveInput`, `ObserveOptions`, `OpenSandboxRunOptions`, `PairwiseOptions`, `PatchDeliverableOptions`, `PlacementInfo`, `PlateauOptions`, `ProgressTrackerOptions`, `PromotionGateOptions`, `PromotionVerdict`, `PublishOptions`, `ReproductionCheck`, `ResolveSandboxClientOptions`, `ResourceRequest`, `RollingDispatchOptions`, `RouterChatResult`, `RouterChatToolsResult`, `RouterConfig`, `RouterToolLoopResult`, `RunAgenticOptions`, `RunAgentRoundsOptions`, `RunGraphOptions`, `SandboxRun`, `ShotSpec`, `SpawnOpts`, `StdioMcpConnection`, `StdioMcpServerSpec`, `SteerableSandboxArgs`, `Strategy`, `StrategyEvolutionConfig`, `StrategyResult`, `StreamAgentTurnOptions`, `StructuralRolloutConfig`, `SuperviseOptions`, `SuperviseSurfaceOptions`, `SupervisorAgentDeps`, `SupervisorOpts`, `SupervisorSpanOptions`, `SupervisorSpanRecorder`, `SurfaceScore`, `ToolSpec`, `ToolStepInput`, `TraceSource`, `TrajectoryAnalysis`, `UntrackedCopyStats`, `ValidationCtx`, `Validator`, `VerifierEnvironmentOptions`, `WatchTraceOptions`, `WaterfallCollector`, `WaterfallReport`, `WaterfallSpan`, `WorkerEvidenceInput`, `Workspace`, `WorkspaceRequest`, `WorkspaceRun`, `WorktreeCliExecutorOptions`, `WorktreeFanoutOptions`, `AgentEnvironmentStatus`, `AgentSessionStatus`, `ChampionPolicy`, `EdgeDeliveryOutcome`, `GraphEdge`, `LoopTraceEvent`, `MakeWorkerAgent`, `RepairStop`, `SandboxControlClient`, `WorkspaceCommit`. ### Environment provider adapters — generic sandbox/compute bridge diff --git a/examples/p1-parity/run-parity.ts b/examples/p1-parity/run-parity.ts index 43bd40de..6897110f 100644 --- a/examples/p1-parity/run-parity.ts +++ b/examples/p1-parity/run-parity.ts @@ -16,6 +16,12 @@ import { parseArgs } from 'node:util' import type { MultishotTransport } from '@tangle-network/agent-eval/multishot' +import { + collectAgentTurn, + createExecutor, + streamAgentTurn, + type ToolSpec, +} from '@tangle-network/agent-runtime/kernel' import type { CellSpec, GraphArmBackend, MultishotArmBackend, ParityRecord } from './arms' import { runGraphArm, runMultishotArm } from './arms' import { offlineGraphBackend, offlineMultishotBackend } from './offline' @@ -100,31 +106,64 @@ function requireBridgeEnv(): BridgeEnv { /** A multishot transport over cli-bridge's OpenAI-compatible chat-completions surface. */ function bridgeTransport(env: BridgeEnv): MultishotTransport { return async (req) => { - const res = await fetch(`${env.url.replace(/\/$/, '')}/v1/chat/completions`, { - method: 'POST', - headers: { - 'content-type': 'application/json', - authorization: `Bearer ${env.bearer}`, - }, - body: JSON.stringify({ - model: req.model, - messages: req.messages, - ...(req.tools !== undefined && req.tools.length > 0 ? { tools: req.tools } : {}), - ...(req.temperature !== undefined ? { temperature: req.temperature } : {}), - ...(req.maxTokens !== undefined ? { max_tokens: req.maxTokens } : {}), - }), - ...(req.signal !== undefined ? { signal: req.signal } : {}), + const messages = req.messages as Array> + const systemPrompt = messages.find((message) => message.role === 'system')?.content + const tools = (req.tools ?? []) as ToolSpec[] + const profile = { + name: 'p1-parity-bridge-turn', + model: { provider: 'cli-bridge', default: req.model }, + ...(typeof systemPrompt === 'string' ? { prompt: { systemPrompt } } : {}), + ...(tools.length > 0 + ? { tools: Object.fromEntries(tools.map((tool) => [tool.function.name, true])) } + : {}), + } + const factory = createExecutor({ + backend: 'router', + routerBaseUrl: `${env.url.replace(/\/$/, '')}/v1`, + routerKey: env.bearer, + model: req.model, + tools, + ...(req.temperature !== undefined ? { temperature: req.temperature } : {}), + ...(req.maxTokens !== undefined ? { maxTokens: req.maxTokens } : {}), }) - if (!res.ok) { - throw new Error(`cli-bridge completion failed: ${res.status} ${await res.text()}`) + const result = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile }, + { messages: messages.filter((message) => message.role !== 'system') }, + req.signal === undefined ? {} : { signal: req.signal }, + ), + ) + if (result.status !== 'completed') { + throw new Error(result.error?.message ?? `bridge turn ended with status ${result.status}`) + } + const message = { + content: result.finalText, + ...(result.toolCalls.length > 0 + ? { + tool_calls: result.toolCalls.map((call) => { + if (call.id === undefined) { + throw new Error('cli-bridge returned a tool call without its required id') + } + return { + id: call.id, + type: 'function' as const, + function: { name: call.name, arguments: call.arguments }, + } + }), + } + : {}), } - const body = (await res.json()) as { - choices?: Array<{ message?: { content?: string | null; tool_calls?: never[] } }> - usage?: { prompt_tokens?: number; completion_tokens?: number } + return { + message, + ...(result.usage.tokensKnown !== false + ? { + usage: { + prompt_tokens: result.usage.input, + completion_tokens: result.usage.output, + }, + } + : {}), } - const message = body.choices?.[0]?.message - if (message === undefined) throw new Error('cli-bridge completion returned no message') - return { message, ...(body.usage !== undefined ? { usage: body.usage } : {}) } } } diff --git a/examples/supervisor-loop/run-supervisor-mcp.ts b/examples/supervisor-loop/run-supervisor-mcp.ts index 54d3c5a1..7f1cc5d2 100644 --- a/examples/supervisor-loop/run-supervisor-mcp.ts +++ b/examples/supervisor-loop/run-supervisor-mcp.ts @@ -30,14 +30,22 @@ * OWN harness tool-loop — that is what makes this the real MCP path, not a scripted driver. */ +import { + type AgentProfile, + harnessTypeSchema, + reasoningEffortSchema, +} from '@tangle-network/agent-interface' import { type Agent, + collectAgentTurn, + createExecutor, createExecutorRegistry, createSupervisor, InMemoryResultBlobStore, InMemorySpawnJournal, type Scope, serveCoordinationMcp, + streamAgentTurn, workerFromBackend, } from '@tangle-network/agent-runtime/kernel' import { buildWorkerBackend, demoCheck, expectedAnswer } from './shared' @@ -58,20 +66,43 @@ async function supervisorBridgeChat(opts: { mcpUrl: string }): Promise { const bridgeBearer = process.env.BRIDGE_BEARER ?? 'local' const model = process.env.SUPERVISOR_MODEL ?? process.env.WORKER_MODEL if (!model) throw new Error('supervisor needs SUPERVISOR_MODEL or WORKER_MODEL set') - const res = await fetch(`${bridgeUrl.replace(/\/$/, '')}/v1/chat/completions`, { - method: 'POST', - headers: { authorization: `Bearer ${bridgeBearer}`, 'content-type': 'application/json' }, - body: JSON.stringify({ - model, - messages: [{ role: 'user', content: supervisorTask }], - // Mount the coordination MCP — the supervisor harness calls spawn_agent through it. - mcp: { mcpServers: { coordination: { type: 'http', url: opts.mcpUrl } } }, - }), + const profile: AgentProfile = { + name: 'supervisor', + harness: harnessTypeSchema.parse(process.env.SUPERVISOR_HARNESS ?? 'pi'), + model: { + provider: process.env.SUPERVISOR_PROVIDER ?? 'tangle-router', + default: model, + reasoningEffort: reasoningEffortSchema.parse( + process.env.SUPERVISOR_REASONING_EFFORT ?? 'ultracode', + ), + }, + prompt: { systemPrompt: supervisorTask }, + mcp: { + coordination: { transport: 'http', url: opts.mcpUrl, enabled: true }, + }, + } + const factory = createExecutor({ + backend: 'bridge', + bridgeUrl, + bridgeBearer, + model, }) - if (!res.ok) - throw new Error(`supervisor bridge ${res.status}: ${(await res.text()).slice(0, 300)}`) - const j = (await res.json()) as { choices?: Array<{ message?: { content?: string } }> } - return j.choices?.[0]?.message?.content ?? '' + const timeoutRaw = process.env.SUPERVISOR_TIMEOUT_MS + const timeoutMs = timeoutRaw === undefined ? undefined : Number(timeoutRaw) + if (timeoutMs !== undefined && (!Number.isSafeInteger(timeoutMs) || timeoutMs <= 0)) { + throw new Error('SUPERVISOR_TIMEOUT_MS must be a positive integer') + } + const turn = await collectAgentTurn( + streamAgentTurn( + { kind: 'executor', factory, profile, agentRunName: profile.name }, + supervisorTask, + timeoutMs === undefined ? {} : { timeoutMs }, + ), + ) + if (turn.status !== 'completed') { + throw new Error(turn.error?.message ?? `supervisor bridge ended with status ${turn.status}`) + } + return turn.finalText } async function main(): Promise { diff --git a/examples/supervisor-loop/run.ts b/examples/supervisor-loop/run.ts index d5499040..29512601 100644 --- a/examples/supervisor-loop/run.ts +++ b/examples/supervisor-loop/run.ts @@ -27,7 +27,7 @@ import { buildWorkerBackend, demoCheck, demoGoal, resolveSupervisorBrain } from async function main(): Promise { // THE ONE KNOB — bridge (local CLIs) or sandbox (real boxes). Everything below is identical. const backend = buildWorkerBackend() - const { brain, label } = resolveSupervisorBrain(1, `${backend.backend}-solver`) + const { brain, model, label } = resolveSupervisorBrain(1, `${backend.backend}-solver`) console.log(`supervisor-loop · ${backend.backend.toUpperCase()} · driver=${label}`) @@ -40,12 +40,22 @@ async function main(): Promise { 'You are a supervisor. Spawn one worker session to produce the required line, await it ' + 'with await_event, and stop once a worker delivered (valid). Do not answer yourself.', }, + ...(model ? { model: { default: model } } : {}), }, demoGoal, { backend, deliverable: { check: demoCheck, describe: 'worker delivers the goal' }, - brain, + ...(brain ? { brain } : {}), + ...(model + ? { + router: { + routerBaseUrl: process.env.ROUTER_BASE_URL ?? 'https://router.tangle.tools/v1', + routerKey: process.env.TANGLE_API_KEY!, + model, + }, + } + : {}), budget: { maxIterations: 100, maxTokens: 2_000_000, maxUsd: 2 }, perWorker: { maxIterations: 1, maxTokens: 200_000 }, maxTurns: 12, diff --git a/examples/supervisor-loop/shared.ts b/examples/supervisor-loop/shared.ts index e137b935..78b77fcf 100644 --- a/examples/supervisor-loop/shared.ts +++ b/examples/supervisor-loop/shared.ts @@ -7,11 +7,10 @@ * these are only the per-example task + the offline brain it can be driven with. */ -import { - type ExecutorConfig, - type SandboxClient as RuntimeSandboxClient, - routerBrain, - type ToolLoopChat, +import type { + ExecutorConfig, + SandboxClient as RuntimeSandboxClient, + ToolLoopChat, } from '@tangle-network/agent-runtime/kernel' import type { BackendType } from '@tangle-network/sandbox' import { Sandbox } from '@tangle-network/sandbox' @@ -144,16 +143,12 @@ export function buildWorkerBackend(): ExecutorConfig { export function resolveSupervisorBrain( workerCount: number, labelPrefix: string, -): { brain: ToolLoopChat; label: string } { +): { brain?: ToolLoopChat; model?: string; label: string } { const routerKey = process.env.TANGLE_API_KEY const driverModel = process.env.DRIVER_MODEL ?? process.env.LOOP_MODEL if (process.env.DRIVER !== 'scripted' && routerKey && driverModel) { return { - brain: routerBrain({ - routerBaseUrl: process.env.ROUTER_BASE_URL ?? 'https://router.tangle.tools/v1', - routerKey, - model: driverModel, - }), + model: driverModel, label: `router(${driverModel})`, } } diff --git a/package.json b/package.json index e1d069fe..109e0b60 100644 --- a/package.json +++ b/package.json @@ -127,6 +127,7 @@ "generate:testing-fixture": "tsx scripts/generate-agent-improvement-proposal-fixtures.ts", "check:testing-fixture": "tsx scripts/generate-agent-improvement-proposal-fixtures.ts --check", "check:skills": "node scripts/check-skills.mjs", + "check:model-execution-boundary": "node scripts/check-model-execution-boundary.mjs", "check:publish-workflow": "node scripts/check-publish-workflow.mjs", "check:version-bump": "node scripts/check-version-bump.mjs", "release:prepare": "node scripts/prepare-release.mjs", @@ -135,7 +136,7 @@ "verify:bench": "pnpm build && pnpm --filter @tangle-network/agent-bench run typecheck:public && pnpm --filter @tangle-network/agent-bench test && pnpm --filter @tangle-network/agent-bench run verify:package:local-runtime", "verify:bench:published": "pnpm build && pnpm --filter @tangle-network/agent-bench run typecheck:public && pnpm --filter @tangle-network/agent-bench test && pnpm --filter @tangle-network/agent-bench run verify:package", "verify:cohort": "node scripts/verify-packed-cohort.mjs", - "verify:package": "pnpm run verify:static-imports && pnpm run build && pnpm run check:testing-fixture && pnpm run check:skills && publint && attw --pack --profile esm-only . && node scripts/verify-package-exports.mjs && pnpm run verify:edge-tool-loop", + "verify:package": "pnpm run check:model-execution-boundary && pnpm run verify:static-imports && pnpm run build && pnpm run check:testing-fixture && pnpm run check:skills && publint && attw --pack --profile esm-only . && node scripts/verify-package-exports.mjs && pnpm run verify:edge-tool-loop", "verify:official-optimizers": "node scripts/verify-official-optimizers.mjs", "verify:primeintellect": "pnpm build && node scripts/verify-primeintellect.mjs", "verify:primeintellect:live": "pnpm build && node scripts/verify-primeintellect-live.mjs", diff --git a/scripts/check-model-execution-boundary.mjs b/scripts/check-model-execution-boundary.mjs new file mode 100644 index 00000000..9fbce2e3 --- /dev/null +++ b/scripts/check-model-execution-boundary.mjs @@ -0,0 +1,233 @@ +#!/usr/bin/env node + +import { readdirSync, readFileSync, statSync } from 'node:fs' +import { relative, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import ts from 'typescript' + +const root = resolve(import.meta.dirname, '..') +const sourceRoots = ['src', 'bench', 'examples', 'scripts'] + +// Runtime owns provider transport. These four bench programs intentionally test the wire itself; +// adding another exception requires editing this reviewed list rather than dropping a magic comment +// beside the bypass. +const directTransportOwners = new Set([ + 'src/backends.ts', + 'src/runtime/router-client.ts', + 'src/runtime/supervise/runtime.ts', + 'bench/src/atom-mcp-e2e.mts', + 'bench/src/egress-probe.mts', + 'bench/src/mcp-mount-probe.mts', + 'bench/src/swe-arena/capacity.ts', +]) + +const sourceExtensions = new Set(['.js', '.mjs', '.cjs', '.ts', '.mts', '.cts', '.py', '.sh']) +const ignoredDirectories = new Set([ + '.git', + 'coverage', + 'dist', + 'fixtures', + 'generated', + 'node_modules', +]) + +function extension(path) { + const match = /\.[^.\/]+$/.exec(path) + return match?.[0] ?? '' +} + +function walk(path) { + const entries = readdirSync(path) + const files = [] + for (const name of entries) { + if (ignoredDirectories.has(name)) continue + const child = resolve(path, name) + const stats = statSync(child) + if (stats.isDirectory()) files.push(...walk(child)) + else if (sourceExtensions.has(extension(name))) files.push(child) + } + return files +} + +function isTestFile(path) { + return /(?:^|\/)[^/]+\.(?:test|spec)\.[cm]?[jt]s$/.test(path) +} + +function sourceLocation(source, node) { + const point = source.getLineAndCharacterOfPosition(node.getStart(source)) + return `${point.line + 1}:${point.character + 1}` +} + +function namesModelEndpoint(text) { + if (/chat\/completions|api\.anthropic\.com/i.test(text)) return true + return ( + /\/responses(?:[?'"`]|$)/i.test(text) && + /openai|anthropic|tangle|router|model|inference|llm/i.test(text) + ) +} + +function isLocalTestTarget(path, target) { + return isTestFile(path) && /(?:localhost|127\.0\.0\.1|\[::1\]|\.test)(?::\d+)?\//i.test(target) +} + +export function checkJavaScript(path, text) { + const kind = path.endsWith('.ts') || path.endsWith('.mts') || path.endsWith('.cts') + ? ts.ScriptKind.TS + : ts.ScriptKind.JS + const source = ts.createSourceFile(path, text, ts.ScriptTarget.Latest, true, kind) + const initializers = new Map() + const failures = [] + + function collect(node) { + if ( + ts.isVariableDeclaration(node) && + ts.isIdentifier(node.name) && + node.initializer !== undefined + ) { + initializers.set(node.name.text, node.initializer) + } + ts.forEachChild(node, collect) + } + collect(source) + + function expressionText(node, seen = new Set()) { + if (ts.isIdentifier(node)) { + if (seen.has(node.text)) return node.getText(source) + const initializer = initializers.get(node.text) + if (initializer !== undefined) { + seen.add(node.text) + return `${node.getText(source)}=${expressionText(initializer, seen)}` + } + } + return node.getText(source) + } + + function inspect(node) { + if (ts.isCallExpression(node)) { + const callee = node.expression.getText(source) + const first = node.arguments[0] + const target = first === undefined ? '' : expressionText(first) + const call = node.getText(source) + const directFetch = + callee === 'fetch' && namesModelEndpoint(target) && !isLocalTestTarget(path, target) + const providerSdk = + /(?:^|\.)(?:chat\.completions\.create|responses\.create|messages\.create|generateContent)$/.test( + callee, + ) + const rawHttp = + /^(?:https?|request|axios)(?:\.|$)/.test(callee) && + namesModelEndpoint(call) && + !isLocalTestTarget(path, call) + if (directFetch || providerSdk || rawHttp) { + failures.push({ node, detail: call.slice(0, 180).replace(/\s+/g, ' ') }) + } + } + if ( + ts.isNewExpression(node) && + /^(?:OpenAI|Anthropic)$/.test(node.expression.getText(source)) + ) { + failures.push({ node, detail: node.getText(source).slice(0, 180).replace(/\s+/g, ' ') }) + } + ts.forEachChild(node, inspect) + } + inspect(source) + return failures.map(({ node, detail }) => ({ location: sourceLocation(source, node), detail })) +} + +function executablePythonLines(text) { + const lines = text.split(/\r?\n/) + let quote = null + return lines.map((line) => { + let code = line + let cursor = 0 + let kept = '' + while (cursor < code.length) { + if (quote !== null) { + const end = code.indexOf(quote, cursor) + if (end === -1) return '' + cursor = end + 3 + quote = null + continue + } + const single = code.indexOf("'''", cursor) + const double = code.indexOf('"""', cursor) + const starts = [single, double].filter((value) => value >= 0) + const start = starts.length === 0 ? -1 : Math.min(...starts) + if (start === -1) { + kept += code.slice(cursor) + break + } + kept += code.slice(cursor, start) + quote = code.slice(start, start + 3) + cursor = start + 3 + } + code = kept.trimStart().startsWith('#') ? '' : kept.replace(/\s+#.*$/, '') + return code + }) +} + +export function checkPython(text) { + const failures = [] + for (const [index, line] of executablePythonLines(text).entries()) { + if ( + /chat\/completions|api\.anthropic\.com|\.chat\.completions\.create\s*\(|\.responses\.create\s*\(|\.messages\.create\s*\(/i.test( + line, + ) + ) { + failures.push({ location: `${index + 1}:1`, detail: line.trim().slice(0, 180) }) + } + } + return failures +} + +export function checkShell(text) { + const failures = [] + for (const [index, line] of text.split(/\r?\n/).entries()) { + const code = line.replace(/^\s*#.*$/, '') + if ( + /(?:curl|wget|http)\b.*(?:chat\/completions|\/responses\b|api\.anthropic\.com)/i.test(code) + ) { + failures.push({ location: `${index + 1}:1`, detail: code.trim().slice(0, 180) }) + } + } + return failures +} + +export function scanRepository() { + const violations = [] + for (const sourceRoot of sourceRoots) { + const path = resolve(root, sourceRoot) + for (const file of walk(path)) { + const repoPath = relative(root, file).replaceAll('\\', '/') + if (directTransportOwners.has(repoPath)) continue + const text = readFileSync(file, 'utf8') + const ext = extension(file) + const failures = + ext === '.py' + ? checkPython(text) + : ext === '.sh' + ? checkShell(text) + : checkJavaScript(repoPath, text) + for (const failure of failures) violations.push({ path: repoPath, ...failure }) + } + } + return violations +} + +function main() { + const violations = scanRepository() + if (violations.length > 0) { + process.stderr.write( + 'Model-provider calls must go through agent-runtime. Use AgentProfile + streamAgentTurn, ' + + 'supervise, or routerChatWithUsage/routerChatWithTools.\n', + ) + for (const violation of violations) { + process.stderr.write(`- ${violation.path}:${violation.location} ${violation.detail}\n`) + } + process.exitCode = 1 + return + } + process.stdout.write('model execution boundary: pass\n') +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main() diff --git a/scripts/check-model-execution-boundary.test.mjs b/scripts/check-model-execution-boundary.test.mjs new file mode 100644 index 00000000..19e8fca9 --- /dev/null +++ b/scripts/check-model-execution-boundary.test.mjs @@ -0,0 +1,51 @@ +import { describe, expect, it } from 'vitest' +import { + checkJavaScript, + checkPython, + checkShell, +} from './check-model-execution-boundary.mjs' + +describe('model execution boundary source check', () => { + it('rejects direct provider HTTP even when the endpoint is held in a variable', () => { + const violations = checkJavaScript( + 'examples/direct.ts', + `const endpoint = 'https://router.tangle.tools/v1/chat/completions'\nawait fetch(endpoint)`, + ) + expect(violations).toHaveLength(1) + expect(violations[0]?.location).toBe('2:7') + }) + + it('rejects provider SDK calls', () => { + expect( + checkJavaScript('examples/direct.ts', `await client.chat.completions.create({ model: 'x' })`), + ).toHaveLength(1) + expect( + checkJavaScript('examples/direct.ts', `const client = new Anthropic({ apiKey: 'x' })`), + ).toHaveLength(1) + }) + + it('ignores comments, inert strings, and ordinary HTTP', () => { + const source = ` + // fetch('https://api.openai.com/v1/chat/completions') + const documentation = "client.responses.create({ model: 'x' })" + await fetch('https://example.com/responses') + await fetch('https://example.com/data') + ` + expect(checkJavaScript('examples/ordinary.ts', source)).toEqual([]) + }) + + it('allows a local fake endpoint only in a test file', () => { + const source = `await fetch('http://127.0.0.1:43123/v1/chat/completions')` + expect(checkJavaScript('tests/local.test.ts', source)).toEqual([]) + expect(checkJavaScript('examples/local.ts', source)).toHaveLength(1) + }) + + it('rejects executable Python and shell calls but ignores comments and docstrings', () => { + expect( + checkPython(`"""requests.post('/v1/chat/completions')"""\nclient.messages.create(model='x')`), + ).toHaveLength(1) + expect(checkPython(`# client.messages.create(model='x')`)).toEqual([]) + expect(checkShell(`# curl https://api.openai.com/v1/chat/completions`)).toEqual([]) + expect(checkShell(`curl https://api.openai.com/v1/chat/completions`)).toHaveLength(1) + }) +}) diff --git a/src/runtime/index.ts b/src/runtime/index.ts index bcadf893..00c1f25a 100644 --- a/src/runtime/index.ts +++ b/src/runtime/index.ts @@ -529,7 +529,12 @@ export { } from './supervise/budget' // The completion-oracle: settled ⟺ DELIVERED. `gateOnDeliverable` wraps an executor so its // settlement `valid` reflects a deployable deliverable check (a test/judge), never self-report. -export { type DeliverableSpec, gateOnDeliverable } from './supervise/completion-gate' +export { + type DeliverableSpec, + type ExecutorResultMapping, + gateOnDeliverable, + mapExecutorResult, +} from './supervise/completion-gate' // The CHEAP / offline driver: an in-process router-tools loop that drives the coordination // verbs over the Scope (no box, no creds). The CAPABLE driver is an external harness with the // coordination verbs mounted as an MCP: `supervise()` wires a local bridge automatically, while a diff --git a/src/runtime/inline-sandbox-client.ts b/src/runtime/inline-sandbox-client.ts index 8980e445..87cea16c 100644 --- a/src/runtime/inline-sandbox-client.ts +++ b/src/runtime/inline-sandbox-client.ts @@ -12,7 +12,10 @@ * `answerOutput`/the kernel's cost ledger already parse — no sessions, no fs, * no fork (those degrade gracefully via the optional `SandboxClient` methods). */ + +import { type AgentProfile, agentProfileSchema } from '@tangle-network/agent-interface' import type { CreateSandboxOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' +import { ValidationError } from '../errors' import type { AgentSpec, Executor, ExecutorFactory, ExecutorResult } from './supervise/types' import type { SandboxClient } from './types' @@ -41,7 +44,10 @@ async function settle( * instantiated fresh per `streamPrompt` (mirrors the per-spawn executor lifecycle): * run once on the prompt, emit the terminal result event, tear down. */ -export function inlineSandboxClient(factory: ExecutorFactory): SandboxClient { +export function inlineSandboxClient( + factory: ExecutorFactory, + defaults: { profile?: AgentProfile } = {}, +): SandboxClient { let seq = 0 return { async create(options?: CreateSandboxOptions): Promise { @@ -70,10 +76,26 @@ export function inlineSandboxClient(factory: ExecutorFactory): SandboxC if (callerSignal.aborted) onAbort() else callerSignal.addEventListener('abort', onAbort, { once: true }) } - const spec: AgentSpec = { profile: { name: id }, harness: null } + const requestedProfile = + defaults.profile ?? + (options?.backend && typeof options.backend === 'object' + ? (options.backend as { profile?: unknown }).profile + : undefined) + const parsedProfile = agentProfileSchema.safeParse(requestedProfile) + if (!parsedProfile.success) { + throw new Error( + 'inlineSandboxClient: an exact AgentProfile is required; pass defaults.profile or create({ backend: { profile } })', + ) + } + const spec: AgentSpec = { profile: parsedProfile.data, harness: null } const exec = factory(spec, { signal: controller.signal, seams: { createOptions } }) try { const artifact = await settle(exec, message, controller.signal) + if (artifact.spent.tokensKnown === false || artifact.spent.usdKnown === false) { + throw new ValidationError( + 'inlineSandboxClient: cannot project unknown executor usage through SandboxEvent without turning unknown into zero', + ) + } const out = artifact.out as { content?: string } | undefined // Speak the runtime's metering protocol: `extractLlmCallEvent` reads // flat `llm_call` events, not the nested result payload — without diff --git a/src/runtime/router-client.complete.test.ts b/src/runtime/router-client.complete.test.ts index 15d1c329..94054fbc 100644 --- a/src/runtime/router-client.complete.test.ts +++ b/src/runtime/router-client.complete.test.ts @@ -15,6 +15,7 @@ describe('RouterConfig.complete — the injected completion transport', () => { // The body is the OpenAI request the client built — assert it threaded the model + messages. expect(body.model).toBe('deepseek-v4-flash') expect(body.messages).toEqual([{ role: 'user', content: 'hi' }]) + expect(body).not.toHaveProperty('max_tokens') return { choices: [{ message: { content: 'pong' } }], usage: { prompt_tokens: 7, completion_tokens: 3 }, @@ -86,6 +87,109 @@ describe('RouterConfig.complete — the injected completion transport', () => { expect(res.content).toBe('live') expect(fetchSpy).toHaveBeenCalledOnce() }) + + it('uses the caller ceiling when set and otherwise leaves the provider default unrestricted', async () => { + const seen: Record[] = [] + const complete = async (body: Record) => { + seen.push(body) + return { choices: [{ message: { content: 'done' } }] } + } + const config = { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'k', + model: 'deepseek-v4-flash', + complete, + } + await routerChatWithUsage(config, [{ role: 'user', content: 'default' }]) + await routerChatWithUsage({ ...config, maxTokens: 16_384 }, [ + { role: 'user', content: 'configured' }, + ]) + await routerChatWithUsage(config, [{ role: 'user', content: 'per-call' }], { + maxTokens: 32_768, + }) + + expect(seen[0]).not.toHaveProperty('max_tokens') + expect(seen[1]?.max_tokens).toBe(16_384) + expect(seen[2]?.max_tokens).toBe(32_768) + }) + + it('accepts multimodal messages and provider fields without letting extras replace canonical fields', async () => { + const content = [ + { type: 'text', text: 'describe this image' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,AA==' } }, + ] + const complete = vi.fn(async (body: Record) => { + expect(body).toMatchObject({ + model: 'deepseek-v4-flash', + messages: [{ role: 'user', content }], + temperature: 0.4, + max_tokens: 32_768, + seed: 7, + thinking: { type: 'enabled' }, + }) + return { choices: [{ message: { content: 'image' } }] } + }) + + await routerChatWithUsage( + { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'k', + model: 'deepseek-v4-flash', + complete, + }, + [{ role: 'user', content }], + { + temperature: 0.4, + maxTokens: 32_768, + seed: 7, + extraBody: { + model: 'must-not-win', + messages: [], + temperature: 999, + max_tokens: 1, + thinking: { type: 'enabled' }, + }, + }, + ) + expect(complete).toHaveBeenCalledOnce() + }) + + it('omits tool fields for a no-tool request and protects canonical fields from provider extras', async () => { + const complete = vi.fn(async (body: Record) => { + expect(body).toMatchObject({ + model: 'deepseek-v4-flash', + messages: [{ role: 'user', content: 'answer' }], + temperature: 0.1, + thinking: { type: 'enabled' }, + }) + expect(body).not.toHaveProperty('tools') + expect(body).not.toHaveProperty('tool_choice') + return { choices: [{ message: { content: 'done' } }] } + }) + + const result = await routerChatWithTools( + { + routerBaseUrl: 'http://router.test/v1', + routerKey: 'k', + model: 'deepseek-v4-flash', + complete, + }, + [{ role: 'user', content: 'answer' }], + [], + { + temperature: 0.1, + extraBody: { + model: 'must-not-win', + messages: [], + tools: [{ type: 'function' }], + tool_choice: 'required', + thinking: { type: 'enabled' }, + }, + }, + ) + expect(result.content).toBe('done') + expect(complete).toHaveBeenCalledOnce() + }) }) describe('reasoning-aware parsing and reasoning_effort forwarding', () => { diff --git a/src/runtime/router-client.ts b/src/runtime/router-client.ts index 47257dd9..42cec354 100644 --- a/src/runtime/router-client.ts +++ b/src/runtime/router-client.ts @@ -11,6 +11,7 @@ */ import { estimateCost, isModelPriced } from '@tangle-network/agent-eval' +import type { ReasoningEffort } from '@tangle-network/agent-interface' import { ValidationError } from '../errors' import { runBrainLoop, type ToolLoopChat } from './tool-loop' @@ -27,7 +28,7 @@ export interface RouterConfig { */ complete?: (body: Record) => Promise /** - * Ceiling for one completion, forwarded as `max_tokens`. Defaults to 8192. + * Optional ceiling for one completion, forwarded as `max_tokens`. * * A REASONING model spends this budget on hidden thinking BEFORE it emits a visible token, so * the default can truncate one mid-thought and return no content at all — observed live with a @@ -79,16 +80,25 @@ export interface RouterChatResult { usage?: { input: number; output: number } /** Derived from usage via `estimateCost` when the model is priced; else undefined. */ costUsd?: number + /** Provider terminal reason (`stop`, `length`, ...), when reported. */ + finishReason?: string } /** One OpenAI-compatible chat completion through the Tangle router, returning text + REAL token usage (`undefined` when the provider omits it — never a fabricated 0). */ export async function routerChatWithUsage( cfg: RouterConfig, - messages: Array<{ role: string; content: string }>, + messages: ReadonlyArray<{ role: string; content: unknown }>, opts?: { temperature?: number signal?: AbortSignal maxTokens?: number + /** OpenAI-compatible deterministic seed. Omit when the provider does not support it. */ + seed?: number + /** + * Provider-specific request fields such as Z.AI's `thinking` object. + * Canonical fields are written after these extras and cannot be overridden here. + */ + extraBody?: Readonly> /** * Reasoning control for thinking models, forwarded as `reasoning_effort`. * 'none' is the load-bearing value: binary/single-token decisions (routing, @@ -97,19 +107,29 @@ export async function routerChatWithUsage( * timeout, not just waste. Providers that ignore the field are handled by * the reasoning/content split in `parseChatResult`. */ - reasoningEffort?: 'none' | 'low' | 'medium' | 'high' + reasoningEffort?: ReasoningEffort }, ): Promise { const url = `${cfg.routerBaseUrl.replace(/\/$/, '')}/chat/completions` const headers = { 'content-type': 'application/json', authorization: `Bearer ${cfg.routerKey}` } let temperature = opts?.temperature ?? 0.2 - // max_tokens default is generous: THINKING models (kimi-k2.6) spend the budget on - // reasoning_content first — a small router default yields EMPTY content. + const maxTokens = opts?.maxTokens ?? cfg.maxTokens const body = (): Record => ({ + ...providerRequestExtras(opts?.extraBody, [ + 'model', + 'messages', + 'temperature', + 'max_tokens', + 'seed', + 'reasoning_effort', + 'stream', + 'stream_options', + ]), model: cfg.model, messages, temperature, - max_tokens: opts?.maxTokens ?? 8192, + ...(maxTokens !== undefined ? { max_tokens: maxTokens } : {}), + ...(opts?.seed !== undefined ? { seed: opts.seed } : {}), ...(opts?.reasoningEffort ? { reasoning_effort: opts.reasoningEffort } : {}), }) // Injected transport short-circuits the network: the offline benchmark seam. It owns its own @@ -151,6 +171,7 @@ function parseChatResult(json: unknown, model: string): RouterChatResult { const data = json as { choices?: Array<{ message?: { content?: string; reasoning?: string; reasoning_content?: string } + finish_reason?: string }> usage?: { prompt_tokens?: number; completion_tokens?: number } } @@ -171,6 +192,9 @@ function parseChatResult(json: unknown, model: string): RouterChatResult { ...(reasoning ? { reasoning } : {}), ...(usage ? { usage } : {}), ...(costUsd !== undefined ? { costUsd } : {}), + ...(typeof data.choices?.[0]?.finish_reason === 'string' + ? { finishReason: data.choices[0].finish_reason } + : {}), } } @@ -255,6 +279,8 @@ export async function routerChatWithTools( signal?: AbortSignal toolChoice?: 'auto' | 'required' | 'none' maxTokens?: number + /** Provider-specific request fields; canonical fields cannot be overridden here. */ + extraBody?: Readonly> }, ): Promise { const body = toolCompletionBody(cfg, messages, tools, opts) @@ -277,6 +303,7 @@ export async function routerChatWithTools( content?: string | null tool_calls?: Array<{ id?: string; function?: { name?: string; arguments?: string } }> } + finish_reason?: string }> usage?: { prompt_tokens?: number; completion_tokens?: number } } @@ -292,6 +319,9 @@ export async function routerChatWithTools( toolCalls, ...(usage ? { usage } : {}), ...(costUsd !== undefined ? { costUsd } : {}), + ...(typeof data.choices?.[0]?.finish_reason === 'string' + ? { finishReason: data.choices[0].finish_reason } + : {}), } } @@ -301,18 +331,42 @@ function toolCompletionBody( cfg: RouterConfig, messages: ReadonlyArray>, tools: ReadonlyArray, - opts?: { temperature?: number; toolChoice?: 'auto' | 'required' | 'none'; maxTokens?: number }, + opts?: { + temperature?: number + toolChoice?: 'auto' | 'required' | 'none' + maxTokens?: number + extraBody?: Readonly> + }, ): Record { return { + ...providerRequestExtras(opts?.extraBody, [ + 'model', + 'messages', + 'tools', + 'tool_choice', + 'temperature', + 'max_tokens', + 'stream', + 'stream_options', + ]), model: cfg.model, messages, - tools, - tool_choice: opts?.toolChoice ?? 'auto', + ...(tools.length > 0 ? { tools, tool_choice: opts?.toolChoice ?? 'auto' } : {}), temperature: opts?.temperature ?? 0.3, ...(opts?.maxTokens ? { max_tokens: opts.maxTokens } : {}), } } +function providerRequestExtras( + extraBody: Readonly> | undefined, + reservedFields: ReadonlyArray, +): Record { + if (!extraBody) return {} + const extras = { ...extraBody } + for (const field of reservedFields) delete extras[field] + return extras +} + /** * REAL usage → the metered pair, or `undefined` when the provider reported none. Never a * fabricated 0: a phantom 0 reads as a free call to the conserved budget pool, which would then @@ -442,6 +496,8 @@ export async function streamRouterChatWithTools( signal?: AbortSignal toolChoice?: 'auto' | 'required' | 'none' maxTokens?: number + /** Provider-specific request fields; canonical streaming fields cannot be overridden here. */ + extraBody?: Readonly> }, ): Promise { if (cfg.complete) { @@ -661,6 +717,7 @@ function chatWithTools( signal?: AbortSignal toolChoice?: 'auto' | 'required' | 'none' maxTokens?: number + extraBody?: Readonly> }, ): Promise { return cfg.stream === true @@ -725,7 +782,7 @@ export async function routerToolLoop( chat: (messages, toolSpecs) => chatWithTools(cfg, messages, toolSpecs, { ...(opts?.temperature !== undefined ? { temperature: opts.temperature } : {}), - ...(opts?.maxTokens ? { maxTokens: opts.maxTokens } : {}), + ...(opts?.maxTokens !== undefined ? { maxTokens: opts.maxTokens } : {}), ...(opts?.signal ? { signal: opts.signal } : {}), }), tools, diff --git a/src/runtime/stream-agent-turn.test.ts b/src/runtime/stream-agent-turn.test.ts index 5952dccf..4fb8a2a1 100644 --- a/src/runtime/stream-agent-turn.test.ts +++ b/src/runtime/stream-agent-turn.test.ts @@ -11,9 +11,12 @@ import type { SandboxEvent } from '@tangle-network/sandbox' import { describe, expect, it } from 'vitest' import type { AgentExecutionBackend, RuntimeStreamEvent } from '../types' import { inProcessSandboxClient } from './in-process-sandbox-client' -import { collectAgentTurn, streamAgentTurn } from './stream-agent-turn' +import { collectAgentTurn, streamAgentTurn, streamObservedAgentTurn } from './stream-agent-turn' +import { attestRuntimeOwnedExecutor } from './supervise/materialization' import type { Executor, ExecutorFactory, ExecutorResult } from './supervise/types' +const TEST_PROFILE = { name: 'stream-agent-turn-test' } as const + function finalOf(events: RuntimeStreamEvent[]): RuntimeStreamEvent & { type: 'final' } { const final = events.at(-1) if (final?.type !== 'final') throw new Error('no terminal final event') @@ -35,7 +38,7 @@ describe('streamAgentTurn: box backend', () => { ] as SandboxEvent[]) const seen: RuntimeStreamEvent[] = [] - for await (const event of streamAgentTurn({ kind: 'box', box }, 'say hello')) { + for await (const event of streamObservedAgentTurn({ kind: 'box', box }, 'say hello')) { seen.push(event) } // Incremental events surface in order, before the terminal event. @@ -62,9 +65,9 @@ describe('streamAgentTurn: box backend', () => { { type: 'done', data: { tokenUsage: { inputTokens: 7, outputTokens: 3 } } }, ] as SandboxEvent[]) - const turn = await collectAgentTurn(streamAgentTurn({ kind: 'box', box }, 'answer')) + const turn = await collectAgentTurn(streamObservedAgentTurn({ kind: 'box', box }, 'answer')) expect(turn.finalText).toBe('42') - expect(turn.usage).toEqual({ input: 7, output: 3 }) + expect(turn.usage).toEqual({ input: 7, output: 3, usdKnown: false }) expect(turn.status).toBe('completed') expect(turn.events.map((e) => e.type)).toEqual([ 'backend_start', @@ -82,7 +85,7 @@ describe('streamAgentTurn: box backend', () => { }, }) const box = await client.create() - const turn = await collectAgentTurn(streamAgentTurn({ kind: 'box', box }, 'boom')) + const turn = await collectAgentTurn(streamObservedAgentTurn({ kind: 'box', box }, 'boom')) expect(turn.status).toBe('failed') expect(turn.error).toMatchObject({ kind: 'backend', message: 'box exploded' }) const types = turn.events.map((e) => e.type) @@ -113,8 +116,12 @@ describe('streamAgentTurn: current Sandbox prompt options', () => { }) const box = await client.create() const turn = await collectAgentTurn( - streamAgentTurn( - { kind: 'box', box, options: { sessionId: 'sess-1', model: 'kimi-k2' } }, + streamObservedAgentTurn( + { + kind: 'box', + box, + options: { sessionId: 'sess-1', model: 'kimi-k2' }, + }, 'do the task', ), ) @@ -141,7 +148,7 @@ describe('streamAgentTurn: current Sandbox prompt options', () => { }) const box = await client.create() const turn = await collectAgentTurn( - streamAgentTurn({ kind: 'box', box }, 'hang', { timeoutMs: 25 }), + streamObservedAgentTurn({ kind: 'box', box }, 'hang', { timeoutMs: 25 }), ) expect(turn.status).toBe('failed') expect(turn.error?.message).toContain('timed out after 25ms') @@ -196,7 +203,7 @@ describe('streamAgentTurn: tool-part preservation (opt-in)', () => { it('preserveToolParts: true surfaces deduped tool_call/tool_result in-stream', async () => { const box = await makeBox(toolFrames) const turn = await collectAgentTurn( - streamAgentTurn({ kind: 'box', box }, 'list files', { preserveToolParts: true }), + streamObservedAgentTurn({ kind: 'box', box }, 'list files', { preserveToolParts: true }), ) expect(turn.events.map((e) => e.type)).toEqual([ 'backend_start', @@ -214,12 +221,12 @@ describe('streamAgentTurn: tool-part preservation (opt-in)', () => { expect(result).toMatchObject({ toolName: 'bash', toolCallId: 'call-1', result: 'file.txt' }) // The projection is additive: text/usage folding is unchanged. expect(turn.finalText).toBe('listed') - expect(turn.usage).toEqual({ input: 5, output: 2 }) + expect(turn.usage).toEqual({ input: 5, output: 2, usdKnown: false }) }) it('default (off) leaves the stream vocabulary unchanged — no tool events', async () => { const box = await makeBox(toolFrames) - const turn = await collectAgentTurn(streamAgentTurn({ kind: 'box', box }, 'list files')) + const turn = await collectAgentTurn(streamObservedAgentTurn({ kind: 'box', box }, 'list files')) expect(turn.events.map((e) => e.type)).toEqual([ 'backend_start', 'text_delta', @@ -244,7 +251,7 @@ describe('streamAgentTurn: tool-part preservation (opt-in)', () => { { type: 'done', data: { tokenUsage: { inputTokens: 1, outputTokens: 1 } } }, ] as SandboxEvent[]) const turn = await collectAgentTurn( - streamAgentTurn({ kind: 'box', box }, 'fetch', { preserveToolParts: true }), + streamObservedAgentTurn({ kind: 'box', box }, 'fetch', { preserveToolParts: true }), ) const types = turn.events.map((e) => e.type) expect(types).toEqual(['backend_start', 'tool_call', 'tool_result', 'llm_call', 'final']) @@ -264,7 +271,7 @@ describe('streamAgentTurn: tool-part preservation (opt-in)', () => { }) const box = await client.create() const turn = await collectAgentTurn( - streamAgentTurn({ kind: 'box', box }, 'search', { preserveToolParts: true }), + streamObservedAgentTurn({ kind: 'box', box }, 'search', { preserveToolParts: true }), ) expect(turn.events.map((e) => e.type)).toEqual([ 'backend_start', @@ -291,7 +298,7 @@ describe('streamAgentTurn: raw-event tap (onRawEvent)', () => { ] as SandboxEvent[], }) const box = await client.create() - const stream = streamAgentTurn({ kind: 'box', box }, 'go', { + const stream = streamObservedAgentTurn({ kind: 'box', box }, 'go', { onRawEvent: async (event) => { // Async on purpose: the drive must AWAIT the tap before projecting. await Promise.resolve() @@ -334,7 +341,7 @@ describe('streamAgentTurn: mid-stream lifecycle (pull-based, no extra API)', () }, }) const box = await client.create() - for await (const event of streamAgentTurn({ kind: 'box', box }, 'go')) { + for await (const event of streamObservedAgentTurn({ kind: 'box', box }, 'go')) { log.push(`consumed:${event.type}`) // The mid-stream escape: arbitrary awaited work (a vault sync, a retry // decision) runs here while the producer is suspended. @@ -380,11 +387,14 @@ describe('streamAgentTurn: mid-stream lifecycle (pull-based, no extra API)', () let synced = false async function* withLifecycle(): AsyncGenerator { - const first = await collectAgentTurn(streamAgentTurn({ kind: 'box', box }, 'attempt')) + const first = await collectAgentTurn(streamObservedAgentTurn({ kind: 'box', box }, 'attempt')) const noop = first.finalText === '' && first.status === 'completed' if (noop) { // Retry with a steering prompt — the first `final` is never forwarded. - for await (const event of streamAgentTurn({ kind: 'box', box }, 'attempt (retry)')) { + for await (const event of streamObservedAgentTurn( + { kind: 'box', box }, + 'attempt (retry)', + )) { if (event.type === 'final') { synced = true // pre-done lifecycle work completes before forwarding } @@ -408,38 +418,61 @@ describe('streamAgentTurn: executor backend', () => { onTeardown?: () => void hangUntilAbort?: boolean }): ExecutorFactory { - return (_spec, ctx): Executor => ({ - runtime: 'inline', - async execute(task, signal): Promise> { - if (opts?.hangUntilAbort) { - await new Promise((_resolve, reject) => { - const onAbort = () => reject(signal.reason ?? new Error('aborted')) - if (signal.aborted) onAbort() - else signal.addEventListener('abort', onAbort, { once: true }) - // ctx.signal must be the same channel — assert linkage indirectly. - if (ctx.signal.aborted) onAbort() - }) - } - return { - outRef: 'stub-1', - out: { content: `echo: ${String(task)}` }, - spent: { iterations: 1, tokens: { input: 11, output: 6 }, usd: 0.005, ms: 1 }, - } - }, - async teardown() { - opts?.onTeardown?.() - return { destroyed: true } - }, - resultArtifact(): ExecutorResult { - throw new Error('one-shot executor: resultArtifact unused') - }, - }) + return (spec, ctx): Executor => { + const attemptId = ctx.node?.attemptId ?? 'stub-attempt' + const executor: Executor = { + runtime: 'inline', + async execute(task, signal): Promise> { + if (opts?.hangUntilAbort) { + await new Promise((_resolve, reject) => { + const onAbort = () => reject(signal.reason ?? new Error('aborted')) + if (signal.aborted) onAbort() + else signal.addEventListener('abort', onAbort, { once: true }) + // ctx.signal must be the same channel — assert linkage indirectly. + if (ctx.signal.aborted) onAbort() + }) + } + return { + outRef: 'stub-1', + out: { content: `echo: ${String(task)}` }, + spent: { iterations: 1, tokens: { input: 11, output: 6 }, usd: 0.005, ms: 1 }, + } + }, + async teardown() { + opts?.onTeardown?.() + return { destroyed: true } + }, + resultArtifact(): ExecutorResult { + throw new Error('one-shot executor: resultArtifact unused') + }, + } + return attestRuntimeOwnedExecutor( + executor, + { + effectiveProfile: spec.profile, + backend: 'inline-test', + model: { status: 'unknown', reason: 'offline test executor has no model' }, + execution: { kind: 'request', id: attemptId }, + materializer: 'offline-test-executor', + plan: { kind: 'offline-test' }, + }, + { + attemptId, + binding: { kind: 'offline-test', attemptId }, + descriptor: { kind: 'offline-test', transport: 'in-process' }, + }, + ) + } } it('runs the factory once and terminates with the executor usage', async () => { let toreDown = 0 const stream = streamAgentTurn( - { kind: 'executor', factory: stubFactory({ onTeardown: () => toreDown++ }) }, + { + kind: 'executor', + factory: stubFactory({ onTeardown: () => toreDown++ }), + profile: TEST_PROFILE, + }, 'ping', ) const turn = await collectAgentTurn(stream) @@ -447,7 +480,12 @@ describe('streamAgentTurn: executor backend', () => { expect(turn.usage).toEqual({ input: 11, output: 6, costUsd: 0.005 }) expect(turn.status).toBe('completed') // Incremental metering surfaces before the terminal event. - expect(turn.events.map((e) => e.type)).toEqual(['backend_start', 'llm_call', 'final']) + expect(turn.events.map((e) => e.type)).toEqual([ + 'backend_start', + 'llm_call', + 'artifact', + 'final', + ]) expect(toreDown).toBe(1) }) @@ -458,6 +496,7 @@ describe('streamAgentTurn: executor backend', () => { { kind: 'executor', factory: stubFactory({ hangUntilAbort: true, onTeardown: () => toreDown++ }), + profile: TEST_PROFILE, }, 'hang', { signal: controller.signal }, @@ -492,7 +531,10 @@ describe('streamAgentTurn: chat backend', () => { it('streams normalized events and terminates with usage + model', async () => { const seen: RuntimeStreamEvent[] = [] - for await (const event of streamAgentTurn({ kind: 'chat', backend: stubChatBackend() }, 'hi')) { + for await (const event of streamObservedAgentTurn( + { kind: 'chat', backend: stubChatBackend() }, + 'hi', + )) { seen.push(event) } expect(seen.map((e) => e.type)).toEqual([ @@ -512,13 +554,14 @@ describe('streamAgentTurn: chat backend', () => { expect(final.metadata).toMatchObject({ tokenUsage: { input: 21, output: 9 }, model: 'glm-4.6', + usdKnown: false, }) expect(final.metadata).not.toHaveProperty('costUsd') }) it('abort mid-stream terminates with status aborted after partial deltas', async () => { const controller = new AbortController() - const stream = streamAgentTurn( + const stream = streamObservedAgentTurn( { kind: 'chat', backend: stubChatBackend({ hangUntilAbort: true }) }, 'hang', { signal: controller.signal }, @@ -539,8 +582,11 @@ describe('streamAgentTurn: chat backend', () => { it('timeoutMs expiry terminates with status failed (not aborted)', async () => { const turn = await collectAgentTurn( - streamAgentTurn( - { kind: 'chat', backend: stubChatBackend({ hangUntilAbort: true }) }, + streamObservedAgentTurn( + { + kind: 'chat', + backend: stubChatBackend({ hangUntilAbort: true }), + }, 'slow', { timeoutMs: 25, diff --git a/src/runtime/stream-agent-turn.ts b/src/runtime/stream-agent-turn.ts index 79f3bb05..6374310b 100644 --- a/src/runtime/stream-agent-turn.ts +++ b/src/runtime/stream-agent-turn.ts @@ -1,8 +1,8 @@ /** * `streamAgentTurn` — the ONE run-a-turn event-stream contract over every * execution substrate: a sandbox box (`SandboxInstance.streamPrompt`), a - * one-shot `Executor` (cli-bridge / router / BYO, via `ExecutorFactory`), and - * an in-process `AgentExecutionBackend` (the `resolveAgentBackend` output). + * one-shot Runtime-owned `Executor` (cli-bridge / router / sandbox, via + * `ExecutorFactory`), and an in-process `AgentExecutionBackend`. * * One function, one vocabulary: every backend kind yields the existing * `RuntimeStreamEvent` union incrementally and ALWAYS terminates with a @@ -14,13 +14,14 @@ * This is a UNIFICATION seam, not a new stream parser — each kind is a thin * adapter over code that already exists and is already hardened: * - `box` — `mapSandboxEvent` + `extractLlmCallEvent` (sandbox-events.ts) - * project the sandbox event stream; nothing is re-mapped here. - * - `executor` — `inlineSandboxClient` (the ONE executor→box adapter) turns - * the factory into a box, then the box path drives it. The - * executor's settle/teardown lifecycle stays in that adapter. + * project the sandbox event stream; its requested profile is + * explicitly recorded as unverified because the box already exists. + * - `executor` — Runtime materializes the exact `AgentProfile`, records its + * identity receipts, drives the executor once, and tears it + * down after capturing the terminal artifact. * - `chat` — the backend's own `stream()` surface, normalized by - * `normalizeBackendStreamEvent` (the same projection - * `runAgentTaskStream` applies). + * `normalizeBackendStreamEvent`; its requested profile is likewise + * unverified because an arbitrary backend cannot attest its setup. * * Distinct from `openSandboxRun` (box-only, session resume over one persistent * artifact, raw `SandboxEvent` deliverables) and from `runAgentTaskStream` @@ -47,11 +48,17 @@ */ import { scoreKnowledgeReadiness } from '@tangle-network/agent-eval' +import { + type AgentProfile, + agentProfileSchema, + canonicalCandidateDigest, +} from '@tangle-network/agent-interface' import type { PromptOptions, SandboxEvent, SandboxInstance } from '@tangle-network/sandbox' import { normalizeBackendStreamEvent } from '../backends' -import { BackendTransportError } from '../errors' +import { BackendTransportError, ValidationError } from '../errors' import { newRuntimeSession, nowIso } from '../sessions' import type { + AgentBackendInput, AgentExecutionBackend, AgentTaskSpec, AgentTaskStatus, @@ -59,9 +66,25 @@ import type { RuntimeSession, RuntimeStreamEvent, } from '../types' -import { inlineSandboxClient } from './inline-sandbox-client' import { createSandboxToolPartState, mapSandboxEvent, mapSandboxToolEvent } from './sandbox-events' -import type { ExecutorFactory } from './supervise/types' +import { + authoredProfileDigest, + knownExecutionBindingReceipt, + knownMaterializationReceipt, + runtimeOwnedExecutorExecutionBinding, + runtimeOwnedExecutorMaterialization, + unknownExecutionBindingReceipt, + unknownMaterializationReceipt, +} from './supervise/materialization' +import type { + ExecutionBindingReceipt, + Executor, + ExecutorFactory, + ExecutorResult, + NodeExecutionIdentity, + ProfileMaterializationReceipt, + UsageEvent, +} from './supervise/types' /** * The execution substrate one turn runs on — a closed discriminated union over @@ -69,7 +92,19 @@ import type { ExecutorFactory } from './supervise/types' * * @experimental */ -export type AgentTurnBackend = +export type AgentTurnBackend = { + /** A Runtime-owned executor factory materialized from this exact canonical profile. */ + kind: 'executor' + factory: ExecutorFactory + /** Exact canonical identity materialized by the executor. */ + profile: AgentProfile + /** Model label stamped on cost-only `llm_call` events. Default `'agent'`. */ + agentRunName?: string +} + +/** Lower-level observation adapters. They normalize an already-created execution surface but do + * not bind an AgentProfile to it, so they are deliberately absent from the public kernel export. */ +type ObservedAgentTurnBackend = | { /** A live sandbox box: the turn is one `box.streamPrompt(prompt)` call. */ kind: 'box' @@ -85,18 +120,6 @@ export type AgentTurnBackend = /** Model label stamped on cost-only `llm_call` events. Default `'agent'`. */ agentRunName?: string } - | { - /** - * A one-shot `Executor` (cli-bridge / router / BYO): the factory is - * instantiated fresh for the turn via `inlineSandboxClient`, run once on - * the prompt, and torn down — the same per-spawn lifecycle the supervise - * runtime gives it. - */ - kind: 'executor' - factory: ExecutorFactory - /** Model label stamped on cost-only `llm_call` events. Default `'agent'`. */ - agentRunName?: string - } | { /** * An in-process `AgentExecutionBackend` (`resolveAgentBackend` output or @@ -106,6 +129,11 @@ export type AgentTurnBackend = backend: AgentExecutionBackend } +/** One prompt or an exact OpenAI-compatible conversation carried as the turn input. */ +export type AgentTurnInput = + | string + | { readonly messages: ReadonlyArray>> } + /** @experimental */ export interface StreamAgentTurnOptions { /** Caller-initiated cancellation. Terminates the stream with `final.status: 'aborted'`. */ @@ -137,18 +165,115 @@ export interface StreamAgentTurnOptions { onRawEvent?: (event: SandboxEvent) => void | Promise } +function turnIntent(input: AgentTurnInput): string { + if (typeof input === 'string') return input + for (let index = input.messages.length - 1; index >= 0; index -= 1) { + const message = input.messages[index] + if (message?.role === 'user' && typeof message.content === 'string') return message.content + } + return 'structured agent turn' +} + +function turnBackendInput(task: AgentTaskSpec, input: AgentTurnInput): AgentBackendInput { + if (typeof input === 'string') return { task, message: input } + return { + task, + messages: input.messages.map((message) => ({ ...message })) as AgentBackendInput['messages'], + } +} + +function executorEvidence( + executor: Executor, + profile: AgentProfile, + attemptId: string, +): { + materialization: ProfileMaterializationReceipt + executionBinding: ExecutionBindingReceipt +} { + const profileDigest = authoredProfileDigest(profile) + const declaration = runtimeOwnedExecutorMaterialization(executor) + if (!profileDigest || !declaration) { + const materialization = unknownMaterializationReceipt({ + ...(profileDigest ? { authoredProfileDigest: profileDigest } : {}), + runtime: executor.runtime, + reason: declaration ? 'invalid-executor-report' : 'executor-did-not-report', + }) + return { + materialization, + executionBinding: unknownExecutionBindingReceipt( + materialization, + attemptId, + declaration ? 'invalid-executor-report' : 'executor-did-not-report', + ), + } + } + try { + const materialization = knownMaterializationReceipt({ + authoredProfileDigest: profileDigest, + runtime: executor.runtime, + declaration, + }) + const binding = runtimeOwnedExecutorExecutionBinding(executor) + if (!binding || binding.attemptId !== attemptId) throw new Error('executor attempt id mismatch') + return { + materialization, + executionBinding: knownExecutionBindingReceipt(materialization, binding), + } + } catch { + const materialization = unknownMaterializationReceipt({ + authoredProfileDigest: profileDigest, + runtime: executor.runtime, + reason: 'invalid-executor-report', + }) + return { + materialization, + executionBinding: unknownExecutionBindingReceipt( + materialization, + attemptId, + 'invalid-executor-report', + ), + } + } +} + +function turnProvenance( + startedAt: number, + timeoutMs: number | undefined, + profileDigest: string | undefined, + taskDigest: string, + materialization: ProfileMaterializationReceipt | undefined, + executionBinding: ExecutionBindingReceipt | undefined, +): Record { + const endedAt = Date.now() + return { + ...(profileDigest ? { identity: { profileDigest, taskDigest } } : { taskDigest }), + ...(materialization ? { materialization } : {}), + ...(executionBinding ? { executionBindings: [executionBinding] } : {}), + budget: { timeoutMs: timeoutMs ?? null }, + timing: { + startedAt: new Date(startedAt).toISOString(), + endedAt: new Date(endedAt).toISOString(), + durationMs: endedAt - startedAt, + }, + } +} + /** * Metered usage of one turn, summed over every cost-bearing event the backend - * emitted. `input`/`output` are token counts (0 when the backend reported - * none — the honest sum, never a fabricated estimate). `costUsd`/`model` are - * present only when the backend actually reported them. + * emitted. `input`/`output` are token counts and are accompanied by + * `tokensKnown: false` when the backend did not report them. `costUsd`/`model` + * are present only when the backend actually reported them. * * @experimental */ export interface AgentTurnUsage { input: number output: number + /** Present when a real turn ran but the provider did not report token usage. */ + tokensKnown?: false costUsd?: number + /** Present when Runtime could not prove the full dollar amount. */ + usdKnown?: false model?: string } @@ -162,6 +287,7 @@ export interface AgentTurnUsage { export interface CollectedAgentTurn { finalText: string usage: AgentTurnUsage + toolCalls: Array<{ id?: string; name: string; arguments: string }> events: RuntimeStreamEvent[] status: AgentTaskStatus error?: BackendErrorDetail @@ -178,7 +304,12 @@ interface TurnAccumulator { input: number output: number costUsd: number + tokensKnown: boolean + usdKnown: boolean + sawLlmCall: boolean model?: string + stopReason?: string + result?: ExecutorResult } /** @@ -193,44 +324,159 @@ interface TurnAccumulator { */ export async function* streamAgentTurn( backend: AgentTurnBackend, - prompt: string, + input: AgentTurnInput, + opts: StreamAgentTurnOptions = {}, +): AsyncGenerator { + yield* streamAgentTurnInternal(backend, input, opts) +} + +/** @internal Normalize a pre-created box/chat stream without asserting an AgentProfile identity. */ +export async function* streamObservedAgentTurn( + backend: ObservedAgentTurnBackend, + input: AgentTurnInput, opts: StreamAgentTurnOptions = {}, +): AsyncGenerator { + yield* streamAgentTurnInternal(backend, input, opts) +} + +async function* streamAgentTurnInternal( + backend: AgentTurnBackend | ObservedAgentTurnBackend, + input: AgentTurnInput, + opts: StreamAgentTurnOptions, ): AsyncGenerator { const label = backend.kind === 'chat' ? backend.backend.kind : backend.kind - const task: AgentTaskSpec = { id: `turn-${crypto.randomUUID()}`, intent: prompt } - const acc: TurnAccumulator = { deltaText: '', input: 0, output: 0, costUsd: 0 } + const profile = + backend.kind === 'executor' ? agentProfileSchema.parse(backend.profile) : undefined + const profileDigest = profile ? canonicalCandidateDigest(profile) : undefined + const taskInput = input + const taskDigest = canonicalCandidateDigest(taskInput) + const task: AgentTaskSpec = { + id: `turn-${crypto.randomUUID()}`, + intent: turnIntent(input), + ...(profileDigest + ? { + metadata: { + identity: { profileDigest, taskDigest } satisfies NodeExecutionIdentity, + }, + } + : {}), + } + const acc: TurnAccumulator = { + deltaText: '', + input: 0, + output: 0, + costUsd: 0, + tokensKnown: false, + usdKnown: false, + sawLlmCall: false, + } const deadline = deriveTurnSignal(opts.signal, opts.timeoutMs ?? 0) + const startedAt = Date.now() let session: RuntimeSession | undefined + let executor: Executor | undefined + let materialization: ProfileMaterializationReceipt | undefined + let executionBinding: ExecutionBindingReceipt | undefined try { - session = await startTurnSession(backend, task, prompt, deadline.signal, label) - yield { type: 'backend_start', task, session, backend: label, timestamp: nowIso() } + const nodeId = task.id + const attemptId = `${nodeId}:attempt:${crypto.randomUUID()}` + if (backend.kind === 'executor') { + executor = backend.factory( + { profile: profile!, harness: null }, + { + signal: deadline.signal, + seams: {}, + node: { + rootId: nodeId, + parentId: nodeId, + nodeId, + attemptId, + identity: { profileDigest: profileDigest!, taskDigest }, + }, + }, + ) + ;({ materialization, executionBinding } = executorEvidence(executor, profile!, attemptId)) + if (materialization.status !== 'known' || executionBinding.status !== 'known') { + throw new ValidationError( + 'streamAgentTurn: exact profile execution requires a Runtime-owned executor with valid materialization and execution binding evidence', + ) + } + if (materialization.effectiveProfileDigest !== materialization.authoredProfileDigest) { + throw new ValidationError( + 'streamAgentTurn: executor changed the authored AgentProfile; exact turn execution refuses profile overlays', + ) + } + } else { + materialization = unknownMaterializationReceipt({ + runtime: backend.kind === 'box' ? 'sandbox' : label, + reason: 'executor-did-not-report', + }) + executionBinding = unknownExecutionBindingReceipt( + materialization, + attemptId, + 'executor-did-not-report', + ) + } + session = await startTurnSession(backend, task, input, deadline.signal, label) + yield { + type: 'backend_start', + task, + session, + backend: executor?.runtime ?? label, + metadata: { + ...(profileDigest ? { identity: { profileDigest, taskDigest } } : { taskDigest }), + ...(materialization ? { materialization } : {}), + ...(executionBinding ? { executionBindings: [executionBinding] } : {}), + budget: { timeoutMs: opts.timeoutMs ?? null }, + timing: { startedAt: new Date(startedAt).toISOString() }, + }, + timestamp: nowIso(), + } const inner = backend.kind === 'chat' - ? driveChatTurn(backend.backend, task, session, prompt, deadline.signal, acc) - : driveBoxTurn( - backend.kind === 'executor' - ? await inlineSandboxClient(backend.factory).create() - : backend.box, - prompt, - deadline.signal, - backend.agentRunName ?? 'agent', - acc, - { - ...(backend.kind !== 'executor' && backend.options - ? { options: backend.options } - : {}), - preserveToolParts: opts.preserveToolParts === true, - ...(opts.onRawEvent ? { onRawEvent: opts.onRawEvent } : {}), - }, - ) + ? driveChatTurn(backend.backend, task, session, input, deadline.signal, acc) + : backend.kind === 'executor' + ? driveExecutorTurn( + executor!, + task, + session, + input, + deadline.signal, + acc, + materializedModel(materialization, profile!), + ) + : driveBoxTurn( + backend.box, + turnIntent(input), + deadline.signal, + backend.agentRunName ?? 'agent', + acc, + { + ...(backend.options ? { options: backend.options } : {}), + preserveToolParts: opts.preserveToolParts === true, + ...(opts.onRawEvent ? { onRawEvent: opts.onRawEvent } : {}), + }, + ) for await (const event of inner) { yield event throwIfAborted(deadline.signal) } - yield buildFinalEvent(task, session, acc, { status: 'completed', reason: 'turn completed' }) + yield buildFinalEvent( + task, + session, + acc, + { status: 'completed', reason: 'turn completed' }, + turnProvenance( + startedAt, + opts.timeoutMs, + profileDigest, + taskDigest, + materialization, + executionBinding, + ), + ) } catch (err) { const callerAborted = opts.signal?.aborted === true const status: AgentTaskStatus = callerAborted ? 'aborted' : 'failed' @@ -249,8 +495,22 @@ export async function* streamAgentTurn( error, timestamp: nowIso(), } - yield buildFinalEvent(task, session, acc, { status, reason: message, error }) + yield buildFinalEvent( + task, + session, + acc, + { status, reason: message, error }, + turnProvenance( + startedAt, + opts.timeoutMs, + profileDigest, + taskDigest, + materialization, + executionBinding, + ), + ) } finally { + await executor?.teardown('brutalKill').catch(() => undefined) deadline.dispose() } } @@ -282,15 +542,25 @@ export async function collectAgentTurn( const usage: AgentTurnUsage = { input: finiteNumber(tokenUsage.input) ?? 0, output: finiteNumber(tokenUsage.output) ?? 0, + ...(metadata.tokensKnown === false ? { tokensKnown: false as const } : {}), } const costUsd = finiteNumber(metadata.costUsd) if (costUsd !== undefined) usage.costUsd = costUsd + if (metadata.usdKnown === false) usage.usdKnown = false if (typeof metadata.model === 'string' && metadata.model.length > 0) { usage.model = metadata.model } + const toolCalls = events + .filter((event) => event.type === 'tool_call') + .map((event) => ({ + ...(event.toolCallId ? { id: event.toolCallId } : {}), + name: event.toolName, + arguments: typeof event.args === 'string' ? event.args : JSON.stringify(event.args ?? {}), + })) return { finalText: final.text ?? '', usage, + toolCalls, events, status: final.status, ...(final.error ? { error: final.error } : {}), @@ -301,17 +571,18 @@ export async function collectAgentTurn( * correlation session otherwise. Box/executor turns carry no server session * here — resume lives in `openSandboxRun`/`SandboxLineage`, not this primitive. */ async function startTurnSession( - backend: AgentTurnBackend, + backend: AgentTurnBackend | ObservedAgentTurnBackend, task: AgentTaskSpec, - prompt: string, + input: AgentTurnInput, signal: AbortSignal, label: string, ): Promise { if (backend.kind === 'chat' && backend.backend.start) { - return backend.backend.start( - { task, message: prompt }, - { task, knowledge: emptyReadiness(task), signal }, - ) + return backend.backend.start(turnBackendInput(task, input), { + task, + knowledge: emptyReadiness(task), + signal, + }) } return newRuntimeSession(label) } @@ -370,19 +641,133 @@ async function* driveChatTurn( backend: AgentExecutionBackend, task: AgentTaskSpec, session: RuntimeSession, - prompt: string, + input: AgentTurnInput, signal: AbortSignal, acc: TurnAccumulator, ): AsyncGenerator { - const input = { task, message: prompt } + const backendInput = turnBackendInput(task, input) const context = { task, knowledge: emptyReadiness(task), session, signal } - for await (const raw of backend.stream(input, context)) { + for await (const raw of backend.stream(backendInput, context)) { const event = normalizeBackendStreamEvent(raw, task, session) foldEvent(event, acc) yield event } } +async function* driveExecutorTurn( + executor: Executor, + task: AgentTaskSpec, + session: RuntimeSession, + input: AgentTurnInput, + signal: AbortSignal, + acc: TurnAccumulator, + declaredModel: string | undefined, +): AsyncGenerator { + const taskValue = typeof input === 'string' ? input : { messages: input.messages } + const run = executor.execute(taskValue, signal) + let result: ExecutorResult + if (isAsyncIterable(run)) { + for await (const _usage of run) { + throwIfAborted(signal) + } + result = executor.resultArtifact() + } else { + result = await run + } + acc.result = result + acc.terminalText = executorResultText(result.out) + acc.input = result.spent.tokens.input + acc.output = result.spent.tokens.output + acc.costUsd = result.spent.usd + acc.tokensKnown = result.spent.tokensKnown !== false + acc.usdKnown = result.spent.usdKnown !== false + acc.sawLlmCall = true + + const model = executorResultModel(result.out) ?? declaredModel + if (model) acc.model = model + acc.stopReason = executorResultStopReason(result.out) + const latencyMs = result.spent.ms + yield { + type: 'llm_call', + task, + session, + model: model ?? executor.runtime, + ...(acc.tokensKnown ? { tokensIn: acc.input, tokensOut: acc.output } : {}), + ...(acc.usdKnown ? { costUsd: acc.costUsd } : {}), + latencyMs, + timestamp: nowIso(), + } + for (const call of executorResultToolCalls(result.out)) { + yield { + type: 'tool_call', + task, + session, + toolName: call.name, + ...(call.id ? { toolCallId: call.id } : {}), + args: call.arguments, + timestamp: nowIso(), + } + } + yield { + type: 'artifact', + task, + session, + artifactId: result.outRef, + name: 'agent-turn-result', + metadata: { + spend: result.spent, + ...(result.verdict ? { verdict: result.verdict } : {}), + }, + timestamp: nowIso(), + } +} + +function isAsyncIterable(value: unknown): value is AsyncIterable { + return typeof value === 'object' && value !== null && Symbol.asyncIterator in value +} + +function executorResultText(value: unknown): string { + if (typeof value === 'string') return value + if (!value || typeof value !== 'object') return '' + const content = (value as Record).content + return typeof content === 'string' ? content : '' +} + +function executorResultModel(value: unknown): string | undefined { + if (!value || typeof value !== 'object') return undefined + const model = (value as Record).model + return typeof model === 'string' && model.length > 0 ? model : undefined +} + +function executorResultStopReason(value: unknown): string | undefined { + if (!value || typeof value !== 'object') return undefined + const reason = (value as Record).finishReason + return typeof reason === 'string' && reason.length > 0 ? reason : undefined +} + +function executorResultToolCalls( + value: unknown, +): Array<{ id?: string; name: string; arguments: string }> { + if (!value || typeof value !== 'object') return [] + const raw = (value as Record).toolCalls + if (!Array.isArray(raw)) return [] + return raw.flatMap((entry) => { + if (!entry || typeof entry !== 'object') return [] + const call = entry as Record + if (typeof call.name !== 'string') return [] + return [ + { + ...(typeof call.id === 'string' ? { id: call.id } : {}), + name: call.name, + arguments: + typeof call.arguments === 'string' + ? call.arguments + : JSON.stringify(call.arguments ?? {}), + }, + ] + }) +} + /** Fold one normalized event into the turn accumulator (text + usage). * `fallbackModelLabel` — a mapper-stamped run label to exclude from * `usage.model` (it is not a backend-reported model). */ @@ -396,6 +781,16 @@ function foldEvent( return } if (event.type === 'llm_call') { + const tokensReported = event.tokensIn !== undefined && event.tokensOut !== undefined + const usdReported = event.costUsd !== undefined + if (!acc.sawLlmCall) { + acc.tokensKnown = tokensReported + acc.usdKnown = usdReported + acc.sawLlmCall = true + } else { + acc.tokensKnown &&= tokensReported + acc.usdKnown &&= usdReported + } acc.input += event.tokensIn ?? 0 acc.output += event.tokensOut ?? 0 acc.costUsd += event.costUsd ?? 0 @@ -424,6 +819,7 @@ function buildFinalEvent( session: RuntimeSession | undefined, acc: TurnAccumulator, outcome: { status: AgentTaskStatus; reason: string; error?: BackendErrorDetail }, + provenance: Record, ): RuntimeStreamEvent { const finalText = acc.terminalText ?? acc.deltaText return { @@ -431,18 +827,39 @@ function buildFinalEvent( task, ...(session ? { session } : {}), status: outcome.status, - reason: outcome.reason, + reason: outcome.status === 'completed' ? (acc.stopReason ?? outcome.reason) : outcome.reason, ...(finalText ? { text: finalText } : {}), metadata: { tokenUsage: { input: acc.input, output: acc.output }, - ...(acc.costUsd > 0 ? { costUsd: acc.costUsd } : {}), + ...(acc.tokensKnown ? {} : { tokensKnown: false }), + ...(acc.usdKnown ? { costUsd: acc.costUsd } : { usdKnown: false }), ...(acc.model ? { model: acc.model } : {}), + ...(acc.stopReason ? { stopReason: acc.stopReason } : {}), + ...(acc.result + ? { + result: { + outRef: acc.result.outRef, + ...(acc.result.verdict ? { verdict: acc.result.verdict } : {}), + spent: acc.result.spent, + }, + } + : {}), + ...provenance, }, ...(outcome.error ? { error: outcome.error } : {}), timestamp: nowIso(), } } +function materializedModel( + receipt: ProfileMaterializationReceipt | undefined, + profile: AgentProfile, +): string | undefined { + if (receipt?.status === 'known' && receipt.model.status === 'known') return receipt.model.id + const fallback = profile.model?.default + return typeof fallback === 'string' && fallback.length > 0 ? fallback : undefined +} + /** Minimal ready-by-construction readiness report for a requirement-free turn. */ function emptyReadiness(task: AgentTaskSpec) { return scoreKnowledgeReadiness({ taskId: task.id, requirements: [] }) diff --git a/src/runtime/supervise/completion-gate.ts b/src/runtime/supervise/completion-gate.ts index 133f12e1..78ef5906 100644 --- a/src/runtime/supervise/completion-gate.ts +++ b/src/runtime/supervise/completion-gate.ts @@ -141,6 +141,68 @@ export function gateOnDeliverable( return inheritRuntimeOwnedExecutorAttestation(inner, wrapped) } +export interface ExecutorResultMapping { + outRef: string + out: Out + verdict?: DefaultVerdict +} + +/** + * Transform a Runtime executor's terminal artifact without losing its private + * profile-materialization attestation or altering its measured spend. This is + * the composition point for deterministic post-processing and grading; callers + * must not rebuild an Executor around a model transport merely to change `out`. + */ +export function mapExecutorResult( + inner: Executor, + map: ( + result: ExecutorResult, + task: unknown, + ) => ExecutorResultMapping | Promise>, +): Executor { + let mapped: ExecutorResult | undefined + + const settle = async ( + result: ExecutorResult, + task: unknown, + ): Promise> => { + const transformed = await map(result, task) + mapped = { + outRef: transformed.outRef, + out: transformed.out, + ...(transformed.verdict ? { verdict: transformed.verdict } : {}), + spent: result.spent, + } + return mapped + } + + const wrapped: Executor = { + runtime: inner.runtime, + ...(inner.budgetExempt !== undefined ? { budgetExempt: inner.budgetExempt } : {}), + ...(inner.deliver ? { deliver: (message: unknown) => inner.deliver?.(message) } : {}), + ...(inner.progress ? { progress: () => inner.progress?.() } : {}), + ...(inner.traceSource ? { traceSource: () => inner.traceSource?.() } : {}), + ...(inner.accounting ? { accounting: () => inner.accounting?.() } : {}), + ...(inner.metered ? { metered: () => inner.metered?.() } : {}), + execute(task, signal) { + const execution = inner.execute(task, signal) + if (isAsyncIterable(execution)) { + return (async function* () { + for await (const event of execution) yield event + await settle(inner.resultArtifact(), task) + })() + } + return (async () => settle(await execution, task))() + }, + teardown: (grace) => inner.teardown(grace), + resultArtifact() { + if (!mapped) throw new Error('mapExecutorResult: resultArtifact() read before execute()') + return mapped + }, + } + return inheritRuntimeOwnedExecutorAttestation(inner, wrapped) +} + function isAsyncIterable(v: unknown): v is AsyncIterable { return ( v != null && diff --git a/src/runtime/supervise/materialization.ts b/src/runtime/supervise/materialization.ts index 508f6a3f..8453a066 100644 --- a/src/runtime/supervise/materialization.ts +++ b/src/runtime/supervise/materialization.ts @@ -91,8 +91,8 @@ export function attestRuntimeOwnedDeferredExecutor( } /** Preserve the runtime-owned attestation when a trusted wrapper changes result semantics only. */ -export function inheritRuntimeOwnedExecutorAttestation( - source: Executor, +export function inheritRuntimeOwnedExecutorAttestation( + source: Executor, wrapper: Executor, ): Executor { const attestation = runtimeOwnedExecutorMaterializations.get(source as object) diff --git a/src/runtime/supervise/runtime.ts b/src/runtime/supervise/runtime.ts index cf5ea6fa..047c29c9 100644 --- a/src/runtime/supervise/runtime.ts +++ b/src/runtime/supervise/runtime.ts @@ -32,10 +32,17 @@ import { Readable } from 'node:stream' import { estimateCost, isModelPriced } from '@tangle-network/agent-eval' import { type AgentProfile, + type AgentProfileResourceRef, agentProfileSchema, mergeAgentProfiles, + profileMaterializationAxes, + type ReasoningEffort, } from '@tangle-network/agent-interface' import type { BackendType, SandboxEvent } from '@tangle-network/sandbox' +import { + assertProfileMaterialization, + defineProfileMaterializationContract, +} from '../../agent/profile-materialization' import { ValidationError } from '../../errors' import type { LocalHarness } from '../../mcp/local-harness' import { mergeTraceEnv } from '../../mcp/trace-propagation' @@ -60,7 +67,7 @@ import { resolveAgentEnvironmentProvider, } from '../environment-provider' import { agentHarness } from '../harness-role' -import { routerChatWithUsage, type ToolSpec } from '../router-client' +import { routerChatWithTools, routerChatWithUsage, type ToolSpec } from '../router-client' import type { RunAgentRoundsOptions } from '../run-loop' import { runAgentRounds } from '../run-loop' import type { @@ -113,14 +120,47 @@ import { createWorktreeCliExecutor } from './worktree-cli-executor' /** * Router/inline connection seam. A direct OpenAI-compatible Router endpoint — - * the cheapest leaf, no box, no tools. `model` overrides the profile's model - * hint when present; otherwise the profile's `model.default` is required. + * the cheapest leaf, no box. `model` is a fallback when the profile delegates + * model selection; two different concrete model declarations are refused. Every + * generation control is optional so the provider default remains available. */ export interface RouterSeam { routerBaseUrl: string routerKey: string model?: string -} + temperature?: number + maxTokens?: number + seed?: number + reasoningEffort?: ReasoningEffort + /** Provider-specific request fields. Canonical fields cannot be overridden. */ + extraBody?: Readonly> + /** When present, return one turn's requested tool calls without executing them. */ + tools?: ReadonlyArray + toolChoice?: 'auto' | 'required' | 'none' +} + +const routerTurnProfileMaterialization = defineProfileMaterializationContract({ + name: 'router-profile-turn', + axes: [ + 'name', + 'description', + 'version', + 'tags', + 'systemPrompt', + 'instructions', + 'modelDefault', + 'modelProvider', + 'modelReasoningEffort', + 'tools', + 'files', + 'resourceTools', + 'skills', + 'resourceAgents', + 'commands', + 'resourceInstructions', + 'metadata', + ], +}) /** * Sandbox executor seam. The `sandboxClient` the composed `runAgentRounds` creates @@ -381,7 +421,7 @@ function unmeteredSpend(ms: number): Spend { */ export const routerInlineExecutor: ExecutorFactory = (spec, ctx) => { const seam = readSeam(ctx, routerSeamKey, 'router/inline') - const model = concreteProfileModel(spec.profile) ?? concreteModelId(seam.model) + const model = exactRouterModel(spec.profile, seam.model, 'routerInlineExecutor') if (!model) { throw new ValidationError( 'routerInlineExecutor: no model — set RouterSeam.model or AgentProfile.model.default', @@ -390,6 +430,7 @@ export const routerInlineExecutor: ExecutorFactory = (spec, ctx) => { if (!seam.routerBaseUrl || !seam.routerKey) { throw new ValidationError('routerInlineExecutor: RouterSeam.routerBaseUrl + routerKey required') } + const profileExecution = routerProfileExecution(spec.profile, seam) const controller = new AbortController() const abortIfSignalled = () => { @@ -406,14 +447,43 @@ export const routerInlineExecutor: ExecutorFactory = (spec, ctx) => { { runtime: 'router' as Runtime, async execute(task, signal): Promise> { - const messages = taskToMessages(task, spec) + const messages = taskToMessages(task, spec, profileExecution.systemPrompt) const started = Date.now() const linked = linkSignals(signal, controller.signal) - const r = await routerChatWithUsage( - { routerBaseUrl: seam.routerBaseUrl, routerKey: seam.routerKey, model }, - messages, - linked ? { signal: linked } : {}, - ) + const extraBody = { + ...(seam.extraBody ?? {}), + ...(seam.seed !== undefined ? { seed: seam.seed } : {}), + ...(profileExecution.reasoningEffort + ? { reasoning_effort: profileExecution.reasoningEffort } + : {}), + } + const r = seam.tools + ? await routerChatWithTools( + { routerBaseUrl: seam.routerBaseUrl, routerKey: seam.routerKey, model }, + messages, + seam.tools, + { + ...(seam.temperature !== undefined ? { temperature: seam.temperature } : {}), + ...(linked ? { signal: linked } : {}), + ...(seam.toolChoice ? { toolChoice: seam.toolChoice } : {}), + ...(seam.maxTokens !== undefined ? { maxTokens: seam.maxTokens } : {}), + ...(Object.keys(extraBody).length > 0 ? { extraBody } : {}), + }, + ) + : await routerChatWithUsage( + { routerBaseUrl: seam.routerBaseUrl, routerKey: seam.routerKey, model }, + messages, + { + ...(seam.temperature !== undefined ? { temperature: seam.temperature } : {}), + ...(linked ? { signal: linked } : {}), + ...(seam.maxTokens !== undefined ? { maxTokens: seam.maxTokens } : {}), + ...(seam.seed !== undefined ? { seed: seam.seed } : {}), + ...(profileExecution.reasoningEffort + ? { reasoningEffort: profileExecution.reasoningEffort } + : {}), + ...(seam.extraBody ? { extraBody: seam.extraBody } : {}), + }, + ) const spent: Spend = { iterations: 1, tokens: r.usage ? { input: r.usage.input, output: r.usage.output } : zeroTokenUsage(), @@ -422,8 +492,18 @@ export const routerInlineExecutor: ExecutorFactory = (spec, ctx) => { ...(r.costUsd === undefined ? { usdKnown: false } : {}), ms: Date.now() - started, } - const out = { content: r.content } as unknown - artifact = { outRef: contentRef('router', { model, content: r.content }), out, spent } + const out = { + content: r.content ?? '', + model, + ...('toolCalls' in r ? { toolCalls: r.toolCalls } : {}), + ...(r.reasoning ? { reasoning: r.reasoning } : {}), + ...(r.finishReason + ? { finishReason: r.finishReason } + : 'toolCalls' in r && r.toolCalls.length > 0 + ? { finishReason: 'tool_calls' } + : {}), + } as unknown + artifact = { outRef: contentRef('router', { model, out }), out, spent } return artifact }, teardown(_grace): Promise<{ destroyed: boolean }> { @@ -446,7 +526,19 @@ export const routerInlineExecutor: ExecutorFactory = (spec, ctx) => { id: executionId, }, materializer: 'router-prompt-model', - plan: { kind: 'openai-chat-completion', model }, + plan: { + kind: 'openai-chat-completion', + model, + provider: spec.profile.model?.provider ?? null, + temperature: seam.temperature ?? null, + maxTokens: seam.maxTokens ?? null, + seed: seam.seed ?? null, + reasoningEffort: profileExecution.reasoningEffort ?? null, + extraBody: seam.extraBody ?? null, + tools: seam.tools ?? null, + toolChoice: seam.toolChoice ?? null, + systemPrompt: profileExecution.systemPrompt || null, + }, }, { attemptId, @@ -478,6 +570,11 @@ export interface RouterToolsSeam { model?: string tools: ReadonlyArray executeToolCall: (name: string, args: Record, task: unknown) => Promise + temperature?: number + maxTokens?: number + toolChoice?: 'auto' | 'required' | 'none' + /** Provider-specific request fields. Canonical fields cannot be overridden. */ + extraBody?: Readonly> /** Online observer of each tool step — the seam a `DetectorMonitor` taps to watch the live pipe * (raise a `finding` when the worker loops/errors). Called after every tool call resolves, with * real per-call wall-clock (`startedAt`/`endedAt`/`durationMs`) so a push `TraceSource` can carry @@ -516,7 +613,7 @@ interface RouterToolsResponse { */ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) => { const seam = readSeam(ctx, routerToolsSeamKey, 'router-tools') - const model = concreteProfileModel(spec.profile) ?? concreteModelId(seam.model) + const model = exactRouterModel(spec.profile, seam.model, 'routerToolsInlineExecutor') if (!model) { throw new ValidationError( 'routerToolsInlineExecutor: no model — set RouterToolsSeam.model or AgentProfile.model.default', @@ -527,6 +624,17 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = 'routerToolsInlineExecutor: RouterToolsSeam.routerBaseUrl + routerKey required', ) } + const profileExecution = routerProfileExecution(spec.profile, { + routerBaseUrl: seam.routerBaseUrl, + routerKey: seam.routerKey, + model, + tools: seam.tools, + ...(seam.temperature !== undefined ? { temperature: seam.temperature } : {}), + ...(seam.maxTokens !== undefined ? { maxTokens: seam.maxTokens } : {}), + ...(seam.toolChoice ? { toolChoice: seam.toolChoice } : {}), + ...(seam.extraBody ? { extraBody: seam.extraBody } : {}), + }) + const enabledToolNames = new Set(seam.tools.map((tool) => tool.function.name)) const maxTurns = seam.maxTurns ?? 200 const controller = new AbortController() @@ -550,7 +658,9 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = async execute(task, signal): Promise> { const started = Date.now() const messages: Array> = [ - ...(taskToMessages(task, spec) as Array>), + ...(taskToMessages(task, spec, profileExecution.systemPrompt) as Array< + Record + >), ] const tokens = zeroTokenUsage() let tokensKnown = true @@ -589,11 +699,16 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = authorization: `Bearer ${seam.routerKey}`, }, body: JSON.stringify({ + ...safeRouterExtraBody(seam.extraBody), model, messages, tools: seam.tools, - tool_choice: 'auto', - temperature: 0.2, + tool_choice: seam.toolChoice ?? 'auto', + ...(seam.temperature !== undefined ? { temperature: seam.temperature } : {}), + ...(seam.maxTokens !== undefined ? { max_tokens: seam.maxTokens } : {}), + ...(profileExecution.reasoningEffort + ? { reasoning_effort: profileExecution.reasoningEffort } + : {}), }), signal: turnController.signal, }) @@ -655,6 +770,20 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = for (let i = 0; i < toolCalls.length; i += 1) { const tc = toolCalls[i] const id = tc?.id ?? `call_${i}` + const toolName = tc?.function?.name ?? '' + if (!enabledToolNames.has(toolName)) { + messages.push({ + role: 'tool', + tool_call_id: id, + content: `error: tool ${JSON.stringify(toolName)} is not enabled by AgentProfile.tools`, + }) + try { + seam.onToolStep?.({ toolName, args: {}, status: 'error' }) + } catch { + // Monitoring cannot authorize or execute a refused tool call. + } + continue + } let args: Record = {} try { args = JSON.parse(tc?.function?.arguments ?? '{}') as Record @@ -668,7 +797,6 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = }) continue } - const toolName = tc?.function?.name ?? '' let result: string let status: 'ok' | 'error' = 'ok' const toolStartedAt = Date.now() @@ -734,7 +862,19 @@ export const routerToolsInlineExecutor: ExecutorFactory = (spec, ctx) = id: executionId, }, materializer: 'router-tools-prompt-model', - plan: { kind: 'openai-tool-loop', model, maxTurns, tools: seam.tools }, + plan: { + kind: 'openai-tool-loop', + model, + provider: spec.profile.model?.provider ?? null, + maxTurns, + tools: seam.tools, + temperature: seam.temperature ?? null, + maxTokens: seam.maxTokens ?? null, + toolChoice: seam.toolChoice ?? 'auto', + extraBody: seam.extraBody ?? null, + reasoningEffort: profileExecution.reasoningEffort ?? null, + systemPrompt: profileExecution.systemPrompt || null, + }, }, { attemptId, @@ -3002,17 +3142,201 @@ function taskToPrompt(task: unknown): string { return JSON.stringify(task) } -/** Router messages from the opaque task + every portable profile prompt instruction. */ -function taskToMessages(task: unknown, spec: AgentSpec): Array<{ role: string; content: string }> { - const messages: Array<{ role: string; content: string }> = [] - const system = [spec.profile.prompt?.systemPrompt, ...(spec.profile.prompt?.instructions ?? [])] - .filter((line): line is string => typeof line === 'string' && line.trim().length > 0) - .join('\n') - if (system.length > 0) { - messages.push({ role: 'system', content: system }) +interface RouterProfileExecution { + systemPrompt: string + reasoningEffort?: ReasoningEffort +} + +/** Validate and render every AgentProfile axis the direct Router path claims to carry. + * Unsupported behavioral axes fail before the HTTP request; inline resources become + * named system-prompt attachments because this executor has no workspace to mount. */ +function routerProfileExecution(profile: AgentProfile, seam: RouterSeam): RouterProfileExecution { + assertProfileMaterialization({ + contract: routerTurnProfileMaterialization, + changedAxes: profileMaterializationAxes(profile), + context: 'routerInlineExecutor', + }) + + if (profile.harness !== undefined && profile.harness !== null) { + throw new ValidationError( + `routerInlineExecutor: AgentProfile.harness ${JSON.stringify(profile.harness)} requires a harness executor; the direct Router executor cannot materialize it`, + ) + } + + const profileEffort = profile.model?.reasoningEffort + const seamEffort = seam.reasoningEffort + if (profileEffort && seamEffort && profileEffort !== seamEffort) { + throw new ValidationError( + `routerInlineExecutor: AgentProfile reasoning effort ${JSON.stringify(profileEffort)} conflicts with RouterSeam.reasoningEffort ${JSON.stringify(seamEffort)}`, + ) + } + const effort = profileEffort ?? seamEffort + + const declaredTools = profile.tools ?? {} + const suppliedTools = seam.tools ?? [] + const suppliedNames = new Set() + for (const tool of suppliedTools) { + const name = tool.function.name + if (!name || suppliedNames.has(name)) { + throw new ValidationError( + `routerInlineExecutor: caller tool names must be non-empty and unique (${JSON.stringify(name)})`, + ) + } + suppliedNames.add(name) + if (declaredTools[name] !== true) { + throw new ValidationError( + `routerInlineExecutor: caller tool ${JSON.stringify(name)} is not enabled by AgentProfile.tools`, + ) + } + } + for (const [name, enabled] of Object.entries(declaredTools)) { + if (enabled && !suppliedNames.has(name)) { + throw new ValidationError( + `routerInlineExecutor: AgentProfile enables tool ${JSON.stringify(name)} but the caller supplied no matching schema`, + ) + } + if (!enabled && suppliedNames.has(name)) { + throw new ValidationError( + `routerInlineExecutor: AgentProfile disables tool ${JSON.stringify(name)}`, + ) + } + } + + return { + systemPrompt: renderRouterProfilePrompt(profile), + ...(effort ? { reasoningEffort: effort } : {}), + } +} + +/** Resolve the one model id that will cross the Router boundary. A seam value is a fallback for + * a profile that delegates selection, never an override hidden from profile identity. */ +function exactRouterModel( + profile: AgentProfile, + seamModel: string | undefined, + context: string, +): string | undefined { + const profileModel = concreteProfileModel(profile) + const configuredModel = concreteModelId(seamModel) + if (profileModel && configuredModel && profileModel !== configuredModel) { + throw new ValidationError( + `${context}: AgentProfile model ${JSON.stringify(profileModel)} conflicts with configured model ${JSON.stringify(configuredModel)}`, + ) + } + return profileModel ?? configuredModel +} + +function renderRouterProfilePrompt(profile: AgentProfile): string { + const sections: string[] = [ + profile.prompt?.systemPrompt, + ...(profile.prompt?.instructions ?? []), + ].filter((value): value is string => typeof value === 'string' && value.trim().length > 0) + const resources = profile.resources + if (!resources) return sections.join('\n') + + if (typeof resources.instructions === 'string') { + if (resources.instructions.trim()) sections.push(resources.instructions) + } else if (resources.instructions) { + sections.push(renderRouterResource('instructions', resources.instructions)) + } + for (const file of resources.files ?? []) { + if (file.executable === true) { + throw new ValidationError( + `routerInlineExecutor: executable resource ${JSON.stringify(file.path)} requires a workspace backend`, + ) + } + sections.push(renderRouterResource(`file ${file.path}`, file.resource)) + } + for (const [kind, refs] of [ + ['tool', resources.tools], + ['skill', resources.skills], + ['agent', resources.agents], + ['command', resources.commands], + ] as const) { + for (const ref of refs ?? []) sections.push(renderRouterResource(kind, ref)) + } + return sections.join('\n\n') +} + +function renderRouterResource(kind: string, resource: AgentProfileResourceRef): string { + if (resource.kind !== 'inline') { + throw new ValidationError( + `routerInlineExecutor: ${kind} resource ${JSON.stringify(resource.name ?? resource.path)} is not inline and cannot be resolved by the direct Router executor`, + ) + } + return `## Attached ${kind}: ${resource.name}\n${resource.content}` +} + +function safeRouterExtraBody( + extraBody: Readonly> | undefined, +): Record { + const safe = { ...(extraBody ?? {}) } + for (const key of [ + 'model', + 'messages', + 'tools', + 'tool_choice', + 'temperature', + 'max_tokens', + 'reasoning_effort', + 'stream', + 'stream_options', + ]) { + delete safe[key] + } + return safe +} + +/** Router messages from the opaque task + every portable profile prompt instruction. + * A structured conversation may carry multimodal/tool content, but any system + * message must exactly equal the canonical profile prompt. This prevents a + * caller from declaring one profile while executing another hidden prompt. */ +function taskToMessages( + task: unknown, + spec: AgentSpec, + resolvedSystem?: string, +): Array<{ role: string; content: unknown } & Record> { + const system = + resolvedSystem ?? + [spec.profile.prompt?.systemPrompt, ...(spec.profile.prompt?.instructions ?? [])] + .filter((line): line is string => typeof line === 'string' && line.trim().length > 0) + .join('\n') + + if ( + task && + typeof task === 'object' && + Array.isArray((task as { messages?: unknown }).messages) + ) { + const supplied = (task as { messages: unknown[] }).messages.map((value, index) => { + if (!value || typeof value !== 'object') { + throw new ValidationError(`routerInlineExecutor: messages[${index}] must be an object`) + } + const message = { ...(value as Record) } + if (typeof message.role !== 'string' || !('content' in message)) { + throw new ValidationError( + `routerInlineExecutor: messages[${index}] requires role and content`, + ) + } + return message as { role: string; content: unknown } & Record + }) + const systemMessages = supplied.filter((message) => message.role === 'system') + if (systemMessages.length > 1) { + throw new ValidationError('routerInlineExecutor: at most one system message is allowed') + } + if (systemMessages.length === 1 && systemMessages[0]?.content !== system) { + throw new ValidationError( + 'routerInlineExecutor: supplied system message must exactly match AgentProfile.prompt', + ) + } + if (systemMessages.length === 0 && system.length > 0) { + return [{ role: 'system', content: system }, ...supplied] + } + return supplied } - messages.push({ role: 'user', content: taskToPrompt(task) }) - return messages + + return [ + ...(system.length > 0 ? [{ role: 'system', content: system }] : []), + { role: 'user', content: taskToPrompt(task) }, + ] } /** A driver that refines a single task up to `maxIterations` times then stops — diff --git a/src/types.ts b/src/types.ts index 7d767cea..64bc3830 100644 --- a/src/types.ts +++ b/src/types.ts @@ -320,6 +320,10 @@ export type RuntimeStreamEvent = task: AgentTaskSpec session: RuntimeSession backend: string + /** Canonical execution identity and materialization evidence for this turn, when Runtime + * owns the selected executor. Generic metadata keeps the event vocabulary open while the + * values use Runtime's existing identity/materialization receipt shapes. */ + metadata?: Record timestamp: string } | {