Skip to content

refactor(runtime): enforce canonical model execution boundary #728

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
drewstone wants to merge 2 commits into from
Closed

refactor(runtime): enforce canonical model execution boundary #728

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3a1406d
refactor(runtime): enforce canonical model execution boundary
drewstone Aug 3, 2026
4108a75
Merge remote-tracking branch 'origin/main' into fix/runtime-model-bou…
drewstone Aug 3, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
169 changes: 0 additions & 169 deletions bench/scripts/appworld_driver.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,177 +9,14 @@

import argparse
import json
import os
import re
import sys
import time


def fail(msg: str) -> None:
print(json.dumps({"error": msg}))
sys.exit(1)


_CODE_RE = re.compile(r"```(?:python|py)?\s*\n(.*?)```", re.DOTALL)


def _extract_code(text: str) -> str:
blocks = _CODE_RE.findall(text or "")
return (blocks[-1] if blocks else "").strip()


def _router_chat(base: str, key: str, model: str, messages: list, timeout: float = 180.0):
"""One router chat-completion with retry on transient/429/5xx. Returns
(content, input_tokens, output_tokens). Raises on exhausted retries."""
import httpx

url = base.rstrip("/") + "/chat/completions"
last = None
for attempt in range(4):
try:
r = httpx.post(
url,
headers={"Authorization": f"Bearer {key}", "Content-Type": "application/json"},
json={"model": model, "messages": messages},
timeout=timeout,
)
if r.status_code in (429, 500, 502, 503, 504):
last = f"{r.status_code}: {r.text[:160]}"
time.sleep(2**attempt)
continue
r.raise_for_status()
d = r.json()
content = (d["choices"][0]["message"].get("content") or "")
usage = d.get("usage") or {}
return content, int(usage.get("prompt_tokens", 0) or 0), int(usage.get("completion_tokens", 0) or 0)
except Exception as e: # noqa: BLE001
last = str(e)
if attempt < 3:
time.sleep(2**attempt)
continue
raise RuntimeError(f"router_chat failed after retries: {last}")
raise RuntimeError(f"router_chat exhausted: {last}")


def _build_system(directive: str, world) -> str:
sup = world.task.supervisor
apps = list(getattr(world.task, "allowed_apps", []) or [])
descs = getattr(world.task, "app_descriptions", "")
desc_str = json.dumps(descs) if isinstance(descs, (dict, list)) else str(descs)
return (
f"You are an AI agent completing a digital task for your supervisor "
f"{getattr(sup, 'first_name', '')} {getattr(sup, 'last_name', '')} "
f"(email {getattr(sup, 'email', '')}, phone {getattr(sup, 'phone_number', '')}) "
"by WRITING PYTHON that calls app APIs (the apis.<app>.<function>(...) surface).\n\n"
f"Available apps: {', '.join(apps)}.\n"
f"App descriptions: {desc_str[:1500]}\n\n"
"How to work, one step per turn:\n"
"- Discover APIs with apis.api_docs.show_api_descriptions(app_name='<app>') and "
"apis.api_docs.show_api_doc(app_name='<app>', api_name='<api>') BEFORE calling them.\n"
"- Get the supervisor's app passwords with apis.supervisor.show_account_passwords(), then log in "
"to each app you use to obtain its access_token.\n"
"- Write ONE short Python code block per turn. After it runs you SEE its OUTPUT (or error "
"traceback) — use that to decide the next step. Print intermediate values you need.\n"
"- Iterate: inspect -> authenticate -> act -> verify. Do not guess API names or arguments.\n"
"- When the task is fully done call apis.supervisor.complete_task(answer=<answer>) (include the "
"answer if the task asks a question, otherwise apis.supervisor.complete_task()).\n"
"- Reply with EXACTLY ONE fenced ```python block per turn and nothing else.\n\n"
f"{directive}"
)


def cmd_react(args) -> None:
"""Multi-turn REPL agent: the model writes a python block, the engine executes
it in the PERSISTENT world, the output is fed back, and it iterates until it
completes the task or hits max-turns. Then AppWorld's own evaluator scores it.
Config (directive, model, router creds, max_turns) arrives as JSON on stdin so
the candidate directive can be arbitrarily long. The directive is the optimized
surface; the loop + contract are fixed."""
cfg = {}
raw = sys.stdin.read()
if raw.strip():
try:
cfg = json.loads(raw)
except Exception as e: # noqa: BLE001
fail(f"react config JSON parse failed: {e}")
directive = str(cfg.get("directive", ""))
model = str(cfg.get("model", "gpt-4o"))
max_turns = int(cfg.get("max_turns", 8))
router_base = str(cfg.get("router_base", "https://router.tangle.tools/v1"))
router_key = str(cfg.get("router_key") or os.environ.get("TANGLE_API_KEY", ""))
if not router_key:
fail("react: router_key/TANGLE_API_KEY required")

try:
from appworld import AppWorld
except Exception as e: # noqa: BLE001
fail(f"appworld import failed: {e}")

in_tok = 0
out_tok = 0
turns = 0
turns_log: list = []
try:
with AppWorld(
task_id=args.task_id,
experiment_name="bench-react",
raise_on_failure=False,
) as world:
messages = [
{"role": "system", "content": _build_system(directive, world)},
{"role": "user", "content": f"Task: {world.task.instruction}"},
]
for turn in range(max_turns):
turns = turn + 1
content, ui, uo = _router_chat(router_base, router_key, model, messages)
in_tok += ui
out_tok += uo
code = _extract_code(content)
messages.append({"role": "assistant", "content": content})
if not code:
messages.append({
"role": "user",
"content": "Reply with exactly one ```python block that makes progress, "
"or call apis.supervisor.complete_task().",
})
continue
output = world.execute(code)
turns_log.append({"code": code[:600], "output": str(output)[:600]})
messages.append({"role": "user", "content": "OUTPUT:\n" + str(output)[:4000]})
if world.task_completed():
break
evaluation = world.evaluate().to_dict()
except Exception as e: # noqa: BLE001
fail(f"react of {args.task_id} failed: {e}")

if "success" not in evaluation or "num_tests" not in evaluation:
fail(f"evaluation dict missing success/num_tests keys: {sorted(evaluation.keys())}")
passes = evaluation.get("passes", [])
failures = evaluation.get("failures", [])
n_pass = len(passes) if isinstance(passes, list) else int(passes or 0)
n_fail = len(failures) if isinstance(failures, list) else int(failures or 0)
print(
json.dumps(
{
"success": bool(evaluation["success"]),
"passes": n_pass,
"fails": n_fail,
"num_tests": int(evaluation["num_tests"]),
# Failed sub-test names — the evidence a trace analyst steers on.
"failure_names": [str(f)[:160] for f in failures][:8]
if isinstance(failures, list)
else [],
"turns": turns,
"input_tokens": in_tok,
"output_tokens": out_tok,
"transcript": "\n---\n".join(
f"CODE:\n{t['code']}\nOUTPUT:\n{t['output']}" for t in turns_log[-3:]
)[:1600],
}
)
)


def cmd_session(args) -> None:
"""Dumb world shim: a persistent AppWorld session driven over stdin JSONL.
NO LLM calls here — the agent loop lives in the runtime (routerToolLoop);
Expand Down Expand Up @@ -336,10 +173,6 @@ def main() -> None:
p_eval.add_argument("--task-id", required=True)
p_eval.add_argument("--split", required=True)

p_react = sub.add_parser("react")
p_react.add_argument("--task-id", required=True)
p_react.add_argument("--split", required=True)

p_session = sub.add_parser("session")
p_session.add_argument("--task-id", required=True)
p_session.add_argument("--split", required=True)
Expand All @@ -349,8 +182,6 @@ def main() -> None:
cmd_load(args)
elif args.cmd == "evaluate":
cmd_evaluate(args)
elif args.cmd == "react":
cmd_react(args)
elif args.cmd == "session":
cmd_session(args)

Expand Down
6 changes: 3 additions & 3 deletions bench/scripts/trata-hedge/README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,8 @@ sparse (1 iff all themes). No deployable ground-truth checker — it's an **orac

## Status (2026-06-06): pipeline PROVEN end-to-end

Our solver → their **real** Gemini-3.1-pro judge → a genuine graded result. Every link
works. The naive **single-shot** baseline (gpt-4o, ~3 of N corpus files in one context
Our Runtime-backed solver → their **real** Gemini-3.1-pro judge → a genuine graded result. Every link
works. The naive **single-shot** baseline (DeepSeek V4 Flash, ~3 of N corpus files in one context
window) scores **0/4** — a floor: it hit only 1/3 moves on a few themes with
hallucinations flagged, because it could not explore the full corpus. The bench is built
for **agentic** exploration; a fair baseline needs our sandbox runtime as the solver
Expand All @@ -35,7 +35,7 @@ for **agentic** exploration; a fair baseline needs our sandbox runtime as the so
git clone https://github.com/Trata-Inc/trata-hedge-bench /tmp/thb

dotenvx run -f ~/company/devops/secrets/.env.keys -f ~/company/devops/secrets/agent-state.env -- \
bash bench/scripts/trata-hedge/run.sh /tmp/thb/environments/<env-name> gpt-4o
bash bench/scripts/trata-hedge/run.sh /tmp/thb/environments/<env-name> deepseek-v4-flash
```

## Gotchas (each cost a debugging cycle)
Expand Down
8 changes: 6 additions & 2 deletions bench/scripts/trata-hedge/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,18 @@
set -euo pipefail

ENV="${1:?usage: run.sh <trata-env-dir> [model]}"
MODEL="${2:-${WORKER_MODEL:-gpt-4o}}"
MODEL="${2:-${WORKER_MODEL:-}}"
if [[ -z "$MODEL" ]]; then
echo "WORKER_MODEL is required (or pass it as argument 2)" >&2
exit 2
fi
HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ANS=/tmp/thb-answer.txt
REWARD=/tmp/thb-reward.txt
DETAILS=/tmp/thb-details.json

echo "[trata] solve: $ENV (model=$MODEL)"
WORKER_MODEL="$MODEL" python3 "$HERE/solve.py" "$ENV" "$ANS"
WORKER_MODEL="$MODEL" pnpm --dir "$HERE/../../.." exec tsx "$HERE/../../src/trata-hedge-solve.mts" "$ENV" "$ANS"

sudo -n mkdir -p /app && sudo -n cp "$ANS" /app/answer.txt && sudo -n chmod 644 /app/answer.txt

Expand Down
83 changes: 0 additions & 83 deletions bench/scripts/trata-hedge/solve.py
View file
Open in desktop

This file was deleted.

23 changes: 18 additions & 5 deletions bench/src/aec-gate.mts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ import { resolveAdapter } from './adapters'
import type { BenchmarkAdapter, BenchTask } from './benchmarks/types'
import { type AttemptRecord, appendRunRecord, buildRunRecordFromAttempts } from './corpus'
import { composeStrategies } from './directives'
import { type RouterConfig, routerChatWithUsage } from '@tangle-network/agent-runtime/kernel'
import type { RouterConfig } from '@tangle-network/agent-runtime/kernel'
import { runBenchRouterTurn } from './router-turn'
import { pool } from './stats.mts'

function must(name: string): string {
Expand Down Expand Up @@ -65,17 +66,29 @@ async function runAttempt(
let lastErr: unknown
for (let attempt = 0; attempt < 3; attempt += 1) {
try {
const res = await routerChatWithUsage(cfg, [{ role: 'user', content: prompt }])
const content = typeof res.content === 'string' ? res.content : ''
const res = await runBenchRouterTurn(
{
routerBaseUrl: cfg.routerBaseUrl,
routerKey: cfg.routerKey,
profile: {
name: 'aec-worker',
model: { provider: 'tangle-router', default: cfg.model },
},
},
prompt,
)
const content = res.finalText
const verdict = await adapter.judge(task, content)
return {
prompt,
output: content,
score: verdict.score,
resolved: verdict.resolved,
wallMs: Date.now() - startedAt,
...(res.costUsd !== undefined ? { costUsd: res.costUsd } : {}),
...(res.usage ? { tokensIn: res.usage.input, tokensOut: res.usage.output } : {}),
...(res.usage.costUsd !== undefined ? { costUsd: res.usage.costUsd } : {}),
...(res.usage.tokensKnown === false
? {}
: { tokensIn: res.usage.input, tokensOut: res.usage.output }),
}
} catch (err) {
lastErr = err
Expand Down
Loading
Loading