diff --git a/benchmarks/harbor/tests/test_atif_compatibility.py b/benchmarks/harbor/tests/test_atif_compatibility.py index 2280ac1..95355b9 100644 --- a/benchmarks/harbor/tests/test_atif_compatibility.py +++ b/benchmarks/harbor/tests/test_atif_compatibility.py @@ -2,9 +2,10 @@ from pathlib import Path +import pytest from harbor.utils.trajectory_validator import TrajectoryValidator from nanopycodeagent.atif import project_atif -from nanopycodeagent.event_journal import EventJournal +from nanopycodeagent.event_journal import EventJournal, NativeEvent def test_projector_output_passes_harbor_atif_validator(): @@ -13,3 +14,33 @@ def test_projector_output_passes_harbor_atif_validator(): validator = TrajectoryValidator() assert validator.validate(trajectory), validator.get_errors() + + +@pytest.mark.parametrize("content", [ + [{"type": "text", "text": "Partial answer"}], + [{"type": "extension", "namespace": "anthropic", "source_type": "thinking", + "value": {"type": "thinking", "thinking": "Still analyzing", "signature": ""}}], + [{"type": "tool_call", "tool_call_id": "call-1", "tool_name": "write", "input": {}}], + [], +]) +def test_truncated_v2_journal_passes_harbor_atif_validator(tmp_path, content): + fixture = Path(__file__).parent / "fixtures" / "atif-journal-v1.jsonl" + with EventJournal.create("run-truncated", directory=tmp_path) as journal: + for entry in EventJournal.replay(fixture): + if entry.type.startswith("tool."): + continue + payload = entry.payload + if entry.type == "model.completed": + payload = payload | { + "stop_reason": "max_tokens", "content": content, + "tool_calls": [block for block in content if block["type"] == "tool_call"], + } + elif entry.type == "run.completed": + payload = payload | {"outcome": "response_truncated"} + journal.append(NativeEvent(entry.type, payload)) + + trajectory = project_atif(EventJournal.replay(journal.path)) + validator = TrajectoryValidator() + assert validator.validate(trajectory), validator.get_errors() + assert trajectory["extra"]["terminal"]["outcome"] == "response_truncated" + assert "observation" not in trajectory["steps"][1] diff --git a/docs/changelogs/0.8.x.md b/docs/changelogs/0.8.x.md index 1448744..021513e 100644 --- a/docs/changelogs/0.8.x.md +++ b/docs/changelogs/0.8.x.md @@ -4,6 +4,13 @@ All notable changes in the **0.8.x** release series are documented here. ## [Unreleased] +### Fixed +- Stop explicitly when a model response reaches `max_tokens`, reporting + `response_truncated` in Event Journals and ATIF trajectories instead of task + completion. Preserve partial output, usage, and cost accounting; skip tools + from the truncated reply and keep subsequent interactive requests valid. + New journals use schema v2, with v1 replay still supported. + ### Changed - Moved detailed CLI and configuration guidance out of the bilingual READMEs into dedicated English and Chinese user references, keeping the READMEs diff --git a/docs/dev_docs/en/event-journal-protocol-v1.md b/docs/dev_docs/en/event-journal-protocol-v1.md index d71c0fc..e4197d7 100644 --- a/docs/dev_docs/en/event-journal-protocol-v1.md +++ b/docs/dev_docs/en/event-journal-protocol-v1.md @@ -1,5 +1,8 @@ # Event Journal Implementation Protocol v1 +> This page preserves the historical v1 contract. The current writer uses +> [v2](event-journal-protocol-v2.md); the reader still supports v1. + > Generated from the Chinese source > [`../zh-CN/event-journal-protocol-v1.md`](../zh-CN/event-journal-protocol-v1.md). > Do not edit by hand. diff --git a/docs/dev_docs/en/event-journal-protocol-v2.md b/docs/dev_docs/en/event-journal-protocol-v2.md new file mode 100644 index 0000000..1fe040e --- /dev/null +++ b/docs/dev_docs/en/event-journal-protocol-v2.md @@ -0,0 +1,80 @@ +# Event Journal Implementation Protocol v2 + +> Generated from the Chinese source +> [`../zh-CN/event-journal-protocol-v2.md`](../zh-CN/event-journal-protocol-v2.md). +> Do not edit by hand. + +v2 is implemented and is the internal Journal protocol used by the current +writer, with `schema_version = 2`. This document defines all changes relative +to [v1](event-journal-protocol-v1.md). Envelope, event types, fields, validation, +ordering, persistence, and projection rules not listed here follow v1. Public +trajectories remain ATIF-v1.7. + +## Response truncation outcome + +`run.completed.payload.outcome` accepts these values: + +| Value | Meaning | +| --- | --- | +| `completed` | The model ended its reply; this does not establish verifier success. | +| `max_turns_exhausted` | The final reply still requests tools, but no reply budget remains, so those tools are skipped. | +| `response_truncated` | The model returned `stop_reason="max_tokens"`, reaching its generation length limit, and the run stopped. | + +`model.completed` means an API call returned its final message and usage. It +does not guarantee that the model finished its reply. Truncated replies still +produce this event, preserving `stop_reason="max_tokens"`, original content, +tool calls, usage, and available provider identifiers. Normal cost +reconciliation still runs during finalization. + +The core recognizes truncation before checking the turn limit or executing +tools. A reply that also spends the last turn therefore records +`response_truncated`. All tools in that reply are skipped: there are no +`tool.started` or `tool.completed` events and no fabricated observations. +Existing text remains on stdout, the truncation diagnostic goes to stderr, +and headless mode exits `0`. The current policy stops without automatically +continuing, retrying, or raising the 8192-token limit. + +This is a budget outcome with an explicit reason, represented by +`run.completed` rather than `run.failed`. ATIF's `extra.terminal.status` +remains `completed`, meaning the run finalized normally; +`extra.terminal.outcome = "response_truncated"` carries the specific result. +The corresponding model step has `extra.stop_reason = "max_tokens"`. +Consumers MUST inspect the outcome to identify truncation rather than infer +task completion from status alone. + +Interactive mode returns to the input prompt. Request history retains only +the reply's text and an explicit truncation notice, removing unexecuted tool +calls and non-text blocks such as potentially unfinished thinking. This keeps +unmatched tool calls and incomplete signatures out of the next request. The +original model reply is still stored under the Journal persistence rules; +history repair does not rewrite runtime facts or invent tool execution events. + +## Compatibility + +- v1 has a closed outcome enum. Adding a value requires a schema increment, + rather than a v1 optional extension. +- The new writer emits `schema_version = 2` for all runs. The new reader/replay + implementation and ATIF projector support both v1 and v2. Existing v1 + journals are not rewritten, and historical `completed` outcomes are not + reclassified retroactively. +- A v1 record containing `response_truncated` is rejected. Old readers + explicitly reject the v2 schema. +- Other unknown schemas are still rejected. The remaining v1 compatibility + rules continue to apply. +- The top-level `truncation` field retains its meaning for Journal string + persistence limits, which are independent of model generation length limits. + +## Implementation and validation + +- [`agent.py`](../../../src/nanopycodeagent/agent.py): explicit RunOutcome, + truncation termination, text diagnostics, and interactive history handling. +- [`event_journal.py`](../../../src/nanopycodeagent/event_journal.py): v2 writer, + v1/v2 replay, and outcome validation. +- [`atif.py`](../../../src/nanopycodeagent/atif.py): projection from both Journal + versions to ATIF-v1.7. +- [`test_truncation.py`](../../../tests/test_truncation.py): text, thinking, + empty replies, partial tool JSON, the final turn, costs, trajectories, and + the next interactive turn. +- [Harbor compatibility tests](../../../benchmarks/harbor/tests/test_atif_compatibility.py): + the old v1 fixture and new v2 truncated trajectories pass the pinned official + ATIF validator. diff --git a/docs/dev_docs/zh-CN/event-journal-protocol-v1.md b/docs/dev_docs/zh-CN/event-journal-protocol-v1.md index cc726a0..11d8a57 100644 --- a/docs/dev_docs/zh-CN/event-journal-protocol-v1.md +++ b/docs/dev_docs/zh-CN/event-journal-protocol-v1.md @@ -1,5 +1,7 @@ # Event Journal 实现协议 v1 +> 此页保留历史 v1 契约；当前 writer 使用 [v2](event-journal-protocol-v2.md)，reader 仍支持 v1。 + > 本文件为**中文源文件**（source of truth）；英文版 > [`../en/event-journal-protocol-v1.md`](../en/event-journal-protocol-v1.md) > 由其生成。 diff --git a/docs/dev_docs/zh-CN/event-journal-protocol-v2.md b/docs/dev_docs/zh-CN/event-journal-protocol-v2.md new file mode 100644 index 0000000..5821cae --- /dev/null +++ b/docs/dev_docs/zh-CN/event-journal-protocol-v2.md @@ -0,0 +1,60 @@ +# Event Journal 实现协议 v2 + +> 本文件为**中文源文件**（source of truth）；英文版 +> [`../en/event-journal-protocol-v2.md`](../en/event-journal-protocol-v2.md) 由其生成。 + +v2 已实现，是当前 writer 使用的内部 Journal 协议，`schema_version = 2`。 +本文完整定义相对 [v1](event-journal-protocol-v1.md) 的变化；未列出的 envelope、 +事件类型、字段、校验、排序、持久化与投影规则沿用 v1。公开 trajectory 仍为 ATIF-v1.7。 + +## 回复截断终态 + +`run.completed.payload.outcome` 的允许值为： + +| 值 | 含义 | +| --- | --- | +| `completed` | 模型结束回复；不代表任务通过 verifier。 | +| `max_turns_exhausted` | 最后一轮仍请求工具，已无下一轮预算，不执行这些工具。 | +| `response_truncated` | 模型返回 `stop_reason="max_tokens"`，本次生成达到长度上限，run 停止。 | + +`model.completed` 表示一次 API 调用已返回最终消息与 usage，不保证模型完成了回复。 +截断回复仍产生该事件，保留 `stop_reason="max_tokens"`、原始 content、tool calls、 +usage 和已有 provider 标识。正常费用补查仍在 run 收尾时执行。 + +core 在检查轮数上限和执行工具之前识别截断，即使该回复恰好用尽最后一轮，也记录 +`response_truncated`。该回复中的工具一律不执行，不产生 `tool.started` 或 +`tool.completed`，也不伪造 observation。已有文本继续保留在 stdout，截断提示写入 +stderr；headless 退出码为 `0`。当前策略是停止，不自动续写、重试或增大 8192 上限。 + +这属于有明确原因的预算终态，使用 `run.completed`，而非 `run.failed`。 +ATIF 的 `extra.terminal.status` 仍为 `completed`，表示 run 已正常收尾； +`extra.terminal.outcome = "response_truncated"` 才是具体结果。 +对应模型 step 的 `extra.stop_reason` 为 `max_tokens`。消费者判断是否截断时必须 +读取 outcome，不能仅凭 status 推断任务完成。 + +交互模式返回输入提示符。供下一次请求使用的会话历史仅保留该回复的 text 与明确的 +截断提示，移除未执行的工具调用和可能未完成的 thinking 等非文本 block，避免下一次 +请求携带无对应结果的工具调用或不完整签名。原始模型回复仍按 Journal 持久化规则保存； +该历史修整不改写运行事实，不产生虚构的工具执行事件。 + +## 兼容性 + +- v1 的 outcome 枚举是封闭的，新增值需要提升 schema，而不是作为 v1 可选扩展。 +- 新 writer 对所有 run 写入 `schema_version = 2`；新 reader/replay 与 ATIF projector + 同时支持 v1 和 v2。已有 v1 Journal 不改写，历史 `completed` 也不会被追溯重分类。 +- v1 记录中出现 `response_truncated` 会被拒绝；旧 reader 会明确拒绝 v2 schema。 +- 其他未知 schema 仍被拒绝；v1 的其余兼容性规则继续适用。 +- Journal 字符串持久化截断的顶层 `truncation` 字段保持原义，与模型生成长度上限 + 是两个独立概念。 + +## 实现与验证 + +- [`agent.py`](../../../src/nanopycodeagent/agent.py)：显式 RunOutcome、截断停止、 + 文本提示与交互历史处理。 +- [`event_journal.py`](../../../src/nanopycodeagent/event_journal.py)：v2 writer、v1/v2 + replay 与 outcome 校验。 +- [`atif.py`](../../../src/nanopycodeagent/atif.py)：两种 Journal 版本到 ATIF-v1.7 的投影。 +- [`test_truncation.py`](../../../tests/test_truncation.py)：文本、thinking、空回复、部分 + 工具 JSON、最后一轮、费用、轨迹与交互下一轮。 +- [Harbor 兼容性测试](../../../benchmarks/harbor/tests/test_atif_compatibility.py)： + 旧 v1 fixture 与新 v2 截断轨迹通过固定版本的官方 ATIF validator。 diff --git a/docs/dev_notes/en/0.8.x.md b/docs/dev_notes/en/0.8.x.md index 9890d06..68f4185 100644 --- a/docs/dev_notes/en/0.8.x.md +++ b/docs/dev_notes/en/0.8.x.md @@ -1,18 +1,18 @@ # Development Notes — 0.8.x -> Generated from the Chinese source [`../zh-CN/0.8.x.md`](../zh-CN/0.8.x.md). Do not edit by hand. +> Generated from the [Chinese source](../zh-CN/0.8.x.md), which is the source of truth. ## 0.8.0 - 2026.09.04 -The basic bash, write, read, and edit tools are now implemented, providing a toolset similar to Pi's. The next step is to run public benchmarks to learn which features need improvement or addition and how to strengthen the harness. Benchmarks used in mainstream model releases provide a way to compare different harnesses with the same model and identify optimization targets. +The basic bash, write, read, and edit tools are now implemented, providing a foundation similar to pi's tools. The next step is to run public benchmarks to learn which optimizations and new features would improve the harness in practice. Benchmarks used by mainstream model releases let us compare different harnesses with the same model and identify concrete improvements. -The first step was research, documented in [code_agent_benchmark](../../research/en/code_agent_benchmark.md). Whichever benchmark we choose, it first needs a non-interactive way to invoke the code agent. Only then can a program run the agent on a task from start to finish and evaluate the result. The interface requirements for the first recommended benchmarks are collected in [benchmark_headless_interface](../../research/en/benchmark_headless_interface.md). +We first researched [code-agent benchmarks](../../research/en/code_agent_benchmark.md). Regardless of the benchmark, the first requirement is a non-interactive agent interface: a program must be able to launch the agent, execute a task end to end, and evaluate the result. We then examined the recommended benchmarks' [headless interfaces](../../research/en/benchmark_headless_interface.md). -The first feature is a **headless CLI**, step one of the research's minimum viable path. It is the only hard blocker: the first three requirements shared by the three benchmarks all depend on it—receive a task and finish in one command, work in the process's current directory, and run without interaction, questions, or confirmation. The current sole entry point is the `input("You> ")` loop in `agent.py`. When container stdin is EOF, the process immediately prints `Bye!` and exits without doing anything. Until this path works, later harness changes cannot be verified programmatically, leaving no basis for optimization. +The first feature is therefore a **headless CLI**, the first step in the research's minimum viable path. It is the only hard blocker. The first three shared requirements—accept a task through one command and exit afterward, operate in the current working directory, and avoid interaction, questions, or confirmation—depend on it. The existing entry point is the `input("You> ")` loop in `agent.py`. In a container, stdin is EOF, so the process immediately prints `Bye!` and exits without doing anything. Until this works, later harness changes cannot be evaluated programmatically. ### Basic headless CLI implementation -Start with the minimum functionality needed to complete one simple benchmark task. Troubleshooting and performance improvements can follow. +Start with the smallest feature that can complete a simple benchmark task, before addressing diagnosis or optimization. **Command-line form:** @@ -22,54 +22,54 @@ nanoPyCodeAgent [-p/--prompt "" | --prompt-file | (stdin pipe)] [--version] ``` -`--version` serves a concrete purpose: Harbor uses the adapter's `get_version_command()` and `parse_version()` to discover and record the agent version on a best-effort basis. Failure does not raise an error. `_package_version()` already exists and only needs a CLI option. +`--version` has a practical purpose: Harbor uses the adapter's `get_version_command()` and `parse_version()` to discover and record the agent version on a best-effort basis. `_package_version()` already exists and only needs to be exposed through the CLI. -**Headless detection:** Supplying `-p` or `--prompt-file` selects headless mode. If neither is supplied and `sys.stdin.isatty()` is False, read all of stdin as the task. Enter the existing REPL only when connected to a tty with no task supplied. All three input paths are needed: Harbor's official Claude Code example uses a stdin pipe, `printf … | claude --print`, to avoid shell escaping and command-length limits; mini-swe-agent uses `--task=` and explicitly connects stdin to `/dev/null`. This also fixes the current behavior of immediately printing `Bye!` and exiting when container stdin is EOF. +**Headless detection:** `-p` or `--prompt-file` selects headless mode. If neither is present and `sys.stdin.isatty()` is false, read all stdin as the task. Enter the existing REPL only when stdin is a terminal and no task is supplied. Support all three input forms: Harbor's Claude Code example pipes `printf … | claude --print`, avoiding shell quoting and command-line length problems; its mini-swe-agent example uses `--task=` and connects stdin to `/dev/null`. This also fixes the immediate `Bye!` on container EOF. **Exit codes:** | Exit code | Situation | | :-: | --- | -| 0 | The model declares completion; the turn budget is exhausted; the task is not solved | +| 0 | The model declares completion; the turn budget is exhausted; the task remains unsolved | | Nonzero | Missing API credentials; invalid arguments; API or transport errors | -Exiting with 0 despite unfinished work may seem counterintuitive. Harbor wraps the agent command in `set -o pipefail`; a nonzero exit raises `NonZeroAgentExitCodeError`, marks the trial as an agent failure, and may trigger a costly retry. The verifier should decide whether the task was solved using reward files under `/logs/verifier/`. The agent's exit code should not make that decision. +Exiting 0 for an unfinished task may seem surprising. Harbor wraps agent commands in `set -o pipefail`; a nonzero status raises `NonZeroAgentExitCodeError`, classifies the trial as an agent failure, and may trigger retries that spend money unnecessarily. The verifier's reward file under `/logs/verifier/` determines whether the task was solved. -This also exposes an existing bug: without API credentials, `run()` prints a message and returns, while `main()` has no return-code handling. The process therefore exits with 0. To the harness, that looks like a completed run that failed to solve the task. A whole batch could silently score zero without revealing the configuration error. `main()` must return an int, which the console script uses as the exit code. +This exposes an existing bug: when credentials are missing, `run()` prints a message and returns, while `main()` has no return-code contract, so the process exits 0. To the harness, every task appears to have run unsuccessfully, hiding the configuration error. `main()` must return an integer, which the console script uses as the process exit code. -**Turn limit:** The inner tool-use loop currently has no limit. In interactive mode, a person can press Ctrl-C when the model repeatedly tries the same command. In headless mode, it can continue spending until an API error stops it. `--max-turns` requires only a counter but is essential for unattended operation, so it belongs with the CLI. Exhausting the budget exits with 0, as above. +**Turn limit:** the inner tool-use loop currently has no limit. An interactive user can press Ctrl-C when the model repeatedly runs the same command; an unattended run otherwise continues spending until the API fails. `--max-turns` needs little more than a counter but is necessary for unattended operation, so it belongs in this feature. Exhaustion exits 0 as specified above. -**Headless system prompt:** The current prompt is written for a conversational assistant. A headless model has nobody to ask. A polite request such as “Should I continue?” ends its turn and leaves the task scoring zero. Headless mode needs a separate system prompt that explicitly requires autonomous decisions, no questions or pauses for confirmation, and a clear declaration when finished. +**Headless system prompt:** the current prompt addresses an interactive assistant. A headless model has nobody to ask; if it politely asks whether to continue, the turn ends and the task scores 0. Use a dedicated prompt requiring independent decisions, no questions or confirmation waits, and a clear completion statement. -**Preserve API error text:** Harbor scans stdout/stderr with regular expressions to classify rate limits, usage limits, overloaded services, context overflow, missing authentication, network interruptions, and other errors. These classifications combine with options such as `--max-retries 3 --retry-include ApiRateLimitError` to decide whether to retry. Emit the original error rather than swallowing or rewriting it so the harness can handle part of the retry work. +**Preserve API error text:** Harbor scans stdout/stderr with regular expressions to classify rate limits, usage limits, overload, context overflow, authentication problems, and network failures. It combines these categories with options such as `--max-retries 3 --retry-include ApiRateLimitError`. Preserve original errors instead of swallowing them so the harness can make retry decisions. -**Acceptance:** One command demonstrates whether this step works: +**Acceptance:** one command establishes the minimum contract: ```bash printf "%s" "create hello.py that prints hi" | nanoPyCodeAgent; echo $? ``` -The task must arrive through the pipe, create the file, and exit with 0. This is the first point at which nanoPyCodeAgent can be called from scripts and connected to benchmarks. +This feature is complete when the task arrives through the pipe, the file is created, and the command exits 0. Only then can nanoPyCodeAgent be called programmatically and connected to benchmarks. -Later work includes `--output-format stream-json` and `--trajectory` (step four of the minimum viable path, when failures need attribution), API retries and backoff (step two), context compaction (step five), `--workdir` (downgraded from P0 because all three benchmarks supply a container WORKDIR, so using the process cwd suffices), and `--timeout`. The harness controls wall-clock timeouts through settings such as `[agent].timeout_sec` in Harbor's `task.toml`; an agent timeout is an additional safeguard rather than an integration prerequisite. +Subsequent work includes `--output-format stream-json` and `--trajectory` (step 4 of the minimum path, needed for failure attribution), API retry/backoff (step 2), context compaction (step 5), `--workdir` (removed from P0 because all three benchmarks provide the container's working directory), and `--timeout` (an additional safeguard; the harness already enforces wall-clock limits, including Harbor's `[agent].timeout_sec` in `task.toml`). ### First end-to-end acceptance with Terminal-Bench -The `hello.py` smoke test establishes that the CLI can accept a task and exit, but does not establish integration with a real benchmark. The goal here is to run the whole path once on a sufficiently simple task that still exercises command execution, file writing, recovery from mistakes, and official verification. +The `hello.py` smoke test only proves that the CLI accepts a task and exits. To validate benchmark integration, choose a simple real task that still exercises commands, file writes, recovery from mistakes, and official verification. The goal is to establish the complete execution chain before reporting benchmark scores. -The selected task is `terminal-bench/openssl-selfsigned-cert` from Terminal-Bench 2.1. Harbor 0.21.0 and the nanoPyCodeAgent adapter were temporarily placed under `/tmp`, without adding benchmark-specific code to the project. The adapter inherits Harbor's `BaseInstalledAgent`. The `module:ClassName` dynamic import syntax in `--agent nanopy_harbor_agent:NanoPyCodeAgent` lets host Harbor execute this sequence: +We selected Terminal-Bench 2.1's `terminal-bench/openssl-selfsigned-cert`. Harbor 0.21.0 and the temporary nanoPyCodeAgent adapter were installed under `/tmp`; no benchmark-specific code was added to the project. The adapter subclasses Harbor's `BaseInstalledAgent`. Its `--agent nanopy_harbor_agent:NanoPyCodeAgent` argument uses the `module:ClassName` import syntax to establish this chain: ```text Harbor - -> dynamically load the temporary adapter - -> start the task's Docker environment - -> install and invoke the nanoPyCodeAgent headless CLI in /app - -> wait for the agent to exit - -> run the official verifier - -> save logs, test results, and reward + -> Dynamically import the temporary adapter + -> Start the task's Docker environment + -> Install and invoke the nanoPyCodeAgent headless CLI in /app + -> Wait for the agent to exit + -> Run the official verifier + -> Save logs, test results, and reward ``` -The core configuration follows. Credentials, base URL, and model configuration were still injected through environment variables rather than written into the command or logs: +The actual core configuration was as follows. Credentials, base URL, and model configuration were supplied through environment variables rather than embedded in command text or logs: ```bash harbor run \ @@ -81,51 +81,51 @@ harbor run \ --n-attempts 1 ``` -The first setup exceeded the default setup timeout while the container downloaded Python and dependencies. No model call had occurred, so this was not a benchmark result. Increasing only the installation timeout, while keeping the task, model, turn budget, and verifier unchanged, produced a complete trial: +Initial setup exceeded the default setup timeout while downloading Python and dependencies for the first time. No model call had occurred, so this was not a benchmark result. After increasing only the installation timeout, preserving the task, model, turn limit, and verifier, the trial completed: | Metric | Result | | --- | --- | | Reward | 1.0 | | Verifier | 6/6 tests passed | -| Agent execution time | 2 minutes 9 seconds | +| Agent execution time | 2 min 9 sec | | Tool calls | 16 bash, 2 write, 1 edit | | Agent / verifier exceptions | 0 | -This establishes the minimum execution loop: Harbor can deliver a real task to the headless CLI; the CLI can call the model and tools unattended in the isolated container's current directory; the model can recover from mistakes; and the official verifier can independently assign a reward after the process exits normally. The simplest benchmark now works from task input through final scoring. Reliable performance across a full suite remains to be established. +This established the minimum complete workflow: Harbor delivers a real task to the headless CLI; the CLI calls the model and tools unattended in an isolated container's working directory; the model can recover from mistakes; and the official verifier independently scores the result after the process exits. It does not yet establish reliable performance across the full benchmark suite. ### Dependency and observability limits exposed by the run -One container installation reported missing `httpx` when starting the CLI. The temporary adapter continued after adding `uv tool install --with httpx`. Two subsequent clean installations with the same uv, Python, and project commit did not reproduce the omission because `anthropic` still installs `httpx` transitively. This does not establish that clean installations always fail. The confirmed issue is that nanoPyCodeAgent directly imports `httpx` without declaring it as a direct dependency. Its operation currently borrows the `anthropic -> httpx` relationship, and that dependency ownership should be corrected. +One container installation reported missing `httpx` when starting the CLI. The temporary adapter continued after installing with `uv tool install --with httpx`. Two subsequent clean installations using the same uv, Python, and project commit did not reproduce the omission because `anthropic` still installed `httpx` transitively. The evidence does not support saying that clean installs always fail. The confirmed problem is dependency ownership: nanoPyCodeAgent imports `httpx` directly without declaring it. Depending on `anthropic -> httpx` to provide it should be corrected. -Even without a trajectory, tool calls could be counted from ordinary text logs because the CLI prints prefixes such as `[bash]`, `[write]`, and `[edit]` before each call. Such counts lack turn boundaries, tool-call IDs, input/output tokens, costs, durations, and structured error states, leaving Harbor's token and cost fields `null`. +Although no trajectory existed yet, tool calls could still be counted using prefixes such as `[bash]`, `[write]`, and `[edit]` in the text logs. These counts lacked turn boundaries, tool-call IDs, input/output tokens, cost, duration, and structured error states, so Harbor's token and cost fields remained `null`. -As explained in [agent_output_and_trajectory](../../research/en/agent_output_and_trajectory.md), a text log cannot simply be renamed a trajectory. Run output is the public content delivered by an invocation; a trajectory is a structured execution path bounded by one task or trial, intended for benchmarking and offline analysis. `--output-format text|json|stream-json` should control stdout's public representation. A separate `--trajectory PATH` should save observations, actions, tool results, outcomes, and usage. Both should be projected from the same internal events while retaining distinct interfaces. +The [output and trajectory research](../../research/en/agent_output_and_trajectory.md) explains why this text log cannot simply be renamed a trajectory. Run output is the invocation's public output; a trajectory is a structured execution path bounded by one task or trial, intended for benchmarking and offline analysis. `--output-format text|json|stream-json` should control stdout's public representation, while `--trajectory PATH` independently saves observations, actions, tool results, outcomes, and usage. Both should project the same internal canonical events while retaining separate interfaces. The next work therefore has three parts: -- Declare `httpx` directly so container installation does not depend on a transitive relationship. -- Bring the Harbor adapter into the repository and version its installation commands, headless invocation, environment forwarding, and version detection for reuse in future benchmarks. -- Design internal events and a trajectory writer so the agent emits ATIF directly and the adapter reliably populates tokens, costs, and steps. Implement `stream-json` later as separate run output. These changes improve installation, reproducibility, and observability of the established execution path. +- Declare `httpx` directly so container installations do not depend on a transitive relationship. +- Add the Harbor adapter to the repository, versioning installation, headless invocation, environment forwarding, and version discovery so future runs can reuse it. +- Design canonical internal events and a trajectory writer so the agent emits ATIF directly and the adapter reliably backfills tokens, cost, and step counts. Implement `stream-json` later as independent run output. These changes improve installation, reproducibility, and observability of the established workflow. -### Bringing the Harbor adapter into the project +### Bringing the Harbor adapter into the repository -The temporary adapter now lives in the independent `benchmarks/harbor/` workspace, exposed as `harbor_adapter:NanoPyCodeAgent`. It serves development and evaluation, stays outside `src/nanopycodeagent`, and does not add Harbor to the ordinary user wheel. Its own `pyproject.toml` and `uv.lock` pin Harbor to 0.21.0, the version used for implementation and tests. Running `uv run --project benchmarks/harbor harbor ...` from the repository root loads the adapter without `/tmp` or a manually configured `PYTHONPATH`. +The temporary acceptance script now lives in an isolated `benchmarks/harbor/` workspace with public import path `harbor_adapter:NanoPyCodeAgent`. It supports development and evaluation, stays outside `src/nanopycodeagent`, and does not add Harbor to the user-facing wheel. The workspace's `pyproject.toml` and `uv.lock` pin Harbor to 0.21.0, the version used for implementation and testing. Running `uv run --project benchmarks/harbor harbor ...` from the repository root loads the adapter without relying on `/tmp` or a manually configured `PYTHONPATH`. -Installation supports two reproducible pins. `--agent-kwarg version=X.Y.Z` runs `uv tool install --force nanoPyCodeAgent==X.Y.Z` inside the task container. `--agent-kwarg git_ref=` installs an exact GitHub revision for benchmarking before release. The two options are mutually exclusive; omitting both installs the latest PyPI version. The uv bootstrap URL is pinned to 0.9.11, and installation immediately runs `nanoPyCodeAgent --version` as a self-check. The temporary `--with httpx` workaround has been removed because the project now declares `httpx` directly. +Installation supports two reproducible pins. `--agent-kwarg version=X.Y.Z` runs `uv tool install --force nanoPyCodeAgent==X.Y.Z` in the task container. `--agent-kwarg git_ref=` installs an exact GitHub revision for unreleased benchmarks. These options are mutually exclusive; omitting both installs the latest PyPI release. The uv bootstrap URL is pinned to 0.9.11, and installation immediately checks `nanoPyCodeAgent --version`. The temporary `--with httpx` workaround was removed because the project now declares the dependency directly. -The instruction is no longer interpolated into shell syntax. The adapter injects the complete task through a temporary environment variable, then pipes it into the headless CLI with `printf`. Quotes, newlines, dollar signs, and arbitrary task lengths do not become shell syntax. The CLI retains its default limit of 50 turns, overridable with `--agent-kwarg max_turns=N`. Under `pipefail`, stdout/stderr are combined and saved with `tee` to `/logs/agent/nanopycodeagent.txt`, allowing Harbor to classify API errors and preserving nonzero CLI exits through the pipeline. +The instruction is no longer interpolated into shell command text. A one-use environment variable carries the complete task into the container, and `printf` pipes it to the headless CLI. Quotes, newlines, dollar signs, and arbitrary task lengths do not become shell syntax. The CLI keeps its 50-turn default, overridden with `--agent-kwarg max_turns=N`. Under `pipefail`, stdout/stderr are combined and saved by `tee` to `/logs/agent/nanopycodeagent.txt`, preserving API error classification and nonzero CLI statuses. -The configuration boundary is also fixed. `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, and `ANTHROPIC_MODEL` are forwarded into the container. When Harbor supplies a `provider/model` name and the provider's key/base URL, the adapter normalizes credentials and endpoint into the Anthropic SDK variables. Unless `ANTHROPIC_MODEL` is explicitly set, it removes the first provider prefix. An explicit `ANTHROPIC_MODEL` takes precedence, preserving the proxy-endpoint model selection used in the first run. Version detection extracts `0.8.0` from `nanoPyCodeAgent 0.8.0` for Harbor. +Configuration boundaries are explicit. `ANTHROPIC_API_KEY`, `ANTHROPIC_BASE_URL`, and `ANTHROPIC_MODEL` pass through to the container. When Harbor supplies a `provider/model` identity and provider-specific credentials or base URL, the adapter normalizes them into the Anthropic SDK's variables and removes the first provider prefix unless `ANTHROPIC_MODEL` is explicit. That explicit override takes precedence, preserving model selection through a proxy endpoint. Version detection parses `nanoPyCodeAgent 0.8.0` into `0.8.0` for Harbor. -Five contract tests using Harbor 0.21.0's real base class first cover the container boundary: release pinning, Git revision pinning, mutually exclusive arguments, the instruction pipeline and logs, and both environment mappings. The same task was then rerun in Docker with the formal adapter. Harbor started the container, installed the pinned Git revision, injected model configuration, invoked the agent, collected logs, and ran the official verifier. Agent/verifier infrastructure exceptions were zero, establishing end-to-end adapter integration. The reward was 0 with 5/6 verifier tests passing: the generated `check_cert.py` depended on `cryptography`, which was absent from the verifier's Python environment. This concerns solution portability rather than adapter infrastructure. +Five contract tests using Harbor 0.21.0's real base class first covered the container boundary: release pinning, Git revision pinning, mutually exclusive options, instruction piping and logs, and the two environment mappings. The same task was then rerun through the official adapter in real Docker. Harbor started the container, installed the specified revision, forwarded model configuration, invoked the agent, collected logs, and ran the verifier with no agent/verifier infrastructure exceptions. The task scored 0 with 5/6 verifier tests passing: the agent's `check_cert.py` depended on `cryptography`, which was unavailable in the verifier's Python environment. This was a portability problem in the solution, rather than an adapter infrastructure failure. -Host integration was also checked through the independent workspace's Harbor version command and dynamic import. Building the root project wheel confirmed that it contains neither the adapter nor a Harbor dependency. The direct `httpx` dependency and formal adapter are now complete. Remaining trajectory work is to define internal events and an Event Journal, emit ATIF from the agent, and reliably populate tokens, costs, and steps in the adapter. `stream-json` is separate run output planned for a later PR. +The workspace's Harbor version command and dynamic import checked host integration, while building the root wheel confirmed it contained neither the adapter nor a Harbor dependency. The first two follow-ups—direct `httpx` ownership and the repository adapter—are complete. The remaining trajectory work is to define canonical events and an Event Journal, emit ATIF directly, and have the adapter backfill tokens, cost, and steps reliably. `stream-json` remains separate run-output work for a later PR. ### Implementing trajectories -The research on [agent output and trajectory boundaries](../../research/en/agent_output_and_trajectory.md), [agent event mapping to ATIF](../../research/en/agent_events_to_atif_examples.md), [OpenRouter cost accounting](../../research/en/openrouter_cost_accounting.md), and [OpenRouter's unified model protocol](../../research/en/openrouter_unified_protocol.md) leads to a replayable internal **Event Journal** and an external `--trajectory` that emits only **ATIF-v1.7**. `stream-json` shares the same underlying runtime facts but remains separate run output. Its `--output-format` CLI belongs to another control surface outside trajectory implementation. +Research into [output and trajectory boundaries](../../research/en/agent_output_and_trajectory.md), [event-to-ATIF mapping](../../research/en/agent_events_to_atif_examples.md), [OpenRouter cost accounting](../../research/en/openrouter_cost_accounting.md), and [OpenRouter's unified protocol](../../research/en/openrouter_unified_protocol.md) led to a replayable internal **Event Journal**, with public `--trajectory` output limited to **ATIF-v1.7**. `stream-json` is independent run output that shares the underlying facts. Its `--output-format` CLI interface belongs to a separate control surface outside this trajectory implementation. -A completed `read` call illustrates the concepts of a `Native Event` and a `Journal Entry`. The agent loop first emits a `Native Event` describing only what happened: +A completed `read` call illustrates the difference between a `Native Event` and a `Journal Entry`. The agent loop first emits a fact describing what happened: ```json { @@ -159,24 +159,24 @@ The journal writer adds the identity, ordering, and recording time needed for pe } ``` -Both describe the same event. The `Native Event` carries facts produced by the core; the `Journal Entry` persists those facts with the run identity, sequence, and recording time. The ATIF projector consumes entries ordered by `seq` and folds multiple facts into trajectory steps. +These describe the same occurrence. A `Native Event` is a runtime fact emitted by the core; a `Journal Entry` is its persistent record, additionally identifying the run, sequence, and recording time. The ATIF projector consumes entries ordered by `seq` and folds multiple facts into trajectory steps. -Implementation is organized around independently usable and verifiable capabilities, rather than seven disconnected technical layers. Each capability defines its implementation and acceptance criteria together: +Implementation is organized around independently usable and verifiable capabilities, rather than seven disconnected technical layers. Each capability defines both implementation and acceptance: -1. **Runtime facts and Event Journal.** Version the `Native Event` and `Journal Entry` contracts. Emit facts at user, model, tool, and run boundaries and append them to the internal Event Journal. Project text output from the same facts while preserving visible behavior. -2. **Public ATIF trajectory.** Implement a one-way Event Journal to ATIF-v1.7 projector enabled in headless runs by `--trajectory PATH`. The path names a complete ATIF JSON snapshot, not the internal journal, and does not change stdout. Map defined messages, tools, timestamps, durations, token/cache usage, and terminal state. Cost mapping depends on the separate provider-cost contract and reconciliation mechanism. -3. **Provider cost collection and reconciliation.** Prefer actual costs in model response `usage.cost`. When costs are absent but a generation ID exists, mark them pending and query the provider's Generation API for `total_cost` during run finalization. Append results without rewriting existing journal entries or changing task outcomes. Map known costs into steps and run totals, explicitly track completeness, and never substitute zero for an unknown cost. -4. **Harbor integration and end-to-end validation.** The adapter supplies the trajectory path, declares and reads agent-generated ATIF, and populates steps, tokens, and costs. It does not convert a native trajectory. Verify collection and metrics with contract tests, then confirm the full path in a real trial. +1. **Runtime facts and the Event Journal.** Establish versioned contracts for `Native Event` and `Journal Entry`. At user, model, tool, and run boundaries, the loop emits facts that are appended to an internal journal. Project text output from the same facts while preserving user-visible behavior. +2. **Public ATIF trajectories.** Implement a one-way Event Journal to ATIF-v1.7 projector, enabled in headless runs by `--trajectory PATH`. The path identifies a complete ATIF JSON snapshot, not the internal journal, and does not change stdout. Map messages, tools, timestamps, durations, token/cache usage, and terminal state. Cost mapping depends on a separate provider-cost contract and reconciliation mechanism. +3. **Provider-reported cost collection and reconciliation.** Prefer actual costs in response `usage.cost`. If cost is absent but a generation ID exists, record a pending state and query the provider's Generation API for `total_cost` during finalization. Append query results without changing existing entries; failures do not change the task outcome. Project known costs into steps and run totals, explicitly reporting completeness and never replacing unknown costs with zero. +4. **Harbor integration and end-to-end acceptance.** The adapter supplies the trajectory path, declares and reads the agent's ATIF, and backfills steps, tokens, and cost without converting a native trajectory. Validate collection and statistics with contract tests, then run a real trial. -`--output-format` and `stream-json` belong to separate run output and are outside these trajectory capabilities. +`--output-format` and `stream-json` remain separate run-output capabilities. -#### Runtime facts and Event Journal +#### Runtime facts and the Event Journal -**What to build:** Version the Native Event and Journal Entry contracts, keep the agent loop concerned with runtime facts, and append a replayable internal Event Journal for each Agent Run. Existing text stdout is another projection of those facts. The Event Journal is sensitive internal reconstruction data, not public run output or a trajectory. +**Scope:** define versioned Native Event and Journal Entry contracts so the loop emits runtime facts and each Agent Run appends a replayable journal. Existing text stdout is projected from the same facts. The Event Journal is sensitive internal reconstruction data, distinct from public run output and trajectories. -**Protocol:** [Event Journal implementation protocol v1](../../dev_docs/en/event-journal-protocol-v1.md) defines schema version 1's wire contract, event semantics, persistence behavior, and compatibility boundary. These development notes do not duplicate its field definitions and validation rules. +**Protocol:** [Event Journal implementation protocol v1](../../dev_docs/en/event-journal-protocol-v1.md) defines schema version 1's wire contract, event semantics, persistence, and compatibility boundaries. Field and validation details are not duplicated here. -**Validation:** Behavioral tests should establish valid contracts and ordering, append/replay support, survival of complete records after interruption, restricted access to sensitive data, and unchanged stdout behavior after event integration: +**Validation:** behavioral tests must establish verifiable contracts and ordering, append/replay behavior, preservation of complete records after interruption, restricted sensitive-data permissions, and unchanged stdout: ```bash uv run pytest \ @@ -185,19 +185,19 @@ uv run pytest \ tests/test_agent.py ``` -Acceptance requires these tests to return 0 and a corresponding test for every mandatory behavior in the protocol document. +Acceptance requires these tests to exit 0 and every mandatory protocol behavior to have test coverage. -#### Public ATIF trajectory +#### Public ATIF trajectories -**What to build:** Implement a one-way Event Journal to ATIF-v1.7 projector that represents one headless Agent Run as a complete ATIF JSON document. Add a separate `--trajectory PATH` option to enable the projector and choose the output file while preserving text stdout. The internal Event Journal stays private. Provider cost collection/reconciliation, Harbor adapter collection, `stream-json`, and interactive trajectories spanning multiple runs are separate capabilities. +**Scope:** implement a one-way Event Journal to ATIF-v1.7 projector that exports one headless Agent Run as a complete JSON document. Add `--trajectory PATH` as an independent opt-in CLI argument specifying the output file, while preserving text stdout. Exposing the internal journal, provider cost collection/reconciliation, Harbor collection, `stream-json`, and trajectories spanning multiple interactive runs are outside this capability. -**Protocol:** Use ATIF-v1.7 as implemented in Harbor 0.21.0: +**Protocol:** target Harbor 0.21.0's ATIF-v1.7: - [Official Harbor ATIF documentation](https://www.harborframework.com/docs/agents/trajectory-format). - [ATIF-v1.7 RFC](https://github.com/harbor-framework/harbor/blob/v0.21.0/rfcs/0001-trajectory-format.md). - [Pydantic reference implementation](https://github.com/harbor-framework/harbor/tree/v0.21.0/src/harbor/models/trajectories) and [trajectory validator](https://github.com/harbor-framework/harbor/blob/v0.21.0/src/harbor/utils/trajectory_validator.py). -**Validation:** Automated tests should show that representative journals preserve user/model/tool information, usage, and run terminal state in projection; stdout remains unchanged; and files are created only when `--trajectory` is supplied. Every generated trajectory must also pass the repository's pinned Harbor 0.21.0 validator: +**Validation:** automated tests must preserve user/model/tool facts, usage, and terminal state from representative journals; leave stdout unchanged; and create files only when `--trajectory` is supplied. Every generated trajectory must also pass the repository's pinned Harbor 0.21.0 validator: ```bash nanoPyCodeAgent -p "read README.md and summarize it" \ @@ -207,60 +207,60 @@ uv run --project benchmarks/harbor \ /tmp/nanopycodeagent-trajectory.json ``` -Acceptance requires validator exit code 0, unchanged text stdout, and a complete JSON output file. Adapter reading and a real trial belong to the later Harbor integration acceptance. +Acceptance requires validator exit 0, unchanged text stdout, and a complete JSON output file. Adapter collection and a real trial belong to subsequent Harbor integration acceptance. -#### Provider cost collection and reconciliation +#### Provider-reported cost collection and reconciliation -**What to build:** Record the provider's actual reported model-call costs in trajectories. Three layers have similar terminology but different roles: +**Scope:** record actual provider-reported model-call costs in trajectories. Three similarly named data layers have different roles: -- **Provider responses** supply the cost facts. `usage.cost` is the `cost` field inside the model response's `usage` object. `X-Generation-Id` is the generation ID in a response HTTP header. `data.total_cost` is the total cost inside the Generation API response body's `data` object. -- **Event Journal** persists facts collected by the agent. `model.completed` is emitted when a model call finishes; its `payload.cost` records that call's cost status. A successful delayed lookup appends `model.cost_resolved`, whose `payload` contains the generation ID, amount, currency, and source. -- **ATIF trajectory** is the public projection of those facts. A model step's `metrics.cost_usd` is its call cost. The top-level `final_metrics` contains run cost totals and completeness information. +- **Provider responses** supply cost facts. `usage.cost` is the response usage object's cost field; `X-Generation-Id` is the generation ID in an HTTP response header; `data.total_cost` is the total-cost field inside a Generation API response's `data` object. +- **The Event Journal** stores facts collected by the agent. `model.completed` records completion of a model call, with its cost state in `payload.cost`. A later successful lookup appends `model.cost_resolved`, whose payload includes generation ID, amount, currency, and source. +- **The ATIF trajectory** is the public projection. A model step's `metrics.cost_usd` holds that call's cost, while top-level `final_metrics` holds run totals and completeness. -There are two collection paths: +Costs have two collection paths: -- **Synchronous collection from responses.** OpenRouter's [Usage Accounting](https://openrouter.ai/docs/cookbook/administration/usage-accounting) defines complete `usage` for OpenAI-compatible Chat Completions and streaming responses. `usage.cost` is the total account charge for that request, returned in the complete non-streaming response or final streaming SSE message. When present, record it immediately in `model.completed.payload.cost` as a provider-reported USD amount with status `resolved`. Token usage alone does not establish that cost is present. To preserve Anthropic schema compatibility, OpenRouter's [Anthropic Messages API](https://openrouter.ai/docs/api/api-reference/anthropic-messages/create-messages) defines token, cache, service-tier, and speed fields in `usage`, but no `cost`. This project currently calls that interface through the Anthropic SDK, so observed usage may contain tokens and extensions such as `speed: "standard"` without a cost. -- **Asynchronous reconciliation by generation ID.** When `usage.cost` is absent but the HTTP response has `X-Generation-Id`, save the identifier in `model.completed.payload.generation_id` and set `payload.cost.status` to `pending`. This covers the current OpenRouter Anthropic Messages path and compatible interfaces that return an identity before the usage record is ready. OpenRouter documents saving the generation ID, calling [`GET /api/v1/generation?id=...`](https://openrouter.ai/docs/api/api-reference/generations/get-generation), and reading `data.total_cost`. The endpoint documents `404`, `429`, and 5xx responses, so a generation ID does not imply an immediately queryable billing record. +- **Synchronous response collection.** OpenRouter's [Usage Accounting](https://openrouter.ai/docs/cookbook/administration/usage-accounting) defines complete usage for OpenAI-compatible Chat Completions and streaming responses. `usage.cost` is the total charged to the account for that request, provided in the complete non-streaming response or final SSE message. When present, save it immediately as provider-reported USD cost in `model.completed.payload.cost` with status `resolved`. Token usage does not imply cost availability. OpenRouter's [Anthropic Messages API](https://openrouter.ai/docs/api/api-reference/anthropic-messages/create-messages) preserves the Anthropic-compatible schema: its usage includes tokens, cache, service tier, and speed, but does not define `cost`. This project currently calls that interface through the Anthropic SDK, so observed usage may contain only tokens and extensions such as `speed: "standard"`. +- **Asynchronous reconciliation by generation ID.** If `usage.cost` is absent but the HTTP header supplies `X-Generation-Id`, record it in `model.completed.payload.generation_id` and mark `payload.cost.status` as `pending`. This covers OpenRouter's current Anthropic Messages path and other compatible interfaces that return an identity before the usage record is ready. OpenRouter documents saving the generation ID and later calling [`GET /api/v1/generation?id=...`](https://openrouter.ai/docs/api/api-reference/generations/get-generation) for `data.total_cost`. Documented responses include `404`, `429`, and 5xx, so an existing generation ID does not guarantee its billing record is queryable yet. -Reconciliation derives a same-origin `v1/generation` endpoint from the current Anthropic SDK base URL. It does not bind a provider by hostname or send credentials to a hardcoded destination; OpenRouter currently verifies this extension protocol. Queries make up to six attempts with `1, 2, 4, 8, 15` seconds of exponential backoff, totaling about 30 seconds of waits. Retry `404`, `408`, `409`, `429`, common 5xx responses, network errors, and HTTP 200 responses that still lack `total_cost`. Stop immediately for permanent failures such as `400`, `401`, `402`, and `403`. On success, append `model.cost_resolved`: put the amount in `payload.amount`, the association key in `payload.generation_id`, and the currency/source in `payload.currency` and `payload.source`. Persisted `model.completed` entries remain unchanged, preserving append-only journal semantics. +Reconciliation derives a same-origin `v1/generation` endpoint from the Anthropic SDK's base URL. It does not bind to a provider hostname or forward credentials to a hardcoded service; OpenRouter currently validates this extension protocol. The query uses six bounded attempts and exponential delays of `1, 2, 4, 8, 15` seconds, totaling about 30 seconds of waiting. Retryable conditions include `404`, `408`, `409`, `429`, common 5xx responses, network errors, and HTTP 200 without `total_cost`. Permanent failures such as `400`, `401`, `402`, and `403` stop immediately. Success appends `model.cost_resolved` with amount, association key, currency, and source in `payload.amount`, `payload.generation_id`, `payload.currency`, and `payload.source`. Existing `model.completed` entries remain unchanged, preserving append-only storage. -Reconciliation observability needs more than `None`. The terminal event optionally carries a `cost_reconciliation` list. Each generation records its final `resolved` or `unresolved` status and individual attempt results: `resolved`, `cost_unavailable` for HTTP 200 without a cost, `http_error` with a status code, `request_error` with an exception type, or `unsupported_endpoint` when the base URL cannot produce a lookup endpoint. Credentials and provider response bodies are excluded. The ATIF projector copies this list into `extra.terminal.cost_reconciliation`, supporting diagnosis from either the internal journal or public trajectory. +Observability cannot depend on `None`. Each run's terminal event may include a `cost_reconciliation` list recording each generation's final `resolved`/`unresolved` state and every attempt: `resolved`, HTTP 200 with `cost_unavailable`, `http_error` with status code, `request_error` with exception type, or `unsupported_endpoint` when the base URL cannot yield a lookup endpoint. Credentials and provider response bodies are omitted. ATIF copies this list to `extra.terminal.cost_reconciliation`, making incomplete costs diagnosable in either representation. -Cost collection is best-effort enrichment. HTTP errors, unavailable billing records, and invalid responses do not change the original task's success or failure. The ATIF projector joins delayed results to model steps through `payload.generation_id` and emits: +Cost collection is best-effort enrichment. HTTP errors, delayed billing records, and invalid responses do not change the original task's success or failure. The projector associates later costs with model steps through `payload.generation_id` and emits: -- Per-call cost in the step's `metrics.cost_usd`, with source and generation ID in `metrics.extra.cost_source` and `metrics.extra.generation_id`. -- Run total in `final_metrics.total_cost_usd` when every billable call has a known cost. -- Otherwise, a known subtotal in `final_metrics.extra.known_cost_usd` and an incomplete flag in `final_metrics.extra.cost_is_partial`. Identifiable unresolved calls are listed in `final_metrics.extra.missing_generation_ids`. +- Per-call cost in `metrics.cost_usd`, with source and generation ID in `metrics.extra.cost_source` and `metrics.extra.generation_id`. +- A complete run total in top-level `final_metrics.total_cost_usd` when all billable calls have costs. +- Otherwise, a known subtotal in `final_metrics.extra.known_cost_usd`, an incompleteness flag in `final_metrics.extra.cost_is_partial`, and identifiable pending calls in `final_metrics.extra.missing_generation_ids`. -Unknown costs remain unknown and are never written as zero. +Unknown costs remain unknown; they are never written as zero. -**Automated acceptance:** Cost unit tests cover direct response collection, pending/unknown states, successful Generation API retries, permanent HTTP failures, bounded unsuccessful lookups, and per-attempt diagnostics. Agent event tests establish that reconciliation happens before the terminal event and that failure diagnostics are persisted. ATIF tests cover step costs, complete and partial totals, and terminal diagnostics. Run the complete suite with: +**Automated acceptance:** cost tests cover direct response collection, pending/unknown states, successful Generation API retries, immediate permanent HTTP failures, bounded failures, and per-attempt diagnostics. Agent event tests verify reconciliation before the terminal event and durable failure diagnostics. ATIF tests cover per-step costs, complete and partial totals, and terminal diagnostics. Run the full suite: ```bash uv run pytest ``` -#### Harbor integration and end-to-end validation +#### Harbor integration and end-to-end acceptance -**What to build:** +**Scope:** -1. Have the repository's Harbor adapter specify an in-container trajectory output path for each trial. After the agent finishes, report the path and format so Harbor reads the agent-generated ATIF-v1.7 file. -2. Populate Harbor with complete ATIF steps and the token, cache-token, cost, and completeness information in step and final metrics. -3. Make the adapter's responsibilities and failure semantics explicit. It handles paths and data handoff, without parsing the Event Journal or maintaining a native trajectory conversion. Missing, invalid, or partial trajectories need clear diagnostics and must not appear as zero consumption or complete results. +1. Have the repository adapter specify a trajectory path inside each trial container, then tell Harbor the path and format after the agent finishes so Harbor reads the agent-generated ATIF-v1.7 file. +2. Backfill complete steps and token, cache-token, cost, and completeness information from step and final metrics. +3. Keep the adapter responsible for path and data handoff. It does not parse Event Journals or maintain a native trajectory converter. Missing, invalid, or partial trajectories must receive explicit diagnostics instead of appearing as zero consumption or complete results. **Validation:** -1. Extend adapter contract tests in `benchmarks/harbor/tests` to verify that the trajectory path reaches the container and that ATIF can be declared and read. -2. Verify step, token, and cost reporting with contract tests, including missing files, failed validation, and incomplete metrics. -3. Run a real Terminal-Bench trial using the pinned Harbor version. Compare agent logs, Harbor's saved trajectory, steps, tokens, cost, and reward to establish the entire path from container installation and task execution through trajectory persistence, Harbor collection, and official scoring. Distinguish solution failures from adapter infrastructure failures. +1. Extend `benchmarks/harbor/tests` to verify container trajectory-path forwarding and ATIF declaration/collection. +2. Test step, token, and cost backfill, including missing files, validation failures, and incomplete metrics. +3. Run one real Terminal-Bench trial with the pinned Harbor version. Compare agent logs, collected trajectory, steps, tokens, cost, and reward across container installation, execution, trajectory writing, Harbor collection, and official verification. Distinguish a failed solution from adapter infrastructure failures. -**End-to-end test procedure:** +**End-to-end procedure:** -1. On a host with Docker installed, run `sudo systemctl start docker.service` to start the daemon. -2. If the current user cannot access the Docker socket, run `sudo usermod -aG docker "$USER"` to join the `docker` group. This system configuration is needed only once. -3. Log in again, or run `newgrp docker` in the current terminal to activate group permissions. Run `docker info` to confirm access to the daemon. -4. Set `ANTHROPIC_API_KEY` and `ANTHROPIC_BASE_URL`, and prepare a full 40-character commit SHA that can be installed remotely. Specify the model only through the next command's `--model`; the adapter converts it into `ANTHROPIC_MODEL` for nanoPyCodeAgent. -5. Run the fixed task from the repository root. `--env docker` lets Harbor pull or build the image, start the task container, and clean up afterward, without manually running `docker compose`: +1. On a host with Docker installed, start the daemon with `sudo systemctl start docker.service`. +2. If the current user cannot access the socket, run `sudo usermod -aG docker "$USER"` once to add the user to the Docker group. +3. Log in again or activate the group in the terminal with `newgrp docker`. Confirm daemon access with `docker info`. +4. Set `ANTHROPIC_API_KEY` and `ANTHROPIC_BASE_URL` and prepare a remotely installable full 40-character commit SHA. Specify the model through the next step's `--model`; the adapter converts it into `ANTHROPIC_MODEL`. +5. Run the pinned task from the repository root. With `--env docker`, Harbor pulls/builds the image, starts the container, and cleans it up afterward; manual `docker compose` commands are unnecessary: ```bash uv run --project benchmarks/harbor harbor run \ @@ -273,15 +273,15 @@ uv run pytest --n-attempts 1 ``` -6. Obtain the result directory from Harbor output, `jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/`, and inspect four artifacts: +6. Locate `jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/` in Harbor's output and inspect four artifacts: - - `agent/nanopycodeagent.txt`: agent stdout/stderr text log. + - `agent/nanopycodeagent.txt`: agent stdout/stderr. - `agent/trajectory.json`: agent-generated ATIF steps, metrics, and terminal state. - - `verifier/test-stdout.txt`: official test details. - - `result.json`: Harbor's aggregate agent metrics, reward, and exception information. + - `verifier/test-stdout.txt`: official test execution details. + - `result.json`: Harbor's aggregated agent metrics, reward, and exceptions. -7. Run `uv run --project benchmarks/harbor python -m harbor.utils.trajectory_validator jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/agent/trajectory.json`. The pinned Harbor 0.21.0 validator checks ATIF schema and cross-field constraints. Exit code 0 indicates successful format and semantic validation. -8. Compare `agent/trajectory.json` with `result.json`: require ATIF-v1.7, complete steps, matching tokens/cache tokens/cost between `final_metrics` and `agent_result`, an empty `exception_info`, and a verifier reward. +7. Run `uv run --project benchmarks/harbor python -m harbor.utils.trajectory_validator jobs/2026-09-03__21-42-56/openssl-selfsigned-cert__5p9AFiW/agent/trajectory.json` to check the ATIF schema and cross-field constraints using pinned Harbor 0.21.0. Exit 0 indicates successful format and semantic validation. +8. Compare the trajectory and result: ATIF-v1.7 schema, complete steps, consistent tokens/cache tokens/cost between `final_metrics` and `agent_result`, empty `exception_info`, and a verifier reward. **Observed results:** @@ -290,17 +290,17 @@ uv run pytest - Harbor exceptions: 0. - Trajectory: 11 steps, including 10 model steps. - Tokens: 49,878 input, 42,240 cache, 5,845 output. -- Cost: 0.00222441 USD; all 10 cost lookups succeeded on their first attempt. +- Cost: 0.00222441 USD; all 10 reconciliations succeeded on the first attempt. -### Running a Terminal-Bench batch and establishing a baseline +### Running Terminal-Bench in batches and establishing a baseline -The single-task validation established the path through the task container, agent, trajectory, cost reconciliation, and official verifier. Next, expand to 20 tasks to observe failure modes and actual consumption before deciding the budget for a baseline across all 89 tasks. These 20 tasks are a small-scale trial run; their pass rate is not the full Terminal-Bench 2.1 score. +Single-task acceptance established the task-container, agent, trajectory, cost-reconciliation, and verifier workflow. Next, expand to 20 tasks to observe failures and actual consumption before setting a budget for all 89 tasks. This pilot's pass rate is not the full Terminal-Bench 2.1 score. -#### Fixed experiment configuration +#### Fixed experimental configuration -This run was prepared on 2026-09-06 with the following fixed conditions. Future harness comparisons should reuse the tasks and execution limits while recording each new agent revision separately. +Prepared on 2026-09-06, this run fixes the following conditions. Later harness comparisons should reuse tasks and limits while recording each new agent revision. -| Item | Configuration for this run | +| Item | Configuration | | --- | --- | | Harbor | 0.21.0, using `benchmarks/harbor/uv.lock` | | Agent commit | `2b8309794cb9e00cb4d3b08baf1e4733153105e0` | @@ -308,21 +308,21 @@ This run was prepared on 2026-09-06 with the following fixed conditions. Future | Dataset content hash | `sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a` | | Model | `openrouter/deepseek/deepseek-v4-flash-0731` | | API endpoint | `https://openrouter.ai/api`, using Anthropic Messages | -| Provider routing | No provider override in requests; account routing settings are also an experimental condition | -| Trial size | First 20 dataset entries, one attempt per task | +| Provider routing | No request-level provider override; account routing settings are also experimental conditions | +| Pilot size | First 20 dataset entries, one attempt each | | Environment | Local Docker, concurrency 2 | | Agent limit | At most 50 model replies per task | -| Per-response limit | `MAX_TOKENS = 8192`; reasoning effort and temperature are not explicitly set | -| Harbor retries | 0; this does not disable the Anthropic SDK's internal request retries | +| Per-response limit | `MAX_TOKENS = 8192`; no explicit reasoning effort or temperature | +| Harbor retries | 0; Anthropic SDK internal request retries remain enabled | | Timeouts and resources | Preserve task defaults | -Harbor 0.21.0's `--n-tasks 20` takes the first 20 entries after filtering; it is not random sampling. Pinning the dataset hash fixes task versions, but the selected task list should still be saved. Repeated `--include-task-name` options can select those exact tasks later. OpenRouter provider routing and server behavior can still change, so pinning a model slug does not make the experiment fully deterministic. +Harbor 0.21.0's `--n-tasks 20` selects the first 20 entries after filtering, rather than sampling randomly. Pinning the dataset hash fixes task versions, but the actual selection should still be saved and later selected with repeated `--include-task-name` arguments. OpenRouter provider routing and server behavior can change, so a pinned model slug does not make the experiment fully deterministic. -Host Harbor uses a lockfile. Inside containers, `uv tool install` pins only the agent source revision and leaves dependency resolution unlocked. Public results retain the Docker version and task image RepoDigests for checking later environments. Host hardware details remain in local records. +Host Harbor uses a lockfile. Inside containers, `uv tool install` pins the agent revision but leaves dependency resolution unlocked. Public results retain Docker version and task-image RepoDigests for checking later environments; host hardware details stay in local records. #### Running the experiment -First confirm that `docker info` succeeds, then run from the repository root. Supply credentials through environment variables rather than command arguments or experiment records. `ANTHROPIC_MODEL` overrides the model derived by the adapter from `--model`, so this run explicitly removes it. +Confirm that `docker info` succeeds, then run from the repository root. Supply credentials through environment variables, without placing them in arguments or experiment records. Because `ANTHROPIC_MODEL` overrides the adapter's model derived from `--model`, explicitly remove it: ```bash export ANTHROPIC_API_KEY="${OPENROUTER_API_KEY:?Set OPENROUTER_API_KEY first}" @@ -346,51 +346,51 @@ uv run --locked --project benchmarks/harbor harbor run \ --job-name tb21-flash0731-pilot20-20260906 ``` -If credentials are already stored in local `~/.nanoPyCodeAgent/settings.json`, call `nanopycodeagent.settings.load_settings_env()` in the Python process launching Harbor, remove `ANTHROPIC_MODEL`, and start the same Harbor command. Relying on the CLI inside the task container to read configuration is insufficient: the host configuration file is not automatically mounted into the container, so host Harbor must first receive the connection environment variables. The actual run used this method, opening a right split in the current Herdr tab while preserving focus in the original pane. +If credentials are stored in `~/.nanoPyCodeAgent/settings.json`, call `nanopycodeagent.settings.load_settings_env()` in the Python process launching Harbor, remove `ANTHROPIC_MODEL`, and start the same command. Letting only the container CLI read settings is insufficient: host settings are not mounted automatically, so host Harbor must first receive the connection environment. This run used that method in a new right split in the current Herdr tab, preserving focus in the original pane. -Use a new `--job-name` for a rerun to distinguish it from existing experiment directories. For all 89 tasks, remove `--n-tasks 20` and choose a separate baseline job name while retaining the remaining conditions. This reruns the full dataset, with the small-scale trial cost counted separately. Analyze this run before executing the full dataset or increasing attempts per task. +Use a new `--job-name` for each rerun. To run all 89 tasks, remove `--n-tasks 20`, choose a separate baseline job name, and preserve the other conditions. This reruns the entire dataset, with pilot costs counted separately. Analyze this run before executing all tasks or increasing attempts. #### Budget and recording plan -The earlier certificate task cost 0.00222441 USD, but one simple task cannot represent the dataset's average cost. According to [OpenRouter model prices](https://openrouter.ai/deepseek/deepseek-v4-flash-0731) and [provider quotes](https://openrouter.ai/api/v1/models/deepseek/deepseek-v4-flash-0731/endpoints) checked on 2026-09-06, Baidu's uncached input/cache-read/output prices were 0.04998/0.009996/0.09996 USD per million tokens. DeepSeek's base prices were 0.22/0.007/0.66 USD. Routing, promotions, and time affect actual prices. +The certificate task cost 0.00222441 USD, but a simple task cannot represent the dataset's average. According to [OpenRouter model prices](https://openrouter.ai/deepseek/deepseek-v4-flash-0731) and [provider quotes](https://openrouter.ai/api/v1/models/deepseek/deepseek-v4-flash-0731/endpoints) checked on 2026-09-06, Baidu's uncached-input/cache-read/output prices were 0.04998/0.009996/0.09996 USD per million tokens. DeepSeek's base prices were 0.22/0.007/0.66 USD. Routing, promotions, and time affect actual quotes. -At one million input tokens and 30,000 output tokens per task with 80% input cache hits, those two price bands imply about 0.4–1.4 USD for 20 tasks. At three million input tokens and 150,000 output tokens per task, the range is about 1.4–5 USD. These are budget scenarios, not measured means or cost ceilings. A 50-turn limit is not a hard dollar cap either. The estimate covers model API charges only, excluding cloud containers, networking, and local execution costs. +At one million input and 30,000 output tokens per task, with 80% input cache hits, 20 tasks would cost about 0.4–1.4 USD at those two price levels. At three million input and 150,000 output tokens, the estimate becomes 1.4–5 USD. These are budget scenarios, not observed averages or cost ceilings. A 50-turn limit is not a dollar cap. These estimates cover only model API costs, excluding cloud containers, networking, and local execution. -Record and review this run in the following order: +Record and verify the run in this order: 1. Save `jobs//config.json`, `lock.json`, task content hashes, and the selected 20-task list. -2. Save each task's `result.json`, `agent/trajectory.json`, agent text log, and verifier log. -3. Count reward-1 tasks, reward-0 tasks, unscored tasks, and infrastructure exceptions separately. Report passes divided by planned tasks without silently reducing the denominator. -4. Check each trajectory's ATIF validity, usage and cost completeness, and Harbor metric population. Missing or partial costs remain unknown; a known subtotal must not be presented as a complete total. -5. Record total input/cache/output tokens, actual USD charges, job wall-clock duration, and per-task durations. Input already includes cached tokens; do not add them again. -6. Distinguish incorrect solutions, exhausted turns, execution timeouts, API errors, and installation/image/verifier environment failures. Preserve the first result and record every rerun separately. -7. Use measured trial consumption to adjust the full 89-task budget, then establish a single-run baseline. To assess variability later, perform additional independent repetitions and preserve every result instead of replacing the first attempt with the best rerun. +2. Preserve each `result.json`, `agent/trajectory.json`, agent text log, and verifier log. +3. Count reward-1 tasks, reward-0 tasks, unscored tasks, and infrastructure exceptions separately. Report passes against planned tasks without silently reducing the denominator. +4. Check ATIF validation, usage/cost completeness, and Harbor metric backfill for every trajectory. Missing or partial costs stay unknown; known subtotals are not complete totals. +5. Record total input/cache/output tokens, actual USD cost, job wall time, and per-task durations. Input already includes cache tokens; do not add them again. +6. Distinguish incorrect solutions, exhausted turns, execution timeouts, API errors, and installation/image/verifier failures. Preserve first results and create separate records for reruns. +7. Adjust the full 89-task budget using pilot consumption, then establish a single-attempt baseline. If later measuring variation, retain every repeated experiment instead of replacing first results with the best rerun. -`/jobs/` is Git-ignored. Archive raw results separately and retain a reviewable summary, per-task results, and artifact locations in the development notes. A local directory name alone does not place the baseline under version control. +`/jobs/` is Git-ignored. Archive raw results separately and preserve reviewable configuration, per-task summaries, and artifact locations in development notes. Recording only local directory names does not place a baseline under version control. -#### Actual results of the 20-task trial run +#### Results of the 20-task pilot -The job is `tb21-flash0731-pilot20-20260906`, and all 20 tasks have finished. The [machine-readable results](../../../benchmarks/harbor/results/tb21-flash0731-pilot20-20260906.json) retain per-task versions, metrics, termination reasons, and cost completeness. Individual billing receipts remain local. +All 20 tasks in `tb21-flash0731-pilot20-20260906` have finished. The [machine-readable results](../../../benchmarks/harbor/results/tb21-flash0731-pilot20-20260906.json) retain per-task versions, metrics, termination reasons, and cost completeness. Individual billing receipts remain local. -| Metric | Measured result | +| Metric | Observed result | | --- | --- | | Run date | 2026-09-06 | -| Job wall-clock duration | 111 minutes 10 seconds | +| Job wall time | 111 min 10 sec | | Passed / failed / unscored | 8 / 12 / 0 | -| Pass rate, denominator of 20 planned tasks | 8 / 20 = 40% | -| Passes without a Harbor exception | 7 / 20 = 35% | +| Pass rate, with 20 planned tasks as denominator | 8 / 20 = 40% | +| Passed without Harbor exceptions | 7 / 20 = 35% | | Harbor exceptions / automatic retries | 1 / 0 | | Model replies / tool calls | 267 / 329 | | Original Harbor input / cache / output tokens | 5,853,395 / 5,050,368 / 311,514 | -| Input / cache / output tokens reaggregated from final ATIF | 6,207,708 / 5,362,944 / 331,041 | -| Valid ATIF files | 20 / 20 | -| Original Harbor metrics matching final ATIF | 19 / 20 | -| Trajectories with incomplete costs before supplemental lookup | 13 / 20 | -| Original Harbor job cost aggregate | 0.045695378 USD | -| Known cost subtotal across all trajectories | 0.11053603 USD | -| Cost after independent supplemental lookup | 0.119200332 USD, complete | +| Input / cache / output tokens recalculated from final ATIF | 6,207,708 / 5,362,944 / 331,041 | +| ATIF files validated | 20 / 20 | +| Original Harbor metrics consistent | 19 / 20 | +| Trajectories with incomplete costs before independent lookup | 13 / 20 | +| Original Harbor job cost total | 0.045695378 USD | +| Known cost subtotal across trajectories | 0.11053603 USD | +| Cost after independent reconciliation | 0.119200332 USD, complete | -The table below uses model metrics from final ATIF files and independently reconciled costs. Original `result.json` and trajectory files were not rewritten. +The table below uses final ATIF model metrics and independently reconciled costs. Original `result.json` files and trajectories were not rewritten. | Task | Reward | Model replies | Last stop reason | Cost (USD) | | --- | ---: | ---: | --- | ---: | @@ -415,150 +415,225 @@ The table below uses model metrics from final ATIF files and independently recon | `circuit-fibsqrt` | 0.0 | 2 | `max_tokens` | 0.001051326 | | `merge-diff-arc-agi-task` | 1.0 | 14 | `end_turn` | 0.002268999 | -`pytorch-model-recovery` has both reward=1 and `AgentTimeoutError`. Preserve Harbor's original pass rate of **8/20 (40%)** and separately report **7/20 (35%) passes without a Harbor exception**, both using the planned 20 tasks as the denominator. This task is not a normally completed success sample. +`pytorch-model-recovery` has both reward 1 and `AgentTimeoutError`. Preserve Harbor's raw pass rate of **8/20 (40%)**, and separately report **7/20 (35%) passing without Harbor exceptions**, both using the planned 20 tasks. This task is not a normally completed success sample. -#### Problems exposed by this run +#### Problems exposed by the pilot -1. **The last reply in 11 failed tasks has `stop_reason=max_tokens`.** The current limit is 8192 tokens per response. Some replies spend most of it on thinking and finish before delivering a complete solution. The model loop in [`agent.py`](../../../src/nanopycodeagent/agent.py) treats every stop reason other than `tool_use` as completion, so these trajectories still have terminal status `completed`. Truncation needs explicit handling and the reasoning/output budget needs evaluation. Pass rate and cost under a higher limit require a separate experiment. -2. **`mteb-leaderboard` exhausts 50 model replies.** Its terminal outcome is `max_turns_exhausted`; the final reply still requests tool use, and `/app/result.txt`, required by the verifier, has not been created. This is a different budget limit from truncating one response. -3. **The agent continues after timeout and encounters a tool argument exception.** Relative to the start of Harbor's agent execution, `pytorch-model-recovery` reaches its limit around second 900 and the verifier starts around second 901. The internal journal records two more model calls starting around seconds 929 and 939. The final `edit` has only `old_text` and `new_text`, with no `path`. The stdout event subscriber directly accesses `arguments['path']`, raising `KeyError`; `run.failed` is persisted around second 1063. This exception occurs after timeout and cannot explain the earlier timeout. Agent execution overlaps verification, limiting how the reward can be interpreted. -4. **Harbor's original aggregate omits the timed-out task's metrics, and costs are delayed.** The trajectory is marked missing during timeout collection. A valid ATIF file later appears in the final directory and matches the independent backup, but Harbor tokens remain null. Original metrics therefore agree with final ATIF in only 19/20 trials. Costs are originally partial in 13 tasks, with 18 generations unresolved by native reconciliation. Independent queries against the same OpenRouter generation endpoint resolve all 18 receipts, so **all 267 recorded model replies have costs**. The original job's 0.045695378 USD is incomplete; the complete measured amount is **0.119200332 USD**, including both calls made after timeout. +1. **The final reply in 11 failed tasks has `max_tokens`.** The 8192-token response limit leaves some replies spending most of their budget on thinking before delivering a complete solution. At the time, the agent interpreted “no more tools requested” as the model finishing its work, without distinguishing replies interrupted by budget exhaustion. Those trajectories therefore still end with outcome `completed`. Explicit truncation handling and reasoning/output budget evaluation are needed; pass rates and costs at a higher limit require another experiment. +2. **`mteb-leaderboard` exhausts 50 replies.** Its outcome is `max_turns_exhausted`; the final reply still requests tools and the verifier-required `/app/result.txt` has not been created. This differs from truncating one response. +3. **The agent keeps running after timeout and encounters a tool-argument exception.** Measured from agent execution start, `pytorch-model-recovery` reaches its limit around second 900 and verification starts around second 901. Its journal records new model calls around seconds 929 and 939. The final `edit` supplies `old_text` and `new_text` but no `path`, causing a `KeyError` while displaying the tool call. The run failure is recorded around second 1063. This exception occurs after the timeout and cannot explain it. Execution overlaps verification, limiting how the reward can be interpreted. +4. **Harbor's original aggregate misses the timed-out task's metrics, and cost availability is delayed.** At timeout collection, its trajectory is marked missing. A valid ATIF file later appears and matches an independent backup, but Harbor tokens remain null. Only 19/20 original metrics match final ATIF. Thirteen trajectories initially have partial costs, with 18 generations unresolved by built-in reconciliation. Independent queries to the same OpenRouter endpoint retrieve all 18 receipts, obtaining costs for **all 267 recorded replies**. The original 0.045695378 USD job aggregate is incomplete; the full observed cost is **0.119200332 USD**, including both calls after timeout. -Complete input is 6,207,708 tokens, including 5,362,944 cache tokens; output is 331,041. Costs come from generation `total_cost`, not displayed-price estimates or the difference in the entire OpenRouter account balance. All 20 tasks have rewards and valid ATIF. Apart from the timeout above, Harbor reports no other trial exceptions. +Complete input is 6,207,708 tokens, including 5,362,944 cached tokens; output is 331,041. Costs come from generation `total_cost`, not advertised-price estimates or changes in the entire account balance. All 20 tasks have scores and valid ATIF. Apart from the timeout, Harbor records no other trial exceptions. -The original job is under `jobs/tb21-flash0731-pilot20-20260906/`. A local archive at `jobs/tb21-flash0731-pilot20-20260906-artifacts.tar.gz` contains the task list, logs, trajectories, generation-level costs, supplemental receipts, Docker image identifiers, and one-off execution and aggregation scripts. It has not been uploaded; its SHA-256 is in the machine-readable results. Two original `torch-pipeline-parallelism` logs contain runtime API credentials. Their permissions are 0600 and archived copies are redacted. Checksums for original and redacted copies remain in local records. +The original job is under `jobs/tb21-flash0731-pilot20-20260906/`. Its local archive, `jobs/tb21-flash0731-pilot20-20260906-artifacts.tar.gz`, contains the task list, logs, trajectories, per-generation costs, supplemental receipts, Docker image identifiers, and one-off execution and aggregation scripts. It has not been uploaded; its SHA-256 is in the machine-readable results. Two original `torch-pipeline-parallelism` logs contain runtime API credentials. Their permissions are 0600 and archived copies are redacted; checksums for both remain in local records. -Version control retains only compact experiment configuration, per-task scores, tokens, costs, durations, and exception summaries. Request IDs, internal run IDs, individual billing receipts, exact call times, host hardware details, and full conversations remain local. This minimization applies to the latest revision; earlier pushed commits still contain reconciliation metadata, and Git history has not been rewritten. +Version control retains only concise experimental configuration, per-task scores, tokens, cost, durations, and exception summaries. Request IDs, internal run IDs, individual billing receipts, precise call times, host hardware, and complete conversations remain local. This minimization applies to the latest version; earlier pushed commits retain reconciliation metadata, and Git history was not rewritten. -#### Next run and cost assessment +#### Next run and budget implications -First fix truncation handling, crashes on missing tool arguments, timeout termination, and trajectory collection, then improve delayed cost lookups. These fixes belong to a new revision. The agent source remained pinned throughout this run. Rerun the same 20 task hashes under a new job name and compare passes without exceptions, termination reasons, tokens, and cost. Preserve this run as the small-scale baseline before fixes. Once execution and accounting are reliable, run all 89 tasks and record that result separately as the full-dataset baseline. +First fix truncation handling, missing-tool-argument crashes, timeout termination, and trajectory collection, then improve delayed cost lookup. These fixes belong to a new revision; the agent source remained pinned throughout this run. Rerun the same 20 task hashes under a new job name, comparing passes without exceptions, termination reasons, tokens, and costs. Preserve this pilot as the pre-fix baseline. Once execution and accounting are reliable, run all 89 tasks and record that result as the full-dataset baseline. -This run costs about 0.12 USD. Multiplying by 89/20 gives about 0.53 USD, but 11 tasks truncate early and the sample consists of the first 20 list entries. That extrapolation cannot budget the full dataset after fixes. Under the earlier scenario of three million input tokens, 150,000 output tokens, and 80% input cache hits per task, 89 tasks cost about 6–22 USD. Adjust the estimate to provider prices at the time and consumption in the next trial run. Only 20 tasks have been executed; the full 89-task run is still pending. +This pilot costs about 0.12 USD. Multiplying mechanically by 89/20 gives about 0.53 USD, but 11 tasks truncate early and the sample is the first 20 list entries, so this cannot budget the full dataset after fixes. The earlier scenario of three million input tokens, 150,000 output tokens, and 80% input cache hits per task implies about 6–22 USD for 89 tasks. Adjust this to provider prices at the time and consumption in later pilots. Only 20 tasks have run; the full 89-task experiment remains pending. -### Addressing problems encountered in the Terminal-Bench batch +### Fixing problems found in batch Terminal-Bench runs -The explanations below record what the problems mean, the current implementation, and proposed fixes. They do not indicate that the corresponding code has already been fixed. +The following notes explain the problems, implementations, and proposed fixes. The first truncation-handling fix is implemented; the remaining sections describe problems and proposals, not completed code fixes. #### Truncation handling -Each model reply is currently limited to 8192 tokens. If the reply exhausts this budget before finishing, the API returns `stop_reason="max_tokens"`, indicating that generation was forced to stop at its length limit. For example, the model may still be analyzing how to modify code when the budget runs out, before generating the subsequent tool call or complete solution. Some replies in this run spent most of their budget on thinking; the final replies in 11 failed tasks all had `max_tokens`. +Each model reply currently has an 8192-token generation limit. If the model exhausts this budget before finishing, the API returns `stop_reason="max_tokens"`, indicating a forced stop at the generation limit. The model may still be analyzing a code change before producing a tool call or complete solution. Some pilot replies spent most of their budget on thinking; all 11 affected failed tasks ended with `max_tokens`. -Currently, `_run_model_loop()` in [`agent.py`](../../../src/nanopycodeagent/agent.py) decides whether the model has ended normally using only this condition: +Previously, the agent interpreted “no more tools requested” as the model finishing its work. However, generation-budget exhaustion can also stop a reply before it produces a tool call. As a result, unfinished tasks were recorded as `completed`. Evaluating a run requires distinguishing a normal model finish, exhaustion of the task's turn budget, and truncation of an individual reply: each explains a different stopping reason. Even a normal model finish leaves correctness for the verifier to establish. -```python -if message.stop_reason != "tool_use": - return True +This change first makes the reported result trustworthy: recognize truncation, preserve evidence, and explicitly say that the task remains unfinished. Stop the run after truncation. Larger generation budgets and automatic recovery are follow-up capabilities whose effectiveness and additional consumption will be evaluated separately. + +**Implemented on 2026-09-07: recognize truncation and stop.** + +Record the stopping reason as `response_truncated`, even on the final allowed turn, so users know that the last reply itself did not finish generating. The agent makes no further model calls and skips every tool call in that reply, including calls with apparently complete arguments, to avoid acting on an unfinished response. + +Generated text, thinking, tool arguments, and usage remain in the execution record, and cost reconciliation still runs. This allows analysis of where the budget went and where progress stopped. Unexecuted tools are not recorded as executed and have no fabricated results. Analysis must distinguish whether a run ended normally from whether its task was completed. + +Users see an explicit truncation diagnostic, and text already output remains visible. Headless still exits 0, leaving the verifier to judge task success. Interactive mode returns to the input prompt so the user can send another message. Subsequent conversation retains generated text and a truncation notice while removing unexecuted tool requests and incomplete thinking, avoiding invalid history in the next request. The original reply remains in the execution record for later inspection. + +The new stopping reason exceeds the old record protocol's allowed values, so new records use Journal v2. Existing v1 records remain readable and convertible, and public trajectories stay at ATIF-v1.7. A model reaching its generation limit is independent of logs shortening content to control storage size. See [Event Journal v2](../../dev_docs/en/event-journal-protocol-v2.md) for fields, event ordering, and compatibility rules. + +Regression tests cover text, thinking-only and empty replies, truncation on the last turn, skipped tools, the next interactive turn, and usage/cost preservation. A locally simulated SSE response exercises partial tool JSON through the real Anthropic SDK. Old v1 and new v2 truncated trajectories pass Harbor's pinned ATIF validator. These tests do not call paid models. At implementation time on 2026-09-07, no benchmark had been rerun and no pass-rate improvement was inferred from the automated tests. The real-model rerun on 2026-09-09 follows. + +Select `regex-log` and `write-compressor` from `tb21-flash0731-pilot20-20260906`. Both previously scored 0 and ended with `max_tokens`, while their old trajectories recorded outcome `completed`. This rerun checks both official verifier scores and accurate truncation reporting on the current branch. + +**Method and configuration.** Open a right split in the current Herdr tab, preserve focus in the original pane, and run from the repository root. Execute the two tasks concurrently with one attempt each, preserving the original job. + +| Item | Rerun configuration | +| --- | --- | +| Agent branch / commit | `fix/handle-truncated-model-responses` / `f7788616e11f3347300ac2fdef6c6400d4eb5611` | +| Installed container package | `0.8.1.dev11+gf7788616e` | +| Harbor | 0.21.0, run through `uv run --locked --project benchmarks/harbor` | +| Dataset | `terminal-bench/terminal-bench-2-1@sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a` | +| Model / API endpoint | `openrouter/deepseek/deepseek-v4-flash-0731` / `https://openrouter.ai/api` | +| Generation limits | 8192 tokens per reply; at most 50 model replies per task; no explicit reasoning effort or temperature | +| Environment and retries | Local Docker, concurrency 2, one attempt per task, Harbor `max_retries=0`; task-default resources and timeouts | +| Job | `tb21-truncation-rerun2-20260909-f778861` | + +Reuse the original task versions and verify identical task identities in the old and new trial configurations: + +| Task | Task hash | +| --- | --- | +| `terminal-bench/regex-log` | `sha256:802c16cfd132e6c457529cb864be5a757c1b23b6cadc57f2d01983cb0110292a` | +| `terminal-bench/write-compressor` | `sha256:d9ddd9a8e925e2c566b37b2492cbf995afecefe58874e4043ef78d7f3c892c7e` | + +The branch had not been pushed, so containers could not install its remote Git revision. Instead, run `uv build --wheel`, compare every packaged Python source file with the commit, and upload the wheel into each container's `/tmp/`. A temporary test adapter changes installation to use this local wheel. Model calls, tools, and trajectory export still use the current branch. The wheel SHA-256 is `3c4ebd9268680ba0798ccbc44e6e56ee91424c5773399171050b7ac24734e9e4`. + +The launcher reads host connection settings, removes the `ANTHROPIC_MODEL` override, and checks that the endpoint is OpenRouter. Credentials pass through the environment without appearing in command arguments. Task filters require the `terminal-bench/` prefix: + +```bash +--include-task-name terminal-bench/regex-log \ +--include-task-name terminal-bench/write-compressor ``` -As a result, `max_tokens` also causes the agent to leave the loop and record `completed` in the trajectory. This conflates an interrupted reply with a normal model finish. `completed` itself does not establish that the solution is correct; the verifier still decides whether the task passes. +The actual local entry point was: + +```bash +uv run --locked --project benchmarks/harbor python \ + jobs/tb21-truncation-rerun2-20260909-f778861-record/workflow/run.py +``` -At minimum, the fix must recognize `max_tokens` separately and accurately record truncation and the stopping reason. Further strategies could attempt to resume generation within the available budget or explicitly record truncation and stop. The recovery method still needs design; unlimited continuation or retries should not be the default. +The script prepares the wheel, records a manifest, and invokes Harbor. It and the temporary adapter are local experiment artifacts, not a committed general-purpose benchmark command. -The value 8192 is currently hardcoded as a module constant in `agent.py`: +Related automated regressions also ran that day: all 54 core tests and 16 Harbor adapter tests passed: -```python -MAX_TOKENS = 8192 +```bash +.venv/bin/python -m pytest -q \ + tests/test_truncation.py tests/test_event_journal.py tests/test_atif.py +benchmarks/harbor/.venv/bin/python -m pytest -q \ + -c benchmarks/harbor/pyproject.toml benchmarks/harbor/tests ``` -There is currently no environment variable or CLI option for adjusting it. +**Results.** Both tasks completed verification without Harbor exceptions. The pass rate was **0/2**, and Harbor reported about **9 min 42 sec** of job wall time. Both truncations were correctly recorded, but neither task was completed. -**Even after increasing this value, `stop_reason="max_tokens"` still needs handling.** The two changes serve different purposes: +| Task | Previous / new reward | Model replies | Final reply tokens | Last stop reason | New terminal outcome | Verifier failure | +| --- | --- | ---: | ---: | --- | --- | --- | +| `regex-log` | 0.0 / 0.0 | 1 | 8192 | `max_tokens` | `response_truncated` | `/app/regex.txt` was not created; 1 test failed | +| `write-compressor` | 0.0 / 0.0 | 2 | 8192 | `max_tokens` | `response_truncated` | `/app/data.comp` was not created; 3 tests failed | -| Change | Purpose | -| --- | --- | -| Increase `MAX_TOKENS` | Give each reply more room to generate, reducing truncation caused by insufficient budget | -| Handle `stop_reason="max_tokens"` | Recognize truncation, decide whether to recover or stop, and accurately record the reason | +| Task | Input tokens | Output tokens | Cost (USD) | +| --- | ---: | ---: | ---: | +| `regex-log` | 1678 | 8192 | 0.00158363 | +| `write-compressor` | 3927 | 8303 | 0.001742255 | +| Total | 5605 | 16495 | 0.003325885 | + +Costs come from actual accounting in the final trajectories and are complete for both tasks. Both ATIF files pass Harbor validation, startup versions contain `f778861`, both logs include explicit truncation diagnostics, and no model call follows the final reply. Neither final reply contains tool calls; skipping partial tool arguments and maintaining valid follow-up history are covered by the automated tests but were not triggered by these two real runs. Harbor's successful exit 0 does not establish task success; verifier rewards remain authoritative. + +This rerun verifies recognition of truncation followed by stopping. It neither increases the response limit nor resumes generation. Both tasks still exhaust their per-response budgets before delivering the required files. These results do not establish pass rates with larger budgets or automatic continuation; those need separate experiments. + +Raw trial logs, trajectories, and verifier output are under `jobs/tb21-truncation-rerun2-20260909-f778861/`. The corresponding `jobs/tb21-truncation-rerun2-20260909-f778861-record/` contains `manifest.json`, `summary.json`, `results.en.md`, the wheel, and run, temporary-installation, and summary scripts under `workflow/`. These directories are Git-ignored and have not been uploaded. This document preserves versions, methods, and results without treating local artifacts as version-controlled files. Future reruns must use new job names and preserve this original 0/2 result. + +#### New requirement: increase the model generation budget (pending) -Any finite limit can be exhausted. Increasing the number alone leaves the incorrect completion classification in place. The proposed order is to implement truncation handling first, then use small-scale trial runs to evaluate whether to raise the limit, by how much, and how to allocate reasoning and output budgets. Truncation in 11 failed tasks does not mean that a higher limit will make all 11 pass. Pass rate, token consumption, and cost require a new experiment. +Added on 2026-09-09. Here, budget primarily means the **token generation limit for one model reply**. The limit is currently fixed at 8192 and cannot be adjusted through environment variables or CLI arguments. This requirement is to raise the default to a value selected through experiments and provide configuration so normal use and Harbor experiments can adjust and record the effective budget. The new default, option names, and configuration precedence remain to be designed. Also evaluate how the budget is divided between thinking and final output. + +**Handling `stop_reason="max_tokens"` must remain even with a higher limit.** Any finite budget can be exhausted. Raising it gives the model more generation room; existing truncation handling records the stopping reason accurately. Both rerun tasks exhaust 8192 tokens, providing a comparison point for larger budgets without guaranteeing they will pass. + +For acceptance, preserve the model, task hashes, and `max_turns=50`, disable automatic continuation, and change only the per-response budget. Verify that the configured value reaches the API request and record the effective value. Rerun these two tasks under a new job, comparing rewards, truncations, input/output tokens, costs, and durations. If truncation occurs again, retain an accurate outcome, usage, and costs without classifying an unfinished reply as task completion. + +#### New requirement: automatically continue after truncation (pending) + +Added on 2026-09-09. The current run ends on `max_tokens` even if model turns remain. The new behavior is to initiate further model calls automatically when configuration permits and sufficient budget remains, continuing the original task so headless execution does not depend on a user manually sending “continue.” + +Expected behavior and constraints: + +- Provide an enable/disable setting and a finite continuation limit. Each additional reply counts against the same run's `max_turns`; recovery must not reset the turn counter. Default enablement, continuation count, and other budget limits remain to be designed. +- Preserve successful tool results and task context while constructing valid request history. Continue skipping tools from truncated replies. Do not replay partial arguments or tool calls without matching results. Thinking-only replies without visible text also need a defined recovery method; continuation prompts and request construction remain to be designed. +- Preserve every `max_tokens` fact in the Journal/ATIF and record subsequent attempts, all usage, and all costs. Do not emit a run terminal event while continuation is still underway. Accurately record normal completion, continuation-limit exhaustion, turn exhaustion, or request failure. Terminal rules and protocol representation for recovery need to be designed together. +- Stop at configured limits, cancellation, or unrecoverable errors with explicit diagnostics, avoiding endless continuation. Continued execution still relies on the verifier to establish correctness. + +Automated acceptance must cover continuation to completion, repeated truncation reaching its limit, exhausted remaining turns, thinking-only replies, and truncated tool arguments neither executing nor being replayed incorrectly. For real-model acceptance, use these two tasks and new job names to compare continuation disabled/enabled at the same response budget, then evaluate larger budgets together with continuation. Report pass rate, additional model calls, total tokens, cost, and duration separately. Both requirements remain unimplemented; this 0/2 experiment is not their acceptance result. #### Crashes caused by missing tool arguments -The model omitted a required tool argument. Instead of returning a tool-call error to the model, the agent raised a Python exception and failed the entire run. In this trial, the failure occurred in the final `edit` call of `pytorch-model-recovery`. +When a model omits a required tool argument, the agent raises a Python exception and fails the entire run instead of returning a tool error the model can correct. In this pilot, the final `edit` call in `pytorch-model-recovery` caused this failure. -The [`edit` tool definition](../../../src/nanopycodeagent/edit_tool.py) requires three arguments: +The [`edit` definition](../../../src/nanopycodeagent/edit_tool.py) requires: - `path`: the file to modify. -- `old_text`: the original text to replace. -- `new_text`: the replacement text. +- `old_text`: the text to replace. +- `new_text`: its replacement. -This call supplied only `old_text` and `new_text`, omitting `path`. It described the replacement without identifying the file. Although the tool schema already declares `path` as required, the actual input was missing it, so the program needs to check arguments before using them. +The call supplied only `old_text` and `new_text`, describing a replacement without identifying the file. The schema declares `path` required, but actual arguments still need validation before use. -Before executing a tool, the agent emits `tool.started`, and `_TextOutputProjector` prints a preview. The `edit` preview directly reads `arguments['path']`; because that key is absent, it raises `KeyError: 'path'`. The failure occurred while printing the preview, before any file modification. It happened after the Harbor timeout and cannot explain the earlier timeout. +This error occurred while displaying the tool call, before any file edit. The missing target path caused even the display of the intended operation to raise `KeyError: 'path'`, ending the entire task. This happened after Harbor's timeout and cannot explain the preceding timeout. -Both preview and execution need protection. The preview must tolerate missing arguments; execution must validate them and return a clear `tool_result` with `is_error=True` instead of executing an invalid call. For example: +Both display and execution need coverage: the display must tolerate missing arguments, and execution must validate required fields. A missing field should prevent that tool from running and return a clear `tool_result` with `is_error=True`, for example: ```text Missing required argument: path. Provide the file path and retry. ``` -The model can then see the error in a later turn, supply the missing argument, and continue within the remaining budget. Fixing the preview alone is insufficient: `_run_one_tool()` still directly reads `block.input["path"]` in its execution branch, and its exception handler records the failure and re-raises it, failing the run. Missing required arguments should be recoverable tool-call errors returned to the model. Other tools should have the same argument-checking and error-return paths. +The model can then supply the missing value in a later turn and continue within the remaining budget. Both display and execution must tolerate incomplete input; otherwise the crash is merely deferred to the next stage. The principle is to return model-correctable argument errors as tool results so the task can continue, rather than ending the entire task. Other tools should follow the same principle. -#### Timeout termination +#### Stopping execution on timeout -Timeout termination means ensuring that the agent inside the container actually stops working when the task reaches its allotted runtime, before verification begins. In this run, Harbor had already declared a timeout while the agent continued making model calls. +Timeout handling must ensure that the agent inside the container actually stops before verification begins. In this pilot, Harbor had already classified the task as timed out while the agent continued calling the model. -This occurred in `pytorch-model-recovery`, whose agent runtime limit was 900 seconds, or 15 minutes. Relative to the start of agent execution, the timeline was: +The affected task, `pytorch-model-recovery`, has a 900-second (15-minute) agent limit. Relative to agent execution start: -| Time | What happened | +| Time | Observed event | | --- | --- | -| Around second 900 | Harbor declared a timeout and recorded `AgentTimeoutError` | -| Around second 901 | The verifier started checking the task result | -| Around seconds 929 and 939 | The agent started two more model calls | -| Around second 1063 | The agent failed because `edit` lacked `path`, recording `run.failed` | +| About 900 sec | Harbor records `AgentTimeoutError` | +| About 901 sec | The verifier begins | +| About 929 and 939 sec | The agent starts two more model calls | +| About 1063 sec | Missing `path` in `edit` causes failure and `run.failed` | -Harbor's timeout decision did not promptly stop the container agent, with two consequences: +Harbor's timeout did not promptly stop container execution, with two consequences: -- The agent remained active during verification and could continue changing files being checked by the verifier, affecting the reliability of the result. Although the task received `reward=1`, it cannot be treated as a normally completed success sample. -- New model calls and charges occurred after the deadline, so the task's time boundary was not enforced. +- During verification, the agent could continue modifying files under examination, weakening result reliability. Reward 1 therefore does not make this a normally completed success sample. +- New model calls and costs occurred after the deadline, so the time boundary was not enforced. -The proposed fix should: +Proposed goals: -1. Stop agent execution at the deadline and prevent new model and tool calls. -2. Clean up tool child processes that need termination so commands cannot continue modifying files after the agent exits. -3. Confirm that execution has stopped before starting the verifier, and accurately record the timeout state. +1. Stop agent execution at the deadline and prevent further model or tool calls. +2. Terminate tool subprocesses that need cleanup so commands cannot keep modifying files after the agent exits. +3. Confirm execution has stopped, then start verification and record timeout state accurately. -The confirmed observation is that the agent continued after timeout. The specific layer where cancellation or process cleanup failed still needs investigation. The purpose is to enforce the time limit; simply extending the 900-second allowance does not solve continued execution after timeout. +Continued execution after timeout is confirmed. Which cancellation or cleanup layer failed remains to be investigated. The goal is to enforce the time limit; extending the 900-second allowance would not fix execution continuing after timeout. #### Trajectory collection -Trajectory collection means saving the agent's structured execution record and making it available to Harbor in time to aggregate steps, tokens, and costs. The issue in this run was that the file appeared later, after Harbor had already tried to collect it for aggregation. - -Normally, the agent continuously appends model replies, tool calls, execution results, and usage to its internal Event Journal. During finalization, `_run_exchange()` in [`agent.py`](../../../src/nanopycodeagent/agent.py) projects the journal into a public ATIF `trajectory.json`. The [Harbor adapter](../../../benchmarks/harbor/src/harbor_adapter/adapter.py) reads and validates the file in `populate_context_post_run()`, records diagnostics such as step counts, and populates tokens and costs in the evaluation result. +Trajectory collection must preserve the structured execution record and make it available to Harbor in time to aggregate steps, tokens, and costs. In this pilot, the file eventually existed but was unavailable when Harbor collected metrics. -Collection for `pytorch-model-recovery` encountered a timing problem: +Normally, the agent continuously records model replies, tool calls, results, and usage, then exports those records as ATIF `trajectory.json` when execution ends. Harbor subsequently reads and validates the trajectory to aggregate steps, tokens, and costs. This creates three stages that must finish in order: the agent stops, the trajectory is saved, and the evaluator collects it. -1. Harbor began collection after declaring timeout, could not obtain the trajectory, and recorded it as `missing`. -2. The agent continued running and wrote a valid ATIF file during its later finalization. -3. The file eventually appeared in the result directory, but tokens in Harbor's original result remained `null`; the file's arrival did not automatically populate them. +For `pytorch-model-recovery`, collection had a timing problem: -Thus, all 20/20 final ATIF files passed validation, while original Harbor metrics matched final ATIF for only 19/20 tasks. A file's eventual existence and validity do not establish that it was collected when evaluation metrics were aggregated. +1. Harbor began collection after timeout, found no trajectory, and recorded it as `missing`. +2. The agent continued running and only wrote valid ATIF during later finalization. +3. The file eventually appeared in the result directory, but original Harbor token metrics remained `null` and were not backfilled automatically. -The proposed fix should preserve as much execution history as possible through the stopping point for normal completion, exceptions, and timeouts, and make it available to Harbor before aggregation. Incomplete records should be explicitly identified. +All 20 final ATIF files therefore validate, but only 19/20 original Harbor metrics match them. A file's eventual existence and validity do not establish successful collection at aggregation time. -This needs to be designed together with timeout termination: stop execution, complete finalization within a bounded time, then collect and aggregate. ATIF writing currently depends on `finally` during finalization. Forcibly killing the process does not guarantee that this code can run. Options such as periodic snapshots or recovering a trajectory from the persisted journal therefore need consideration. The specific solution is undecided; the objective is to keep actual execution records, saved trajectories, and Harbor aggregates consistent. +The proposed goal is to preserve as much execution history as possible up to stopping, whether the run completes, fails, or times out, and let Harbor read it before aggregation. Incomplete records must be labeled explicitly. -#### Improving delayed cost lookups +Design this together with timeout termination: stop execution, finalize within a bounded time, then collect and aggregate. Trajectory export currently depends on the agent completing finalization. Forcibly terminating the process may leave no time to save it. Periodic snapshots or reconstruction from persisted journals therefore need consideration. The specific approach is undecided; the objective is consistency among actual execution, saved trajectories, and Harbor metrics. -Improving delayed cost lookups means being able to query model-call costs that remain unavailable when a task ends and complete the accounting later. +#### Improving delayed cost reconciliation -A model reply may already have arrived without an actual cost, while the billing endpoint still cannot provide an amount. The agent saves that call's generation ID to associate a later billing lookup with the call and query its `total_cost`. A temporarily unavailable charge must remain pending or unknown, not be treated as free. +Costs unavailable at task completion should remain queryable later so accounting can be completed. -The existing `_reconcile_costs()` in [`agent.py`](../../../src/nanopycodeagent/agent.py) automatically checks calls with missing costs and known generation IDs one by one during finalization. By default, `resolve_generation_cost()` in [`cost.py`](../../../src/nanopycodeagent/cost.py) makes up to six attempts per generation, with retry delays of `1, 2, 4, 8, 15` seconds. Those delays total 30 seconds, in addition to the time spent on requests themselves, so 30 seconds is not a limit on the entire lookup process. Exhausting the attempts leaves costs incomplete. This finalization lookup is already a supported agent capability; there is no supported independent entry point for subsequent lookups yet. +A model response may lack actual cost, and the billing endpoint may not yet return an amount. Save the call's generation ID as the association key and query its `total_cost` later. Temporary unavailability means pending or unknown cost, not free execution. -In this run, 13 tasks had incomplete costs in their original trajectories, with 18 calls unresolved during finalization. During and after the batch on 2026-09-06, an independent script queried the same cost endpoint in batches and eventually resolved all 18 charges. +During finalization, the agent already queries calls with generation IDs and missing costs individually. Each lookup defaults to six attempts, separated by `1, 2, 4, 8, 15` seconds. These delays total 30 seconds, plus request durations; 30 seconds is not a wall-clock cap on the complete reconciliation process. Exhausting the attempts preserves an incomplete-cost state. Finalization reconciliation is already a supported agent feature, while an independent later-reconciliation interface is not yet provided. -**Those independent lookups used a temporary script that is not part of the regular workflow.** The local script is `jobs/tb21-flash0731-pilot20-20260906-record/workflow/reconcile.py`. It lives under Git-ignored `jobs/` and is archived as a one-off workflow record for this trial. It must be run separately and is not integrated into the supported CLI, Harbor adapter, or a standard evaluation post-processing step. Obtaining complete costs through this script does not mean that the agent already performs subsequent lookups automatically. +In the pilot, 13 original trajectories had incomplete costs, with 18 calls unresolved during finalization. During and after the 2026-09-06 batch run, an independent script queried the same endpoint in batches and eventually obtained all 18 costs. -Costs before and after the supplemental lookups were: +**Those independent lookups used a temporary script that is not part of the supported workflow.** The script is `jobs/tb21-flash0731-pilot20-20260906-record/workflow/reconcile.py`, under Git-ignored `jobs/`, archived as a one-off workflow record. It must be run separately and is not integrated into the official CLI, Harbor adapter, or a standard evaluation post-processing stage. Completing costs through that script does not mean the agent automatically performs later lookups. | Accounting basis | Cost (USD) | | --- | ---: | | Known cost subtotal in final trajectories | 0.11053603 | -| Complete cost after independent lookups | 0.119200332 | +| Complete cost after independent reconciliation | 0.119200332 | -The reconciled amount covers all 267 recorded model replies. Bounded attempts during finalization can leave charges requiring later lookup; the subtotal known at that point must not be treated as the complete total. +The reconciled total covers all 267 recorded model replies. This shows that bounded finalization attempts can leave costs requiring later lookup; the known subtotal at that time is not a complete total. -The proposed direction is to turn the temporary subsequent lookup work into a reusable supported capability that can integrate into the evaluation workflow and run independently after a task ends. Read generation IDs with unresolved costs, query them later, save receipts, and generate updated cost aggregates. Preserve original results, avoid counting the same charge twice, and explicitly distinguish known subtotals from complete totals. Unavailable amounts stay unknown rather than zero, and lookup failures must not change task scores. +The proposed direction is to turn this one-off follow-up into a reusable supported capability that integrates with evaluation and can run independently after tasks finish. Read unresolved generation IDs, query later, preserve receipts, and produce updated aggregates. Preserve original results, avoid counting any charge twice, and distinguish known subtotals from complete totals. Unresolved amounts remain unknown rather than zero, and reconciliation failures must not change task scores. -Whether this becomes an independent command or an automatic step after batch evaluation remains undecided. The objective is to allow lookups to continue beyond task finalization without delaying agent exit for a long time while waiting for billing records. +Whether this becomes a separate command or an automatic post-benchmark stage is undecided. The goal is to continue reconciliation beyond task finalization without delaying agent exit indefinitely while waiting for billing records. diff --git a/docs/dev_notes/zh-CN/0.8.x.md b/docs/dev_notes/zh-CN/0.8.x.md index 6cb6e04..853b2a9 100644 --- a/docs/dev_notes/zh-CN/0.8.x.md +++ b/docs/dev_notes/zh-CN/0.8.x.md @@ -457,8 +457,8 @@ Baidu 当时的未缓存输入／缓存读取／输出价格分别为每百万 t 1. **11 道未通过题的末次回复均为 `max_tokens`。** 当前每次回复最多 8192 tokens， 部分回复的预算主要消耗在 thinking，尚未交付完整解答就结束。 - [`agent.py`](../../../src/nanopycodeagent/agent.py) 的模型循环把所有非 `tool_use` - 的 stop reason 都当作完成，所以这些轨迹的终态仍为 `completed`。这说明需要显式 + 当时 agent 将“不再请求工具”视为模型完成处理，未区分因预算耗尽而中断的回复， + 所以这些轨迹的终态仍为 `completed`。这说明需要显式 处理截断，并评估 reasoning／output 预算；增加上限后的通过率和费用需要另行实测。 2. **`mteb-leaderboard` 用尽 50 次模型回复。** 终态 outcome 为 `max_turns_exhausted`，末次回复仍要求调用工具，verifier 所需的 `/app/result.txt` @@ -466,8 +466,8 @@ Baidu 当时的未缓存输入／缓存读取／输出价格分别为每百万 t 3. **超时后 agent 仍在运行，且发生工具参数异常。** 以 Harbor 开始执行 agent 为 起点，`pytorch-model-recovery` 在约第 900 秒达到限制，verifier 约第 901 秒开始； 内部 journal 记录 agent 在约第 929 秒和第 939 秒又启动两次模型调用。最后一次 - `edit` 只有 `old_text`、`new_text`，缺少 `path`，stdout 事件订阅者直接读取 - `arguments['path']`，触发 `KeyError`；`run.failed` 约在第 1063 秒落盘。 + `edit` 只有 `old_text`、`new_text`，缺少 `path`，在显示工具调用信息时触发 + `KeyError`；运行失败记录约在第 1063 秒落盘。 因而该异常发生在超时之后，不能用它解释之前的超时。运行与验证发生重叠，该题的 reward 需要带着这一限制解读。 4. **Harbor 原始汇总漏掉超时题的计量，费用也有延迟。** 超时采集时 trajectory @@ -508,51 +508,180 @@ API 凭据，原件权限已设为 0600，归档副本已脱敏；原件与脱 ### 修复批量运行 Terminal-Bench 时遇到的问题 -以下问答记录本轮问题的含义、当前实现与拟议修复方向，尚不表示对应代码已修复。 +以下记录本轮问题的含义、实现与修复方向。“截断处理”已完成第一步修复；其余小节 +仍描述问题与拟议方向，不表示对应代码已修复。 #### 截断处理 - 当前每次模型回复最多生成 8192 tokens。模型尚未输出完就用尽这次预算时，接口 返回 `stop_reason="max_tokens"`，表示达到生成长度上限，被迫停止。例如，模型 可能仍在分析如何修改代码，后面的工具调用或完整解答还没有生成，预算就耗尽了。 本轮部分回复的预算主要消耗在 thinking；11 道未通过题的末次回复均为 `max_tokens`。 -当前 [`agent.py`](../../../src/nanopycodeagent/agent.py) 的 `_run_model_loop()` -只用以下判断决定模型是否正常结束： +过去 agent 将“不再请求工具”视为模型完成处理，但生成预算耗尽也会让回复停在 +没有工具调用的位置。结果是任务尚未做完，轨迹却将它记为 `completed`。判断运行 +结果时，需要区分模型正常结束、任务轮数用尽和单次回复被截断；它们说明的是不同 +的停止原因。即使模型正常结束，解答是否正确仍要由 verifier 判断。 -```python -if message.stop_reason != "tool_use": - return True -``` +本次先让运行结果可信：识别截断、保留证据，并明确告知任务尚未完成。遇到截断后 +停止本次运行；增加生成预算和自动恢复作为后续能力，分别评估效果与额外消耗。 -因此，`max_tokens` 也会使 agent 退出循环，并在 trajectory 中记录 `completed`。 -这里混淆了“回复被迫中断”和“模型正常结束”。`completed` 本身也不代表题目做对了， -题目是否通过仍由 verifier 判断。 +**2026-09-07 实现：识别截断并停止。** -修复的最低要求是单独识别 `max_tokens`，准确记录截断事实和停止原因。进一步的 -策略可以是在预算允许时尝试恢复生成，或者明确记录截断后停止；具体采用何种恢复 -方式仍需设计，不能把无限续写或重试当成默认方案。 +截断后的停止原因记录为 `response_truncated`。即使它恰好发生在最后一轮，也保留 +这个原因，让使用者知道最后一次回复本身没有生成完。agent 不再发起新的模型调用， +并跳过该回复中的全部工具调用，包括参数看似完整的调用，避免根据未完成的回复 +执行操作。 -8192 目前直接写在 `agent.py` 的模块常量中： +已经生成的文本、思考内容、工具参数和用量仍保存在执行记录中，费用继续补查， +便于分析预算花在哪里、任务停在哪一步。未执行的工具不会被记成已执行，也不会 +产生虚构的结果。分析结果时，应把“运行是否正常结束”和“任务是否完成”分开判断。 -```python -MAX_TOKENS = 8192 -``` +使用者会看到明确的截断提示，已输出文字保持可见。Headless 仍以 `0` 退出，由 +verifier 判断任务是否通过。交互模式回到输入提示，用户可以继续发消息；后续对话 +保留已生成文字和截断说明，移除未执行的工具请求与不完整的思考内容，避免下次 +请求带入无法衔接的历史。原始回复仍保留在执行记录中供追溯。 + +新增停止原因超出了旧记录协议允许的范围，因此新记录升级为 Journal v2；已有 v1 +记录仍可读取和转换，公开轨迹继续使用 ATIF-v1.7。模型生成到达上限与日志为控制 +体积而缩短内容是两件独立的事。具体字段、事件顺序和兼容规则见 +[Event Journal v2](../../dev_docs/zh-CN/event-journal-protocol-v2.md)。 + +回归测试覆盖文本、仅 thinking、空回复、最后一轮截断、工具跳过、交互下一轮、 +usage 与费用保留，并通过真实 Anthropic SDK 的本地模拟 SSE 验证半截工具 JSON。 +旧 v1 与新 v2 截断轨迹均通过 Harbor 固定版本的 ATIF validator。测试没有调用付费 +模型；2026-09-07 实现时尚未重跑 benchmark，也没有将这些自动化测试解释为通过率 +提升。2026-09-09 的真实模型复跑记录如下。 -当前没有环境变量或 CLI 参数可以调整它。 +从 `tb21-flash0731-pilot20-20260906` 中选择 `regex-log` 和 `write-compressor`。 +两题上次 reward 都为 0，末次回复都是 `max_tokens`，但旧轨迹将 outcome 记录为 +`completed`。本次同时检查官方 verifier 的评分，以及当前分支是否准确记录截断。 -**即使增大这个值，也必须处理 `stop_reason="max_tokens"`。** 两项工作解决的问题不同： +**测试方法与配置。** 在当前 Herdr tab 中向右新建 split，保留原 pane 的焦点， +从仓库根目录运行。两题并发执行，每题只尝试一次，原始 job 保持不变。 -| 工作 | 作用 | +| 项目 | 本次配置 | | --- | --- | -| 增大 `MAX_TOKENS` | 给一次回复更多生成空间，减少因预算不足而截断的情况 | -| 处理 `stop_reason="max_tokens"` | 发生截断时，正确识别、决定恢复或停止，并准确记录原因 | +| Agent 分支／提交 | `fix/handle-truncated-model-responses` ／ `f7788616e11f3347300ac2fdef6c6400d4eb5611` | +| 容器中的包版本 | `0.8.1.dev11+gf7788616e` | +| Harbor | 0.21.0，通过 `uv run --locked --project benchmarks/harbor` 运行 | +| Dataset | `terminal-bench/terminal-bench-2-1@sha256:7d7bdc1cbedad549fc1140404bd4dc45e5fd0ea7c4186773687d177ad3a0699a` | +| Model／API endpoint | `openrouter/deepseek/deepseek-v4-flash-0731` ／ `https://openrouter.ai/api` | +| 生成限制 | 每次回复 8192 tokens；每题最多 50 次模型回复；未显式设置 reasoning effort 或 temperature | +| 环境与重试 | 本地 Docker，并发 2，每题 1 次尝试，Harbor `max_retries=0`；保留任务默认资源与超时 | +| Job | `tb21-truncation-rerun2-20260909-f778861` | + +两题沿用原始任务版本，并检查新旧 trial config 中的任务标识完全一致： + +| 任务 | Task hash | +| --- | --- | +| `terminal-bench/regex-log` | `sha256:802c16cfd132e6c457529cb864be5a757c1b23b6cadc57f2d01983cb0110292a` | +| `terminal-bench/write-compressor` | `sha256:d9ddd9a8e925e2c566b37b2492cbf995afecefe58874e4043ef78d7f3c892c7e` | + +当时分支尚未推送，容器无法按远端 Git revision 安装，因此先对当前提交执行 +`uv build --wheel`，逐文件核对 wheel 中的 Python 源码与该提交一致，再上传到各自 +容器的 `/tmp/`。测试通过临时适配器将安装来源改为这个本地 wheel；模型调用、 +工具执行和 trajectory 导出仍使用当前分支。 +wheel 的 SHA-256 为 +`3c4ebd9268680ba0798ccbc44e6e56ee91424c5773399171050b7ac24734e9e4`。 + +启动脚本先读取宿主机连接配置，移除 `ANTHROPIC_MODEL` +覆盖值并确认 endpoint 为 OpenRouter；凭据通过环境传递，不写入命令参数。 +Harbor 用以下过滤参数选择题目，名称必须包含 `terminal-bench/` 前缀： + +```bash +--include-task-name terminal-bench/regex-log \ +--include-task-name terminal-bench/write-compressor +``` + +本机实际运行入口如下。脚本内部准备 wheel、记录 manifest，再调用 Harbor；脚本与 +临时适配器都属于这次实验的本地产物，并非已提交的通用 benchmark 命令。 + +```bash +uv run --locked --project benchmarks/harbor python \ + jobs/tb21-truncation-rerun2-20260909-f778861-record/workflow/run.py +``` + +同日还运行了相关自动化回归测试，核心 54 项、Harbor 适配器 16 项全部通过： + +```bash +.venv/bin/python -m pytest -q \ + tests/test_truncation.py tests/test_event_journal.py tests/test_atif.py +benchmarks/harbor/.venv/bin/python -m pytest -q \ + -c benchmarks/harbor/pyproject.toml benchmarks/harbor/tests +``` -任何有限上限都有可能被用尽。只调大数值，当前“截断也当作正常完成”的错误判断 -仍然存在。拟议顺序是先补上截断处理，再通过小规模试跑评估是否提高上限、提高到 -多少，以及如何分配思考与输出预算。11 道失败题出现截断，不等于提高上限后它们就 -一定能通过；通过率、token 消耗和费用都需要重新实验验证。 +**测试结果。** 两题均完成判题，无 Harbor 异常，通过率为 **0/2**；Harbor 报告的 +job 墙钟耗时约 **9 分 42 秒**。两题的截断状态都正确记录，但仍未完成任务。 + +| 任务 | 上次／本次 reward | 本次模型回复数 | 末次回复 tokens | 末次 stop reason | 本次 terminal outcome | Verifier 失败原因 | +| --- | --- | ---: | ---: | --- | --- | --- | +| `regex-log` | 0.0 ／ 0.0 | 1 | 8192 | `max_tokens` | `response_truncated` | 未生成 `/app/regex.txt`，1 项测试失败 | +| `write-compressor` | 0.0 ／ 0.0 | 2 | 8192 | `max_tokens` | `response_truncated` | 未生成 `/app/data.comp`，3 项测试失败 | + +| 任务 | Input tokens | Output tokens | 费用（USD） | +| --- | ---: | ---: | ---: | +| `regex-log` | 1678 | 8192 | 0.00158363 | +| `write-compressor` | 3927 | 8303 | 0.001742255 | +| 合计 | 5605 | 16495 | 0.003325885 | + +费用来自最终 trajectory 中的实际计量，两题费用均完整。两份 ATIF 都通过 Harbor +validator，启动版本包含 `f778861`，日志都有明确的截断提示，末次回复后没有继续 +调用模型。两题末次回复均无工具调用；半截工具参数的跳过与后续历史处理由上述 +自动化测试覆盖，未在这两次实测中触发。Harbor 正常结束且退出码为 0，不代表题目 +通过，评分仍以 verifier 的 reward 为准。 + +这次复跑验证了“识别截断并停止”的行为；它没有提高生成上限，也没有自动续写， +两题仍在交付所需文件前耗尽单次预算。这两次结果不能说明增加预算或自动续写后的 +通过率，后续需要分别实验。 + +原始 trial 日志、trajectory 与 verifier 输出保存在 +`jobs/tb21-truncation-rerun2-20260909-f778861/`。对应的 +`jobs/tb21-truncation-rerun2-20260909-f778861-record/` 保存 `manifest.json`、 +`summary.json`、`results.en.md`、wheel 和 `workflow/` 下的运行、临时安装与汇总脚本。 +这些目录被 Git 忽略，未上传；本文保存版本、方法与结果汇总，不将本地产物视为 +已经进入版本管理的文件。以后重跑须使用新的 job 名，保留这次 0/2 的原始结果。 + +#### 新需求：增加模型生成预算（待实现） + +2026-09-09 补充。这里的预算首先指**单次模型回复的生成 token 上限**。当前上限 +固定为 8192，不能通过环境变量或 CLI 参数调整。本需求要提高经过实测选择的默认上限， +并提供配置入口，使日常使用与 Harbor 实验可以调整、记录实际预算。新的默认值、 +参数名称与配置优先级仍待设计；同时评估 thinking 与最终输出的预算分配。 + +**即使增大上限，也必须保留 `stop_reason="max_tokens"` 的处理。** 任何有限预算 +都有可能耗尽。提高上限可以给模型更多生成空间；现有截断处理负责准确识别停止原因。 +这次两题均用尽 8192 tokens，为评估更高预算提供了对照，但不能保证提高后一定通过。 + +验收时保持模型、题目 hash 和 `max_turns=50` 一致，先关闭自动续写，只改变单次预算。 +检查配置值确实进入 API 请求，并在实验记录中保存生效值；用新 job 重跑上述两题， +比较 reward、截断情况、input／output tokens、费用和耗时。遇到再次截断时，仍须 +准确记录 outcome、保留 usage 与费用，不得将未完成回复当成任务完成。 + +#### 新需求：截断后自动续写（待实现） + +2026-09-09 补充。当前收到 `max_tokens` 就结束本次 run，即使还有剩余模型轮数也 +不会继续。新需求是在配置允许且仍有剩余预算时，由 agent 自动发起后续模型调用， +继续完成原任务，使 headless 运行不必依赖用户手动发送“继续”。 + +预期行为与约束： + +- 提供自动续写的配置开关及有限的续写次数上限。每次后续模型回复计入同一次 run + 的 `max_turns`，不能通过恢复流程重置轮数；默认开关、次数与其他预算限制待设计。 +- 保留已经成功执行的工具结果和任务上下文，为后续请求构造有效历史。截断回复中的 + 工具调用仍不执行，不把半截参数或无匹配结果的工具调用直接重放；只有 thinking、 + 没有可见文本的截断也需要有明确恢复方式。具体续写提示和请求构造方式待设计。 +- 在 Journal／ATIF 中保留每一次 `max_tokens` 的事实，记录后续尝试及全部 usage、 + 费用。继续生成期间不提前发出 run 终态；最终正常结束、达到续写限制、轮数耗尽或 + 请求失败时，准确记录原因。恢复后的终态规则与协议表达需要一并设计。 +- 达到配置上限、收到取消或发生无法恢复的错误时停止，并给出明确诊断，避免无限 + 续写。自动续写能继续处理任务，是否做对仍由 verifier 判断。 + +自动化验收至少覆盖：截断后继续并完成、连续截断后达到限制、剩余轮数耗尽、只有 +thinking 的回复，以及截断工具参数不被执行或错误重放。真实模型验收使用上述两题 +和新的 job 名，在相同单次预算下比较关闭／开启续写的结果，再评估“增加预算＋ +自动续写”的组合；分别报告通过率、追加模型调用数、总 tokens、费用和耗时。 +两个需求都尚未实现，本次 0/2 的实测结果不能当作它们的验收结果。 #### 缺失工具参数导致的崩溃 @@ -570,9 +699,8 @@ MAX_TOKENS = 8192 文字，却没有指定文件。工具 schema 已经把 `path` 声明为必填，但本次收到的实际 参数仍然缺失，因此程序需要在使用参数前检查。 -agent 执行工具前会先发出 `tool.started` 事件，由 `_TextOutputProjector` 打印调用 -预览。打印 `edit` 预览时，它直接读取 `arguments['path']`；字典里没有这个字段， -于是抛出 `KeyError: 'path'`。这次尚未执行文件修改，就在打印预览时失败了。 +这次错误发生在显示工具调用信息时，尚未执行文件修改。因为缺少目标路径，连展示 +这次调用的意图都触发了 `KeyError: 'path'`，导致整个任务退出。 异常发生在 Harbor 超时之后，不能用它解释此前的超时。 需要同时覆盖预览和执行两处：打印预览时能容忍缺失参数；执行工具前校验参数， @@ -584,9 +712,9 @@ Missing required argument: path. Provide the file path and retry. ``` 这样模型可以在后续轮次看到错误，补齐参数重新调用，agent 在剩余预算内继续工作。 -只修打印逻辑不够：`_run_one_tool()` 的执行分支仍然直接读取 `block.input["path"]`， -其异常处理会记录失败后重新抛出，依然导致整次运行失败。缺失必填参数应作为可恢复 -的工具调用错误反馈给模型；其他工具也应覆盖相同的参数检查与错误返回路径。 +展示和执行两个环节都需要容忍不完整输入，否则只是把崩溃推迟到下一步。处理原则 +是把模型可纠正的参数错误反馈为工具结果，让任务继续，而不是直接结束整个任务； +其他工具也应遵循这一原则。 #### 超时停止 @@ -625,13 +753,10 @@ Harbor 的“已经超时”没有及时转化为容器内 agent 的“停止执 及时读取，用来汇总执行步骤、token 和费用。本轮暴露的问题是文件后来有了， 但 Harbor 汇总时没有拿到。 -正常情况下，agent 运行时持续写入内部 Event Journal，记录模型回复、工具调用、 -执行结果和用量；运行收尾时，再由 -[`agent.py`](../../../src/nanopycodeagent/agent.py) 的 `_run_exchange()` 将 Journal -转换成公开的 ATIF 格式 `trajectory.json`。 -[`Harbor adapter`](../../../benchmarks/harbor/src/harbor_adapter/adapter.py) 的 -`populate_context_post_run()` 读取并校验这个文件，记录步骤数等诊断信息，并把 -token 和费用回填到评测结果中。 +正常情况下，agent 运行时持续记录模型回复、工具调用、执行结果和用量，结束时 +将这些记录导出为 ATIF 格式的 `trajectory.json`。Harbor 随后读取并校验轨迹， +据此汇总步骤、token 和费用。这意味着“agent 已停止”“轨迹已保存”“评测已采集” +是需要按顺序完成的三个环节。 本轮 `pytorch-model-recovery` 的采集过程出现了时序问题： @@ -647,8 +772,8 @@ ATIF 一致。文件最终存在、格式有效，并不表示评测汇总时已 记录，并让 Harbor 在汇总前读到它；记录不完整时，也要明确标注。 这需要与“超时停止”一起设计：先让执行停止并在有限时间内完成收尾，再采集和 -汇总。当前 ATIF 写入依赖收尾时的 `finally`；如果直接强制杀死进程，不能保证这段 -代码有机会执行。因此还需考虑定期保存快照，或从已落盘的 Journal 恢复轨迹等 +汇总。当前轨迹导出依赖 agent 完成收尾；如果直接强制杀死进程，轨迹可能来不及 +保存。因此还需考虑定期保存快照，或从已落盘的 Journal 恢复轨迹等 方案。具体方案尚未确定，修复目标是让实际执行记录、保存的 trajectory 和 Harbor 汇总指标能够对应起来。 @@ -661,10 +786,8 @@ ATIF 一致。文件最终存在、格式有效，并不表示评测汇总时已 此时 agent 保存该次调用的 generation ID，作为后续关联账单的标识，再用它查询 这次调用的 `total_cost`。暂时查不到只能标记为待查或未知，不能当成没有费用。 -当前 [`agent.py`](../../../src/nanopycodeagent/agent.py) 的 `_reconcile_costs()` -已经会在运行收尾时，自动对缺少费用且有 generation ID 的调用逐个补查。 -[`cost.py`](../../../src/nanopycodeagent/cost.py) 中的 `resolve_generation_cost()` -默认对每个 generation 最多尝试 6 次，重试间隔为 `1、2、4、8、15` 秒。这些间隔 +当前 agent 已经会在运行收尾时，自动对缺少费用且有 generation ID 的调用逐笔 +补查。每笔默认最多尝试 6 次，重试间隔为 `1、2、4、8、15` 秒。这些间隔 累计为 30 秒，还要加上查询请求本身的耗时，因此 30 秒并非整个补查过程的耗时 上限。超过尝试次数仍未查到，就保留费用不完整的状态。上述收尾补查已经是 agent 的正式能力；当前尚未提供正式的独立后续补查入口。 diff --git a/docs/user_docs/en/cli_reference.md b/docs/user_docs/en/cli_reference.md index 57ed671..f9081af 100644 --- a/docs/user_docs/en/cli_reference.md +++ b/docs/user_docs/en/cli_reference.md @@ -79,10 +79,18 @@ calls. If reply `N` still requests tools, those tools are not run because no reply remains to consume their results. Reaching the limit prints a diagnostic to stderr but is still a normal headless exit. +Each reply has a separate fixed limit of 8192 generated tokens. If the provider +returns `stop_reason="max_tokens"`, the agent stops that run, prints a truncation +diagnostic to stderr, and skips all tools from that reply. It preserves partial +text and records `response_truncated` as the trajectory terminal outcome. +Headless mode still exits `0`; it does not retry or continue automatically. +Interactive mode returns to `You>` with the partial text and a truncation notice +in conversation history, so a later user message can continue the conversation. + ## Output channels During a headless run, stdout carries the streamed model text plus echoed tool -calls and tool results. The startup banner, turn-limit diagnostic, and API +calls and tool results. The startup banner, budget-limit diagnostics, and API errors go to stderr. This separation lets callers capture the run output while retaining operational diagnostics. diff --git a/docs/user_docs/zh-CN/cli_reference.md b/docs/user_docs/zh-CN/cli_reference.md index dbdd455..2758fb1 100644 --- a/docs/user_docs/zh-CN/cli_reference.md +++ b/docs/user_docs/zh-CN/cli_reference.md @@ -70,10 +70,16 @@ nanoPyCodeAgent -p "fix the failing tests" 工具,这些工具不会执行,因为已经没有下一轮回复可以使用工具结果。达到上限时,命令 会在 stderr 打印诊断,但仍属于一次正常的 headless 退出。 +每次回复还受独立的 8192 生成 token 固定上限约束。如果 provider 返回 +`stop_reason="max_tokens"`,agent 会停止本次 run,向 stderr 打印截断诊断,并跳过该 +回复中的所有工具调用。已输出的文本会保留,trajectory 的终态 outcome 记录为 +`response_truncated`。Headless 模式仍退出 `0`,不会自动重试或续写。交互模式会 +返回 `You>`,会话历史中保留部分文本和截断提示,用户可以在后续消息中继续对话。 + ## 输出通道 Headless run 期间,stdout 包含流式模型文本以及回显的工具调用和工具结果。启动 banner、 -轮数上限诊断和 API 错误写入 stderr。调用方因此可以捕获 run output,同时保留运行 +预算上限诊断和 API 错误写入 stderr。调用方因此可以捕获 run output,同时保留运行 诊断。 `--trajectory` 不会改变 stdout。目前没有 JSON 或 JSONL stdout 模式。 diff --git a/src/nanopycodeagent/agent.py b/src/nanopycodeagent/agent.py index 4de0df9..50ba8b1 100644 --- a/src/nanopycodeagent/agent.py +++ b/src/nanopycodeagent/agent.py @@ -54,6 +54,7 @@ JsonObject, JsonValue, NativeEvent, + RunOutcome, utc_now, ) from .read_tool import READ_TOOL, run_read @@ -66,6 +67,11 @@ DEFAULT_MODEL = "claude-sonnet-4-6" MAX_TOKENS = 8192 +_TRUNCATION_NOTICE = ( + "[response truncated: reached max_tokens; stopped without finishing the task. " + "Tool calls from this response were not executed.]" +) + # How many model replies one headless task may spend before the run stops on # its own. The interactive loop needs no such cap — a human watching the # visible output can interrupt a model that keeps retrying the same command — @@ -192,6 +198,8 @@ def __call__(self, event: NativeEvent) -> None: model_call_id = str(event.payload["model_call_id"]) if model_call_id in self._model_calls_with_text: print() + if event.payload["stop_reason"] == "max_tokens": + print(_TRUNCATION_NOTICE, file=sys.stderr) elif event.type == "tool.started": tool_name = str(event.payload["tool_name"]) arguments = event.payload["input"] @@ -354,13 +362,13 @@ def _run_exchange( max_turns: int | None = None, reply_prefix: str = "\nAgent> ", trajectory_path: Path | None = None, -) -> bool: +) -> RunOutcome: """Reply to the conversation so far, running tools until the model stops. - Appends every assistant reply and tool result to ``messages`` in place. - Returns True when the model ended a reply without asking for tools, and - False when ``max_turns`` replies were spent while it was still calling - them — the caller decides what an exhausted budget means. + Appends assistant replies and tool results to ``messages`` in place. + A truncated reply retains only its text and a notice in request history; + the original response is kept in the journal. Returns the stopping outcome, + distinguishing completion, turn-budget exhaustion, and response truncation. """ run_id = f"run-{uuid.uuid4()}" run_started_ns = time.perf_counter_ns() @@ -390,7 +398,7 @@ def _run_exchange( }, ) try: - finished = _run_model_loop( + outcome = _run_model_loop( client, model, messages, @@ -421,7 +429,7 @@ def _run_exchange( emitter.emit( "run.completed", { - "outcome": "completed" if finished else "max_turns_exhausted", + "outcome": outcome, "duration_ms": (time.perf_counter_ns() - run_started_ns) / 1_000_000, **( @@ -438,7 +446,7 @@ def _run_exchange( project_atif(EventJournal.replay(journal.path)), trajectory_path, ) - return finished + return outcome def _run_model_loop( @@ -449,7 +457,7 @@ def _run_model_loop( *, emitter: EventEmitter, max_turns: int | None, -) -> bool: +) -> RunOutcome: """Run model replies and tool calls for an already-started Agent Run.""" turns = 0 while True: @@ -517,13 +525,27 @@ def _run_model_loop( emitter.emit("model.completed", payload) turns += 1 + if message.stop_reason == "max_tokens": + # Do not execute partial tool calls or replay them without results. + # Thinking may also be cut off before its signature arrives. Keep + # visible text and an explicit notice for the next interactive turn. + text = "".join( + block.text for block in message.content if block.type == "text" + ) + messages.append( + { + "role": "assistant", + "content": (f"{text}\n\n" if text else "") + _TRUNCATION_NOTICE, + } + ) + return "response_truncated" messages.append({"role": "assistant", "content": message.content}) if message.stop_reason != "tool_use": - return True + return "completed" if max_turns is not None and turns >= max_turns: # Stop before running the tools: their results would only be # useful to a reply this budget can no longer pay for. - return False + return "max_turns_exhausted" # Every tool_use block needs a matching tool_result in the next # user message, or the API rejects the request. results = [ @@ -655,7 +677,7 @@ def run_headless( messages: list[MessageParam] = [{"role": "user", "content": task}] try: - finished = _run_exchange( + outcome = _run_exchange( client, model, messages, @@ -671,7 +693,7 @@ def run_headless( # swallowing it, throws that away. print(f"API error: {exc}", file=sys.stderr) return 1 - if not finished: + if outcome == "max_turns_exhausted": turns = "turn" if max_turns == 1 else "turns" print( f"[stopped after {max_turns} {turns} without finishing the task]", diff --git a/src/nanopycodeagent/atif.py b/src/nanopycodeagent/atif.py index 0396bbb..dd68cca 100644 --- a/src/nanopycodeagent/atif.py +++ b/src/nanopycodeagent/atif.py @@ -9,7 +9,12 @@ from decimal import Decimal from pathlib import Path -from .event_journal import JsonObject, JsonValue, JournalEntry, SCHEMA_VERSION +from .event_journal import ( + SUPPORTED_SCHEMA_VERSIONS, + JsonObject, + JsonValue, + JournalEntry, +) ATIF_SCHEMA_VERSION = "ATIF-v1.7" @@ -226,7 +231,9 @@ def project_atif(entries: Sequence[JournalEntry]) -> JsonObject: """Fold a complete headless Event Journal into one ATIF-v1.7 document.""" if not entries: raise AtifProjectionError("cannot project an empty Event Journal") - if any(entry.schema_version != SCHEMA_VERSION for entry in entries): + if any( + entry.schema_version not in SUPPORTED_SCHEMA_VERSIONS for entry in entries + ): raise AtifProjectionError("unsupported Event Journal schema") run_ids = {entry.run_id for entry in entries} if len(run_ids) != 1: diff --git a/src/nanopycodeagent/event_journal.py b/src/nanopycodeagent/event_journal.py index 5957819..592de86 100644 --- a/src/nanopycodeagent/event_journal.py +++ b/src/nanopycodeagent/event_journal.py @@ -15,13 +15,16 @@ from datetime import UTC, datetime from decimal import Decimal, InvalidOperation from pathlib import Path -from typing import Callable, Mapping +from typing import Callable, Literal, Mapping from . import settings -SCHEMA_VERSION = 1 +SCHEMA_VERSION = 2 +SUPPORTED_SCHEMA_VERSIONS = frozenset({1, 2}) DEFAULT_MAX_STRING_CHARS = 100_000 +type RunOutcome = Literal["completed", "max_turns_exhausted", "response_truncated"] + EVENT_TYPES = frozenset( { "run.started", @@ -410,7 +413,9 @@ def _validate_native_payload(event_type: str, payload: JsonObject) -> None: ) _validate_tool_error(payload.get("error")) elif event_type == "run.completed": - if payload["outcome"] not in {"completed", "max_turns_exhausted"}: + if payload["outcome"] not in { + "completed", "max_turns_exhausted", "response_truncated" + }: raise ValueError("run.completed.outcome is unsupported") _validate_cost_reconciliation(payload, event_type) elif event_type == "run.failed": @@ -421,7 +426,7 @@ def _validate_native_payload(event_type: str, payload: JsonObject) -> None: @dataclass(frozen=True, slots=True) class NativeEvent: - """One version-one runtime fact produced by the agent core.""" + """One runtime fact produced by the agent core.""" type: str payload: JsonObject @@ -439,7 +444,7 @@ def __post_init__(self) -> None: _validate_native_payload(self.type, self.payload) def to_dict(self) -> JsonObject: - """Return the version-one wire representation of this fact.""" + """Return the wire representation of this fact.""" return {"type": self.type, "payload": self.payload} @@ -491,7 +496,7 @@ def to_dict(self) -> JsonObject: @classmethod def from_dict(cls, value: Mapping[str, object]) -> JournalEntry: - """Validate and rebuild one version-one Journal Entry.""" + """Validate and rebuild one supported Journal Entry.""" schema_version = value.get("schema_version") run_id = value.get("run_id") seq = value.get("seq") @@ -502,7 +507,7 @@ def from_dict(cls, value: Mapping[str, object]) -> JournalEntry: if ( not isinstance(schema_version, int) or isinstance(schema_version, bool) - or schema_version != SCHEMA_VERSION + or schema_version not in SUPPORTED_SCHEMA_VERSIONS ): raise ValueError(f"unsupported Journal Entry schema: {schema_version}") if not isinstance(run_id, str) or not run_id: @@ -517,6 +522,12 @@ def from_dict(cls, value: Mapping[str, object]) -> JournalEntry: if truncation is not None: _validate_truncation(truncation) event = NativeEvent(event_type, payload) + if ( + schema_version == 1 + and event.type == "run.completed" + and event.payload["outcome"] == "response_truncated" + ): + raise ValueError("response_truncated requires Journal Entry schema 2") return cls( schema_version=schema_version, run_id=run_id, diff --git a/tests/test_event_journal.py b/tests/test_event_journal.py index affbd7f..3b90cab 100644 --- a/tests/test_event_journal.py +++ b/tests/test_event_journal.py @@ -54,7 +54,7 @@ def test_journal_entry_wraps_the_native_event_with_ordering_metadata(tmp_path): }, } assert entry.to_dict() == { - "schema_version": 1, + "schema_version": 2, "run_id": "run-123", "seq": 1, "recorded_at": "2026-08-23T08:00:01.420Z", @@ -336,11 +336,12 @@ def test_native_event_contract_rejects_non_json_values(content): ) -def test_journal_entry_rejects_boolean_schema_version(): +@pytest.mark.parametrize("schema_version", [True, 0, 3, "2"]) +def test_journal_entry_rejects_unsupported_schema_version(schema_version): with pytest.raises(ValueError, match="unsupported Journal Entry schema"): JournalEntry.from_dict( { - "schema_version": True, + "schema_version": schema_version, "run_id": "run-1", "seq": 1, "recorded_at": "2026-08-23T08:00:00.000Z", diff --git a/tests/test_truncation.py b/tests/test_truncation.py new file mode 100644 index 0000000..f8120a2 --- /dev/null +++ b/tests/test_truncation.py @@ -0,0 +1,192 @@ +"""Response-budget exhaustion must remain distinct from task completion.""" + +import json +from types import SimpleNamespace + +import anthropic +import httpx +import pytest +from anthropic.types import ThinkingBlock + +from nanopycodeagent import agent, cli, settings +from nanopycodeagent.event_journal import EventJournal, JournalEntry + +from helpers import ( + FakeClient, + FakeMessages, + FakeStream, + patch_client, + patch_client_and_input, + text_block, + write_tool_use_block, +) + + +def _journal_entries(): + paths = list((settings.SETTINGS_PATH.parent / "journals").glob("*.jsonl")) + assert len(paths) == 1 + return EventJournal.replay(paths[0]) + + +@pytest.mark.parametrize("max_turns", [1, 5]) +@pytest.mark.parametrize("content", [ + [text_block("Partial answer")], + [ThinkingBlock(type="thinking", thinking="Still analyzing", signature="")], + [], +]) +def test_truncation_stops_with_usage_cost_and_distinct_terminal( + monkeypatch, tmp_path, capsys, content, max_turns +): + reply = FakeStream( + content, + stop_reason="max_tokens", + usage=SimpleNamespace(input_tokens=10, output_tokens=8192), + response_headers={"x-generation-id": "gen-truncated"}, + ) + messages = FakeMessages([reply]) + patch_client(monkeypatch, FakeClient(messages)) + reconciled = [] + + def resolve(base_url, generation_id, credential, **kwargs): + reconciled.append(generation_id) + return { + "generation_id": generation_id, + "amount": "0.01", + "currency": "USD", + "source": "provider_generation.total_cost", + } + + monkeypatch.setattr(agent, "resolve_generation_cost", resolve) + trajectory_path = tmp_path / "trajectory.json" + assert cli.main([ + "-p", "fix it", "--max-turns", str(max_turns), + "--trajectory", str(trajectory_path), + ]) == 0 + + assert len(messages.calls) == 1 + assert messages.kwargs[0]["max_tokens"] == 8192 + captured = capsys.readouterr() + assert captured.out == ("Partial answer\n" if content and content[0].type == "text" else "") + assert "response truncated" in captured.err + assert "max_tokens" in captured.err + assert "stopped after" not in captured.err + assert reconciled == ["gen-truncated"] + + entries = _journal_entries() + assert all(entry.schema_version == 2 for entry in entries) + assert entries[-1].type == "run.completed" + assert entries[-1].payload["outcome"] == "response_truncated" + completed = next(entry for entry in entries if entry.type == "model.completed") + assert completed.payload["stop_reason"] == "max_tokens" + assert completed.payload["usage"]["output_tokens"] == 8192 + assert completed.payload["content"] == agent._native_content_blocks(content) + assert not any(entry.type.startswith("tool.") for entry in entries) + + trajectory = json.loads(trajectory_path.read_text()) + assert trajectory["schema_version"] == "ATIF-v1.7" + assert trajectory["extra"]["terminal"]["outcome"] == "response_truncated" + assert trajectory["steps"][1]["extra"]["stop_reason"] == "max_tokens" + assert trajectory["final_metrics"]["total_prompt_tokens"] == 10 + assert trajectory["final_metrics"]["total_completion_tokens"] == 8192 + assert trajectory["final_metrics"]["total_cost_usd"] == 0.01 + + +def test_truncated_tools_are_not_executed_or_replayed_on_the_next_user_turn( + monkeypatch, tmp_path, capsys +): + target = tmp_path / "must-not-exist.txt" + truncated = FakeStream([ + ThinkingBlock(type="thinking", thinking="Unfinished thinking", signature=""), + text_block("I was about to write"), + write_tool_use_block("complete-input", path=str(target), content="unsafe"), + write_tool_use_block("partial-input", path=str(target)), + ], stop_reason="max_tokens") + messages = FakeMessages([truncated, [text_block("New answer")]]) + patch_client_and_input( + monkeypatch, client=FakeClient(messages), inputs=["write it", "continue", "/exit"] + ) + + assert agent.run() == 0 + + assert len(messages.calls) == 2 + assert not target.exists() + followup = messages.calls[1] + assert followup[0] == {"role": "user", "content": "write it"} + assert followup[-1] == {"role": "user", "content": "continue"} + assert followup[1]["role"] == "assistant" + assert isinstance(followup[1]["content"], str) + assert followup[1]["content"].startswith("I was about to write\n\n") + assert "response truncated" in followup[1]["content"] + assert "not executed" in followup[1]["content"] + assert "Unfinished thinking" not in followup[1]["content"] + captured = capsys.readouterr() + assert "New answer" in captured.out + assert "[write]" not in captured.out + assert captured.err.count("response truncated") == 1 + + +def test_sdk_stream_with_partial_tool_json_preserves_truncation( + monkeypatch, tmp_path, capsys +): + """Use the real SDK accumulator with an input JSON delta cut mid-string.""" + events = [ + {"type": "message_start", "message": { + "id": "msg-truncated", "type": "message", "role": "assistant", + "model": "test-model", "content": [], "stop_reason": None, + "stop_sequence": None, "usage": {"input_tokens": 10, "output_tokens": 0}, + }}, + {"type": "content_block_start", "index": 0, "content_block": { + "type": "tool_use", "id": "partial-tool", "name": "write", "input": {}, + }}, + {"type": "content_block_delta", "index": 0, "delta": { + "type": "input_json_delta", "partial_json": '{"path": "unfinished', + }}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": { + "stop_reason": "max_tokens", "stop_sequence": None, + }, "usage": {"output_tokens": 8192}}, + {"type": "message_stop"}, + ] + wire = "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events) + requests = [] + + def respond(request): + requests.append(request) + return httpx.Response( + 200, headers={"content-type": "text/event-stream"}, content=wire + ) + + with anthropic.Anthropic( + api_key="test-key", base_url="https://example.test", + http_client=httpx.Client(transport=httpx.MockTransport(respond)), + ) as client: + patch_client(monkeypatch, client) + assert agent.run_headless( + "write a file", trajectory_path=tmp_path / "trajectory.json" + ) == 0 + + assert len(requests) == 1 + assert "response truncated" in capsys.readouterr().err + entries = _journal_entries() + assert entries[-1].payload["outcome"] == "response_truncated" + completed = next(entry for entry in entries if entry.type == "model.completed") + assert completed.payload["tool_calls"][0]["input"] == {} + assert not any(entry.type.startswith("tool.") for entry in entries) + trajectory = json.loads((tmp_path / "trajectory.json").read_text()) + assert trajectory["steps"][1]["tool_calls"][0]["arguments"] == {} + assert "observation" not in trajectory["steps"][1] + + +@pytest.mark.parametrize("schema_version", [1, 2]) +@pytest.mark.parametrize("outcome", ["completed", "max_turns_exhausted", "response_truncated"]) +def test_journal_outcome_versions(schema_version, outcome): + record = { + "schema_version": schema_version, "run_id": "run-1", "seq": 1, + "recorded_at": "2026-09-07T00:00:00.000Z", "type": "run.completed", + "payload": {"outcome": outcome, "duration_ms": 1, "source_timestamp": None}, + } + if schema_version == 1 and outcome == "response_truncated": + with pytest.raises(ValueError, match="requires Journal Entry schema 2"): + JournalEntry.from_dict(record) + else: + assert JournalEntry.from_dict(record).to_dict() == record