From 7a610cbf87ee3ac27f8cdb8a4329076f210dee8d Mon Sep 17 00:00:00 2001 From: minixalpha Date: Wed, 9 Sep 2026 22:41:34 +0800 Subject: [PATCH 1/4] feat: make model generation budgets configurable --- benchmarks/harbor/README.md | 8 ++ benchmarks/harbor/README.zh-CN.md | 6 + .../harbor/src/harbor_adapter/adapter.py | 9 +- benchmarks/harbor/tests/test_adapter.py | 17 +++ docs/user_docs/en/cli_reference.md | 19 ++- docs/user_docs/en/configuration.md | 19 ++- docs/user_docs/zh-CN/cli_reference.md | 17 ++- docs/user_docs/zh-CN/configuration.md | 17 ++- src/nanopycodeagent/agent.py | 22 ++-- src/nanopycodeagent/atif.py | 5 + src/nanopycodeagent/cli.py | 17 ++- src/nanopycodeagent/event_journal.py | 9 ++ src/nanopycodeagent/settings.py | 20 +++ tests/conftest.py | 5 +- tests/test_generation_budget.py | 117 ++++++++++++++++++ tests/test_truncation.py | 16 ++- 16 files changed, 295 insertions(+), 28 deletions(-) create mode 100644 tests/test_generation_budget.py diff --git a/benchmarks/harbor/README.md b/benchmarks/harbor/README.md index f1151c4..de342bb 100644 --- a/benchmarks/harbor/README.md +++ b/benchmarks/harbor/README.md @@ -45,6 +45,14 @@ container's current directory, and saves combined stdout/stderr to `/logs/agent/nanopycodeagent.txt`. It uses the CLI's 50-turn default; override that with `--agent-kwarg max_turns=20`. +For the per-reply generation limit, pass `--agent-kwarg max_tokens=32768`. +This becomes `--max-tokens 32768` in the container and overrides the forwarded +`ANTHROPIC_MAX_TOKENS` environment variable. When omitted, the adapter sends no +token flag, so the installed agent's environment/settings/default applies +(32768 in the version introducing this option). Older releases require omitting +the new option. The effective budget is recorded in the startup log and in +`agent.extra.max_tokens` in the ATIF trajectory. + The adapter also asks the agent to write an ATIF-v1.7 trajectory directly to `/logs/agent/trajectory.json`. Harbor collects that file as the trial's native ATIF output and backfills prompt, completion, cache-token, and cost totals into diff --git a/benchmarks/harbor/README.zh-CN.md b/benchmarks/harbor/README.zh-CN.md index 1af3f2f..15620c4 100644 --- a/benchmarks/harbor/README.zh-CN.md +++ b/benchmarks/harbor/README.zh-CN.md @@ -41,6 +41,12 @@ adapter 通过 stdin 发送任务指令，在 task 容器的当前目录中运 的 stdout/stderr 保存到 `/logs/agent/nanopycodeagent.txt`。它默认沿用 CLI 的 50 轮限制；可以通过 `--agent-kwarg max_turns=20` 覆盖此设置。 +每次回复的生成上限可通过 `--agent-kwarg max_tokens=32768` 指定，它会转换为容器内的 +`--max-tokens 32768`，优先于透传的 `ANTHROPIC_MAX_TOKENS` 环境变量。未指定时， +adapter 不添加该 flag，沿用已安装 agent 的环境变量／配置文件／默认值（引入该参数的 +版本默认为 32768）。安装旧版本时需省略新参数。生效预算会记录到启动日志和 ATIF +trajectory 的 `agent.extra.max_tokens`。 + adapter 还会要求 agent 将 ATIF-v1.7 trajectory 直接写入 `/logs/agent/trajectory.json`。Harbor 会把该文件作为 trial 的原生 ATIF 输出采集， 并将 prompt、completion、cache token 和 cost 汇总回填到 agent result。trajectory diff --git a/benchmarks/harbor/src/harbor_adapter/adapter.py b/benchmarks/harbor/src/harbor_adapter/adapter.py index b4c9e0e..805810f 100644 --- a/benchmarks/harbor/src/harbor_adapter/adapter.py +++ b/benchmarks/harbor/src/harbor_adapter/adapter.py @@ -37,7 +37,8 @@ class NanoPyCodeAgent(BaseInstalledAgent): cli="--max-turns", type="int", default=_DEFAULT_MAX_TURNS, - ) + ), + CliFlag("max_tokens", cli="--max-tokens", type="int"), ] def __init__(self, *args, git_ref: str | None = None, **kwargs): @@ -54,6 +55,9 @@ def __init__(self, *args, git_ref: str | None = None, **kwargs): raise ValueError("version and git_ref are mutually exclusive") self._git_ref = git_ref super().__init__(*args, **kwargs) + max_tokens = self._resolved_flags.get("max_tokens") + if max_tokens is not None and max_tokens < 1: + raise ValueError("max_tokens must be a positive integer") @staticmethod @override @@ -112,6 +116,9 @@ def _runtime_env(self) -> dict[str, str]: model = self.model_name.split("/", 1)[-1] if model: env["ANTHROPIC_MODEL"] = model + max_tokens = self._get_env("ANTHROPIC_MAX_TOKENS") + if max_tokens is not None: + env["ANTHROPIC_MAX_TOKENS"] = max_tokens return env @override diff --git a/benchmarks/harbor/tests/test_adapter.py b/benchmarks/harbor/tests/test_adapter.py index fafe681..a4c5a4a 100644 --- a/benchmarks/harbor/tests/test_adapter.py +++ b/benchmarks/harbor/tests/test_adapter.py @@ -114,6 +114,7 @@ def test_run_pipes_the_instruction_and_forwards_anthropic_configuration(tmp_path "ANTHROPIC_MODEL": "deepseek/deepseek-v4-flash-0731", }, max_turns=20, + max_tokens=65536, ) environment = RecordingEnvironment() @@ -124,6 +125,7 @@ def test_run_pipes_the_instruction_and_forwards_anthropic_configuration(tmp_path assert instruction not in command assert 'printf "%s" "$harbor_nanopycodeagent_instruction_' in command assert "nanoPyCodeAgent --max-turns 20" in command + assert "--max-tokens 65536" in command assert "--trajectory /logs/agent/trajectory.json" in command assert command.endswith("2>&1 | tee /logs/agent/nanopycodeagent.txt") @@ -160,6 +162,21 @@ def test_run_normalizes_harbor_provider_configuration_for_the_anthropic_sdk( assert run_env["ANTHROPIC_BASE_URL"] == "https://openrouter.example/api" assert run_env["ANTHROPIC_MODEL"] == "deepseek/deepseek-v4-flash-0731" assert "OPENROUTER_API_KEY" not in run_env + assert "--max-tokens" not in environment.calls[-1]["command"] + + +def test_run_forwards_the_environment_budget_without_forcing_a_cli_default(tmp_path): + adapter = make_adapter(tmp_path, extra_env={"ANTHROPIC_MAX_TOKENS": "16384"}) + environment = RecordingEnvironment() + asyncio.run(adapter.run("fix it", environment, SimpleNamespace())) + assert environment.calls[-1]["env"]["ANTHROPIC_MAX_TOKENS"] == "16384" + assert "--max-tokens" not in environment.calls[-1]["command"] + + +@pytest.mark.parametrize("value", [0, -1, True, "bad", 3.5]) +def test_adapter_rejects_invalid_generation_budgets(tmp_path, value): + with pytest.raises(ValueError, match="max_tokens"): + make_adapter(tmp_path, max_tokens=value) def test_adapter_declares_atif_support_and_populates_complete_context(tmp_path): diff --git a/docs/user_docs/en/cli_reference.md b/docs/user_docs/en/cli_reference.md index f9081af..679ea44 100644 --- a/docs/user_docs/en/cli_reference.md +++ b/docs/user_docs/en/cli_reference.md @@ -10,7 +10,7 @@ task. ## Synopsis ```text -nanoPyCodeAgent [-h] [-p TEXT | --prompt-file PATH] [--max-turns N] +nanoPyCodeAgent [-h] [-p TEXT | --prompt-file PATH] [--max-turns N] [--max-tokens N] [--trajectory PATH] [--version] ``` @@ -71,6 +71,7 @@ working directory. | `-p TEXT`, `--prompt TEXT` | — | Run `TEXT` as one headless task. | | `--prompt-file PATH` | — | Read one headless task from a UTF-8 file. The file must be readable and contain a non-empty task. | | `--max-turns N` | `50` | Allow at most `N` model replies in a headless run. `N` must be an integer of at least `1`. | +| `--max-tokens N` | `ANTHROPIC_MAX_TOKENS` or `32768` | Maximum generated tokens per model reply in either mode. `N` must be a positive integer; the CLI value overrides environment and settings-file values. | | `--trajectory PATH` | disabled | Write the headless run as one ATIF-v1.7 JSON document. See [Trajectory output](#trajectory-output). | | `--version` | — | Print `nanoPyCodeAgent VERSION` and exit successfully. | @@ -79,7 +80,14 @@ calls. If reply `N` still requests tools, those tools are not run because no reply remains to consume their results. Reaching the limit prints a diagnostic to stderr but is still a normal headless exit. -Each reply has a separate fixed limit of 8192 generated tokens. If the provider +Each reply has a separate generation limit, defaulting to **32768** tokens. +Use `--max-tokens 65536` to override it for one invocation; use +`ANTHROPIC_MAX_TOKENS` in the environment or settings file for a persistent +default. The provider must accept the requested limit for the chosen model. +The limit applies to the whole generated reply, including thinking where the +provider counts it; it is not a guaranteed amount of visible answer text. + +If the provider returns `stop_reason="max_tokens"`, the agent stops that run, prints a truncation diagnostic to stderr, and skips all tools from that reply. It preserves partial text and records `response_truncated` as the trajectory terminal outcome. @@ -103,7 +111,7 @@ stdout mode. | --- | --- | | `0` | Help or version output completed; an interactive session ended normally; or a headless run started and returned control, even if the model gave up, left work incomplete, or exhausted `--max-turns`. | | `1` | A runtime or infrastructure failure prevented a normal run, including missing API credentials or an Anthropic/HTTP API failure. | -| `2` | Command-line usage was invalid, including conflicting or empty task input, an invalid turn limit, an unreadable prompt file, or an invalid trajectory destination. | +| `2` | Command-line usage or the generation-budget setting was invalid, including conflicting or empty task input, an invalid turn or token limit, an unreadable prompt file, or an invalid trajectory destination. | Exit status `0` does not certify that a headless task succeeded. A script or benchmark must inspect the resulting workspace or run its own verifier. @@ -127,6 +135,11 @@ cost information when available, and terminal state describe a single Agent Run. A caught API failure after the run has started produces a partial trajectory with a failed terminal state. +The startup banner and `run.started.max_tokens` in the Journal record the +effective per-reply limit. ATIF exposes it as `agent.extra.max_tokens`, separately +from actual token usage. Older journals without this field still export; +their trajectories omit the unknown limit. + The path contract is: - `--trajectory` requires a headless task and cannot be used in interactive diff --git a/docs/user_docs/en/configuration.md b/docs/user_docs/en/configuration.md index df6f3c4..b607cfa 100644 --- a/docs/user_docs/en/configuration.md +++ b/docs/user_docs/en/configuration.md @@ -24,6 +24,10 @@ The settings file is loaded before the Anthropic client is created and before the model is selected, so the same precedence applies in interactive and headless modes. +The per-reply generation limit also accepts `--max-tokens N`. For this setting, +the order is **CLI > environment > settings file > 32768**. It is resolved once +at startup and applies to every model reply in that invocation. + ## Supported settings | Variable | Required | Default | Description | @@ -32,13 +36,14 @@ headless modes. | `ANTHROPIC_AUTH_TOKEN` | One credential is required | none | Bearer token used for services that authenticate with `Authorization: Bearer`, such as OpenRouter's Anthropic-compatible endpoint. | | `ANTHROPIC_BASE_URL` | No | `https://api.anthropic.com` | Base URL used by the Anthropic SDK. Set it for a compatible proxy or third-party endpoint; leave it unset for the official API. | | `ANTHROPIC_MODEL` | No | `claude-sonnet-4-6` | Model passed to every Messages API call. | +| `ANTHROPIC_MAX_TOKENS` | No | `32768` | Positive integer controlling the generated tokens per model reply. Overridden by `--max-tokens`. | At least one of `ANTHROPIC_API_KEY` or `ANTHROPIC_AUTH_TOKEN` must provide a usable credential. If neither is available, the command reports the missing credentials on stderr and exits with status `1` before starting an Agent Run. The settings-file loader accepts any key whose name begins with `ANTHROPIC_`. -The four variables above are the nanoPyCodeAgent configuration contract; +The five variables above are the nanoPyCodeAgent configuration contract; additional variables are interpreted, if at all, by the installed Anthropic Python SDK and can change with that dependency. @@ -83,7 +88,8 @@ the `env` field in [Claude Code settings](https://code.claude.com/docs/en/settin "ANTHROPIC_API_KEY": "", "ANTHROPIC_AUTH_TOKEN": "", "ANTHROPIC_BASE_URL": "", - "ANTHROPIC_MODEL": "" + "ANTHROPIC_MODEL": "", + "ANTHROPIC_MAX_TOKENS": "" } } ``` @@ -96,7 +102,8 @@ rest empty or remove their keys. For example: "env": { "ANTHROPIC_AUTH_TOKEN": "your-token", "ANTHROPIC_BASE_URL": "https://example.com/anthropic", - "ANTHROPIC_MODEL": "provider/model-name" + "ANTHROPIC_MODEL": "provider/model-name", + "ANTHROPIC_MAX_TOKENS": "32768" } } ``` @@ -144,6 +151,12 @@ supply it: unset ANTHROPIC_API_KEY ANTHROPIC_AUTH_TOKEN ANTHROPIC_BASE_URL ANTHROPIC_MODEL ``` +For `ANTHROPIC_MAX_TOKENS`, a final environment value that is empty, non-integer, +zero, or negative is a usage error (exit `2`) before any model request. A valid +CLI override takes precedence even over an invalid budget in the environment. +Settings-file entries follow the string-only and placeholder rules above; +write the budget as `"32768"`, not a JSON number. + ## Precedence example Given this file: diff --git a/docs/user_docs/zh-CN/cli_reference.md b/docs/user_docs/zh-CN/cli_reference.md index 2758fb1..ebd03e3 100644 --- a/docs/user_docs/zh-CN/cli_reference.md +++ b/docs/user_docs/zh-CN/cli_reference.md @@ -9,7 +9,7 @@ ## 命令格式 ```text -nanoPyCodeAgent [-h] [-p TEXT | --prompt-file PATH] [--max-turns N] +nanoPyCodeAgent [-h] [-p TEXT | --prompt-file PATH] [--max-turns N] [--max-tokens N] [--trajectory PATH] [--version] ``` @@ -63,6 +63,7 @@ nanoPyCodeAgent -p "fix the failing tests" | `-p TEXT`、`--prompt TEXT` | 无 | 把 `TEXT` 作为一次 headless 任务运行。 | | `--prompt-file PATH` | 无 | 从 UTF-8 文件读取一次 headless 任务;文件必须可读并包含非空任务。 | | `--max-turns N` | `50` | 一次 headless run 最多允许 `N` 轮模型回复;`N` 必须是大于或等于 `1` 的整数。 | +| `--max-tokens N` | `ANTHROPIC_MAX_TOKENS` 或 `32768` | 两种模式下每次模型回复的最大生成 token 数。`N` 必须是正整数；CLI 值优先于环境变量与 settings 文件。 | | `--trajectory PATH` | 禁用 | 把 headless run 写成一份 ATIF-v1.7 JSON 文档;参见[Trajectory 输出](#trajectory-输出)。 | | `--version` | 无 | 打印 `nanoPyCodeAgent VERSION` 并成功退出。 | @@ -70,7 +71,13 @@ nanoPyCodeAgent -p "fix the failing tests" 工具,这些工具不会执行,因为已经没有下一轮回复可以使用工具结果。达到上限时,命令 会在 stderr 打印诊断,但仍属于一次正常的 headless 退出。 -每次回复还受独立的 8192 生成 token 固定上限约束。如果 provider 返回 +每次回复还受独立的生成上限约束，默认为 **32768** tokens。可用 +`--max-tokens 65536` 覆盖本次调用，或通过环境变量、settings 文件中的 +`ANTHROPIC_MAX_TOKENS` 设置持久默认值。所选模型与 provider 必须接受该上限。 +它限制整次生成；provider 将 thinking 计入生成预算时，thinking 也占用这一上限， +因此它不保证相同数量的可见回答文本。 + +如果 provider 返回 `stop_reason="max_tokens"`,agent 会停止本次 run,向 stderr 打印截断诊断,并跳过该 回复中的所有工具调用。已输出的文本会保留,trajectory 的终态 outcome 记录为 `response_truncated`。Headless 模式仍退出 `0`,不会自动重试或续写。交互模式会 @@ -90,7 +97,7 @@ Headless run 期间,stdout 包含流式模型文本以及回显的工具调用 | --- | --- | | `0` | help 或 version 输出完成;交互会话正常结束;或者 headless run 已经启动并交回控制权,即使模型放弃、工作未完成或用尽了 `--max-turns`。 | | `1` | runtime 或基础设施故障阻止了正常运行,包括缺少 API 凭据或 Anthropic/HTTP API 失败。 | -| `2` | 命令行用法无效,包括任务输入冲突或为空、轮数上限无效、prompt file 无法读取,或者 trajectory 目标无效。 | +| `2` | 命令行用法无效,包括任务输入冲突或为空、轮数或生成 token 上限无效、prompt file 无法读取,或者 trajectory 目标无效。 | 退出状态 `0` 不证明 headless 任务成功。脚本或 benchmark 必须检查产生的 workspace, 或者运行自己的 verifier。CLI 未处理的意外故障也可能让进程以非零状态和 traceback @@ -111,6 +118,10 @@ Trajectory 是独立 artifact,不会替代或重定向 stdout。它描述单次 模型回复、工具参数、工具结果、时间、用量、可获得的成本信息和终态。如果 run 启动后 发生被捕获的 API 失败,仍会产生带失败终态的 partial trajectory。 +启动提示和 Journal 的 `run.started.max_tokens` 记录最终生效的单次生成上限; +ATIF 将它保存在 `agent.extra.max_tokens` 中。这是请求预算,实际消耗仍由 usage +字段记录。旧 Journal 没有预算信息时,转换后的 ATIF 会省略此字段。 + 路径契约如下: - `--trajectory` 需要 headless 任务,不能在交互模式中使用。 diff --git a/docs/user_docs/zh-CN/configuration.md b/docs/user_docs/zh-CN/configuration.md index 1fb0a7f..de09e38 100644 --- a/docs/user_docs/zh-CN/configuration.md +++ b/docs/user_docs/zh-CN/configuration.md @@ -21,6 +21,10 @@ nanoPyCodeAgent 不加载项目 `.env` 文件,没有项目级 settings 文件, Settings 文件在 Anthropic client 创建和模型选择之前加载,所以交互模式与 headless 模式遵循同一套优先级。 +单次回复的生成上限还支持 `--max-tokens N`。此设置的优先级为 +**CLI > 环境变量 > 配置文件 > 32768**。启动时解析一次,本次调用中的每次模型回复 +都使用同一上限。 + ## 支持的设置 | 变量 | 是否必需 | 默认值 | 说明 | @@ -29,11 +33,12 @@ Settings 文件在 Anthropic client 创建和模型选择之前加载,所以交 | `ANTHROPIC_AUTH_TOKEN` | 两种凭据至少提供一种 | 无 | 需要以 `Authorization: Bearer` 认证的服务所使用的 bearer token,例如 OpenRouter 的 Anthropic-compatible endpoint。 | | `ANTHROPIC_BASE_URL` | 否 | `https://api.anthropic.com` | Anthropic SDK 使用的 base URL。兼容的 proxy 或第三方 endpoint 需要设置此项;使用官方 API 时保持未设置。 | | `ANTHROPIC_MODEL` | 否 | `claude-sonnet-4-6` | 每次 Messages API 调用所使用的模型。 | +| `ANTHROPIC_MAX_TOKENS` | 否 | `32768` | 控制每次模型回复生成 token 数的正整数,可由 `--max-tokens` 覆盖。 | `ANTHROPIC_API_KEY` 与 `ANTHROPIC_AUTH_TOKEN` 中至少要有一个提供可用凭据。两者都 不可用时,命令会在 stderr 报告缺少凭据,并在 Agent Run 启动前以状态 `1` 退出。 -Settings 文件 loader 接受名称以 `ANTHROPIC_` 开头的任何键。上面的四个变量是 +Settings 文件 loader 接受名称以 `ANTHROPIC_` 开头的任何键。上面的五个变量是 nanoPyCodeAgent 的配置契约;其他变量是否生效由安装的 Anthropic Python SDK 决定, 并可能随着该依赖变化。 @@ -76,7 +81,8 @@ nanoPyCodeAgent -p "run the test suite" "ANTHROPIC_API_KEY": "", "ANTHROPIC_AUTH_TOKEN": "", "ANTHROPIC_BASE_URL": "", - "ANTHROPIC_MODEL": "" + "ANTHROPIC_MODEL": "", + "ANTHROPIC_MAX_TOKENS": "" } } ``` @@ -88,7 +94,8 @@ nanoPyCodeAgent -p "run the test suite" "env": { "ANTHROPIC_AUTH_TOKEN": "your-token", "ANTHROPIC_BASE_URL": "https://example.com/anthropic", - "ANTHROPIC_MODEL": "provider/model-name" + "ANTHROPIC_MODEL": "provider/model-name", + "ANTHROPIC_MAX_TOKENS": "32768" } } ``` @@ -129,6 +136,10 @@ chmod 600 ~/.nanoPyCodeAgent/settings.json unset ANTHROPIC_API_KEY ANTHROPIC_AUTH_TOKEN ANTHROPIC_BASE_URL ANTHROPIC_MODEL ``` +`ANTHROPIC_MAX_TOKENS` 的最终环境值为空、非整数、零或负数时,会在任何模型请求前 +报用法错误并以状态 `2` 退出。有效的 CLI 覆盖值优先于环境中的非法预算值。 +配置文件仍遵循上述字符串与占位符规则;预算应写成 `"32768"`,而不是 JSON 数字。 + ## 优先级示例 假设文件内容如下: diff --git a/src/nanopycodeagent/agent.py b/src/nanopycodeagent/agent.py index 50ba8b1..111f51e 100644 --- a/src/nanopycodeagent/agent.py +++ b/src/nanopycodeagent/agent.py @@ -58,14 +58,13 @@ utc_now, ) from .read_tool import READ_TOOL, run_read -from .settings import load_settings_env +from .settings import DEFAULT_MAX_TOKENS, load_settings_env, resolve_max_tokens from .terminal import Spinner, print_tool_output, print_tool_use from .write_tool import WRITE_TOOL, content_preview, run_write # The model used when ANTHROPIC_MODEL is set in neither the environment nor # the config file. DEFAULT_MODEL = "claude-sonnet-4-6" -MAX_TOKENS = 8192 _TRUNCATION_NOTICE = ( "[response truncated: reached max_tokens; stopped without finishing the task. " @@ -360,6 +359,7 @@ def _run_exchange( system: str, *, max_turns: int | None = None, + max_tokens: int = DEFAULT_MAX_TOKENS, reply_prefix: str = "\nAgent> ", trajectory_path: Path | None = None, ) -> RunOutcome: @@ -381,6 +381,7 @@ def _run_exchange( "mode": "headless" if max_turns is not None else "interactive", "model": model, "max_turns": max_turns, + "max_tokens": max_tokens, "producer": { "name": "nanoPyCodeAgent", "version": _package_version(), @@ -405,6 +406,7 @@ def _run_exchange( system, emitter=emitter, max_turns=max_turns, + max_tokens=max_tokens, ) except BaseException as exc: cost_reconciliation = _reconcile_costs(client, journal, emitter) @@ -457,6 +459,7 @@ def _run_model_loop( *, emitter: EventEmitter, max_turns: int | None, + max_tokens: int, ) -> RunOutcome: """Run model replies and tool calls for an already-started Agent Run.""" turns = 0 @@ -478,7 +481,7 @@ def _run_model_loop( # the accumulated message for the conversation history. with Spinner() as spinner, client.messages.stream( model=model, - max_tokens=MAX_TOKENS, + max_tokens=max_tokens, system=system, tools=TOOLS, messages=messages, @@ -605,20 +608,22 @@ def _reconcile_costs( return outcomes -def run() -> int: +def run(*, max_tokens: int | None = None) -> int: """Start the read → ask → answer loop until the user types ``/exit``. A reply may include tool calls; they are executed and their results fed back to the model until it finishes the turn without tool use. Returns the process exit code. """ + max_tokens = resolve_max_tokens(max_tokens) client = _create_client() if client is None: return 1 model = _resolve_model() print( - f"nanoPyCodeAgent v{_package_version()} — model {model} " + f"nanoPyCodeAgent v{_package_version()} — model {model}, " + f"max tokens {max_tokens} " "(set ANTHROPIC_MODEL to override)." ) print("Type a message to chat, or /exit to quit.") @@ -640,7 +645,7 @@ def run() -> int: break messages.append({"role": "user", "content": user_input}) - _run_exchange(client, model, messages, SYSTEM_PROMPT) + _run_exchange(client, model, messages, SYSTEM_PROMPT, max_tokens=max_tokens) print("Bye!") return 0 @@ -650,6 +655,7 @@ def run_headless( task: str, *, max_turns: int = DEFAULT_MAX_TURNS, + max_tokens: int | None = None, trajectory_path: Path | None = None, ) -> int: """Work ``task`` to completion without a user, and return the exit code. @@ -662,6 +668,7 @@ def run_headless( scores the result. Only a run that could not happen at all — no credentials, an API that keeps refusing — exits non-zero. """ + max_tokens = resolve_max_tokens(max_tokens) client = _create_client() if client is None: return 1 @@ -671,7 +678,7 @@ def run_headless( # model's prose and the echoed tool calls, nothing else. print( f"nanoPyCodeAgent v{_package_version()} — model {model}, " - f"max turns {max_turns}", + f"max turns {max_turns}, max tokens {max_tokens}", file=sys.stderr, ) @@ -683,6 +690,7 @@ def run_headless( messages, HEADLESS_SYSTEM_PROMPT, max_turns=max_turns, + max_tokens=max_tokens, reply_prefix="", trajectory_path=trajectory_path, ) diff --git a/src/nanopycodeagent/atif.py b/src/nanopycodeagent/atif.py index dd68cca..ca1a5b5 100644 --- a/src/nanopycodeagent/atif.py +++ b/src/nanopycodeagent/atif.py @@ -424,6 +424,11 @@ def project_atif(entries: Sequence[JournalEntry]) -> JsonObject: "extra": { "mode": run_payload["mode"], "max_turns": run_payload["max_turns"], + **( + {"max_tokens": run_payload["max_tokens"]} + if "max_tokens" in run_payload + else {} + ), }, }, "steps": steps, diff --git a/src/nanopycodeagent/cli.py b/src/nanopycodeagent/cli.py index c895275..93173de 100644 --- a/src/nanopycodeagent/cli.py +++ b/src/nanopycodeagent/cli.py @@ -17,6 +17,7 @@ from pathlib import Path from .agent import DEFAULT_MAX_TURNS, _package_version, run, run_headless +from .settings import DEFAULT_MAX_TOKENS, resolve_max_tokens # Reserved by argparse for a misuse of the command line itself, and used here # for the same: a task that cannot be read is a mistake in how the agent was @@ -54,6 +55,15 @@ def _build_parser() -> argparse.ArgumentParser: f"(default: {DEFAULT_MAX_TURNS})" ), ) + parser.add_argument( + "--max-tokens", + type=int, + metavar="N", + help=( + "maximum generated tokens per model reply in either mode " + f"(default: ANTHROPIC_MAX_TOKENS or {DEFAULT_MAX_TOKENS})" + ), + ) parser.add_argument( "--trajectory", type=Path, @@ -122,15 +132,20 @@ def main(argv: list[str] | None = None) -> int: args = parser.parse_args(argv) if args.max_turns < 1: parser.error("--max-turns must be at least 1") + try: + max_tokens = resolve_max_tokens(args.max_tokens) + except ValueError as exc: + parser.error(str(exc)) task = _read_task(args, parser) if task is None: if args.trajectory is not None: parser.error("--trajectory requires a headless task") - return run() + return run(max_tokens=max_tokens) trajectory_path = _trajectory_path(args.trajectory, parser) return run_headless( task, max_turns=args.max_turns, + max_tokens=max_tokens, trajectory_path=trajectory_path, ) diff --git a/src/nanopycodeagent/event_journal.py b/src/nanopycodeagent/event_journal.py index 592de86..edac9d0 100644 --- a/src/nanopycodeagent/event_journal.py +++ b/src/nanopycodeagent/event_journal.py @@ -299,6 +299,15 @@ def _validate_native_payload(event_type: str, payload: JsonObject) -> None: raise ValueError(f"{event_type}.duration_ms must be non-negative") if event_type == "run.started": + # Optional so journals written before configurable budgets still replay. + if "max_tokens" in payload: + max_tokens = payload["max_tokens"] + if ( + not isinstance(max_tokens, int) + or isinstance(max_tokens, bool) + or max_tokens < 1 + ): + raise ValueError("run.started.max_tokens must be a positive integer") if payload["mode"] not in {"interactive", "headless"}: raise ValueError("run.started.mode must be interactive or headless") _require_string(payload, "model", event_type) diff --git a/src/nanopycodeagent/settings.py b/src/nanopycodeagent/settings.py index 167a26e..76a2dea 100644 --- a/src/nanopycodeagent/settings.py +++ b/src/nanopycodeagent/settings.py @@ -10,6 +10,26 @@ from pathlib import Path SETTINGS_PATH = Path.home() / ".nanoPyCodeAgent" / "settings.json" +DEFAULT_MAX_TOKENS = 32768 + + +def resolve_max_tokens(override: int | None = None) -> int: + """Resolve the per-reply limit: explicit value, environment, file, default.""" + if override is not None: + if isinstance(override, bool) or not isinstance(override, int) or override < 1: + raise ValueError("--max-tokens must be a positive integer") + return override + load_settings_env() + value = os.environ.get("ANTHROPIC_MAX_TOKENS") + if value is None: + return DEFAULT_MAX_TOKENS + try: + limit = int(value) + except ValueError: + raise ValueError("ANTHROPIC_MAX_TOKENS must be a positive integer") from None + if limit < 1: + raise ValueError("ANTHROPIC_MAX_TOKENS must be a positive integer") + return limit def load_settings_env(path: Path | None = None) -> None: diff --git a/tests/conftest.py b/tests/conftest.py index 8ffeafa..435ae64 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,7 +6,10 @@ from nanopycodeagent import settings -_MANAGED_ENV = ("ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "ANTHROPIC_MODEL") +_MANAGED_ENV = ( + "ANTHROPIC_API_KEY", "ANTHROPIC_BASE_URL", "ANTHROPIC_MODEL", + "ANTHROPIC_MAX_TOKENS", +) @pytest.fixture(autouse=True) diff --git a/tests/test_generation_budget.py b/tests/test_generation_budget.py new file mode 100644 index 0000000..415a253 --- /dev/null +++ b/tests/test_generation_budget.py @@ -0,0 +1,117 @@ +"""Generation limits must reach every request and remain observable.""" + +import io +import json +from types import SimpleNamespace + +import pytest + +from nanopycodeagent import agent, cli, settings +from nanopycodeagent.event_journal import EventJournal, NativeEvent + +from helpers import ( + FakeClient, + FakeMessages, + FakeStream, + patch_client, + patch_client_and_input, + read_tool_use_block, + text_block, + write_settings, +) + + +class TtyStdin(io.StringIO): + def isatty(self): + return True + + +@pytest.mark.parametrize("interactive", [False, True]) +@pytest.mark.parametrize("file_value,env_value,cli_value,expected", [ + (None, None, None, 32768), + ("16384", None, None, 16384), + ("8192", "65536", None, 65536), + ("invalid", "16384", None, 16384), + ("invalid", "invalid", "1024", 1024), + (None, None, "1", 1), +]) +def test_budget_precedence_reaches_every_request_and_execution_record( + monkeypatch, tmp_path, capsys, interactive, file_value, env_value, cli_value, expected +): + if file_value is not None: + write_settings(settings.SETTINGS_PATH, {"ANTHROPIC_MAX_TOKENS": file_value}) + if env_value is not None: + monkeypatch.setenv("ANTHROPIC_MAX_TOKENS", env_value) + target = tmp_path / "input.txt" + target.write_text("hello") + usage = SimpleNamespace(input_tokens=10, output_tokens=1) + messages = FakeMessages([ + FakeStream([read_tool_use_block("read-1", path=str(target))], + stop_reason="tool_use", usage=usage), + FakeStream([text_block("done")], usage=usage), + ]) + patch_client_and_input(monkeypatch, client=FakeClient(messages), inputs=["read it", "/exit"]) + trajectory_path = tmp_path / "trajectory.json" + if interactive: + monkeypatch.setattr(cli.sys, "stdin", TtyStdin()) + args = [] + else: + args = ["-p", "read it", "--trajectory", str(trajectory_path)] + if cli_value is not None: + args += ["--max-tokens", cli_value] + + assert cli.main(args) == 0 + + assert [request["max_tokens"] for request in messages.kwargs] == [expected, expected] + captured = capsys.readouterr() + assert f"max tokens {expected}" in (captured.out if interactive else captured.err) + journal_path, = (tmp_path / "journals").glob("*.jsonl") + entries = EventJournal.replay(journal_path) + assert entries[0].payload["max_tokens"] == expected + assert entries[-1].payload["outcome"] == "completed" + if not interactive: + trajectory = json.loads(trajectory_path.read_text()) + assert trajectory["agent"]["extra"]["max_tokens"] == expected + assert trajectory["final_metrics"]["total_completion_tokens"] == 2 + + +@pytest.mark.parametrize("source", ["cli", "env", "file"]) +@pytest.mark.parametrize("value", ["0", "-1", "3.5", "invalid"]) +def test_invalid_budget_fails_before_starting_a_run(monkeypatch, tmp_path, capsys, source, value): + args = ["-p", "do not run"] + if source == "cli": + args += ["--max-tokens", value] + elif source == "env": + monkeypatch.setenv("ANTHROPIC_MAX_TOKENS", value) + else: + write_settings(settings.SETTINGS_PATH, {"ANTHROPIC_MAX_TOKENS": value}) + messages = FakeMessages([]) + patch_client(monkeypatch, FakeClient(messages)) + + with pytest.raises(SystemExit) as excinfo: + cli.main(args) + + assert excinfo.value.code == cli.EXIT_USAGE + diagnostic = "--max-tokens" if source == "cli" else "ANTHROPIC_MAX_TOKENS" + assert diagnostic in capsys.readouterr().err + assert messages.calls == [] + assert not (tmp_path / "journals").exists() + + +def test_direct_headless_entry_point_honors_environment(monkeypatch): + monkeypatch.setenv("ANTHROPIC_MAX_TOKENS", "16384") + messages = FakeMessages([[text_block("done")]]) + patch_client(monkeypatch, FakeClient(messages)) + assert agent.run_headless("say hi") == 0 + assert messages.kwargs[0]["max_tokens"] == 16384 + + +@pytest.mark.parametrize("value", [0, -1, True, 1.5, "32768", None]) +def test_journal_rejects_invalid_recorded_budgets(value): + with pytest.raises(ValueError, match="run.started.max_tokens"): + NativeEvent("run.started", { + "mode": "headless", "model": "test", "max_turns": 50, + "max_tokens": value, + "producer": {"name": "nanoPyCodeAgent", "version": "test"}, + "source_timestamp": None, + }) diff --git a/tests/test_truncation.py b/tests/test_truncation.py index f8120a2..4da9f48 100644 --- a/tests/test_truncation.py +++ b/tests/test_truncation.py @@ -29,18 +29,19 @@ def _journal_entries(): @pytest.mark.parametrize("max_turns", [1, 5]) +@pytest.mark.parametrize("max_tokens", [8192, 32768]) @pytest.mark.parametrize("content", [ [text_block("Partial answer")], [ThinkingBlock(type="thinking", thinking="Still analyzing", signature="")], [], ]) def test_truncation_stops_with_usage_cost_and_distinct_terminal( - monkeypatch, tmp_path, capsys, content, max_turns + monkeypatch, tmp_path, capsys, content, max_turns, max_tokens ): reply = FakeStream( content, stop_reason="max_tokens", - usage=SimpleNamespace(input_tokens=10, output_tokens=8192), + usage=SimpleNamespace(input_tokens=10, output_tokens=max_tokens), response_headers={"x-generation-id": "gen-truncated"}, ) messages = FakeMessages([reply]) @@ -60,11 +61,12 @@ def resolve(base_url, generation_id, credential, **kwargs): trajectory_path = tmp_path / "trajectory.json" assert cli.main([ "-p", "fix it", "--max-turns", str(max_turns), + "--max-tokens", str(max_tokens), "--trajectory", str(trajectory_path), ]) == 0 assert len(messages.calls) == 1 - assert messages.kwargs[0]["max_tokens"] == 8192 + assert messages.kwargs[0]["max_tokens"] == max_tokens captured = capsys.readouterr() assert captured.out == ("Partial answer\n" if content and content[0].type == "text" else "") assert "response truncated" in captured.err @@ -78,7 +80,7 @@ def resolve(base_url, generation_id, credential, **kwargs): assert entries[-1].payload["outcome"] == "response_truncated" completed = next(entry for entry in entries if entry.type == "model.completed") assert completed.payload["stop_reason"] == "max_tokens" - assert completed.payload["usage"]["output_tokens"] == 8192 + assert completed.payload["usage"]["output_tokens"] == max_tokens assert completed.payload["content"] == agent._native_content_blocks(content) assert not any(entry.type.startswith("tool.") for entry in entries) @@ -87,7 +89,8 @@ def resolve(base_url, generation_id, credential, **kwargs): assert trajectory["extra"]["terminal"]["outcome"] == "response_truncated" assert trajectory["steps"][1]["extra"]["stop_reason"] == "max_tokens" assert trajectory["final_metrics"]["total_prompt_tokens"] == 10 - assert trajectory["final_metrics"]["total_completion_tokens"] == 8192 + assert trajectory["agent"]["extra"]["max_tokens"] == max_tokens + assert trajectory["final_metrics"]["total_completion_tokens"] == max_tokens assert trajectory["final_metrics"]["total_cost_usd"] == 0.01 @@ -162,10 +165,11 @@ def respond(request): ) as client: patch_client(monkeypatch, client) assert agent.run_headless( - "write a file", trajectory_path=tmp_path / "trajectory.json" + "write a file", max_tokens=65536, trajectory_path=tmp_path / "trajectory.json" ) == 0 assert len(requests) == 1 + assert json.loads(requests[0].content)["max_tokens"] == 65536 assert "response truncated" in capsys.readouterr().err entries = _journal_entries() assert entries[-1].payload["outcome"] == "response_truncated" From 9b5b0f1ef92549cce16134be37aed09f52890fb0 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Wed, 9 Sep 2026 23:01:37 +0800 Subject: [PATCH 2/4] docs: record generation budget research and initial validation --- docs/changelogs/0.8.x.md | 7 +++ docs/dev_notes/en/0.8.x.md | 59 +++++++++++++++--- docs/dev_notes/zh-CN/0.8.x.md | 110 +++++++++++++++++++++++++++++----- 3 files changed, 153 insertions(+), 23 deletions(-) diff --git a/docs/changelogs/0.8.x.md b/docs/changelogs/0.8.x.md index 021513e..51b034b 100644 --- a/docs/changelogs/0.8.x.md +++ b/docs/changelogs/0.8.x.md @@ -4,6 +4,12 @@ All notable changes in the **0.8.x** release series are documented here. ## [Unreleased] +### Added +- Configure per-reply generation budgets through `--max-tokens` or + `ANTHROPIC_MAX_TOKENS`, including the user settings file and Harbor adapter. + Record the effective limit in startup output, Event Journals, and ATIF + trajectories while retaining compatibility with older journals. + ### Fixed - Stop explicitly when a model response reaches `max_tokens`, reporting `response_truncated` in Event Journals and ATIF trajectories instead of task @@ -12,6 +18,7 @@ All notable changes in the **0.8.x** release series are documented here. New journals use schema v2, with v1 replay still supported. ### Changed +- Increase the default per-reply generation limit from 8192 to 32768 tokens. - Moved detailed CLI and configuration guidance out of the bilingual READMEs into dedicated English and Chinese user references, keeping the READMEs concise while documenting task input, options, exit statuses, trajectories, diff --git a/docs/dev_notes/en/0.8.x.md b/docs/dev_notes/en/0.8.x.md index 68f4185..ffdda53 100644 --- a/docs/dev_notes/en/0.8.x.md +++ b/docs/dev_notes/en/0.8.x.md @@ -528,17 +528,62 @@ This rerun verifies recognition of truncation followed by stopping. It neither i Raw trial logs, trajectories, and verifier output are under `jobs/tb21-truncation-rerun2-20260909-f778861/`. The corresponding `jobs/tb21-truncation-rerun2-20260909-f778861-record/` contains `manifest.json`, `summary.json`, `results.en.md`, the wheel, and run, temporary-installation, and summary scripts under `workflow/`. These directories are Git-ignored and have not been uploaded. This document preserves versions, methods, and results without treating local artifacts as version-controlled files. Future reruns must use new job names and preserve this original 0/2 result. -#### New requirement: increase the model generation budget (pending) +#### Increasing the model generation budget -Added on 2026-09-09. Here, budget primarily means the **token generation limit for one model reply**. The limit is currently fixed at 8192 and cannot be adjusted through environment variables or CLI arguments. This requirement is to raise the default to a value selected through experiments and provide configuration so normal use and Harbor experiments can adjust and record the effective budget. The new default, option names, and configuration precedence remain to be designed. Also evaluate how the budget is divided between thinking and final output. +**Per-reply generation budgets are now configurable, with the default raised from 8192 to 32768 for evaluation.** This budget is the **generated-token limit for one model reply**, not cumulative task usage or the context window. Both preceding cases exhausted 8192 before delivering their files; this experiment reruns the same tasks to evaluate the higher budget. -**Handling `stop_reason="max_tokens"` must remain even with a higher limit.** Any finite budget can be exhausted. Raising it gives the model more generation room; existing truncation handling records the stopping reason accurately. Both rerun tasks exhaust 8192 tokens, providing a comparison point for larger budgets without guaranteeing they will pass. +**Research conclusion: 32768 and 65536 are suitable experimental candidates, but public sources do not establish a universal default to copy.** TB 2.1 evaluations explicitly use both values. Code agents include defaults around 32k/64k, model-dependent limits, and implementations that omit an explicit limit. Evaluation settings and product defaults are recorded separately below so model capacity is not mistaken for an actual request parameter. -For acceptance, preserve the model, task hashes, and `max_turns=50`, disable automatic continuation, and change only the per-response budget. Verify that the configured value reaches the API request and record the effective value. Rerun these two tasks under a new job, comparing rewards, truncations, input/output tokens, costs, and durations. If truncation occurs again, retain an accurate outcome, usage, and costs without classifying an unfinished reply as task completion. +**Settings used by other agents/models on Terminal-Bench.** These first-party evaluation descriptions were checked on 2026-09-09, focusing on this project's **TB 2.1**; TB 2.0 is supplementary. Values are per-generation limits disclosed by the publisher, not cumulative task consumption or independently captured production requests. -#### New requirement: automatically continue after truncation (pending) +| Benchmark | Model and agent | Disclosed per-generation limit | Comparison conditions and sources | +| --- | --- | --- | --- | +| **TB 2.1** | GLM-5.3 and GLM-5.3-Flash / Claude Code **2.1.207** | **65536** (`max_new_tokens`) | Z.ai specifies `temperature=1.0`, `top_p=1`, and a six-hour timeout. [GLM-5.3 notes](https://huggingface.co/zai-org/GLM-5.3/blob/main/README.md#footnotes), [Flash notes](https://huggingface.co/zai-org/GLM-5.3-Flash/blob/main/README.md#footnotes) | +| **TB 2.1** | GLM-5.3-Flash-NVFP4 / Terminus-2 | **32768** (`max_tokens`) | RadixArk's quantized-model evaluation uses `max` effort and **disables task deadlines**. Some trials could not run or finish, so its pass rate is not a comparison under this project's conditions. [Evaluation notes](https://huggingface.co/RadixArk/GLM-5.3-Flash-NVFP4#evaluation) | +| TB 2.0 (supplementary) | Qwen3.6-27B / Harbor + Terminus-2 | **80K** (`max_tokens`, preserving the original unit) | Qwen specifies 256K context, a three-hour timeout, 32 CPUs / 48 GB RAM, and a five-run average. [Model-card footnotes](https://huggingface.co/Qwen/Qwen3.6-27B#evaluation) | +| TB 2.0 (supplementary) | Kimi-K2.6 / Tinker Cookbook's simplified agent | **8192** (`max_tokens`) | Thinking Machines uses 32K context, 200 turns, and no context compaction, and reports many context-limit failures. This establishes that 8192 has been used, not that it suits this project. [Configuration and results](https://github.com/thinking-machines-lab/tinker-cookbook/blob/main/tinker_cookbook/recipes/harbor_rl/README.md#evaluation) | -Added on 2026-09-09. The current run ends on `max_tokens` even if model turns remain. The new behavior is to initiate further model calls automatically when configuration permits and sufficient budget remains, continuing the original task so headless execution does not depend on a user manually sending “continue.” +DeepSeek-V4 is closer to this project's model. Its technical report describes an internal bash + file-edit harness, at most 500 steps, and 512K **context**, but that section does not disclose per-reply `max_tokens` and evaluates TB 2.0. The 512K value therefore cannot be treated as a generation budget or establish the appropriate setting for `deepseek-v4-flash-0731` on TB 2.1. [Report §5.3.1](https://arxiv.org/html/2606.19348v1#S5.SS3.SSS1) + +**Defaults in other code agents.** Defaults require a version, model, and configuration path. Source links below are pinned to the checked commits; official documentation reflects the 2026-09-09 lookup. + +| Agent / inspected scope | Behavior without an explicit override | Evidence and limits | +| --- | --- | --- | +| Claude Code / Opus 4.6 | **64k** | The **v2.1.77 release notes** explicitly raise this model's default to 64k. The same entry's 128k is an available upper limit, not the default, and this finding cannot be generalized to every model. [Release notes](https://github.com/anthropics/claude-code/releases/tag/v2.1.77) | +| Claude Code / unrecognized model IDs, such as custom gateway names | **32000** | Current official documentation specifies this fallback, overridable with `CLAUDE_CODE_MAX_OUTPUT_TOKENS`. Defaults and ceilings for recognized models vary. [Environment variables](https://code.claude.com/docs/en/env-vars#variables) | +| OpenCode / `830d5eb` | **`min(model output limit, 32000)`**; missing or zero model limits fall back to 32000 | The default request-preparation path uses this calculation; environment variables and plugins can override it. [Calculation](https://github.com/anomalyco/opencode/blob/830d5eb5354874105cc31599635a80c1662609e8/packages/opencode/src/provider/transform.ts#L1468), [request construction](https://github.com/anomalyco/opencode/blob/830d5eb5354874105cc31599635a80c1662609e8/packages/opencode/src/session/llm/request.ts#L117) | +| pi / ordinary model-request path at `acaa253` | **Determined by the model's `maxTokens`**, then tightened to remaining context | `buildBaseOptions` uses `options.maxTokens ?? model.maxTokens` and reserves 4096 tokens in its context estimate. There is no common fixed value for all models. Thinking budgets also undergo provider-specific processing. [Source](https://github.com/earendil-works/pi/blob/acaa253cc8e3f159e6100b6f3874861b1f0bfc99/packages/ai/src/api/simple-options.ts#L11) | +| mini-swe-agent / default configuration and LiteLLM path at `04d809c` | **No explicit uniform `max_tokens`** | The default YAML omits it; the model layer forwards `model_kwargs`. The selected configuration, LiteLLM, and server determine the actual value. Omitting the parameter does not mean unlimited generation. [Default configuration](https://github.com/SWE-agent/mini-swe-agent/blob/04d809ceab9df28f9adaed044884180159172930/src/minisweagent/config/default.yaml), [model call](https://github.com/SWE-agent/mini-swe-agent/blob/04d809ceab9df28f9adaed044884180159172930/src/minisweagent/models/litellm_model.py#L64) | +| Codex CLI / official configuration reference | **No uniform default number confirmed in this research** | The inspected configuration reference does not give a default per-generation limit. API model-page output capacity cannot substitute for the CLI's actual default. [Configuration reference](https://learn.chatgpt.com/docs/config-file/config-reference) | + +**Record 32000 and 32768 separately**, rather than reducing both to “32k.” Preserve original units where sources disclose only `64k` or `80K`. Thinking limits and effort levels such as `high`/`max` also need to remain distinct from total generation limits; a separate thinking parameter has not been designed. + +**Implementation and automated acceptance.** Implementation commit: `7a610cbf8`. Configuration uses `--max-tokens`, `ANTHROPIC_MAX_TOKENS`, and the existing `settings.json` `env` object, with precedence **CLI > environment > settings file > 32768**. Both interactive and headless modes resolve the value once at startup and use it for every reply. The effective value must be a positive integer; invalid values fail before any request with exit code `2`. Settings entries retain the existing string rules, for example `"ANTHROPIC_MAX_TOKENS": "32768"`. + +Harbor forwards `--agent-kwarg max_tokens=N` and `ANTHROPIC_MAX_TOKENS`. Omitting the kwarg does not insert a CLI default, allowing environment configuration and the installed version's default to apply. Startup output, Journal `run.started.max_tokens`, and ATIF `agent.extra.max_tokens` record the effective limit separately from actual usage. Older journals without the field still convert; ATIF does not invent a historical budget. + +```bash +nanoPyCodeAgent --max-tokens 65536 -p "fix the failing tests" +# Harbor: add --agent-kwarg max_tokens=65536 to the run command +``` + +All **242 core tests** and **22 Harbor adapter tests** passed. Coverage includes precedence, budgets on every interactive/headless request, early rejection of invalid values, separation of recorded limits from usage, and real SDK request serialization. **65536 has only been tested for parameter forwarding; no real-model effectiveness comparison has run at that setting.** + +**Handling `stop_reason="max_tokens"` must remain even with a higher limit.** Any finite budget can be exhausted. Raising it gives the model more generation room; existing truncation handling records the stopping reason accurately. Both preceding cases exhausted 8192, providing a comparison point without guaranteeing success at a higher limit. + +Truncation regressions also pass at higher budgets: `max_tokens` still produces `response_truncated`, preserves usage and costs, and skips tools from that reply. This change adds neither a separate thinking parameter nor automatic continuation. + +**Real-model experiment at 32768.** New job `tb21-budget32768-20260909-7a610cb` compares against the 8192 results in `tb21-truncation-rerun2-20260909-f778861`. It retains the same TB 2.1 dataset, task hashes, OpenRouter `deepseek/deepseek-v4-flash-0731`, `max_turns=50`, native 900-second task timeouts, concurrency 2, one attempt per task, and no retries. A local wheel was built from the implementation commit, with each packaged Python source verified against that commit. The launcher removes `ANTHROPIC_MAX_TOKENS` and omits `--max-tokens` to test the built-in default. + +The first `regex-log` trial exceeded the default 360-second installation timeout before model execution and received no verifier score; that record remains in the original job. New job `tb21-budget32768-regex-setup-retry-20260909-7a610cb` reruns only that task with an installation timeout of 1080 seconds (`--agent-setup-timeout-multiplier 3`), retaining its 900-second task deadline. + +The initial `write-compressor` trial entered model execution and completed tool calls in its first reply. Its second call received the server error `stream closed before completion` after about 421 seconds of agent execution. Harbor records `UnknownApiError`, and the verifier reward is 0. The response did not return `max_tokens`, so this does not establish budget exhaustion. The completed first reply used 1528 input / 103 output tokens and cost $0.00011786; complete usage and cost for the interrupted call are unknown. That known subtotal must not be reported as the task's complete cost. + +To check whether the service failure was transient, one additional attempt uses the same model, budget, and task deadline under `tb21-budget32768-compressor-api-retry-20260909-7a610cb`, also with a 1080-second installation window. Both model runs are retained rather than replacing the initial failure. Retry results are pending; these two cases measure improvement on these truncation failures, not benefits across the full benchmark. + +#### Automatically continuing after truncation + +The current run ends on `max_tokens` even if model turns remain. The new behavior is to initiate further model calls automatically when configuration permits and sufficient budget remains, continuing the original task so headless execution does not depend on a user manually sending “continue.” Expected behavior and constraints: @@ -547,7 +592,7 @@ Expected behavior and constraints: - Preserve every `max_tokens` fact in the Journal/ATIF and record subsequent attempts, all usage, and all costs. Do not emit a run terminal event while continuation is still underway. Accurately record normal completion, continuation-limit exhaustion, turn exhaustion, or request failure. Terminal rules and protocol representation for recovery need to be designed together. - Stop at configured limits, cancellation, or unrecoverable errors with explicit diagnostics, avoiding endless continuation. Continued execution still relies on the verifier to establish correctness. -Automated acceptance must cover continuation to completion, repeated truncation reaching its limit, exhausted remaining turns, thinking-only replies, and truncated tool arguments neither executing nor being replayed incorrectly. For real-model acceptance, use these two tasks and new job names to compare continuation disabled/enabled at the same response budget, then evaluate larger budgets together with continuation. Report pass rate, additional model calls, total tokens, cost, and duration separately. Both requirements remain unimplemented; this 0/2 experiment is not their acceptance result. +Automated acceptance must cover continuation to completion, repeated truncation reaching its limit, exhausted remaining turns, thinking-only replies, and truncated tool arguments neither executing nor being replayed incorrectly. For real-model acceptance, use these two tasks and new job names to compare continuation disabled/enabled at the same response budget, then evaluate larger budgets together with continuation. Report pass rate, additional model calls, total tokens, cost, and duration separately. Automatic continuation remains unimplemented; neither the preceding 0/2 result at 8192 nor the higher-budget experiment constitutes acceptance for automatic continuation. #### Crashes caused by missing tool arguments diff --git a/docs/dev_notes/zh-CN/0.8.x.md b/docs/dev_notes/zh-CN/0.8.x.md index 853b2a9..314f28c 100644 --- a/docs/dev_notes/zh-CN/0.8.x.md +++ b/docs/dev_notes/zh-CN/0.8.x.md @@ -642,25 +642,103 @@ validator，启动版本包含 `f778861`，日志都有明确的截断提示， 这些目录被 Git 忽略，未上传；本文保存版本、方法与结果汇总，不将本地产物视为 已经进入版本管理的文件。以后重跑须使用新的 job 名，保留这次 0/2 的原始结果。 -#### 新需求：增加模型生成预算（待实现） +#### 增加模型生成预算 + +**已实现单次生成预算配置，并将默认值从 8192 提高到 32768 进行试用。** +这里的预算指**单次模型回复的生成 token 上限**，不是整题累计 token 或上下文窗口。 +前述两题都曾在交付文件前用尽 8192；本次用相同题目重跑，检验提高预算后的效果。 + +**调研结论：32768、65536 适合作为下一轮实验的候选档位，但公开资料没有给出一个 +可以直接照搬的统一默认值。** TB 2.1 已有明确采用这两个值的评测；code agent 则 +既有约 32k／64k 的默认设置，也有按模型决定或不显式设置的实现。下面分别记录 +评测参数和产品默认行为，避免把模型能力上限当成实际请求参数。 + +**其他 agent／模型跑 Terminal-Bench 时设置多少。** 以下为 2026-09-09 查到的 +第一方评测说明，主看本项目使用的 **TB 2.1**；TB 2.0 仅作补充。数字是发布方 +披露的单次生成上限，不是整道题累计消耗，也不是对其线上请求的独立抓包验证。 + +| Benchmark | 模型与 agent | 披露的单次生成上限 | 影响比较的条件与来源 | +| --- | --- | --- | --- | +| **TB 2.1** | GLM-5.3、GLM-5.3-Flash／Claude Code **2.1.207** | **65536**（`max_new_tokens`） | Z.ai 披露 `temperature=1.0`、`top_p=1`、6 小时 timeout。[GLM-5.3 说明](https://huggingface.co/zai-org/GLM-5.3/blob/main/README.md#footnotes)、[Flash 说明](https://huggingface.co/zai-org/GLM-5.3-Flash/blob/main/README.md#footnotes) | +| **TB 2.1** | GLM-5.3-Flash-NVFP4／Terminus-2 | **32768**（`max_tokens`） | RadixArk 的量化版本评测，effort 为 `max`；**关闭了任务 deadline**，存在无法运行及未完成的 trial，不能视为本项目同条件的通过率对照。[评测说明](https://huggingface.co/RadixArk/GLM-5.3-Flash-NVFP4#evaluation) | +| TB 2.0（补充） | Qwen3.6-27B／Harbor + Terminus-2 | **80K**（`max_tokens`，保留原文单位） | Qwen 披露 256K context、3 小时 timeout、32 CPU／48 GB RAM，取 5 次运行平均。[模型卡脚注](https://huggingface.co/Qwen/Qwen3.6-27B#evaluation) | +| TB 2.0（补充） | Kimi-K2.6／Tinker Cookbook 的简化 agent | **8192**（`max_tokens`） | Thinking Machines 的实验使用 32K context、200 轮、无上下文压缩；说明中记录了大量上下文超限。它证明 8192 也被实际采用过，不代表这一预算适合本项目。[实验配置与结果](https://github.com/thinking-machines-lab/tinker-cookbook/blob/main/tinker_cookbook/recipes/harbor_rl/README.md#evaluation) | + +与本项目更接近的 DeepSeek-V4，技术报告披露了代码 agent 使用内部 bash + file-edit +harness、最多 500 步和 512K **上下文**，但该段没有披露单次 `max_tokens`，评测 +还是 TB 2.0。因此，不能把 512K 当生成预算，也不能用它确认本项目 +`deepseek-v4-flash-0731` 在 TB 2.1 上应设置多少。[报告 §5.3.1](https://arxiv.org/html/2606.19348v1#S5.SS3.SSS1) + +**其他 code agent 的默认值。** 默认值必须同时注明版本、模型和配置路径。 +下面的源码链接固定到本次检查的 commit；官方文档按 2026-09-09 查询结果记录。 + +| Agent／检查范围 | 未主动覆盖时的行为 | 依据与边界 | +| --- | --- | --- | +| Claude Code／Opus 4.6 | **64k** | **v2.1.77 发布说明**明确将该模型的默认值提高到 64k；同条中的 128k 是可提高到的上界，不能当成默认值，也不能将该版本结论推广到所有模型。[发布说明](https://github.com/anthropics/claude-code/releases/tag/v2.1.77) | +| Claude Code／无法识别的模型 ID，例如网关自定义名称 | **32000** | 当前官方文档明确列出这个 fallback；可用 `CLAUDE_CODE_MAX_OUTPUT_TOKENS` 覆盖，已识别模型的默认值和上界因模型而异。[环境变量说明](https://code.claude.com/docs/en/env-vars#variables) | +| OpenCode／`830d5eb` | **`min(模型输出上限, 32000)`**；模型输出上限缺失或为 0 时回退到 32000 | 默认请求准备路径使用这一计算结果；环境变量和插件可覆盖。[计算函数](https://github.com/anomalyco/opencode/blob/830d5eb5354874105cc31599635a80c1662609e8/packages/opencode/src/provider/transform.ts#L1468)、[请求构造](https://github.com/anomalyco/opencode/blob/830d5eb5354874105cc31599635a80c1662609e8/packages/opencode/src/session/llm/request.ts#L117) | +| pi／`acaa253` 的普通模型请求路径 | **按模型的 `maxTokens` 决定**，再按剩余上下文收紧 | 当前 `buildBaseOptions` 采用 `options.maxTokens ?? model.maxTokens`，并为上下文估算留出 4096 tokens；不是所有模型共用一个固定数值。thinking 的预算还会经过对应 provider 的处理。[源码](https://github.com/earendil-works/pi/blob/acaa253cc8e3f159e6100b6f3874861b1f0bfc99/packages/ai/src/api/simple-options.ts#L11) | +| mini-swe-agent／`04d809c` 的默认配置与 LiteLLM 路径 | **不显式设置统一的 `max_tokens`** | 默认 YAML 未指定，模型层透传 `model_kwargs`；实际值由选用配置、LiteLLM 和服务端共同决定，省略参数不等于无限生成。[默认配置](https://github.com/SWE-agent/mini-swe-agent/blob/04d809ceab9df28f9adaed044884180159172930/src/minisweagent/config/default.yaml)、[模型调用](https://github.com/SWE-agent/mini-swe-agent/blob/04d809ceab9df28f9adaed044884180159172930/src/minisweagent/models/litellm_model.py#L64) | +| Codex CLI／官方配置参考 | **本次未确认统一的默认数字** | 所查官方配置参考未给出单次生成上限的默认值；不能拿 API 模型页上的最大输出能力代替 CLI 的实际默认设置。[配置参考](https://learn.chatgpt.com/docs/config-file/config-reference) | + +这些数字也提醒我们：**32000 与 32768 要分别记录**，不能都简写成“32k”后当成同一 +配置；只披露 `64k`、`80K` 的来源则保留原文单位。thinking 上限、`high`／`max` +这类 effort 档位也要与总生成上限分开记录；本次尚未确定独立 thinking 参数的设计。 + +**实现与自动化验收。** 实现提交为 `7a610cbf8`。配置入口采用 `--max-tokens`、 +`ANTHROPIC_MAX_TOKENS` 和现有 `settings.json` 的 `env`，优先级为 +**CLI > 环境变量 > 配置文件 > 32768**；交互和 headless 都生效，启动时解析一次， +每轮请求使用同一值。最终值必须是正整数，非法值在请求前以退出码 `2` 报错。 +配置文件沿用现有 `env` 字符串规则，例如 `"ANTHROPIC_MAX_TOKENS": "32768"`。 + +Harbor 支持 `--agent-kwarg max_tokens=N` 和 `ANTHROPIC_MAX_TOKENS` 透传；省略 +kwarg 时不强制插入 CLI 默认值，以便环境配置及所安装版本的默认值生效。 +启动提示、Journal 的 `run.started.max_tokens` 和 ATIF 的 `agent.extra.max_tokens` +都记录生效上限，实际 usage 另行记录。旧 Journal 缺少此字段时仍可转换，ATIF 不会 +为历史记录补写一个推测的预算。 -2026-09-09 补充。这里的预算首先指**单次模型回复的生成 token 上限**。当前上限 -固定为 8192，不能通过环境变量或 CLI 参数调整。本需求要提高经过实测选择的默认上限， -并提供配置入口，使日常使用与 Harbor 实验可以调整、记录实际预算。新的默认值、 -参数名称与配置优先级仍待设计；同时评估 thinking 与最终输出的预算分配。 +```bash +nanoPyCodeAgent --max-tokens 65536 -p "fix the failing tests" +# Harbor: add --agent-kwarg max_tokens=65536 to the run command +``` + +核心 **242 项测试**、Harbor adapter **22 项测试**通过，覆盖配置优先级、交互与 +headless 每轮请求的预算、非法值提前失败、记录与实际 usage 分离，以及真实 SDK +请求序列化。**65536 目前只验证了参数透传，未进行真实模型效果对比。** **即使增大上限，也必须保留 `stop_reason="max_tokens"` 的处理。** 任何有限预算 都有可能耗尽。提高上限可以给模型更多生成空间；现有截断处理负责准确识别停止原因。 -这次两题均用尽 8192 tokens，为评估更高预算提供了对照，但不能保证提高后一定通过。 - -验收时保持模型、题目 hash 和 `max_turns=50` 一致，先关闭自动续写，只改变单次预算。 -检查配置值确实进入 API 请求，并在实验记录中保存生效值；用新 job 重跑上述两题， -比较 reward、截断情况、input／output tokens、费用和耗时。遇到再次截断时，仍须 -准确记录 outcome、保留 usage 与费用，不得将未完成回复当成任务完成。 - -#### 新需求：截断后自动续写（待实现） - -2026-09-09 补充。当前收到 `max_tokens` 就结束本次 run，即使还有剩余模型轮数也 +上次两题均用尽 8192 tokens，为评估更高预算提供了对照，但不能保证提高后一定通过。 + +增大预算后的截断回归也已通过：遇到 `max_tokens` 仍记录 `response_truncated`， +保留 usage 与费用，跳过该回复中的工具调用。本次没有新增独立 thinking 参数或 +自动续写。 + +**32768 的真实模型实验。** 使用新 job +`tb21-budget32768-20260909-7a610cb`，以 +`tb21-truncation-rerun2-20260909-f778861` 的 8192 结果作为对照。保留同一 +TB 2.1 dataset、两题 hash、OpenRouter 的 `deepseek/deepseek-v4-flash-0731`、 +`max_turns=50`、每题 900 秒原生 timeout、并发 2、每题 1 次且不重试。 +从实现提交构建本地 wheel，并逐个核对包内 Python 源文件与该提交一致。 +实验清除 `ANTHROPIC_MAX_TOKENS`，也不传 `--max-tokens`，验证内置默认值。 +`regex-log` 首次在安装阶段超过默认 360 秒，尚未进入模型执行，也没有 verifier +评分；该记录保留在原 job。另建 +`tb21-budget32768-regex-setup-retry-20260909-7a610cb` 仅补跑这题，将安装 timeout +放大为 1080 秒（`--agent-setup-timeout-multiplier 3`），解题 timeout 仍为 900 秒。 +`write-compressor` 首次进入模型执行，第一轮正常完成工具调用；整次执行约 421 秒 +时，第二轮收到服务端 `stream closed before completion`，Harbor 记录 `UnknownApiError`， +verifier reward 为 0。它没有返回 `max_tokens`，不能作为预算耗尽的证据。 +已完成首轮的 usage 为 input 1528／output 103，费用为 $0.00011786；中断调用的 +完整 usage 与费用未知，不能把这一已知小计当成整题总费用。 + +为排除一次性服务故障，再以相同模型、预算和解题时限补跑一次，保存为 +`tb21-budget32768-compressor-api-retry-20260909-7a610cb`；安装窗口同样为 1080 秒。 +两次模型运行均保留，不以补跑覆盖首次失败。补跑结果待回填；这两题只用于验证 +本次截断问题的改善，不代表整套 benchmark 的收益。 + +#### 截断后自动续写 + +当前收到 `max_tokens` 就结束本次 run，即使还有剩余模型轮数也 不会继续。新需求是在配置允许且仍有剩余预算时，由 agent 自动发起后续模型调用， 继续完成原任务，使 headless 运行不必依赖用户手动发送“继续”。 @@ -681,7 +759,7 @@ validator，启动版本包含 `f778861`，日志都有明确的截断提示， thinking 的回复，以及截断工具参数不被执行或错误重放。真实模型验收使用上述两题 和新的 job 名，在相同单次预算下比较关闭／开启续写的结果，再评估“增加预算＋ 自动续写”的组合；分别报告通过率、追加模型调用数、总 tokens、费用和耗时。 -两个需求都尚未实现，本次 0/2 的实测结果不能当作它们的验收结果。 +自动续写尚未实现；前述 8192 下的 0/2 结果及提高预算后的实验均不属于自动续写验收。 #### 缺失工具参数导致的崩溃 From fa8074a27c40b16523cf47d3bad21f79144a8b84 Mon Sep 17 00:00:00 2001 From: minixalpha Date: Thu, 10 Sep 2026 20:24:38 +0800 Subject: [PATCH 3/4] docs: record generation budget benchmark results --- docs/dev_notes/en/0.8.x.md | 19 ++++++++++++++++-- docs/dev_notes/zh-CN/0.8.x.md | 37 ++++++++++++++++++++++++++++++++--- 2 files changed, 51 insertions(+), 5 deletions(-) diff --git a/docs/dev_notes/en/0.8.x.md b/docs/dev_notes/en/0.8.x.md index ffdda53..5fc1c63 100644 --- a/docs/dev_notes/en/0.8.x.md +++ b/docs/dev_notes/en/0.8.x.md @@ -530,7 +530,7 @@ Raw trial logs, trajectories, and verifier output are under `jobs/tb21-truncatio #### Increasing the model generation budget -**Per-reply generation budgets are now configurable, with the default raised from 8192 to 32768 for evaluation.** This budget is the **generated-token limit for one model reply**, not cumulative task usage or the context window. Both preceding cases exhausted 8192 before delivering their files; this experiment reruns the same tasks to evaluate the higher budget. +**Per-reply generation budgets are now configurable, with the default raised from 8192 to 32768 for evaluation.** This budget is the **generated-token limit for one model reply**, not cumulative task usage or the context window. Both preceding cases exhausted 8192 before delivering their files. **Both higher-budget retries timed out at 900 seconds, remaining 0/2; the 32768 setting took effect, but improvement on these cases has not been established.** **Research conclusion: 32768 and 65536 are suitable experimental candidates, but public sources do not establish a universal default to copy.** TB 2.1 evaluations explicitly use both values. Code agents include defaults around 32k/64k, model-dependent limits, and implementations that omit an explicit limit. Evaluation settings and product defaults are recorded separately below so model capacity is not mistaken for an actual request parameter. @@ -579,7 +579,22 @@ The first `regex-log` trial exceeded the default 360-second installation timeout The initial `write-compressor` trial entered model execution and completed tool calls in its first reply. Its second call received the server error `stream closed before completion` after about 421 seconds of agent execution. Harbor records `UnknownApiError`, and the verifier reward is 0. The response did not return `max_tokens`, so this does not establish budget exhaustion. The completed first reply used 1528 input / 103 output tokens and cost $0.00011786; complete usage and cost for the interrupted call are unknown. That known subtotal must not be reported as the task's complete cost. -To check whether the service failure was transient, one additional attempt uses the same model, budget, and task deadline under `tb21-budget32768-compressor-api-retry-20260909-7a610cb`, also with a 1080-second installation window. Both model runs are retained rather than replacing the initial failure. Retry results are pending; these two cases measure improvement on these truncation failures, not benefits across the full benchmark. +To check whether the service failure was transient, one additional attempt uses the same model, budget, and task deadline under `tb21-budget32768-compressor-api-retry-20260909-7a610cb`, also with a 1080-second installation window. Both model runs are retained rather than replacing the initial failure. + +**Final results (executed 2026-09-09, verified and archived 09-10).** Times below are Harbor's agent execution durations, excluding installation and verification. The initial installation failure has no model execution time or score. + +| Task | 8192 baseline | Initial 32768 trial | 32768 retry | +| --- | --- | --- | --- | +| `regex-log` | Reward **0**; `response_truncated`; 187.3 sec | `AgentSetupTimeoutError`; no model execution or score | Reward **0**; `AgentTimeoutError`; 900.9 sec | +| `write-compressor` | Reward **0**; `response_truncated`; 304.4 sec | Reward **0**; `UnknownApiError`; 421.4 sec | Reward **0**; `AgentTimeoutError`; 900.9 sec | + +Both retry startup logs confirm `max tokens 32768`; agent versions and task hashes match the experimental configuration. Verifiers report missing `/app/regex.txt` and `/app/data.comp`, respectively. Including the initial server interruption, **three runs entered model execution and received scores, all 0**; one additional trial failed during installation. No retry result replaces an initial failure. + +**A complete usage and cost comparison was not possible.** Neither timed-out retry exported final ATIF. Harbor records trajectory status `missing` and `null` input/output tokens and costs. Model-call counts, final `stop_reason`, and actual truncation counts also cannot be fully reconstructed from the available records; missing values must not become zeroes. Complete baseline costs at 8192 were $0.00158363 and $0.001742255. At 32768, only the initial `write-compressor` trial's completed first reply has the known subtotal of $0.00011786, so no reliable total cost increase can be calculated. + +**Keep 32768 as the experimental default, without describing this experiment as successful acceptance of the higher budget.** Configuration, request forwarding, and truncation regressions are verified. Real-model evaluation remains affected by a server interruption, task timeouts, and missing trajectories after timeout. Without complete final replies, the experiment cannot establish that a higher budget eliminates `max_tokens` truncation or imply a gain across the full benchmark. + +All three jobs used the same wheel, SHA-256 `91996b1d4c70af8062318a38b3af9235c85d93c9685e10ac1323f3bef3625425`. Raw logs, verifier output, and exceptions remain in the corresponding jobs; each `-record/` directory contains its manifest, wheel, and launcher. The consolidated summary is under `jobs/tb21-budget32768-20260909-7a610cb-record/` in `summary.json`, `results.en.md`, and `workflow/summarize.py`. These `jobs/` artifacts are Git-ignored and have not been uploaded. The associated containers have been cleaned up. #### Automatically continuing after truncation diff --git a/docs/dev_notes/zh-CN/0.8.x.md b/docs/dev_notes/zh-CN/0.8.x.md index 314f28c..9d6173d 100644 --- a/docs/dev_notes/zh-CN/0.8.x.md +++ b/docs/dev_notes/zh-CN/0.8.x.md @@ -646,7 +646,8 @@ validator，启动版本包含 `f778861`，日志都有明确的截断提示， **已实现单次生成预算配置，并将默认值从 8192 提高到 32768 进行试用。** 这里的预算指**单次模型回复的生成 token 上限**，不是整题累计 token 或上下文窗口。 -前述两题都曾在交付文件前用尽 8192；本次用相同题目重跑，检验提高预算后的效果。 +前述两题都曾在交付文件前用尽 8192。**本次两题补跑均在 900 秒处超时，仍为 0/2； +32768 已作为配置生效，但尚未证明能改善这两题。** **调研结论：32768、65536 适合作为下一轮实验的候选档位，但公开资料没有给出一个 可以直接照搬的统一默认值。** TB 2.1 已有明确采用这两个值的评测；code agent 则 @@ -733,8 +734,38 @@ verifier reward 为 0。它没有返回 `max_tokens`，不能作为预算耗尽 为排除一次性服务故障，再以相同模型、预算和解题时限补跑一次，保存为 `tb21-budget32768-compressor-api-retry-20260909-7a610cb`；安装窗口同样为 1080 秒。 -两次模型运行均保留，不以补跑覆盖首次失败。补跑结果待回填；这两题只用于验证 -本次截断问题的改善，不代表整套 benchmark 的收益。 +两次模型运行均保留，不以补跑覆盖首次失败。 + +**最终结果（2026-09-09 执行，09-10 核对归档）。** 下表的时间是 Harbor 记录的 +agent 执行时间，不含安装与 verifier；首次安装失败没有模型执行时间或评分。 + +| 题目 | 8192 基线 | 32768 首次运行 | 32768 补跑 | +| --- | --- | --- | --- | +| `regex-log` | reward **0**；`response_truncated`；187.3 秒 | `AgentSetupTimeoutError`；未调用模型，未评分 | reward **0**；`AgentTimeoutError`；900.9 秒 | +| `write-compressor` | reward **0**；`response_truncated`；304.4 秒 | reward **0**；`UnknownApiError`；421.4 秒 | reward **0**；`AgentTimeoutError`；900.9 秒 | + +两次补跑的启动日志均确认 `max tokens 32768`，agent 版本和题目 hash 与实验配置 +相符。verifier 分别报告 `/app/regex.txt` 和 `/app/data.comp` 不存在。 +包括首次服务端中断在内，共有 **3 次进入模型执行且获评分的运行，全部为 0**；另有 +1 次安装失败。没有挑选一次补跑结果来替换原始失败。 + +**完整用量与成本比较未能完成。** 两次超时均未导出最终 ATIF，Harbor 将 trajectory +记为 `missing`，input／output tokens 和 cost 都为 `null`。模型调用次数、末次 +`stop_reason` 和实际截断次数也无法从现存记录完整恢复，不能将缺失值写成 0。 +8192 基线的完整费用分别为 $0.00158363、$0.001742255；32768 仅有前述首次 +`write-compressor` 已完成首轮的 $0.00011786 小计，无法计算可靠的总费用增幅。 + +因此，**保留 32768 作为本轮试用默认值，但不将这次实验写成提高预算后的成功验收**。 +参数配置、请求透传和截断回归已验证；真实模型实验仍受服务端中断、任务 timeout +和超时后 trajectory 缺失影响。没有完整的末次回复，不能断言更高预算消除了 +`max_tokens` 截断，也不能由这两题推导整套 benchmark 的收益。 + +三个 job 的 wheel 完全相同，SHA-256 为 +`91996b1d4c70af8062318a38b3af9235c85d93c9685e10ac1323f3bef3625425`。 +原始日志、verifier 与异常保存在上述各 job；对应的 `-record/` 保存 manifest、 +wheel 和运行脚本。统一汇总位于 +`jobs/tb21-budget32768-20260909-7a610cb-record/summary.json`、`results.en.md` +及 `workflow/summarize.py`。这些 `jobs/` 产物被 Git 忽略，未上传；相关容器已清理。 #### 截断后自动续写 From c6e0d1ea0533aaf142cee5fb4e91e85536fe24be Mon Sep 17 00:00:00 2001 From: minixalpha Date: Thu, 10 Sep 2026 22:17:44 +0800 Subject: [PATCH 4/4] docs: record extended regex benchmark diagnosis --- docs/dev_notes/en/0.8.x.md | 25 +++++++++++++++-- docs/dev_notes/zh-CN/0.8.x.md | 53 +++++++++++++++++++++++++++++++++-- 2 files changed, 73 insertions(+), 5 deletions(-) diff --git a/docs/dev_notes/en/0.8.x.md b/docs/dev_notes/en/0.8.x.md index 5fc1c63..c4df9a3 100644 --- a/docs/dev_notes/en/0.8.x.md +++ b/docs/dev_notes/en/0.8.x.md @@ -530,7 +530,7 @@ Raw trial logs, trajectories, and verifier output are under `jobs/tb21-truncatio #### Increasing the model generation budget -**Per-reply generation budgets are now configurable, with the default raised from 8192 to 32768 for evaluation.** This budget is the **generated-token limit for one model reply**, not cumulative task usage or the context window. Both preceding cases exhausted 8192 before delivering their files. **Both higher-budget retries timed out at 900 seconds, remaining 0/2; the 32768 setting took effect, but improvement on these cases has not been established.** +**Per-reply generation budgets are now configurable, with the default raised from 8192 to 32768 for evaluation.** This budget is the **generated-token limit for one model reply**, not cumulative task usage or the context window. Both preceding cases exhausted 8192 before delivering their files. Both retries on 09-09 timed out at 900 seconds, remaining 0/2. **On 09-10, a fully traced `regex-log` run with a 3600-second execution allowance passed in about 580 seconds. Most time went into the first reply's sustained thinking output. The earlier timeout was not reproduced, so passing cannot be attributed to the longer deadline.** **Research conclusion: 32768 and 65536 are suitable experimental candidates, but public sources do not establish a universal default to copy.** TB 2.1 evaluations explicitly use both values. Code agents include defaults around 32k/64k, model-dependent limits, and implementations that omit an explicit limit. Evaluation settings and product defaults are recorded separately below so model capacity is not mistaken for an actual request parameter. @@ -592,10 +592,31 @@ Both retry startup logs confirm `max tokens 32768`; agent versions and task hash **A complete usage and cost comparison was not possible.** Neither timed-out retry exported final ATIF. Harbor records trajectory status `missing` and `null` input/output tokens and costs. Model-call counts, final `stop_reason`, and actual truncation counts also cannot be fully reconstructed from the available records; missing values must not become zeroes. Complete baseline costs at 8192 were $0.00158363 and $0.001742255. At 32768, only the initial `write-compressor` trial's completed first reply has the known subtotal of $0.00011786, so no reliable total cost increase can be calculated. -**Keep 32768 as the experimental default, without describing this experiment as successful acceptance of the higher budget.** Configuration, request forwarding, and truncation regressions are verified. Real-model evaluation remains affected by a server interruption, task timeouts, and missing trajectories after timeout. Without complete final replies, the experiment cannot establish that a higher budget eliminates `max_tokens` truncation or imply a gain across the full benchmark. +**Keep 32768 as the experimental default, without describing the 09-09 experiment as successful acceptance of the higher budget.** Configuration, request forwarding, and truncation regressions are verified. Real-model evaluation remains affected by a server interruption, task timeouts, and missing trajectories after timeout. Without complete final replies, the experiment cannot establish that a higher budget eliminates `max_tokens` truncation or imply a gain across the full benchmark. All three jobs used the same wheel, SHA-256 `91996b1d4c70af8062318a38b3af9235c85d93c9685e10ac1323f3bef3625425`. Raw logs, verifier output, and exceptions remain in the corresponding jobs; each `-record/` directory contains its manifest, wheel, and launcher. The consolidated summary is under `jobs/tb21-budget32768-20260909-7a610cb-record/` in `summary.json`, `results.en.md`, and `workflow/summarize.py`. These `jobs/` artifacts are Git-ignored and have not been uploaded. The associated containers have been cleaned up. +**Extended deadline with a complete trajectory (2026-09-10).** Trial `regex-log__2A9PJgL` in new job `tb21-regex-3600s-trace-20260910-7a610cb` received **reward 1.0**, passing the official verifier's `test_regex_matches_dates`. Harbor records **580.4 seconds** of agent execution, excluding installation and verification. This success did not consume time beyond the original 900-second deadline, so it does not establish that extending the deadline resolved the earlier timeout. + +The run retained the same wheel, task hash, model, built-in `max_tokens=32768`, and 50-turn limit, with one task, one attempt, and no retries. Agent execution was allowed **3600 seconds**, followed by 120 seconds for interruption and export. Harbor's outer allowance was 3780 seconds (`--agent-timeout-multiplier 4.2`) to avoid preempting cleanup. Installation remained at 1080 seconds; the verifier deadline was unchanged. A temporary diagnostic wrapper saved an unshortened Journal, HTTP response bodies, per-chunk arrival times, and stack snapshots every 120 seconds directly to persistent logs, retaining the comparison's agent source, prompt, and tools. This run finished naturally without interruption or trajectory reconstruction. + +| Observation | Result | +| --- | --- | +| Model replies / tool calls | **5 / 6**; the first four replies ended with `tool_use`, the last with `end_turn`; no `max_tokens` truncation | +| Total model duration | **567.3 seconds**, approximately **97.9%** of native run time | +| First / subsequent reply durations | **502.4 seconds** / 35.1, 2.8, 4.2, 22.8 seconds | +| Total local tool duration | **0.023 seconds** | +| Complete usage | **74650** prompt tokens, including **53504** cached tokens; **17264** completion tokens | +| Complete cost | **$0.005338074**, with costs reconciled for all five calls | + +**The evidence points to sustained first-reply thinking generation as this run's main delay, rather than local tool execution.** Thinking began about 2.5 seconds after the first request and continued through 500.8 seconds; tool arguments began at 501.7 seconds. The longest interval between nonempty generation deltas was only **3.4 seconds**, with no long stream gap observed. That reply retained 47007 thinking characters. The provider reported **15123 output tokens**, with **12026** in `output_tokens_details.thinking_tokens`. Character counts and provider token accounting differ; subtracting those token fields does not independently measure visible-text usage. + +Integrity checks passed: **147 Journal entries, none truncated**; native ATIF passed Harbor schema validation and exactly matched reconstruction from the Journal. All five streams contained `message_stop`, and all five requests actually sent 32768. Stream analysis uses monotonic clocks to avoid small system-clock adjustments. The diagnostic wrapper also passed three offline scenarios: normal completion, interruption after a simulated stall, and recovery after forced termination. + +**The exact cause of the historical timeout remains undetermined.** Without that run's complete reply and stream timeline, prolonged generation, provider/network stalls, and a different solution path cannot be distinguished. The new run recorded Anthropic SDK **1.4.0**, httpx **0.28.1**, and httpx2 **2.12.0** in its container; corresponding versions were not retained for the earlier container. Equal wheel bytes therefore do not establish identical runtime conditions. This result demonstrates that this task can finish at 32768, but one success cannot establish a causal benefit from extending the deadline or generalize to the other task or full benchmark. + +Complete ATIF is in `regex-log__2A9PJgL/agent/trajectory.json` under the new job. Adjacent `journals/`, `http/`, `http-events.jsonl`, and `python-stacks.log` retain the raw diagnostic evidence. The corresponding `-record/` directory contains `results.en.md`, `summary.json`, `validation.json`, and `workflow/` for the report, integrity checks, and reproduction scripts. Artifacts remain local under Git-ignored `jobs/` and have not been uploaded. The container has been cleaned up, and earlier failure records remain intact. + #### Automatically continuing after truncation The current run ends on `max_tokens` even if model turns remain. The new behavior is to initiate further model calls automatically when configuration permits and sufficient budget remains, continuing the original task so headless execution does not depend on a user manually sending “continue.” diff --git a/docs/dev_notes/zh-CN/0.8.x.md b/docs/dev_notes/zh-CN/0.8.x.md index 9d6173d..ab54a45 100644 --- a/docs/dev_notes/zh-CN/0.8.x.md +++ b/docs/dev_notes/zh-CN/0.8.x.md @@ -646,8 +646,9 @@ validator，启动版本包含 `f778861`，日志都有明确的截断提示， **已实现单次生成预算配置，并将默认值从 8192 提高到 32768 进行试用。** 这里的预算指**单次模型回复的生成 token 上限**，不是整题累计 token 或上下文窗口。 -前述两题都曾在交付文件前用尽 8192。**本次两题补跑均在 900 秒处超时，仍为 0/2； -32768 已作为配置生效，但尚未证明能改善这两题。** +前述两题都曾在交付文件前用尽 8192。09-09 两题补跑均在 900 秒处超时，仍为 0/2。 +**09-10 将解题时限提高到 3600 秒并完整记录 `regex-log`，该题约 580 秒通过； +主要耗时是第一轮模型持续输出 thinking。旧超时未复现，不能把通过归因于延长时限。** **调研结论：32768、65536 适合作为下一轮实验的候选档位，但公开资料没有给出一个 可以直接照搬的统一默认值。** TB 2.1 已有明确采用这两个值的评测；code agent 则 @@ -755,7 +756,7 @@ agent 执行时间，不含安装与 verifier；首次安装失败没有模型 8192 基线的完整费用分别为 $0.00158363、$0.001742255；32768 仅有前述首次 `write-compressor` 已完成首轮的 $0.00011786 小计，无法计算可靠的总费用增幅。 -因此，**保留 32768 作为本轮试用默认值，但不将这次实验写成提高预算后的成功验收**。 +因此，**保留 32768 作为本轮试用默认值，但不将 09-09 的实验写成提高预算后的成功验收**。 参数配置、请求透传和截断回归已验证；真实模型实验仍受服务端中断、任务 timeout 和超时后 trajectory 缺失影响。没有完整的末次回复，不能断言更高预算消除了 `max_tokens` 截断，也不能由这两题推导整套 benchmark 的收益。 @@ -767,6 +768,52 @@ wheel 和运行脚本。统一汇总位于 `jobs/tb21-budget32768-20260909-7a610cb-record/summary.json`、`results.en.md` 及 `workflow/summarize.py`。这些 `jobs/` 产物被 Git 忽略，未上传；相关容器已清理。 +**延长时限并保留完整 trajectory（2026-09-10）。** 新 job +`tb21-regex-3600s-trace-20260910-7a610cb` 的 `regex-log__2A9PJgL` 得到 +**reward 1.0**，官方 verifier 的 `test_regex_matches_dates` 通过。 +Harbor 记录 agent 执行 **580.4 秒**，不含安装和验证；因此，这次成功并未用到 +原 900 秒以外的时间，不能据此认定延长时限解决了此前超时。 + +继续使用上述同一 wheel、题目 hash、模型、内置 `max_tokens=32768` 和 50 轮上限， +单题、单次、不重试。将解题执行窗口改为 **3600 秒**，另留 120 秒用于中断和导出； +Harbor 外层窗口设为 3780 秒（`--agent-timeout-multiplier 4.2`），避免提前杀死 +清理过程。安装仍为 1080 秒，verifier 时限不变。临时诊断包装器将未截短的 Journal、 +HTTP 响应体、逐块到达时间和每 120 秒的线程栈直接写入持久日志目录；agent 源码、 +提示词和工具保持本次对照配置。本次自然完成，未触发中断或轨迹重建。 + +| 观测项 | 结果 | +| --- | --- | +| 模型回复／工具调用 | **5 轮／6 次**；前 4 轮为 `tool_use`，末轮为 `end_turn`，无 `max_tokens` 截断 | +| 模型累计耗时 | **567.3 秒**，占原生 run 时间约 **97.9%** | +| 首轮／后续各轮耗时 | **502.4 秒**／35.1、2.8、4.2、22.8 秒 | +| 本地工具累计耗时 | **0.023 秒** | +| 完整 usage | prompt **74650**（含 cache **53504**），completion **17264** | +| 完整费用 | **$0.005338074**，5 次调用均已完成费用核对 | + +**证据指向本次首轮持续生成 thinking，而非本地工具耗时。** 首轮在请求后约 +2.5 秒开始返回 thinking，持续到第 500.8 秒，第 501.7 秒才开始输出工具参数； +非空生成片段之间最长间隔仅 **3.4 秒**，没有观察到长时间断流。该轮保留了 +47007 个 thinking 字符，服务端报告 output **15123 tokens**，其中 +`output_tokens_details.thinking_tokens` 为 **12026**。字符数与服务端 token +口径不同，不能把两项 token 数的差额直接当成可见文本量。 + +完整性检查通过：**147 条 Journal 记录均未截短**，原生 ATIF 通过 Harbor schema +校验，并与 Journal 重建结果完全一致；5 个响应流都有 `message_stop`，5 次请求均 +实际发送 32768。记录使用单调时钟分析流式耗时，避免系统时钟微调影响。诊断包装器的 +正常结束、模拟卡住后中断、强制终止后恢复三个离线场景也均通过。 + +**仍未确定旧超时的真正原因。** 旧运行缺少完整回复与流式时间线，无法区分当时的 +长时间生成、服务端／网络停滞或不同解题路径。这次记录了实际容器依赖 +Anthropic SDK **1.4.0**、httpx **0.28.1**、httpx2 **2.12.0**；旧容器未保存对应 +版本，因此不能认为同一 wheel 就等于全部运行条件相同。该结果证明这一题在 32768 +下能够完成，但单次成功不能证明延长时限的因果收益，也不能推广到另一题或整套评测。 + +完整 ATIF 位于上述 job 的 `regex-log__2A9PJgL/agent/trajectory.json`；同级 +`journals/`、`http/`、`http-events.jsonl` 和 `python-stacks.log` 保留原始诊断证据。 +对应 `-record/` 下的 `results.en.md`、`summary.json`、`validation.json` 和 +`workflow/` 保存诊断报告、完整性核对和复现脚本。产物保留在本地 `jobs/`，被 Git +忽略且未上传；容器已清理，此前失败记录保留。 + #### 截断后自动续写 当前收到 `max_tokens` 就结束本次 run，即使还有剩余模型轮数也