Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions benchmarks/harbor/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,14 @@ container's current directory, and saves combined stdout/stderr to
`/logs/agent/nanopycodeagent.txt`. It uses the CLI's 50-turn default; override
that with `--agent-kwarg max_turns=20`.

For the per-reply generation limit, pass `--agent-kwarg max_tokens=32768`.
This becomes `--max-tokens 32768` in the container and overrides the forwarded
`ANTHROPIC_MAX_TOKENS` environment variable. When omitted, the adapter sends no
token flag, so the installed agent's environment/settings/default applies
(32768 in the version introducing this option). Older releases require omitting
the new option. The effective budget is recorded in the startup log and in
`agent.extra.max_tokens` in the ATIF trajectory.

The adapter also asks the agent to write an ATIF-v1.7 trajectory directly to
`/logs/agent/trajectory.json`. Harbor collects that file as the trial's native
ATIF output and backfills prompt, completion, cache-token, and cost totals into
Expand Down
6 changes: 6 additions & 0 deletions benchmarks/harbor/README.zh-CN.md
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,12 @@ adapter 通过 stdin 发送任务指令,在 task 容器的当前目录中运
的 stdout/stderr 保存到 `/logs/agent/nanopycodeagent.txt`。它默认沿用 CLI 的 50
轮限制;可以通过 `--agent-kwarg max_turns=20` 覆盖此设置。

每次回复的生成上限可通过 `--agent-kwarg max_tokens=32768` 指定,它会转换为容器内的
`--max-tokens 32768`,优先于透传的 `ANTHROPIC_MAX_TOKENS` 环境变量。未指定时,
adapter 不添加该 flag,沿用已安装 agent 的环境变量/配置文件/默认值(引入该参数的
版本默认为 32768)。安装旧版本时需省略新参数。生效预算会记录到启动日志和 ATIF
trajectory 的 `agent.extra.max_tokens`。

adapter 还会要求 agent 将 ATIF-v1.7 trajectory 直接写入
`/logs/agent/trajectory.json`。Harbor 会把该文件作为 trial 的原生 ATIF 输出采集,
并将 prompt、completion、cache token 和 cost 汇总回填到 agent result。trajectory
Expand Down
9 changes: 8 additions & 1 deletion benchmarks/harbor/src/harbor_adapter/adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,8 @@ class NanoPyCodeAgent(BaseInstalledAgent):
cli="--max-turns",
type="int",
default=_DEFAULT_MAX_TURNS,
)
),
CliFlag("max_tokens", cli="--max-tokens", type="int"),
]

def __init__(self, *args, git_ref: str | None = None, **kwargs):
Expand All @@ -54,6 +55,9 @@ def __init__(self, *args, git_ref: str | None = None, **kwargs):
raise ValueError("version and git_ref are mutually exclusive")
self._git_ref = git_ref
super().__init__(*args, **kwargs)
max_tokens = self._resolved_flags.get("max_tokens")
if max_tokens is not None and max_tokens < 1:
raise ValueError("max_tokens must be a positive integer")

@staticmethod
@override
Expand Down Expand Up @@ -112,6 +116,9 @@ def _runtime_env(self) -> dict[str, str]:
model = self.model_name.split("/", 1)[-1]
if model:
env["ANTHROPIC_MODEL"] = model
max_tokens = self._get_env("ANTHROPIC_MAX_TOKENS")
if max_tokens is not None:
env["ANTHROPIC_MAX_TOKENS"] = max_tokens
return env

@override
Expand Down
17 changes: 17 additions & 0 deletions benchmarks/harbor/tests/test_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@ def test_run_pipes_the_instruction_and_forwards_anthropic_configuration(tmp_path
"ANTHROPIC_MODEL": "deepseek/deepseek-v4-flash-0731",
},
max_turns=20,
max_tokens=65536,
)
environment = RecordingEnvironment()

Expand All @@ -124,6 +125,7 @@ def test_run_pipes_the_instruction_and_forwards_anthropic_configuration(tmp_path
assert instruction not in command
assert 'printf "%s" "$harbor_nanopycodeagent_instruction_' in command
assert "nanoPyCodeAgent --max-turns 20" in command
assert "--max-tokens 65536" in command
assert "--trajectory /logs/agent/trajectory.json" in command
assert command.endswith("2>&1 | tee /logs/agent/nanopycodeagent.txt")

Expand Down Expand Up @@ -160,6 +162,21 @@ def test_run_normalizes_harbor_provider_configuration_for_the_anthropic_sdk(
assert run_env["ANTHROPIC_BASE_URL"] == "https://openrouter.example/api"
assert run_env["ANTHROPIC_MODEL"] == "deepseek/deepseek-v4-flash-0731"
assert "OPENROUTER_API_KEY" not in run_env
assert "--max-tokens" not in environment.calls[-1]["command"]


def test_run_forwards_the_environment_budget_without_forcing_a_cli_default(tmp_path):
adapter = make_adapter(tmp_path, extra_env={"ANTHROPIC_MAX_TOKENS": "16384"})
environment = RecordingEnvironment()
asyncio.run(adapter.run("fix it", environment, SimpleNamespace()))
assert environment.calls[-1]["env"]["ANTHROPIC_MAX_TOKENS"] == "16384"
assert "--max-tokens" not in environment.calls[-1]["command"]


@pytest.mark.parametrize("value", [0, -1, True, "bad", 3.5])
def test_adapter_rejects_invalid_generation_budgets(tmp_path, value):
with pytest.raises(ValueError, match="max_tokens"):
make_adapter(tmp_path, max_tokens=value)


def test_adapter_declares_atif_support_and_populates_complete_context(tmp_path):
Expand Down
7 changes: 7 additions & 0 deletions docs/changelogs/0.8.x.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,12 @@ All notable changes in the **0.8.x** release series are documented here.

## [Unreleased]

### Added
- Configure per-reply generation budgets through `--max-tokens` or
`ANTHROPIC_MAX_TOKENS`, including the user settings file and Harbor adapter.
Record the effective limit in startup output, Event Journals, and ATIF
trajectories while retaining compatibility with older journals.

### Fixed
- Stop explicitly when a model response reaches `max_tokens`, reporting
`response_truncated` in Event Journals and ATIF trajectories instead of task
Expand All @@ -12,6 +18,7 @@ All notable changes in the **0.8.x** release series are documented here.
New journals use schema v2, with v1 replay still supported.

### Changed
- Increase the default per-reply generation limit from 8192 to 32768 tokens.
- Moved detailed CLI and configuration guidance out of the bilingual READMEs
into dedicated English and Chinese user references, keeping the READMEs
concise while documenting task input, options, exit statuses, trajectories,
Expand Down
Loading