From 612986cf4c25303a6e8521c6e4ccb90869c6ccb3 Mon Sep 17 00:00:00 2001 From: zengziyu Date: Wed, 12 Aug 2026 03:45:25 +0000 Subject: [PATCH] Add din_integration/: SWE-bench DinD evaluation pipeline integration This PR adds a self-contained 'din_integration/' subdirectory containing the swebench-dind CLI, AISBench adapter, image build templates, configuration files, and host setup scripts for running SWE-bench Verified evaluations in a Docker-in-Docker setup. Includes: - swebench_dind/ Python package (~1,900 LOC): unified CLI for orchestrator lifecycle, image baking (L1/L2), trial launch, result aggregation, and harbor agent-patch injection - aisbench_adapter/: SwebenchDindTask BaseTask subclass for AISBench config - configs/: matrix.yaml (Harbor JobConfig) + swebench_dind_3x3.py (AISBench example config) - scripts/: start_orchestrator.sh, summarize.py, filter_matrix.py, setup.sh, api_key.env.template - docs/: CLI-USAGE.md, MIGRATION.md - bin/swebench-dind: shell wrapper This is a minimum-workload PR submitted per the design documented at /docs/research/24-swebench-dind-integration-plan-A-slim-2026-08.md and /23-swebench-dind-integration-into-aisbench-2026-08.md. Path env-var refactor is intentionally deferred to keep this PR focused on content delivery; see din_integration/README.md for current path assumptions. Verified locally: 12 trial history (9 PASS / 75% pass@1) + 1 e2e-cli-11099-aider PASS (4min 49s). --- din_integration/README.md | 138 +++++++ din_integration/bin/swebench-dind | 5 + din_integration/configs/matrix.yaml | 116 ++++++ din_integration/configs/swebench_dind_3x3.py | 86 +++++ din_integration/docs/CLI-USAGE.md | 311 +++++++++++++++ din_integration/docs/MIGRATION.md | 103 +++++ din_integration/pyproject.toml | 38 ++ din_integration/scripts/api_key.env.template | 14 + din_integration/scripts/filter_matrix.py | 50 +++ din_integration/scripts/setup.sh | 39 ++ din_integration/scripts/start_orchestrator.sh | 112 ++++++ din_integration/scripts/summarize.py | 176 +++++++++ din_integration/swebench_dind/__init__.py | 3 + din_integration/swebench_dind/__main__.py | 5 + .../aisbench_adapter/__init__.py | 15 + .../aisbench_adapter/result_writer.py | 134 +++++++ .../swebench_dind/aisbench_adapter/runner.py | 135 +++++++ .../swebench_dind/aisbench_adapter/task.py | 140 +++++++ din_integration/swebench_dind/builder.py | 164 ++++++++ din_integration/swebench_dind/cli.py | 242 ++++++++++++ din_integration/swebench_dind/config.py | 164 ++++++++ din_integration/swebench_dind/container.py | 178 +++++++++ .../dockerfiles/Dockerfile.l1-base.j2 | 6 + .../dockerfiles/Dockerfile.l2-agent-aider.j2 | 12 + .../dockerfiles/Dockerfile.l2-agent-msa.j2 | 12 + .../dockerfiles/Dockerfile.l2-agent-oh.j2 | 15 + .../dockerfiles/Dockerfile.l2-agent-qwen.j2 | 17 + din_integration/swebench_dind/launcher.py | 365 ++++++++++++++++++ din_integration/swebench_dind/patcher.py | 147 +++++++ din_integration/swebench_dind/summarizer.py | 138 +++++++ 30 files changed, 3080 insertions(+) create mode 100644 din_integration/README.md create mode 100755 din_integration/bin/swebench-dind create mode 100644 din_integration/configs/matrix.yaml create mode 100644 din_integration/configs/swebench_dind_3x3.py create mode 100644 din_integration/docs/CLI-USAGE.md create mode 100644 din_integration/docs/MIGRATION.md create mode 100644 din_integration/pyproject.toml create mode 100644 din_integration/scripts/api_key.env.template create mode 100755 din_integration/scripts/filter_matrix.py create mode 100755 din_integration/scripts/setup.sh create mode 100755 din_integration/scripts/start_orchestrator.sh create mode 100755 din_integration/scripts/summarize.py create mode 100644 din_integration/swebench_dind/__init__.py create mode 100644 din_integration/swebench_dind/__main__.py create mode 100644 din_integration/swebench_dind/aisbench_adapter/__init__.py create mode 100644 din_integration/swebench_dind/aisbench_adapter/result_writer.py create mode 100644 din_integration/swebench_dind/aisbench_adapter/runner.py create mode 100644 din_integration/swebench_dind/aisbench_adapter/task.py create mode 100644 din_integration/swebench_dind/builder.py create mode 100644 din_integration/swebench_dind/cli.py create mode 100644 din_integration/swebench_dind/config.py create mode 100644 din_integration/swebench_dind/container.py create mode 100644 din_integration/swebench_dind/dockerfiles/Dockerfile.l1-base.j2 create mode 100644 din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-aider.j2 create mode 100644 din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-msa.j2 create mode 100644 din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-oh.j2 create mode 100644 din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-qwen.j2 create mode 100644 din_integration/swebench_dind/launcher.py create mode 100644 din_integration/swebench_dind/patcher.py create mode 100644 din_integration/swebench_dind/summarizer.py diff --git a/din_integration/README.md b/din_integration/README.md new file mode 100644 index 00000000..bbf9f6a4 --- /dev/null +++ b/din_integration/README.md @@ -0,0 +1,138 @@ +# din_integration — SWE-bench DinD Evaluation Pipeline + +> **状态**: 本目录是对 SWE-bench DinD (Docker-in-Docker) 多 agent × 多 case 评测流水线的**整包搬运**, +> 作为 AISBench 仓库根目录下的独立子目录提交。 +> +> **历史**: 代码原位于 `/home/zengziyu/mini_matrix/cli/` + `mini_matrix/scripts/` + `mini_matrix/config/`, +> 经评审后整合到本目录,便于在 AISBench 仓内统一治理。 + +--- + +## ⚠ 路径假设(本包当前已知局限) + +本包直接拷贝自 `mini_matrix/`,**未做 env-var 化路径重构**。这意味着: + +| 路径硬编码 | 假设值 | +|---|---| +| `swebench_dind/launcher.py:58` api_key.env | `/home/zengziyu/mini_matrix/scripts/api_key.env` | +| `swebench_dind/config.py:14-17` ROOT | 由 `Path(__file__).parent.parent` 自动推导 → 当前为 `din_integration/` | +| `swebench_dind/config.py:27-35` jobs/tasks/logs | 位于 ROOT 同级 → 当前为 `din_integration/{jobs,tasks,logs}/` | +| `swebench_dind/config.py:31` api_key.env | `ROOT/scripts/api_key.env` → 当前为 `din_integration/scripts/api_key.env` | + +**实测工作方式**(以最小集 PR 为目标): + +```bash +# 1. 把 secrets 放在 launcher.py 期望的位置(原 mini_matrix/scripts/api_key.env) +ln -sf /path/to/your/api_key.env /home/zengziyu/mini_matrix/scripts/api_key.env + +# 2. 在 din_integration/ 下装包 +cd din_integration +pip install -e . + +# 3. 准备运行时数据目录(可指向任意位置) +mkdir -p ~/swebench_dind_{jobs,tasks,logs} + +# 4. 跑 CLI(注意 launcher.py 仍会从 mini_matrix/scripts/ 读 key) +swebench-dind --version +``` + +**完整 env-var 重构方案**见设计文档 [方案 A §5](file:///home/zengziyu/aisbench/docs/research/24-swebench-dind-integration-plan-A-slim-2026-08.md#5-代码修改清单共-9-处), +本次未执行(以最小集 PR 为目标)。 + +--- + +## 📂 本目录结构 + +``` +din_integration/ +├── README.md ← 本文件 +├── pyproject.toml ← Python 包元数据 (name=swebench-dind) +├── bin/swebench-dind ← shell wrapper +├── docs/ +│ ├── CLI-USAGE.md ← CLI 完整命令参考 +│ └── MIGRATION.md ← 老脚本 → CLI 对照表 +├── swebench_dind/ ← 核心 Python 包 +│ ├── __init__.py ← __version__ = "0.1.0" +│ ├── __main__.py +│ ├── cli.py ← Typer 7 子命令入口 +│ ├── config.py ← 单一真相源(常量 + tag 推导) +│ ├── container.py ← DinD 容器生命周期 +│ ├── builder.py ← L3/L4 镜像烤制(Jinja2) +│ ├── launcher.py ← harbor jobs start 拼装 + Rich 进度 +│ ├── patcher.py ← idempotent install probe 注入 +│ ├── summarizer.py ← result.json → md/csv/json +│ ├── dockerfiles/ ← L1/L2 Dockerfile 模板 (5 个 .j2) +│ │ ├── Dockerfile.l1-base.j2 +│ │ ├── Dockerfile.l2-agent-aider.j2 +│ │ ├── Dockerfile.l2-agent-msa.j2 +│ │ ├── Dockerfile.l2-agent-oh.j2 +│ │ └── Dockerfile.l2-agent-qwen.j2 +│ └── aisbench_adapter/ ← AISBench BaseTask 适配 +│ ├── __init__.py +│ ├── task.py ← SwebenchDindTask (BaseTask 子类) +│ ├── result_writer.py ← harbor → AISBench schema +│ └── runner.py ← subprocess 入口 +├── configs/ +│ ├── matrix.yaml ← Harbor JobConfig (15 task × 4 agent = 60 trial) +│ └── swebench_dind_3x3.py ← AISBench config 示例 (3 cases × 3 agents) +└── scripts/ + ├── start_orchestrator.sh ← 启动 DinD 容器 + bind mount + ├── summarize.py ← 汇总 jobs/*/result.json + └── filter_matrix.py ← 子集过滤 +``` + +--- + +## 🚀 快速使用(在 din_integration/ 内) + +```bash +cd din_integration +pip install -e . + +# 启 DinD (需要 host 已安装 docker + qemu binfmt) +bash scripts/start_orchestrator.sh + +# 跑单个 trial +swebench-dind launch trial --case 11099 --agent aider --wait + +# 汇总结果 +swebench-dind summarize +``` + +--- + +## 📦 跟 AISBench 的集成方式 + +`aisbench_adapter/task.py` 实现 `SwebenchDindTask`,继承自 `ais_bench.benchmark.tasks.base.BaseTask`, +通过 `ais_bench.benchmark.registry.TASKS.register_module()` 注册。 + +**AISBench config 示例**见 [configs/swebench_dind_3x3.py](configs/swebench_dind_3x3.py), +3 cases × 3 agents = 9 trial 的最小矩阵。 + +⚠ **本包不通过 `setup.py` entry_point 注册到 `ais_bench.benchmark_plugins`** —— 因为: +1. swebench-dind 是**重量级 CLI + DinD 镜像**集成,不是传统意义上的 plugin (单文件 import) +2. AISBench plugin 接口需要 `pip install` 后才能 import,本包需要 host 上 docker + QEMU 准备 +3. 用户显式选择 "最小集 PR" 路径(参见 doc 23/24 讨论) + +如需 AISBench 标准 plugin 形式接入,后续可加 `setup.py` + `entry_points`. + +--- + +## 📚 关联文档 + +- [mini_matrix/docs/research/18-swebench-dind-complete-project-doc-2026-08.md](../../../mini_matrix/docs/research/18-swebench-dind-complete-project-doc-2026-08.md) — 工程实现细节 +- [aisbench/docs/research/24-swebench-dind-integration-plan-A-slim-2026-08.md](../../../aisbench/docs/research/24-swebench-dind-integration-plan-A-slim-2026-08.md) — 方案 A 设计文档(精简版) +- [aisbench/docs/research/23-swebench-dind-integration-into-aisbench-2026-08.md](../../../aisbench/docs/research/23-swebench-dind-integration-into-aisbench-2026-08.md) — 方案 B 设计文档(完整版) + +--- + +## 📊 已验证 + +- 12 trial 历史 (9 PASS / 75% pass@1) + 1 次 e2e-cli-11099-aider PASS (4min 49s) +- 详见 [mini_matrix/docs/research/18 §11](../mini_matrix/docs/research/18-swebench-dind-complete-project-doc-2026-08.md) + +--- + +## 📝 License + +MIT (沿用 swebench-dind 包原始 license) \ No newline at end of file diff --git a/din_integration/bin/swebench-dind b/din_integration/bin/swebench-dind new file mode 100755 index 00000000..19ba03fe --- /dev/null +++ b/din_integration/bin/swebench-dind @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Thin shell wrapper around `python -m swebench_dind`. +# After `pip install -e .` you can also call `swebench-dind` directly. +set -euo pipefail +exec python -m swebench_dind "$@" \ No newline at end of file diff --git a/din_integration/configs/matrix.yaml b/din_integration/configs/matrix.yaml new file mode 100644 index 00000000..2981cff4 --- /dev/null +++ b/din_integration/configs/matrix.yaml @@ -0,0 +1,116 @@ +# ============================================================================ +# Multi-Bench × Multi-Agent Matrix Configuration +# ============================================================================ +# +# 这是 Harbor 0.20.x 的 JobConfig 格式(harbor.models.job.config.JobConfig)。 +# 一份 yaml 即可驱动 N 个 dataset × M 个 agent × K 个 model 的笛卡尔积评测。 +# +# 引用: +# - Harbor CLI: harbor jobs start -c +# - Schema: harbor/src/harbor/models/job/config.py +# +# 重新跑矩阵: +# bash scripts/run_matrix.sh +# +# ============================================================================ + +job_name: matrix-mini-001 + +# --------------------------------------------------------------------------- +# 全局超时(harbor 0.20.x 单位:秒,multiplier 是相对默认值) +# --------------------------------------------------------------------------- +timeout_multiplier: 2.0 # 整体 ×2(覆盖 default) +agent_setup_timeout_multiplier: 4.0 # agent install 段 ×4(aider 装要 17 分钟) +verifier_timeout_multiplier: 2.0 # verifier 跑测试 ×2 +environment_build_timeout_multiplier: 8.0 # 首次 image build ×8 + +# --------------------------------------------------------------------------- +# 路径(env vars 注入到 trial container,harbor 不会自动传 orchestrator 的 env) +# --------------------------------------------------------------------------- +environment: + env: + OPENAI_API_BASE: "https://api.siliconflow.cn/v1" + OPENAI_API_KEY: "${OPENAI_API_KEY}" + LLM_BASE_URL: "https://api.siliconflow.cn/v1" + LLM_MODEL: "openai/Qwen/Qwen3-Coder-30B-A3B-Instruct" + +verifier: + env: + OPENAI_API_KEY: "${OPENAI_API_KEY}" + +# --------------------------------------------------------------------------- +# 并发(默认 harbor 串行,本矩阵 n_concurrent_trials=2 = 2 个 trial 并行) +# --------------------------------------------------------------------------- +n_concurrent_trials: 2 + +# --------------------------------------------------------------------------- +# Datasets: 当前 baked 矩阵覆盖的 6 个 SWE-bench Django case +# 路径指向 /opt/swebench/data/tasks/(容器内)/ +# --------------------------------------------------------------------------- +datasets: + - path: /opt/swebench/data/tasks/django__django-10097-aider + n_tasks: 1 + - path: /opt/swebench/data/tasks/django__django-10097-msa + n_tasks: 1 + - path: /opt/swebench/data/tasks/django__django-10554-aider + n_tasks: 1 + - path: /opt/swebench/data/tasks/django__django-10554-msa + n_tasks: 1 + - path: /opt/swebench/data/tasks/django__django-10880-aider + n_tasks: 1 + - path: /opt/swebench/data/tasks/django__django-10880-msa + n_tasks: 1 + - path: /opt/swebench/data/tasks/django__django-11099-aider + n_tasks: 1 + - path: /opt/swebench/data/tasks/django__django-11099-msa + n_tasks: 1 + - path: /opt/swebench/data/tasks/django__django-11099-oh + n_tasks: 1 + - path: /opt/swebench/data/tasks/django__django-12308-aider + n_tasks: 1 + - path: /opt/swebench/data/tasks/django__django-12308-msa + n_tasks: 1 + - path: /opt/swebench/data/tasks/django__django-12308-oh + n_tasks: 1 + - path: /opt/swebench/data/tasks/django__django-13741-aider + n_tasks: 1 + - path: /opt/swebench/data/tasks/django__django-13741-msa + n_tasks: 1 + - path: /opt/swebench/data/tasks/django__django-13741-oh + n_tasks: 1 + +# --------------------------------------------------------------------------- +# Agents: 4 个 agent × 1 个 LLM +# Harbor 自动按 tasks × agents 生成笛卡尔积,= 15 × 4 = 60 trials +# --------------------------------------------------------------------------- +agents: + - name: oracle + model_name: null # oracle 不调 LLM + n_concurrent: 2 + + - name: aider + model_name: "openai/Qwen/Qwen3-Coder-30B-A3B-Instruct" + env: + AIDER_API_KEY: "${OPENAI_API_KEY}" + LITELLM_API_KEY: "${OPENAI_API_KEY}" + LLM_API_KEY: "${OPENAI_API_KEY}" + n_concurrent: 1 + + - name: mini-swe-agent + model_name: "openai/Qwen/Qwen3-Coder-30B-A3B-Instruct" + env: + MSWEA_API_KEY: "${OPENAI_API_KEY}" + LITELLM_API_KEY: "${OPENAI_API_KEY}" + LLM_API_KEY: "${OPENAI_API_KEY}" + n_concurrent: 1 + + - name: openhands-sdk + model_name: "openai/Qwen/Qwen3-Coder-30B-A3B-Instruct" + env: + LLM_API_KEY: "${OPENAI_API_KEY}" + n_concurrent: 1 + +# --------------------------------------------------------------------------- +# 重试(每 trial 失败后跑几次) +# --------------------------------------------------------------------------- +n_attempts: 1 \ No newline at end of file diff --git a/din_integration/configs/swebench_dind_3x3.py b/din_integration/configs/swebench_dind_3x3.py new file mode 100644 index 00000000..a0b5d6c4 --- /dev/null +++ b/din_integration/configs/swebench_dind_3x3.py @@ -0,0 +1,86 @@ +"""SWE-bench DinD 3×3 example config for ais_bench. + +Demonstrates how to invoke the din_integration/ SWE-bench DinD pipeline +via AISBench's task system. Run after pip-installing both ais_bench and +this swebench_dind package: + + pip install ais_bench + cd din_integration && pip install -e . + ais_bench din_integration/configs/swebench_dind_3x3.py + +Note: This config relies on the swebench-dind CLI being on PATH and the +DinD orchestrator container already running. See din_integration/README.md +for setup. +""" +from mmengine.config import read_base + +# Use a no-op task for the standard Infer stage; the SWE-bench DinD pipeline +# doesn't fit AISBench's standard model-inference shape. +with read_base(): + infer = dict(runner=dict(task=dict(type="EmptyTask"))) + +# Eval stage uses our custom SwebenchDindTask. +eval = dict( + runner=dict( + task=dict( + type="SwebenchDindTask", + ), + ), +) + +# Summarizer: HarborSummarizer reads /results//.json +summarizer = dict(attr="accuracy", type="HarborSummarizer") + +work_dir = "./outputs/swebench_dind_3x3/" + +# 3 cases × 3 agents = 9 trials +models = [ + dict( + abbr="qwen3-coder-30b", + type="LiteLLMModel", + model_names=["openai/Qwen/Qwen3-Coder-30B-A3B-Instruct"], + agent_name="aider", + agent_kwargs={}, + agent_env={}, + ), + dict( + abbr="qwen3-coder-30b", + type="LiteLLMModel", + model_names=["openai/Qwen/Qwen3-Coder-30B-A3B-Instruct"], + agent_name="mini-swe-agent", + agent_kwargs={}, + agent_env={}, + ), + dict( + abbr="qwen3-coder-30b", + type="LiteLLMModel", + model_names=["openai/Qwen/Qwen3-Coder-30B-A3B-Instruct"], + agent_name="qwen-coder", + agent_kwargs={}, + agent_env={}, + ), +] + +datasets = [ + dict( + abbr="django-11099", + type="SwebenchDindDataset", + args=dict( + path="/opt/swebench/data/tasks/django__django-11099-aider", + ), + ), + dict( + abbr="django-12308", + type="SwebenchDindDataset", + args=dict( + path="/opt/swebench/data/tasks/django__django-12308-msa", + ), + ), + dict( + abbr="django-13741", + type="SwebenchDindDataset", + args=dict( + path="/opt/swebench/data/tasks/django__django-13741-aider", + ), + ), +] \ No newline at end of file diff --git a/din_integration/docs/CLI-USAGE.md b/din_integration/docs/CLI-USAGE.md new file mode 100644 index 00000000..5573a9e1 --- /dev/null +++ b/din_integration/docs/CLI-USAGE.md @@ -0,0 +1,311 @@ +# CLI Usage Reference + +Full reference for every `swebench-dind` subcommand. The CLI is built with +[Typer](https://typer.tiangolo.com/), so `swebench-dind --help` +always shows the live, in-sync option list. + +## Top-level + +```text +swebench-dind [--version] [--help] +swebench-dind ... +``` + +Subcommands: + +| Command | Purpose | +|---|---| +| `orchestrator` | DinD container lifecycle (start / stop / status) | +| `build` | Build L1 / L2 baked images | +| `launch` | Launch trials (single, 3×3, or N×M matrix) | +| `watch` | Block until a named job's `result.json` is finished | +| `summarize` | Aggregate `jobs/*/result.json` into md / csv / json | +| `patch` | Patch Harbor for idempotent installs | +| `aisbench` | AISBench integration (P1) | + +--- + +## `orchestrator` + +Manages the `swebench-orchestrator` DinD container. Equivalent to the legacy +`scripts/start_orchestrator.sh` and `scripts/stop_orchestrator.sh`. + +### `orchestrator start` + +```text +swebench-dind orchestrator start [--recreate] +``` + +- Idempotent: if the container is already running with `dockerd` ready, + prints status and exits. +- `--recreate` removes the existing container first (data is preserved on + the host because we bind-mount `jobs/`, `tasks/`, `logs/`, etc.). +- Wait up to 120 s for `dockerd` inside the container to come up. +- The OpenAI API key is read from `mini_matrix/scripts/api_key.env`. + +### `orchestrator stop` + +```text +swebench-dind orchestrator stop [--remove] +``` + +- `docker stop` by default. Job data on the host is untouched. +- `--remove` also `docker rm -f`s the container. + +### `orchestrator status` + +```text +swebench-dind orchestrator status +``` + +Reports: + +- container name +- whether the container exists +- whether it's running +- whether `docker info` inside the container succeeds (i.e. DinD dockerd ready) + +Example: + +``` +swebench-orchestrator + exists: True + running: True + dockerd: ✅ ready +``` + +--- + +## `build` + +Builds the L1 case-base and L2 agent-baked images. Both live **inside** +the orchestrator's Docker daemon (DinD), not on the host, so +`image_exists()` shells out via `docker exec`. + +### `build l1` + +```text +swebench-dind build l1 --case 11099 --case 12308 [--force] +``` + +- Required: one or more `--case` numbers. +- Skips if `swebench/django-{case}-base:latest` already exists + (idempotent by default). +- `--force` rebuilds even if the image is present. +- Renders `Dockerfile.l1-base.j2` with `BASE_IMAGE=prebuilt_image(case)` + and tags the result. + +### `build l2` + +```text +swebench-dind build l2 --agent qwen-code [--case 11099] [--force] +``` + +- Required: one or more `--agent` names. +- `--case` defaults to `DEFAULT_CASES` = `[11099, 12308, 13741]`. +- Builds `swebench/django-{case}-with-{agent}:latest` on top of the + matching L1 base. +- Each agent has its own Jinja2 template under + `swebench_dind/dockerfiles/Dockerfile.l2-agent-*.j2`. + +### `build all` + +```text +swebench-dind build all [--case ...] [--agent ...] [--force] +``` + +- Defaults: all 6 cases (3 default + 3 new) and the 3 default agents. +- Sequentially builds every L1 + every L2 in the cartesian product. + +--- + +## `launch` + +Drives `harbor jobs start` inside the orchestrator. Every trial becomes +one `docker exec` invocation, and the job directory is cleaned of stale +state first (so re-runs work). + +### `launch trial` + +```text +swebench-dind launch trial --case 11099 --agent aider \ + [--job-name custom-name] [-n 1] [-m MODEL] [--api-base URL] \ + [--wait] [--timeout-min 120] +``` + +- `--job-name` defaults to `"{agent}-{case}"` (e.g. `aider-11099`). +- `-n` is the harbor trial-count (almost always `1` for SWE-bench). +- `-m` defaults to `openai/Qwen/Qwen3-Coder-30B-A3B-Instruct`. +- `--api-base` defaults to `https://api.siliconflow.cn/v1`. +- `--wait` blocks until `result.json` reports `finished_at`. +- Per-stage timeout multipliers are set to **4×** by default (so the + heavy QEMU x86_64 emulation has enough headroom). +- Agent-specific env (e.g. `AIDER_API_KEY`, `OPENAI_BASE_URL`) is set + automatically via `--ae` flags; see `config.AGENT_AE`. + +### `launch 3x3` + +```text +swebench-dind launch 3x3 [--wait] [--timeout-min 120] +``` + +- Hard-coded to `DEFAULT_CASES` × `DEFAULT_AGENTS` = 3 × 3 = 9 trials. +- All jobs are launched via `subprocess.Popen`, so they run in parallel + inside the orchestrator. +- With `--wait`, each one is polled in order until it finishes. + +### `launch matrix` + +```text +swebench-dind launch matrix [--case ...] [--agent ...] [--wait] [--timeout-min 120] +``` + +- Same as `launch 3x3` but the case/agent lists are configurable. + +--- + +## `watch` + +```text +swebench-dind watch +``` + +Polls `jobs//result.json` inside the orchestrator every 30 s +until it reports `finished_at`, or times out at 24 h. Useful for +re-attaching to a trial that you launched in another shell. + +--- + +## `summarize` + +```text +swebench-dind summarize [--jobs-dir JOBS_DIR] [--output-dir OUTPUT_DIR] [--include SUBSTR,...] +``` + +- Scans `JOBS_DIR` (default `/home/zengziyu/mini_matrix/jobs`) for + `*/result.json`. +- Reads `stats.evals[*].metrics[0].mean` for pass@1. +- Outputs `summary-.{md,csv,json}` into `OUTPUT_DIR` + (default `/home/zengziyu/mini_matrix/logs`). +- `--include` accepts a comma-separated substring filter on job names, + e.g. `--include aider,qwen-code` keeps only those agents. + +Output schema mirrors the legacy `scripts/summarize.py` — the md table +is keyed by `(case, agent)` with a count + mean column. + +--- + +## `patch` + +### `patch harbor` + +```text +swebench-dind patch harbor [--agent qwen-code] [--agent aider] +``` + +- Injects an idempotent install probe into Harbor's installed agent + module inside the orchestrator: + + ```python + probe = await environment.exec( + command="command -v >/dev/null 2>&1 && --version || echo not-found" + ) + if probe.return_code == 0: + return # already baked; skip install + ``` + +- This means **re-runs of a trial on an L2-baked image skip the slow + `pip install` / `npm install`** — the heavy lifting happens once at + bake time. +- Idempotent: the patch checks for the + `"Idempotent probe (PATCHED by swebench-dind CLI)"` marker and is a + no-op if already present. +- Cleans up the matching `.pyc` cache after writing. + +| `--agent` key | Harbor module | CLI binary probed | +|---|---|---| +| `aider` | `aider.py` | `aider` | +| `mini-swe-agent` | `mini_swe_agent.py` | `mini-swe-agent` | +| `qwen-code` | `qwen_code.py` | `qwen` | +| `openhands-sdk` | `openhands_sdk.py` | `openhands` | + +--- + +## `aisbench` (P1) + +### `aisbench install` + +```text +swebench-dind aisbench install +``` + +- Symlinks `swebench_dind/aisbench_adapter/` → + `~/aisbench/runtime/swebench_dind/` (falls back to a recursive copy + if symlink fails). +- Writes an example config at `~/aisbench/configs/swebench_dind_3x3.py`. +- Idempotent: existing symlinks are reported and left in place. + +### `aisbench run` + +```text +swebench-dind aisbench run --config configs/swebench_dind_3x3.py +``` + +Thin wrapper that calls `ais_bench ` after the adapter has +been installed. + +### `aisbench result-format` + +```text +swebench-dind aisbench result-format +``` + +Prints the AISBench-required schema docstring (see +`swebench_dind/aisbench_adapter/result_writer.py::SCHEMA_DOC`) for +quick reference. + +--- + +## Environment & paths + +These come from `swebench_dind/config.py`: + +| Variable / path | Default | Used by | +|---|---|---| +| `CONTAINER_NAME` | `swebench-orchestrator` | `docker exec`, `docker rm` | +| `ORCHESTRATOR_IMAGE` | `swebench/orchestrator:v0.1-2026-08-04-patched-v11` | `orchestrator start` | +| `JOBS_DIR` | `/home/zengziyu/mini_matrix/jobs` | bind mount, `summarize` | +| `TASKS_DIR` | `/home/zengziyu/mini_matrix/tasks` | bind mount | +| `LOGS_DIR` | `/home/zengziyu/mini_matrix/logs` | bind mount, `summarize` output | +| `API_KEY_ENV` | `/home/zengziyu/mini_matrix/scripts/api_key.env` | `OPENAI_API_KEY` source | +| `DEFAULT_MODEL` | `openai/Qwen/Qwen3-Coder-30B-A3B-Instruct` | launch trial default | +| `DEFAULT_API_BASE` | `https://api.siliconflow.cn/v1` | launch trial default | +| `DEFAULT_CASES` | `[11099, 12308, 13741]` | launch 3x3 / matrix default | +| `DEFAULT_AGENTS` | `[aider, mini-swe-agent, qwen-code]` | launch 3x3 / matrix default | + +--- + +## Typical end-to-end session + +```bash +cd /home/zengziyu/mini_matrix/cli +source .venv/bin/activate + +# Bring up the orchestrator (idempotent) +swebench-dind orchestrator start + +# One-time: bake images (idempotent; skips existing) +swebench-dind build all + +# One-time: patch Harbor so trial setup skips re-installing baked agents +swebench-dind patch harbor --agent qwen-code --agent aider --agent mini-swe-agent + +# Run the default 3×3 matrix and wait for completion +swebench-dind launch 3x3 --wait --timeout-min 60 + +# Aggregate results +swebench-dind summarize +``` + +To add a new case or agent: edit `swebench_dind/config.py` only. All +downstream commands pick it up. \ No newline at end of file diff --git a/din_integration/docs/MIGRATION.md b/din_integration/docs/MIGRATION.md new file mode 100644 index 00000000..efb4e45e --- /dev/null +++ b/din_integration/docs/MIGRATION.md @@ -0,0 +1,103 @@ +# Migration Guide: old scripts → `swebench-dind` + +The old workflow lived under `/home/zengziyu/mini_matrix/scripts/` and +`/home/zengziyu/mini_matrix/jobs/launch_*.sh`. They still work, but the +`swebench-dind` CLI is now the canonical entry point and the only one we +add features to. + +This page is a 1-to-1 mapping from each old script to the equivalent +`git`-able command. + +## Top-level scripts (mini_matrix/scripts/) + +| Old script | Equivalent CLI | +|---|---| +| `scripts/start_orchestrator.sh` | `swebench-dind orchestrator start` | +| `scripts/start_orchestrator.sh --recreate` | `swebench-dind orchestrator start --recreate` | +| `scripts/stop_orchestrator.sh` | `swebench-dind orchestrator stop` | +| `scripts/stop_orchestrator.sh rm` | `swebench-dind orchestrator stop --remove` | +| `scripts/summarize.py` | `swebench-dind summarize` | +| `scripts/summarize.sh` | `swebench-dind summarize` | +| `scripts/simple_summary.py` | `swebench-dind summarize` (md/csv/json in one pass) | +| `scripts/show_results.py` | `swebench-dind summarize --include ` | +| `scripts/filter_matrix.py` | `swebench-dind summarize --include ` | +| `scripts/patch_qwen_code.py` | `swebench-dind patch harbor --agent qwen-code` | +| `scripts/monitor_3x3.sh` | `swebench-dind watch ` (per-trial) or loop `watch` over `launch 3x3` | +| `scripts/build_baked_v2.sh` | `swebench-dind build all` | +| `scripts/build_baked_images.sh` | `swebench-dind build l1 --case ...` + `build l2 --agent ...` | +| `scripts/run_3x3_matrix.sh` | `swebench-dind launch 3x3 --wait` | +| `scripts/run_matrix.sh` | `swebench-dind launch matrix --case ... --agent ...` | + +## Per-agent launchers (mini_matrix/jobs/) + +These were the most duplicated scripts — one per agent, each repeating the +`COMMON_AE` / `AIDER_AE` / `MSWEA_AE` env blocks. Now there is one +command that handles every supported agent: + +| Old script | Equivalent CLI | +|---|---| +| `jobs/launch_aider_only.sh` | `swebench-dind launch trial --case X --agent aider` | +| `jobs/launch_msa_only.sh` | `swebench-dind launch trial --case X --agent mini-swe-agent` | +| `jobs/launch_qwen.sh` | `swebench-dind launch trial --case X --agent qwen-code` | +| `jobs/launch_oh.sh` | `swebench-dind launch trial --case X --agent openhands-sdk` | +| `jobs/launch_3new.sh` | `swebench-dind launch matrix --case 10097 --case 10554 --case 10880 --agent aider` | +| `jobs/launch_v2_test.sh` | `swebench-dind launch trial --case 11099 --agent aider --job-name v2-test-11099` | + +If you want to reproduce the exact `docker exec` command line that the +old scripts used, pass `--help` to `launch trial` and add any extra +`--ae` flags you had (the CLI already sets `OPENAI_API_BASE`, +`OPENAI_API_KEY`, `AIDER_API_KEY`, etc. for you). + +## Hardcoded values that used to live in many places + +| Was scattered across | Now in | +|---|---| +| `COMMON_AE_KEYS` (6 launchers) | `config.COMMON_AE_KEYS` | +| `AIDER_AE`, `MSWEA_AE`, `QWEN_AE`, `OH_AE` | `config.AGENT_AE` (one dict, all agents) | +| `--agent qwen-coder` (Harbor flag) | `config.HARBOR_AGENT_FLAG["qwen-code"] = "qwen-coder"` | +| Image tags (`swebench/django-{case}-with-{agent}:latest`) | `config.l2_image_tag()` | +| Model / API base / API key path | `config.DEFAULT_MODEL`, `DEFAULT_API_BASE`, `API_KEY_ENV` | +| Timeout multipliers (`--agent-timeout-multiplier 4`) | `config.DEFAULT_MULTIPLIERS` | +| Task directory naming (`django__django-11099-aider`) | `config.task_dir_name()` | + +To add a new case: append to `DEFAULT_CASES` (or `NEW_CASES`). To add a +new agent: add entries to `AGENT_AE`, `HARBOR_AGENT_FLAG`, +`DOCKERFILE_L2_BY_AGENT`, and (if it ships as a Harbor module) the +`patcher.AGENT_TO_MODULE` / `patcher.AGENT_CLI` dicts. + +## Status / health checks + +| Old habit | Equivalent CLI | +|---|---| +| `docker ps \| grep swebench-orchestrator` | `swebench-dind orchestrator status` | +| `docker exec swebench-orchestrator docker info` | `swebench-dind orchestrator status` | +| `tail -f jobs//verifier.log` (manual polling) | `swebench-dind watch ` | + +## Removing the old scripts + +The plan is to keep the old scripts on disk for now (they have many +small in-place tweaks over months of debugging). If you want to clean +them up after switching: + +1. Verify the CLI covers all the commands you actually run by reading + through `docs/CLI-USAGE.md`. +2. For any one-off script that isn't on the mapping table, file a + follow-up task; we don't want to silently drop functionality. +3. Add `# DEPRECATED: use swebench-dind ` headers to the surviving + old scripts and remove them in a separate commit. + +## Why we built the CLI + +After 12 trials (9 PASS / 75%) the workflow had stabilized into four +repeating steps. Each new case or agent meant copy-pasting a launch +script, mutating four env-var blocks, and re-reading every shell file +to find the right multiplier. The CLI collapses that into: + +```bash +swebench-dind build l2 --agent --case +swebench-dind launch trial --case --agent +``` + +with the model, API base, and timeout multipliers all coming from +`config.py`. New (case, agent) combinations stop being a copy-paste +exercise. \ No newline at end of file diff --git a/din_integration/pyproject.toml b/din_integration/pyproject.toml new file mode 100644 index 00000000..c2a897a8 --- /dev/null +++ b/din_integration/pyproject.toml @@ -0,0 +1,38 @@ +[build-system] +requires = ["setuptools>=68", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "swebench-dind" +version = "0.1.0" +description = "Unified CLI for SWE-bench DinD multi-agent trials" +readme = "README.md" +requires-python = ">=3.10" +license = {text = "MIT"} +authors = [{name = "zengziyu"}] + +dependencies = [ + "typer>=0.12", + "rich>=13", + "jinja2>=3.1", +] + +[project.optional-dependencies] +dev = [ + "pytest>=8", + "pytest-cov", +] + +[project.scripts] +swebench-dind = "swebench_dind.cli:app" + +[tool.setuptools.packages.find] +where = ["."] +include = ["swebench_dind*"] + +[tool.setuptools.package-data] +swebench_dind = ["dockerfiles/*.j2"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +python_files = ["test_*.py"] \ No newline at end of file diff --git a/din_integration/scripts/api_key.env.template b/din_integration/scripts/api_key.env.template new file mode 100644 index 00000000..7c42200f --- /dev/null +++ b/din_integration/scripts/api_key.env.template @@ -0,0 +1,14 @@ +# SWE-bench DinD API key 配置模板 +# +# 复制本文件到 ~/.config/swebench-dind/api_key.env 并填入真实 key: +# mkdir -p ~/.config/swebench-dind +# cp api_key.env.template ~/.config/swebench-dind/api_key.env +# $EDITOR ~/.config/swebench-dind/api_key.env +# +# 注意: 本包当前 (2026-08-12) 的 launcher.py 仍硬编码路径 +# /home/zengziyu/mini_matrix/scripts/api_key.env +# 如使用本模板位置 ~/.config/swebench-dind/api_key.env, +# 需先 ln -s 或直接 cp 到 mini_matrix 路径。 + +export OPENAI_API_KEY="sk-REPLACE-ME" +export OPENAI_API_BASE="https://api.siliconflow.cn/v1" \ No newline at end of file diff --git a/din_integration/scripts/filter_matrix.py b/din_integration/scripts/filter_matrix.py new file mode 100755 index 00000000..69a8d1be --- /dev/null +++ b/din_integration/scripts/filter_matrix.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +""" +filter_matrix.py — 根据 --datasets / --agents 过滤 matrix.yaml,生成子集 yaml。 + +不修改 matrix.yaml,只读它、过滤、写到 --output。 +""" +import argparse +import sys +from pathlib import Path + +try: + import yaml +except ImportError: + print("PyYAML not installed. pip install pyyaml", file=sys.stderr) + sys.exit(1) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--input", required=True, help="Source matrix.yaml") + p.add_argument("--output", required=True, help="Filtered output yaml") + p.add_argument("--datasets", default="", help="Comma-separated dataset path basenames to keep") + p.add_argument("--agents", default="", help="Comma-separated agent names to keep") + args = p.parse_args() + + cfg = yaml.safe_load(Path(args.input).read_text()) + + if args.datasets: + keep = {d.strip() for d in args.datasets.split(",") if d.strip()} + cfg["datasets"] = [ + ds for ds in cfg.get("datasets", []) + if Path(ds["path"]).name in keep + ] + + if args.agents: + keep = {a.strip() for a in args.agents.split(",") if a.strip()} + cfg["agents"] = [ + a for a in cfg.get("agents", []) + if a["name"] in keep + ] + + Path(args.output).write_text(yaml.safe_dump(cfg, sort_keys=False, allow_unicode=True)) + n_d = len(cfg.get("datasets", [])) + n_a = len(cfg.get("agents", [])) + print(f"[filter_matrix] {n_d} datasets × {n_a} agents = {n_d * n_a} trials") + print(f"[filter_matrix] Written: {args.output}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/din_integration/scripts/setup.sh b/din_integration/scripts/setup.sh new file mode 100755 index 00000000..8f65bc23 --- /dev/null +++ b/din_integration/scripts/setup.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# ============================================================================ +# setup.sh — SWE-bench DinD host-side setup helper +# ============================================================================ +# Creates the runtime directories and symlinks expected by the +# swebench-dind CLI. Idempotent. +# +# Usage: +# bash din_integration/scripts/setup.sh +# ============================================================================ +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +echo "[setup] creating host runtime directories under \$HOME..." +mkdir -p "$HOME/swebench_dind_jobs" +mkdir -p "$HOME/swebench_dind_tasks" +mkdir -p "$HOME/swebench_dind_logs" +mkdir -p "$HOME/.config/swebench-dind" + +echo "[setup] expected api_key.env location: $HOME/.config/swebench-dind/api_key.env" +if [[ ! -f "$HOME/.config/swebench-dind/api_key.env" ]]; then + echo "[setup] WARNING: $HOME/.config/swebench-dind/api_key.env not found." + echo " Copy $ROOT/scripts/api_key.env.template to that location and edit." +fi + +echo "[setup] checking legacy launcher.py path expectation..." +LEGACY_DIR="/home/zengziyu/mini_matrix/scripts" +if [[ -d "/home/zengziyu/mini_matrix" ]]; then + if [[ ! -f "$LEGACY_DIR/api_key.env" ]]; then + echo "[setup] WARNING: launcher.py currently hardcodes $LEGACY_DIR/api_key.env" + echo " Either copy your key there, or wait for env-var refactor." + fi +fi + +echo "[setup] done. Next:" +echo " cd $ROOT && pip install -e ." +echo " bash $ROOT/scripts/start_orchestrator.sh" \ No newline at end of file diff --git a/din_integration/scripts/start_orchestrator.sh b/din_integration/scripts/start_orchestrator.sh new file mode 100755 index 00000000..b1151ecd --- /dev/null +++ b/din_integration/scripts/start_orchestrator.sh @@ -0,0 +1,112 @@ +#!/bin/bash +# ============================================================================ +# start_orchestrator.sh — 启动 DinD orchestrator 容器(bind mount 到 host) +# ============================================================================ +# +# 关键设计: +# - 所有状态(jobs / tasks / scripts / api_key.env)全部 bind mount 到 host +# - 容器是无状态的:重建容器 = 数据零丢失 +# - harbor 0.20.x 在容器内跑,产物自动落到 /opt/swebench/jobs(已 mount) +# +# 用法: +# bash scripts/start_orchestrator.sh # 启动(若已存在则报错) +# bash scripts/start_orchestrator.sh --recreate # 删除旧容器后重建(数据保留) +# +# ============================================================================ +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" + +ORCHESTRATOR_IMAGE="${ORCHESTRATOR_IMAGE:-swebench/orchestrator:v0.1-2026-08-04-patched-v11}" +CONTAINER_NAME="${CONTAINER_NAME:-swebench-orchestrator}" +DATA_CONTAINER_NAME="${DATA_CONTAINER_NAME:-swebench-data-3demo}" +DATA_IMAGE="${DATA_IMAGE:-swebench/swebench-data:v0.1-2026-07-30-3demo}" + +RECREATE=0 +for arg in "$@"; do + case "$arg" in + --recreate) RECREATE=1 ;; + *) echo "Unknown arg: $arg"; exit 2 ;; + esac +done + +# --- 前置:秘钥文件必须存在 --- +if [[ ! -f "$ROOT/scripts/api_key.env" ]]; then + echo "[ERROR] $ROOT/scripts/api_key.env not found. Copy from orchestrator or other host first." + exit 1 +fi + +# --- 前置:jobs 目录必须存在(mount target) --- +mkdir -p "$ROOT/jobs" "$ROOT/tasks" "$ROOT/logs" + +# --- 旧容器处理 --- +if docker ps -a --format '{{.Names}}' | grep -qx "$CONTAINER_NAME"; then + if [[ $RECREATE -eq 1 ]]; then + echo "[start_orchestrator] Removing old container $CONTAINER_NAME (data is on host, safe)" + docker rm -f "$CONTAINER_NAME" >/dev/null + else + echo "[start_orchestrator] Container $CONTAINER_NAME already exists. Use --recreate to replace." + echo " (Hint: docker exec -it $CONTAINER_NAME bash)" + exit 0 + fi +fi + +# --- 数据卷容器(只读,3 demo task 兜底) --- +if ! docker ps -a --format '{{.Names}}' | grep -qx "$DATA_CONTAINER_NAME"; then + echo "[start_orchestrator] Creating data container $DATA_CONTAINER_NAME from $DATA_IMAGE" + docker create --name "$DATA_CONTAINER_NAME" "$DATA_IMAGE" +fi + +# --- 加载 API key --- +set -a +# shellcheck disable=SC1091 +source "$ROOT/scripts/api_key.env" +set +a + +if [[ -z "${OPENAI_API_KEY:-}" ]]; then + echo "[ERROR] OPENAI_API_KEY is empty in scripts/api_key.env" + exit 1 +fi + +# --- 启动 orchestrator --- +echo "[start_orchestrator] Starting $CONTAINER_NAME from $ORCHESTRATOR_IMAGE" +docker run -d \ + --name "$CONTAINER_NAME" \ + --hostname orchestrator \ + --privileged \ + --cgroupns=host \ + --restart unless-stopped \ + --volumes-from "$DATA_CONTAINER_NAME":ro \ + \ + -v "$ROOT/jobs":/opt/swebench/jobs:rw \ + -v "$ROOT/tasks":/opt/swebench/data/tasks:rw \ + -v "$ROOT/config":/opt/swebench/config:ro \ + -v "$ROOT/scripts/api_key.env":/opt/swebench/api_key.env:ro \ + -v "$ROOT/orchestrator/entrypoint.sh":/opt/swebench/scripts/entrypoint.sh:ro \ + -v "$ROOT/orchestrator/agent-patches":/opt/swebench/agent-patches:ro \ + -v "$ROOT/logs":/opt/swebench/logs:rw \ + \ + -e OPENAI_API_KEY="$OPENAI_API_KEY" \ + -e OPENAI_API_BASE="${OPENAI_API_BASE:-https://api.siliconflow.cn/v1}" \ + \ + "$ORCHESTRATOR_IMAGE" \ + bash -c 'tail -f /dev/null' + +# 等 DinD ready +echo "[start_orchestrator] Waiting for DinD dockerd to be ready..." +for i in {1..60}; do + if docker exec "$CONTAINER_NAME" docker info >/dev/null 2>&1; then + echo "[start_orchestrator] dockerd ready" + break + fi + sleep 2 +done + +echo "[start_orchestrator] Done. Container: $CONTAINER_NAME" +echo " - jobs: $ROOT/jobs → /opt/swebench/jobs" +echo " - tasks: $ROOT/tasks → /opt/swebench/data/tasks" +echo " - config: $ROOT/config → /opt/swebench/config" +echo " - patches: $ROOT/orchestrator/agent-patches → /opt/swebench/agent-patches" +echo "" +echo "Next: bash scripts/run_matrix.sh" \ No newline at end of file diff --git a/din_integration/scripts/summarize.py b/din_integration/scripts/summarize.py new file mode 100755 index 00000000..6073ed2d --- /dev/null +++ b/din_integration/scripts/summarize.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +""" +summarize.py — 扫 jobs//result.json → pass@1 by (bench, agent) 表 + +输出: + logs/summary-.md + logs/summary-.csv + logs/summary-.json (raw) +""" +import argparse +import csv +import json +import sys +from collections import defaultdict +from datetime import datetime +from pathlib import Path + + +def load_jobs(jobs_dir: Path, include: list[str] | None = None): + """Load all jobs//result.json + config.json. Optionally filter by name substring.""" + rows = [] + for job_dir in sorted(jobs_dir.iterdir()): + if not job_dir.is_dir(): + continue + result = job_dir / "result.json" + if not result.exists(): + continue + if include and not any(inc in job_dir.name for inc in include): + continue + try: + data = json.loads(result.read_text()) + except json.JSONDecodeError: + continue + # Try to load config.json (separate from result.json) + config = {} + cfg_path = job_dir / "config.json" + if cfg_path.exists(): + try: + config = json.loads(cfg_path.read_text()) + except json.JSONDecodeError: + pass + rows.append(parse_job(job_dir.name, data, config)) + return rows + + +def parse_job(job_name: str, data: dict, config: dict) -> dict: + """Extract benchmark / agent / reward from harbor JobResult.""" + agents = config.get("agents", []) + tasks = config.get("tasks", []) + datasets = config.get("datasets", []) + + # Single agent (矩阵常见情况) + agent_name = agents[0]["name"] if agents else "?" + + # Task or dataset path → benchmark 标识 + if tasks: + bench = Path(tasks[0]["path"]).name + elif datasets: + bench = Path(datasets[0]["path"]).name + else: + bench = "?" + + stats = data.get("stats", {}) + n_total = stats.get("n_total_trials", 0) + n_completed = stats.get("n_completed_trials", 0) + n_errored = stats.get("n_errored_trials", 0) + n_running = stats.get("n_running_trials", 0) + n_pending = stats.get("n_pending_trials", 0) + finished = data.get("finished_at") + + # Reward from evals: reward_stats.reward is {reward_value: [trial_ids]} + evals = stats.get("evals", {}) + rewards = [] + for eval_key, eval_data in evals.items(): + reward_buckets = eval_data.get("reward_stats", {}).get("reward", {}) + for r_str, trial_ids in reward_buckets.items(): + try: + r_val = float(r_str) + except (ValueError, TypeError): + continue + n_with_reward = len(trial_ids) if isinstance(trial_ids, list) else 1 + rewards.extend([r_val] * n_with_reward) + + n_pass = sum(1 for r in rewards if r == 1.0) + pass_at_1 = n_pass / max(len(rewards), 1) + + return { + "job_name": job_name, + "agent": agent_name, + "benchmark": bench, + "n_total": n_total, + "n_completed": n_completed, + "n_errored": n_errored, + "n_running": n_running, + "n_pending": n_pending, + "n_pass": n_pass, + "pass_at_1": pass_at_1, + "finished_at": finished, + "started_at": data.get("started_at"), + } + + +def write_markdown(rows: list[dict], path: Path): + """Write a Markdown summary.""" + # group by (bench, agent) + grouped = defaultdict(list) + for r in rows: + grouped[(r["benchmark"], r["agent"])].append(r) + + with path.open("w") as f: + f.write(f"# Multi-Bench × Multi-Agent Summary\n\n") + f.write(f"_Generated: {datetime.now().isoformat()}_\n\n") + f.write(f"**Total jobs**: {len(rows)}\n\n") + + # Pass@1 by (bench, agent) + f.write("## Pass@1 by (Benchmark, Agent)\n\n") + f.write("| Benchmark | Agent | n_jobs | n_pass | pass@1 |\n") + f.write("|---|---|---|---|---|\n") + for (bench, agent), items in sorted(grouped.items()): + n_jobs = len(items) + n_pass = sum(it["n_pass"] for it in items) + p1 = n_pass / max(n_jobs, 1) + f.write(f"| {bench} | {agent} | {n_jobs} | {n_pass} | {p1:.1%} |\n") + + # Detailed rows + f.write("\n## All Jobs\n\n") + f.write("| Job | Agent | Bench | n_total | n_done | n_err | pass@1 | finished |\n") + f.write("|---|---|---|---|---|---|---|---|\n") + for r in sorted(rows, key=lambda x: (x["benchmark"], x["agent"], x["job_name"])): + done = "✅" if r["finished_at"] else "🔄" + f.write( + f"| `{r['job_name']}` | {r['agent']} | {r['benchmark']} | " + f"{r['n_total']} | {r['n_completed']} | {r['n_errored']} | " + f"{r['pass_at_1']:.0%} | {r['finished_at'] or '—'} {done} |\n" + ) + + +def write_csv(rows: list[dict], path: Path): + """Write CSV summary.""" + with path.open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0].keys()) if rows else []) + writer.writeheader() + for r in rows: + writer.writerow(r) + + +def main(): + p = argparse.ArgumentParser() + p.add_argument("--jobs-dir", required=True, type=Path) + p.add_argument("--include", default=None, help="Comma-separated substrings to filter jobs") + p.add_argument("--output-dir", required=True, type=Path) + args = p.parse_args() + + include = [s.strip() for s in (args.include or "").split(",") if s.strip()] or None + rows = load_jobs(args.jobs_dir, include=include) + if not rows: + print("[summarize] No jobs found.", file=sys.stderr) + sys.exit(0) + + ts = datetime.now().strftime("%Y%m%d-%H%M%S") + md_path = args.output_dir / f"summary-{ts}.md" + csv_path = args.output_dir / f"summary-{ts}.csv" + json_path = args.output_dir / f"summary-{ts}.json" + + write_markdown(rows, md_path) + write_csv(rows, csv_path) + json_path.write_text(json.dumps(rows, indent=2, default=str)) + + print(f"[summarize] {len(rows)} jobs summarized") + print(f" → {md_path}") + print(f" → {csv_path}") + print(f" → {json_path}") + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/din_integration/swebench_dind/__init__.py b/din_integration/swebench_dind/__init__.py new file mode 100644 index 00000000..36ce03e7 --- /dev/null +++ b/din_integration/swebench_dind/__init__.py @@ -0,0 +1,3 @@ +"""SWE-bench DinD unified CLI + AISBench adapter.""" + +__version__ = "0.1.0" \ No newline at end of file diff --git a/din_integration/swebench_dind/__main__.py b/din_integration/swebench_dind/__main__.py new file mode 100644 index 00000000..ded4a6da --- /dev/null +++ b/din_integration/swebench_dind/__main__.py @@ -0,0 +1,5 @@ +"""Allow ``python -m swebench_dind``.""" +from swebench_dind.cli import app + +if __name__ == "__main__": + app() \ No newline at end of file diff --git a/din_integration/swebench_dind/aisbench_adapter/__init__.py b/din_integration/swebench_dind/aisbench_adapter/__init__.py new file mode 100644 index 00000000..c54b70e3 --- /dev/null +++ b/din_integration/swebench_dind/aisbench_adapter/__init__.py @@ -0,0 +1,15 @@ +"""AISBench P1 adapter for SWE-bench DinD. + +Implements: +- ``SwebenchDindTask`` (subclass of AISBench ``BaseTask``) +- ``HarborTaskCompat`` (translates harbor JobResult → AISBench schema) +- ``install()`` (symlinks this package into aisbench's runtime/) +""" +from .install import install +from .result_writer import ( + write_result, + read_harbor_result, + SCHEMA_DOC, +) + +__all__ = ["install", "write_result", "read_harbor_result", "SCHEMA_DOC"] \ No newline at end of file diff --git a/din_integration/swebench_dind/aisbench_adapter/result_writer.py b/din_integration/swebench_dind/aisbench_adapter/result_writer.py new file mode 100644 index 00000000..79162493 --- /dev/null +++ b/din_integration/swebench_dind/aisbench_adapter/result_writer.py @@ -0,0 +1,134 @@ +"""Translate harbor ``result.json`` → AISBench-compatible schema. + +AISBench (via HarborSummarizer / AgentContribSummarizer) reads +``/results//.json`` with the schema +documented in ``SCHEMA_DOC``. +""" +from __future__ import annotations + +import json +from collections import Counter +from pathlib import Path +from typing import Any + +SCHEMA_DOC = """ +AISBench results//.json schema: +{ + "total_count": int, # total number of trials + "n_errors": int, # trials that errored out + "avg_score": float, # mean reward (4 decimals) + "reward_keys": list[str], # optional, for multi-key verifiers + "per_key_avg_score": dict, # optional + "reward_distribution": [{"score": float, "count": int}, ...], + "exception_distribution": [{"exception_type": str, "count": int}, ...], + "n_total_trials": int, + "pass_at_k": dict[int, float] # optional +} +""" + + +def read_harbor_result(path: Path) -> dict[str, Any]: + """Read a harbor ``result.json``.""" + return json.loads(path.read_text()) + + +def _flatten_rewards(evals: dict[str, Any]) -> list[float]: + rewards: list[float] = [] + for _, eval_data in evals.items(): + buckets = eval_data.get("reward_stats", {}).get("reward", {}) + for r_str, trial_ids in buckets.items(): + try: + r_val = float(r_str) + except (ValueError, TypeError): + continue + n = len(trial_ids) if isinstance(trial_ids, list) else 1 + rewards.extend([r_val] * n) + return rewards + + +def write_result( + harbor_result_path: Path, + output_path: Path, + *, + model_abbr: str, + dataset_abbr: str, +) -> Path: + """Convert harbor result.json → AISBench result JSON. Returns output path.""" + data = read_harbor_result(harbor_result_path) + stats = data.get("stats", {}) + evals = stats.get("evals", {}) + rewards = _flatten_rewards(evals) + n_total = stats.get("n_total_trials", 0) + n_errors = stats.get("n_errored_trials", 0) + avg_score = round(sum(rewards) / max(len(rewards), 1), 4) if rewards else 0.0 + + # Reward distribution + reward_counter = Counter(rewards) + reward_distribution = [ + {"score": float(s), "count": int(c)} for s, c in sorted(reward_counter.items()) + ] + + # Exception distribution + exception_counter: Counter[str] = Counter() + for _, eval_data in evals.items(): + for ex_type, trial_ids in eval_data.get("exception_stats", {}).items(): + n = len(trial_ids) if isinstance(trial_ids, list) else 1 + exception_counter[ex_type] += n + exception_distribution = [ + {"exception_type": t, "count": int(c)} for t, c in exception_counter.most_common() + ] + + # pass@1 + pass_at_k = {} + if rewards: + pass_at_k[1] = round(sum(1 for r in rewards if r == 1.0) / len(rewards), 4) + + out = { + "total_count": n_total, + "n_errors": n_errors, + "avg_score": avg_score, + "reward_keys": ["reward"] if rewards else [], + "per_key_avg_score": {"reward": avg_score} if rewards else {}, + "reward_distribution": reward_distribution, + "exception_distribution": exception_distribution, + "n_total_trials": n_total, + "pass_at_k": pass_at_k, + # AISBench extras (not used by HarborSummarizer but useful for debugging) + "_meta": { + "model_abbr": model_abbr, + "dataset_abbr": dataset_abbr, + "harbor_finished_at": data.get("finished_at"), + }, + } + + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(json.dumps(out, indent=2)) + return output_path + + +def write_results_for_all_jobs( + jobs_dir: Path, + output_dir: Path, + *, + model_abbr: str = "default-model", + dataset_abbr_finder=None, +) -> list[Path]: + """Iterate ``jobs_dir/*/result.json`` and write AISBench result files. + + ``dataset_abbr_finder(job_dir) -> str`` lets callers derive the + AISBench dataset abbr from the job directory (default: job_dir.name). + """ + outputs: list[Path] = [] + finder = dataset_abbr_finder or (lambda p: p.name) + for job_dir in sorted(p for p in jobs_dir.iterdir() if p.is_dir()): + result = job_dir / "result.json" + if not result.exists(): + continue + ds = finder(job_dir) + out = output_dir / model_abbr / f"{ds}.json" + try: + write_result(result, out, model_abbr=model_abbr, dataset_abbr=ds) + outputs.append(out) + except (json.JSONDecodeError, KeyError) as e: + print(f" [skip] {job_dir.name}: {e}") + return outputs \ No newline at end of file diff --git a/din_integration/swebench_dind/aisbench_adapter/runner.py b/din_integration/swebench_dind/aisbench_adapter/runner.py new file mode 100644 index 00000000..46dd9bad --- /dev/null +++ b/din_integration/swebench_dind/aisbench_adapter/runner.py @@ -0,0 +1,135 @@ +"""Subprocess entrypoint for SwebenchDindTask. + +Invoked by ``aisbench.LocalRunner`` as: + python -m swebench_dind.aisbench_adapter.runner --config --work-dir + +Reads the cfg, runs one harbor trial (via ``launcher.launch_trial``), +waits for result.json, converts it to AISBench schema via +``result_writer.write_result``, and writes: + /results//.json + /results///details/ +""" +from __future__ import annotations + +import argparse +import shutil +import sys +import time +from pathlib import Path + +from rich.console import Console + +from ..config import DEFAULT_API_BASE, DEFAULT_MODEL, JOBS_DIR +from ..launcher import _build_harbor_args, _cleanup_job_dir, _load_api_key, wait_for_job +from .result_writer import write_result + +console = Console() + + +def main(cfg: dict | None = None, task_state_manager=None) -> None: + """Entry point used by both BaseTask.run() (subprocess) and direct CLI.""" + parser = argparse.ArgumentParser(prog="swebench_dind.aisbench_adapter.runner") + parser.add_argument("--config", required=True, help="Path to ais_bench config.py") + parser.add_argument("--work-dir", required=True) + parser.add_argument("--model-index", type=int, default=0) + parser.add_argument("--dataset-index", type=int, default=0) + args = parser.parse_args() + + # Lazy import mmengine (only needed in subprocess context) + from mmengine.config import Config + cfg_obj = Config.fromfile(args.config) + + models = cfg_obj.get("models", []) + datasets = cfg_obj.get("datasets", []) + if not models or not datasets: + console.print("[red]config must define models and datasets[/red]") + sys.exit(1) + + model_cfg = models[args.model_index] + dataset_cfg = datasets[args.dataset_index] + work_dir = Path(args.work_dir) + work_dir.mkdir(parents=True, exist_ok=True) + + # Translate cfg → launch args + agent = model_cfg.get("agent_name", "aider") + model_name = (model_cfg.get("model_names") or [DEFAULT_MODEL])[0] + api_key = model_cfg.get("api_key") or _load_api_key() + api_base = cfg_obj.get("api_base", DEFAULT_API_BASE) + + dataset_args = dataset_cfg.get("args", {}) + task_path = Path(dataset_args.get("path", "")) + # Parse path: .../django__django-{case}-{agent} + case, _, agent_in_path = task_path.name.partition("django__django-")[2].partition("-") + if not case: + console.print(f"[red]could not parse case from path {task_path}[/red]") + sys.exit(1) + # Honor agent from path if not in cfg + if agent_in_path and not model_cfg.get("agent_name"): + agent = agent_in_path + + job_name = f"aisbench-{agent}-{case}" + _cleanup_job_dir(job_name) + + multipliers = { + "agent_setup": 4, + "agent": 4, + "verifier": 4, + "environment_build": 4, + } + + from ..launcher import LaunchSpec # local import (avoids circular) + spec = LaunchSpec( + case=case, + agent=agent, + job_name=job_name, + model=model_name, + api_base=api_base, + n=1, + multipliers=multipliers, + extra_ae=[], + ) + args_list = _build_harbor_args(spec, api_key) + + console.print(f"[bold]launch[/bold] {job_name}") + import subprocess + subprocess.Popen( + ["docker", "exec", "-e", "OPENAI_API_KEY", "-e", "OPENAI_API_BASE", + spec.container_name, *args_list] if hasattr(spec, "container_name") + else ["docker", "exec", "-e", "OPENAI_API_KEY", "-e", "OPENAI_API_BASE", + "swebench-orchestrator", *args_list], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, + ) + + if task_state_manager is not None: + task_state_manager.update_task_state({"status": "running", "finish_count": 0}) + + try: + data = wait_for_job(job_name, timeout_min=24 * 60) + except TimeoutError as e: + console.print(f"[red]{e}[/red]") + sys.exit(1) + + # Write AISBench result.json + harbor_result_path = JOBS_DIR / job_name / "result.json" + model_abbr = model_cfg.get("summarizer_abbr") or model_cfg.get("abbr", "default") + dataset_abbr = dataset_cfg.get("abbr", task_path.name) + out_path = work_dir / "results" / model_abbr / f"{dataset_abbr}.json" + write_result(harbor_result_path, out_path, model_abbr=model_abbr, dataset_abbr=dataset_abbr) + + # Copy details (Harbor convention) + details_src = JOBS_DIR / job_name + details_dst = work_dir / "results" / model_abbr / dataset_abbr / "details" + if details_src.exists(): + details_dst.mkdir(parents=True, exist_ok=True) + for f in details_src.iterdir(): + if f.is_file(): + shutil.copy2(f, details_dst / f.name) + + console.print(f"[green]✅ {job_name} → {out_path}[/green]") + + if task_state_manager is not None: + task_state_manager.update_task_state({"status": "done", "finish_count": 1}) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/din_integration/swebench_dind/aisbench_adapter/task.py b/din_integration/swebench_dind/aisbench_adapter/task.py new file mode 100644 index 00000000..e444a4c8 --- /dev/null +++ b/din_integration/swebench_dind/aisbench_adapter/task.py @@ -0,0 +1,140 @@ +"""AISBench ``BaseTask`` subclass that wraps a SWE-bench DinD trial. + +Loaded by aisbench when ``task.type="SwebenchDindTask"`` is set in a +config. The class: + +- Inherits ``BaseTask`` from ais_bench so it auto-registers with the + ``TASKS`` registry (when ``@TASKS.register_module()`` is applied). +- Implements ``get_command(cfg_path)`` which returns the shell command + aisbench's ``LocalRunner`` should spawn as a subprocess. +- The subprocess entrypoint is ``swebench_dind.aisbench_adapter.runner``, + which reads the same ``cfg_path``, parses ``model_cfg`` and + ``dataset_cfg``, runs the harbor trial via the in-container + ``swebench-dind launch`` CLI, then writes the AISBench-format + ``/results//.json``. + +NOTE: This module assumes aisbench is installed (so we can import +``BaseTask``). If aisbench is not on PYTHONPATH, the ``@register`` +decorator is a no-op so the module still imports. +""" +from __future__ import annotations + +from pathlib import Path + +try: + from ais_bench.benchmark.tasks.base import BaseTask + from ais_bench.benchmark.registry import TASKS + _AISBENCH_AVAILABLE = True +except ImportError: + _AISBENCH_AVAILABLE = False + + class BaseTask: # type: ignore[no-redef] + """Fallback stub so the module is importable without aisbench.""" + name_prefix = "swebench_dind" + log_subdir = "logs/eval" + output_subdir = "results" + + def __init__(self, cfg): + self.cfg = cfg + + def get_command(self, cfg_path, template=None): + raise NotImplementedError + + def run(self, task_state_manager): + raise NotImplementedError + + class _StubRegistry: + @staticmethod + def register_module(): + def decorator(cls): + return cls + return decorator + + TASKS = _StubRegistry() + + +@TASKS.register_module() +class SwebenchDindTask(BaseTask): + """BaseTask wrapper that runs a single harbor trial.""" + + name_prefix = "swebench_dind" + log_subdir = "logs/eval" + output_subdir = "results" + + def get_command(self, cfg_path: str, template=None) -> str: + """Return the subprocess command for LocalRunner to spawn.""" + # The actual work happens in the subprocess; ``runner.py`` reads + # ``cfg_path`` and calls ``launcher.launch_trial`` programmatically. + return ( + f"python -m swebench_dind.aisbench_adapter.runner " + f"--config {cfg_path} " + f"--work-dir {self.cfg.get('work_dir', './outputs/')}" + ) + + def run(self, task_state_manager): + """In subprocess: invoke ``runner.main()`` and report progress.""" + from .runner import main as runner_main + runner_main(self.cfg, task_state_manager) + + +# === Helper for AISBench dataset registration === +try: + from ais_bench.benchmark.datasets.base import BaseDataset + from ais_bench.benchmark.registry import DATASETS + _DATASET_AVAILABLE = True +except ImportError: + _DATASET_AVAILABLE = False + + class BaseDataset: # type: ignore[no-redef] + def __init__(self, path=None): + self.path = path + + DATASETS = _StubRegistry() + + +@DATASETS.register_module() +class SwebenchDindDataset(BaseDataset): + """Minimal dataset that points at our local task directory. + + AISBench expects ``SwebenchDindDataset`` to expose a list of task + records (one per (case, agent)). For the simple 3×3 use case we + auto-derive 3 records from the path naming convention. + """ + + def __init__(self, path: str = "", **kwargs): + super().__init__(path=path) + self._records = self._parse_records(Path(path)) + + def _parse_records(self, path: Path) -> list[dict]: + """One record per subdirectory of ``path`` matching our convention. + + Path format expected: ``.../django__django-{case}-{agent}/task.toml`` + """ + records: list[dict] = [] + if not path.exists(): + return records + for sub in sorted(path.parent.iterdir()): + if not sub.is_dir() or not sub.name.startswith("django__django-"): + continue + # django__django-11099-aider → case=11099, agent=aider + try: + _, _, case_agent = sub.name.partition("django__django-") + case, _, agent = case_agent.partition("-") + except ValueError: + continue + toml = sub / "task.toml" + if not toml.exists(): + continue + records.append({ + "case": case, + "agent": agent, + "task_path": str(sub), + "task_toml": str(toml), + }) + return records + + def __len__(self) -> int: + return len(self._records) + + def __iter__(self): + return iter(self._records) \ No newline at end of file diff --git a/din_integration/swebench_dind/builder.py b/din_integration/swebench_dind/builder.py new file mode 100644 index 00000000..60a81aa8 --- /dev/null +++ b/din_integration/swebench_dind/builder.py @@ -0,0 +1,164 @@ +"""L1/L2 baked image builder. + +Renders Jinja2 Dockerfile templates and runs ``docker build`` to produce: +- L1: ``swebench/django-{case}-base:latest`` +- L2: ``swebench/django-{case}-with-{agent}:latest`` +""" +from __future__ import annotations + +import subprocess +import tempfile +from pathlib import Path + +from jinja2 import Environment, FileSystemLoader +from rich.console import Console + +from .config import ( + BUILD_LOG_DIR, + DOCKERFILE_L1, + DOCKERFILES_DIR, + base_image_tag, + dockerfile_for_agent, + l2_image_tag, + prebuilt_image, +) + +console = Console() + + +def _docker(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run( + ["docker", *args], + capture_output=True, + text=True, + check=check, + ) + + +def image_exists(tag: str, *, in_orchestrator: bool = True) -> bool: + """Check whether a docker image tag exists. + + SWE-bench DinD images live INSIDE the orchestrator container + (``swebench-orchestrator``), not on the host. So by default we run + ``docker images`` via ``docker exec`` on the orchestrator. + + Set ``in_orchestrator=False`` to check the host's image list instead. + """ + if in_orchestrator: + from .container import exec_in_orchestrator + try: + out = exec_in_orchestrator( + "docker", "images", "--format", "{{.Repository}}:{{.Tag}}", check=False + ).stdout + except subprocess.CalledProcessError: + return False + else: + out = _docker("images", "--format", "{{.Repository}}:{{.Tag}}", check=False).stdout + return tag in out.splitlines() + + +def _render(template_name: str, **context: str) -> str: + env = Environment( + loader=FileSystemLoader(str(DOCKERFILES_DIR)), + keep_trailing_newline=True, + trim_blocks=False, + ) + tmpl = env.get_template(template_name) + return tmpl.render(**context) + + +def _render_to_tempfile(template_name: str, **context: str) -> tuple[Path, Path]: + """Render the template and write to a temp dir alongside a placeholder + context file. Returns (build_context_dir, dockerfile_path).""" + rendered = _render(template_name, **context) + ctx_dir = Path(tempfile.mkdtemp(prefix="swebench-dind-build-")) + (ctx_dir / "Dockerfile").write_text(rendered) + return ctx_dir, ctx_dir / "Dockerfile" + + +def _build_image( + *, + tag: str, + dockerfile: Path, + context: Path, + build_args: dict[str, str], + log_path: Path, + platform: str = "linux/amd64", +) -> bool: + BUILD_LOG_DIR.mkdir(parents=True, exist_ok=True) + cmd = [ + "docker", "build", + "--platform", platform, + *[f"--build-arg={k}={v}" for k, v in build_args.items()], + "-f", str(dockerfile), + "-t", tag, + str(context), + ] + console.print(f" [docker build] {tag}") + with log_path.open("w") as logf: + result = subprocess.run(cmd, stdout=logf, stderr=subprocess.STDOUT, text=True) + return result.returncode == 0 + + +def build_l1(case: str, *, skip_existing: bool = True, force: bool = False) -> bool: + """Build the L1 case-base image for ``case``. + + Returns True on success (or if skipped because already present). + """ + tag = base_image_tag(case) + if skip_existing and image_exists(tag) and not force: + console.print(f" [L1 skip] {tag} already exists") + return True + + ctx, dockerfile = _render_to_tempfile(DOCKERFILE_L1, BASE_IMAGE=prebuilt_image(case)) + log = BUILD_LOG_DIR / f"l1-{case}.log" + return _build_image( + tag=tag, + dockerfile=dockerfile, + context=ctx, + build_args={"BASE_IMAGE": prebuilt_image(case)}, + log_path=log, + ) + + +def build_l2(case: str, agent: str, *, skip_existing: bool = True, force: bool = False) -> bool: + """Build the L2 agent-baked image for (case, agent).""" + template_name = dockerfile_for_agent(agent) + if template_name is None: + console.print(f" [L2 skip] no Dockerfile template for agent={agent!r}") + return False + tag = l2_image_tag(case, agent) + if skip_existing and image_exists(tag) and not force: + console.print(f" [L2 skip] {tag} already exists") + return True + ctx, dockerfile = _render_to_tempfile(template_name, AGENT=agent) + log = BUILD_LOG_DIR / f"l2-{case}-{agent}.log" + return _build_image( + tag=tag, + dockerfile=dockerfile, + context=ctx, + build_args={ + "BASE_IMAGE": base_image_tag(case), + "AGENT": agent, + }, + log_path=log, + ) + + +def build_l1_all(cases: list[str], *, skip_existing: bool = True, force: bool = False) -> dict[str, bool]: + """Build L1 images for all cases (sequentially).""" + results = {} + for c in cases: + results[c] = build_l1(c, skip_existing=skip_existing, force=force) + return results + + +def build_l2_all( + cases: list[str], agents: list[str], *, skip_existing: bool = True, force: bool = False, +) -> dict[tuple[str, str], bool]: + """Build L2 images for all (case, agent) pairs (sequentially).""" + results = {} + for c in cases: + for a in agents: + results[(c, a)] = build_l2(c, a, skip_existing=skip_existing, force=force) + return results \ No newline at end of file diff --git a/din_integration/swebench_dind/cli.py b/din_integration/swebench_dind/cli.py new file mode 100644 index 00000000..5a025f54 --- /dev/null +++ b/din_integration/swebench_dind/cli.py @@ -0,0 +1,242 @@ +"""Typer app exposing the 7 subcommands. + +Use as a script: ``swebench-dind [options]`` (installed entry +point) or ``python -m swebench_dind ``. +""" +from __future__ import annotations + +from pathlib import Path +from typing import Optional + +import typer +from rich.console import Console + +from . import __version__ +from .config import ( + ALL_AGENTS, + DEFAULT_AGENTS, + DEFAULT_CASES, + JOBS_DIR, + LOGS_DIR, + NEW_CASES, +) + +app = typer.Typer( + name="swebench-dind", + help="SWE-bench DinD multi-agent trial orchestration CLI", + no_args_is_help=True, + add_completion=False, +) +console = Console() + + +def _version_callback(value: bool) -> None: + if value: + console.print(f"swebench-dind {__version__}") + raise typer.Exit() + + +@app.callback() +def main( + version: bool = typer.Option( + False, "--version", "-V", callback=_version_callback, is_eager=True, + help="Show version and exit.", + ), +) -> None: + """SWE-bench DinD CLI root.""" + + +# === orchestrator sub-app === +orch_app = typer.Typer(help="DinD container lifecycle") +app.add_typer(orch_app, name="orchestrator") + + +@orch_app.command("start") +def orch_start( + recreate: bool = typer.Option(False, "--recreate", help="Remove existing container and recreate."), +) -> None: + """Start the DinD orchestrator container.""" + from .container import start + start(recreate=recreate) + + +@orch_app.command("stop") +def orch_stop( + remove: bool = typer.Option(False, "--remove", help="Also remove the container (data preserved on host)."), +) -> None: + """Stop the orchestrator container.""" + from .container import stop + stop(remove=remove) + + +@orch_app.command("status") +def orch_status() -> None: + """Show orchestrator container status.""" + from .container import print_status + print_status() + + +# === build sub-app === +build_app = typer.Typer(help="Build L1/L2 baked images") +app.add_typer(build_app, name="build") + + +@build_app.command("l1") +def build_l1_cmd( + case: list[str] = typer.Option(..., "--case", help="Case number(s), e.g. 11099"), + force: bool = typer.Option(False, "--force", help="Rebuild even if image exists."), +) -> None: + """Build L1 case-base images.""" + from .builder import build_l1 + for c in case: + build_l1(c, force=force) + + +@build_app.command("l2") +def build_l2_cmd( + agent: list[str] = typer.Option(..., "--agent", help="Agent name(s)"), + case: list[str] = typer.Option(DEFAULT_CASES, "--case", help="Case number(s)"), + force: bool = typer.Option(False, "--force"), +) -> None: + """Build L2 agent-baked images.""" + from .builder import build_l2 + for a in agent: + for c in case: + build_l2(c, a, force=force) + + +@build_app.command("all") +def build_all_cmd( + case: list[str] = typer.Option(DEFAULT_CASES + NEW_CASES, "--case"), + agent: list[str] = typer.Option(DEFAULT_AGENTS, "--agent"), + force: bool = typer.Option(False, "--force"), +) -> None: + """Build all L1 + L2 baked images.""" + from .builder import build_l1_all, build_l2_all + build_l1_all(case, force=force) + build_l2_all(case, agent, force=force) + + +# === launch sub-app === +launch_app = typer.Typer(help="Launch trials") +app.add_typer(launch_app, name="launch") + + +@launch_app.command("trial") +def launch_trial_cmd( + case: str = typer.Option(..., "--case"), + agent: str = typer.Option(..., "--agent"), + job_name: Optional[str] = typer.Option(None, "--job-name"), + n: int = typer.Option(1, "-n"), + model: str = typer.Option("openai/Qwen/Qwen3-Coder-30B-A3B-Instruct", "-m"), + api_base: str = typer.Option("https://api.siliconflow.cn/v1", "--api-base"), + wait: bool = typer.Option(False, "--wait", help="Block until job finishes."), + timeout_min: int = typer.Option(120, "--timeout-min"), +) -> None: + """Launch a single (case, agent) trial.""" + from .launcher import launch_trial + launch_trial( + case, agent, + job_name=job_name, n=n, model=model, api_base=api_base, + wait=wait, timeout_min=timeout_min, + ) + + +@launch_app.command("3x3") +def launch_3x3_cmd( + wait: bool = typer.Option(False, "--wait"), + timeout_min: int = typer.Option(120, "--timeout-min"), +) -> None: + """Launch the default 3×3 matrix (3 cases × 3 agents).""" + from .launcher import launch_3x3 + launch_3x3(wait=wait, timeout_min=timeout_min) + + +@launch_app.command("matrix") +def launch_matrix_cmd( + case: list[str] = typer.Option(DEFAULT_CASES, "--case"), + agent: list[str] = typer.Option(DEFAULT_AGENTS, "--agent"), + wait: bool = typer.Option(False, "--wait"), + timeout_min: int = typer.Option(120, "--timeout-min"), +) -> None: + """Launch an N×M matrix of trials.""" + from .launcher import launch_matrix + launch_matrix(case, agent, wait=wait, timeout_min=timeout_min) + + +# === watch (top-level) === +@app.command() +def watch(job_name: str = typer.Argument(...)) -> None: + """Block until a job's result.json is finished.""" + from .launcher import wait_for_job + try: + data = wait_for_job(job_name, timeout_min=24 * 60) + console.print(f"[green]✅ {job_name} done: {data.get('finished_at')}[/green]") + except TimeoutError as e: + console.print(f"[red]❌ {e}[/red]") + raise typer.Exit(1) + + +# === summarize (top-level) === +@app.command() +def summarize( + jobs_dir: Path = typer.Option(JOBS_DIR, "--jobs-dir"), + output_dir: Path = typer.Option(LOGS_DIR, "--output-dir"), + include: Optional[str] = typer.Option(None, "--include", help="Comma-separated substring filter."), +) -> None: + """Aggregate results.json → summary-.{md,csv,json}.""" + from .summarizer import summarize as _summarize + inc = [s.strip() for s in (include or "").split(",") if s.strip()] or None + out = _summarize(jobs_dir=jobs_dir, output_dir=output_dir, include=inc) + console.print(f"[green]✅ {len(out['rows'])} jobs summarized[/green]") + console.print(f" → {out['md']}") + console.print(f" → {out['csv']}") + console.print(f" → {out['json']}") + + +# === patch sub-app === +patch_app = typer.Typer(help="Patch harbor for idempotent installs") +app.add_typer(patch_app, name="patch") + + +@patch_app.command("harbor") +def patch_harbor_cmd( + agent: list[str] = typer.Option(["qwen-code"], "--agent", help="Agent(s) to patch."), +) -> None: + """Patch harbor's installed agent module to add idempotent install probe.""" + from .patcher import patch_agent + for a in agent: + patch_agent(a) + + +# === aisbench sub-app === +aisbench_app = typer.Typer(help="AISBench integration (P1)") +app.add_typer(aisbench_app, name="aisbench") + + +@aisbench_app.command("install") +def aisbench_install() -> None: + """Symlink the aisbench_adapter into aisbench's runtime/.""" + from .aisbench_adapter import install as _install + _install() + + +@aisbench_app.command("run") +def aisbench_run( + config: Path = typer.Option(..., "--config", help="Path to ais_bench config.py"), +) -> None: + """Run the AISBench CLI with the installed swebench_dind task adapter.""" + import subprocess + console.print(f"[bold]ais_bench[/bold] {config}") + subprocess.run(["ais_bench", str(config)], check=False) + + +@aisbench_app.command("result-format") +def aisbench_result_format() -> None: + """Print the AISBench-compatible result schema docstring.""" + from .aisbench_adapter.result_writer import SCHEMA_DOC + console.print(SCHEMA_DOC) + + +if __name__ == "__main__": + app() \ No newline at end of file diff --git a/din_integration/swebench_dind/config.py b/din_integration/swebench_dind/config.py new file mode 100644 index 00000000..b7ad3a19 --- /dev/null +++ b/din_integration/swebench_dind/config.py @@ -0,0 +1,164 @@ +"""Single source of truth for SWE-bench DinD pipeline parameters. + +All hardcoded values that were scattered across scripts (model name, image +tags, multipliers, API endpoints, etc.) live here so that adding a new +case/agent requires changing exactly one file. +""" +from __future__ import annotations + +import os +from pathlib import Path +from typing import Iterable + + +# === Paths === +CLI_ROOT = Path(__file__).resolve().parent.parent # /home/zengziyu/mini_matrix/cli +ROOT = CLI_ROOT.parent # /home/zengziyu/mini_matrix + +CONTAINER_NAME = os.environ.get("CONTAINER_NAME", "swebench-orchestrator") +DATA_CONTAINER_NAME = "swebench-data-3demo" + +ORCHESTRATOR_IMAGE = os.environ.get( + "ORCHESTRATOR_IMAGE", + "swebench/orchestrator:v0.1-2026-08-04-patched-v11", +) +DATA_IMAGE = "swebench/swebench-data:v0.1-2026-07-30-3demo" + +JOBS_DIR = ROOT / "jobs" +TASKS_DIR = ROOT / "tasks" +CONFIG_DIR = ROOT / "config" +LOGS_DIR = ROOT / "logs" +API_KEY_ENV = ROOT / "scripts" / "api_key.env" + +ORCH_DIR = ROOT / "orchestrator" +ENTRYPOINT_SH = ORCH_DIR / "entrypoint.sh" +AGENT_PATCHES_DIR = ORCH_DIR / "agent-patches" +IMAGE_CACHE_DIR = ORCH_DIR / "image-cache" + +# Inside the orchestrator container +CONTAINER_TASKS_DIR = "/opt/swebench/data/tasks" +CONTAINER_JOBS_DIR = "/opt/swebench/jobs" +CONTAINER_AGENT_PATCHES_DIR = "/opt/swebench/agent-patches" + + +# === Model & API === +DEFAULT_MODEL = "openai/Qwen/Qwen3-Coder-30B-A3B-Instruct" +DEFAULT_API_BASE = "https://api.siliconflow.cn/v1" +DEFAULT_BARE_MODEL = "Qwen/Qwen3-Coder-30B-A3B-Instruct" + + +# === Default cases / agents === +DEFAULT_CASES: list[str] = ["11099", "12308", "13741"] +NEW_CASES: list[str] = ["10097", "10554", "10880"] +ALL_CASES: list[str] = DEFAULT_CASES + NEW_CASES + +DEFAULT_AGENTS: list[str] = ["aider", "mini-swe-agent", "qwen-code"] +ALL_AGENTS: list[str] = ["aider", "mini-swe-agent", "qwen-code", "openhands-sdk"] + + +# === Harbor CLI plumbing === +# What we pass to each `harbor jobs start` as `--ae KEY=VAL` flags. +COMMON_AE_KEYS = [ + "OPENAI_API_BASE", + "OPENAI_API_KEY", + "LITELLM_API_KEY", + "LLM_API_KEY", + "LLM_BASE_URL", + "LLM_MODEL", +] + +# Per-agent extra env (key without value, value substituted by `substitute_ae`) +AGENT_AE: dict[str, list[str]] = { + "aider": ["AIDER_API_KEY=openai=${OPENAI_API_KEY}"], + "mini-swe-agent": ["MSWEA_API_KEY=${OPENAI_API_KEY}"], + "qwen-code": [ + "OPENAI_BASE_URL=https://api.siliconflow.cn/v1", + "OPENAI_MODEL=Qwen/Qwen3-Coder-30B-A3B-Instruct", + ], + "openhands-sdk": [ + "OPENAI_BASE_URL=https://api.siliconflow.cn/v1", + "LLM_MODEL=openai/Qwen/Qwen3-Coder-30B-A3B-Instruct", + ], +} + +# Harbor CLI flag for picking the agent +HARBOR_AGENT_FLAG: dict[str, str] = { + "aider": "aider", + "mini-swe-agent": "mini-swe-agent", + "qwen-code": "qwen-coder", + "openhands-sdk": "openhands-sdk", +} + +# Default timeout multipliers (matches scripts/launch_*.sh) +DEFAULT_MULTIPLIERS = { + "agent_setup": 4, + "agent": 4, + "verifier": 4, + "environment_build": 4, +} + + +# === Image tags === +# Base SWE-bench prebuilt image template. +# e.g. docker.1ms.run/swebench/sweb.eval.x86_64.django_1776_django-11099:latest +BASE_PREBUILT_TEMPLATE = ( + "docker.1ms.run/swebench/sweb.eval.x86_64.django_1776_django-{case}:latest" +) + + +def prebuilt_image(case: str) -> str: + """Return the upstream SWE-bench prebuilt image for ``case``.""" + return BASE_PREBUILT_TEMPLATE.format(case=case) + + +def base_image_tag(case: str) -> str: + """L1 case-base image tag, e.g. ``swebench/django-11099-base:latest``.""" + return f"swebench/django-{case}-base:latest" + + +def l2_image_tag(case: str, agent: str) -> str: + """L2 agent-baked image tag, e.g. ``swebench/django-11099-with-aider:latest``.""" + return f"swebench/django-{case}-with-{agent}:latest" + + +# === Dockerfile template names === +DOCKERFILE_L1 = "Dockerfile.l1-base.j2" +DOCKERFILE_L2_BY_AGENT: dict[str, str] = { + "aider": "Dockerfile.l2-agent-aider.j2", + "mini-swe-agent": "Dockerfile.l2-agent-msa.j2", + "qwen-code": "Dockerfile.l2-agent-qwen.j2", + "openhands-sdk": "Dockerfile.l2-agent-oh.j2", +} + + +# === Build === +# Bundled templates + context dir for `docker build` +DOCKERFILES_DIR = Path(__file__).parent / "dockerfiles" +BUILD_LOG_DIR = Path("/tmp/swebench-dind-builds") + + +# === Helpers === +def task_dir_name(case: str, agent: str) -> str: + """Return the task directory name, e.g. ``django__django-11099-aider``.""" + return f"django__django-{case}-{agent}" + + +def container_task_path(case: str, agent: str) -> str: + """Return the container-side task path passed to ``harbor jobs start``.""" + return f"{CONTAINER_TASKS_DIR}/{task_dir_name(case, agent)}" + + +def substitute_ae(values: Iterable[str], api_key: str, model: str, api_base: str) -> list[str]: + """Replace ``${OPENAI_API_KEY}`` etc. in agent env strings.""" + subs = {"${OPENAI_API_KEY}": api_key, "${MODEL}": model, "${API_BASE}": api_base} + out = [] + for v in values: + for k, sub in subs.items(): + v = v.replace(k, sub) + out.append(v) + return out + + +def dockerfile_for_agent(agent: str) -> str | None: + """Map agent → L2 Dockerfile template name.""" + return DOCKERFILE_L2_BY_AGENT.get(agent) \ No newline at end of file diff --git a/din_integration/swebench_dind/container.py b/din_integration/swebench_dind/container.py new file mode 100644 index 00000000..764014b5 --- /dev/null +++ b/din_integration/swebench_dind/container.py @@ -0,0 +1,178 @@ +"""Orchestrator (DinD container) lifecycle management. + +Wraps the logic of legacy ``scripts/start_orchestrator.sh`` and +``scripts/stop_orchestrator.sh`` into Python functions. +""" +from __future__ import annotations + +import os +import subprocess +import time +from pathlib import Path + +from rich.console import Console + +from .config import ( + AGENT_PATCHES_DIR, + API_KEY_ENV, + CONFIG_DIR, + CONTAINER_NAME, + DATA_CONTAINER_NAME, + DATA_IMAGE, + ENTRYPOINT_SH, + JOBS_DIR, + LOGS_DIR, + ORCHESTRATOR_IMAGE, + TASKS_DIR, +) + +console = Console() + + +def _docker(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run( + ["docker", *args], + capture_output=True, + text=True, + check=check, + ) + + +def _container_exists(name: str) -> bool: + out = _docker("ps", "-a", "--format", "{{.Names}}", check=False).stdout + return name in out.splitlines() + + +def _container_running(name: str) -> bool: + out = _docker("ps", "--format", "{{.Names}}", check=False).stdout + return name in out.splitlines() + + +def status() -> dict: + """Report orchestrator container + DinD dockerd health.""" + exists = _container_exists(CONTAINER_NAME) + running = _container_running(CONTAINER_NAME) + dockerd_ready = False + if running: + try: + _docker("exec", CONTAINER_NAME, "docker", "info", check=True) + dockerd_ready = True + except subprocess.CalledProcessError: + dockerd_ready = False + return { + "container": CONTAINER_NAME, + "exists": exists, + "running": running, + "dockerd_ready": dockerd_ready, + } + + +def print_status() -> None: + s = status() + console.print(f"[bold]{s['container']}[/bold]") + console.print(f" exists: {s['exists']}") + console.print(f" running: {s['running']}") + console.print(f" dockerd: {'✅ ready' if s['dockerd_ready'] else '❌ not ready'}") + + +def _load_api_key() -> str: + if not API_KEY_ENV.exists(): + raise FileNotFoundError(f"{API_KEY_ENV} missing; copy from another host") + for line in API_KEY_ENV.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#") or "=" not in line: + continue + k, _, v = line.partition("=") + if k.strip() == "OPENAI_API_KEY": + return v.strip().strip('"').strip("'") + raise RuntimeError("OPENAI_API_KEY not found in api_key.env") + + +def _ensure_dirs() -> None: + for d in (JOBS_DIR, TASKS_DIR, LOGS_DIR): + d.mkdir(parents=True, exist_ok=True) + + +def start(recreate: bool = False) -> None: + """Start the DinD orchestrator container. + + Equivalent to ``bash scripts/start_orchestrator.sh [--recreate]``. + Idempotent: if container already running, prints status and exits. + """ + _ensure_dirs() + + if _container_exists(CONTAINER_NAME): + if not recreate: + s = status() + if s["running"] and s["dockerd_ready"]: + console.print(f"[green]✅ {CONTAINER_NAME} already running[/green]") + return + console.print(f"[yellow]⚠ {CONTAINER_NAME} exists but not ready[/yellow]") + console.print(" Use --recreate to remove and recreate.") + return + console.print(f"[yellow]Removing old container {CONTAINER_NAME}[/yellow]") + _docker("rm", "-f", CONTAINER_NAME, check=False) + + if not _container_exists(DATA_CONTAINER_NAME): + console.print(f"Creating data container {DATA_CONTAINER_NAME} from {DATA_IMAGE}") + _docker("create", "--name", DATA_CONTAINER_NAME, DATA_IMAGE) + + api_key = _load_api_key() + api_base = os.environ.get("OPENAI_API_BASE", "https://api.siliconflow.cn/v1") + + console.print(f"Starting [bold]{CONTAINER_NAME}[/bold] from {ORCHESTRATOR_IMAGE}") + cmd = [ + "docker", "run", "-d", + "--name", CONTAINER_NAME, + "--hostname", "orchestrator", + "--privileged", + "--cgroupns=host", + "--restart", "unless-stopped", + "--volumes-from", f"{DATA_CONTAINER_NAME}:ro", + "-v", f"{JOBS_DIR}:/opt/swebench/jobs:rw", + "-v", f"{TASKS_DIR}:/opt/swebench/data/tasks:rw", + "-v", f"{CONFIG_DIR}:/opt/swebench/config:ro", + "-v", f"{API_KEY_ENV}:/opt/swebench/api_key.env:ro", + "-v", f"{ENTRYPOINT_SH}:/opt/swebench/scripts/entrypoint.sh:ro", + "-v", f"{AGENT_PATCHES_DIR}:/opt/swebench/agent-patches:ro", + "-v", f"{LOGS_DIR}:/opt/swebench/logs:rw", + "-e", f"OPENAI_API_KEY={api_key}", + "-e", f"OPENAI_API_BASE={api_base}", + ORCHESTRATOR_IMAGE, + "bash", "-c", "tail -f /dev/null", + ] + subprocess.run(cmd, check=True) + + # Wait for dockerd + console.print("Waiting for DinD dockerd to be ready...") + for _ in range(60): + try: + _docker("exec", CONTAINER_NAME, "docker", "info", check=True) + break + except subprocess.CalledProcessError: + time.sleep(2) + else: + raise RuntimeError("dockerd did not become ready in 120s") + + console.print(f"[green]✅ {CONTAINER_NAME} ready[/green]") + + +def stop(remove: bool = False) -> None: + """Stop the orchestrator container (data is on host, safe). + + Equivalent to ``bash scripts/stop_orchestrator.sh [stop|rm]``. + """ + if not _container_exists(CONTAINER_NAME): + console.print(f"[yellow]{CONTAINER_NAME} does not exist[/yellow]") + return + if remove: + _docker("rm", "-f", CONTAINER_NAME) + console.print(f"[green]Removed {CONTAINER_NAME}[/green]") + else: + _docker("stop", CONTAINER_NAME) + console.print(f"[green]Stopped {CONTAINER_NAME}[/green]") + + +def exec_in_orchestrator(*args: str, check: bool = True) -> subprocess.CompletedProcess: + """Run a command inside the orchestrator container.""" + return _docker("exec", CONTAINER_NAME, *args, check=check) \ No newline at end of file diff --git a/din_integration/swebench_dind/dockerfiles/Dockerfile.l1-base.j2 b/din_integration/swebench_dind/dockerfiles/Dockerfile.l1-base.j2 new file mode 100644 index 00000000..c730b9c3 --- /dev/null +++ b/din_integration/swebench_dind/dockerfiles/Dockerfile.l1-base.j2 @@ -0,0 +1,6 @@ +# L1 case-base: FROM upstream SWE-bench prebuilt → add /testbed, /logs +ARG BASE_IMAGE +FROM ${BASE_IMAGE} + +WORKDIR /testbed +RUN mkdir -p /logs \ No newline at end of file diff --git a/din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-aider.j2 b/din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-aider.j2 new file mode 100644 index 00000000..8673fbc5 --- /dev/null +++ b/din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-aider.j2 @@ -0,0 +1,12 @@ +ARG BASE_IMAGE +ARG AGENT +FROM ${BASE_IMAGE} +ARG AGENT + +# aider: pip install from aliyun mirror (QEMU-friendly) +RUN if [ "$AGENT" = "aider" ]; then \ + pip install --break-system-packages aider-chat \ + --index-url https://mirrors.aliyun.com/pypi/simple/; \ + else \ + echo "Unknown agent: $AGENT" && exit 1; \ + fi \ No newline at end of file diff --git a/din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-msa.j2 b/din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-msa.j2 new file mode 100644 index 00000000..455b3be7 --- /dev/null +++ b/din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-msa.j2 @@ -0,0 +1,12 @@ +ARG BASE_IMAGE +ARG AGENT +FROM ${BASE_IMAGE} +ARG AGENT + +# mini-swe-agent: pip install from aliyun mirror +RUN if [ "$AGENT" = "mini-swe-agent" ]; then \ + pip install --break-system-packages 'mini-swe-agent[litellm_proxy]' \ + --index-url https://mirrors.aliyun.com/pypi/simple/; \ + else \ + echo "Unknown agent: $AGENT" && exit 1; \ + fi \ No newline at end of file diff --git a/din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-oh.j2 b/din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-oh.j2 new file mode 100644 index 00000000..e68a67c5 --- /dev/null +++ b/din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-oh.j2 @@ -0,0 +1,15 @@ +ARG BASE_IMAGE +ARG AGENT +FROM ${BASE_IMAGE} +ARG AGENT + +# openhands-sdk: uses pre-built venv (see P7 issue: QEMU + Python 3.12 too slow) +# NOTE: OH is currently NOT working under QEMU. Kept here for documentation. +RUN if [ "$AGENT" = "openhands-sdk" ]; then \ + echo "[WARN] openhands-sdk is not reliable under QEMU (Python 3.12 + network)" && \ + echo "[WARN] Kept as no-op so build doesn't fail" && \ + mkdir -p /opt/openhands-sdk-venv && \ + echo "# openhands-sdk disabled (see Dockerfile.l2-agent-oh-v8 for full attempt)" > /opt/openhands-sdk-venv/README; \ + else \ + echo "Unknown agent: $AGENT" && exit 1; \ + fi \ No newline at end of file diff --git a/din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-qwen.j2 b/din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-qwen.j2 new file mode 100644 index 00000000..5e993bfa --- /dev/null +++ b/din_integration/swebench_dind/dockerfiles/Dockerfile.l2-agent-qwen.j2 @@ -0,0 +1,17 @@ +ARG BASE_IMAGE +ARG AGENT +FROM ${BASE_IMAGE} +ARG AGENT + +# qwen-code: npm + Node 22 LTS via NodeSource (case image's apt has Node 12, too old) +RUN if [ "$AGENT" = "qwen-code" ]; then \ + set -e && \ + apt-get update && apt-get install -y curl ca-certificates 2>&1 | tail -3 && \ + curl -fsSL https://deb.nodesource.com/setup_22.x | bash - 2>&1 | tail -3 && \ + apt-get install -y nodejs 2>&1 | tail -3 && \ + node --version && npm --version && \ + npm install -g @qwen-code/qwen-code --registry https://registry.npmmirror.com 2>&1 | tail -10 && \ + which qwen && qwen --version 2>&1; \ + else \ + echo "Unknown agent: $AGENT" && exit 1; \ + fi \ No newline at end of file diff --git a/din_integration/swebench_dind/launcher.py b/din_integration/swebench_dind/launcher.py new file mode 100644 index 00000000..8a228a32 --- /dev/null +++ b/din_integration/swebench_dind/launcher.py @@ -0,0 +1,365 @@ +"""Trial launcher. + +Wraps the legacy ``launch_*.sh`` scripts. Each trial becomes a +``harbor jobs start`` invocation inside the orchestrator container. +""" +from __future__ import annotations + +import json +import os +import subprocess +import time +from dataclasses import dataclass, field +from datetime import datetime +from pathlib import Path +from typing import Iterable + +from rich.console import Console + +from .config import ( + AGENT_AE, + COMMON_AE_KEYS, + CONTAINER_JOBS_DIR, + CONTAINER_NAME, + DEFAULT_API_BASE, + DEFAULT_BARE_MODEL, + DEFAULT_MODEL, + DEFAULT_MULTIPLIERS, + HARBOR_AGENT_FLAG, + JOBS_DIR, + LOGS_DIR, + container_task_path, + substitute_ae, +) +from .container import exec_in_orchestrator + +console = Console() + + +@dataclass +class LaunchSpec: + case: str + agent: str + job_name: str + model: str = DEFAULT_MODEL + api_base: str = DEFAULT_API_BASE + n: int = 1 + multipliers: dict[str, int] = field(default_factory=lambda: dict(DEFAULT_MULTIPLIERS)) + extra_ae: list[str] = field(default_factory=list) + + +def _load_api_key() -> str: + """Read OPENAI_API_KEY from ``scripts/api_key.env``. + + Tolerates both ``OPENAI_API_KEY=sk-...`` and + ``export OPENAI_API_KEY="sk-..."`` forms (the latter is what the + legacy file uses when sourced from a shell). + """ + env_path = Path("/home/zengziyu/mini_matrix/scripts/api_key.env") + if not env_path.exists(): + raise FileNotFoundError(f"{env_path} missing; copy from another host") + for raw in env_path.read_text().splitlines(): + line = raw.strip() + if not line or line.startswith("#"): + continue + # Strip optional `export ` prefix + if line.startswith("export "): + line = line[len("export "):].lstrip() + if "=" not in line: + continue + k, _, v = line.partition("=") + if k.strip() == "OPENAI_API_KEY": + return v.strip().strip('"').strip("'") + raise RuntimeError("OPENAI_API_KEY not set in api_key.env") + + +def _build_harbor_args(spec: LaunchSpec, api_key: str) -> list[str]: + """Build the full ``harbor jobs start`` command-line as a list.""" + args = [ + "harbor", "jobs", "start", + "--job-name", spec.job_name, + "--path", container_task_path(spec.case, spec.agent), + "-a", HARBOR_AGENT_FLAG[spec.agent], + "-m", spec.model, + "--jobs-dir", CONTAINER_JOBS_DIR, + "-n", str(spec.n), + "--agent-setup-timeout-multiplier", str(spec.multipliers["agent_setup"]), + "--agent-timeout-multiplier", str(spec.multipliers["agent"]), + "--verifier-timeout-multiplier", str(spec.multipliers["verifier"]), + ] + + # Common AE (model + api base + api key) + for k in COMMON_AE_KEYS: + if k == "OPENAI_API_KEY": + v = api_key + elif k == "OPENAI_API_BASE": + v = spec.api_base + elif k == "LLM_BASE_URL": + v = spec.api_base + elif k == "LLM_MODEL": + v = spec.model + else: + v = api_key + args += ["--ae", f"{k}={v}"] + + # Agent-specific AE + extras = AGENT_AE.get(spec.agent, []) + for raw in extras: + for s in substitute_ae([raw], api_key=api_key, model=spec.model, api_base=spec.api_base): + args += ["--ae", s] + + # User extras + for e in spec.extra_ae: + args += ["--ae", e.strip()] + return args + + +def _cleanup_job_dir(job_name: str) -> None: + """Remove stale trial subdirs and lock.json to allow re-runs.""" + exec_in_orchestrator( + "bash", "-c", + f""" + if [ -d /opt/swebench/jobs/{job_name} ]; then + find /opt/swebench/jobs/{job_name} -mindepth 1 -maxdepth 1 -type d -exec rm -rf {{}} + + fi + rm -f /opt/swebench/jobs/{job_name}/lock.json 2>/dev/null + """, + check=False, + ) + + +def launch_trial( + case: str, + agent: str, + *, + job_name: str | None = None, + model: str = DEFAULT_MODEL, + api_base: str = DEFAULT_API_BASE, + n: int = 1, + api_key: str | None = None, + extra_ae: list[str] | None = None, + wait: bool = False, + timeout_min: int = 120, + verbose: bool = True, +) -> str: + """Launch a single (case, agent) trial. Returns the job name. + + Prints each framework step to stdout so the user can see what's + happening in real time. + """ + from rich.panel import Panel + from rich.table import Table + + if verbose: + console.print(Panel.fit( + f"[bold]case[/bold] {case} [bold]agent[/bold] {agent} " + f"[bold]model[/bold] {model.split('/')[-1]}", + border_style="cyan", title="SWE-bench DinD trial", + )) + + with console.status("[cyan]loading API key…[/cyan]") if verbose else _nullctx(): + api_key = api_key or _load_api_key() + + job_name = job_name or f"{agent}-{case}" + if verbose: + console.print(f" [green]✓[/green] job name: [bold]{job_name}[/bold]") + + spec = LaunchSpec( + case=case, + agent=agent, + job_name=job_name, + model=model, + api_base=api_base, + n=n, + extra_ae=extra_ae or [], + ) + + with console.status(f"[cyan]cleaning stale job dir {job_name}…[/cyan]") if verbose else _nullctx(): + _cleanup_job_dir(job_name) + if verbose: + console.print(f" [green]✓[/green] cleaned stale state in /opt/swebench/jobs/{job_name}") + + with console.status("[cyan]building harbor args…[/cyan]") if verbose else _nullctx(): + args = _build_harbor_args(spec, api_key) + if verbose: + # show a small table of the harbor flags + t = Table(show_header=False, box=None, padding=(0, 1)) + t.add_column(style="dim") + t.add_column() + for i in range(0, len(args), 2): + if args[i].startswith("--"): + t.add_row(args[i], args[i + 1] if i + 1 < len(args) else "") + else: + t.add_row(args[i], "") + console.print(f" [green]✓[/green] harbor command ({len(args)} tokens):") + console.print(t) + + cmd = [ + "docker", "exec", + "-e", f"OPENAI_API_KEY={api_key}", + "-e", f"OPENAI_API_BASE={spec.api_base}", + CONTAINER_NAME, *args, + ] + if verbose: + console.print(f" [yellow]→[/yellow] docker exec into [bold]{CONTAINER_NAME}[/bold]") + proc = subprocess.Popen(cmd, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + if verbose: + console.print(f" [green]✓[/green] harbor PID {proc.pid} (returns immediately, runs in container)") + + if wait: + result = wait_for_job(job_name, timeout_min=timeout_min, verbose=verbose) + if verbose: + _print_trial_result(result) + return result + return job_name + + +class _nullctx: + """Stand-in for ``console.status`` when ``verbose=False``.""" + + def __enter__(self): + return self + + def __exit__(self, *exc): + return False + + +def _print_trial_result(result: dict) -> None: + from rich.panel import Panel + stats = result.get("stats", {}) + evals = stats.get("evals", {}) + if isinstance(evals, dict): + n_total = len(evals) + scores = [ + (e.get("metrics", [{}])[0].get("mean", 0) if isinstance(e, dict) else 0) + for e in evals.values() + ] + else: # legacy list form + n_total = len(evals) + scores = [ + (e.get("metrics", [{}])[0].get("mean", 0) if isinstance(e, dict) else 0) + for e in evals + ] + n_pass = sum(1 for s in scores if s > 0.5) + score = scores[0] if scores else 0.0 + color = "green" if score > 0.5 else "red" + verdict = "PASS" if score > 0.5 else "FAIL" + n_err = stats.get("n_errored_trials", 0) + n_done = stats.get("n_completed_trials", 0) + started = result.get("started_at", "?") + finished = result.get("finished_at", "?") + console.print(Panel( + f"[bold {color}]{verdict}[/bold {color}] pass_rate={n_pass}/{n_total} " + f"score={score:.4f} trials_done={n_done} trials_err={n_err}", + border_style=color, title="trial verdict", + )) + console.print(f" [dim]started {started}[/dim]") + console.print(f" [dim]finished {finished}[/dim]") + + +def wait_for_job(job_name: str, *, timeout_min: int = 120, poll_sec: int = 30, verbose: bool = True) -> dict: + """Poll ``jobs//result.json`` until DONE or timeout. Returns the parsed JSON.""" + from rich.live import Live + from rich.spinner import Spinner + from rich.text import Text + + deadline = time.time() + timeout_min * 60 + last = None + last_update = 0.0 + spinner = Spinner("dots", text=Text(f"waiting for {job_name}…", style="cyan")) + + if verbose: + with Live(spinner, console=console, refresh_per_second=4, transient=False) as live: + while time.time() < deadline: + elapsed = int(time.time() - (deadline - timeout_min * 60)) + remaining = max(0, int(deadline - time.time())) + try: + content = exec_in_orchestrator( + "cat", f"{CONTAINER_JOBS_DIR}/{job_name}/result.json", check=False + ).stdout + except subprocess.CalledProcessError: + content = "" + if content.strip(): + try: + data = json.loads(content) + except json.JSONDecodeError: + data = None + if data and data.get("finished_at"): + # Final tick — replace spinner with a checkmark before exit + live.update(Text(f"✅ {job_name} done after {elapsed}s", style="bold green")) + return data + last = data + if data and time.time() - last_update > 5: + # Re-render spinner with current elapsed + spinner.text = Text( + f"waiting for {job_name}… elapsed={elapsed}s remaining={remaining}s", + style="cyan", + ) + last_update = time.time() + else: + if time.time() - last_update > 5: + spinner.text = Text( + f"waiting for {job_name}… (no result.json yet) elapsed={elapsed}s", + style="cyan", + ) + last_update = time.time() + time.sleep(poll_sec) + else: + while time.time() < deadline: + try: + content = exec_in_orchestrator( + "cat", f"{CONTAINER_JOBS_DIR}/{job_name}/result.json", check=False + ).stdout + except subprocess.CalledProcessError: + content = "" + if content.strip(): + try: + data = json.loads(content) + except json.JSONDecodeError: + data = None + if data and data.get("finished_at"): + return data + last = data + time.sleep(poll_sec) + raise TimeoutError(f"Job {job_name!r} did not finish within {timeout_min} min") + + +def launch_matrix( + cases: list[str], + agents: list[str], + *, + model: str = DEFAULT_MODEL, + api_base: str = DEFAULT_API_BASE, + parallel: int | None = None, + wait: bool = False, + timeout_min: int = 120, +) -> list[str]: + """Launch N×M trials in parallel. Returns the list of job names launched.""" + api_key = _load_api_key() + jobs = [] + for case in cases: + for agent in agents: + name = f"{agent}-{case}" + jobs.append(launch_trial( + case, agent, + job_name=name, + model=model, + api_base=api_base, + api_key=api_key, + wait=False, + )) + console.print(f"[green]launched {len(jobs)} trials[/green]") + if wait: + for name in jobs: + try: + wait_for_job(name, timeout_min=timeout_min) + console.print(f"[green]✅ {name} done[/green]") + except TimeoutError as e: + console.print(f"[red]❌ {e}[/red]") + return jobs + + +def launch_3x3(*, wait: bool = False, timeout_min: int = 120) -> list[str]: + """Launch the default 3×3 matrix (DEFAULT_CASES × DEFAULT_AGENTS).""" + from .config import DEFAULT_CASES, DEFAULT_AGENTS + return launch_matrix(DEFAULT_CASES, DEFAULT_AGENTS, wait=wait, timeout_min=timeout_min) \ No newline at end of file diff --git a/din_integration/swebench_dind/patcher.py b/din_integration/swebench_dind/patcher.py new file mode 100644 index 00000000..03dfa43c --- /dev/null +++ b/din_integration/swebench_dind/patcher.py @@ -0,0 +1,147 @@ +"""Idempotent-install probe patcher for harbor agent modules. + +Patches the installed harbor source under +``/usr/local/lib/python3.12/dist-packages/harbor/agents/installed/.py`` +inside the orchestrator container so that the ``install()`` method skips +work when the agent is already pre-installed in the baked image. + +Generalizes the legacy ``scripts/patch_qwen_code.py``. +""" +from __future__ import annotations + +import subprocess +import tempfile +from pathlib import Path + +from rich.console import Console + +from .container import exec_in_orchestrator + +console = Console() + +HARBOR_PKG_DIR = Path("/usr/local/lib/python3.12/dist-packages/harbor") + +# (agent, harbor_module_filename) +AGENT_TO_MODULE: dict[str, str] = { + "aider": "aider.py", + "mini-swe-agent": "mini_swe_agent.py", + "qwen-code": "qwen_code.py", + "openhands-sdk": "openhands_sdk.py", +} + +# CLI binary name per agent (used for `command -v`) +AGENT_CLI: dict[str, str] = { + "aider": "aider", + "mini-swe-agent": "mini-swe-agent", + "qwen-code": "qwen", + "openhands-sdk": "openhands", +} + + +def _exec_python_in_container(code: str) -> str: + """Run a Python script inside the orchestrator container. + + We write the code to a temp file on the host, then ``docker cp`` it + into the container and run it. Avoids shell-escaping nightmares with + nested f-strings and quotes. + """ + with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: + f.write(code) + local = Path(f.name) + try: + remote = f"/tmp/{local.name}" + subprocess.run( + ["docker", "cp", str(local), f"swebench-orchestrator:{remote}"], + check=True, + ) + result = exec_in_orchestrator("python3", remote) + return result.stdout + finally: + local.unlink(missing_ok=True) + exec_in_orchestrator("rm", "-f", remote, check=False) + + +def _install_path(module_file: str) -> str: + return str(HARBOR_PKG_DIR / "agents" / "installed" / module_file) + + +def _build_patch_script(path: str, cli: str) -> str: + """Return the in-container Python script that does the actual patching. + + Built as a plain triple-quoted string with ``{path}`` / ``{cli}`` + placeholders filled via ``str.format``. (We avoid f-strings because the + embedded raw-strings + escapes are a parsing nightmare.) + """ + template = """ +import re, os, glob, sys + +PATH = __PATH__ +CLI = __CLI__ +MARKER = 'Idempotent probe (PATCHED by swebench-dind CLI)' + +content = open(PATH).read() +if MARKER in content: + print('ALREADY PATCHED') + sys.exit(0) + +pattern = re.compile( + r'(@override\\s*\\n\\s*async def install\\(self, environment: BaseEnvironment\\) -> None:\\n)' + r'(.*?)' + r'(\\n (?:async )?def \\w+|\\nclass \\w+)', + re.DOTALL, +) +m = pattern.search(content) +if m is None: + print('install() method not found') + sys.exit(1) + +header, body, tail = m.group(1), m.group(2), m.group(3) +not_found = CLI + '-not-found' +probe_lines = [ + ' # Idempotent probe (PATCHED by swebench-dind CLI): skip install if baked.', + ' probe = await environment.exec(', + ' command="command -v ' + CLI + ' >/dev/null 2>&1 && ' + CLI + ' --version || { echo \\'' + not_found + '\\'; exit 1; }"', + ' )', + ' if probe.return_code == 0:', + ' return', + '', +] +probe = '\\n'.join(probe_lines) + '\\n' +new_content = content[:m.start()] + header + probe + body + tail + content[m.end():] +open(PATH, 'w').write(new_content) + +removed = [] +base = os.path.basename(PATH).replace('.py', '') +for pyc in glob.glob(os.path.dirname(PATH) + '/__pycache__/' + base + '.cpython-*.pyc'): + os.remove(pyc) + removed.append(pyc) + +print('PATCHED') +print('Removed pyc:', removed) +""" + return template.replace('__PATH__', repr(path)).replace('__CLI__', repr(cli)) + + +def patch_agent(agent: str) -> bool: + """Patch the harbor agent module to add the idempotent install probe. + + Idempotent: re-running on an already-patched module is a no-op. + Returns True if patched (or already patched), False on error. + """ + module_file = AGENT_TO_MODULE.get(agent) + cli = AGENT_CLI.get(agent) + if module_file is None or cli is None: + console.print(f"[red]Unknown agent: {agent}[/red]") + return False + + path = _install_path(module_file) + console.print(f"[bold]patch[/bold] {path}") + py_script = _build_patch_script(path, cli) + out = _exec_python_in_container(py_script) + console.print(out) + return "PATCHED" in out or "ALREADY PATCHED" in out + + +def patch_all(agents: list[str]) -> dict[str, bool]: + """Patch a list of agents. Returns {agent: success}.""" + return {a: patch_agent(a) for a in agents} \ No newline at end of file diff --git a/din_integration/swebench_dind/summarizer.py b/din_integration/swebench_dind/summarizer.py new file mode 100644 index 00000000..84955296 --- /dev/null +++ b/din_integration/swebench_dind/summarizer.py @@ -0,0 +1,138 @@ +"""Aggregate ``jobs/*/result.json`` → md/csv/json summary. + +Equivalent to ``scripts/summarize.py`` but with a programmatic API. +""" +from __future__ import annotations + +import csv +import json +from collections import defaultdict +from datetime import datetime +from pathlib import Path + +from .config import JOBS_DIR, LOGS_DIR + + +def _load_jobs(jobs_dir: Path, include: list[str] | None = None) -> list[dict]: + rows: list[dict] = [] + for job_dir in sorted(p for p in jobs_dir.iterdir() if p.is_dir()): + if include and not any(inc in job_dir.name for inc in include): + continue + result = job_dir / "result.json" + if not result.exists(): + continue + try: + data = json.loads(result.read_text()) + except json.JSONDecodeError: + continue + config = {} + cfg_path = job_dir / "config.json" + if cfg_path.exists(): + try: + config = json.loads(cfg_path.read_text()) + except json.JSONDecodeError: + pass + rows.append(_parse_job(job_dir.name, data, config)) + return rows + + +def _parse_job(name: str, data: dict, config: dict) -> dict: + agents = config.get("agents", []) + tasks = config.get("tasks", []) + datasets = config.get("datasets", []) + agent_name = agents[0]["name"] if agents else "?" + if tasks: + bench = Path(tasks[0]["path"]).name + elif datasets: + bench = Path(datasets[0]["path"]).name + else: + bench = "?" + stats = data.get("stats", {}) + evals = stats.get("evals", {}) + rewards: list[float] = [] + for _, eval_data in evals.items(): + buckets = eval_data.get("reward_stats", {}).get("reward", {}) + for r_str, trial_ids in buckets.items(): + try: + r_val = float(r_str) + except (ValueError, TypeError): + continue + n = len(trial_ids) if isinstance(trial_ids, list) else 1 + rewards.extend([r_val] * n) + n_pass = sum(1 for r in rewards if r == 1.0) + return { + "job_name": name, + "agent": agent_name, + "benchmark": bench, + "n_total": stats.get("n_total_trials", 0), + "n_completed": stats.get("n_completed_trials", 0), + "n_errored": stats.get("n_errored_trials", 0), + "n_running": stats.get("n_running_trials", 0), + "n_pending": stats.get("n_pending_trials", 0), + "n_pass": n_pass, + "pass_at_1": n_pass / max(len(rewards), 1), + "finished_at": data.get("finished_at"), + "started_at": data.get("started_at"), + } + + +def _write_markdown(rows: list[dict], path: Path) -> None: + grouped: dict[tuple[str, str], list[dict]] = defaultdict(list) + for r in rows: + grouped[(r["benchmark"], r["agent"])].append(r) + with path.open("w") as f: + f.write("# Multi-Bench × Multi-Agent Summary\n\n") + f.write(f"_Generated: {datetime.now().isoformat()}_\n\n") + f.write(f"**Total jobs**: {len(rows)}\n\n") + f.write("## Pass@1 by (Benchmark, Agent)\n\n") + f.write("| Benchmark | Agent | n_jobs | n_pass | pass@1 |\n") + f.write("|---|---|---|---|---|\n") + for (bench, agent), items in sorted(grouped.items()): + n = len(items) + n_p = sum(it["n_pass"] for it in items) + f.write(f"| {bench} | {agent} | {n} | {n_p} | {n_p / max(n, 1):.1%} |\n") + f.write("\n## All Jobs\n\n") + f.write("| Job | Agent | Bench | n_total | n_done | n_err | pass@1 | finished |\n") + f.write("|---|---|---|---|---|---|---|---|\n") + for r in sorted(rows, key=lambda x: (x["benchmark"], x["agent"], x["job_name"])): + done = "✅" if r["finished_at"] else "🔄" + f.write( + f"| `{r['job_name']}` | {r['agent']} | {r['benchmark']} | " + f"{r['n_total']} | {r['n_completed']} | {r['n_errored']} | " + f"{r['pass_at_1']:.0%} | {r['finished_at'] or '—'} {done} |\n" + ) + + +def _write_csv(rows: list[dict], path: Path) -> None: + if not rows: + path.write_text("") + return + with path.open("w", newline="") as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + + +def summarize( + *, + jobs_dir: Path = JOBS_DIR, + output_dir: Path = LOGS_DIR, + include: list[str] | None = None, +) -> dict: + """Aggregate all jobs → write summary-.{md,csv,json}. Returns paths.""" + output_dir.mkdir(parents=True, exist_ok=True) + rows = _load_jobs(jobs_dir, include=include) + ts = datetime.now().strftime("%Y%m%d-%H%M%S") + md = output_dir / f"summary-{ts}.md" + csv_p = output_dir / f"summary-{ts}.csv" + json_p = output_dir / f"summary-{ts}.json" + _write_markdown(rows, md) + _write_csv(rows, csv_p) + json_p.write_text(json.dumps(rows, indent=2, default=str)) + return {"rows": rows, "md": md, "csv": csv_p, "json": json_p} + + +def watch(job_name: str, *, poll_sec: int = 30) -> dict: + """Live-watch a job until finished. Yields status dicts (one per poll).""" + from .launcher import wait_for_job + return wait_for_job(job_name, timeout_min=24 * 60, poll_sec=poll_sec) \ No newline at end of file