From dcaced2bb31298bd319c6e216388f67a6e6ee1bd Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 09:02:30 -0700 Subject: [PATCH 01/13] feat: add SkillsBench external lift adapter --- benchmarks/external/skillsbench_adapter.py | 666 +++++++++++++++++++++ 1 file changed, 666 insertions(+) create mode 100644 benchmarks/external/skillsbench_adapter.py diff --git a/benchmarks/external/skillsbench_adapter.py b/benchmarks/external/skillsbench_adapter.py new file mode 100644 index 0000000..4737113 --- /dev/null +++ b/benchmarks/external/skillsbench_adapter.py @@ -0,0 +1,666 @@ +#!/usr/bin/env python3 +"""Run a paired Practical Coding ablation on SkillsBench.""" + +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import json +import os +import random +import re +import shutil +import statistics +import subprocess +import tempfile +from collections import defaultdict +from pathlib import Path +from typing import Any, Iterable + +DATASET = "skillsbench@1.1" +DATASET_NAME = "skillsbench" +DATASET_VERSION = "1.1" +SKILLSBENCH_REPO = "https://github.com/benchflow-ai/skillsbench.git" +SKILLSBENCH_REF = "v1.1" +BENCHFLOW_VERSION = "0.6.2" +AGENT = "codex-acp" +MODEL = "gpt-5.6-luna" +REASONING = "medium" +ROOT = Path(__file__).resolve().parents[2] +SMOKE_TASKS = ( + "fix-build-agentops", + "spring-boot-jakarta-migration", + "react-performance-debugging", +) +AUTH_ENV_VARS = ("OPENAI_API_KEY", "CODEX_API_KEY", "CODEX_ACCESS_TOKEN") + + +def run_command(command: list[str], cwd: Path | None = None, timeout: float | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + command, + cwd=str(cwd) if cwd else None, + text=True, + encoding="utf-8", + errors="replace", + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + timeout=timeout, + check=False, + ) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + digest.update(path.read_bytes()) + return digest.hexdigest() + + +def skill_bundle_sha256(root: Path) -> str: + digest = hashlib.sha256() + paths = [root / "SKILL.md", *sorted((root / "references").glob("*.md"))] + for path in paths: + if not path.is_file(): + continue + digest.update(str(path.relative_to(root)).replace("\\", "/").encode()) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +def resolve_uvx() -> list[str]: + uvx = shutil.which("uvx") + if uvx: + return [uvx, "--from", f"benchflow=={BENCHFLOW_VERSION}", "bench"] + uv = shutil.which("uv") + if uv: + return [uv, "tool", "run", "--from", f"benchflow=={BENCHFLOW_VERSION}", "bench"] + raise FileNotFoundError("uv/uvx is required. Install uv, then rerun the external benchmark.") + + +def check_prerequisites(sandbox: str) -> dict[str, str]: + if not shutil.which("git"): + raise FileNotFoundError("git is required") + codex = shutil.which("codex") + if not codex: + raise FileNotFoundError("Codex CLI is required for the codex-acp SkillsBench arm") + uv_prefix = resolve_uvx() + auth_json = Path.home() / ".codex" / "auth.json" + if not any(os.environ.get(name) for name in AUTH_ENV_VARS) and not auth_json.is_file(): + raise RuntimeError("Codex authentication was not found. Run `codex login` or provide a supported Codex/OpenAI credential.") + versions: dict[str, str] = {} + for name, command in ( + ("git", [shutil.which("git") or "git", "--version"]), + ("codex", [codex, "--version"]), + ("uv", [uv_prefix[0], "--version"]), + ): + result = run_command(command) + if result.returncode: + raise RuntimeError(f"failed to inspect {name}: {result.stdout[-1000:]}") + versions[name] = result.stdout.strip() + if sandbox == "docker": + docker = shutil.which("docker") + if not docker: + raise FileNotFoundError("Docker is required for --sandbox docker") + info = run_command([docker, "info"], timeout=30) + if info.returncode: + raise RuntimeError("Docker is installed but the daemon is unavailable") + versions["docker"] = run_command([docker, "--version"]).stdout.strip() + return versions + + +def ensure_skillsbench_checkout(cache_root: Path) -> tuple[Path, str]: + checkout = cache_root / f"skillsbench-{DATASET_VERSION}" + checkout.parent.mkdir(parents=True, exist_ok=True) + if not checkout.exists(): + clone = run_command([ + shutil.which("git") or "git", + "clone", + "--filter=blob:none", + "--depth", + "1", + "--branch", + SKILLSBENCH_REF, + SKILLSBENCH_REPO, + str(checkout), + ]) + if clone.returncode: + raise RuntimeError(f"failed to clone SkillsBench {SKILLSBENCH_REF}: {clone.stdout[-3000:]}") + head = run_command([shutil.which("git") or "git", "rev-parse", "HEAD"], checkout) + tag = run_command([shutil.which("git") or "git", "describe", "--tags", "--exact-match"], checkout) + if head.returncode or tag.returncode or tag.stdout.strip() != SKILLSBENCH_REF: + raise RuntimeError(f"SkillsBench metadata checkout must be exactly {SKILLSBENCH_REF}") + return checkout, head.stdout.strip() + + +def load_dataset_roster(checkout: Path) -> list[str]: + registry = json.loads((checkout / "registry.json").read_text(encoding="utf-8")) + for entry in registry: + if entry.get("name") == DATASET_NAME and str(entry.get("version")) == DATASET_VERSION: + names = [str(task["name"]) for task in entry.get("tasks", [])] + if not names: + raise RuntimeError(f"{DATASET} has an empty registry roster") + return names + raise RuntimeError(f"{DATASET} is missing from SkillsBench registry.json") + + +def task_category(task_md: Path) -> str | None: + text = task_md.read_text(encoding="utf-8", errors="replace") + frontmatter = text.split("---", 2) + if len(frontmatter) < 3: + return None + match = re.search(r"(?m)^\s*category:\s*['\"]?([a-z0-9-]+)", frontmatter[1]) + return match.group(1) if match else None + + +def discover_tasks(checkout: Path, profile: str, explicit: Iterable[str] = ()) -> list[str]: + roster = load_dataset_roster(checkout) + roster_set = set(roster) + explicit = [item for item in explicit if item] + if explicit: + unknown = sorted(set(explicit) - roster_set) + if unknown: + raise ValueError(f"tasks are not in {DATASET}: {unknown}") + return list(dict.fromkeys(explicit)) + software = [ + name + for name in roster + if task_category(checkout / "tasks" / name / "task.md") == "software-engineering" + ] + if profile == "smoke": + preferred = [name for name in SMOKE_TASKS if name in software] + return preferred if len(preferred) == len(SMOKE_TASKS) else software[:3] + if profile == "standard": + if not software: + raise RuntimeError("no software-engineering tasks were discovered") + return software + if profile == "full": + return roster + raise ValueError(profile) + + +def stage_practical_skill(output: Path) -> Path: + skills_root = output / "staged-skills" + destination = skills_root / "practical-coding" + if destination.exists(): + shutil.rmtree(destination) + destination.mkdir(parents=True) + shutil.copy2(ROOT / "SKILL.md", destination / "SKILL.md") + references = ROOT / "references" + if references.is_dir(): + shutil.copytree(references, destination / "references") + return skills_root + + +def bench_command( + *, + jobs_dir: Path, + tasks: list[str], + sandbox: str, + workers: int, + model: str, + reasoning: str, + skill_mode: str, + skills_root: Path | None = None, +) -> list[str]: + command = [ + *resolve_uvx(), + "eval", + "run", + "-d", + DATASET, + "--agent", + AGENT, + "--model", + model, + "--reasoning-effort", + reasoning, + "--sandbox", + sandbox, + "--concurrency", + str(workers), + "--jobs-dir", + str(jobs_dir), + "--skill-mode", + skill_mode, + "--quiet", + ] + if skill_mode == "with-skill": + if skills_root is None: + raise ValueError("skills_root is required for with-skill") + command += ["--skills-dir", str(skills_root)] + for task in tasks: + command += ["--include", task] + return command + + +def oracle_command(*, jobs_dir: Path, tasks: list[str], sandbox: str, workers: int) -> list[str]: + command = [ + *resolve_uvx(), + "eval", + "run", + "-d", + DATASET, + "--agent", + "oracle", + "--sandbox", + sandbox, + "--concurrency", + str(workers), + "--jobs-dir", + str(jobs_dir), + "--skill-mode", + "no-skill", + "--quiet", + ] + for task in tasks: + command += ["--include", task] + return command + + +def _read_json(path: Path) -> dict[str, Any] | None: + try: + value = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + return value if isinstance(value, dict) else None + + +def _reward(result: dict[str, Any]) -> float | None: + rewards = result.get("rewards") + if isinstance(rewards, dict): + value = rewards.get("reward") + if isinstance(value, (int, float)) and not isinstance(value, bool): + return float(value) + value = result.get("reward") + if isinstance(value, (int, float)) and not isinstance(value, bool): + return float(value) + return None + + +def _task_id(result: dict[str, Any], rollout_dir: Path) -> str: + direct = result.get("task_name") or result.get("task_id") + if direct: + return str(direct) + task = result.get("task") + if isinstance(task, dict) and task.get("id"): + return str(task["id"]) + config = _read_json(rollout_dir / "config.json") or {} + direct = config.get("task_name") or config.get("task_id") + if direct: + return str(direct) + task = config.get("task") + if isinstance(task, dict) and task.get("id"): + return str(task["id"]) + return rollout_dir.name + + +def _excluded_reason(result: dict[str, Any], reward: float | None) -> str | None: + if result.get("healthy") is False: + return "unhealthy" + if result.get("partial_trajectory") is True: + return "partial_trajectory" + if reward is not None: + return None + for field in ("error", "verifier_error", "export_error"): + if result.get(field): + return field + return "unscored" + + +def load_job_rewards(job_dir: Path) -> dict[str, dict[str, Any]]: + by_task: dict[str, dict[str, Any]] = {} + duplicates: dict[str, list[str]] = defaultdict(list) + for result_path in sorted(job_dir.rglob("result.json")): + result = _read_json(result_path) + if result is None: + continue + reward = _reward(result) + task_id = _task_id(result, result_path.parent) + row = { + "task_id": task_id, + "reward": reward, + "passed": reward == 1.0, + "excluded_reason": _excluded_reason(result, reward), + "path": str(result_path), + } + if task_id in by_task: + duplicates[task_id].extend([by_task[task_id]["path"], str(result_path)]) + else: + by_task[task_id] = row + if duplicates: + details = "; ".join(f"{task}: {sorted(set(paths))}" for task, paths in sorted(duplicates.items())) + raise RuntimeError(f"duplicate SkillsBench result.json files for a task: {details}") + return by_task + + +def validate_oracle(job_dir: Path, tasks: list[str]) -> tuple[bool, dict[str, Any]]: + rows = load_job_rewards(job_dir) + missing = sorted(set(tasks) - set(rows)) + failed = sorted(task for task in tasks if task in rows and rows[task]["reward"] != 1.0) + unhealthy = sorted(task for task in tasks if task in rows and rows[task]["excluded_reason"] is not None) + return not missing and not failed and not unhealthy, { + "expected": len(tasks), + "observed": len(rows), + "missing": missing, + "failed": failed, + "unhealthy": unhealthy, + } + + +def collect_pairs(output: Path, tasks: list[str], runs: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: + pairs: list[dict[str, Any]] = [] + gaps: list[dict[str, Any]] = [] + for repetition in range(1, runs + 1): + rep = output / "runs" / f"r{repetition:03d}" + base = load_job_rewards(rep / "no-skill") if (rep / "no-skill").exists() else {} + practical = load_job_rewards(rep / "practical") if (rep / "practical").exists() else {} + for task in tasks: + left = base.get(task) + right = practical.get(task) + if not left or not right or left["excluded_reason"] or right["excluded_reason"]: + gaps.append({ + "task": task, + "repetition": repetition, + "baseline": left, + "practical": right, + }) + continue + pairs.append({ + "task": task, + "repetition": repetition, + "reward_base": left["reward"], + "reward_practical": right["reward"], + "passed_base": left["passed"], + "passed_practical": right["passed"], + }) + return pairs, gaps + + +def percentile(values: list[float], p: float) -> float | None: + if not values: + return None + values = sorted(values) + if len(values) == 1: + return values[0] + position = (len(values) - 1) * p + lower = int(position) + upper = min(lower + 1, len(values) - 1) + fraction = position - lower + return values[lower] * (1 - fraction) + values[upper] * fraction + + +def cluster_bootstrap(pairs: list[dict[str, Any]], samples: int = 5000, seed: int = 0) -> dict[str, list[float | None]]: + groups: dict[str, list[dict[str, Any]]] = defaultdict(list) + for pair in pairs: + groups[pair["task"]].append(pair) + tasks = sorted(groups) + if not tasks or samples <= 0: + return {"pass_rate_delta": [None, None], "mean_reward_delta": [None, None]} + rng = random.Random(seed) + pass_deltas: list[float] = [] + reward_deltas: list[float] = [] + for _ in range(samples): + sample_pairs: list[dict[str, Any]] = [] + for task in (rng.choice(tasks) for _ in tasks): + sample_pairs.extend(groups[task]) + pass_deltas.append( + statistics.mean(float(row["passed_practical"]) - float(row["passed_base"]) for row in sample_pairs) + ) + reward_deltas.append( + statistics.mean(float(row["reward_practical"]) - float(row["reward_base"]) for row in sample_pairs) + ) + return { + "pass_rate_delta": [percentile(pass_deltas, 0.025), percentile(pass_deltas, 0.975)], + "mean_reward_delta": [percentile(reward_deltas, 0.025), percentile(reward_deltas, 0.975)], + } + + +def summarize_pairs(pairs: list[dict[str, Any]], tasks: list[str], runs: int, gaps: list[dict[str, Any]], oracle_ok: bool) -> dict[str, Any]: + if pairs: + pass_base = statistics.mean(float(row["passed_base"]) for row in pairs) + pass_practical = statistics.mean(float(row["passed_practical"]) for row in pairs) + reward_base = statistics.mean(float(row["reward_base"]) for row in pairs) + reward_practical = statistics.mean(float(row["reward_practical"]) for row in pairs) + else: + pass_base = pass_practical = reward_base = reward_practical = 0.0 + delta = pass_practical - pass_base + normalized_gain = (delta / (1.0 - pass_base)) if delta > 0 and pass_base < 1.0 else delta + by_task: list[dict[str, Any]] = [] + for task in tasks: + rows = [row for row in pairs if row["task"] == task] + if not rows: + by_task.append({"task": task, "n": 0}) + continue + by_task.append({ + "task": task, + "n": len(rows), + "pass_rate_base": statistics.mean(float(row["passed_base"]) for row in rows), + "pass_rate_practical": statistics.mean(float(row["passed_practical"]) for row in rows), + "mean_reward_base": statistics.mean(float(row["reward_base"]) for row in rows), + "mean_reward_practical": statistics.mean(float(row["reward_practical"]) for row in rows), + }) + expected_pairs = len(tasks) * runs + stable = runs >= 3 and oracle_ok and len(pairs) == expected_pairs and not gaps and all(row["n"] == runs for row in by_task) + return { + "stable": stable, + "expected_pairs": expected_pairs, + "paired_rollouts": len(pairs), + "gap_count": len(gaps), + "pass_rate_base": pass_base, + "pass_rate_practical": pass_practical, + "pass_rate_delta": delta, + "normalized_gain": normalized_gain, + "mean_reward_base": reward_base, + "mean_reward_practical": reward_practical, + "mean_reward_delta": reward_practical - reward_base, + "wins": sum((not row["passed_base"]) and row["passed_practical"] for row in pairs), + "losses": sum(row["passed_base"] and (not row["passed_practical"]) for row in pairs), + "ties": sum(row["passed_base"] == row["passed_practical"] for row in pairs), + "ci95": cluster_bootstrap(pairs), + "by_task": by_task, + } + + +def _fmt_pct(value: float | None) -> str: + return "—" if value is None else f"{100 * value:.1f}%" + + +def _fmt_signed_pp(value: float | None) -> str: + return "—" if value is None else f"{100 * value:+.1f} pp" + + +def render_report(manifest: dict[str, Any], summary: dict[str, Any], gaps: list[dict[str, Any]]) -> str: + ci = summary["ci95"] + pass_ci = ci["pass_rate_delta"] + reward_ci = ci["mean_reward_delta"] + lines = [ + "# Practical Coding × SkillsBench external lift", + "", + f"- Dataset: `{manifest['dataset']}`", + f"- Profile: `{manifest['profile']}` ({len(manifest['tasks'])} tasks)", + f"- Agent/model: `{manifest['agent']}` / `{manifest['model']}` ({manifest['reasoning']})", + f"- Runs per task/arm: `{manifest['runs']}`", + f"- Evidence: **{'STABLE' if summary['stable'] else 'PROVISIONAL'}**", + "- Treatment: baseline is `no-skill`; trained arm mounts only `practical-coding` as the custom Skill directory.", + "- This is a Practical-owned ablation on the immutable SkillsBench dataset, not an official SkillsBench leaderboard submission using each task's curated Skill.", + "", + "## Overall", + "", + "| Metric | No Skill | + Practical | Delta | 95% cluster-bootstrap CI |", + "|---|---:|---:|---:|---|", + f"| Pass rate | {_fmt_pct(summary['pass_rate_base'])} | {_fmt_pct(summary['pass_rate_practical'])} | {_fmt_signed_pp(summary['pass_rate_delta'])} | {_fmt_signed_pp(pass_ci[0])} … {_fmt_signed_pp(pass_ci[1])} |", + f"| Mean reward | {summary['mean_reward_base']:.3f} | {summary['mean_reward_practical']:.3f} | {summary['mean_reward_delta']:+.3f} | {(f'{reward_ci[0]:+.3f}' if reward_ci[0] is not None else '—')} … {(f'{reward_ci[1]:+.3f}' if reward_ci[1] is not None else '—')} |", + "", + f"Paired rollouts: **{summary['paired_rollouts']}/{summary['expected_pairs']}**. Pass flips: **{summary['wins']} wins / {summary['losses']} losses / {summary['ties']} ties**. Normalized gain: **{100 * summary['normalized_gain']:+.1f}%**.", + "", + "## Per task", + "", + "| Task | n | No Skill pass | Practical pass | No Skill reward | Practical reward |", + "|---|---:|---:|---:|---:|---:|", + ] + for row in summary["by_task"]: + if row["n"] == 0: + lines.append(f"| {row['task']} | 0 | — | — | — | — |") + else: + lines.append( + f"| {row['task']} | {row['n']} | {_fmt_pct(row['pass_rate_base'])} | {_fmt_pct(row['pass_rate_practical'])} | {row['mean_reward_base']:.3f} | {row['mean_reward_practical']:.3f} |" + ) + if gaps: + lines += ["", "## Missing / unhealthy pairs", ""] + for gap in gaps: + lines.append(f"- `{gap['task']}` r{gap['repetition']:03d}") + lines += [ + "", + "## Interpretation", + "", + "A positive delta means the same Codex/Luna setup solved more of the selected SkillsBench tasks when Practical Coding was mounted. The confidence interval resamples task IDs as clusters and keeps repeated trials for a task together. Public benchmark exposure still means this is external evidence, not a private holdout.", + "", + ] + return "\n".join(lines) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--profile", choices=("smoke", "standard", "full"), default="standard") + parser.add_argument("--task", action="append", default=[]) + parser.add_argument("--runs", type=int, default=0) + parser.add_argument("--workers", type=int, default=3) + parser.add_argument("--sandbox", choices=("docker", "daytona", "modal", "apple-container", "agentcore"), default="docker") + parser.add_argument("--model", default=MODEL) + parser.add_argument("--reasoning", default=REASONING) + parser.add_argument("--output", type=Path) + parser.add_argument("--cache-root", type=Path) + parser.add_argument("--skip-oracle", action="store_true") + parser.add_argument("--require-stable-ranking", action="store_true") + parser.add_argument("--self-test", action="store_true") + return parser.parse_args() + + +def self_test() -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + jobs = root / "jobs" + for task, base, trained in (("a", 0.0, 1.0), ("b", 1.0, 1.0)): + for arm, reward in (("base", base), ("trained", trained)): + path = jobs / arm / task + path.mkdir(parents=True, exist_ok=True) + (path / "result.json").write_text(json.dumps({"task_name": task, "rewards": {"reward": reward}}), encoding="utf-8") + assert load_job_rewards(jobs / "base")["a"]["reward"] == 0.0 + pairs = [ + {"task": "a", "repetition": 1, "reward_base": 0.0, "reward_practical": 1.0, "passed_base": False, "passed_practical": True}, + {"task": "b", "repetition": 1, "reward_base": 1.0, "reward_practical": 1.0, "passed_base": True, "passed_practical": True}, + ] + summary = summarize_pairs(pairs, ["a", "b"], 1, [], True) + assert summary["pass_rate_delta"] == 0.5 + assert not summary["stable"] + stable_pairs = [] + for repetition in (1, 2, 3): + for task in ("a", "b"): + stable_pairs.append({"task": task, "repetition": repetition, "reward_base": 1.0, "reward_practical": 1.0, "passed_base": True, "passed_practical": True}) + assert summarize_pairs(stable_pairs, ["a", "b"], 3, [], True)["stable"] + print("SkillsBench adapter self-test: PASS") + + +def main() -> int: + args = parse_args() + if args.self_test: + self_test() + return 0 + runs = args.runs or (1 if args.profile == "smoke" else 3) + if runs < 1 or args.workers < 1: + raise SystemExit("runs and workers must be positive") + if args.require_stable_ranking and runs < 3: + raise SystemExit("stable external ranking requires at least 3 runs") + versions = check_prerequisites(args.sandbox) + cache_root = (args.cache_root or Path(os.environ.get("LOCALAPPDATA") or tempfile.gettempdir()) / "practical-coding-benchmarks" / "external").resolve() + checkout, source_commit = ensure_skillsbench_checkout(cache_root) + tasks = discover_tasks(checkout, args.profile, args.task) + stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S") + output = (args.output or ROOT / "benchmark-results" / "external" / f"skillsbench-{stamp}").resolve() + output.mkdir(parents=True, exist_ok=False) + staged_skills = stage_practical_skill(output) + started = dt.datetime.now(dt.timezone.utc) + manifest: dict[str, Any] = { + "schema_version": 1, + "started_at": started.isoformat(), + "dataset": DATASET, + "skillsbench_metadata_ref": SKILLSBENCH_REF, + "skillsbench_metadata_commit": source_commit, + "benchflow_version": BENCHFLOW_VERSION, + "agent": AGENT, + "model": args.model, + "reasoning": args.reasoning, + "sandbox": args.sandbox, + "profile": args.profile, + "runs": runs, + "workers": args.workers, + "tasks": tasks, + "skill_bundle_sha256": skill_bundle_sha256(ROOT), + "adapter_sha256": sha256_file(Path(__file__)), + "environment": versions, + "commands": [], + } + (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + + oracle_ok = True + oracle_summary: dict[str, Any] = {"skipped": True} + if not args.skip_oracle: + oracle_dir = output / "oracle" + command = oracle_command(jobs_dir=oracle_dir, tasks=tasks, sandbox=args.sandbox, workers=args.workers) + manifest["commands"].append({"kind": "oracle", "command": command}) + result = run_command(command, ROOT) + (output / "oracle.log").write_text(result.stdout, encoding="utf-8") + if result.returncode: + raise RuntimeError(f"SkillsBench oracle command failed: {result.stdout[-4000:]}") + oracle_ok, oracle_summary = validate_oracle(oracle_dir, tasks) + if not oracle_ok: + raise RuntimeError(f"SkillsBench oracle did not pass the selected task set: {oracle_summary}") + + for repetition in range(1, runs + 1): + rep = output / "runs" / f"r{repetition:03d}" + order = ("no-skill", "practical") if repetition % 2 else ("practical", "no-skill") + for arm in order: + jobs_dir = rep / arm + mode = "no-skill" if arm == "no-skill" else "with-skill" + command = bench_command( + jobs_dir=jobs_dir, + tasks=tasks, + sandbox=args.sandbox, + workers=args.workers, + model=args.model, + reasoning=args.reasoning, + skill_mode=mode, + skills_root=staged_skills if arm == "practical" else None, + ) + manifest["commands"].append({"kind": arm, "repetition": repetition, "command": command}) + result = run_command(command, ROOT) + rep.mkdir(parents=True, exist_ok=True) + (rep / f"{arm}.log").write_text(result.stdout, encoding="utf-8") + if result.returncode: + manifest.setdefault("infrastructure_failures", []).append({"arm": arm, "repetition": repetition, "returncode": result.returncode}) + (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + raise RuntimeError(f"SkillsBench {arm} r{repetition} failed: {result.stdout[-4000:]}") + + pairs, gaps = collect_pairs(output, tasks, runs) + summary = summarize_pairs(pairs, tasks, runs, gaps, oracle_ok) + manifest.update({ + "completed_at": dt.datetime.now(dt.timezone.utc).isoformat(), + "oracle": oracle_summary, + "paired_rollouts": len(pairs), + "stable": summary["stable"], + }) + (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + (output / "pairs.json").write_text(json.dumps(pairs, indent=2) + "\n", encoding="utf-8") + (output / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + (output / "report.md").write_text(render_report(manifest, summary, gaps), encoding="utf-8") + print(f"wrote {output}") + print(f"SkillsBench pass lift: {_fmt_signed_pp(summary['pass_rate_delta'])} ({'STABLE' if summary['stable'] else 'PROVISIONAL'})") + if args.require_stable_ranking and not summary["stable"]: + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From a1cd509d6975c265b544b1fb5509650d090bdd0a Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 09:02:40 -0700 Subject: [PATCH 02/13] chore: make external benchmarks importable --- benchmarks/external/__init__.py | 1 + 1 file changed, 1 insertion(+) create mode 100644 benchmarks/external/__init__.py diff --git a/benchmarks/external/__init__.py b/benchmarks/external/__init__.py new file mode 100644 index 0000000..9f0a6ca --- /dev/null +++ b/benchmarks/external/__init__.py @@ -0,0 +1 @@ +"""External benchmark adapters for Practical Coding.""" From 8afa6dc52ee0eb2f31b944fbd70747a21767b5fd Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 09:02:53 -0700 Subject: [PATCH 03/13] feat: add one-command external benchmark entrypoint --- benchmarks/run_external.ps1 | 55 +++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 benchmarks/run_external.ps1 diff --git a/benchmarks/run_external.ps1 b/benchmarks/run_external.ps1 new file mode 100644 index 0000000..be797cc --- /dev/null +++ b/benchmarks/run_external.ps1 @@ -0,0 +1,55 @@ +param( + [ValidateSet("skillsbench")] + [string]$Benchmark = "skillsbench", + [ValidateSet("smoke", "standard", "full")] + [string]$Profile = "standard", + [int]$Runs = 0, + [int]$Workers = 3, + [ValidateSet("docker", "daytona", "modal", "apple-container", "agentcore")] + [string]$Sandbox = "docker", + [string]$Model = "gpt-5.6-luna", + [string]$Reasoning = "medium", + [string]$Output = "", + [string]$CacheRoot = "", + [string[]]$Task = @(), + [switch]$SkipOracle, + [switch]$RequireStableRanking, + [switch]$SelfTest +) + +$ErrorActionPreference = "Stop" +$env:PYTHONUTF8 = "1" +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$repoRoot = Split-Path -Parent $scriptDir + +if ($Benchmark -ne "skillsbench") { + Write-Error "Unsupported external benchmark: $Benchmark" + exit 2 +} + +$adapter = Join-Path $scriptDir "external/skillsbench_adapter.py" +$arguments = @( + $adapter, + "--profile", $Profile, + "--workers", $Workers, + "--sandbox", $Sandbox, + "--model", $Model, + "--reasoning", $Reasoning +) + +if ($Runs -gt 0) { $arguments += @("--runs", $Runs) } +if ($Output) { $arguments += @("--output", $Output) } +if ($CacheRoot) { $arguments += @("--cache-root", $CacheRoot) } +foreach ($item in $Task) { $arguments += @("--task", $item) } +if ($SkipOracle) { $arguments += "--skip-oracle" } +if ($RequireStableRanking) { $arguments += "--require-stable-ranking" } +if ($SelfTest) { $arguments += "--self-test" } + +Push-Location $repoRoot +try { + & python @arguments + exit $LASTEXITCODE +} +finally { + Pop-Location +} From 0e3979f28295f9637cab0599eb3a5d3eff5c75ce Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 09:03:25 -0700 Subject: [PATCH 04/13] test: cover SkillsBench adapter and stable lift gate --- benchmarks/test_external_skillsbench.py | 161 ++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100644 benchmarks/test_external_skillsbench.py diff --git a/benchmarks/test_external_skillsbench.py b/benchmarks/test_external_skillsbench.py new file mode 100644 index 0000000..88dd234 --- /dev/null +++ b/benchmarks/test_external_skillsbench.py @@ -0,0 +1,161 @@ +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from benchmarks.external import skillsbench_adapter as adapter + + +class SkillsBenchAdapterTests(unittest.TestCase): + def test_discovers_versioned_roster_and_software_engineering_profile(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + registry = [ + { + "name": "skillsbench", + "version": "1.1", + "tasks": [ + {"name": "se-a"}, + {"name": "office-a"}, + {"name": "se-b"}, + ], + } + ] + (root / "registry.json").write_text(json.dumps(registry), encoding="utf-8") + for name, category in (("se-a", "software-engineering"), ("office-a", "office-white-collar"), ("se-b", "software-engineering")): + task = root / "tasks" / name + task.mkdir(parents=True) + (task / "task.md").write_text( + f"---\nmetadata:\n category: {category}\n---\nTask\n", + encoding="utf-8", + ) + + self.assertEqual(adapter.discover_tasks(root, "standard"), ["se-a", "se-b"]) + self.assertEqual(adapter.discover_tasks(root, "full"), ["se-a", "office-a", "se-b"]) + self.assertEqual(adapter.discover_tasks(root, "standard", ["se-b"]), ["se-b"]) + with self.assertRaises(ValueError): + adapter.discover_tasks(root, "standard", ["missing"]) + + def test_bench_commands_pin_dataset_and_mount_only_custom_skill(self): + with patch.object(adapter, "resolve_uvx", return_value=["uvx", "--from", "benchflow==0.6.2", "bench"]): + baseline = adapter.bench_command( + jobs_dir=Path("base"), + tasks=["a", "b"], + sandbox="docker", + workers=2, + model="gpt-5.6-luna", + reasoning="medium", + skill_mode="no-skill", + ) + trained = adapter.bench_command( + jobs_dir=Path("trained"), + tasks=["a", "b"], + sandbox="docker", + workers=2, + model="gpt-5.6-luna", + reasoning="medium", + skill_mode="with-skill", + skills_root=Path("skills"), + ) + self.assertIn("skillsbench@1.1", baseline) + self.assertIn("codex-acp", baseline) + self.assertEqual(baseline.count("--include"), 2) + self.assertNotIn("--skills-dir", baseline) + self.assertIn("--skills-dir", trained) + self.assertIn("skills", trained) + self.assertIn("with-skill", trained) + + def test_job_parser_supports_benchflow_reward_shapes(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + first = root / "a" + second = root / "b" + first.mkdir() + second.mkdir() + (first / "result.json").write_text( + json.dumps({"task_name": "a", "rewards": {"reward": 1.0}}), + encoding="utf-8", + ) + (second / "result.json").write_text( + json.dumps({"task_id": "b", "reward": 0.25}), + encoding="utf-8", + ) + rows = adapter.load_job_rewards(root) + self.assertTrue(rows["a"]["passed"]) + self.assertEqual(rows["b"]["reward"], 0.25) + self.assertIsNone(rows["b"]["excluded_reason"]) + + def test_job_parser_rejects_duplicate_rollouts(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for index in (1, 2): + cell = root / str(index) + cell.mkdir() + (cell / "result.json").write_text( + json.dumps({"task_name": "same", "reward": 1.0}), + encoding="utf-8", + ) + with self.assertRaises(RuntimeError): + adapter.load_job_rewards(root) + + def test_stable_gate_requires_three_complete_paired_runs_and_oracle(self): + tasks = ["a", "b"] + pairs = [] + for repetition in (1, 2, 3): + for task in tasks: + pairs.append( + { + "task": task, + "repetition": repetition, + "reward_base": 0.0 if task == "a" else 1.0, + "reward_practical": 1.0, + "passed_base": task != "a", + "passed_practical": True, + } + ) + stable = adapter.summarize_pairs(pairs, tasks, 3, [], True) + self.assertTrue(stable["stable"]) + self.assertEqual(stable["paired_rollouts"], 6) + self.assertEqual(stable["pass_rate_delta"], 0.5) + self.assertFalse(adapter.summarize_pairs(pairs[:2], tasks, 1, [], True)["stable"]) + self.assertFalse(adapter.summarize_pairs(pairs, tasks, 3, [], False)["stable"]) + self.assertFalse(adapter.summarize_pairs(pairs[:-1], tasks, 3, [{"task": "b"}], True)["stable"]) + + def test_cluster_bootstrap_is_deterministic_and_clustered_by_task(self): + pairs = [] + for repetition in (1, 2, 3): + pairs.extend( + [ + {"task": "a", "passed_base": False, "passed_practical": True, "reward_base": 0.0, "reward_practical": 1.0}, + {"task": "b", "passed_base": True, "passed_practical": True, "reward_base": 1.0, "reward_practical": 1.0}, + ] + ) + one = adapter.cluster_bootstrap(pairs, samples=200, seed=7) + two = adapter.cluster_bootstrap(pairs, samples=200, seed=7) + self.assertEqual(one, two) + self.assertLessEqual(one["pass_rate_delta"][0], 0.5) + self.assertGreaterEqual(one["pass_rate_delta"][1], 0.5) + + def test_stage_skill_copies_entrypoint_and_references(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) / "skill" + output = Path(tmp) / "out" + (root / "references").mkdir(parents=True) + (root / "SKILL.md").write_text("# skill\n", encoding="utf-8") + (root / "references" / "debugging.md").write_text("# debug\n", encoding="utf-8") + (root / "README.md").write_text("not part of the skill bundle\n", encoding="utf-8") + original = adapter.ROOT + adapter.ROOT = root + try: + skills_root = adapter.stage_practical_skill(output) + finally: + adapter.ROOT = original + staged = skills_root / "practical-coding" + self.assertTrue((staged / "SKILL.md").is_file()) + self.assertTrue((staged / "references" / "debugging.md").is_file()) + self.assertFalse((staged / "README.md").exists()) + + +if __name__ == "__main__": + unittest.main() From 0bd32f73fcc8206d8b0c5efa368718700a6f471e Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 09:03:37 -0700 Subject: [PATCH 05/13] ci: validate external benchmark adapter --- .github/workflows/validate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index 6f21950..6e3227d 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -22,7 +22,7 @@ jobs: run: skills-ref validate ./practical-coding - name: Run benchmark harness tests working-directory: practical-coding - run: python -m unittest benchmarks.test_benchmarks benchmarks.test_stability benchmarks.test_catalog + run: python -m unittest benchmarks.test_benchmarks benchmarks.test_stability benchmarks.test_catalog benchmarks.test_external_skillsbench - name: Check Codex default_prompt references the skill as $skill-name run: grep -qF '$practical-coding' practical-coding/agents/openai.yaml - name: Ensure legacy local graph runtime is not reintroduced From a4724c292026e317f242854d53d6cda8bb68764f Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 09:04:16 -0700 Subject: [PATCH 06/13] docs: document one-command SkillsBench evaluation --- benchmarks/external/README.md | 112 ++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100644 benchmarks/external/README.md diff --git a/benchmarks/external/README.md b/benchmarks/external/README.md new file mode 100644 index 0000000..4d35fe2 --- /dev/null +++ b/benchmarks/external/README.md @@ -0,0 +1,112 @@ +# External benchmarks + +Practical Coding keeps external evidence separate from its project-owned regression suites. The first executable adapter targets the immutable `skillsbench@1.1` dataset through BenchFlow. + +## SkillsBench + +The adapter compares the same Codex/model configuration under two treatments: + +- `no-skill`: no Agent Skill is mounted. +- `practical`: only the current repository's `practical-coding` Skill bundle is mounted through BenchFlow's custom `--skills-dir` path. + +This deliberately does **not** use each SkillsBench task's curated Skill. The result measures Practical Coding's lift on SkillsBench tasks; it is not an official SkillsBench leaderboard submission for the benchmark's curated-Skill condition. + +### Prerequisites + +- Python 3.12+ +- Git +- `uv`/`uvx` +- Codex CLI authenticated with `codex login` or a supported Codex/OpenAI credential +- Docker for the default `docker` sandbox; Daytona, Modal, Apple Container, and AgentCore can be selected explicitly when configured + +BenchFlow itself is launched on demand through `uvx` and pinned to `benchflow==0.6.2`, which is inside the compatibility range declared by SkillsBench v1.1. The adapter also keeps a metadata checkout at the exact `v1.1` tag for category selection. Actual benchmark execution uses `-d skillsbench@1.1`, so BenchFlow resolves the versioned dataset and validates its task digests. + +### Fast instrument check + +No model, network, or Docker calls: + +```powershell +pwsh -File benchmarks/run_external.ps1 -Benchmark skillsbench -SelfTest +``` + +### Smoke + +Three software-engineering tasks, one run per arm. This is only a plumbing check and is always provisional: + +```powershell +pwsh -File benchmarks/run_external.ps1 ` + -Benchmark skillsbench ` + -Profile smoke +``` + +### Stable software-engineering lift + +`standard` discovers every `software-engineering` task in the SkillsBench v1.1 registry roster and runs three separately materialized paired repetitions by default: + +```powershell +pwsh -File benchmarks/run_external.ps1 ` + -Benchmark skillsbench ` + -Profile standard ` + -Runs 3 ` + -Workers 3 ` + -RequireStableRanking +``` + +Before any model calls, the adapter runs the SkillsBench oracle across the selected task set. A stable result requires: + +1. oracle reward `1.0` for every selected task; +2. at least three repetitions; +3. exactly one healthy `no-skill` and one healthy `practical` result for every task/repetition pair; +4. no missing or unhealthy pair. + +Behavioral failures remain valid data. Infrastructure failures or missing rewards make the evidence provisional or abort the run. + +### Full cross-domain interference run + +`full` uses the complete `skillsbench@1.1` registry roster. This is intentionally expensive and is mainly useful for measuring whether a general coding Skill causes irrelevant-domain interference: + +```powershell +pwsh -File benchmarks/run_external.ps1 ` + -Benchmark skillsbench ` + -Profile full ` + -Runs 3 ` + -Workers 3 ` + -RequireStableRanking +``` + +Use `-Task ` repeatedly to run an explicit versioned subset. + +### Outputs + +Artifacts are written under `benchmark-results/external/skillsbench-/` unless `-Output` is supplied: + +```text +manifest.json dataset/model/environment pins, task roster, Skill hash, commands +oracle/ BenchFlow oracle jobs +oracle.log oracle command output +runs/ + r001/ + no-skill/ raw BenchFlow jobs + practical/ raw BenchFlow jobs + no-skill.log + practical.log + ... +pairs.json healthy task/repetition pairs used for comparison +summary.json pass/reward lift, win/loss/tie counts, 95% CIs, per-task rates +report.md human-readable external lift report +staged-skills/ exact Practical Coding bundle mounted into BenchFlow +``` + +The two model arms alternate execution order on successive repetitions. Confidence intervals use a deterministic task-cluster bootstrap: task IDs are resampled as clusters while all repeated trials for the sampled task stay together. + +### Interpretation + +The primary result is pass-rate lift: + +```text +Codex/Luna no Skill + vs +Codex/Luna + Practical Coding +``` + +A positive delta is external evidence that Practical Coding improves resolution on the selected public SkillsBench tasks. Because SkillsBench is public, this remains external public evidence rather than a private holdout. Do not fold these scores into the project-owned Router/Decision/Debug regression rollups. From dda425cbba8aefca433821a36fb625baad2b8253 Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 09:04:51 -0700 Subject: [PATCH 07/13] docs: expose executable SkillsBench lift workflow --- benchmarks/README.md | 47 +++++++++++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 11 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index ba40048..579ca89 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -2,9 +2,9 @@ This chain runs isolated Codex sessions directly against `gpt-5.6-luna`, preserves every prompt/transcript/workspace, applies mechanical graders, and writes JSON plus Markdown summaries. It follows the mature evaluation shape used by Agent Skills and Ponytail: realistic cases, fixed sources, clean sessions, repeated paired arms, deterministic assertions where possible, tokens/time, and raw evidence. -For prerequisites, pinned revisions, exact reproduction commands, evidence boundaries, and the published v1.11 calibration results, see [`REPRODUCING.md`](REPRODUCING.md). For the external benchmark landscape and the public-regression/external/held-out evidence model, see [`../docs/evaluations/2026-08-24-benchmark-landscape.md`](../docs/evaluations/2026-08-24-benchmark-landscape.md). +For prerequisites, pinned revisions, exact reproduction commands, evidence boundaries, and the published v1.11 calibration results, see [`REPRODUCING.md`](REPRODUCING.md). For the external benchmark landscape and the public-regression/external/held-out evidence model, see [`../docs/evaluations/2026-08-24-benchmark-landscape.md`](../docs/evaluations/2026-08-24-benchmark-landscape.md). The executable external SkillsBench workflow is documented in [`external/README.md`](external/README.md). -## Run +## Run the project-owned benchmark ```powershell pwsh -File benchmarks/run.ps1 -Profile smoke @@ -15,9 +15,9 @@ pwsh -File benchmarks/run.ps1 -Profile smoke -Suite router -Case direct-artifact pwsh -File benchmarks/run.ps1 -Rescore D:\path\to\benchmark-results\20260824-203839 ``` -`run.ps1` is the canonical entrypoint. It loads the core runner through `run_catalog.py`, which installs the extended public case catalog before execution. This keeps benchmark mechanics separate from the evolving task corpus. `manifest.json` fingerprints the complete benchmark runtime bundle (core runner + case catalog + canonical wrapper), so task/scorer changes cannot masquerade as the same benchmark revision. +`run.ps1` is the canonical internal entrypoint. It loads the core runner through `run_catalog.py`, which installs the extended public case catalog before execution. This keeps benchmark mechanics separate from the evolving task corpus. `manifest.json` fingerprints the complete benchmark runtime bundle (core runner + case catalog + canonical wrapper), so task/scorer changes cannot masquerade as the same benchmark revision. -For a result that will be presented as a stable ranking, opt into the evidence gate: +For an internal result that will be presented as a stable ranking, opt into the evidence gate: ```powershell pwsh -File benchmarks/run.ps1 -Profile standard -Runs 3 -Workers 3 -RequireStableRanking @@ -32,7 +32,29 @@ python benchmarks/check_stability.py benchmark-results\v111-delivery-n1-core-rev That command intentionally reports the published v1.11 Delivery `n=1` artifact as `PROVISIONAL`; it must not be used for a stable ranking until the same cells are rerun with at least three distinct repetitions. -## Profiles +## Run the external SkillsBench lift + +The external adapter uses BenchFlow's immutable `skillsbench@1.1` dataset and compares the same Codex/Luna configuration with no Skill against a custom Skill directory containing only Practical Coding. + +```powershell +# Instrument-only self-test; no model calls. +pwsh -File benchmarks/run_external.ps1 -Benchmark skillsbench -SelfTest + +# Fast three-task plumbing check; n=1 and provisional. +pwsh -File benchmarks/run_external.ps1 -Benchmark skillsbench -Profile smoke + +# Stable external software-engineering lift. +pwsh -File benchmarks/run_external.ps1 ` + -Benchmark skillsbench ` + -Profile standard ` + -Runs 3 ` + -Workers 3 ` + -RequireStableRanking +``` + +`standard` dynamically selects every `software-engineering` task in the SkillsBench v1.1 registry roster. `full` runs the complete versioned roster and is intended mainly as a cross-domain interference check. The adapter pins BenchFlow, runs the SkillsBench oracle before model calls, alternates arm order across repetitions, preserves raw BenchFlow jobs, and reports pass/reward lift with task-cluster bootstrap confidence intervals. It is a Practical-owned custom-Skill ablation on SkillsBench, not an official SkillsBench leaderboard submission using the benchmark's per-task curated Skills. + +## Internal profiles | Profile | Delivery | Router | Decision | Debug | Default runs | Cells without previous/no-Skill arm | |---|---:|---:|---:|---:|---:|---:| @@ -42,7 +64,7 @@ That command intentionally reports the published v1.11 Delivery `n=1` artifact a `standard` is the normal release gate. `full` carries the complete public regression matrix. The extra Router cases span all six routes; the expanded Debug set covers twelve cases across parsing, normalization, tenant isolation, pagination, units, row handling, state invariants, TTL semantics, URL handling, and the upstream transfer/amount tasks. Decision grows from four to ten two-turn decisions in `full`. -Useful options: +Useful internal options: - `-BaselineSkill ` adds a previous Practical snapshot to every suite. The directory must contain `SKILL.md` and its `references` directory. - `-BaselineRef ` materializes `SKILL.md` plus `references/` from a commit into the run artifact and adds it as `practical-previous`. This is the simplest before/after gate for dirty candidate edits. @@ -60,10 +82,11 @@ By default, run artifacts are written under `benchmark-results/` and ignored by The fast harness regression suite is also runnable without model calls: ```powershell -python -m unittest benchmarks.test_benchmarks benchmarks.test_stability benchmarks.test_catalog +python -m unittest benchmarks.test_benchmarks benchmarks.test_stability benchmarks.test_catalog benchmarks.test_external_skillsbench +python benchmarks/external/skillsbench_adapter.py --self-test ``` -The output directory contains: +The internal output directory contains: ```text manifest.json fixed model, commits, profile, cases, and skill hashes @@ -73,7 +96,7 @@ comparisons.json Practical-minus-comparator behavioral and efficiency del rollups.json suite/arm totals across cases rollup-comparisons.json suite-level Practical-minus-comparator deltas report.md human-readable comparison and Practical deltas -cells/ prompt, raw JSONL, stderr, answer, and code workspace per cell +cells/ prompt, raw JSONL, stderr, answers, workspaces per cell ``` ## Suites and scoring @@ -89,6 +112,8 @@ cells/ prompt, raw JSONL, stderr, answer, and code workspace pe Use repeated paired results. A candidate is not accepted merely because its prose matches a Skill contract. Require no correctness/build regression, then compare delivered code and behavior. Treat LOC, tokens, and time as secondary within equally correct artifacts. `n=1` is a smoke result, not a stable ranking. -A published stable ranking must pass `benchmarks/check_stability.py` with the default minimum `n=3`. The gate checks distinct repetition IDs, complete-run metadata, and infrastructure errors. Behavioral or build failures remain valid benchmark observations and therefore do not invalidate the sample by themselves. +A published internal stable ranking must pass `benchmarks/check_stability.py` with the default minimum `n=3`. The gate checks distinct repetition IDs, complete-run metadata, and infrastructure errors. Behavioral or build failures remain valid benchmark observations and therefore do not invalidate the sample by themselves. + +A published SkillsBench external lift must pass the adapter's independent stable gate: oracle success, at least three runs, and one healthy result from each arm for every selected task/repetition pair. -The public catalog is a **regression suite**, not a hidden generalization test. Once a case has influenced Skill wording, its future 100% score should be treated as a ceiling check. External benchmarks and a private held-out set are required for stronger claims. +The public catalog is a **regression suite**, not a hidden generalization test. Once a case has influenced Skill wording, its future 100% score should be treated as a ceiling check. SkillsBench provides external public evidence; a private held-out set is still required for the strongest generalization claims. From 909b9ba9029a066f718b84b99c7675f7a9145179 Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 09:05:19 -0700 Subject: [PATCH 08/13] docs: mark SkillsBench external adapter executable --- .../2026-08-24-benchmark-landscape.md | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/evaluations/2026-08-24-benchmark-landscape.md b/docs/evaluations/2026-08-24-benchmark-landscape.md index 8b3e4f8..00bb44d 100644 --- a/docs/evaluations/2026-08-24-benchmark-landscape.md +++ b/docs/evaluations/2026-08-24-benchmark-landscape.md @@ -21,7 +21,7 @@ This note separates project regression tests from external benchmark evidence. P Use three layers instead of one leaderboard number: 1. **Public regression layer** — Router, Decision, Debug and the Ponytail-derived Delivery tasks in this repository. These should stay deterministic and are allowed to encode previously observed failures. Their job is to prevent regression, not prove unseen-task generalization. -2. **External benchmark layer** — run the Skill as an augmentation treatment on independent public suites. SkillsBench is the first target because it explicitly measures with-Skills versus without-Skills. FeatureBench and a standardized SWE-bench/Terminal-Bench harness provide broader coding-agent validity. +2. **External benchmark layer** — run the Skill as an augmentation treatment on independent public suites. SkillsBench is executable through `benchmarks/run_external.ps1`: the same Codex/Luna configuration is paired as `no-skill` versus a custom Skill directory containing only Practical Coding. FeatureBench and a standardized SWE-bench/Terminal-Bench harness remain broader coding-agent validity targets. 3. **Held-out layer** — keep a small private task set that is not read while editing `SKILL.md` or references. Rotate or refresh it from recent real PRs/bugs. Only this layer should be used for claims that a prompt iteration generalized beyond the public regression corpus. ## Task-authoring rules adopted here @@ -36,6 +36,25 @@ The expanded public catalog follows these rules: - `standard` is a bounded release gate while `full` contains the complete public regression matrix; - stable rankings still require at least `n=3`; public 100% results are described as regression ceilings, not generalization proof. +## SkillsBench adapter status + +Implemented in `benchmarks/external/skillsbench_adapter.py` with the canonical PowerShell entrypoint `benchmarks/run_external.ps1`. + +The adapter: + +- pins execution to the immutable `skillsbench@1.1` dataset through BenchFlow; +- keeps a `v1.1` SkillsBench metadata checkout only for category/roster selection; +- pins BenchFlow to `0.6.2`, inside the dataset's declared compatibility range; +- runs the benchmark oracle before model calls by default; +- compares `codex-acp` + `gpt-5.6-luna` + medium reasoning under `no-skill` and `with-skill` treatments; +- mounts only the current Practical Coding bundle in the treatment arm rather than each task's curated SkillsBench Skill; +- alternates arm execution order across repetitions; +- requires three complete paired repetitions for stable evidence; +- records raw BenchFlow jobs, the selected task roster, source/version pins, adapter hash, Skill bundle hash, and exact commands; +- reports pass-rate lift, mean-reward lift, pass flips, normalized gain, per-task rates, and task-cluster-bootstrap 95% confidence intervals. + +This is intentionally described as a **Practical-owned ablation on SkillsBench**, not an official SkillsBench leaderboard row. + ## Next external-validation milestone -The highest-value next step is not another Core rule. It is an adapter that can run `practical-coding` as a treatment on an independent benchmark without rewriting that benchmark's tasks. Start with SkillsBench because its paired methodology matches the project's goal, then add a small FeatureBench fast-split experiment. Keep model, harness, reasoning effort, task version, and Skill bundle hash fixed for every paired comparison. +After a stable SkillsBench software-engineering run is recorded, the next useful adapter is FeatureBench's fast/lite split for larger feature implementation and exploration behavior. That should preserve the same evidence discipline: fixed model/harness/reasoning, immutable dataset version or commit, paired no-Skill/Practical arms, oracle/gold validation, raw artifacts, and a stable repeated-run gate. From 604aef9778754717c18c68a10da6dd87d072a829 Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 09:08:46 -0700 Subject: [PATCH 09/13] fix: align external sandbox options with BenchFlow 0.6.5 --- benchmarks/run_external.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmarks/run_external.ps1 b/benchmarks/run_external.ps1 index be797cc..1f0cc77 100644 --- a/benchmarks/run_external.ps1 +++ b/benchmarks/run_external.ps1 @@ -5,7 +5,7 @@ param( [string]$Profile = "standard", [int]$Runs = 0, [int]$Workers = 3, - [ValidateSet("docker", "daytona", "modal", "apple-container", "agentcore")] + [ValidateSet("docker", "daytona", "modal")] [string]$Sandbox = "docker", [string]$Model = "gpt-5.6-luna", [string]$Reasoning = "medium", From d171a4262ad83e2aefa3909c6492d0a937591326 Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 09:09:14 -0700 Subject: [PATCH 10/13] test: pin tested BenchFlow CLI contract --- benchmarks/test_external_skillsbench.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmarks/test_external_skillsbench.py b/benchmarks/test_external_skillsbench.py index 88dd234..46aa8b5 100644 --- a/benchmarks/test_external_skillsbench.py +++ b/benchmarks/test_external_skillsbench.py @@ -38,7 +38,7 @@ def test_discovers_versioned_roster_and_software_engineering_profile(self): adapter.discover_tasks(root, "standard", ["missing"]) def test_bench_commands_pin_dataset_and_mount_only_custom_skill(self): - with patch.object(adapter, "resolve_uvx", return_value=["uvx", "--from", "benchflow==0.6.2", "bench"]): + with patch.object(adapter, "resolve_uvx", return_value=["uvx", "--from", "benchflow==0.6.5", "bench"]): baseline = adapter.bench_command( jobs_dir=Path("base"), tasks=["a", "b"], @@ -58,10 +58,12 @@ def test_bench_commands_pin_dataset_and_mount_only_custom_skill(self): skill_mode="with-skill", skills_root=Path("skills"), ) + self.assertIn("benchflow==0.6.5", baseline) self.assertIn("skillsbench@1.1", baseline) self.assertIn("codex-acp", baseline) self.assertEqual(baseline.count("--include"), 2) self.assertNotIn("--skills-dir", baseline) + self.assertNotIn("--quiet", baseline) self.assertIn("--skills-dir", trained) self.assertIn("skills", trained) self.assertIn("with-skill", trained) From 687006ab579ee61d747fa7c8953267b3949efbba Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 09:09:34 -0700 Subject: [PATCH 11/13] docs: pin external runner to compatible BenchFlow release --- benchmarks/external/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/external/README.md b/benchmarks/external/README.md index 4d35fe2..1088a67 100644 --- a/benchmarks/external/README.md +++ b/benchmarks/external/README.md @@ -17,9 +17,9 @@ This deliberately does **not** use each SkillsBench task's curated Skill. The re - Git - `uv`/`uvx` - Codex CLI authenticated with `codex login` or a supported Codex/OpenAI credential -- Docker for the default `docker` sandbox; Daytona, Modal, Apple Container, and AgentCore can be selected explicitly when configured +- Docker for the default `docker` sandbox; Daytona or Modal can be selected explicitly when configured -BenchFlow itself is launched on demand through `uvx` and pinned to `benchflow==0.6.2`, which is inside the compatibility range declared by SkillsBench v1.1. The adapter also keeps a metadata checkout at the exact `v1.1` tag for category selection. Actual benchmark execution uses `-d skillsbench@1.1`, so BenchFlow resolves the versioned dataset and validates its task digests. +BenchFlow itself is launched on demand through `uvx` and pinned to `benchflow==0.6.5`, a tested 0.6.x release compatible with SkillsBench v1.1 and its `bench eval run` / versioned-dataset / custom-Skill CLI contract. The adapter also keeps a metadata checkout at the exact `v1.1` tag for category selection. Actual benchmark execution uses `-d skillsbench@1.1`, so BenchFlow resolves the versioned dataset and validates its task digests. ### Fast instrument check From b38f171c22fbefafbbd1a58f7cffabf16145e472 Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 09:09:57 -0700 Subject: [PATCH 12/13] docs: record compatible external runner pin --- docs/evaluations/2026-08-24-benchmark-landscape.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/evaluations/2026-08-24-benchmark-landscape.md b/docs/evaluations/2026-08-24-benchmark-landscape.md index 00bb44d..3849205 100644 --- a/docs/evaluations/2026-08-24-benchmark-landscape.md +++ b/docs/evaluations/2026-08-24-benchmark-landscape.md @@ -44,7 +44,7 @@ The adapter: - pins execution to the immutable `skillsbench@1.1` dataset through BenchFlow; - keeps a `v1.1` SkillsBench metadata checkout only for category/roster selection; -- pins BenchFlow to `0.6.2`, inside the dataset's declared compatibility range; +- pins BenchFlow to `0.6.5`, a tested 0.6.x release whose CLI supports versioned datasets, `bench eval run`, custom `--skills-dir`, repeated `--include`, and reasoning effort; - runs the benchmark oracle before model calls by default; - compares `codex-acp` + `gpt-5.6-luna` + medium reasoning under `no-skill` and `with-skill` treatments; - mounts only the current Practical Coding bundle in the treatment arm rather than each task's curated SkillsBench Skill; From 99e0147a78c9dbd436204bb64b17cb8ea4b49b18 Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 09:11:46 -0700 Subject: [PATCH 13/13] fix: pin SkillsBench adapter to verified BenchFlow 0.6.5 contract --- benchmarks/external/skillsbench_adapter.py | 446 +++++++++++++++------ 1 file changed, 329 insertions(+), 117 deletions(-) diff --git a/benchmarks/external/skillsbench_adapter.py b/benchmarks/external/skillsbench_adapter.py index 4737113..0f58f24 100644 --- a/benchmarks/external/skillsbench_adapter.py +++ b/benchmarks/external/skillsbench_adapter.py @@ -23,7 +23,7 @@ DATASET_VERSION = "1.1" SKILLSBENCH_REPO = "https://github.com/benchflow-ai/skillsbench.git" SKILLSBENCH_REF = "v1.1" -BENCHFLOW_VERSION = "0.6.2" +BENCHFLOW_VERSION = "0.6.5" AGENT = "codex-acp" MODEL = "gpt-5.6-luna" REASONING = "medium" @@ -36,7 +36,9 @@ AUTH_ENV_VARS = ("OPENAI_API_KEY", "CODEX_API_KEY", "CODEX_ACCESS_TOKEN") -def run_command(command: list[str], cwd: Path | None = None, timeout: float | None = None) -> subprocess.CompletedProcess[str]: +def run_command( + command: list[str], cwd: Path | None = None, timeout: float | None = None +) -> subprocess.CompletedProcess[str]: return subprocess.run( command, cwd=str(cwd) if cwd else None, @@ -75,23 +77,38 @@ def resolve_uvx() -> list[str]: return [uvx, "--from", f"benchflow=={BENCHFLOW_VERSION}", "bench"] uv = shutil.which("uv") if uv: - return [uv, "tool", "run", "--from", f"benchflow=={BENCHFLOW_VERSION}", "bench"] - raise FileNotFoundError("uv/uvx is required. Install uv, then rerun the external benchmark.") + return [ + uv, + "tool", + "run", + "--from", + f"benchflow=={BENCHFLOW_VERSION}", + "bench", + ] + raise FileNotFoundError( + "uv/uvx is required. Install uv, then rerun the external benchmark." + ) def check_prerequisites(sandbox: str) -> dict[str, str]: - if not shutil.which("git"): - raise FileNotFoundError("git is required") + git = shutil.which("git") codex = shutil.which("codex") + if not git: + raise FileNotFoundError("git is required") if not codex: - raise FileNotFoundError("Codex CLI is required for the codex-acp SkillsBench arm") + raise FileNotFoundError( + "Codex CLI is required for the codex-acp SkillsBench arm" + ) uv_prefix = resolve_uvx() auth_json = Path.home() / ".codex" / "auth.json" if not any(os.environ.get(name) for name in AUTH_ENV_VARS) and not auth_json.is_file(): - raise RuntimeError("Codex authentication was not found. Run `codex login` or provide a supported Codex/OpenAI credential.") + raise RuntimeError( + "Codex authentication was not found. Run `codex login` or provide a supported Codex/OpenAI credential." + ) + versions: dict[str, str] = {} for name, command in ( - ("git", [shutil.which("git") or "git", "--version"]), + ("git", [git, "--version"]), ("codex", [codex, "--version"]), ("uv", [uv_prefix[0], "--version"]), ): @@ -99,6 +116,7 @@ def check_prerequisites(sandbox: str) -> dict[str, str]: if result.returncode: raise RuntimeError(f"failed to inspect {name}: {result.stdout[-1000:]}") versions[name] = result.stdout.strip() + if sandbox == "docker": docker = shutil.which("docker") if not docker: @@ -114,23 +132,34 @@ def ensure_skillsbench_checkout(cache_root: Path) -> tuple[Path, str]: checkout = cache_root / f"skillsbench-{DATASET_VERSION}" checkout.parent.mkdir(parents=True, exist_ok=True) if not checkout.exists(): - clone = run_command([ - shutil.which("git") or "git", - "clone", - "--filter=blob:none", - "--depth", - "1", - "--branch", - SKILLSBENCH_REF, - SKILLSBENCH_REPO, - str(checkout), - ]) + clone = run_command( + [ + shutil.which("git") or "git", + "clone", + "--filter=blob:none", + "--depth", + "1", + "--branch", + SKILLSBENCH_REF, + SKILLSBENCH_REPO, + str(checkout), + ] + ) if clone.returncode: - raise RuntimeError(f"failed to clone SkillsBench {SKILLSBENCH_REF}: {clone.stdout[-3000:]}") - head = run_command([shutil.which("git") or "git", "rev-parse", "HEAD"], checkout) - tag = run_command([shutil.which("git") or "git", "describe", "--tags", "--exact-match"], checkout) + raise RuntimeError( + f"failed to clone SkillsBench {SKILLSBENCH_REF}: {clone.stdout[-3000:]}" + ) + head = run_command( + [shutil.which("git") or "git", "rev-parse", "HEAD"], checkout + ) + tag = run_command( + [shutil.which("git") or "git", "describe", "--tags", "--exact-match"], + checkout, + ) if head.returncode or tag.returncode or tag.stdout.strip() != SKILLSBENCH_REF: - raise RuntimeError(f"SkillsBench metadata checkout must be exactly {SKILLSBENCH_REF}") + raise RuntimeError( + f"SkillsBench metadata checkout must be exactly {SKILLSBENCH_REF}" + ) return checkout, head.stdout.strip() @@ -150,11 +179,15 @@ def task_category(task_md: Path) -> str | None: frontmatter = text.split("---", 2) if len(frontmatter) < 3: return None - match = re.search(r"(?m)^\s*category:\s*['\"]?([a-z0-9-]+)", frontmatter[1]) + match = re.search( + r"(?m)^\s*category:\s*['\"]?([a-z0-9-]+)", frontmatter[1] + ) return match.group(1) if match else None -def discover_tasks(checkout: Path, profile: str, explicit: Iterable[str] = ()) -> list[str]: +def discover_tasks( + checkout: Path, profile: str, explicit: Iterable[str] = () +) -> list[str]: roster = load_dataset_roster(checkout) roster_set = set(roster) explicit = [item for item in explicit if item] @@ -163,10 +196,12 @@ def discover_tasks(checkout: Path, profile: str, explicit: Iterable[str] = ()) - if unknown: raise ValueError(f"tasks are not in {DATASET}: {unknown}") return list(dict.fromkeys(explicit)) + software = [ name for name in roster - if task_category(checkout / "tasks" / name / "task.md") == "software-engineering" + if task_category(checkout / "tasks" / name / "task.md") + == "software-engineering" ] if profile == "smoke": preferred = [name for name in SMOKE_TASKS if name in software] @@ -193,6 +228,13 @@ def stage_practical_skill(output: Path) -> Path: return skills_root +def _selection_args(tasks: list[str]) -> list[str]: + args = ["--expected-tasks", str(len(tasks))] + for task in tasks: + args += ["--include", task] + return args + + def bench_command( *, jobs_dir: Path, @@ -224,19 +266,19 @@ def bench_command( str(jobs_dir), "--skill-mode", skill_mode, - "--quiet", + *_selection_args(tasks), ] if skill_mode == "with-skill": if skills_root is None: raise ValueError("skills_root is required for with-skill") command += ["--skills-dir", str(skills_root)] - for task in tasks: - command += ["--include", task] return command -def oracle_command(*, jobs_dir: Path, tasks: list[str], sandbox: str, workers: int) -> list[str]: - command = [ +def oracle_command( + *, jobs_dir: Path, tasks: list[str], sandbox: str, workers: int +) -> list[str]: + return [ *resolve_uvx(), "eval", "run", @@ -252,11 +294,8 @@ def oracle_command(*, jobs_dir: Path, tasks: list[str], sandbox: str, workers: i str(jobs_dir), "--skill-mode", "no-skill", - "--quiet", + *_selection_args(tasks), ] - for task in tasks: - command += ["--include", task] - return command def _read_json(path: Path) -> dict[str, Any] | None: @@ -330,16 +369,29 @@ def load_job_rewards(job_dir: Path) -> dict[str, dict[str, Any]]: else: by_task[task_id] = row if duplicates: - details = "; ".join(f"{task}: {sorted(set(paths))}" for task, paths in sorted(duplicates.items())) - raise RuntimeError(f"duplicate SkillsBench result.json files for a task: {details}") + details = "; ".join( + f"{task}: {sorted(set(paths))}" + for task, paths in sorted(duplicates.items()) + ) + raise RuntimeError( + f"duplicate SkillsBench result.json files for a task: {details}" + ) return by_task -def validate_oracle(job_dir: Path, tasks: list[str]) -> tuple[bool, dict[str, Any]]: +def validate_oracle( + job_dir: Path, tasks: list[str] +) -> tuple[bool, dict[str, Any]]: rows = load_job_rewards(job_dir) missing = sorted(set(tasks) - set(rows)) - failed = sorted(task for task in tasks if task in rows and rows[task]["reward"] != 1.0) - unhealthy = sorted(task for task in tasks if task in rows and rows[task]["excluded_reason"] is not None) + failed = sorted( + task for task in tasks if task in rows and rows[task]["reward"] != 1.0 + ) + unhealthy = sorted( + task + for task in tasks + if task in rows and rows[task]["excluded_reason"] is not None + ) return not missing and not failed and not unhealthy, { "expected": len(tasks), "observed": len(rows), @@ -349,32 +401,50 @@ def validate_oracle(job_dir: Path, tasks: list[str]) -> tuple[bool, dict[str, An } -def collect_pairs(output: Path, tasks: list[str], runs: int) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: +def collect_pairs( + output: Path, tasks: list[str], runs: int +) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: pairs: list[dict[str, Any]] = [] gaps: list[dict[str, Any]] = [] for repetition in range(1, runs + 1): rep = output / "runs" / f"r{repetition:03d}" - base = load_job_rewards(rep / "no-skill") if (rep / "no-skill").exists() else {} - practical = load_job_rewards(rep / "practical") if (rep / "practical").exists() else {} + base = ( + load_job_rewards(rep / "no-skill") + if (rep / "no-skill").exists() + else {} + ) + practical = ( + load_job_rewards(rep / "practical") + if (rep / "practical").exists() + else {} + ) for task in tasks: - left = base.get(task) - right = practical.get(task) - if not left or not right or left["excluded_reason"] or right["excluded_reason"]: - gaps.append({ + left, right = base.get(task), practical.get(task) + if ( + not left + or not right + or left["excluded_reason"] + or right["excluded_reason"] + ): + gaps.append( + { + "task": task, + "repetition": repetition, + "baseline": left, + "practical": right, + } + ) + continue + pairs.append( + { "task": task, "repetition": repetition, - "baseline": left, - "practical": right, - }) - continue - pairs.append({ - "task": task, - "repetition": repetition, - "reward_base": left["reward"], - "reward_practical": right["reward"], - "passed_base": left["passed"], - "passed_practical": right["passed"], - }) + "reward_base": left["reward"], + "reward_practical": right["reward"], + "passed_base": left["passed"], + "passed_practical": right["passed"], + } + ) return pairs, gaps @@ -391,13 +461,18 @@ def percentile(values: list[float], p: float) -> float | None: return values[lower] * (1 - fraction) + values[upper] * fraction -def cluster_bootstrap(pairs: list[dict[str, Any]], samples: int = 5000, seed: int = 0) -> dict[str, list[float | None]]: +def cluster_bootstrap( + pairs: list[dict[str, Any]], samples: int = 5000, seed: int = 0 +) -> dict[str, list[float | None]]: groups: dict[str, list[dict[str, Any]]] = defaultdict(list) for pair in pairs: groups[pair["task"]].append(pair) tasks = sorted(groups) if not tasks or samples <= 0: - return {"pass_rate_delta": [None, None], "mean_reward_delta": [None, None]} + return { + "pass_rate_delta": [None, None], + "mean_reward_delta": [None, None], + } rng = random.Random(seed) pass_deltas: list[float] = [] reward_deltas: list[float] = [] @@ -406,43 +481,83 @@ def cluster_bootstrap(pairs: list[dict[str, Any]], samples: int = 5000, seed: in for task in (rng.choice(tasks) for _ in tasks): sample_pairs.extend(groups[task]) pass_deltas.append( - statistics.mean(float(row["passed_practical"]) - float(row["passed_base"]) for row in sample_pairs) + statistics.mean( + float(row["passed_practical"]) - float(row["passed_base"]) + for row in sample_pairs + ) ) reward_deltas.append( - statistics.mean(float(row["reward_practical"]) - float(row["reward_base"]) for row in sample_pairs) + statistics.mean( + float(row["reward_practical"]) - float(row["reward_base"]) + for row in sample_pairs + ) ) return { - "pass_rate_delta": [percentile(pass_deltas, 0.025), percentile(pass_deltas, 0.975)], - "mean_reward_delta": [percentile(reward_deltas, 0.025), percentile(reward_deltas, 0.975)], + "pass_rate_delta": [ + percentile(pass_deltas, 0.025), + percentile(pass_deltas, 0.975), + ], + "mean_reward_delta": [ + percentile(reward_deltas, 0.025), + percentile(reward_deltas, 0.975), + ], } -def summarize_pairs(pairs: list[dict[str, Any]], tasks: list[str], runs: int, gaps: list[dict[str, Any]], oracle_ok: bool) -> dict[str, Any]: +def summarize_pairs( + pairs: list[dict[str, Any]], + tasks: list[str], + runs: int, + gaps: list[dict[str, Any]], + oracle_ok: bool, +) -> dict[str, Any]: if pairs: pass_base = statistics.mean(float(row["passed_base"]) for row in pairs) - pass_practical = statistics.mean(float(row["passed_practical"]) for row in pairs) + pass_practical = statistics.mean( + float(row["passed_practical"]) for row in pairs + ) reward_base = statistics.mean(float(row["reward_base"]) for row in pairs) - reward_practical = statistics.mean(float(row["reward_practical"]) for row in pairs) + reward_practical = statistics.mean( + float(row["reward_practical"]) for row in pairs + ) else: pass_base = pass_practical = reward_base = reward_practical = 0.0 + delta = pass_practical - pass_base - normalized_gain = (delta / (1.0 - pass_base)) if delta > 0 and pass_base < 1.0 else delta + normalized_gain = delta / (1.0 - pass_base) if pass_base < 1.0 else 0.0 by_task: list[dict[str, Any]] = [] for task in tasks: rows = [row for row in pairs if row["task"] == task] if not rows: by_task.append({"task": task, "n": 0}) continue - by_task.append({ - "task": task, - "n": len(rows), - "pass_rate_base": statistics.mean(float(row["passed_base"]) for row in rows), - "pass_rate_practical": statistics.mean(float(row["passed_practical"]) for row in rows), - "mean_reward_base": statistics.mean(float(row["reward_base"]) for row in rows), - "mean_reward_practical": statistics.mean(float(row["reward_practical"]) for row in rows), - }) + by_task.append( + { + "task": task, + "n": len(rows), + "pass_rate_base": statistics.mean( + float(row["passed_base"]) for row in rows + ), + "pass_rate_practical": statistics.mean( + float(row["passed_practical"]) for row in rows + ), + "mean_reward_base": statistics.mean( + float(row["reward_base"]) for row in rows + ), + "mean_reward_practical": statistics.mean( + float(row["reward_practical"]) for row in rows + ), + } + ) + expected_pairs = len(tasks) * runs - stable = runs >= 3 and oracle_ok and len(pairs) == expected_pairs and not gaps and all(row["n"] == runs for row in by_task) + stable = ( + runs >= 3 + and oracle_ok + and len(pairs) == expected_pairs + and not gaps + and all(row["n"] == runs for row in by_task) + ) return { "stable": stable, "expected_pairs": expected_pairs, @@ -455,9 +570,15 @@ def summarize_pairs(pairs: list[dict[str, Any]], tasks: list[str], runs: int, ga "mean_reward_base": reward_base, "mean_reward_practical": reward_practical, "mean_reward_delta": reward_practical - reward_base, - "wins": sum((not row["passed_base"]) and row["passed_practical"] for row in pairs), - "losses": sum(row["passed_base"] and (not row["passed_practical"]) for row in pairs), - "ties": sum(row["passed_base"] == row["passed_practical"] for row in pairs), + "wins": sum( + (not row["passed_base"]) and row["passed_practical"] for row in pairs + ), + "losses": sum( + row["passed_base"] and (not row["passed_practical"]) for row in pairs + ), + "ties": sum( + row["passed_base"] == row["passed_practical"] for row in pairs + ), "ci95": cluster_bootstrap(pairs), "by_task": by_task, } @@ -471,16 +592,25 @@ def _fmt_signed_pp(value: float | None) -> str: return "—" if value is None else f"{100 * value:+.1f} pp" -def render_report(manifest: dict[str, Any], summary: dict[str, Any], gaps: list[dict[str, Any]]) -> str: - ci = summary["ci95"] - pass_ci = ci["pass_rate_delta"] - reward_ci = ci["mean_reward_delta"] +def render_report( + manifest: dict[str, Any], + summary: dict[str, Any], + gaps: list[dict[str, Any]], +) -> str: + pass_ci = summary["ci95"]["pass_rate_delta"] + reward_ci = summary["ci95"]["mean_reward_delta"] + reward_ci_text = ( + f"{reward_ci[0]:+.3f} … {reward_ci[1]:+.3f}" + if reward_ci[0] is not None and reward_ci[1] is not None + else "—" + ) lines = [ "# Practical Coding × SkillsBench external lift", "", f"- Dataset: `{manifest['dataset']}`", f"- Profile: `{manifest['profile']}` ({len(manifest['tasks'])} tasks)", f"- Agent/model: `{manifest['agent']}` / `{manifest['model']}` ({manifest['reasoning']})", + f"- BenchFlow: `{manifest['benchflow_version']}`", f"- Runs per task/arm: `{manifest['runs']}`", f"- Evidence: **{'STABLE' if summary['stable'] else 'PROVISIONAL'}**", "- Treatment: baseline is `no-skill`; trained arm mounts only `practical-coding` as the custom Skill directory.", @@ -491,7 +621,7 @@ def render_report(manifest: dict[str, Any], summary: dict[str, Any], gaps: list[ "| Metric | No Skill | + Practical | Delta | 95% cluster-bootstrap CI |", "|---|---:|---:|---:|---|", f"| Pass rate | {_fmt_pct(summary['pass_rate_base'])} | {_fmt_pct(summary['pass_rate_practical'])} | {_fmt_signed_pp(summary['pass_rate_delta'])} | {_fmt_signed_pp(pass_ci[0])} … {_fmt_signed_pp(pass_ci[1])} |", - f"| Mean reward | {summary['mean_reward_base']:.3f} | {summary['mean_reward_practical']:.3f} | {summary['mean_reward_delta']:+.3f} | {(f'{reward_ci[0]:+.3f}' if reward_ci[0] is not None else '—')} … {(f'{reward_ci[1]:+.3f}' if reward_ci[1] is not None else '—')} |", + f"| Mean reward | {summary['mean_reward_base']:.3f} | {summary['mean_reward_practical']:.3f} | {summary['mean_reward_delta']:+.3f} | {reward_ci_text} |", "", f"Paired rollouts: **{summary['paired_rollouts']}/{summary['expected_pairs']}**. Pass flips: **{summary['wins']} wins / {summary['losses']} losses / {summary['ties']} ties**. Normalized gain: **{100 * summary['normalized_gain']:+.1f}%**.", "", @@ -523,11 +653,15 @@ def render_report(manifest: dict[str, Any], summary: dict[str, Any], gaps: list[ def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--profile", choices=("smoke", "standard", "full"), default="standard") + parser.add_argument( + "--profile", choices=("smoke", "standard", "full"), default="standard" + ) parser.add_argument("--task", action="append", default=[]) parser.add_argument("--runs", type=int, default=0) parser.add_argument("--workers", type=int, default=3) - parser.add_argument("--sandbox", choices=("docker", "daytona", "modal", "apple-container", "agentcore"), default="docker") + parser.add_argument( + "--sandbox", choices=("docker", "daytona", "modal"), default="docker" + ) parser.add_argument("--model", default=MODEL) parser.add_argument("--reasoning", default=REASONING) parser.add_argument("--output", type=Path) @@ -546,11 +680,28 @@ def self_test() -> None: for arm, reward in (("base", base), ("trained", trained)): path = jobs / arm / task path.mkdir(parents=True, exist_ok=True) - (path / "result.json").write_text(json.dumps({"task_name": task, "rewards": {"reward": reward}}), encoding="utf-8") + (path / "result.json").write_text( + json.dumps({"task_name": task, "rewards": {"reward": reward}}), + encoding="utf-8", + ) assert load_job_rewards(jobs / "base")["a"]["reward"] == 0.0 pairs = [ - {"task": "a", "repetition": 1, "reward_base": 0.0, "reward_practical": 1.0, "passed_base": False, "passed_practical": True}, - {"task": "b", "repetition": 1, "reward_base": 1.0, "reward_practical": 1.0, "passed_base": True, "passed_practical": True}, + { + "task": "a", + "repetition": 1, + "reward_base": 0.0, + "reward_practical": 1.0, + "passed_base": False, + "passed_practical": True, + }, + { + "task": "b", + "repetition": 1, + "reward_base": 1.0, + "reward_practical": 1.0, + "passed_base": True, + "passed_practical": True, + }, ] summary = summarize_pairs(pairs, ["a", "b"], 1, [], True) assert summary["pass_rate_delta"] == 0.5 @@ -558,7 +709,16 @@ def self_test() -> None: stable_pairs = [] for repetition in (1, 2, 3): for task in ("a", "b"): - stable_pairs.append({"task": task, "repetition": repetition, "reward_base": 1.0, "reward_practical": 1.0, "passed_base": True, "passed_practical": True}) + stable_pairs.append( + { + "task": task, + "repetition": repetition, + "reward_base": 1.0, + "reward_practical": 1.0, + "passed_base": True, + "passed_practical": True, + } + ) assert summarize_pairs(stable_pairs, ["a", "b"], 3, [], True)["stable"] print("SkillsBench adapter self-test: PASS") @@ -568,23 +728,36 @@ def main() -> int: if args.self_test: self_test() return 0 + runs = args.runs or (1 if args.profile == "smoke" else 3) if runs < 1 or args.workers < 1: raise SystemExit("runs and workers must be positive") if args.require_stable_ranking and runs < 3: raise SystemExit("stable external ranking requires at least 3 runs") + versions = check_prerequisites(args.sandbox) - cache_root = (args.cache_root or Path(os.environ.get("LOCALAPPDATA") or tempfile.gettempdir()) / "practical-coding-benchmarks" / "external").resolve() + cache_root = ( + args.cache_root + or Path(os.environ.get("LOCALAPPDATA") or tempfile.gettempdir()) + / "practical-coding-benchmarks" + / "external" + ).resolve() checkout, source_commit = ensure_skillsbench_checkout(cache_root) tasks = discover_tasks(checkout, args.profile, args.task) + if not tasks: + raise RuntimeError("task selection is empty") + stamp = dt.datetime.now().strftime("%Y%m%d-%H%M%S") - output = (args.output or ROOT / "benchmark-results" / "external" / f"skillsbench-{stamp}").resolve() + output = ( + args.output + or ROOT / "benchmark-results" / "external" / f"skillsbench-{stamp}" + ).resolve() output.mkdir(parents=True, exist_ok=False) staged_skills = stage_practical_skill(output) - started = dt.datetime.now(dt.timezone.utc) + manifest: dict[str, Any] = { "schema_version": 1, - "started_at": started.isoformat(), + "started_at": dt.datetime.now(dt.timezone.utc).isoformat(), "dataset": DATASET, "skillsbench_metadata_ref": SKILLSBENCH_REF, "skillsbench_metadata_commit": source_commit, @@ -602,25 +775,40 @@ def main() -> int: "environment": versions, "commands": [], } - (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") + (output / "manifest.json").write_text( + json.dumps(manifest, indent=2) + "\n", encoding="utf-8" + ) oracle_ok = True oracle_summary: dict[str, Any] = {"skipped": True} if not args.skip_oracle: oracle_dir = output / "oracle" - command = oracle_command(jobs_dir=oracle_dir, tasks=tasks, sandbox=args.sandbox, workers=args.workers) + command = oracle_command( + jobs_dir=oracle_dir, + tasks=tasks, + sandbox=args.sandbox, + workers=args.workers, + ) manifest["commands"].append({"kind": "oracle", "command": command}) result = run_command(command, ROOT) (output / "oracle.log").write_text(result.stdout, encoding="utf-8") if result.returncode: - raise RuntimeError(f"SkillsBench oracle command failed: {result.stdout[-4000:]}") + raise RuntimeError( + f"SkillsBench oracle command failed: {result.stdout[-4000:]}" + ) oracle_ok, oracle_summary = validate_oracle(oracle_dir, tasks) if not oracle_ok: - raise RuntimeError(f"SkillsBench oracle did not pass the selected task set: {oracle_summary}") + raise RuntimeError( + f"SkillsBench oracle did not pass the selected task set: {oracle_summary}" + ) for repetition in range(1, runs + 1): rep = output / "runs" / f"r{repetition:03d}" - order = ("no-skill", "practical") if repetition % 2 else ("practical", "no-skill") + order = ( + ("no-skill", "practical") + if repetition % 2 + else ("practical", "no-skill") + ) for arm in order: jobs_dir = rep / arm mode = "no-skill" if arm == "no-skill" else "with-skill" @@ -634,29 +822,53 @@ def main() -> int: skill_mode=mode, skills_root=staged_skills if arm == "practical" else None, ) - manifest["commands"].append({"kind": arm, "repetition": repetition, "command": command}) + manifest["commands"].append( + {"kind": arm, "repetition": repetition, "command": command} + ) result = run_command(command, ROOT) rep.mkdir(parents=True, exist_ok=True) (rep / f"{arm}.log").write_text(result.stdout, encoding="utf-8") if result.returncode: - manifest.setdefault("infrastructure_failures", []).append({"arm": arm, "repetition": repetition, "returncode": result.returncode}) - (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") - raise RuntimeError(f"SkillsBench {arm} r{repetition} failed: {result.stdout[-4000:]}") + manifest.setdefault("infrastructure_failures", []).append( + { + "arm": arm, + "repetition": repetition, + "returncode": result.returncode, + } + ) + (output / "manifest.json").write_text( + json.dumps(manifest, indent=2) + "\n", encoding="utf-8" + ) + raise RuntimeError( + f"SkillsBench {arm} r{repetition} failed: {result.stdout[-4000:]}" + ) pairs, gaps = collect_pairs(output, tasks, runs) summary = summarize_pairs(pairs, tasks, runs, gaps, oracle_ok) - manifest.update({ - "completed_at": dt.datetime.now(dt.timezone.utc).isoformat(), - "oracle": oracle_summary, - "paired_rollouts": len(pairs), - "stable": summary["stable"], - }) - (output / "manifest.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8") - (output / "pairs.json").write_text(json.dumps(pairs, indent=2) + "\n", encoding="utf-8") - (output / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") - (output / "report.md").write_text(render_report(manifest, summary, gaps), encoding="utf-8") + manifest.update( + { + "completed_at": dt.datetime.now(dt.timezone.utc).isoformat(), + "oracle": oracle_summary, + "paired_rollouts": len(pairs), + "stable": summary["stable"], + } + ) + (output / "manifest.json").write_text( + json.dumps(manifest, indent=2) + "\n", encoding="utf-8" + ) + (output / "pairs.json").write_text( + json.dumps(pairs, indent=2) + "\n", encoding="utf-8" + ) + (output / "summary.json").write_text( + json.dumps(summary, indent=2) + "\n", encoding="utf-8" + ) + (output / "report.md").write_text( + render_report(manifest, summary, gaps), encoding="utf-8" + ) print(f"wrote {output}") - print(f"SkillsBench pass lift: {_fmt_signed_pp(summary['pass_rate_delta'])} ({'STABLE' if summary['stable'] else 'PROVISIONAL'})") + print( + f"SkillsBench pass lift: {_fmt_signed_pp(summary['pass_rate_delta'])} ({'STABLE' if summary['stable'] else 'PROVISIONAL'})" + ) if args.require_stable_ranking and not summary["stable"]: return 2 return 0