From 03d1d7c919fc7d33c575fa046ae585aa34bd176b Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 08:44:32 -0700 Subject: [PATCH 01/12] feat: expand benchmark case catalog --- benchmarks/case_catalog.py | 291 +++++++++++++++++++++++++++++++++++++ 1 file changed, 291 insertions(+) create mode 100644 benchmarks/case_catalog.py diff --git a/benchmarks/case_catalog.py b/benchmarks/case_catalog.py new file mode 100644 index 0000000..2202263 --- /dev/null +++ b/benchmarks/case_catalog.py @@ -0,0 +1,291 @@ +"""Extended benchmark cases kept separate from the core runner. + +The catalog favors distinct failure mechanisms and routing boundaries over paraphrases of +existing cases. Every custom debug task has a deterministic verifier with a reported +caller (`correct`) and a sibling/shared-boundary check (`safe`). +""" + +from __future__ import annotations + +import importlib.util +from pathlib import Path +from typing import Any + + +EXTRA_ROUTER_CASES = { + "direct-existing-prop": ( + "DIRECT", + "The existing Button already has a loading prop. Make CheckoutButton use it while submit is pending and run its existing component test.", + ), + "direct-private-dto": ( + "DIRECT", + "All three known internal consumers already accept user_id. Rename the private DTO field and update those known callers.", + ), + "decision-job-runner": ( + "DECISION", + "We need scheduled jobs in this service. Pick a separate queue worker or the database-backed scheduler; operational ownership and latency needs have not been decided.", + ), + "decision-webhook-compat": ( + "DECISION", + "We need to change a public webhook payload shape, but we do not know whether external consumers can migrate together or need a compatibility window.", + ), + "debug-flaky-retry": ( + "DEBUGGING", + "The retry test fails about one run in twenty with an assertion mismatch. We have not established which state first becomes wrong.", + ), + "debug-performance-regression": ( + "DEBUGGING", + "Opening the analytics dashboard became roughly five times slower after last week's change; no hotspot has been profiled yet.", + ), + "implementation-event-tenant": ( + "IMPLEMENTATION", + "Add tenant_id to a versioned event across its producer, schema, consumers, and replay path; the exact coordinated contract surface has not been mapped yet.", + ), + "implementation-known-callers": ( + "DIRECT", + "The new helper signature is already decided and the four affected callers are known. Update those callers and run their focused tests.", + ), + "exploration-auth-context": ( + "EXPLORATION", + "Before changing AuthContext in this monorepo, identify every package that creates it, transforms it, or depends on the transformed shape.", + ), + "exploration-plugin-loading": ( + "EXPLORATION", + "Find which payment adapter implementations can be loaded at runtime and where their registration and dispatch happen across this large repository.", + ), + "verification-payment-retry": ( + "VERIFICATION", + "The retry-policy patch is complete. Decide what evidence is sufficient to support the claim that it cannot create duplicate charges.", + ), + "verification-performance-claim": ( + "VERIFICATION", + "The optimization is implemented. Decide whether unit timings, a benchmark, or a load test is sufficient evidence for the performance claim.", + ), +} + + +EXTRA_DECISION_CASES = { + "pagination-contract": { + "prompt": "Choose cursor or offset pagination for a public activity API. Dataset size, mutation rate, stable continuation needs, and client ergonomics are unresolved.", + "reply": "The dataset is large and changes continuously; clients need stable continuation while new rows arrive and can store an opaque token. Resolve the decision now.", + "expected": ("cursor", "opaque token"), + }, + "file-storage": { + "prompt": "Choose database blobs or object storage for user-uploaded media. Size distribution, delivery path, transactional coupling, and lifecycle needs are unresolved.", + "reply": "Files reach hundreds of MB, are immutable after upload, are served through a CDN, and only metadata must be transactional with the database. Resolve the decision now.", + "expected": ("object storage", "object store"), + }, + "deployment-rollout": { + "prompt": "Choose blue-green or rolling deployment for this service. Capacity headroom, compatibility between versions, rollback expectations, and session behavior are unresolved.", + "reply": "We cannot afford double capacity; adjacent versions are backward compatible; instances are stateless; gradual rollback is acceptable. Resolve the decision now.", + "expected": ("rolling", "rollout"), + }, + "distributed-consistency": { + "prompt": "Choose a distributed transaction or a saga for an order workflow spanning payment and inventory services. Failure recovery and atomicity requirements are unresolved.", + "reply": "The services own separate databases, partial progress is acceptable temporarily, and every step has a tested compensating action. Resolve the decision now.", + "expected": ("saga", "compensat"), + }, + "identifier-strategy": { + "prompt": "Choose database-generated integer IDs or UUIDs for records created by intermittently connected clients. Merge behavior, ordering, and privacy requirements are unresolved.", + "reply": "Clients must create IDs offline without coordination; merges happen later; exposing record counts is undesirable; sortable IDs are preferred but not mandatory. Resolve the decision now.", + "expected": ("uuid", "uuidv7", "uuid v7"), + }, + "sync-async-boundary": { + "prompt": "Choose a synchronous service call or asynchronous queue for invoice validation. Latency, failure coupling, throughput, and user-visible consistency are unresolved.", + "reply": "The user must receive validation errors in the same request; p99 is under 80 ms; both services share the same availability target; throughput is modest. Resolve the decision now.", + "expected": ("synchronous", "sync", "direct call"), + }, +} + + +EXTRA_DEBUG_CASES = { + "trace-header-normalize": { + "prompt": "auth_header() misses Authorization when a proxy supplies header names with surrounding whitespace. Fix the bug without breaking other header lookups.", + "files": { + "headers.py": """def normalize_header(name):\n return name.lower()\n\ndef get_header(headers, target):\n wanted = normalize_header(target)\n for name, value in headers.items():\n if normalize_header(name) == wanted:\n return value\n return None\n\ndef auth_header(headers):\n return get_header(headers, \"Authorization\")\n\ndef trace_header(headers):\n return get_header(headers, \"X-Trace-Id\")\n""", + }, + "score": "headers", + }, + "trace-cache-tenant": { + "prompt": "profile_cache_key() collides for users with the same id in different tenants. Fix the bug; other cache namespaces use the same key builder.", + "files": { + "cache_keys.py": """def cache_key(tenant, namespace, key):\n return f\"{namespace}:{key}\"\n\ndef profile_cache_key(tenant, user_id):\n return cache_key(tenant, \"profile\", user_id)\n\ndef invoice_cache_key(tenant, invoice_id):\n return cache_key(tenant, \"invoice\", invoice_id)\n""", + }, + "score": "cache-tenant", + }, + "trace-page-window": { + "prompt": "list_orders() returns the second page when page=1. Fix the pagination bug without changing the public one-based page contract used by sibling lists.", + "files": { + "paging.py": """def page_bounds(page, size):\n start = page * size\n return start, start + size\n\ndef list_orders(rows, page, size):\n start, end = page_bounds(page, size)\n return rows[start:end]\n\ndef list_users(rows, page, size):\n start, end = page_bounds(page, size)\n return rows[start:end]\n""", + }, + "score": "paging", + }, + "trace-duration-units": { + "prompt": "request_timeout() crashes when REQUEST_TIMEOUT is configured as '2500ms'. Fix duration parsing; background jobs use the same configuration format.", + "files": { + "timeouts.py": """def parse_duration(value):\n return float(value)\n\ndef request_timeout(env):\n return parse_duration(env.get(\"REQUEST_TIMEOUT\", \"5\"))\n\ndef background_timeout(env):\n return parse_duration(env.get(\"BACKGROUND_TIMEOUT\", \"30\"))\n""", + }, + "score": "duration", + }, + "trace-csv-blank": { + "prompt": "invoice_rows() produces empty records for blank lines in uploaded CSV text. Fix the shared row parsing behavior without breaking audit imports.", + "files": { + "rows.py": """def parse_rows(text):\n return [line.split(\",\") for line in text.split(\"\\n\")]\n\ndef invoice_rows(text):\n return parse_rows(text)\n\ndef audit_rows(text):\n return parse_rows(text)\n""", + }, + "score": "csv-blank", + }, + "trace-stock-debit": { + "prompt": "reserve() can drive stock negative when quantity exceeds availability. Fix the invariant; another inventory path shares the debit primitive.", + "files": { + "inventory.py": """def _debit(stock, sku, quantity):\n stock[sku] = stock.get(sku, 0) - quantity\n return True\n\ndef reserve(stock, sku, quantity):\n return _debit(stock, sku, quantity)\n\ndef consume(stock, sku, quantity):\n return _debit(stock, sku, quantity)\n""", + }, + "score": "stock", + }, + "trace-ttl-zero": { + "prompt": "session_ttl() ignores an explicit TTL of zero and silently restores the default. Fix TTL parsing without changing sibling cache semantics.", + "files": { + "ttl.py": """def parse_ttl(value, default):\n if value is None:\n return default\n return int(value) or default\n\ndef session_ttl(env):\n return parse_ttl(env.get(\"SESSION_TTL\"), 300)\n\ndef cache_ttl(env):\n return parse_ttl(env.get(\"CACHE_TTL\"), 60)\n""", + }, + "score": "ttl-zero", + }, + "trace-null-sort": { + "prompt": "sort_products() crashes when a product has no name; unnamed entries should sort last. Fix the shared ordering rule without breaking customer sorting.", + "files": { + "sorting.py": """def name_key(value):\n return value.lower()\n\ndef sort_products(rows):\n return sorted(rows, key=lambda row: name_key(row.get(\"name\")))\n\ndef sort_customers(rows):\n return sorted(rows, key=lambda row: name_key(row.get(\"name\")))\n""", + }, + "score": "null-sort", + }, +} + + +STANDARD_EXTRA_DEBUG = [ + "trace-header-normalize", + "trace-cache-tenant", + "trace-page-window", + "trace-duration-units", +] + +STANDARD_EXTRA_DECISION = [ + "pagination-contract", + "file-storage", +] + + +def _load_module(workspace: Path, filename: str, name: str) -> Any: + path = workspace / filename + spec = importlib.util.spec_from_file_location(name, path) + if spec is None or spec.loader is None: + raise RuntimeError(f"cannot import {filename}") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _result(correct: bool, safe: bool, reason: str) -> dict[str, Any]: + return {"correct": int(bool(correct)), "safe": int(bool(safe)), "reason": reason} + + +def score_extra_debug(case: str, workspace: Path) -> dict[str, Any]: + spec = EXTRA_DEBUG_CASES[case] + score = spec["score"] + try: + if score == "headers": + module = _load_module(workspace, "headers.py", f"debug_{score}") + correct = module.auth_header({" Authorization ": "Bearer abc"}) == "Bearer abc" + safe = module.trace_header({" X-Trace-Id ": "trace-1"}) == "trace-1" + return _result(correct, safe, "shared header normalization") + + if score == "cache-tenant": + module = _load_module(workspace, "cache_keys.py", "debug_cache_tenant") + correct = module.profile_cache_key("tenant-a", "42") != module.profile_cache_key("tenant-b", "42") + safe = module.invoice_cache_key("tenant-a", "9") != module.invoice_cache_key("tenant-b", "9") + return _result(correct, safe, "tenant isolation in shared cache_key") + + if score == "paging": + module = _load_module(workspace, "paging.py", "debug_paging") + correct = module.list_orders(["o1", "o2", "o3", "o4"], 1, 2) == ["o1", "o2"] + safe = module.list_users(["u1", "u2", "u3", "u4"], 2, 2) == ["u3", "u4"] + return _result(correct, safe, "one-based page_bounds shared by callers") + + if score == "duration": + module = _load_module(workspace, "timeouts.py", "debug_duration") + correct = abs(float(module.request_timeout({"REQUEST_TIMEOUT": "2500ms"})) - 2.5) < 1e-9 + safe = ( + abs(float(module.background_timeout({"BACKGROUND_TIMEOUT": "1500ms"})) - 1.5) < 1e-9 + and abs(float(module.background_timeout({"BACKGROUND_TIMEOUT": "3"})) - 3.0) < 1e-9 + ) + return _result(correct, safe, "shared duration parser preserves seconds and ms") + + if score == "csv-blank": + module = _load_module(workspace, "rows.py", "debug_csv_blank") + correct = module.invoice_rows("a,b\n\nc,d\n") == [["a", "b"], ["c", "d"]] + safe = module.audit_rows("x,y\n\nq,r") == [["x", "y"], ["q", "r"]] + return _result(correct, safe, "shared row parser drops blank records") + + if score == "stock": + module = _load_module(workspace, "inventory.py", "debug_stock") + reported = {"sku": 3} + try: + outcome = module.reserve(reported, "sku", 5) + except Exception: + outcome = False + correct = reported == {"sku": 3} and outcome is not True + + sibling = {"sku": 2} + try: + sibling_outcome = module.consume(sibling, "sku", 4) + except Exception: + sibling_outcome = False + safe = sibling == {"sku": 2} and sibling_outcome is not True + return _result(correct, safe, "shared debit boundary preserves non-negative stock") + + if score == "ttl-zero": + module = _load_module(workspace, "ttl.py", "debug_ttl_zero") + correct = module.session_ttl({"SESSION_TTL": "0"}) == 0 + safe = module.cache_ttl({"CACHE_TTL": "0"}) == 0 and module.cache_ttl({}) == 60 + return _result(correct, safe, "explicit zero survives shared TTL parsing") + + if score == "null-sort": + module = _load_module(workspace, "sorting.py", "debug_null_sort") + products = module.sort_products([{"name": "Beta"}, {"name": None}, {"name": "alpha"}]) + customers = module.sort_customers([{"name": None}, {"name": "Zed"}, {"name": "amy"}]) + correct = [row.get("name") for row in products] == ["alpha", "Beta", None] + safe = [row.get("name") for row in customers] == ["amy", "Zed", None] + return _result(correct, safe, "shared nullable name ordering") + + raise KeyError(score) + except Exception as error: + return {"correct": 0, "safe": 0, "reason": str(error)} + + +def install(bench: Any) -> None: + """Install the extended cases into run_benchmarks without duplicating the runner.""" + if getattr(bench, "_extended_case_catalog_installed", False): + return + + base_decision = list(bench.DECISION_CASES) + base_debug = list(bench.PROFILE_CASES["full"]["debug"]) + + bench.ROUTER_CASES.update(EXTRA_ROUTER_CASES) + bench.DECISION_CASES.update(EXTRA_DECISION_CASES) + bench.CUSTOM_DEBUG.update(EXTRA_DEBUG_CASES) + + original_debug_score = bench.custom_debug_score + + def combined_debug_score(case: str, workspace: Path) -> dict[str, Any]: + if case in EXTRA_DEBUG_CASES: + return score_extra_debug(case, workspace) + return original_debug_score(case, workspace) + + bench.custom_debug_score = combined_debug_score + + # Smoke remains deliberately tiny. Standard is broader but bounded; full carries the + # complete public regression matrix. + bench.PROFILE_CASES["standard"]["router"] = list(bench.ROUTER_CASES) + bench.PROFILE_CASES["full"]["router"] = list(bench.ROUTER_CASES) + bench.PROFILE_CASES["standard"]["decision"] = [*base_decision, *STANDARD_EXTRA_DECISION] + bench.PROFILE_CASES["full"]["decision"] = list(bench.DECISION_CASES) + bench.PROFILE_CASES["standard"]["debug"] = [*base_debug, *STANDARD_EXTRA_DEBUG] + bench.PROFILE_CASES["full"]["debug"] = [*base_debug, *EXTRA_DEBUG_CASES] + + bench._extended_case_catalog_installed = True From 3e963b6cad7669aa8957ff4a10cbffb33d95a3be Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 08:44:40 -0700 Subject: [PATCH 02/12] feat: load extended benchmark catalog --- benchmarks/run_catalog.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 benchmarks/run_catalog.py diff --git a/benchmarks/run_catalog.py b/benchmarks/run_catalog.py new file mode 100644 index 0000000..128ce65 --- /dev/null +++ b/benchmarks/run_catalog.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""Canonical benchmark entrypoint with the extended public case catalog installed.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +if str(HERE) not in sys.path: + sys.path.insert(0, str(HERE)) + +import run_benchmarks as bench +from case_catalog import install + + +if __name__ == "__main__": + install(bench) + raise SystemExit(bench.main()) From 62c247d072a63efeffd06b66ea02b0bfb35d5444 Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 08:45:05 -0700 Subject: [PATCH 03/12] test: add oracle fixes for expanded debug cases --- benchmarks/debug_oracles.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 benchmarks/debug_oracles.py diff --git a/benchmarks/debug_oracles.py b/benchmarks/debug_oracles.py new file mode 100644 index 0000000..67fa2b3 --- /dev/null +++ b/benchmarks/debug_oracles.py @@ -0,0 +1,28 @@ +"""Reference fixes used only to validate the expanded deterministic debug scorers.""" + +DEBUG_ORACLES = { + "trace-header-normalize": { + "headers.py": """def normalize_header(name):\n return name.strip().lower()\n\ndef get_header(headers, target):\n wanted = normalize_header(target)\n for name, value in headers.items():\n if normalize_header(name) == wanted:\n return value\n return None\n\ndef auth_header(headers):\n return get_header(headers, \"Authorization\")\n\ndef trace_header(headers):\n return get_header(headers, \"X-Trace-Id\")\n""", + }, + "trace-cache-tenant": { + "cache_keys.py": """def cache_key(tenant, namespace, key):\n return f\"{tenant}:{namespace}:{key}\"\n\ndef profile_cache_key(tenant, user_id):\n return cache_key(tenant, \"profile\", user_id)\n\ndef invoice_cache_key(tenant, invoice_id):\n return cache_key(tenant, \"invoice\", invoice_id)\n""", + }, + "trace-page-window": { + "paging.py": """def page_bounds(page, size):\n start = (page - 1) * size\n return start, start + size\n\ndef list_orders(rows, page, size):\n start, end = page_bounds(page, size)\n return rows[start:end]\n\ndef list_users(rows, page, size):\n start, end = page_bounds(page, size)\n return rows[start:end]\n""", + }, + "trace-duration-units": { + "timeouts.py": """def parse_duration(value):\n value = str(value).strip().lower()\n if value.endswith(\"ms\"):\n return float(value[:-2]) / 1000.0\n return float(value)\n\ndef request_timeout(env):\n return parse_duration(env.get(\"REQUEST_TIMEOUT\", \"5\"))\n\ndef background_timeout(env):\n return parse_duration(env.get(\"BACKGROUND_TIMEOUT\", \"30\"))\n""", + }, + "trace-csv-blank": { + "rows.py": """def parse_rows(text):\n return [line.split(\",\") for line in text.splitlines() if line.strip()]\n\ndef invoice_rows(text):\n return parse_rows(text)\n\ndef audit_rows(text):\n return parse_rows(text)\n""", + }, + "trace-stock-debit": { + "inventory.py": """def _debit(stock, sku, quantity):\n available = stock.get(sku, 0)\n if quantity > available:\n return False\n stock[sku] = available - quantity\n return True\n\ndef reserve(stock, sku, quantity):\n return _debit(stock, sku, quantity)\n\ndef consume(stock, sku, quantity):\n return _debit(stock, sku, quantity)\n""", + }, + "trace-ttl-zero": { + "ttl.py": """def parse_ttl(value, default):\n if value is None:\n return default\n return int(value)\n\ndef session_ttl(env):\n return parse_ttl(env.get(\"SESSION_TTL\"), 300)\n\ndef cache_ttl(env):\n return parse_ttl(env.get(\"CACHE_TTL\"), 60)\n""", + }, + "trace-null-sort": { + "sorting.py": """def name_key(value):\n return (value is None, \"\" if value is None else value.lower())\n\ndef sort_products(rows):\n return sorted(rows, key=lambda row: name_key(row.get(\"name\")))\n\ndef sort_customers(rows):\n return sorted(rows, key=lambda row: name_key(row.get(\"name\")))\n""", + }, +} From 972719be5964251278c3e00b78b51e78e5bcdcc5 Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 08:45:18 -0700 Subject: [PATCH 04/12] test: validate expanded benchmark catalog --- benchmarks/test_catalog.py | 68 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 benchmarks/test_catalog.py diff --git a/benchmarks/test_catalog.py b/benchmarks/test_catalog.py new file mode 100644 index 0000000..3c034af --- /dev/null +++ b/benchmarks/test_catalog.py @@ -0,0 +1,68 @@ +import tempfile +import unittest +from pathlib import Path + +from benchmarks import run_benchmarks as bench +from benchmarks.case_catalog import ( + EXTRA_DEBUG_CASES, + EXTRA_DECISION_CASES, + EXTRA_ROUTER_CASES, + install, + score_extra_debug, +) +from benchmarks.debug_oracles import DEBUG_ORACLES + + +class ExpandedCatalogTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + install(bench) + + def test_public_matrix_is_materially_broader(self): + self.assertEqual(len(bench.ROUTER_CASES), 28) + self.assertEqual(len(bench.DECISION_CASES), 10) + self.assertEqual(len(bench.PROFILE_CASES["standard"]["decision"]), 6) + self.assertEqual(len(bench.PROFILE_CASES["full"]["decision"]), 10) + self.assertEqual(len(bench.PROFILE_CASES["standard"]["debug"]), 8) + self.assertEqual(len(bench.PROFILE_CASES["full"]["debug"]), 12) + + def test_expansion_is_not_just_one_route_or_bug_shape(self): + extra_routes = {expected for expected, _ in EXTRA_ROUTER_CASES.values()} + self.assertEqual( + extra_routes, + {"DIRECT", "DECISION", "DEBUGGING", "IMPLEMENTATION", "EXPLORATION", "VERIFICATION"}, + ) + self.assertEqual(len({case["score"] for case in EXTRA_DEBUG_CASES.values()}), len(EXTRA_DEBUG_CASES)) + self.assertGreaterEqual(len(EXTRA_DECISION_CASES), 6) + + def test_profiles_have_no_duplicate_case_ids(self): + for profile in ("standard", "full"): + for suite in ("router", "decision", "debug"): + cases = bench.PROFILE_CASES[profile][suite] + self.assertEqual(len(cases), len(set(cases)), f"duplicates in {profile}/{suite}") + + def test_each_debug_seed_fails_and_oracle_passes(self): + self.assertEqual(set(EXTRA_DEBUG_CASES), set(DEBUG_ORACLES)) + for case, spec in EXTRA_DEBUG_CASES.items(): + with self.subTest(case=case), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + for name, content in spec["files"].items(): + (root / name).write_text(content, encoding="utf-8") + seeded = score_extra_debug(case, root) + self.assertFalse(seeded["correct"] == 1 and seeded["safe"] == 1) + + for name, content in DEBUG_ORACLES[case].items(): + (root / name).write_text(content, encoding="utf-8") + oracle = score_extra_debug(case, root) + self.assertEqual((oracle["correct"], oracle["safe"]), (1, 1), oracle["reason"]) + + def test_decision_cases_have_two_turn_resolution_contract(self): + for case, spec in bench.DECISION_CASES.items(): + with self.subTest(case=case): + self.assertTrue(spec["prompt"].strip()) + self.assertIn("Resolve the decision now", spec["reply"]) + self.assertTrue(spec["expected"]) + + +if __name__ == "__main__": + unittest.main() From c5454b37962780df6fcbfdb746dcd5fa1838f459 Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 08:45:35 -0700 Subject: [PATCH 05/12] feat: run expanded benchmark catalog --- benchmarks/run.ps1 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/run.ps1 b/benchmarks/run.ps1 index 0bf3668..5f3bd15 100644 --- a/benchmarks/run.ps1 +++ b/benchmarks/run.ps1 @@ -27,7 +27,7 @@ $repoRoot = Split-Path -Parent $scriptDir if ($SelfTest) { Push-Location $repoRoot try { - & python -m unittest benchmarks.test_benchmarks benchmarks.test_stability + & python -m unittest benchmarks.test_benchmarks benchmarks.test_stability benchmarks.test_catalog if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } } finally { @@ -63,7 +63,7 @@ if ($RequireStableRanking -and -not $Output) { } $arguments = @( - (Join-Path $scriptDir "run_benchmarks.py"), + (Join-Path $scriptDir "run_catalog.py"), "--profile", $Profile, "--workers", $Workers ) From 12367ffa782cb173ac8d58d765cd9dd5a6ef5d09 Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 08:45:50 -0700 Subject: [PATCH 06/12] test: run expanded benchmark catalog checks in CI --- .github/workflows/validate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index e1349c5..6f21950 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -22,7 +22,7 @@ jobs: run: skills-ref validate ./practical-coding - name: Run benchmark harness tests working-directory: practical-coding - run: python -m unittest benchmarks.test_benchmarks benchmarks.test_stability + run: python -m unittest benchmarks.test_benchmarks benchmarks.test_stability benchmarks.test_catalog - name: Check Codex default_prompt references the skill as $skill-name run: grep -qF '$practical-coding' practical-coding/agents/openai.yaml - name: Ensure legacy local graph runtime is not reintroduced From 099f131602eba54523f0f12e256b6b5abd051e90 Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 08:46:55 -0700 Subject: [PATCH 07/12] docs: map external benchmark landscape --- .../2026-08-24-benchmark-landscape.md | 41 +++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 docs/evaluations/2026-08-24-benchmark-landscape.md diff --git a/docs/evaluations/2026-08-24-benchmark-landscape.md b/docs/evaluations/2026-08-24-benchmark-landscape.md new file mode 100644 index 0000000..8b3e4f8 --- /dev/null +++ b/docs/evaluations/2026-08-24-benchmark-landscape.md @@ -0,0 +1,41 @@ +# Coding-agent and Agent-Skill benchmark landscape + +Date: 2026-08-24 + +This note separates project regression tests from external benchmark evidence. Public regression cases are useful for preventing known failures from returning, but repeated prompt iteration against them eventually makes them unsuitable as evidence of generalization. + +## Most relevant public projects + +| Project | What it measures | Dataset / repetition shape | Why it matters to Practical Coding | +|---|---|---|---| +| [SkillsBench](https://skillsbench.ai/) / [benchflow-ai/skillsbench](https://github.com/benchflow-ai/skillsbench) | Skill effectiveness and agent ability to use Skills | v1.1 leaderboard: 87 tasks, paired with/without-Skills, up to 3 trials, deterministic verifiers, 95% CIs | Closest external benchmark to this project. Its primary question is the same one Practical should answer: does adding the Skill improve a fixed agent/model? | +| [langchain-ai/skills-benchmarks](https://github.com/langchain-ai/skills-benchmarks) | Skill documentation layout, treatment A/B, split/merged Skills, distractor/noise effects | Docker tasks decoupled from treatments; repeated pytest runs; trajectory logging | Strong reference for treatment design and testing routing/interference rather than only final task success. | +| [SWE-bench Verified](https://www.swebench.com/verified.html) | Real GitHub issue resolution | 500 human-filtered issues; public agent leaderboard plus a standardized mini-SWE-agent view | Canonical external coding-agent signal. Useful for testing whether Practical improves a neutral coding harness, but less specific to Skill design. | +| [Terminal-Bench](https://github.com/harbor-framework/terminal-bench) | Long-horizon terminal-agent work | Continuous tagged dataset releases; containerized tasks; oracle solutions; leaderboard; oracle is recommended to run 5x | Strong model for benchmark operations: version datasets, continuously validate tasks, keep oracle checks, and treat benchmark maintenance as a product. | +| [FeatureBench](https://github.com/LiberCoders/FeatureBench) | End-to-end feature development | 200 full tasks, 100 fast tasks, 30 lite tasks; executable environments and an open data-generation pipeline | Useful external evidence for Implementation/Exploration behavior because tasks span larger feature surfaces than bug-only benchmarks. | +| [Senior SWE-Bench](https://senior-swe-bench.snorkel.ai/) | Senior-level design/build and investigate/fix behavior from realistic messages | 100 initial tasks: 50 public and 50 private; recent real PRs; behavioral verification and quality gates | Best reference for contamination control and realistic debugging prompts. The private half is especially relevant once Practical's public regression set is used for prompt iteration. | +| [ProgramBench](https://programbench.com/) | Rebuilding complete programs from behavioral contracts | 200 tasks and a large hidden behavioral-test suite | Useful as a hard design/implementation ceiling, but less directly targeted at a procedural coding Skill. | + +## Evidence model for Practical Coding + +Use three layers instead of one leaderboard number: + +1. **Public regression layer** — Router, Decision, Debug and the Ponytail-derived Delivery tasks in this repository. These should stay deterministic and are allowed to encode previously observed failures. Their job is to prevent regression, not prove unseen-task generalization. +2. **External benchmark layer** — run the Skill as an augmentation treatment on independent public suites. SkillsBench is the first target because it explicitly measures with-Skills versus without-Skills. FeatureBench and a standardized SWE-bench/Terminal-Bench harness provide broader coding-agent validity. +3. **Held-out layer** — keep a small private task set that is not read while editing `SKILL.md` or references. Rotate or refresh it from recent real PRs/bugs. Only this layer should be used for claims that a prompt iteration generalized beyond the public regression corpus. + +## Task-authoring rules adopted here + +The expanded public catalog follows these rules: + +- prefer a new failure mechanism over another paraphrase of an existing prompt; +- each custom Debug case has a deterministic reported-caller check and a sibling/shared-boundary check; +- every custom Debug seed must fail and an oracle fix must pass before model calls; +- Router additions cover all six routes rather than concentrating on the last observed failure; +- Decision cases use two turns: first expose a real decision frontier, then provide enough facts to require convergence; +- `standard` is a bounded release gate while `full` contains the complete public regression matrix; +- stable rankings still require at least `n=3`; public 100% results are described as regression ceilings, not generalization proof. + +## Next external-validation milestone + +The highest-value next step is not another Core rule. It is an adapter that can run `practical-coding` as a treatment on an independent benchmark without rewriting that benchmark's tasks. Start with SkillsBench because its paired methodology matches the project's goal, then add a small FeatureBench fast-split experiment. Keep model, harness, reasoning effort, task version, and Skill bundle hash fixed for every paired comparison. From 1af2064ad8f6b5e325bd384156d400e67174fad9 Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 08:47:25 -0700 Subject: [PATCH 08/12] docs: document expanded benchmark matrix --- benchmarks/README.md | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index 994f66c..a616491 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -2,7 +2,7 @@ This chain runs isolated Codex sessions directly against `gpt-5.6-luna`, preserves every prompt/transcript/workspace, applies mechanical graders, and writes JSON plus Markdown summaries. It follows the mature evaluation shape used by Agent Skills and Ponytail: realistic cases, fixed sources, clean sessions, repeated paired arms, deterministic assertions where possible, tokens/time, and raw evidence. -For prerequisites, pinned revisions, exact reproduction commands, evidence boundaries, and the published v1.11 calibration results, see [`REPRODUCING.md`](REPRODUCING.md). +For prerequisites, pinned revisions, exact reproduction commands, evidence boundaries, and the published v1.11 calibration results, see [`REPRODUCING.md`](REPRODUCING.md). For the external benchmark landscape and the public-regression/external/held-out evidence model, see [`../docs/evaluations/2026-08-24-benchmark-landscape.md`](../docs/evaluations/2026-08-24-benchmark-landscape.md). ## Run @@ -15,6 +15,8 @@ pwsh -File benchmarks/run.ps1 -Profile smoke -Suite router -Case direct-artifact pwsh -File benchmarks/run.ps1 -Rescore D:\path\to\benchmark-results\20260824-203839 ``` +`run.ps1` is the canonical entrypoint. It loads the core runner through `run_catalog.py`, which installs the extended public case catalog before execution. This keeps benchmark mechanics separate from the evolving task corpus. + For a result that will be presented as a stable ranking, opt into the evidence gate: ```powershell @@ -30,11 +32,15 @@ python benchmarks/check_stability.py benchmark-results\v111-delivery-n1-core-rev That command intentionally reports the published v1.11 Delivery `n=1` artifact as `PROVISIONAL`; it must not be used for a stable ranking until the same cells are rerun with at least three distinct repetitions. -Profiles: +## Profiles + +| Profile | Delivery | Router | Decision | Debug | Default runs | Cells without previous/no-Skill arm | +|---|---:|---:|---:|---:|---:|---:| +| `smoke` | 3 | 4 | 1 | 1 | 1 | 13 | +| `standard` | 9 | 28 | 6 | 8 | 3 | 222 | +| `full` | 18 | 28 | 10 | 12 | 3 | 324 | -- `smoke`: one run by default; quick harness and model sanity check. -- `standard`: three runs by default; balanced release comparison. -- `full`: three runs by default; expanded Ponytail delivery, router, multi-round Decision, and debugging matrix. +`standard` is the normal release gate. `full` carries the complete public regression matrix. The extra Router cases span all six routes; the expanded Debug set covers twelve cases across parsing, normalization, tenant isolation, pagination, units, row handling, state invariants, TTL semantics, URL handling, and the upstream transfer/amount tasks. Decision grows from four to ten two-turn decisions in `full`. Useful options: @@ -44,30 +50,30 @@ Useful options: - `-SourcesRoot ` reuses pinned competitor checkouts. Without it, sources are cached under the user-local application data directory and cloned as needed. - `-IncludeBaseline` adds a no-skill delivery arm. - `-NoBuilds` skips runner-owned frontend production builds. It is rejected for a stable Delivery ranking. -- `-SelfTest` runs the local harness regression tests and validates fixtures, upstream scorers, source pins, and reporting without model calls. +- `-SelfTest` runs the local harness regression tests and validates fixtures, upstream scorers, expanded custom scorers, source pins, and reporting without model calls. - `-FailOnCellFailure` makes any behavioral cell failure return exit code 2. By default only harness/infrastructure failures are non-zero, because a valid comparison may intentionally expose competitor or candidate failures. - `-RequireStableRanking` requires at least three distinct repetitions per selected suite/case/arm and rejects incomplete or infrastructure-failed runs before they are called stable. - `-Rescore ` reapplies the current mechanical graders to saved workspaces/transcripts without another model call; the manifest records the new runner hash and rescore time. By default, run artifacts are written under `benchmark-results/` and ignored by Git, so transcripts and generated workspaces remain inspectable across commands without entering commits. Use `-Output` for an explicit location. -The fast harness regression suite is also runnable without sources or model access: +The fast harness regression suite is also runnable without model calls: ```powershell -python -m unittest benchmarks.test_benchmarks benchmarks.test_stability +python -m unittest benchmarks.test_benchmarks benchmarks.test_stability benchmarks.test_catalog ``` The output directory contains: ```text -manifest.json fixed model, commits, profile, cases, and skill hashes -results.json one record per cell -summary.json grouped rates, medians, means, and standard deviations -comparisons.json Practical-minus-comparator behavioral and efficiency deltas -rollups.json suite/arm totals across cases +manifest.json fixed model, commits, profile, cases, and skill hashes +results.json one record per cell +summary.json grouped rates, medians, means, and standard deviations +comparisons.json Practical-minus-comparator behavioral and efficiency deltas +rollups.json suite/arm totals across cases rollup-comparisons.json suite-level Practical-minus-comparator deltas -report.md human-readable comparison and Practical deltas -cells/ prompt, raw JSONL, stderr, answer, and code workspace per cell +report.md human-readable comparison and Practical deltas +cells/ prompt, raw JSONL, stderr, answer, and code workspace per cell ``` ## Suites and scoring @@ -75,7 +81,7 @@ cells/ prompt, raw JSONL, stderr, answer, and code workspace per - `delivery`: Ponytail's published agentic tasks and deterministic scorer. Reports correctness, safety, production LOC, test LOC, files, tokens, time, tool calls, and optional frontend build result. - `router`: exact classification across Direct, Decision, Debugging, Implementation, Exploration, and Verification, including overlap and negative-boundary cases. - `decision`: Practical versus Matt Pocock `grilling`. Uses a real resumed second turn and gates on frontier questions, one recommendation per question, no premature implementation, and convergence after scripted user decisions. Trade-off language is reported diagnostically but is not a declared grilling contract gate. -- `debug`: shared-root-cause tasks scored on the repaired invariant and sibling callers. Tests/TDD process receives no bonus. +- `debug`: shared-root-cause tasks scored on the repaired invariant and sibling callers. Tests/TDD process receives no bonus. Each Practical-owned Debug seed is required to fail its deterministic scorer, and a separate oracle fixture must pass it before the case is accepted into the catalog. `total_tokens` includes cached input because that is how Codex reports turn input. The report therefore also separates cached input, uncached input, output, and reasoning tokens. `duration_seconds` is per-cell process duration; suite elapsed time is recorded separately and is not obtained by summing concurrent cell durations. @@ -84,3 +90,5 @@ cells/ prompt, raw JSONL, stderr, answer, and code workspace per Use repeated paired results. A candidate is not accepted merely because its prose matches a Skill contract. Require no correctness/build regression, then compare delivered code and behavior. Treat LOC, tokens, and time as secondary within equally correct artifacts. `n=1` is a smoke result, not a stable ranking. A published stable ranking must pass `benchmarks/check_stability.py` with the default minimum `n=3`. The gate checks distinct repetition IDs, complete-run metadata, and infrastructure errors. Behavioral or build failures remain valid benchmark observations and therefore do not invalidate the sample by themselves. + +The public catalog is a **regression suite**, not a hidden generalization test. Once a case has influenced Skill wording, its future 100% score should be treated as a ceiling check. External benchmarks and a private held-out set are required for stronger claims. From 61ed4941e97580d5ddea09b152b746dcd08d479d Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 08:48:03 -0700 Subject: [PATCH 09/12] docs: update reproduction matrix for expanded cases --- benchmarks/REPRODUCING.md | 50 ++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/benchmarks/REPRODUCING.md b/benchmarks/REPRODUCING.md index a3c58fa..9846ad6 100644 --- a/benchmarks/REPRODUCING.md +++ b/benchmarks/REPRODUCING.md @@ -12,10 +12,12 @@ The chain deliberately separates three evidence types: | Router | Practical-owned exact classification corpus covering Direct, Decision, Debugging, Implementation, Exploration, and Verification | Current and previous Practical | Project regression benchmark | | Decision | Practical-owned two-turn scenarios and mechanical contract grader | Practical and Matt Pocock `grilling`, plus optional previous Practical | Comparative benchmark; `grilling` has no declared upstream behavior benchmark | | Debug | Ponytail `trace-transfer`/`trace-amount` plus Practical-owned shared-boundary cases; deterministic invariant and sibling-caller grading | Practical and Superpowers, plus optional previous Practical | Mixed upstream/custom comparative benchmark | -| Harness tests | Six Python tests for question/recommendation counting, delta direction, fixed Luna configuration, route coverage, and Decision-module injection | Runner only | Local benchmark-infrastructure regression tests | +| Harness tests | Python unit tests for runner mechanics, stability gating, catalog breadth, duplicate detection, scorer seed rejection, and oracle acceptance | Runner only | Local benchmark-infrastructure regression tests | The Decision and Debug comparisons must not be described as official Matt Pocock or Superpowers benchmark results. They are controlled Codex/Luna comparisons against those Skills' relevant behavior. Tests, TDD phases, planning prose, and workflow completeness receive no quality points; only the delivered behavior, safety invariant, build, and artifact metrics are scored. +The public Router/Decision/Debug catalog is a regression corpus. Cases that have influenced Skill wording remain useful for preventing regressions but are no longer independent evidence of generalization. See [`../docs/evaluations/2026-08-24-benchmark-landscape.md`](../docs/evaluations/2026-08-24-benchmark-landscape.md) for the external and held-out evidence plan. + ## Pinned upstream sources The runner verifies these exact commits before spending model calls: @@ -55,7 +57,7 @@ Set-Location practical-coding pwsh -NoProfile -File benchmarks/run.ps1 -SelfTest ``` -`-SelfTest` makes no model calls. It runs the six harness regression tests, verifies every profile case ID, then proves that the Ponytail and custom scorers accept their good fixtures and reject their bad fixtures. +`-SelfTest` makes no model calls. It runs the harness/stability/catalog unit tests, verifies every profile case ID, proves the pinned Ponytail scorers reject their bad references, proves every Practical-owned Debug seed fails, and proves each expanded Debug oracle passes its deterministic scorer. ## 2. Run the standard comparison @@ -66,19 +68,28 @@ pwsh -NoProfile -File benchmarks/run.ps1 ` -Workers 3 ``` -This runs 150 isolated cells when no previous-version or no-Skill arm is requested: +The current public `standard` profile runs 222 isolated cells when no previous-version or no-Skill arm is requested: -- 9 Delivery cases × 2 arms × 3 runs; -- 16 Router cases × 1 arm × 3 runs; -- 4 Decision cases × 2 arms × 3 runs; -- 4 Debug cases × 2 arms × 3 runs. +- 9 Delivery cases × 2 arms × 3 runs = 54; +- 28 Router cases × 1 arm × 3 runs = 84; +- 6 Decision cases × 2 arms × 3 runs = 36; +- 8 Debug cases × 2 arms × 3 runs = 48. -For the expanded 18-case Ponytail delivery matrix: +For the complete public matrix: ```powershell pwsh -NoProfile -File benchmarks/run.ps1 -Profile full -Runs 3 -Workers 3 ``` +`full` runs 324 cells without a previous-version or no-Skill arm: + +- 18 Delivery cases × 2 arms × 3 runs = 108; +- 28 Router cases × 1 arm × 3 runs = 84; +- 10 Decision cases × 2 arms × 3 runs = 60; +- 12 Debug cases × 2 arms × 3 runs = 72. + +The `smoke` profile intentionally remains small and defaults to one repetition. It is for harness/model sanity only. + ## 3. Run a before/after candidate gate While editing a dirty candidate, compare it with the checked-in version: @@ -102,6 +113,8 @@ pwsh -NoProfile -File benchmarks/run.ps1 ` The materialized baseline Skill is copied into the run directory. Both entrypoint and complete Skill-bundle hashes are recorded in `manifest.json`, preventing an ambiguous "previous version" comparison. +For a comparison that will be published as a stable ranking, add `-RequireStableRanking`. It rejects effective `n<3`, incomplete runs, infrastructure failures, and Delivery rankings without production-build evidence. + ## 4. Run a focused regression Selectors are repeatable: @@ -114,9 +127,9 @@ pwsh -NoProfile -File benchmarks/run.ps1 ` -BaselineRef HEAD pwsh -NoProfile -File benchmarks/run.ps1 ` - -Profile smoke ` - -Suite decision ` - -Case service-boundary ` + -Profile full ` + -Suite debug ` + -Case trace-cache-tenant ` -Runs 3 ``` @@ -148,18 +161,20 @@ The rescore timestamp and current runner hash are written to the manifest. Raw r ## Published v1.11 calibration -The following results were produced on Windows 11 with Python 3.13.14, Codex CLI 0.145.0, `gpt-5.6-luna`, reasoning `medium`, and three parallel workers. They are calibration results, not claims about every repository or model. +The following table is a **historical calibration produced before the public catalog expansion in this document**. It remains the evidence for the v1.11 iteration and must not be relabeled as results on the new 28/10/12 matrix. + +The run used Windows 11 with Python 3.13.14, Codex CLI 0.145.0, `gpt-5.6-luna`, reasoning `medium`, and three parallel workers. -| Matrix | Practical v1.11 | Frozen v1.10 | Comparator | Main difference | +| Historical matrix | Practical v1.11 | Frozen v1.10 | Comparator | Main difference | |---|---:|---:|---:|---| -| Router, 16 cases × n=3 | 48/48 | 48/48 | — | Both arms reached the current harness ceiling; the new negative rules caused no regression but did not prove a gain | +| Router, 16 cases × n=3 | 48/48 | 48/48 | — | Both arms reached that harness ceiling; the new negative rules caused no regression but did not prove a gain | | Debug, 4 cases × n=3 | 12/12 | 12/12 | Superpowers 10/12 | Superpowers twice repaired only the named caller and missed the sibling/shared invariant | | Decision, 4 cases × n=3 | 12/12 | 12/12 | grilling 10/12 | Both Practical versions converged after the scripted reply; grilling reopened `api-migration` twice | | Delivery, 6 differentiating cases × n=1 | 5/6 | 5/6 | Ponytail 5/6 | All arms scored 6/6 correct/safe; production builds separated the pass rate and remain unstable at `n=1` | -For Debug, suite median time was 39.1 seconds for v1.11, 44.5 seconds for v1.10, and 78.8 seconds for Superpowers; median total tokens were 80,940, 88,053, and 245,966 respectively. These secondary efficiency metrics matter only after correctness and safety. Delivery total LOC at `n=1` was 376 for v1.11, 363 for v1.10, and 343 for Ponytail, so the current data does **not** support a claim that v1.11 matches Ponytail's compactness. +For historical Debug, suite median time was 39.1 seconds for v1.11, 44.5 seconds for v1.10, and 78.8 seconds for Superpowers; median total tokens were 80,940, 88,053, and 245,966 respectively. These secondary efficiency metrics matter only after correctness and safety. Historical Delivery total LOC at `n=1` was 376 for v1.11, 363 for v1.10, and 343 for Ponytail, so those data do **not** support a claim that v1.11 matches Ponytail's compactness. -The published comparison used `-BaselineSkill docs/evaluations/snapshots/practical-v1.10`, not commit `75d5013`. See [`../docs/evaluations/2026-08-24-practical-v111-iteration.md`](../docs/evaluations/2026-08-24-practical-v111-iteration.md) for the complete per-case tables and acceptance decisions. +The published comparison used `-BaselineSkill docs/evaluations/snapshots/practical-v1.10`, not commit `75d5013`. See [`../docs/evaluations/2026-08-24-practical-v111-iteration.md`](../docs/evaluations/2026-08-24-practical-v111-iteration.md) for the complete historical per-case tables and acceptance decisions. ## Reproducibility limits @@ -168,4 +183,5 @@ The published comparison used `-BaselineSkill docs/evaluations/snapshots/practic - `input_tokens` already includes cached input. Compare `uncached_input_tokens`, `output_tokens`, and reasoning tokens separately when discussing cost. - A successful Ponytail-derived Delivery score proves the reused deterministic contract, not equivalence with Ponytail's original Claude runtime. - The runner disables normal user Skills, plugins, apps, memories, and multi-agent behavior, then embeds exactly one selected Skill arm. This isolates the comparison but differs from a user's fully configured interactive session. -- Do not generalize the focused v1.11 calibration rows into a whole-Skill ranking. Run the standard or full matrix after each material prompt change. +- The public Practical-owned cases are visible regression tests. Do not use their saturation as proof of unseen-task generalization. +- Do not generalize the focused v1.11 historical calibration rows into a whole-Skill ranking. Run the current standard/full matrix and an independent external or held-out suite after material prompt changes. From 40fddc3851dcd090a98b37ffbfadbcbb0f08b076 Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 08:49:22 -0700 Subject: [PATCH 10/12] fix: fingerprint the full benchmark runner bundle --- benchmarks/run_catalog.py | 34 +++++++++++++++++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/benchmarks/run_catalog.py b/benchmarks/run_catalog.py index 128ce65..2b6e910 100644 --- a/benchmarks/run_catalog.py +++ b/benchmarks/run_catalog.py @@ -3,6 +3,7 @@ from __future__ import annotations +import hashlib import sys from pathlib import Path @@ -14,6 +15,37 @@ from case_catalog import install -if __name__ == "__main__": +_CORE_SHA256 = bench.sha256 +_RUNTIME_FILES = ( + Path(bench.__file__).resolve(), + (HERE / "case_catalog.py").resolve(), + Path(__file__).resolve(), +) + + +def runner_bundle_sha256() -> str: + digest = hashlib.sha256() + for path in _RUNTIME_FILES: + digest.update(path.name.encode("utf-8")) + digest.update(b"\0") + digest.update(path.read_bytes()) + digest.update(b"\0") + return digest.hexdigest() + + +def catalog_aware_sha256(path: Path) -> str: + resolved = Path(path).resolve() + if resolved == Path(bench.__file__).resolve(): + return runner_bundle_sha256() + return _CORE_SHA256(resolved) + + +def configure() -> None: install(bench) + bench.VERSION = "1.3" + bench.sha256 = catalog_aware_sha256 + + +if __name__ == "__main__": + configure() raise SystemExit(bench.main()) From 7d769559931551fc1275eb023b4d48059d4cc126 Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 08:51:06 -0700 Subject: [PATCH 11/12] test: cover canonical benchmark fingerprint --- benchmarks/test_catalog.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/benchmarks/test_catalog.py b/benchmarks/test_catalog.py index 3c034af..5d2a8a9 100644 --- a/benchmarks/test_catalog.py +++ b/benchmarks/test_catalog.py @@ -3,6 +3,7 @@ from pathlib import Path from benchmarks import run_benchmarks as bench +from benchmarks import run_catalog from benchmarks.case_catalog import ( EXTRA_DEBUG_CASES, EXTRA_DECISION_CASES, @@ -63,6 +64,16 @@ def test_decision_cases_have_two_turn_resolution_contract(self): self.assertIn("Resolve the decision now", spec["reply"]) self.assertTrue(spec["expected"]) + def test_canonical_runner_fingerprint_includes_catalog(self): + raw_core = bench.sha256(Path(bench.__file__)) + bundled = run_catalog.runner_bundle_sha256() + self.assertEqual(len(bundled), 64) + self.assertNotEqual(bundled, raw_core) + self.assertEqual( + run_catalog.catalog_aware_sha256(Path(run_catalog.bench.__file__)), + bundled, + ) + if __name__ == "__main__": unittest.main() From 6d81f86da36fad6e11dae1b43b47534abd82c9aa Mon Sep 17 00:00:00 2001 From: Hubujiu Date: Mon, 24 Aug 2026 08:51:32 -0700 Subject: [PATCH 12/12] docs: fix matrix count and runner fingerprint note --- benchmarks/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/benchmarks/README.md b/benchmarks/README.md index a616491..ba40048 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -15,7 +15,7 @@ pwsh -File benchmarks/run.ps1 -Profile smoke -Suite router -Case direct-artifact pwsh -File benchmarks/run.ps1 -Rescore D:\path\to\benchmark-results\20260824-203839 ``` -`run.ps1` is the canonical entrypoint. It loads the core runner through `run_catalog.py`, which installs the extended public case catalog before execution. This keeps benchmark mechanics separate from the evolving task corpus. +`run.ps1` is the canonical entrypoint. It loads the core runner through `run_catalog.py`, which installs the extended public case catalog before execution. This keeps benchmark mechanics separate from the evolving task corpus. `manifest.json` fingerprints the complete benchmark runtime bundle (core runner + case catalog + canonical wrapper), so task/scorer changes cannot masquerade as the same benchmark revision. For a result that will be presented as a stable ranking, opt into the evidence gate: @@ -36,7 +36,7 @@ That command intentionally reports the published v1.11 Delivery `n=1` artifact a | Profile | Delivery | Router | Decision | Debug | Default runs | Cells without previous/no-Skill arm | |---|---:|---:|---:|---:|---:|---:| -| `smoke` | 3 | 4 | 1 | 1 | 1 | 13 | +| `smoke` | 3 | 4 | 1 | 1 | 1 | 14 | | `standard` | 9 | 28 | 6 | 8 | 3 | 222 | | `full` | 18 | 28 | 10 | 12 | 3 | 324 |