From e1dbdb5a71b9e30992fa20d9bd1434ed170a967b Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Sat, 8 Aug 2026 17:36:25 -0700 Subject: [PATCH 1/9] feat(swe-bench): content-addressed unit plan + durable mkdir-atomic work queue Adds the two foundations of the distributed SWE-bench harness: - units.py: shards an instance-id list into immutable, content-addressed units. The sha256 digest covers the ordered id list, so a plan cannot be silently reused across a different run, instance list, or ordering. - queue.py: a filesystem work queue whose claim is a bare os.mkdir (never makedirs(exist_ok=True), which hands a unit to every caller). available() is plan - claims - results, so deleting a result alone does NOT requeue a unit; requeue() is the only supported path and removes the result, the claim and the attempt records together. Env faults are ledgered separately from counted attempts, and abandoning a unit publishes a terminal result AND releases the claim so claims/ and results/ never disagree. --- .../swe_bench_distributed/__init__.py | 30 ++ .../evaluation/swe_bench_distributed/queue.py | 495 ++++++++++++++++++ .../evaluation/swe_bench_distributed/units.py | 187 +++++++ .../test_units_and_queue.py | 246 +++++++++ 4 files changed, 958 insertions(+) create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/queue.py create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/units.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_units_and_queue.py diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py new file mode 100644 index 000000000..2042c3d55 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Distributed SWE-bench execution across a fleet of SWE-bench services. + +The single-service :class:`~inference_endpoint.evaluation.swe_bench_scorer.SWEBenchScorer` +issues one run covering every instance. This package shards the instance list +into units, dispatches units across several services concurrently, classifies +infrastructure damage separately from genuine model failures, and refuses to +emit an accuracy number unless every planned instance id is accounted for +exactly once. +""" + +from .queue import ( + ClaimError, + UnitOutcome, + UnitResult, + WorkQueue, +) +from .units import Unit, UnitPlan, plan_units + +__all__ = [ + "ClaimError", + "Unit", + "UnitOutcome", + "UnitPlan", + "UnitResult", + "WorkQueue", + "plan_units", +] diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/queue.py b/src/inference_endpoint/evaluation/swe_bench_distributed/queue.py new file mode 100644 index 000000000..3feddae63 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/queue.py @@ -0,0 +1,495 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Durable work queue for distributed SWE-bench units. + +The queue is a directory tree so that a client crash costs nothing but the +in-flight units, and so that the merge gate reads durable records rather than +process memory. + +Layout under ``root``:: + + units.json immutable plan (see units.py) + claims//owner json owner record, written temp+rename + claims//hb heartbeat, mtime only + results/.json terminal record (succeeded OR abandoned) + failed/..json one record per *counted* attempt + failed/env/.*.json environment faults, NOT counted + failed/artifacts/... evidence snapshot taken before a retry + +Two invariants are load-bearing and are enforced here rather than by +convention: + +1. ``claim()`` is ``os.mkdir`` and nothing else. ``mkdir`` on an existing + directory fails atomically with ``EEXIST`` on every filesystem we run on, + including Lustre, so exactly one of N racing callers wins. ``makedirs(..., + exist_ok=True)`` would hand the unit to every caller. +2. ``requeue()`` is the only way to make a terminal unit runnable again, and it + removes the result, the claim tombstone *and* the counted attempt records + together. Deleting a result file by hand does not requeue a unit -- the + claim tombstone still hides it -- and that misunderstanding has cost real + campaign time. +""" + +from __future__ import annotations + +import errno +import logging +import os +import shutil +import socket +import time +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path +from typing import Any + +import msgspec + +from .units import PLAN_FILENAME, UnitPlan, read_plan + +logger = logging.getLogger(__name__) + +_OWNER = "owner" +_HEARTBEAT = "hb" +_CLAIM_CONTENTS = frozenset({_OWNER, _HEARTBEAT}) + +# Small files that explain a failure. Snapshotted before a retry reuses the +# unit's run directory: a unit that fails and then succeeds otherwise leaves +# only the success's artifacts, and a post-mortem then reads the wrong run. +EVIDENCE_FILES = ( + "status.json", + "swe_bench_results.json", + "preds.json", + "swe_bench_service_status.json", +) +EVIDENCE_LOG_TAIL_BYTES = 200_000 +EVIDENCE_LOGS = ("swe_bench_agent.log", "swe_bench_eval.log") + + +class ClaimError(RuntimeError): + """A claim operation could not be performed.""" + + +class UnitOutcome(StrEnum): + """How an attempt at a unit ended. + + ``ENV_FAULT`` is deliberately separate from ``FAILED``: a broken service, an + unreachable endpoint or a refused gate is a property of the *worker*, not of + the unit. Charging it to the unit's attempt budget abandons perfectly good + units because they happened to land on a sick host. + """ + + SUCCEEDED = "succeeded" + INFRA = "infra" + FAILED = "failed" + ENV_FAULT = "env_fault" + + +#: Outcomes that consume one of the unit's ``max_attempts``. +COUNTED_OUTCOMES = frozenset({UnitOutcome.INFRA, UnitOutcome.FAILED}) + + +@dataclass(slots=True) +class UnitResult: + """A terminal or attempt record for one unit.""" + + unit_id: str + run_id: str + plan_digest: str + outcome: UnitOutcome + accounted_instance_ids: tuple[str, ...] = () + resolved_instance_ids: tuple[str, ...] = () + infra_error_count: int = 0 + genuine_error_count: int = 0 + error_kinds: dict[str, int] = field(default_factory=dict) + service_url: str | None = None + endpoint_fingerprint: str | None = None + service_run_id: str | None = None + attempt: int = 0 + abandoned: bool = False + duration_s: float = 0.0 + detail: str | None = None + finished_at: float = field(default_factory=time.time) + + def to_dict(self) -> dict[str, Any]: + return { + "unit_id": self.unit_id, + "run_id": self.run_id, + "plan_digest": self.plan_digest, + "outcome": self.outcome.value, + "accounted_instance_ids": list(self.accounted_instance_ids), + "resolved_instance_ids": list(self.resolved_instance_ids), + "infra_error_count": self.infra_error_count, + "genuine_error_count": self.genuine_error_count, + "error_kinds": dict(self.error_kinds), + "service_url": self.service_url, + "endpoint_fingerprint": self.endpoint_fingerprint, + "service_run_id": self.service_run_id, + "attempt": self.attempt, + "abandoned": self.abandoned, + "duration_s": self.duration_s, + "detail": self.detail, + "finished_at": self.finished_at, + } + + @classmethod + def from_dict(cls, raw: dict[str, Any]) -> UnitResult: + return cls( + unit_id=str(raw["unit_id"]), + run_id=str(raw["run_id"]), + plan_digest=str(raw["plan_digest"]), + outcome=UnitOutcome(str(raw["outcome"])), + accounted_instance_ids=tuple( + str(x) for x in raw.get("accounted_instance_ids") or () + ), + resolved_instance_ids=tuple( + str(x) for x in raw.get("resolved_instance_ids") or () + ), + infra_error_count=int(raw.get("infra_error_count") or 0), + genuine_error_count=int(raw.get("genuine_error_count") or 0), + error_kinds=dict(raw.get("error_kinds") or {}), + service_url=raw.get("service_url"), + endpoint_fingerprint=raw.get("endpoint_fingerprint"), + service_run_id=raw.get("service_run_id"), + attempt=int(raw.get("attempt") or 0), + abandoned=bool(raw.get("abandoned")), + duration_s=float(raw.get("duration_s") or 0.0), + detail=raw.get("detail"), + finished_at=float(raw.get("finished_at") or 0.0), + ) + + +@dataclass(slots=True) +class OwnerRecord: + unit_id: str + host: str + pid: int + boot_id: str + plan_digest: str + claimed_at: float + endpoint_fingerprint: str | None = None + slurm_job_id: str | None = None + slurm_step_id: str | None = None + + def to_dict(self) -> dict[str, Any]: + return { + "unit_id": self.unit_id, + "host": self.host, + "pid": self.pid, + "boot_id": self.boot_id, + "plan_digest": self.plan_digest, + "claimed_at": self.claimed_at, + "endpoint_fingerprint": self.endpoint_fingerprint, + "slurm_job_id": self.slurm_job_id, + "slurm_step_id": self.slurm_step_id, + } + + @classmethod + def from_dict(cls, raw: dict[str, Any]) -> OwnerRecord: + return cls( + unit_id=str(raw["unit_id"]), + host=str(raw.get("host") or ""), + pid=int(raw.get("pid") or 0), + boot_id=str(raw.get("boot_id") or ""), + plan_digest=str(raw.get("plan_digest") or ""), + claimed_at=float(raw.get("claimed_at") or 0.0), + endpoint_fingerprint=raw.get("endpoint_fingerprint"), + slurm_job_id=raw.get("slurm_job_id"), + slurm_step_id=raw.get("slurm_step_id"), + ) + + +def boot_id() -> str: + """Identify this boot of this host. + + A pid alone is not proof of liveness: after a reboot the same pid can belong + to something else entirely, and the reaper would then conclude a dead owner + is alive and leave its unit blocked forever. + """ + try: + return Path("/proc/sys/kernel/random/boot_id").read_text().strip() + except OSError: + try: + return str(int(time.time() - time.monotonic())) + except (OSError, ValueError): # pragma: no cover - defensive + return "unknown" + + +def _atomic_write_json(path: Path, payload: dict[str, Any]) -> None: + tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp") + tmp.write_bytes(msgspec.json.encode(payload)) + tmp.replace(path) + + +class WorkQueue: + """Filesystem-backed queue over an immutable :class:`UnitPlan`.""" + + def __init__(self, root: os.PathLike[str] | str, plan: UnitPlan) -> None: + self.root = Path(root) + self.plan = plan + self.claims_dir = self.root / "claims" + self.results_dir = self.root / "results" + self.failed_dir = self.root / "failed" + self.env_failed_dir = self.failed_dir / "env" + self.artifacts_dir = self.failed_dir / "artifacts" + for directory in ( + self.root, + self.claims_dir, + self.results_dir, + self.failed_dir, + self.env_failed_dir, + self.artifacts_dir, + ): + directory.mkdir(parents=True, exist_ok=True) + self.plan.write(self.root) + self._boot_id = boot_id() + + # ------------------------------------------------------------------ open -- + + @classmethod + def open(cls, root: os.PathLike[str] | str) -> WorkQueue: + """Reopen an existing queue, reading its plan from disk.""" + root_path = Path(root) + return cls(root_path, read_plan(root_path / PLAN_FILENAME)) + + # ----------------------------------------------------------- inspection -- + + def claimed_unit_ids(self) -> set[str]: + try: + return {entry.name for entry in self.claims_dir.iterdir() if entry.is_dir()} + except FileNotFoundError: # pragma: no cover - created in __init__ + return set() + + def completed_unit_ids(self) -> set[str]: + return {path.stem for path in self.results_dir.glob("*.json")} + + def available_unit_ids(self) -> list[str]: + """Units that are neither claimed nor terminal, in plan order. + + Subtracting *both* claims and results is what makes a hand-deleted + result file a no-op: the claim tombstone still hides the unit. Use + :meth:`requeue`. + """ + taken = self.claimed_unit_ids() | self.completed_unit_ids() + return [unit_id for unit_id in self.plan.unit_ids if unit_id not in taken] + + def attempts(self, unit_id: str) -> int: + """Number of *counted* attempts recorded for a unit.""" + return len(list(self.failed_dir.glob(f"{unit_id}.*.json"))) + + def owner(self, unit_id: str) -> OwnerRecord | None: + path = self.claims_dir / unit_id / _OWNER + try: + raw = msgspec.json.decode(path.read_bytes(), type=dict) + except (OSError, msgspec.DecodeError): + return None + try: + return OwnerRecord.from_dict(raw) + except (KeyError, TypeError, ValueError): + return None + + def heartbeat_age(self, unit_id: str, *, now: float | None = None) -> float | None: + path = self.claims_dir / unit_id / _HEARTBEAT + try: + mtime = path.stat().st_mtime + except OSError: + return None + return (time.time() if now is None else now) - mtime + + def result(self, unit_id: str) -> UnitResult | None: + path = self.results_dir / f"{unit_id}.json" + try: + raw = msgspec.json.decode(path.read_bytes(), type=dict) + except (OSError, msgspec.DecodeError): + return None + try: + return UnitResult.from_dict(raw) + except (KeyError, TypeError, ValueError): + return None + + def results(self) -> dict[str, UnitResult]: + found: dict[str, UnitResult] = {} + for path in sorted(self.results_dir.glob("*.json")): + result = self.result(path.stem) + if result is not None: + found[path.stem] = result + return found + + # ---------------------------------------------------------------- claim -- + + def claim( + self, + unit_id: str, + *, + endpoint_fingerprint: str | None = None, + ) -> OwnerRecord | None: + """Take exclusive ownership of ``unit_id``; ``None`` if someone else has it.""" + if unit_id not in self.plan.unit_ids: + raise ClaimError( + f"{unit_id!r} is not in the plan for run {self.plan.run_id}" + ) + claim_dir = self.claims_dir / unit_id + try: + # THE RACE IS DECIDED HERE. mkdir, never makedirs/exist_ok. + os.mkdir(claim_dir) + except FileExistsError: + return None + except OSError as exc: + if exc.errno == errno.EEXIST: # pragma: no cover - platform variance + return None + raise ClaimError(f"could not claim {unit_id}: {exc}") from exc + + record = OwnerRecord( + unit_id=unit_id, + host=socket.gethostname(), + pid=os.getpid(), + boot_id=self._boot_id, + plan_digest=self.plan.digest, + claimed_at=time.time(), + endpoint_fingerprint=endpoint_fingerprint, + slurm_job_id=os.environ.get("SLURM_JOB_ID") or None, + slurm_step_id=os.environ.get("SLURM_STEP_ID") or None, + ) + # Sole owner from here, but still temp+rename so the reaper never reads + # a half-written owner record and calls it malformed. + _atomic_write_json(claim_dir / _OWNER, record.to_dict()) + (claim_dir / _HEARTBEAT).touch() + return record + + def beat(self, unit_id: str) -> None: + path = self.claims_dir / unit_id / _HEARTBEAT + try: + path.touch() + except OSError: + logger.debug("could not refresh heartbeat for %s", unit_id, exc_info=True) + + def release(self, unit_id: str) -> bool: + """Hand a claimed unit back to the queue. + + Removes the claim *directory*. Removing only the ``owner`` file leaves an + ownerless directory, which still hides the unit and merely relabels the + problem. + """ + claim_dir = self.claims_dir / unit_id + if not claim_dir.exists(): + return False + shutil.rmtree(claim_dir, ignore_errors=True) + return True + + def is_pure_bookkeeping(self, unit_id: str) -> bool: + """True when a claim directory holds only ``owner``/``hb``.""" + claim_dir = self.claims_dir / unit_id + try: + contents = {entry.name for entry in claim_dir.iterdir()} + except OSError: + return False + return not (contents - _CLAIM_CONTENTS) + + # -------------------------------------------------------------- publish -- + + def publish(self, result: UnitResult) -> None: + """Record a terminal result and release the claim. + + Releasing here is not optional. The abandon path once published a result + but kept the claim directory, so ``claims/`` and ``results/`` disagreed + for the rest of the campaign and every reaper pass had a phantom to + reason about. Releasing is safe because :meth:`available_unit_ids` + subtracts results as well as claims. + """ + self._check_digest(result) + _atomic_write_json( + self.results_dir / f"{result.unit_id}.json", result.to_dict() + ) + self.release(result.unit_id) + + def record_attempt(self, result: UnitResult) -> int: + """Record a non-terminal attempt. Returns the counted-attempt total. + + ``ENV_FAULT`` attempts are written to ``failed/env/`` and do not + increment the counter. + """ + self._check_digest(result) + if result.outcome is UnitOutcome.ENV_FAULT: + path = self.env_failed_dir / f"{result.unit_id}.{time.time_ns()}.json" + _atomic_write_json(path, result.to_dict()) + return self.attempts(result.unit_id) + count = self.attempts(result.unit_id) + 1 + result.attempt = count + _atomic_write_json( + self.failed_dir / f"{result.unit_id}.{count}.json", result.to_dict() + ) + return count + + def snapshot_evidence(self, unit_id: str, source_dir: Path, attempt: int) -> Path: + """Copy the small files that explain a failure before a retry overwrites them.""" + target = self.artifacts_dir / f"{unit_id}.attempt{attempt}" + target.mkdir(parents=True, exist_ok=True) + for name in EVIDENCE_FILES: + candidate = source_dir / name + if candidate.is_file(): + try: + shutil.copy2(candidate, target / name) + except OSError: + logger.debug("could not snapshot %s", candidate, exc_info=True) + for name in EVIDENCE_LOGS: + candidate = source_dir / name + if not candidate.is_file(): + continue + try: + with candidate.open("rb") as handle: + handle.seek(0, os.SEEK_END) + size = handle.tell() + handle.seek(max(0, size - EVIDENCE_LOG_TAIL_BYTES)) + (target / f"{name}.tail").write_bytes(handle.read()) + except OSError: + logger.debug("could not snapshot %s", candidate, exc_info=True) + return target + + def abandon(self, result: UnitResult) -> None: + """Publish a terminal, explicitly-abandoned result.""" + result.abandoned = True + self.publish(result) + + # -------------------------------------------------------------- requeue -- + + def requeue(self, unit_id: str) -> dict[str, list[str]]: + """Make a unit runnable again. The *only* supported way. + + Removes, together: the terminal result, the claim tombstone, and every + counted attempt record. Removing any subset leaves the unit invisible or + already out of attempts, which is how "I deleted the result, why is it + not rerunning?" happens. + """ + if unit_id not in self.plan.unit_ids: + raise ClaimError( + f"{unit_id!r} is not in the plan for run {self.plan.run_id}" + ) + removed: dict[str, list[str]] = {"results": [], "claims": [], "attempts": []} + result_path = self.results_dir / f"{unit_id}.json" + if result_path.exists(): + result_path.unlink() + removed["results"].append(str(result_path)) + claim_dir = self.claims_dir / unit_id + if claim_dir.exists(): + shutil.rmtree(claim_dir, ignore_errors=True) + removed["claims"].append(str(claim_dir)) + for path in sorted(self.failed_dir.glob(f"{unit_id}.*.json")): + path.unlink() + removed["attempts"].append(str(path)) + return removed + + # ---------------------------------------------------------------- utils -- + + def _check_digest(self, result: UnitResult) -> None: + if result.plan_digest != self.plan.digest: + raise ClaimError( + f"refusing to record {result.unit_id}: plan digest " + f"{result.plan_digest[:12]} does not match this queue's " + f"{self.plan.digest[:12]}" + ) + if result.run_id != self.plan.run_id: + raise ClaimError( + f"refusing to record {result.unit_id}: run id {result.run_id!r} " + f"does not match this queue's {self.plan.run_id!r}" + ) diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/units.py b/src/inference_endpoint/evaluation/swe_bench_distributed/units.py new file mode 100644 index 000000000..9a163a037 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/units.py @@ -0,0 +1,187 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Shard plan: the immutable binding between units and instance ids. + +The plan is content-addressed. Every unit result carries the plan digest, and +the merge gate refuses to combine results whose digest differs from the plan +being merged. That is what makes it impossible to accidentally merge results +from a different run, a different instance list, or a different ordering into +one accuracy number. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import msgspec + +PLAN_FILENAME = "units.json" + + +class PlanError(ValueError): + """The requested shard plan cannot be built, or a plan file is invalid.""" + + +@dataclass(frozen=True, slots=True) +class Unit: + """One dispatchable shard of a run.""" + + unit_id: str + run_id: str + shard: int + instance_ids: tuple[str, ...] + + def to_dict(self) -> dict[str, Any]: + return { + "unit_id": self.unit_id, + "run_id": self.run_id, + "shard": self.shard, + "instance_ids": list(self.instance_ids), + } + + +@dataclass(frozen=True, slots=True) +class UnitPlan: + """The full, immutable set of units for one run id.""" + + run_id: str + shard_size: int + digest: str + units: tuple[Unit, ...] + + @property + def instance_ids(self) -> tuple[str, ...]: + return tuple( + instance_id for unit in self.units for instance_id in unit.instance_ids + ) + + def unit(self, unit_id: str) -> Unit: + for candidate in self.units: + if candidate.unit_id == unit_id: + return candidate + raise KeyError(unit_id) + + @property + def unit_ids(self) -> tuple[str, ...]: + return tuple(unit.unit_id for unit in self.units) + + def to_dict(self) -> dict[str, Any]: + return { + "run_id": self.run_id, + "shard_size": self.shard_size, + "digest": self.digest, + "units": [unit.to_dict() for unit in self.units], + } + + def write(self, directory: Path) -> Path: + """Write the plan once. Rewriting an existing, different plan is an error.""" + directory.mkdir(parents=True, exist_ok=True) + path = directory / PLAN_FILENAME + if path.exists(): + existing = read_plan(path) + if existing.digest != self.digest or existing.run_id != self.run_id: + raise PlanError( + f"refusing to overwrite plan at {path}: existing run_id=" + f"{existing.run_id!r} digest={existing.digest[:12]} differs from " + f"new run_id={self.run_id!r} digest={self.digest[:12]}" + ) + return path + tmp = path.with_name(f".{path.name}.tmp") + tmp.write_bytes(msgspec.json.encode(self.to_dict())) + tmp.replace(path) + return path + + +def plan_digest(run_id: str, instance_ids: list[str] | tuple[str, ...]) -> str: + """Digest over the run id and the ordered instance list. + + Order is included deliberately: two plans over the same ids in a different + order produce different shards, so they are different plans. + """ + hasher = hashlib.sha256() + hasher.update(run_id.encode()) + hasher.update(b"\0") + for instance_id in instance_ids: + hasher.update(instance_id.encode()) + hasher.update(b"\n") + return hasher.hexdigest() + + +def plan_units( + run_id: str, + instance_ids: list[str] | tuple[str, ...], + *, + shard_size: int = 10, +) -> UnitPlan: + """Split ``instance_ids`` into fixed-size shards, in order. + + The final shard is short when the count is not a multiple of ``shard_size``; + it is never padded and never merged into its neighbour, because the merge + gate compares id sets and a padded shard would claim ids it never ran. + """ + if not run_id or "/" in run_id or run_id in {".", ".."}: + raise PlanError(f"invalid run_id: {run_id!r}") + if shard_size < 1: + raise PlanError(f"shard_size must be >= 1; got {shard_size}") + ordered = [str(instance_id) for instance_id in instance_ids] + if not ordered: + raise PlanError("cannot plan a run with no instance ids") + duplicates = sorted({x for x in ordered if ordered.count(x) > 1}) + if duplicates: + raise PlanError( + "instance ids must be unique; duplicated: " + ", ".join(duplicates[:10]) + ) + + digest = plan_digest(run_id, ordered) + units: list[Unit] = [] + for shard, start in enumerate(range(0, len(ordered), shard_size)): + chunk = tuple(ordered[start : start + shard_size]) + units.append( + Unit( + unit_id=f"{run_id}.s{shard:02d}", + run_id=run_id, + shard=shard, + instance_ids=chunk, + ) + ) + return UnitPlan( + run_id=run_id, shard_size=shard_size, digest=digest, units=tuple(units) + ) + + +def read_plan(path: Path) -> UnitPlan: + try: + raw = msgspec.json.decode(path.read_bytes(), type=dict) + except (OSError, msgspec.DecodeError) as exc: + raise PlanError(f"could not read unit plan at {path}") from exc + try: + units = tuple( + Unit( + unit_id=str(entry["unit_id"]), + run_id=str(entry["run_id"]), + shard=int(entry["shard"]), + instance_ids=tuple(str(x) for x in entry["instance_ids"]), + ) + for entry in raw["units"] + ) + plan = UnitPlan( + run_id=str(raw["run_id"]), + shard_size=int(raw["shard_size"]), + digest=str(raw["digest"]), + units=units, + ) + except (KeyError, TypeError, ValueError) as exc: + raise PlanError(f"malformed unit plan at {path}") from exc + + recomputed = plan_digest(plan.run_id, list(plan.instance_ids)) + if recomputed != plan.digest: + raise PlanError( + f"unit plan at {path} is inconsistent: recorded digest " + f"{plan.digest[:12]} does not match its own instance list " + f"({recomputed[:12]})" + ) + return plan diff --git a/tests/unit/evaluation/swe_bench_distributed/test_units_and_queue.py b/tests/unit/evaluation/swe_bench_distributed/test_units_and_queue.py new file mode 100644 index 000000000..6343b7536 --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_units_and_queue.py @@ -0,0 +1,246 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Unit plan and work-queue semantics.""" + +from __future__ import annotations + +import threading + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed.queue import ( + ClaimError, + UnitOutcome, + UnitResult, + WorkQueue, +) +from inference_endpoint.evaluation.swe_bench_distributed.units import ( + PlanError, + plan_units, + read_plan, +) + +pytestmark = pytest.mark.unit + + +def make_ids(n: int) -> list[str]: + return [f"repo__proj-{i:03d}" for i in range(n)] + + +@pytest.fixture +def queue(tmp_path): + plan = plan_units("run-a", make_ids(25), shard_size=10) + return WorkQueue(tmp_path / "wq", plan) + + +def result_for(queue: WorkQueue, unit_id: str, **overrides) -> UnitResult: + unit = queue.plan.unit(unit_id) + payload = { + "unit_id": unit_id, + "run_id": unit.run_id, + "plan_digest": queue.plan.digest, + "outcome": UnitOutcome.SUCCEEDED, + "accounted_instance_ids": unit.instance_ids, + "resolved_instance_ids": unit.instance_ids[:1], + } + payload.update(overrides) + return UnitResult(**payload) + + +class TestPlan: + def test_shards_in_order_with_a_short_tail(self): + plan = plan_units("run-a", make_ids(25), shard_size=10) + assert [len(unit.instance_ids) for unit in plan.units] == [10, 10, 5] + assert plan.unit_ids == ("run-a.s00", "run-a.s01", "run-a.s02") + # The short tail is never padded: a padded shard would claim ids it + # never ran and the merge gate compares ids, not counts. + assert plan.instance_ids == tuple(make_ids(25)) + + def test_digest_depends_on_order(self): + ids = make_ids(20) + assert ( + plan_units("r", ids).digest != plan_units("r", list(reversed(ids))).digest + ) + + def test_digest_depends_on_run_id(self): + ids = make_ids(20) + assert plan_units("r1", ids).digest != plan_units("r2", ids).digest + + def test_duplicate_instance_ids_are_refused(self): + with pytest.raises(PlanError, match="unique"): + plan_units("r", ["a", "b", "a"]) + + def test_empty_plan_is_refused(self): + with pytest.raises(PlanError): + plan_units("r", []) + + def test_plan_round_trips_and_self_verifies(self, tmp_path): + plan = plan_units("run-a", make_ids(12), shard_size=5) + path = plan.write(tmp_path) + assert read_plan(path).digest == plan.digest + + def test_rewriting_a_different_plan_is_refused(self, tmp_path): + plan_units("run-a", make_ids(10)).write(tmp_path) + with pytest.raises(PlanError, match="refusing to overwrite"): + plan_units("run-a", make_ids(11)).write(tmp_path) + + def test_tampered_plan_file_is_detected(self, tmp_path): + plan = plan_units("run-a", make_ids(10)) + path = plan.write(tmp_path) + raw = path.read_text().replace(plan.digest, "0" * 64) + path.write_text(raw) + with pytest.raises(PlanError, match="inconsistent"): + read_plan(path) + + +class TestClaims: + def test_a_second_claim_loses(self, queue): + assert queue.claim("run-a.s00") is not None + assert queue.claim("run-a.s00") is None + + def test_exactly_one_thread_wins_a_contested_claim(self, queue): + winners: list[object] = [] + barrier = threading.Barrier(8) + + def contend(): + barrier.wait() + if queue.claim("run-a.s01") is not None: + winners.append(object()) + + threads = [threading.Thread(target=contend) for _ in range(8)] + for thread in threads: + thread.start() + for thread in threads: + thread.join() + assert len(winners) == 1 + + def test_claiming_a_unit_outside_the_plan_is_an_error(self, queue): + with pytest.raises(ClaimError): + queue.claim("other-run.s00") + + def test_release_removes_the_whole_directory(self, queue): + queue.claim("run-a.s00") + assert queue.release("run-a.s00") + # Removing only `owner` would leave an ownerless directory that still + # hides the unit -- a relabelled problem, not a fix. + assert not (queue.claims_dir / "run-a.s00").exists() + assert "run-a.s00" in queue.available_unit_ids() + + def test_owner_record_carries_identity(self, queue): + record = queue.claim("run-a.s00") + stored = queue.owner("run-a.s00") + assert stored is not None + assert stored.pid == record.pid + assert stored.boot_id == record.boot_id + assert stored.plan_digest == queue.plan.digest + + +class TestAvailability: + def test_claims_and_results_both_hide_a_unit(self, queue): + queue.claim("run-a.s00") + queue.publish(result_for(queue, "run-a.s01")) + assert queue.available_unit_ids() == ["run-a.s02"] + + def test_publish_releases_the_claim(self, queue): + queue.claim("run-a.s00") + queue.publish(result_for(queue, "run-a.s00")) + # An abandoned or published unit that keeps its claim makes claims/ and + # results/ disagree for the rest of the run. + assert queue.claimed_unit_ids() == set() + + def test_abandon_publishes_and_releases(self, queue): + queue.claim("run-a.s00") + queue.abandon(result_for(queue, "run-a.s00", outcome=UnitOutcome.FAILED)) + stored = queue.result("run-a.s00") + assert stored is not None and stored.abandoned + assert queue.claimed_unit_ids() == set() + + def test_result_from_another_plan_is_refused(self, queue): + bad = result_for(queue, "run-a.s00", plan_digest="0" * 64) + with pytest.raises(ClaimError, match="plan digest"): + queue.publish(bad) + + +class TestRequeue: + def test_deleting_only_the_result_does_not_requeue(self, queue): + queue.claim("run-a.s00") + queue.publish(result_for(queue, "run-a.s00")) + # Re-claim so a tombstone exists, mimicking an interrupted retry. + queue.claim("run-a.s00") + (queue.results_dir / "run-a.s00.json").unlink() + assert "run-a.s00" not in queue.available_unit_ids() + + def test_deleting_only_the_claim_does_not_requeue(self, queue): + queue.claim("run-a.s00") + queue.publish(result_for(queue, "run-a.s00")) + queue.release("run-a.s00") + assert "run-a.s00" not in queue.available_unit_ids() + + def test_requeue_removes_result_claim_and_attempts(self, queue): + queue.claim("run-a.s00") + queue.record_attempt(result_for(queue, "run-a.s00", outcome=UnitOutcome.FAILED)) + queue.record_attempt(result_for(queue, "run-a.s00", outcome=UnitOutcome.INFRA)) + queue.publish(result_for(queue, "run-a.s00")) + queue.claim("run-a.s00") + + removed = queue.requeue("run-a.s00") + + assert len(removed["results"]) == 1 + assert len(removed["claims"]) == 1 + assert len(removed["attempts"]) == 2 + assert "run-a.s00" in queue.available_unit_ids() + assert queue.attempts("run-a.s00") == 0 + + def test_requeue_outside_the_plan_is_an_error(self, queue): + with pytest.raises(ClaimError): + queue.requeue("other-run.s00") + + +class TestAttemptLedger: + def test_environment_faults_do_not_consume_the_budget(self, queue): + for _ in range(5): + queue.record_attempt( + result_for(queue, "run-a.s00", outcome=UnitOutcome.ENV_FAULT) + ) + # A broken host is a property of the host, not of the unit. Charging it + # to the unit abandons good units for landing in the wrong place. + assert queue.attempts("run-a.s00") == 0 + + def test_counted_failures_increment(self, queue): + assert ( + queue.record_attempt( + result_for(queue, "run-a.s00", outcome=UnitOutcome.FAILED) + ) + == 1 + ) + assert ( + queue.record_attempt( + result_for(queue, "run-a.s00", outcome=UnitOutcome.INFRA) + ) + == 2 + ) + + def test_evidence_is_snapshotted_before_a_retry_overwrites_it( + self, queue, tmp_path + ): + source = tmp_path / "unit-run" + source.mkdir() + (source / "status.json").write_text('{"attempt": 1}') + (source / "swe_bench_agent.log").write_text("first attempt log") + + target = queue.snapshot_evidence("run-a.s00", source, attempt=1) + + # The retry reuses the run directory, so a unit that fails then succeeds + # would otherwise leave only the success's artifacts behind. + (source / "status.json").write_text('{"attempt": 2}') + assert (target / "status.json").read_text() == '{"attempt": 1}' + assert (target / "swe_bench_agent.log.tail").read_text() == "first attempt log" + + +class TestReopen: + def test_reopen_reads_the_plan_from_disk(self, queue): + queue.publish(result_for(queue, "run-a.s00")) + reopened = WorkQueue.open(queue.root) + assert reopened.plan.digest == queue.plan.digest + assert reopened.completed_unit_ids() == {"run-a.s00"} From d53af867eadb4713bf17f4b043c5258b2c59e7db Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Sat, 8 Aug 2026 17:39:43 -0700 Subject: [PATCH 2/9] feat(swe-bench): all-or-nothing merge gate scoped to exactly one run id merge_run(wq, run_id) refuses to emit an accuracy number unless every planned unit has a terminal result, none is abandoned, every unit accounts for exactly its planned instance IDS (a set comparison, never a count), the union equals the plan with no cross-shard duplicates, every plan_digest matches, and no unit carries an infra error. Refusal is a structured MergeRefusal naming the offending units and ids; there is no force flag and no partial-credit path. There is deliberately no --all: merge_run takes a required run id and treats a foreign run id or digest as a hard error, not a skip. verify_inventory() cross-checks claims, results and the id-union as independent producers, so a blind spot shared by one instrument cannot certify itself. --- .../swe_bench_distributed/__init__.py | 5 + .../evaluation/swe_bench_distributed/merge.py | 230 ++++++++++++++++++ .../swe_bench_distributed/test_merge.py | 189 ++++++++++++++ 3 files changed, 424 insertions(+) create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/merge.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_merge.py diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py index 2042c3d55..3507289bb 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py @@ -11,6 +11,7 @@ exactly once. """ +from .merge import MergeRefusal, MergeResult, merge_run, verify_inventory from .queue import ( ClaimError, UnitOutcome, @@ -21,10 +22,14 @@ __all__ = [ "ClaimError", + "MergeRefusal", + "MergeResult", "Unit", "UnitOutcome", "UnitPlan", "UnitResult", "WorkQueue", + "merge_run", "plan_units", + "verify_inventory", ] diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/merge.py b/src/inference_endpoint/evaluation/swe_bench_distributed/merge.py new file mode 100644 index 000000000..a8dbee7cf --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/merge.py @@ -0,0 +1,230 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The merge gate: refuse to emit an accuracy unless every id is accounted for. + +The single most important property of a sharded accuracy run is that it never +divides the results of 190 instances by 200. The gate is all-or-nothing by +design: there is no force flag and no partial-credit path, because a partial +number is indistinguishable from a real one once it leaves this module. + +The gate is also scoped to exactly one run. There is no ``merge_all``. Merging +"every run that looks finished" once re-merged hundreds of banked results +belonging to unrelated configurations into one number. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from .queue import UnitOutcome, UnitResult, WorkQueue +from .units import UnitPlan + + +class MergeRefusal(RuntimeError): + """The gate refused to produce an accuracy number. + + ``reasons`` lists every independent failure, so one merge attempt reports + everything wrong rather than the first thing wrong. + """ + + def __init__(self, run_id: str, reasons: list[str]) -> None: + self.run_id = run_id + self.reasons = reasons + super().__init__( + f"refusing to score run {run_id!r}: " + + "; ".join(reasons[:10]) + + (f" (+{len(reasons) - 10} more)" if len(reasons) > 10 else "") + ) + + +@dataclass(slots=True) +class MergeResult: + run_id: str + plan_digest: str + total_instances: int + resolved_instances: int + unit_count: int + + @property + def resolved_rate(self) -> float: + return self.resolved_instances / self.total_instances + + def to_dict(self) -> dict[str, Any]: + return { + "run_id": self.run_id, + "plan_digest": self.plan_digest, + "total_instances": self.total_instances, + "resolved_instances": self.resolved_instances, + "resolved_rate": self.resolved_rate, + "unit_count": self.unit_count, + } + + +@dataclass(slots=True) +class InventoryReport: + """Cross-check of three independently produced views of the same run. + + The units the plan asked for, the units the queue recorded results for, and + the instance ids those results claim to have covered are produced by + different code paths. Checking one against itself is how a verification pass + can agree with a broken system: the instrument shares the blind spot. These + must agree with each other. + """ + + missing_units: list[str] = field(default_factory=list) + foreign_units: list[str] = field(default_factory=list) + unreadable_units: list[str] = field(default_factory=list) + ownerless_claims: list[str] = field(default_factory=list) + claims_without_results: list[str] = field(default_factory=list) + + @property + def consistent(self) -> bool: + return not ( + self.missing_units + or self.foreign_units + or self.unreadable_units + or self.ownerless_claims + ) + + +def verify_inventory(queue: WorkQueue) -> InventoryReport: + """Compare the plan, the claim directory and the result directory.""" + report = InventoryReport() + plan_units = set(queue.plan.unit_ids) + + result_files = {path.stem for path in queue.results_dir.glob("*.json")} + report.foreign_units = sorted(result_files - plan_units) + report.missing_units = sorted(plan_units - result_files) + for unit_id in sorted(result_files & plan_units): + if queue.result(unit_id) is None: + report.unreadable_units.append(unit_id) + + for unit_id in sorted(queue.claimed_unit_ids()): + if queue.owner(unit_id) is None: + report.ownerless_claims.append(unit_id) + if unit_id not in result_files: + report.claims_without_results.append(unit_id) + return report + + +def merge_run(queue: WorkQueue, run_id: str) -> MergeResult: + """Score one run, or refuse. + + ``run_id`` is required and must match the queue's plan. Passing another + run's id is an error, not a filter. + """ + plan: UnitPlan = queue.plan + if run_id != plan.run_id: + raise MergeRefusal( + run_id, + [ + f"queue at {queue.root} holds run {plan.run_id!r}, not {run_id!r}; " + "a merge is always scoped to exactly one run" + ], + ) + + reasons: list[str] = [] + inventory = verify_inventory(queue) + if inventory.foreign_units: + reasons.append( + "results present for units outside the plan: " + + ", ".join(inventory.foreign_units[:5]) + ) + if inventory.unreadable_units: + reasons.append( + "unreadable result records: " + ", ".join(inventory.unreadable_units[:5]) + ) + if inventory.ownerless_claims: + reasons.append( + "claims with no readable owner: " + + ", ".join(inventory.ownerless_claims[:5]) + ) + if inventory.missing_units: + reasons.append( + f"{len(inventory.missing_units)} of {len(plan.units)} units have no " + "result: " + ", ".join(inventory.missing_units[:5]) + ) + + results: dict[str, UnitResult] = queue.results() + seen_ids: dict[str, str] = {} + resolved: set[str] = set() + + for unit in plan.units: + result = results.get(unit.unit_id) + if result is None: + continue + if result.plan_digest != plan.digest: + reasons.append( + f"{unit.unit_id}: result belongs to plan {result.plan_digest[:12]}, " + f"not {plan.digest[:12]}" + ) + continue + if result.abandoned: + reasons.append(f"{unit.unit_id}: abandoned after {result.attempt} attempts") + continue + if result.outcome is not UnitOutcome.SUCCEEDED: + reasons.append(f"{unit.unit_id}: outcome {result.outcome.value}") + continue + if result.infra_error_count > 0: + reasons.append( + f"{unit.unit_id}: {result.infra_error_count} instance(s) lost to " + "infrastructure" + ) + continue + + expected = set(unit.instance_ids) + accounted = set(result.accounted_instance_ids) + if len(result.accounted_instance_ids) != len(accounted): + reasons.append(f"{unit.unit_id}: duplicate instance ids in its own result") + continue + # Compare ids, never counts. A shard with one duplicate and one missing + # id has the right count and the wrong content. + if accounted != expected: + missing = sorted(expected - accounted) + extra = sorted(accounted - expected) + detail = [] + if missing: + detail.append(f"missing {', '.join(missing[:5])}") + if extra: + detail.append(f"unplanned {', '.join(extra[:5])}") + reasons.append(f"{unit.unit_id}: " + "; ".join(detail)) + continue + + for instance_id in result.accounted_instance_ids: + previous = seen_ids.get(instance_id) + if previous is not None: + reasons.append( + f"instance {instance_id} accounted for by both {previous} and " + f"{unit.unit_id}" + ) + continue + seen_ids[instance_id] = unit.unit_id + unplanned_resolved = set(result.resolved_instance_ids) - expected + if unplanned_resolved: + reasons.append( + f"{unit.unit_id}: resolved ids outside its shard: " + + ", ".join(sorted(unplanned_resolved)[:5]) + ) + continue + resolved.update(result.resolved_instance_ids) + + planned_ids = set(plan.instance_ids) + if not reasons and set(seen_ids) != planned_ids: + unaccounted = sorted(planned_ids - set(seen_ids)) + reasons.append( + f"{len(unaccounted)} planned instance(s) unaccounted for: " + + ", ".join(unaccounted[:5]) + ) + + if reasons: + raise MergeRefusal(run_id, reasons) + + return MergeResult( + run_id=run_id, + plan_digest=plan.digest, + total_instances=len(planned_ids), + resolved_instances=len(resolved), + unit_count=len(plan.units), + ) diff --git a/tests/unit/evaluation/swe_bench_distributed/test_merge.py b/tests/unit/evaluation/swe_bench_distributed/test_merge.py new file mode 100644 index 000000000..528ab9335 --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_merge.py @@ -0,0 +1,189 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The merge gate: all-or-nothing, id-based, scoped to one run.""" + +from __future__ import annotations + +import inspect + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed.merge import ( + MergeRefusal, + merge_run, + verify_inventory, +) +from inference_endpoint.evaluation.swe_bench_distributed.queue import ( + UnitOutcome, + UnitResult, + WorkQueue, +) +from inference_endpoint.evaluation.swe_bench_distributed.units import plan_units + +pytestmark = pytest.mark.unit + +IDS = [f"repo__proj-{i:02d}" for i in range(20)] + + +@pytest.fixture +def queue(tmp_path): + return WorkQueue(tmp_path / "wq", plan_units("run-a", IDS, shard_size=10)) + + +def publish(queue: WorkQueue, unit_id: str, **overrides) -> None: + unit = queue.plan.unit(unit_id) + payload = { + "unit_id": unit_id, + "run_id": unit.run_id, + "plan_digest": queue.plan.digest, + "outcome": UnitOutcome.SUCCEEDED, + "accounted_instance_ids": unit.instance_ids, + "resolved_instance_ids": unit.instance_ids[:3], + } + payload.update(overrides) + queue.publish(UnitResult(**payload)) + + +def publish_all(queue: WorkQueue) -> None: + for unit_id in queue.plan.unit_ids: + publish(queue, unit_id) + + +class TestHappyPath: + def test_full_accounting_scores(self, queue): + publish_all(queue) + result = merge_run(queue, "run-a") + assert result.total_instances == 20 + assert result.resolved_instances == 6 + assert result.resolved_rate == pytest.approx(0.3) + assert result.unit_count == 2 + + +class TestRefusals: + def test_a_missing_unit_refuses(self, queue): + publish(queue, "run-a.s00") + # 10 results must never be divided by 20. + with pytest.raises(MergeRefusal, match="have no result"): + merge_run(queue, "run-a") + + def test_an_abandoned_unit_refuses(self, queue): + publish(queue, "run-a.s00") + publish(queue, "run-a.s01", abandoned=True, attempt=3) + with pytest.raises(MergeRefusal, match="abandoned"): + merge_run(queue, "run-a") + + def test_a_non_success_outcome_refuses(self, queue): + publish(queue, "run-a.s00") + publish(queue, "run-a.s01", outcome=UnitOutcome.FAILED) + with pytest.raises(MergeRefusal, match="outcome failed"): + merge_run(queue, "run-a") + + def test_infrastructure_damage_refuses(self, queue): + publish(queue, "run-a.s00") + publish(queue, "run-a.s01", infra_error_count=2) + with pytest.raises(MergeRefusal, match="lost to infrastructure"): + merge_run(queue, "run-a") + + def test_a_missing_id_refuses_even_though_the_count_is_wrong_by_one(self, queue): + publish(queue, "run-a.s00") + unit = queue.plan.unit("run-a.s01") + publish(queue, "run-a.s01", accounted_instance_ids=unit.instance_ids[:-1]) + with pytest.raises(MergeRefusal, match="missing"): + merge_run(queue, "run-a") + + def test_a_swapped_id_refuses_although_the_count_matches(self, queue): + # The whole point of comparing ids rather than counts: this shard has + # exactly ten entries and the wrong content. + publish(queue, "run-a.s00") + unit = queue.plan.unit("run-a.s01") + swapped = unit.instance_ids[:-1] + ("some__other-99",) + publish(queue, "run-a.s01", accounted_instance_ids=swapped) + with pytest.raises(MergeRefusal, match="unplanned"): + merge_run(queue, "run-a") + + def test_a_duplicated_id_within_one_unit_refuses(self, queue): + publish(queue, "run-a.s00") + unit = queue.plan.unit("run-a.s01") + duped = unit.instance_ids[:-1] + (unit.instance_ids[0],) + publish(queue, "run-a.s01", accounted_instance_ids=duped) + with pytest.raises(MergeRefusal, match="duplicate"): + merge_run(queue, "run-a") + + def test_resolved_ids_outside_the_shard_refuse(self, queue): + publish(queue, "run-a.s00") + unit = queue.plan.unit("run-a.s01") + publish( + queue, + "run-a.s01", + resolved_instance_ids=(*unit.instance_ids[:2], IDS[0]), + ) + with pytest.raises(MergeRefusal, match="outside its shard"): + merge_run(queue, "run-a") + + def test_a_foreign_plan_digest_refuses(self, queue): + publish(queue, "run-a.s00") + publish(queue, "run-a.s01") + path = queue.results_dir / "run-a.s01.json" + path.write_text(path.read_text().replace(queue.plan.digest, "f" * 64)) + with pytest.raises(MergeRefusal, match="belongs to plan"): + merge_run(queue, "run-a") + + def test_a_result_outside_the_plan_refuses(self, queue): + publish_all(queue) + (queue.results_dir / "other-run.s00.json").write_text("{}") + with pytest.raises(MergeRefusal, match="outside the plan"): + merge_run(queue, "run-a") + + def test_an_unreadable_result_refuses(self, queue): + publish_all(queue) + (queue.results_dir / "run-a.s00.json").write_text("not json") + with pytest.raises(MergeRefusal, match="unreadable"): + merge_run(queue, "run-a") + + def test_every_reason_is_reported_at_once(self, queue): + publish(queue, "run-a.s00", infra_error_count=1) + with pytest.raises(MergeRefusal) as excinfo: + merge_run(queue, "run-a") + assert len(excinfo.value.reasons) >= 2 + + +class TestScoping: + def test_a_merge_is_always_scoped_to_one_run(self, queue): + publish_all(queue) + with pytest.raises(MergeRefusal, match="scoped to exactly one run"): + merge_run(queue, "some-other-run") + + def test_there_is_no_merge_all(self): + # "Merge everything that looks finished" once combined hundreds of + # banked results from unrelated configurations into one number. + signature = inspect.signature(merge_run) + assert "run_id" in signature.parameters + assert signature.parameters["run_id"].default is inspect.Parameter.empty + assert not hasattr( + __import__( + "inference_endpoint.evaluation.swe_bench_distributed.merge", + fromlist=["merge"], + ), + "merge_all", + ) + + +class TestInventory: + def test_a_complete_run_is_consistent(self, queue): + publish_all(queue) + assert verify_inventory(queue).consistent + + def test_an_ownerless_claim_is_an_inventory_error(self, queue): + publish_all(queue) + claim_dir = queue.claims_dir / "run-a.s00" + claim_dir.mkdir(parents=True) + # Checking `owner` files with one tool and claim directories with + # another is how a verification pass agrees with a broken system. + report = verify_inventory(queue) + assert report.ownerless_claims == ["run-a.s00"] + assert not report.consistent + + def test_claims_without_results_are_reported(self, queue): + queue.claim("run-a.s00") + assert verify_inventory(queue).claims_without_results == ["run-a.s00"] From 76319c392d5e5e742066fff00a0d3a63bee98689 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Sat, 8 Aug 2026 17:40:25 -0700 Subject: [PATCH 3/9] feat(swe-bench): conservative claim reaper + PID-only memory guard reaper.py releases a stale claim only when it has no result, its heartbeat is past stale_after, AND its owner is provably gone. Liveness is a pluggable protocol: LocalProcessLiveness pairs pid with boot id so a recycled pid on a rebooted host is not read as a live owner, and SlurmStepLiveness treats a step missing from scontrol inside a live job as dead, because the job-level rule alone deadlocks the queue forever. An indeterminate probe releases NOTHING - a false reap creates two owners, duplicate results and a wrong denominator. guards.py kills a runaway graded test only under a full conjunction (RSS over threshold AND cwd inside the testbed AND a container-supervisor ancestor). Kills are by PID and refuse self and any ancestor of self; there is no pattern-kill path in the module at all, and a test greps the source to keep it that way. Each term reports its evidence count, and HealthVerdict.combine returns INDETERMINATE rather than UNHEALTHY when a term has zero evidence, so a conjunctive guard cannot collapse into its weakest clause. --- .../swe_bench_distributed/__init__.py | 11 + .../swe_bench_distributed/guards.py | 314 ++++++++++++++++++ .../swe_bench_distributed/reaper.py | 242 ++++++++++++++ .../test_guards_and_reaper.py | 295 ++++++++++++++++ .../swe_bench_distributed/test_liveness.py | 163 +++++++++ 5 files changed, 1025 insertions(+) create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/guards.py create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/reaper.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_guards_and_reaper.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_liveness.py diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py index 3507289bb..73fd2cbab 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py @@ -11,6 +11,7 @@ exactly once. """ +from .guards import HealthTerm, HealthVerdict, MemoryGuard, combine_terms, kill_by_pid from .merge import MergeRefusal, MergeResult, merge_run, verify_inventory from .queue import ( ClaimError, @@ -18,18 +19,28 @@ UnitResult, WorkQueue, ) +from .reaper import LocalProcessLiveness, OwnerLiveness, SlurmStepLiveness, reap from .units import Unit, UnitPlan, plan_units __all__ = [ "ClaimError", + "HealthTerm", + "HealthVerdict", + "LocalProcessLiveness", + "MemoryGuard", "MergeRefusal", "MergeResult", + "OwnerLiveness", + "SlurmStepLiveness", "Unit", "UnitOutcome", "UnitPlan", "UnitResult", "WorkQueue", + "combine_terms", + "kill_by_pid", "merge_run", "plan_units", + "reap", "verify_inventory", ] diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/guards.py b/src/inference_endpoint/evaluation/swe_bench_distributed/guards.py new file mode 100644 index 000000000..92801fe28 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/guards.py @@ -0,0 +1,314 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Resource guards for graded SWE-bench evaluation. + +A graded test runs inside an evaluation container with no memory limit. A model +patch that makes a test allocate without bound will take the host down, and when +a whole client fleet shares one scheduler step, one host's OOM destroys every +peer's work along with it -- ``--kill-on-bad-exit=0`` does **not** prevent that, +because the scheduler escalates OOM separately from task exit codes. + +Killing such a process is correct, not a distortion. A patch that makes a graded +test allocate without bound is a failing patch, exactly as a patch that makes it +loop forever is; the alternative to killing was never "the test passes", it was +"the host dies and the instance still never completes". The kill is recorded as +a marker file and the classifier books it as a genuine failure. + +TWO RULES THAT ARE ENFORCED BY CONSTRUCTION HERE: + +1. **Kill by pid, never by pattern.** A pattern such as ``runtests.py`` can + appear in the guard's own command line, and a long-lived daemon can carry a + dead process's argv for days. This module contains no ``pkill``/``pgrep`` + path at all, and :func:`kill_by_pid` refuses self and its own ancestors. +2. **A conjunctive guard must not degenerate.** When one honest term of an + AND-guard permanently loses its data source, the conjunction collapses into + its remaining, weaker clauses and starts firing on healthy targets -- that is + how an idle watchdog killed a live bring-up. :func:`combine_terms` therefore + returns ``INDETERMINATE``, never ``UNHEALTHY``, if any term has no evidence. +""" + +from __future__ import annotations + +import json +import logging +import os +import signal +import time +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path + +logger = logging.getLogger(__name__) + +DEFAULT_KILL_BYTES = 150 * 1024**3 +DEFAULT_WARN_BYTES = 100 * 1024**3 +#: Ancestors that prove a process is inside a container supervisor. +CONTAINER_SUPERVISORS = ("conmon", "containerd-shim", "runc", "enroot", "crun") +_ANCESTOR_DEPTH = 6 + + +class HealthVerdict(StrEnum): + HEALTHY = "healthy" + UNHEALTHY = "unhealthy" + INDETERMINATE = "indeterminate" + + +@dataclass(slots=True) +class HealthTerm: + """One clause of a conjunctive guard, with its evidence count. + + ``evidence`` is the number of observations the term actually made. A term + that made none cannot vote, and must not be silently read as ``HEALTHY`` + (which would let the conjunction fire on the strength of the other clauses + alone) nor as ``UNHEALTHY``. + """ + + name: str + verdict: HealthVerdict + evidence: int + detail: str = "" + + +def combine_terms(terms: list[HealthTerm]) -> tuple[HealthVerdict, str]: + """AND the terms, refusing to act on an unevidenced conjunction.""" + if not terms: + return HealthVerdict.INDETERMINATE, "no terms" + blind = [term.name for term in terms if term.evidence <= 0] + if blind: + return ( + HealthVerdict.INDETERMINATE, + "no evidence for term(s): " + + ", ".join(blind) + + " -- a conjunction with a blind term cannot be trusted to be true", + ) + indeterminate = [ + term.name for term in terms if term.verdict is HealthVerdict.INDETERMINATE + ] + if indeterminate: + return ( + HealthVerdict.INDETERMINATE, + "indeterminate term(s): " + ", ".join(indeterminate), + ) + healthy = [term.name for term in terms if term.verdict is HealthVerdict.HEALTHY] + if healthy: + return HealthVerdict.HEALTHY, "healthy term(s): " + ", ".join(healthy) + return HealthVerdict.UNHEALTHY, "; ".join( + f"{term.name}: {term.detail}" for term in terms + ) + + +class SelfKillRefused(RuntimeError): + """Refused to signal this process or one of its ancestors.""" + + +def ancestors( + pid: int, *, depth: int = _ANCESTOR_DEPTH, proc: Path | None = None +) -> list[int]: + """Parent pids of ``pid``, nearest first.""" + root = proc if proc is not None else Path("/proc") + found: list[int] = [] + current = pid + for _ in range(depth): + try: + stat = (root / str(current) / "status").read_text() + except OSError: + break + parent = None + for line in stat.splitlines(): + if line.startswith("PPid:"): + try: + parent = int(line.split()[1]) + except (IndexError, ValueError): + parent = None + break + if parent is None or parent <= 0 or parent in found: + break + found.append(parent) + current = parent + return found + + +def kill_by_pid( + pid: int, *, sig: int = signal.SIGKILL, proc: Path | None = None +) -> bool: + """Signal exactly one pid. + + Refuses this process and any of its ancestors. There is deliberately no + pattern-matching variant of this function: matching by command line is how a + guard kills itself, or kills whatever inherited a stale argv. + """ + if pid <= 0: + raise SelfKillRefused(f"refusing to signal pid {pid}") + if pid == os.getpid(): + raise SelfKillRefused("refusing to signal self") + if pid in ancestors(os.getpid(), proc=proc): + raise SelfKillRefused(f"refusing to signal ancestor pid {pid}") + try: + os.kill(pid, sig) + except ProcessLookupError: + return False + except OSError: + logger.warning("could not signal pid %d", pid, exc_info=True) + return False + return True + + +@dataclass(slots=True) +class ProcessSample: + pid: int + rss_bytes: int + #: Name of the container this process belongs to, if it could be resolved. + #: This is what determines the phase, so an unresolvable name is not "not a + #: test" -- see :meth:`MemoryGuard.phase_for`. + container_name: str | None = None + ancestor_names: tuple[str, ...] = () + #: Advisory only. Deliberately NOT a predicate: see MemoryGuard's docstring. + cwd: str = "" + + +@dataclass(slots=True) +class GuardAction: + pid: int + rss_bytes: int + verdict: HealthVerdict + reason: str + killed: bool = False + terms: list[HealthTerm] = field(default_factory=list) + + +class MemoryGuard: + """Kill a runaway graded test, and only a runaway graded test. + + A process is a candidate only when **both** terms hold: + + * resident memory at or above ``kill_bytes`` (default 150 GiB; a healthy + graded test uses single-digit GiB, so the headroom is roughly thirty-fold) + * it has a container-supervisor ancestor -- it is inside a container + + THERE IS DELIBERATELY NO WORKING-DIRECTORY TERM. An earlier version required + the process's cwd to be inside the testbed, on the reasoning that a graded + test runs there. It does not always: a runaway that had grown to 667 GiB was + skipped for 105 minutes because its cwd was ``/tmp``. Every additional + conjunct is another way for the guard to miss what it exists to catch, so + the predicate set is the smallest one that cannot match a benchmark client, + an engine, a login shell or the guard itself -- all of which fail the + container term. ``cwd`` is still sampled, as advisory detail only. + """ + + def __init__( + self, + *, + kill_bytes: int = DEFAULT_KILL_BYTES, + warn_bytes: int = DEFAULT_WARN_BYTES, + killed_dir: Path | None = None, + supervisors: tuple[str, ...] = CONTAINER_SUPERVISORS, + eval_container_prefixes: tuple[str, ...] = ("sweb.eval",), + agent_container_prefixes: tuple[str, ...] = ("minisweagent",), + ) -> None: + self.kill_bytes = kill_bytes + self.warn_bytes = warn_bytes + self.killed_dir = killed_dir + self.supervisors = supervisors + self.eval_container_prefixes = eval_container_prefixes + self.agent_container_prefixes = agent_container_prefixes + + def phase_for(self, sample: ProcessSample) -> str: + """Which phase a runaway belongs to, from its container name. + + Fails closed to ``"unknown"``. An unresolvable container name must not + stop the kill -- the process is still a confirmed runaway inside a + container -- but it must also not be booked as an eval kill, because + only an eval kill turns an instance's error into a genuine failure. + """ + name = sample.container_name or "" + if any(name.startswith(prefix) for prefix in self.eval_container_prefixes): + return "eval" + if any(name.startswith(prefix) for prefix in self.agent_container_prefixes): + return "agent" + return "unknown" + + def evaluate(self, sample: ProcessSample) -> GuardAction: + terms = [ + HealthTerm( + name="rss", + verdict=( + HealthVerdict.UNHEALTHY + if sample.rss_bytes >= self.kill_bytes + else HealthVerdict.HEALTHY + ), + evidence=1 if sample.rss_bytes >= 0 else 0, + detail=f"{sample.rss_bytes / 1024**3:.1f} GiB", + ), + HealthTerm( + name="in_container", + verdict=( + HealthVerdict.UNHEALTHY + if any(name in self.supervisors for name in sample.ancestor_names) + else HealthVerdict.HEALTHY + ), + evidence=len(sample.ancestor_names), + detail=f"ancestors={list(sample.ancestor_names)}", + ), + ] + verdict, reason = combine_terms(terms) + return GuardAction( + pid=sample.pid, + rss_bytes=sample.rss_bytes, + verdict=verdict, + reason=reason, + terms=terms, + ) + + def act( + self, + sample: ProcessSample, + *, + instance_id: str | None = None, + phase: str | None = None, + apply: bool = False, + ) -> GuardAction: + """Evaluate and, when ``apply``, kill by pid and record a marker. + + The marker is written *before* the kill: a SIGKILLed test leaves an + ambiguous log, so the record of having killed it is the only reliable + evidence, and it has to exist even if the process dies first. + """ + action = self.evaluate(sample) + if action.verdict is not HealthVerdict.UNHEALTHY or not apply: + return action + resolved_phase = phase if phase is not None else self.phase_for(sample) + if self.killed_dir is not None and instance_id: + self.record_kill(instance_id, sample, phase=resolved_phase) + action.killed = kill_by_pid(sample.pid) + return action + + def record_kill( + self, instance_id: str, sample: ProcessSample, *, phase: str = "eval" + ) -> Path: + """Write the ``....json`` marker. + + Phase is load-bearing: only ``eval`` markers make an instance's error a + genuine failure. An ``agent`` kill merely makes one tool call return an + error observation and the agent carries on, so it must never influence + classification. + """ + import socket + + assert self.killed_dir is not None + self.killed_dir.mkdir(parents=True, exist_ok=True) + host = socket.gethostname() + path = self.killed_dir / f"{phase}.{instance_id}.{host}.{sample.pid}.json" + path.write_text( + json.dumps( + { + "phase": phase, + "instance_id": instance_id, + "host": host, + "pid": sample.pid, + "rss_bytes": sample.rss_bytes, + "killed_at": time.time(), + } + ) + ) + return path diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/reaper.py b/src/inference_endpoint/evaluation/swe_bench_distributed/reaper.py new file mode 100644 index 000000000..0905cee42 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/reaper.py @@ -0,0 +1,242 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Return orphaned claims to the queue. Nothing else. + +A false reap is the worst thing this system can do. Releasing a claim whose +owner is still running puts the unit back in the queue while it is executing, a +second worker takes it, both write results, and the run has duplicate work, a +wrong denominator, and no error anywhere -- the exact silent corruption the +atomic claim exists to prevent, reintroduced by the janitor. + +Therefore the reaper is conservative in one specific direction: **uncertainty +never escalates.** If liveness cannot be determined, nothing is released. +""" + +from __future__ import annotations + +import logging +import os +import subprocess +from dataclasses import dataclass, field +from enum import StrEnum +from typing import Protocol + +from .queue import OwnerRecord, WorkQueue + +logger = logging.getLogger(__name__) + +_PROBE_TIMEOUT_S = 60 + + +class Liveness(StrEnum): + ALIVE = "alive" + DEAD = "dead" + #: Could not tell. Treated as ALIVE for the purpose of reaping. + INDETERMINATE = "indeterminate" + + +@dataclass(frozen=True, slots=True) +class LivenessVerdict: + state: Liveness + #: Which layer decided. ``"step"`` gets a shorter staleness threshold: a + #: step that died inside a live job took its tasks with it immediately, so + #: there is no reason to wait an hour to believe it. + scope: str = "process" + detail: str = "" + + +class OwnerLiveness(Protocol): + """Decides whether the process that claimed a unit still exists.""" + + def probe(self, owner: OwnerRecord) -> LivenessVerdict: ... + + +class LocalProcessLiveness: + """Liveness by pid, scoped to one host and one boot. + + A pid on its own is not evidence: after a reboot the same number can belong + to something unrelated, so an owner from a different boot of this host is + dead, and an owner from a different host is indeterminate (we cannot see it). + """ + + def __init__(self, *, host: str | None = None, boot: str | None = None) -> None: + import socket + + from .queue import boot_id + + self.host = host if host is not None else socket.gethostname() + self.boot = boot if boot is not None else boot_id() + + def probe(self, owner: OwnerRecord) -> LivenessVerdict: + if owner.host != self.host: + return LivenessVerdict( + Liveness.INDETERMINATE, "process", f"owner is on {owner.host}" + ) + if owner.boot_id and owner.boot_id != self.boot: + return LivenessVerdict( + Liveness.DEAD, "process", "host rebooted since claim" + ) + if owner.pid <= 0: + return LivenessVerdict(Liveness.INDETERMINATE, "process", "no pid recorded") + try: + os.kill(owner.pid, 0) + except ProcessLookupError: + return LivenessVerdict(Liveness.DEAD, "process", "pid gone") + except PermissionError: + # Exists, owned by someone else. + return LivenessVerdict(Liveness.ALIVE, "process", "pid exists") + except OSError: + return LivenessVerdict(Liveness.INDETERMINATE, "process", "kill(0) failed") + return LivenessVerdict(Liveness.ALIVE, "process", "pid exists") + + +class SlurmStepLiveness: + """Liveness by SLURM job *and step*. + + An owner is dead when its job is absent from ``squeue``, **or** when the job + is alive but its step is gone. The second clause is not optional: a step can + die inside a live job (a killed srun, an OOM-terminated step) and SLURM + kills that step's tasks, but the job never leaves ``squeue``, so a + job-level-only rule blocks those units for the entire life of the + allocation. + + Step liveness comes from ``scontrol show step``, never ``squeue -s``: on the + clusters this was built for ``squeue -s`` reports only ``.extern`` and never + the worker step, so using it would mark every live step dead and falsely + reap every claim. + + Every failure to read SLURM yields ``INDETERMINATE``. An unavailable + ``squeue`` must never be read as "no jobs are running". + """ + + def __init__(self, *, timeout_s: int = _PROBE_TIMEOUT_S) -> None: + self.timeout_s = timeout_s + + def _run(self, argv: list[str]) -> str | None: + try: + completed = subprocess.run( + argv, capture_output=True, text=True, timeout=self.timeout_s + ) + except (OSError, subprocess.SubprocessError): + logger.warning("reaper: %s unavailable; releasing nothing", argv[0]) + return None + if completed.returncode != 0: + return None + return completed.stdout + + def live_job_ids(self) -> set[str] | None: + out = self._run(["squeue", "-h", "-o", "%i"]) + if out is None: + return None + ids: set[str] = set() + for token in out.split(): + token = token.strip() + if not token: + continue + ids.add(token) + ids.add(token.split("_")[0].split(".")[0]) + if not ids and os.environ.get("SLURM_JOB_ID"): + # An empty queue is legitimate in general, but not while we are + # ourselves inside a job. That is what a broken squeue looks like. + logger.warning( + "reaper: squeue returned empty while inside a job; releasing nothing" + ) + return None + return ids + + def live_step_ids(self, job_id: str) -> set[str] | None: + out = self._run(["scontrol", "show", "step", str(job_id)]) + if out is None: + return None + steps = { + token.split("=", 1)[1].split(".", 1)[1] + for token in out.split() + if token.startswith("StepId=") and "." in token.split("=", 1)[1] + } + # A successful scontrol listing no step at all is implausible while the + # job exists (there is always .extern): indeterminate, not empty. + return steps or None + + def probe(self, owner: OwnerRecord) -> LivenessVerdict: + if not owner.slurm_job_id: + return LivenessVerdict(Liveness.INDETERMINATE, "job", "no job id recorded") + jobs = self.live_job_ids() + if jobs is None: + return LivenessVerdict(Liveness.INDETERMINATE, "job", "squeue unreadable") + if owner.slurm_job_id not in jobs: + return LivenessVerdict(Liveness.DEAD, "job", "job absent from squeue") + if not owner.slurm_step_id: + return LivenessVerdict( + Liveness.ALIVE, "job", "job present, no step recorded" + ) + steps = self.live_step_ids(owner.slurm_job_id) + if steps is None: + # Indeterminate step liveness must not become MORE aggressive than + # the job-level answer, which is "alive". + return LivenessVerdict(Liveness.ALIVE, "job", "step list unreadable") + if owner.slurm_step_id in steps: + return LivenessVerdict(Liveness.ALIVE, "step", "step present") + return LivenessVerdict(Liveness.DEAD, "step", "step gone inside a live job") + + +@dataclass(slots=True) +class ReapReport: + released: list[str] = field(default_factory=list) + kept: dict[str, str] = field(default_factory=dict) + dry_run: bool = True + + def __bool__(self) -> bool: # pragma: no cover - convenience + return bool(self.released) + + +def reap( + queue: WorkQueue, + liveness: OwnerLiveness, + *, + stale_after_s: float = 3600.0, + step_stale_after_s: float = 900.0, + apply: bool = False, + now: float | None = None, +) -> ReapReport: + """Release claims whose owner is provably gone and which produced no result. + + All three conditions must hold: no result, a stale-enough heartbeat, and a + ``DEAD`` liveness verdict. A verdict scoped to ``"step"`` uses the shorter + ``step_stale_after_s``: when a step dies inside a job that stays in the + queue, the job-level rule alone never fires and those units stay blocked for + the entire life of the allocation. + """ + report = ReapReport(dry_run=not apply) + completed = queue.completed_unit_ids() + for unit_id in sorted(queue.claimed_unit_ids()): + if unit_id in completed: + # Claims for completed units are harmless bookkeeping. + report.kept[unit_id] = "has result" + continue + age = queue.heartbeat_age(unit_id, now=now) + if age is None: + report.kept[unit_id] = "no heartbeat to age" + continue + owner = queue.owner(unit_id) + if owner is None: + # We cannot prove anything about an unreadable owner, so age alone + # decides. A claim holding anything but pure bookkeeping is not ours + # to reason about at all. + if not queue.is_pure_bookkeeping(unit_id): + report.kept[unit_id] = "claim holds non-bookkeeping contents" + continue + verdict = LivenessVerdict(Liveness.DEAD, "process", "owner unreadable") + else: + verdict = liveness.probe(owner) + threshold = step_stale_after_s if verdict.scope == "step" else stale_after_s + if age < threshold: + report.kept[unit_id] = f"heartbeat {age:.0f}s < {threshold:.0f}s" + continue + if verdict.state is not Liveness.DEAD: + report.kept[unit_id] = f"owner {verdict.state.value}: {verdict.detail}" + continue + report.released.append(unit_id) + if apply: + queue.release(unit_id) + return report diff --git a/tests/unit/evaluation/swe_bench_distributed/test_guards_and_reaper.py b/tests/unit/evaluation/swe_bench_distributed/test_guards_and_reaper.py new file mode 100644 index 000000000..aebdec290 --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_guards_and_reaper.py @@ -0,0 +1,295 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Resource guards and the claim reaper.""" + +from __future__ import annotations + +import os +import time + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed import guards as guards_mod +from inference_endpoint.evaluation.swe_bench_distributed.guards import ( + DEFAULT_KILL_BYTES, + HealthTerm, + HealthVerdict, + MemoryGuard, + ProcessSample, + SelfKillRefused, + combine_terms, + kill_by_pid, +) +from inference_endpoint.evaluation.swe_bench_distributed.queue import ( + UnitOutcome, + UnitResult, + WorkQueue, +) +from inference_endpoint.evaluation.swe_bench_distributed.reaper import ( + Liveness, + LivenessVerdict, + reap, +) +from inference_endpoint.evaluation.swe_bench_distributed.units import plan_units + +pytestmark = pytest.mark.unit + +GIB = 1024**3 + + +def executable_source(module) -> str: + """Module source with comments and string literals removed.""" + import tokenize + + kept = [] + with open(module.__file__, "rb") as handle: + for token in tokenize.tokenize(handle.readline): + if token.type in {tokenize.COMMENT, tokenize.STRING}: + continue + kept.append(token.string) + return " ".join(kept) + + +class FakeLiveness: + def __init__(self, verdict: LivenessVerdict) -> None: + self.verdict = verdict + + def probe(self, owner): + return self.verdict + + +@pytest.fixture +def queue(tmp_path): + plan = plan_units("run-a", [f"i-{i}" for i in range(20)], shard_size=10) + return WorkQueue(tmp_path / "wq", plan) + + +class TestConjunction: + def test_all_unhealthy_terms_fire(self): + terms = [ + HealthTerm("a", HealthVerdict.UNHEALTHY, evidence=1), + HealthTerm("b", HealthVerdict.UNHEALTHY, evidence=1), + ] + assert combine_terms(terms)[0] is HealthVerdict.UNHEALTHY + + def test_a_blind_term_makes_the_conjunction_indeterminate(self): + # When an honest term permanently loses its data source, an AND-guard + # collapses into its remaining, weaker clauses and starts firing on + # healthy targets. That is how an idle watchdog killed a live bring-up. + terms = [ + HealthTerm("loud", HealthVerdict.UNHEALTHY, evidence=1), + HealthTerm("blind", HealthVerdict.UNHEALTHY, evidence=0), + ] + verdict, reason = combine_terms(terms) + assert verdict is HealthVerdict.INDETERMINATE + assert "blind" in reason + + def test_a_blind_term_never_yields_unhealthy(self): + terms = [HealthTerm("blind", HealthVerdict.UNHEALTHY, evidence=0)] + assert combine_terms(terms)[0] is not HealthVerdict.UNHEALTHY + + def test_one_healthy_term_spares_the_target(self): + terms = [ + HealthTerm("a", HealthVerdict.UNHEALTHY, evidence=1), + HealthTerm("b", HealthVerdict.HEALTHY, evidence=1), + ] + assert combine_terms(terms)[0] is HealthVerdict.HEALTHY + + def test_no_terms_is_indeterminate(self): + assert combine_terms([])[0] is HealthVerdict.INDETERMINATE + + +class TestKillDiscipline: + def test_there_is_no_pattern_kill_path_in_the_module(self): + # A pattern such as "runtests.py" can appear in the guard's own command + # line, and a long-lived daemon can carry a dead process's argv for days. + # Executable code is inspected with comments and strings removed, so the + # docstring explaining the rule cannot satisfy the test for it. + code = executable_source(guards_mod) + assert "pkill" not in code + assert "pgrep" not in code + + def test_the_guard_never_shells_out(self): + # There is no command line to match against in the first place. + code = executable_source(guards_mod) + assert "subprocess" not in code + assert "os.system" not in code + + def test_killing_self_is_refused(self): + with pytest.raises(SelfKillRefused, match="self"): + kill_by_pid(os.getpid()) + + def test_killing_an_ancestor_is_refused(self): + with pytest.raises(SelfKillRefused, match="ancestor"): + kill_by_pid(os.getppid()) + + def test_a_nonsense_pid_is_refused(self): + with pytest.raises(SelfKillRefused): + kill_by_pid(0) + + +class TestMemoryGuard: + def runaway(self, **overrides): + payload = { + "pid": 4242, + "rss_bytes": 200 * GIB, + "container_name": "sweb.eval.arm64.repo__proj-1", + "ancestor_names": ("bash", "conmon"), + } + payload.update(overrides) + return ProcessSample(**payload) + + def test_a_runaway_graded_test_is_unhealthy(self): + action = MemoryGuard().evaluate(self.runaway()) + assert action.verdict is HealthVerdict.UNHEALTHY + + def test_a_runaway_outside_the_testbed_is_still_caught(self): + # An earlier version required cwd inside /testbed. A runaway that had + # grown to 667 GiB was skipped for 105 minutes because its cwd was /tmp, + # so cwd is advisory detail and never a predicate. + action = MemoryGuard().evaluate(self.runaway(cwd="/tmp")) + assert action.verdict is HealthVerdict.UNHEALTHY + assert {term.name for term in action.terms} == {"rss", "in_container"} + + def test_a_large_process_outside_a_container_is_spared(self): + action = MemoryGuard().evaluate(self.runaway(ancestor_names=("bash", "sshd"))) + assert action.verdict is HealthVerdict.HEALTHY + + def test_a_normal_test_is_spared(self): + action = MemoryGuard().evaluate(self.runaway(rss_bytes=3 * GIB)) + assert action.verdict is HealthVerdict.HEALTHY + + def test_unreadable_ancestry_is_indeterminate_not_a_kill(self): + action = MemoryGuard().evaluate(self.runaway(ancestor_names=())) + assert action.verdict is HealthVerdict.INDETERMINATE + + def test_the_default_threshold_leaves_wide_headroom(self): + assert DEFAULT_KILL_BYTES >= 100 * GIB + + @pytest.mark.parametrize( + ("container_name", "phase"), + [ + ("sweb.eval.arm64.repo__proj-1", "eval"), + ("minisweagent-abc123", "agent"), + ("something-else", "unknown"), + (None, "unknown"), + ], + ) + def test_phase_comes_from_the_container_name_and_fails_closed( + self, container_name, phase + ): + # Only an eval kill turns an instance's error into a genuine failure, so + # an unresolvable name must not be booked as one. + guard = MemoryGuard() + assert guard.phase_for(self.runaway(container_name=container_name)) == phase + + def test_the_marker_is_written_before_the_kill(self, tmp_path, monkeypatch): + killed_dir = tmp_path / "killed" + order: list[str] = [] + monkeypatch.setattr( + guards_mod, + "kill_by_pid", + lambda pid, **kwargs: order.append("kill") or True, + ) + guard = MemoryGuard(killed_dir=killed_dir) + original = guard.record_kill + + def traced(*args, **kwargs): + order.append("marker") + return original(*args, **kwargs) + + monkeypatch.setattr(guard, "record_kill", traced) + guard.act(self.runaway(), instance_id="repo__proj-1", apply=True) + + # A SIGKILLed test leaves an ambiguous log, so the record of having + # killed it must survive even if the process dies first. + assert order == ["marker", "kill"] + assert list(killed_dir.glob("eval.repo__proj-1.*.json")) + + def test_dry_evaluation_does_not_kill(self, tmp_path, monkeypatch): + monkeypatch.setattr( + guards_mod, "kill_by_pid", lambda *a, **k: pytest.fail("killed") + ) + action = MemoryGuard(killed_dir=tmp_path).act( + self.runaway(), instance_id="x", apply=False + ) + assert not action.killed + + +class TestReaper: + def stale_claim(self, queue, unit_id="run-a.s00", age=7200.0): + queue.claim(unit_id) + heartbeat = queue.claims_dir / unit_id / "hb" + past = time.time() - age + os.utime(heartbeat, (past, past)) + + def test_a_dead_owner_with_no_result_is_released(self, queue): + self.stale_claim(queue) + report = reap(queue, FakeLiveness(LivenessVerdict(Liveness.DEAD)), apply=True) + assert report.released == ["run-a.s00"] + assert "run-a.s00" in queue.available_unit_ids() + + def test_a_live_owner_is_never_released(self, queue): + self.stale_claim(queue) + report = reap(queue, FakeLiveness(LivenessVerdict(Liveness.ALIVE)), apply=True) + # A false reap gives one unit two owners, duplicate results and a wrong + # denominator, with no error anywhere. + assert report.released == [] + + def test_an_indeterminate_probe_releases_nothing(self, queue): + self.stale_claim(queue) + report = reap( + queue, + FakeLiveness(LivenessVerdict(Liveness.INDETERMINATE)), + apply=True, + ) + assert report.released == [] + assert "indeterminate" in report.kept["run-a.s00"] + + def test_a_fresh_heartbeat_is_never_released(self, queue): + queue.claim("run-a.s00") + report = reap(queue, FakeLiveness(LivenessVerdict(Liveness.DEAD)), apply=True) + assert report.released == [] + + def test_a_claim_with_a_result_is_never_released(self, queue): + self.stale_claim(queue) + unit = queue.plan.unit("run-a.s00") + queue.results_dir.joinpath("run-a.s00.json").write_text( + UnitResult( + unit_id="run-a.s00", + run_id="run-a", + plan_digest=queue.plan.digest, + outcome=UnitOutcome.SUCCEEDED, + accounted_instance_ids=unit.instance_ids, + ).to_dict() + and '{"unit_id": "run-a.s00"}' + ) + report = reap(queue, FakeLiveness(LivenessVerdict(Liveness.DEAD)), apply=True) + assert report.released == [] + + def test_a_dead_step_uses_the_shorter_threshold(self, queue): + # A step that dies inside a live job takes its tasks with it at once, so + # waiting an hour would block those units for the whole allocation. + self.stale_claim(queue, age=1200.0) + report = reap( + queue, + FakeLiveness(LivenessVerdict(Liveness.DEAD, scope="step")), + stale_after_s=3600.0, + step_stale_after_s=900.0, + apply=True, + ) + assert report.released == ["run-a.s00"] + + def test_dry_run_reports_without_releasing(self, queue): + self.stale_claim(queue) + report = reap(queue, FakeLiveness(LivenessVerdict(Liveness.DEAD))) + assert report.released == ["run-a.s00"] + assert queue.claimed_unit_ids() == {"run-a.s00"} + + def test_a_claim_with_unexpected_contents_is_left_alone(self, queue): + self.stale_claim(queue) + (queue.claims_dir / "run-a.s00" / "surprise").write_text("x") + (queue.claims_dir / "run-a.s00" / "owner").unlink() + report = reap(queue, FakeLiveness(LivenessVerdict(Liveness.DEAD)), apply=True) + assert report.released == [] diff --git a/tests/unit/evaluation/swe_bench_distributed/test_liveness.py b/tests/unit/evaluation/swe_bench_distributed/test_liveness.py new file mode 100644 index 000000000..e65b32b9b --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_liveness.py @@ -0,0 +1,163 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Owner-liveness probes. Uncertainty must never escalate to DEAD.""" + +from __future__ import annotations + +import os +import socket +import subprocess + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed.queue import OwnerRecord +from inference_endpoint.evaluation.swe_bench_distributed.reaper import ( + Liveness, + LocalProcessLiveness, + SlurmStepLiveness, +) + +pytestmark = pytest.mark.unit + + +def owner(**overrides) -> OwnerRecord: + payload = { + "unit_id": "run-a.s00", + "host": socket.gethostname(), + "pid": os.getpid(), + "boot_id": "boot-1", + "plan_digest": "d" * 64, + "claimed_at": 0.0, + } + payload.update(overrides) + return OwnerRecord(**payload) + + +class TestLocalProcessLiveness: + def test_a_live_pid_on_this_boot_is_alive(self): + probe = LocalProcessLiveness(boot="boot-1") + assert probe.probe(owner()).state is Liveness.ALIVE + + def test_a_missing_pid_is_dead(self): + probe = LocalProcessLiveness(boot="boot-1") + # 2**22 is above the default pid_max on Linux, so it cannot exist. + assert probe.probe(owner(pid=2**22)).state is Liveness.DEAD + + def test_a_different_boot_is_dead(self): + # After a reboot the same pid number can belong to something unrelated, + # so a live-looking pid is not evidence that the owner survived. + probe = LocalProcessLiveness(boot="boot-2") + assert probe.probe(owner()).state is Liveness.DEAD + + def test_another_host_is_indeterminate_not_dead(self): + probe = LocalProcessLiveness(boot="boot-1") + assert probe.probe(owner(host="elsewhere")).state is Liveness.INDETERMINATE + + def test_a_missing_pid_record_is_indeterminate(self): + probe = LocalProcessLiveness(boot="boot-1") + assert probe.probe(owner(pid=0)).state is Liveness.INDETERMINATE + + +class FakeSlurm(SlurmStepLiveness): + def __init__(self, responses): + super().__init__() + self.responses = responses + self.calls: list[list[str]] = [] + + def _run(self, argv): + self.calls.append(argv) + response = self.responses.get(argv[0]) + if isinstance(response, Exception): + raise response + return response + + +class TestSlurmStepLiveness: + def slurm_owner(self, **overrides): + return owner(slurm_job_id="1000", slurm_step_id="3", **overrides) + + def test_a_job_absent_from_squeue_is_dead(self): + probe = FakeSlurm({"squeue": "2000\n"}) + verdict = probe.probe(self.slurm_owner()) + assert verdict.state is Liveness.DEAD + assert verdict.scope == "job" + + def test_a_live_job_and_step_is_alive(self): + probe = FakeSlurm( + { + "squeue": "1000\n", + "scontrol": "StepId=1000.3 State=RUNNING StepId=1000.extern", + } + ) + assert probe.probe(self.slurm_owner()).state is Liveness.ALIVE + + def test_a_dead_step_inside_a_live_job_is_dead(self): + # A step can die inside a live job -- a killed srun, an OOM-terminated + # step -- and the job never leaves the queue, so a job-level-only rule + # blocks those units for the entire allocation. + probe = FakeSlurm( + {"squeue": "1000\n", "scontrol": "StepId=1000.extern State=RUNNING"} + ) + verdict = probe.probe(self.slurm_owner()) + assert verdict.state is Liveness.DEAD + assert verdict.scope == "step" + + def test_step_liveness_uses_scontrol_not_squeue_s(self): + probe = FakeSlurm( + {"squeue": "1000\n", "scontrol": "StepId=1000.3 State=RUNNING"} + ) + probe.probe(self.slurm_owner()) + # `squeue -s` reports only `.extern` on the clusters this targets, so it + # would mark every live step dead and falsely reap every claim. + assert ["scontrol", "show", "step", "1000"] in probe.calls + assert not any("-s" in argv for argv in probe.calls if argv[0] == "squeue") + + def test_an_unreadable_squeue_is_indeterminate(self): + probe = FakeSlurm({"squeue": None}) + assert probe.probe(self.slurm_owner()).state is Liveness.INDETERMINATE + + def test_an_unreadable_step_list_falls_back_to_the_job_answer(self): + # Indeterminate step liveness must never be more aggressive than the + # job-level answer, which is "alive". + probe = FakeSlurm({"squeue": "1000\n", "scontrol": None}) + assert probe.probe(self.slurm_owner()).state is Liveness.ALIVE + + def test_an_empty_scontrol_listing_is_treated_as_unreadable(self): + probe = FakeSlurm({"squeue": "1000\n", "scontrol": "no steps here"}) + assert probe.probe(self.slurm_owner()).state is Liveness.ALIVE + + def test_an_owner_without_a_job_id_is_indeterminate(self): + probe = FakeSlurm({"squeue": "1000\n"}) + assert probe.probe(owner()).state is Liveness.INDETERMINATE + + def test_an_empty_queue_inside_a_job_is_implausible(self, monkeypatch): + # An empty successful squeue is what a broken squeue looks like. It must + # never be read as "no jobs are running" while we are inside a job. + monkeypatch.setenv("SLURM_JOB_ID", "1000") + probe = FakeSlurm({"squeue": ""}) + assert probe.live_job_ids() is None + + def test_an_empty_queue_outside_a_job_is_trusted(self, monkeypatch): + monkeypatch.delenv("SLURM_JOB_ID", raising=False) + probe = FakeSlurm({"squeue": ""}) + assert probe.live_job_ids() == set() + + def test_array_job_ids_are_matched_by_base_id(self): + probe = FakeSlurm( + {"squeue": "1000_4\n", "scontrol": "StepId=1000.3 State=RUNNING"} + ) + assert probe.probe(self.slurm_owner()).state is Liveness.ALIVE + + def test_a_failing_command_is_indeterminate_not_dead(self): + probe = SlurmStepLiveness(timeout_s=1) + + def boom(argv, **kwargs): + raise subprocess.SubprocessError("no slurm here") + + original = subprocess.run + subprocess.run = boom + try: + assert probe.live_job_ids() is None + finally: + subprocess.run = original From b56729a07cfba5692e074d261bc14d517135d6f4 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Sat, 8 Aug 2026 17:40:57 -0700 Subject: [PATCH 4/9] feat(swe-bench): eval-phase infra-vs-genuine error classifier The Pyxis sentinel only covers the agent phase; eval-phase error_ids were counted as real outcomes and never retried, which is what produced 24 of 25 permanently-bad runs on the source cluster. classify.py reads the SWE-bench report's error_ids and each instance's run_instance.log and classifies them through an ORDERED rule list, first match wins. The order is load-bearing: BuildImageError is checked before everything because its message embeds the other rules' needles, CONMON_EAGAIN and TEST_TIMEOUT precede WEDGE_EVAL, and PATCH_APPLY_FAILED is last. Anything unclassifiable is UNKNOWN and UNKNOWN is GENUINE, asserted by a membership test: a false bad-run costs one redo, a false retry biases the measurement toward optimism. Memory-kill markers are consumed by phase - an eval-phase kill is a genuine failure (an unbounded allocation is a failing patch), an agent-phase kill is recorded for audit only, since the agent merely gets an error observation and the instance still reaches a real outcome. --- .../swe_bench_distributed/__init__.py | 14 ++ .../swe_bench_distributed/classify.py | 234 ++++++++++++++++++ .../swe_bench_distributed/test_classify.py | 171 +++++++++++++ 3 files changed, 419 insertions(+) create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/classify.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_classify.py diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py index 73fd2cbab..c2711f071 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py @@ -11,6 +11,14 @@ exactly once. """ +from .classify import ( + GENUINE_KINDS, + INFRA_KINDS, + ErrorKind, + UnitClassification, + classify_eval_log, + classify_unit, +) from .guards import HealthTerm, HealthVerdict, MemoryGuard, combine_terms, kill_by_pid from .merge import MergeRefusal, MergeResult, merge_run, verify_inventory from .queue import ( @@ -23,7 +31,10 @@ from .units import Unit, UnitPlan, plan_units __all__ = [ + "GENUINE_KINDS", + "INFRA_KINDS", "ClaimError", + "ErrorKind", "HealthTerm", "HealthVerdict", "LocalProcessLiveness", @@ -33,10 +44,13 @@ "OwnerLiveness", "SlurmStepLiveness", "Unit", + "UnitClassification", "UnitOutcome", "UnitPlan", "UnitResult", "WorkQueue", + "classify_eval_log", + "classify_unit", "combine_terms", "kill_by_pid", "merge_run", diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/classify.py b/src/inference_endpoint/evaluation/swe_bench_distributed/classify.py new file mode 100644 index 000000000..b7c06ac25 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/classify.py @@ -0,0 +1,234 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Split a unit's error instances into infrastructure damage and genuine failures. + +A SWE-bench run reports three per-instance outcomes: resolved, unresolved, and +*error*. The agent phase has an infrastructure sentinel (the Pyxis +``infrastructure_failure_path``), but the eval phase has none: an instance whose +evaluation container wedged is booked as ``error``, which counts as "accounted +for", so the unit is published successful and is never retried. Those instances +silently poison a run that can then never reach a full result. + +Classification exists to catch exactly that. It reads each error instance's +``run_instance.log`` and assigns one kind. + +BIAS RULE -- this is the whole design and it is deliberately asymmetric. If an +error cannot be classified confidently it is treated as GENUINE, never as +infrastructure. A false bad-run costs one redo; a false retry silently biases +the measurement toward optimism, and an optimistic accuracy number is worse than +no number. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass, field +from enum import StrEnum +from pathlib import Path + +logger = logging.getLogger(__name__) + + +class ErrorKind(StrEnum): + """One classification of an error instance.""" + + # Infrastructure: a defect in the runtime we provided, safe to retry. + CONTAINER_EXEC_REFUSED = "container_exec_refused" + CONTAINER_FORK_EAGAIN = "container_fork_eagain" + RUNTIME_READ_TIMEOUT = "runtime_read_timeout" + IMAGE_BUILD_TIMEOUT = "image_build_timeout" + IMAGE_BUILD_ERROR = "image_build_error" + STEP_INFRASTRUCTURE_FAILURE = "step_infrastructure_failure" + ENDPOINT_CHANGED = "endpoint_changed" + + # Genuine: a real outcome of the model's patch, or unreadable. Never retried. + TEST_TIMEOUT = "test_timeout" + TEST_MEMORY_EXCEEDED = "test_memory_exceeded" + PATCH_APPLY_FAILED = "patch_apply_failed" + UNKNOWN = "unknown" + + +#: Retryable. Every member is a defect in infrastructure we control. +INFRA_KINDS: frozenset[ErrorKind] = frozenset( + { + ErrorKind.CONTAINER_EXEC_REFUSED, + ErrorKind.CONTAINER_FORK_EAGAIN, + ErrorKind.RUNTIME_READ_TIMEOUT, + ErrorKind.IMAGE_BUILD_TIMEOUT, + ErrorKind.IMAGE_BUILD_ERROR, + ErrorKind.STEP_INFRASTRUCTURE_FAILURE, + ErrorKind.ENDPOINT_CHANGED, + } +) + +#: Never retried. +#: +#: ``TEST_TIMEOUT`` is a plausible model outcome: a patch that makes the suite +#: loop is a failing patch. ``TEST_MEMORY_EXCEEDED`` is its exact parallel -- a +#: patch that makes a graded test allocate without bound is a failing patch, and +#: the alternative to killing it was never "the test passes", it was "the host +#: OOMs and the instance still never completes". ``PATCH_APPLY_FAILED`` is the +#: model emitting a diff that does not apply; SWE-bench books it as ``error`` +#: rather than ``unresolved``, but it is model behaviour. ``UNKNOWN`` is the bias +#: rule. +GENUINE_KINDS: frozenset[ErrorKind] = frozenset( + { + ErrorKind.TEST_TIMEOUT, + ErrorKind.TEST_MEMORY_EXCEEDED, + ErrorKind.PATCH_APPLY_FAILED, + ErrorKind.UNKNOWN, + } +) + +# ORDERED. First match wins, and the order is load-bearing. +# +# CONTAINER_FORK_EAGAIN and TEST_TIMEOUT come BEFORE CONTAINER_EXEC_REFUSED: a +# timed-out or fork-failed evaluation frequently *also* emits "container state +# improper" while the harness tears the container down, and reading that as a +# wedge would retry a genuine model outcome. +# +# PATCH_APPLY_FAILED is checked LAST: if a container also wedged, the wedge +# wins, because a wedged container's verdict is unreliable either way. Only a +# log with no infrastructure signature at all reaches this rule. +_RULES: tuple[tuple[ErrorKind, tuple[str, ...]], ...] = ( + ( + ErrorKind.CONTAINER_FORK_EAGAIN, + ("fork/exec /usr/bin/conmon: resource temporarily unavailable",), + ), + (ErrorKind.TEST_TIMEOUT, ("Test timed out after",)), + ( + ErrorKind.CONTAINER_EXEC_REFUSED, + ( + "can only create exec sessions on running containers", + "container state improper", + ), + ), + (ErrorKind.RUNTIME_READ_TIMEOUT, ("Read timed out. (read timeout=",)), + ( + ErrorKind.PATCH_APPLY_FAILED, + ( + "Reversed (or previously applied) patch detected", + ">>>>> Patch Apply Failed", + "hunk FAILED", + "hunk failed", + ), + ), +) + + +def classify_eval_log(text: str) -> ErrorKind: + """Classify one instance's evaluation log. + + ``BuildImageError`` is checked before the ordered rules because its message + embeds the same "Read timed out" / "500" strings the other rules look for, + so any other order misattributes a build failure. + """ + if "BuildImageError" in text: + if "Read timed out" in text: + return ErrorKind.IMAGE_BUILD_TIMEOUT + return ErrorKind.IMAGE_BUILD_ERROR + for kind, needles in _RULES: + if any(needle in text for needle in needles): + return kind + return ErrorKind.UNKNOWN + + +@dataclass(slots=True) +class UnitClassification: + """Per-kind counts for one unit's error instances.""" + + kinds: dict[ErrorKind, int] = field(default_factory=dict) + error_instance_ids: tuple[str, ...] = () + #: False when the run's report could not be read at all. "Not measured" and + #: "measured zero" are different, and conflating them once let a damaged + #: unit into a clean set. + measured: bool = False + + @property + def infra_count(self) -> int: + return sum(count for kind, count in self.kinds.items() if kind in INFRA_KINDS) + + @property + def genuine_count(self) -> int: + return sum(count for kind, count in self.kinds.items() if kind in GENUINE_KINDS) + + @property + def should_retry(self) -> bool: + return self.infra_count > 0 + + def as_counts(self) -> dict[str, int]: + return {kind.value: count for kind, count in sorted(self.kinds.items())} + + +def _find_instance_log(output_dir: Path, instance_id: str) -> Path | None: + patterns = ( + f"logs/run_evaluation/*/*/{instance_id}/run_instance.log", + f"logs/run_evaluation/*/*/*/{instance_id}/run_instance.log", + ) + for pattern in patterns: + for match in sorted(output_dir.glob(pattern)): + return match + return None + + +def memory_kill_markers(killed_dir: Path, instance_id: str) -> bool: + """True only for an *eval*-phase memory kill. + + Phase is load-bearing and the two cases must never be collapsed. An eval + kill destroyed a graded result, so the instance's error is a genuine + failure. An agent kill merely makes one tool call return an error + observation and the agent carries on, so the instance still reaches a real + outcome; that marker exists for audit and must not influence classification. + + A marker beats any log heuristic: a SIGKILLed test leaves an ambiguous log, + but the kill itself is a fact recorded before acting. + """ + return any(killed_dir.glob(f"eval.{instance_id}.*.json")) + + +def classify_unit( + output_dir: Path, + error_instance_ids: list[str] | tuple[str, ...] | None, + *, + killed_dir: Path | None = None, + infrastructure_failure: bool = False, + endpoint_changed: bool = False, +) -> UnitClassification: + """Classify every error instance of one unit. + + ``infrastructure_failure`` carries the Pyxis agent-phase sentinel, and + ``endpoint_changed`` carries a mismatch between the inference endpoint + fingerprint recorded at claim time and at publish time -- an engine + restarted under a live client produces a plausible-looking run that must not + be scored. + """ + kinds: dict[ErrorKind, int] = {} + + if infrastructure_failure: + kinds[ErrorKind.STEP_INFRASTRUCTURE_FAILURE] = ( + kinds.get(ErrorKind.STEP_INFRASTRUCTURE_FAILURE, 0) + 1 + ) + if endpoint_changed: + kinds[ErrorKind.ENDPOINT_CHANGED] = kinds.get(ErrorKind.ENDPOINT_CHANGED, 0) + 1 + + if error_instance_ids is None: + return UnitClassification(kinds=kinds, error_instance_ids=(), measured=False) + + ids = tuple(str(x) for x in error_instance_ids) + for instance_id in ids: + if killed_dir is not None and memory_kill_markers(killed_dir, instance_id): + kinds[ErrorKind.TEST_MEMORY_EXCEEDED] = ( + kinds.get(ErrorKind.TEST_MEMORY_EXCEEDED, 0) + 1 + ) + continue + log_path = _find_instance_log(output_dir, instance_id) + kind = ErrorKind.UNKNOWN + if log_path is not None: + try: + kind = classify_eval_log(log_path.read_text(errors="replace")) + except OSError: + logger.debug("could not read %s", log_path, exc_info=True) + kinds[kind] = kinds.get(kind, 0) + 1 + + return UnitClassification(kinds=kinds, error_instance_ids=ids, measured=True) diff --git a/tests/unit/evaluation/swe_bench_distributed/test_classify.py b/tests/unit/evaluation/swe_bench_distributed/test_classify.py new file mode 100644 index 000000000..2b079b315 --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_classify.py @@ -0,0 +1,171 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Infrastructure-versus-genuine classification of error instances.""" + +from __future__ import annotations + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed.classify import ( + GENUINE_KINDS, + INFRA_KINDS, + ErrorKind, + classify_eval_log, + classify_unit, +) + +pytestmark = pytest.mark.unit + + +def write_log(output_dir, instance_id: str, text: str) -> None: + log_dir = output_dir / "logs" / "run_evaluation" / "run-1" / "model" / instance_id + log_dir.mkdir(parents=True, exist_ok=True) + (log_dir / "run_instance.log").write_text(text) + + +class TestBiasRule: + def test_unknown_is_genuine_never_infra(self): + # A false bad-run costs one redo; a false retry biases the measurement + # toward optimism. Unclassifiable therefore means "keep the result". + assert ErrorKind.UNKNOWN in GENUINE_KINDS + assert ErrorKind.UNKNOWN not in INFRA_KINDS + + def test_kinds_are_partitioned(self): + assert not (INFRA_KINDS & GENUINE_KINDS) + assert INFRA_KINDS | GENUINE_KINDS == set(ErrorKind) + + def test_model_outcomes_are_genuine(self): + for kind in ( + ErrorKind.TEST_TIMEOUT, + ErrorKind.TEST_MEMORY_EXCEEDED, + ErrorKind.PATCH_APPLY_FAILED, + ): + assert kind in GENUINE_KINDS + + +class TestLogRules: + @pytest.mark.parametrize( + ("text", "expected"), + [ + ( + "fork/exec /usr/bin/conmon: resource temporarily unavailable", + ErrorKind.CONTAINER_FORK_EAGAIN, + ), + ("Test timed out after 1800s", ErrorKind.TEST_TIMEOUT), + ( + "can only create exec sessions on running containers", + ErrorKind.CONTAINER_EXEC_REFUSED, + ), + ("container state improper", ErrorKind.CONTAINER_EXEC_REFUSED), + ("Read timed out. (read timeout=60)", ErrorKind.RUNTIME_READ_TIMEOUT), + ( + "Reversed (or previously applied) patch detected", + ErrorKind.PATCH_APPLY_FAILED, + ), + ("1 out of 3 hunk FAILED", ErrorKind.PATCH_APPLY_FAILED), + ("nothing recognisable here", ErrorKind.UNKNOWN), + ], + ) + def test_each_rule(self, text, expected): + assert classify_eval_log(text) is expected + + def test_timeout_wins_over_teardown_noise(self): + # A timed-out evaluation also emits "container state improper" while the + # harness tears the container down. Reading that as a wedge would retry + # a genuine model outcome, so rule order is load-bearing. + text = "Test timed out after 1800s\ncontainer state improper\n" + assert classify_eval_log(text) is ErrorKind.TEST_TIMEOUT + + def test_fork_failure_wins_over_teardown_noise(self): + text = ( + "fork/exec /usr/bin/conmon: resource temporarily unavailable\n" + "can only create exec sessions on running containers\n" + ) + assert classify_eval_log(text) is ErrorKind.CONTAINER_FORK_EAGAIN + + def test_a_wedge_wins_over_patch_apply(self): + # A wedged container's verdict is unreliable either way, so the wedge + # decides and the unit is retried. + text = "container state improper\nhunk FAILED\n" + assert classify_eval_log(text) is ErrorKind.CONTAINER_EXEC_REFUSED + + def test_build_error_is_checked_before_every_other_rule(self): + # BuildImageError's message embeds the same needles the other rules look + # for, so any other ordering misattributes a build failure. + assert ( + classify_eval_log("BuildImageError: Read timed out. (read timeout=60)") + is ErrorKind.IMAGE_BUILD_TIMEOUT + ) + assert ( + classify_eval_log("BuildImageError: 500 Server Error") + is ErrorKind.IMAGE_BUILD_ERROR + ) + + +class TestClassifyUnit: + def test_infra_and_genuine_are_counted_separately(self, tmp_path): + write_log(tmp_path, "a-1", "container state improper") + write_log(tmp_path, "a-2", "Test timed out after 1800s") + + classification = classify_unit(tmp_path, ["a-1", "a-2"]) + + assert classification.infra_count == 1 + assert classification.genuine_count == 1 + assert classification.should_retry + + def test_only_genuine_errors_do_not_trigger_a_retry(self, tmp_path): + write_log(tmp_path, "a-1", "Test timed out after 1800s") + assert not classify_unit(tmp_path, ["a-1"]).should_retry + + def test_a_missing_log_is_unknown_and_therefore_genuine(self, tmp_path): + classification = classify_unit(tmp_path, ["absent"]) + assert classification.kinds == {ErrorKind.UNKNOWN: 1} + assert not classification.should_retry + + def test_none_error_ids_means_not_measured(self, tmp_path): + # "We did not measure" and "we measured zero" are different; conflating + # them once let a damaged unit into a clean set. + classification = classify_unit(tmp_path, None) + assert classification.measured is False + + def test_empty_error_ids_means_measured_zero(self, tmp_path): + classification = classify_unit(tmp_path, []) + assert classification.measured is True + assert classification.infra_count == 0 + + def test_eval_memory_kill_marker_is_genuine(self, tmp_path): + killed = tmp_path / "killed" + killed.mkdir() + (killed / "eval.a-1.host.999.json").write_text("{}") + write_log(tmp_path, "a-1", "container state improper") + + classification = classify_unit(tmp_path, ["a-1"], killed_dir=killed) + + # The marker beats the log: a SIGKILLed test leaves an ambiguous log, + # but the kill is a fact recorded before acting. + assert classification.kinds == {ErrorKind.TEST_MEMORY_EXCEEDED: 1} + assert not classification.should_retry + + def test_agent_phase_kill_marker_does_not_classify(self, tmp_path): + killed = tmp_path / "killed" + killed.mkdir() + (killed / "agent.a-1.host.999.json").write_text("{}") + write_log(tmp_path, "a-1", "Test timed out after 1800s") + + # An agent kill only makes one tool call return an error observation; + # the instance still reaches a real outcome, so the marker is audit-only. + classification = classify_unit(tmp_path, ["a-1"], killed_dir=killed) + assert classification.kinds == {ErrorKind.TEST_TIMEOUT: 1} + + def test_step_infrastructure_failure_forces_a_retry(self, tmp_path): + classification = classify_unit(tmp_path, [], infrastructure_failure=True) + assert classification.should_retry + assert classification.kinds == {ErrorKind.STEP_INFRASTRUCTURE_FAILURE: 1} + + def test_a_changed_endpoint_forces_a_retry(self, tmp_path): + # An engine restarted under a live client produces a plausible run that + # scores near zero and exits successfully. + classification = classify_unit(tmp_path, [], endpoint_changed=True) + assert classification.should_retry + assert classification.kinds == {ErrorKind.ENDPOINT_CHANGED: 1} From c57b08966d9792c56f5bf151a149cf27e9543634 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Sat, 8 Aug 2026 17:41:31 -0700 Subject: [PATCH 5/9] feat(swe-bench): pre-dispatch gates that must prove their own scale run_gates() calls assert_scale() before check() and treats GateScaleError as a gate FAILURE, never a skip. This is the code-level form of the most expensive lesson available: a tool-call gate that exercised the right operation at a 278-token prompt passed, while prompts over 2k tokens silently returned empty, and the run scored 0/80. - CheckpointIdentityGate probes /get_model_info then /v1/models and compares the served model path with == , never startswith or in: the bf16 path is a strict prefix of the fp8 path, so any substring test passes an FP8 engine as bf16. Unidentifiable or ambiguous endpoints fail closed. - ToolCallGate requires a well-formed bash tool call at a prompt of at least min_prompt_tokens measured with the server's own /tokenize, not estimated from characters. No tokenizer means the gate cannot prove its scale, so it fails. - EndpointFingerprintGate records a per-endpoint identity the dispatcher re-checks at publish time, so an engine restarted under a live client cannot yield a 0%-accuracy run that still exits rc=0. --- .../swe_bench_distributed/__init__.py | 18 + .../evaluation/swe_bench_distributed/gates.py | 423 ++++++++++++++++++ .../swe_bench_distributed/test_gates.py | 224 ++++++++++ 3 files changed, 665 insertions(+) create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/gates.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_gates.py diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py index c2711f071..44324e27f 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/__init__.py @@ -19,6 +19,16 @@ classify_eval_log, classify_unit, ) +from .gates import ( + CheckpointIdentityGate, + EndpointFingerprintGate, + Gate, + GateFailure, + GateReport, + GateScaleError, + ToolCallGate, + run_gates, +) from .guards import HealthTerm, HealthVerdict, MemoryGuard, combine_terms, kill_by_pid from .merge import MergeRefusal, MergeResult, merge_run, verify_inventory from .queue import ( @@ -33,8 +43,14 @@ __all__ = [ "GENUINE_KINDS", "INFRA_KINDS", + "CheckpointIdentityGate", "ClaimError", + "EndpointFingerprintGate", "ErrorKind", + "Gate", + "GateFailure", + "GateReport", + "GateScaleError", "HealthTerm", "HealthVerdict", "LocalProcessLiveness", @@ -43,6 +59,7 @@ "MergeResult", "OwnerLiveness", "SlurmStepLiveness", + "ToolCallGate", "Unit", "UnitClassification", "UnitOutcome", @@ -56,5 +73,6 @@ "merge_run", "plan_units", "reap", + "run_gates", "verify_inventory", ] diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py new file mode 100644 index 000000000..9330fe012 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py @@ -0,0 +1,423 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pre-dispatch gates on the inference endpoints. + +A gate proves, before a single instance is dispatched, that the endpoints under +test can actually do the thing the benchmark requires. Gates fail closed: an +endpoint that cannot be identified or reached is a failure, never a pass. + +THE SCALE RULE. Every gate must first prove it is testing at the scale it +claims, via :meth:`Gate.assert_scale`, and a scale failure is a *gate failure*, +not a skip. This is not defensive programming; it is the most expensive lesson +in this codebase's history. A tool-call gate that exercised exactly the right +operation with a 278-token prompt passed cleanly while every prompt above 2000 +tokens silently returned an empty completion -- and SWE-bench prompts are all +far larger than 2000 tokens. The gate was green and the run scored zero. A gate +that cannot prove its scale is not a gate. +""" + +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from typing import Any, Protocol +from urllib import error as urllib_error +from urllib import request as urllib_request + +logger = logging.getLogger(__name__) + +_DEFAULT_TIMEOUT_S = 60.0 +#: SWE-bench prompts are far larger than this; the threshold is a floor, not a +#: target. +DEFAULT_MIN_PROMPT_TOKENS = 2000 + + +class GateFailure(RuntimeError): + """A gate refused to let the run start.""" + + +class GateScaleError(GateFailure): + """A gate could not prove it was testing at the scale it claims.""" + + +@dataclass(slots=True) +class GateReport: + name: str + passed: bool + checked: int = 0 + failures: list[tuple[str, str]] = field(default_factory=list) + notes: list[str] = field(default_factory=list) + data: dict[str, Any] = field(default_factory=dict) + + def summary(self) -> str: + head = ( + f"{self.name}: {'pass' if self.passed else 'FAIL'} ({self.checked} checked)" + ) + detail = "".join( + f"\n {target} -> {reason}" for target, reason in self.failures[:8] + ) + notes = "".join(f"\n note: {note}" for note in self.notes) + return head + detail + notes + + +class Gate(Protocol): + name: str + + def assert_scale(self, targets: list[str]) -> None: + """Prove this gate tests what it claims. Raise :class:`GateScaleError`.""" + ... + + def check(self, targets: list[str]) -> GateReport: ... + + +def _http_json( + url: str, + payload: dict[str, Any] | None = None, + *, + timeout_s: float = _DEFAULT_TIMEOUT_S, + api_key: str | None = None, +) -> dict[str, Any]: + data = None + headers = {"Accept": "application/json"} + if api_key: + headers["Authorization"] = f"Bearer {api_key}" + if payload is not None: + data = json.dumps(payload).encode() + headers["Content-Type"] = "application/json" + request = urllib_request.Request(url, data=data, headers=headers) + with urllib_request.urlopen(request, timeout=timeout_s) as response: + return json.loads(response.read()) + + +def run_gates(gates: list[Gate], targets: list[str]) -> list[GateReport]: + """Run every gate; raise :class:`GateFailure` if any refused. + + Every gate runs even after one fails, so one preflight reports every problem + rather than sending the operator round the loop once per endpoint. + """ + reports: list[GateReport] = [] + for gate in gates: + try: + gate.assert_scale(targets) + except GateScaleError as exc: + reports.append( + GateReport( + name=gate.name, + passed=False, + failures=[("", str(exc))], + notes=[ + "a gate that cannot prove its scale is a failing gate, " + "not a skipped one" + ], + ) + ) + continue + reports.append(gate.check(targets)) + + failed = [report for report in reports if not report.passed] + if failed: + raise GateFailure( + "pre-dispatch gate(s) refused:\n" + + "\n".join(report.summary() for report in failed) + ) + return reports + + +class CheckpointIdentityGate: + """Every endpoint must serve exactly the expected checkpoint. + + Two traps, both of which produced silently contaminated results: + + 1. ``/v1/models`` echoes ``--served-model-name``, which operators routinely + set identically for two different checkpoints (e.g. an FP8 and a BF16 + build of the same model). ``/get_model_info`` reports the real model + path, so it is tried first. + 2. Checkpoint names nest: ``Org/Model`` is a strict prefix of + ``Org/Model-FP8``. Any ``startswith``/``in`` test therefore accepts an + FP8 endpoint as BF16. Comparison is ``==`` and nothing else. + """ + + name = "checkpoint_identity" + + def __init__( + self, + expected_model: str, + *, + timeout_s: float = 10.0, + api_key: str | None = None, + ) -> None: + if not expected_model: + raise ValueError("expected_model is required") + self.expected_model = expected_model + self.timeout_s = timeout_s + self.api_key = api_key + + def assert_scale(self, targets: list[str]) -> None: + if not targets: + raise GateScaleError("no endpoints to identify") + + def probe(self, url: str) -> tuple[str | None, str]: + base = url.rstrip("/") + try: + info = _http_json( + f"{base}/get_model_info", + timeout_s=self.timeout_s, + api_key=self.api_key, + ) + model_path = info.get("model_path") + if model_path: + return str(model_path), "get_model_info" + except (urllib_error.URLError, OSError, ValueError, TimeoutError): + pass # not an SGLang endpoint; fall through to the OpenAI route + try: + listing = _http_json( + f"{base}/v1/models", timeout_s=self.timeout_s, api_key=self.api_key + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError) as exc: + return None, f"unreachable: {type(exc).__name__}" + ids = [ + entry.get("id") for entry in listing.get("data") or [] if entry.get("id") + ] + if len(ids) == 1: + return str(ids[0]), "v1/models" + if len(ids) > 1: + return None, f"ambiguous /v1/models: {ids!r}" + return None, "no model id from either endpoint" + + def check(self, targets: list[str]) -> GateReport: + report = GateReport(name=self.name, passed=True, checked=len(targets)) + sources: set[str] = set() + for url in targets: + identity, source = self.probe(url) + if identity is None: + report.failures.append((url, source)) + elif identity != self.expected_model: # EXACT; never startswith/in + report.failures.append( + (url, f"serves {identity!r}, expected {self.expected_model!r}") + ) + else: + sources.add(source) + if "v1/models" in sources: + report.notes.append( + "identity came from /v1/models, which echoes --served-model-name; " + "that string can be identical across checkpoints, so it cannot " + "separate two builds that share a served name" + ) + report.passed = not report.failures + report.data["expected_model"] = self.expected_model + return report + + +class ToolCallGate: + """Every endpoint must return a well-formed tool call at SWE-bench scale. + + The prompt is measured with the *server's own* tokenizer (``/tokenize``), + never estimated from character count, and a prompt that measures below + ``min_prompt_tokens`` fails the scale assertion rather than passing the + gate. + """ + + name = "tool_call" + + def __init__( + self, + model: str, + *, + min_prompt_tokens: int = DEFAULT_MIN_PROMPT_TOKENS, + prompt: str | None = None, + timeout_s: float = 180.0, + api_key: str | None = None, + tool_name: str = "bash", + ) -> None: + self.model = model + self.min_prompt_tokens = min_prompt_tokens + self.prompt = prompt if prompt is not None else build_scale_prompt() + self.timeout_s = timeout_s + self.api_key = api_key + self.tool_name = tool_name + self._measured: dict[str, int] = {} + + @property + def tools(self) -> list[dict[str, Any]]: + return [ + { + "type": "function", + "function": { + "name": self.tool_name, + "description": "Run a shell command", + "parameters": { + "type": "object", + "properties": {"command": {"type": "string"}}, + "required": ["command"], + }, + }, + } + ] + + def count_tokens(self, url: str) -> int | None: + try: + response = _http_json( + f"{url.rstrip('/')}/tokenize", + {"model": self.model, "prompt": self.prompt}, + timeout_s=self.timeout_s, + api_key=self.api_key, + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError): + return None + count = response.get("count") + if count is None: + tokens = response.get("tokens") + count = len(tokens) if isinstance(tokens, list) else None + try: + return int(count) if count is not None else None + except (TypeError, ValueError): + return None + + def assert_scale(self, targets: list[str]) -> None: + if not targets: + raise GateScaleError("no endpoints to gate") + measured = False + for url in targets: + count = self.count_tokens(url) + if count is None: + continue + self._measured[url] = count + measured = True + if count < self.min_prompt_tokens: + raise GateScaleError( + f"{url}: gate prompt measures {count} tokens, below the " + f"{self.min_prompt_tokens}-token floor this gate claims to " + "test. A tool-call gate that passes at a small prompt says " + "nothing about SWE-bench-sized prompts." + ) + if not measured and self.min_prompt_tokens > 0: + raise GateScaleError( + "no endpoint exposed /tokenize, so the gate cannot prove the " + f"prompt reaches {self.min_prompt_tokens} tokens. Serve a " + "tokenizer endpoint or set min_prompt_tokens=0 to accept an " + "unverified prompt size." + ) + + def check(self, targets: list[str]) -> GateReport: + report = GateReport(name=self.name, passed=True, checked=len(targets)) + for url in targets: + tokens = self._measured.get(url) + try: + response = _http_json( + f"{url.rstrip('/')}/v1/chat/completions", + { + "model": self.model, + "messages": [{"role": "user", "content": self.prompt}], + "tools": self.tools, + "tool_choice": "auto", + "max_tokens": 256, + "temperature": 0.0, + }, + timeout_s=self.timeout_s, + api_key=self.api_key, + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError) as exc: + report.failures.append((url, f"{type(exc).__name__}: {exc}")) + continue + failure = self._validate(response) + if failure is not None: + report.failures.append((url, f"tokens={tokens}: {failure}")) + report.passed = not report.failures + report.data["measured_tokens"] = dict(self._measured) + return report + + def _validate(self, response: dict[str, Any]) -> str | None: + try: + message = response["choices"][0]["message"] + except (KeyError, IndexError, TypeError): + return "malformed chat completion response" + tool_calls = message.get("tool_calls") + if not tool_calls: + content = (message.get("content") or "")[:120] + return f"no tool_calls; content={content!r}" + function = tool_calls[0].get("function") or {} + if function.get("name") != self.tool_name: + return f"wrong tool {function.get('name')!r}" + try: + arguments = json.loads(function.get("arguments") or "") + except (TypeError, ValueError): + return f"arguments are not valid JSON: {function.get('arguments')!r}" + command = arguments.get("command") + if not isinstance(command, str) or not command.strip(): + return f"malformed arguments {function.get('arguments')!r}" + return None + + +def build_scale_prompt(repetitions: int = 120) -> str: + """A prompt long enough to exercise the large-context path.""" + filler = "\n".join( + f"def helper_{index}(path, flags=None):\n" + f" # legacy shim retained for compatibility with the v{index} api\n" + " result = compute_checksum(path, flags or DEFAULT_FLAGS)\n" + " return normalise(result), path, flags\n" + for index in range(repetitions) + ) + return ( + "You are working in a Python repository checked out at /testbed.\n" + "Below is the current content of /testbed/legacy/helpers.py.\n\n" + "\n" + filler + "\n\n" + "Before proposing any change you must inspect the repository.\n" + "List the files in /testbed using the shell tool. Call the tool; do not " + "answer in prose." + ) + + +class EndpointFingerprintGate: + """Record a per-endpoint fingerprint for later comparison. + + An engine restarted under a live client yields a run that scores near zero + and still exits successfully -- nothing in the result distinguishes it from + a genuinely bad model. The dispatcher therefore records each endpoint's + fingerprint when a unit is claimed and re-reads it when the unit is + published; a change means the unit was scored against something other than + what it was dispatched to, and the unit is requeued rather than counted. + """ + + name = "endpoint_fingerprint" + + def __init__(self, *, timeout_s: float = 10.0, api_key: str | None = None) -> None: + self.timeout_s = timeout_s + self.api_key = api_key + self.fingerprints: dict[str, str] = {} + + def assert_scale(self, targets: list[str]) -> None: + if not targets: + raise GateScaleError("no endpoints to fingerprint") + + def fingerprint(self, url: str) -> str | None: + base = url.rstrip("/") + parts: list[str] = [] + for path in ("/get_model_info", "/v1/models"): + try: + payload = _http_json( + base + path, timeout_s=self.timeout_s, api_key=self.api_key + ) + except (urllib_error.URLError, OSError, ValueError, TimeoutError): + continue + parts.append(json.dumps(payload, sort_keys=True, default=str)) + if not parts: + return None + import hashlib + + return hashlib.sha256("|".join(parts).encode()).hexdigest()[:16] + + def check(self, targets: list[str]) -> GateReport: + report = GateReport(name=self.name, passed=True, checked=len(targets)) + for url in targets: + value = self.fingerprint(url) + if value is None: + report.failures.append( + (url, "could not read an identity to fingerprint") + ) + continue + self.fingerprints[url] = value + report.passed = not report.failures + report.data["fingerprints"] = dict(self.fingerprints) + return report diff --git a/tests/unit/evaluation/swe_bench_distributed/test_gates.py b/tests/unit/evaluation/swe_bench_distributed/test_gates.py new file mode 100644 index 000000000..607b04d3f --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_gates.py @@ -0,0 +1,224 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Pre-dispatch gates, including the scale rule.""" + +from __future__ import annotations + +import json + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed import gates as gates_mod +from inference_endpoint.evaluation.swe_bench_distributed.gates import ( + CheckpointIdentityGate, + EndpointFingerprintGate, + GateFailure, + GateScaleError, + ToolCallGate, + build_scale_prompt, + run_gates, +) + +pytestmark = pytest.mark.unit + +ENDPOINT = "http://engine-1:8000" + + +def install_http(monkeypatch, routes): + """Route ``_http_json`` by URL suffix; a missing route raises like a network error.""" + + def fake(url, payload=None, *, timeout_s=60.0, api_key=None): + for suffix, response in routes.items(): + if url.endswith(suffix): + if isinstance(response, Exception): + raise response + if callable(response): + return response(payload) + return response + raise OSError(f"no route for {url}") + + monkeypatch.setattr(gates_mod, "_http_json", fake) + + +def tool_call_response(command="ls /testbed", name="bash", arguments=None): + return { + "choices": [ + { + "message": { + "tool_calls": [ + { + "function": { + "name": name, + "arguments": ( + arguments + if arguments is not None + else json.dumps({"command": command}) + ), + } + } + ] + } + } + ] + } + + +class TestCheckpointIdentity: + def test_exact_match_passes(self, monkeypatch): + install_http(monkeypatch, {"/get_model_info": {"model_path": "Org/Model-FP8"}}) + report = CheckpointIdentityGate("Org/Model-FP8").check([ENDPOINT]) + assert report.passed + + def test_a_prefix_is_not_a_match(self, monkeypatch): + # "Org/Model" is a strict prefix of "Org/Model-FP8", so any + # startswith/in test would accept an FP8 engine as the BF16 build. + install_http(monkeypatch, {"/get_model_info": {"model_path": "Org/Model-FP8"}}) + report = CheckpointIdentityGate("Org/Model").check([ENDPOINT]) + assert not report.passed + assert "Org/Model-FP8" in report.failures[0][1] + + def test_get_model_info_is_preferred_over_v1_models(self, monkeypatch): + # /v1/models echoes --served-model-name, which operators routinely set + # identically for two different checkpoints. + install_http( + monkeypatch, + { + "/get_model_info": {"model_path": "Org/Model-FP8"}, + "/v1/models": {"data": [{"id": "Org/Model"}]}, + }, + ) + assert CheckpointIdentityGate("Org/Model-FP8").check([ENDPOINT]).passed + + def test_v1_models_fallback_warns_about_its_own_ambiguity(self, monkeypatch): + install_http( + monkeypatch, + { + "/get_model_info": OSError("not sglang"), + "/v1/models": {"data": [{"id": "Org/Model"}]}, + }, + ) + report = CheckpointIdentityGate("Org/Model").check([ENDPOINT]) + assert report.passed + assert any("served-model-name" in note for note in report.notes) + + def test_an_unreachable_endpoint_fails_closed(self, monkeypatch): + install_http( + monkeypatch, + {"/get_model_info": OSError("down"), "/v1/models": OSError("down")}, + ) + assert not CheckpointIdentityGate("Org/Model").check([ENDPOINT]).passed + + def test_ambiguous_model_listing_fails(self, monkeypatch): + install_http( + monkeypatch, + { + "/get_model_info": OSError("not sglang"), + "/v1/models": {"data": [{"id": "a"}, {"id": "b"}]}, + }, + ) + report = CheckpointIdentityGate("a").check([ENDPOINT]) + assert not report.passed + assert "ambiguous" in report.failures[0][1] + + +class TestToolCallScale: + def test_a_small_prompt_fails_the_scale_assertion(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": {"count": 278}}) + gate = ToolCallGate("Org/Model", prompt="tiny", min_prompt_tokens=2000) + # A gate exercising the right operation at a 278-token prompt passed + # while every prompt above 2000 tokens silently returned nothing. + with pytest.raises(GateScaleError, match="278 tokens"): + gate.assert_scale([ENDPOINT]) + + def test_a_scale_failure_is_a_gate_failure_not_a_skip(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": {"count": 100}}) + gate = ToolCallGate("Org/Model", min_prompt_tokens=2000) + with pytest.raises(GateFailure, match="failing gate"): + run_gates([gate], [ENDPOINT]) + + def test_no_tokenizer_means_the_gate_cannot_prove_its_scale(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": OSError("404")}) + gate = ToolCallGate("Org/Model", min_prompt_tokens=2000) + with pytest.raises(GateScaleError, match="cannot prove"): + gate.assert_scale([ENDPOINT]) + + def test_scale_can_be_waived_explicitly(self, monkeypatch): + install_http(monkeypatch, {"/tokenize": OSError("404")}) + ToolCallGate("Org/Model", min_prompt_tokens=0).assert_scale([ENDPOINT]) + + def test_the_default_prompt_is_large(self): + assert len(build_scale_prompt()) > 20_000 + + +class TestToolCallCheck: + def _gate(self, monkeypatch, chat_response): + install_http( + monkeypatch, + {"/tokenize": {"count": 4096}, "/v1/chat/completions": chat_response}, + ) + gate = ToolCallGate("Org/Model") + gate.assert_scale([ENDPOINT]) + return gate + + def test_a_well_formed_call_passes(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response()) + report = gate.check([ENDPOINT]) + assert report.passed + assert report.data["measured_tokens"][ENDPOINT] == 4096 + + def test_an_empty_completion_fails(self, monkeypatch): + gate = self._gate(monkeypatch, {"choices": [{"message": {"content": ""}}]}) + report = gate.check([ENDPOINT]) + assert not report.passed + assert "no tool_calls" in report.failures[0][1] + + def test_the_wrong_tool_fails(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response(name="python")) + assert not gate.check([ENDPOINT]).passed + + def test_unparseable_arguments_fail(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response(arguments="{not json")) + report = gate.check([ENDPOINT]) + assert "not valid JSON" in report.failures[0][1] + + def test_an_empty_command_fails(self, monkeypatch): + gate = self._gate(monkeypatch, tool_call_response(command=" ")) + assert not gate.check([ENDPOINT]).passed + + +class TestFingerprint: + def test_the_fingerprint_changes_with_the_served_model(self, monkeypatch): + install_http(monkeypatch, {"/get_model_info": {"model_path": "A"}}) + first = EndpointFingerprintGate().fingerprint(ENDPOINT) + install_http(monkeypatch, {"/get_model_info": {"model_path": "B"}}) + second = EndpointFingerprintGate().fingerprint(ENDPOINT) + assert first is not None and first != second + + def test_an_unidentifiable_endpoint_fails(self, monkeypatch): + install_http(monkeypatch, {}) + assert not EndpointFingerprintGate().check([ENDPOINT]).passed + + +class TestRunGates: + def test_every_gate_runs_even_after_one_fails(self, monkeypatch): + install_http( + monkeypatch, + { + "/get_model_info": {"model_path": "Wrong/Model"}, + "/tokenize": {"count": 4096}, + "/v1/chat/completions": {"choices": [{"message": {}}]}, + }, + ) + with pytest.raises(GateFailure) as excinfo: + run_gates( + [CheckpointIdentityGate("Org/Model"), ToolCallGate("Org/Model")], + [ENDPOINT], + ) + message = str(excinfo.value) + assert "checkpoint_identity" in message + assert "tool_call" in message + + def test_no_targets_is_a_failure_not_a_pass(self, monkeypatch): + with pytest.raises(GateFailure): + run_gates([CheckpointIdentityGate("Org/Model")], []) From 79b866c67086882e6c253ba7f8c5984e42660d17 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Mon, 10 Aug 2026 03:51:04 -0700 Subject: [PATCH 6/9] fix(swe-bench): fingerprint endpoint identity, not the time of asking EndpointFingerprintGate hashed the whole /v1/models payload. vLLM stamps that response with a request-time `created` field and mints a fresh `permission[].id` on every call, so two reads of one healthy, untouched engine produce two different fingerprints -- four calls, four values. The dispatcher records a fingerprint when a unit is claimed and re-reads it when the unit is published, and treats any difference as `endpoint_changed`: an infrastructure fault, which requeues the unit. With an unstable fingerprint that comparison is always true, so every unit is retried until it exhausts max_attempts, is published as abandoned, and the merge gate refuses the run. The failure costs the full agent and evaluation time of every attempt first, and reports itself as infrastructure damage rather than as a bug here. Hash only the identity-bearing fields by dropping the per-request ones. The gate still fails closed on an endpoint whose identity cannot be read at all, which is the property it exists to provide. --- .../evaluation/swe_bench_distributed/gates.py | 28 ++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py index 9330fe012..302ce3d3e 100644 --- a/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/gates.py @@ -369,6 +369,30 @@ def build_scale_prompt(repetitions: int = 120) -> str: ) +#: Response fields that change on every request and carry no checkpoint +#: identity. vLLM's ``/v1/models`` stamps ``created`` with the request time and +#: mints a fresh ``permission[].id`` per call, so hashing the raw payload makes +#: the fingerprint differ between any two reads of a perfectly healthy engine. +#: The dispatcher compares the claim-time and publish-time fingerprints and +#: treats a difference as ``endpoint_changed`` -- an infrastructure fault -- so +#: an unstable fingerprint retries and then abandons every unit, and the merge +#: gate can never produce a number. +_VOLATILE_IDENTITY_KEYS = frozenset({"created", "created_at", "permission"}) + + +def _strip_volatile(value: Any) -> Any: + """Drop per-request fields so a fingerprint reflects identity, not time.""" + if isinstance(value, dict): + return { + key: _strip_volatile(item) + for key, item in value.items() + if key not in _VOLATILE_IDENTITY_KEYS + } + if isinstance(value, list): + return [_strip_volatile(item) for item in value] + return value + + class EndpointFingerprintGate: """Record a per-endpoint fingerprint for later comparison. @@ -401,7 +425,9 @@ def fingerprint(self, url: str) -> str | None: ) except (urllib_error.URLError, OSError, ValueError, TimeoutError): continue - parts.append(json.dumps(payload, sort_keys=True, default=str)) + parts.append( + json.dumps(_strip_volatile(payload), sort_keys=True, default=str) + ) if not parts: return None import hashlib From e5003ecce3c8f6b83f0e48f842ab0519ba2d9c74 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Sat, 8 Aug 2026 17:42:09 -0700 Subject: [PATCH 7/9] feat(swe-bench): SWEBenchFleetScorer - fan out units across a service fleet Wires the pieces into a scorer registered as eval_method: swe_bench_fleet. It is a scheduler in front of the existing SWE-bench service protocol, not a new runtime: a unit is one RunRequest over a shard, so exact instance binding, per-instance containers, artifact allow-listing and cancellation are reused rather than reimplemented. preflight() runs the gates against the inference endpoints and /health against every service, raising SetupError before a single instance is dispatched. score() plans, dispatches with one in-flight run per service, classifies every unit, requeues any unit with infra_error_count > 0 even when the service reported succeeded, and takes the accuracy number only past the merge gate - self.complete comes from the gate, never a count heuristic. Stall quarantine verifies effect rather than status: a service that is /health-OK but has completed no unit within stall_timeout_s is quarantined and its in-flight unit requeued. Also adds scripts/swe_bench_wq.py {status,merge,requeue,reap} for operators. reap is dry-run by default and requeue prints exactly which result, claim and attempt records it removed, because the failure mode in the field was an operator believing a delete had requeued something. --- AGENTS.md | 3 + docs/evaluation/SWE_BENCH_DISTRIBUTED.md | 233 ++++++++++ scripts/swe_bench_wq.py | 174 +++++++ src/inference_endpoint/config/schema.py | 7 +- src/inference_endpoint/evaluation/scoring.py | 5 +- .../evaluation/swe_bench_distributed/fleet.py | 434 ++++++++++++++++++ .../evaluation/swe_bench_fleet_scorer.py | 392 ++++++++++++++++ .../swe_bench_distributed/test_fleet.py | 293 ++++++++++++ .../test_fleet_scorer.py | 78 ++++ 9 files changed, 1617 insertions(+), 2 deletions(-) create mode 100644 docs/evaluation/SWE_BENCH_DISTRIBUTED.md create mode 100644 scripts/swe_bench_wq.py create mode 100644 src/inference_endpoint/evaluation/swe_bench_distributed/fleet.py create mode 100644 src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_fleet.py create mode 100644 tests/unit/evaluation/swe_bench_distributed/test_fleet_scorer.py diff --git a/AGENTS.md b/AGENTS.md index 9ef6f83ef..9ff10ac22 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -100,6 +100,7 @@ Dataset Manager --> Load Generator --> Endpoint Client --> External Endpoint | **VideoGen** | `src/inference_endpoint/videogen/` | Adapter for video-generation endpoints (e.g. trtllm-serve `POST /v1/videos/generations`, used by MLPerf WAN2.2-T2V-A14B). Defaults to `response_format=video_path` (server saves video to shared storage and returns path) to avoid large byte payloads. Accuracy mode also runs on `video_path`: the adapter mirrors the path into `response_output` so the event log carries it to `VBenchScorer` (see `evaluation/scoring.py`), which scores videos via VBench from a sibling `uv` subproject at `examples/09_Wan22_VideoGen_Example/accuracy/` (vbench's `transformers==4.33.2` + `numpy<2` pins are incompatible with the parent env, so it runs out-of-process via `uv run --project`). Dataset is ingested via the generic JSONL loader. | | **SWE-bench** | `src/inference_endpoint/dataset_manager/predefined/swe_bench/`, `src/inference_endpoint/evaluation/swe_bench_scorer.py`, `src/inference_endpoint/evaluation/swebench_service/` | `SWEBench` predefined dataset (HuggingFace `princeton-nlp/SWE-bench_Verified` or `_Lite`; `ACCURACY_ONLY=True`). `SWEBenchScorer` sets `SKIP_ENDPOINT_PHASE=True` and bypasses the built-in accuracy phase entirely: it delegates agent execution and grading to the configured SWE-bench service via `accuracy_config.extras.swebench_service_url`. The service is an isolated `uv` subproject; its host owns Docker/runtime execution, artifacts, and credentials, while the benchmark client remains the report-producing entrypoint. | | **Compliance (submission checker)** | `src/inference_endpoint/compliance/checker.py`, `scripts/check_compliance.py` | Validates a completed run's report directory against a registered ruleset. `check_submission(report_dir, ruleset, model)` reads the resolved `config.yaml` plus scorer output (`accuracy/accuracy_results.json` for accuracy, `scores.json` for the agentic perf run) and runs config-lock (deterministic + single-stream), the accuracy gate (`score >= factor x reference`, factor 0.97 for Edge-Agentic), and run validity (0 dropped turns). Server-side launch flags (`--reasoning off`, `--ctx-size`) aren't in client artifacts, so they're surfaced as manual attestations. CLI: `scripts/check_compliance.py REPORT_DIR` (exit 0 = pass). | +| **SWE-bench (distributed)** | `src/inference_endpoint/evaluation/swe_bench_distributed/`, `src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py`, `scripts/swe_bench_wq.py` | `SWEBenchFleetScorer` (`eval_method: swe_bench_fleet`) shards the instance list into units and runs them across several SWE-bench services concurrently, reusing the same service HTTP protocol. Adds what the single-service path lacks: a durable `mkdir`-atomic work queue (resume after a client crash), an eval-phase infra-vs-genuine classifier driving in-unit retry, pre-dispatch gates on the inference endpoints (checkpoint identity, tool call at >=2k-token scale, endpoint fingerprint), a memory guard, and an all-or-nothing merge gate that compares instance **ids** (never counts) and is scoped to exactly one run id. Operator CLI: `scripts/swe_bench_wq.py {status,merge,requeue,reap}` — `requeue` is the only way to re-run a unit; deleting a result leaves the claim tombstone in place. See `docs/evaluation/SWE_BENCH_DISTRIBUTED.md`. | | **Compliance (audit tests)** | `src/inference_endpoint/compliance/`, `commands/audit.py` | MLPerf compliance audits. `AuditTest` protocol + `AuditRunSpec`/`AuditRunArtifacts` + registry (`compliance/__init__.py`); `OutputCachingAudit` (`compliance/audit_test/output_caching_test.py`, which also owns the QPS-specific `AuditRunStats`) implements MLPerf **TEST04** output-caching detection — reference phase (distinct samples) vs. fixed-sample audit phase, comparing QPS against `threshold`. `commands/audit.py:run_audit` runs phases via `AuditTest.plan_runs`/`validate`, writing `audit_result.json`/`verify_.txt` atomically via `compliance/result.py`. Enabled by the `audit:` YAML block; `cli._run` runs it after the main benchmark (upstream MLPerf order: perf run, then TEST04), or standalone with `audit.only: true`. Perf-only by default (a phase may opt into accuracy via `AuditRunSpec.test_mode`, but this is unused today). | ### Hot-Path Architecture @@ -265,6 +266,8 @@ src/inference_endpoint/ │ └── adapter.py # VideoGenAdapter (HttpRequestAdapter) + VideoGenAccumulator (no-op) ├── evaluation/ # Accuracy evaluation (extractor, scoring, livecodebench) │ └── swebench_service/ # Isolated uv service for Docker-backed SWE-bench runs +│ ├── swe_bench_distributed/ # Fleet dispatch: unit plan, work queue, reaper, classifier, gates, guards, merge gate +│ └── swe_bench_fleet_scorer.py # SWEBenchFleetScorer (scorer_id swe_bench_fleet) ├── compliance/ # Submission compliance checks (config-lock, accuracy gate, run validity) │ ├── __init__.py │ └── checker.py # check_submission() + Check/ComplianceReport (Edge-Agentic ruleset) diff --git a/docs/evaluation/SWE_BENCH_DISTRIBUTED.md b/docs/evaluation/SWE_BENCH_DISTRIBUTED.md new file mode 100644 index 000000000..898d4e15b --- /dev/null +++ b/docs/evaluation/SWE_BENCH_DISTRIBUTED.md @@ -0,0 +1,233 @@ +# Distributed SWE-bench (`swe_bench_fleet`) + +> Shards a SWE-bench accuracy run across several SWE-bench services, classifies +> infrastructure damage separately from genuine model failures, and refuses to +> emit an accuracy number unless every planned instance is accounted for exactly +> once. + +`swe_bench_scorer` runs the whole instance list as one service run against one +endpoint. That is the right shape for a hundred instances on one Docker host. It +does not survive a 200-instance run spread over many hours and many hosts: a +client crash loses everything, a host that dies takes its instances with it, and +an evaluation container that wedges is booked as an ordinary `error` — accounted +for, never retried, and silently subtracted from the score. + +`swe_bench_fleet` addresses those six gaps and nothing else. It reuses the +service HTTP protocol, the Docker/Pyxis runtimes, the exact instance-id binding, +the artifact allow-list and the secret redaction unchanged. + +## Configuration + +```yaml +datasets: + - name: swe_bench + accuracy_config: + eval_method: swe_bench_fleet + extras: + swebench_service_urls: + - http://swe-host-1:18080 + - http://swe-host-2:18080 + swebench_service_auth_token: ${SWEBENCH_TOKEN} + num_instances: 200 + shard_size: 10 # instances per unit; 200 / 10 = 20 units + max_attempts: 3 + expected_model: Org/Model-FP8 # optional; gates checkpoint identity + min_prompt_tokens: 2000 # tool-call gate scale floor + stall_timeout_s: 10800 +``` + +| Extra | Default | Meaning | +| --- | --- | --- | +| `swebench_service_urls` | required | One URL per service host. Duplicates are refused: two entries for one host is not extra capacity, it is two runs contending for the same container runtime. | +| `shard_size` | 10 | Instances per unit. | +| `max_attempts` | 3 | Counted attempts before a unit is abandoned. Environment faults are not counted. | +| `expected_model` | none | When set, every endpoint must serve exactly this checkpoint. | +| `min_prompt_tokens` | 2000 | Floor the tool-call gate must prove it reaches. `0` waives the proof. | +| `stall_timeout_s` | 10800 | A service completing no unit in this long is quarantined even if healthy. | + +## What runs, in order + +1. **Preflight gates** (`preflight()`, before the benchmark starts) — every + service's `/health`, plus the endpoint gates below. Any failure raises + `SetupError` before a single instance is dispatched, and every gate runs so + one preflight reports every problem. +2. **Plan** — the instance list is split into units, and the plan is written + once to `units.json` with a digest over the run id and the ordered ids. +3. **Dispatch** — one in-flight service run per service; each service loop + claims a unit, submits it, polls, downloads artifacts, classifies, and + publishes or retries. +4. **Merge gate** — `merge_run(queue, run_id)`. All-or-nothing. + +## The gates + +| Gate | Refuses | +| --- | --- | +| `CheckpointIdentityGate` | Any endpoint not serving *exactly* `expected_model`. `/get_model_info` is preferred over `/v1/models`, because the latter echoes `--served-model-name`, which is routinely identical across checkpoints. Comparison is `==`: `Org/Model` is a strict prefix of `Org/Model-FP8`, so any `startswith`/`in` test accepts FP8 as BF16. Unidentifiable means fail. | +| `ToolCallGate` | Any endpoint that does not return a well-formed tool call — right name, `arguments` parse as JSON, non-empty `command`. | +| `EndpointFingerprintGate` | Any endpoint whose identity cannot be read at all. Records a fingerprint compared again at publish time. | + +**The scale rule.** Every gate implements `assert_scale()`, it runs before the +gate's own check, and a scale failure is a *gate failure*, never a skip. +`ToolCallGate` measures its prompt with the server's own `/tokenize` and fails if +the prompt is below `min_prompt_tokens`. This exists because a tool-call gate +that exercised exactly the right operation with a 278-token prompt passed +cleanly while every prompt above 2000 tokens silently returned an empty +completion. SWE-bench prompts are all far above 2000 tokens; the gate was green +and the run scored zero. **A gate that cannot prove its scale is not a gate.** + +## The work queue + +Under `/swe_bench_wq/`: + +``` +units.json immutable plan + digest +claims//owner host, pid, boot id, plan digest, SLURM job/step +claims//hb heartbeat (mtime only) +results/.json terminal record (succeeded OR abandoned) +failed/..json one per counted attempt +failed/env/.*.json environment faults (not counted) +failed/artifacts/.attemptN/ evidence snapshot taken before a retry +``` + +A unit is available when it is in the plan and has **neither** a claim **nor** a +result. Claiming is `os.mkdir` and nothing else — `makedirs(exist_ok=True)` would +hand the unit to every caller. + +### Re-running a unit + +```bash +python scripts/swe_bench_wq.py requeue REPORT_DIR run-a.s07 +``` + +`requeue` is the **only** supported way. Deleting the result file does not +requeue anything: the claim tombstone still hides the unit. `requeue` removes the +result, the claim and the counted attempt records together, and prints exactly +what it removed. + +### Reaping abandoned claims + +```bash +python scripts/swe_bench_wq.py reap REPORT_DIR # dry run +python scripts/swe_bench_wq.py reap REPORT_DIR --apply --slurm +``` + +A claim is released only when it has no result, its heartbeat is stale, **and** +its owner is provably gone. Uncertainty never escalates: if the liveness probe +fails, times out, or returns an implausible answer, nothing is released. A false +reap gives one unit two owners, duplicate results, and a wrong denominator, with +no error anywhere. + +`SlurmStepLiveness` treats an owner as dead when its job is absent from `squeue` +**or** its step is absent from `scontrol show step` while the job lives — a step +can die inside a live job, and the job-level rule alone then blocks those units +for the whole allocation. Step liveness never uses `squeue -s`, which reports +only `.extern` on the clusters this targets and would mark every live step dead. + +## Classification and retry + +Every unit is classified after the service reports success. The rule list is +ordered and first-match-wins; the order is load-bearing. + +| Kind | Class | Why | +| --- | --- | --- | +| `container_fork_eagain`, `container_exec_refused`, `runtime_read_timeout`, `image_build_timeout`, `image_build_error`, `step_infrastructure_failure`, `endpoint_changed` | infra → **retry** | Defects in infrastructure we provided. | +| `test_timeout` | genuine | A patch that makes the suite loop is a failing patch. | +| `test_memory_exceeded` | genuine | A patch that makes a graded test allocate without bound is a failing patch. The alternative to killing it was never "the test passes", it was "the host OOMs and the instance still never completes". | +| `patch_apply_failed` | genuine | The model emitted a diff that does not apply. SWE-bench books it as `error`, but it is model behaviour. | +| `unknown` | genuine | **The bias rule.** | + +**The bias rule is deliberately asymmetric.** An error that cannot be classified +confidently is genuine, never infrastructure. A false bad-run costs one redo; a +false retry biases the measurement toward optimism, and an optimistic accuracy +number is worse than no number. + +`endpoint_changed` deserves its own note: an engine restarted under a live client +yields a run that scores near zero and exits successfully, and nothing in the +result distinguishes it from a genuinely bad model. The endpoint fingerprint +recorded at claim time is re-read at publish time; a change requeues the unit. + +### Attempt accounting + +* **Environment fault** (service unreachable, submit failed) — recorded under + `failed/env/`, does **not** consume the attempt budget, and counts toward + quarantining that service. A broken host is a property of the host, not of the + unit. +* **Infra / failed** — counted. After `max_attempts` the unit is published as + `abandoned` and its claim released, so it stops burning capacity and shows up + loudly in the merge gate instead of spinning forever. + +Before every retry, the small files that explain the failure are snapshotted to +`failed/artifacts/.attemptN/`, because the unit's run directory is reused +and a unit that fails then succeeds would otherwise leave only the success's +artifacts behind. + +## The merge gate + +`merge_run(queue, run_id)` produces a number only when **all** hold: + +1. every planned unit has a terminal result; +2. no result is abandoned; +3. every unit's accounted instance **ids** equal its planned ids exactly — a set + comparison, never a count, because a shard with one duplicate and one missing + id has the right count and the wrong content; +4. the union across units equals the plan, with no id claimed twice; +5. every result carries the plan's digest; +6. no unit lost instances to infrastructure. + +Otherwise it raises `MergeRefusal` listing every reason. There is no force flag, +no partial-credit path, and **no `merge_all`**: `run_id` is required, and merging +"everything that looks finished" once combined hundreds of banked results from +unrelated configurations into one number. + +`verify_inventory()` cross-checks three independently produced views — the plan, +the claim directory and the result directory — and treats disagreement as an +error. Checking one view against itself is how a verification pass agrees with a +broken system. + +## Resource guards + +`MemoryGuard` kills a graded test only when its resident memory is at or above +`kill_bytes` (default 150 GiB) **and** it has a container-supervisor ancestor. +There is deliberately no working-directory term: an earlier version required a +cwd inside the testbed and skipped a runaway that had grown to 667 GiB because +its cwd was `/tmp`. Every extra conjunct is another way for the guard to miss +what it exists to catch. + +Two rules are enforced by construction and by test: + +* **Kill by pid, never by pattern.** There is no `pkill`/`pgrep` path in the + module and it never shells out; `kill_by_pid` refuses this process and its + ancestors. A pattern can match the guard's own command line, and a long-lived + daemon can carry a dead process's argv for days. +* **A conjunctive guard must not degenerate.** `combine_terms()` returns + `INDETERMINATE`, never `UNHEALTHY`, when any term has zero evidence. An + AND-guard whose honest term loses its data source collapses into its weaker + clauses and starts firing on healthy targets. + +The kill marker is written *before* the signal, and its phase is load-bearing: +only `eval.*` markers make an instance's error a genuine failure. An `agent` +kill merely makes one tool call return an error observation, so it is recorded +for audit and must not influence classification. An unresolvable container name +fails closed to `unknown`. + +## Operational notes + +* `--kill-on-bad-exit=0` does **not** prevent a scheduler force-terminating a + whole step when one node OOMs; OOM escalation is separate from task exit codes. + The defence is small, independently retryable units plus `MemoryGuard` acting + before the host dies — not the flag. +* A service that answers `/health` while completing nothing is the silent + failure. Verify the effect, never the status: the dispatcher quarantines a + service that has completed no unit within `stall_timeout_s` and requeues its + work. +* In shell tooling around this queue, remember `grep -c` exits 1 on zero + matches; a pipeline under `set -e` will abort on an empty, correct answer. + +## What is intentionally not here + +Cluster-lifecycle machinery — allocation rotation, holder chaining, image-store +construction and distribution, node suspect lists, multi-configuration campaign +bookkeeping — is out of scope for a benchmark client. The Pyxis runtime pulls +per-instance images from a registry and never builds a store, so that whole +class of image-corruption failure is architecturally absent rather than worked +around. diff --git a/scripts/swe_bench_wq.py b/scripts/swe_bench_wq.py new file mode 100644 index 000000000..b82648df5 --- /dev/null +++ b/scripts/swe_bench_wq.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Operator tool for a distributed SWE-bench work queue. + + swe_bench_wq.py status REPORT_DIR + swe_bench_wq.py merge REPORT_DIR --run-id RUN + swe_bench_wq.py requeue REPORT_DIR UNIT_ID [UNIT_ID ...] + swe_bench_wq.py reap REPORT_DIR [--apply] + +Two deliberate omissions: + +* There is no ``merge --all``. A merge is always scoped to one run id; merging + "everything that looks finished" once combined hundreds of banked results + from unrelated configurations into a single number. +* There is no way to re-run a unit other than ``requeue``. Deleting a result + file does not requeue anything, because the claim tombstone still hides the + unit; ``requeue`` removes the result, the claim and the attempt records + together and prints exactly what it removed. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src")) + +from inference_endpoint.evaluation.swe_bench_distributed.fleet import ( # noqa: E402 + QUEUE_DIRNAME, +) +from inference_endpoint.evaluation.swe_bench_distributed.merge import ( # noqa: E402 + MergeRefusal, + merge_run, + verify_inventory, +) +from inference_endpoint.evaluation.swe_bench_distributed.queue import ( # noqa: E402 + WorkQueue, +) +from inference_endpoint.evaluation.swe_bench_distributed.reaper import ( # noqa: E402 + LocalProcessLiveness, + SlurmStepLiveness, + reap, +) + + +def _open(report_dir: Path) -> WorkQueue: + root = report_dir / QUEUE_DIRNAME + if not root.exists(): + root = report_dir + return WorkQueue.open(root) + + +def cmd_status(args: argparse.Namespace) -> int: + queue = _open(args.report_dir) + results = queue.results() + claimed = queue.claimed_unit_ids() + inventory = verify_inventory(queue) + print(f"run: {queue.plan.run_id}") + print(f"plan digest: {queue.plan.digest[:16]}") + print(f"units: {len(queue.plan.units)}") + print(f" with result: {len(results)}") + print(f" claimed: {len(claimed)}") + print(f" available: {len(queue.available_unit_ids())}") + abandoned = [uid for uid, result in results.items() if result.abandoned] + if abandoned: + print(f" ABANDONED: {len(abandoned)} -> {', '.join(sorted(abandoned)[:8])}") + infra = [uid for uid, result in results.items() if result.infra_error_count] + if infra: + print(f" infra-damaged:{len(infra)} -> {', '.join(sorted(infra)[:8])}") + if not inventory.consistent: + print("\nINVENTORY DISAGREEMENT (claims, results and ids do not agree):") + for label, values in ( + ("missing results", inventory.missing_units), + ("results outside the plan", inventory.foreign_units), + ("unreadable results", inventory.unreadable_units), + ("ownerless claims", inventory.ownerless_claims), + ): + if values: + print(f" {label}: {len(values)} -> {', '.join(values[:8])}") + return 0 + + +def cmd_merge(args: argparse.Namespace) -> int: + queue = _open(args.report_dir) + try: + result = merge_run(queue, args.run_id) + except MergeRefusal as exc: + print(f"REFUSED to score run {exc.run_id}:") + for reason in exc.reasons: + print(f" - {reason}") + return 1 + print(json.dumps(result.to_dict(), indent=2)) + return 0 + + +def cmd_requeue(args: argparse.Namespace) -> int: + queue = _open(args.report_dir) + for unit_id in args.unit_ids: + removed = queue.requeue(unit_id) + total = sum(len(paths) for paths in removed.values()) + print(f"{unit_id}: removed {total} record(s)") + for kind, paths in removed.items(): + for path in paths: + print(f" {kind}: {path}") + if total == 0: + print(" (nothing to remove; the unit was already runnable)") + return 0 + + +def cmd_reap(args: argparse.Namespace) -> int: + queue = _open(args.report_dir) + liveness = SlurmStepLiveness() if args.slurm else LocalProcessLiveness() + report = reap( + queue, + liveness, + stale_after_s=args.stale, + step_stale_after_s=args.step_stale, + apply=args.apply, + ) + verb = "released" if args.apply else "would release" + print(f"{verb} {len(report.released)} claim(s)") + for unit_id in report.released: + print(f" {unit_id}") + if args.verbose: + for unit_id, reason in sorted(report.kept.items()): + print(f" kept {unit_id}: {reason}") + if not args.apply and report.released: + print("\nthis was a dry run; pass --apply to release") + return 0 + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + sub = parser.add_subparsers(dest="command", required=True) + + status = sub.add_parser("status", help="summarise the queue") + status.add_argument("report_dir", type=Path) + status.set_defaults(func=cmd_status) + + merge = sub.add_parser("merge", help="score exactly one run") + merge.add_argument("report_dir", type=Path) + merge.add_argument( + "--run-id", + required=True, + help="required; a merge is always scoped to one run", + ) + merge.set_defaults(func=cmd_merge) + + requeue = sub.add_parser( + "requeue", help="make units runnable again (result + claim + attempts)" + ) + requeue.add_argument("report_dir", type=Path) + requeue.add_argument("unit_ids", nargs="+") + requeue.set_defaults(func=cmd_requeue) + + reap_parser = sub.add_parser("reap", help="release claims whose owner is gone") + reap_parser.add_argument("report_dir", type=Path) + reap_parser.add_argument("--apply", action="store_true", help="actually release") + reap_parser.add_argument("--slurm", action="store_true", help="use SLURM liveness") + reap_parser.add_argument("--stale", type=float, default=3600.0) + reap_parser.add_argument("--step-stale", type=float, default=900.0) + reap_parser.add_argument("--verbose", action="store_true") + reap_parser.set_defaults(func=cmd_reap) + + args = parser.parse_args(argv) + return args.func(args) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/inference_endpoint/config/schema.py b/src/inference_endpoint/config/schema.py index e9a69e5d2..70a0e906a 100644 --- a/src/inference_endpoint/config/schema.py +++ b/src/inference_endpoint/config/schema.py @@ -149,6 +149,7 @@ class ScorerMethod(str, Enum): BFCL_V4 = "bfcl_v4" LEGACY_MLPERF_DEEPSEEK_R1 = "legacy_mlperf_deepseek_r1" SWE_BENCH = "swe_bench_scorer" + SWE_BENCH_FLEET = "swe_bench_fleet" class AuditTestId(str, Enum): @@ -1154,6 +1155,9 @@ def _resolve_and_validate(self) -> Self: if not self.model_params.name: raise ValueError("Required: --model-params.name [--model]") + # Only the single-service scorer is limited to one endpoint. The fleet + # scorer runs many service runs, each against one endpoint, so it has + # no such restriction. uses_swe_bench = any( dataset.accuracy_config is not None and dataset.accuracy_config.eval_method == ScorerMethod.SWE_BENCH @@ -1299,7 +1303,8 @@ def _resolve_and_validate(self) -> Self: acc = ds.accuracy_config if ( acc is not None - and acc.eval_method == ScorerMethod.SWE_BENCH + and acc.eval_method + in (ScorerMethod.SWE_BENCH, ScorerMethod.SWE_BENCH_FLEET) and (acc.extras is None or acc.extras.get("workers") is None) ): new_extras = {**(acc.extras or {}), "workers": concurrency} diff --git a/src/inference_endpoint/evaluation/scoring.py b/src/inference_endpoint/evaluation/scoring.py index 790a17bcb..d626bf9d5 100644 --- a/src/inference_endpoint/evaluation/scoring.py +++ b/src/inference_endpoint/evaluation/scoring.py @@ -2143,5 +2143,8 @@ def score_breakdown(self) -> dict[str, Any] | None: return self._breakdown -# Late import registers the extracted scorer without introducing a cycle. +# Late imports register the extracted scorers without introducing a cycle. +from .swe_bench_fleet_scorer import ( # noqa: E402 + SWEBenchFleetScorer as SWEBenchFleetScorer, +) from .swe_bench_scorer import SWEBenchScorer as SWEBenchScorer # noqa: E402 diff --git a/src/inference_endpoint/evaluation/swe_bench_distributed/fleet.py b/src/inference_endpoint/evaluation/swe_bench_distributed/fleet.py new file mode 100644 index 000000000..44723dabc --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_distributed/fleet.py @@ -0,0 +1,434 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Fan a SWE-bench accuracy run out across a fleet of SWE-bench services.""" + +from __future__ import annotations + +import logging +import threading +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any +from urllib.parse import urljoin + +import msgspec +import yaml + +from ...exceptions import SetupError +from .classify import classify_unit +from .gates import ( + CheckpointIdentityGate, + EndpointFingerprintGate, + Gate, + GateFailure, + ToolCallGate, + run_gates, +) +from .merge import MergeRefusal, merge_run +from .queue import UnitOutcome, UnitResult, WorkQueue +from .reaper import LocalProcessLiveness, reap +from .units import Unit, plan_units + +logger = logging.getLogger(__name__) + +QUEUE_DIRNAME = "swe_bench_wq" +UNITS_DIRNAME = "units" + +#: Buckets a SWE-bench run report uses for instances that reached an outcome. +#: ``incomplete_ids`` is deliberately absent: an incomplete instance is exactly +#: what "not accounted for" means, and it must fail the merge gate. +ACCOUNTED_ID_KEYS = ( + "resolved_ids", + "unresolved_ids", + "empty_patch_ids", + "error_ids", +) + + +class ServiceQuarantined(RuntimeError): + """A service was withdrawn from the fleet.""" + + +@dataclass(slots=True) +class ServiceState: + """Per-service bookkeeping for the dispatcher.""" + + url: str + completed_units: int = 0 + consecutive_env_faults: int = 0 + last_progress_at: float = field(default_factory=time.monotonic) + quarantined_reason: str | None = None + + @property + def available(self) -> bool: + return self.quarantined_reason is None + + +@dataclass(slots=True) +class DispatchOutcome: + """What one attempt at one unit produced.""" + + result: UnitResult + terminal: bool + + +class FleetDispatcher: + """Claim units, run them on services, classify, retry, and merge. + + Concurrency is one in-flight service run per service. The service itself + parallelises within a run (``workers`` / ``max_eval_workers``), so a second + concurrent run per service would only contend for the same host. + """ + + def __init__( + self, + *, + queue: WorkQueue, + service_urls: list[str], + submit: Any, + poll: Any, + collect: Any, + fingerprint: Any = None, + max_attempts: int = 3, + stall_timeout_s: float = 3 * 60 * 60, + max_consecutive_env_faults: int = 3, + idle_poll_s: float = 1.0, + unit_root: Path | None = None, + killed_dir: Path | None = None, + ) -> None: + if not service_urls: + raise SetupError("the SWE-bench fleet needs at least one service URL") + self.queue = queue + self.services = {url: ServiceState(url=url) for url in service_urls} + self.submit = submit + self.poll = poll + self.collect = collect + self.fingerprint = fingerprint + self.max_attempts = max_attempts + self.stall_timeout_s = stall_timeout_s + self.max_consecutive_env_faults = max_consecutive_env_faults + self.idle_poll_s = idle_poll_s + self.unit_root = unit_root + self.killed_dir = killed_dir + self._lock = threading.Lock() + self._in_flight = 0 + + # ------------------------------------------------------------ dispatch -- + + def run(self) -> None: + """Drive every planned unit to a terminal result.""" + with ThreadPoolExecutor(max_workers=len(self.services)) as pool: + futures = [ + pool.submit(self._service_loop, url) for url in list(self.services) + ] + for future in futures: + future.result() + + def _service_loop(self, url: str) -> None: + state = self.services[url] + while state.available: + fingerprint = self._fingerprint(url) + unit = self._take_unit(fingerprint) + if unit is None: + # An empty queue does not mean the run is finished. A unit in + # flight on another service can be released back at any moment + # -- a failed attempt, a quarantined peer -- and a worker that + # exits on the first empty poll leaves that unit for nobody. + # Only "nothing available AND nothing in flight" ends the run. + if self._idle_is_terminal(): + return + time.sleep(self.idle_poll_s) + continue + try: + outcome = self._attempt(unit, state, fingerprint) + self._settle(unit, state, outcome) + finally: + with self._lock: + self._in_flight -= 1 + self._check_stall(state) + + def _take_unit(self, fingerprint: str | None) -> Unit | None: + """Claim the next available unit and mark it in flight, atomically. + + Claiming and counting must happen under one lock. With the increment + after the claim there is a window in which a peer sees "nothing + available" (this unit is claimed) and "nothing in flight" (not yet + counted), concludes the run is over, and exits -- leaving the unit with + nobody to retry it if this attempt fails. + """ + with self._lock: + for unit_id in self.queue.available_unit_ids(): + if self.queue.claim(unit_id, endpoint_fingerprint=fingerprint) is None: + continue # another process won the filesystem claim + self._in_flight += 1 + return self.queue.plan.unit(unit_id) + return None + + def _idle_is_terminal(self) -> bool: + with self._lock: + return self._in_flight == 0 and not self.queue.available_unit_ids() + + def _fingerprint(self, url: str) -> str | None: + if self.fingerprint is None: + return None + try: + return self.fingerprint() + except Exception: # noqa: BLE001 - a fingerprint is advisory at claim time + logger.debug("could not fingerprint endpoints for %s", url, exc_info=True) + return None + + def _attempt( + self, unit: Unit, state: ServiceState, claim_fingerprint: str | None + ) -> DispatchOutcome: + started = time.monotonic() + base = UnitResult( + unit_id=unit.unit_id, + run_id=unit.run_id, + plan_digest=self.queue.plan.digest, + outcome=UnitOutcome.FAILED, + service_url=state.url, + endpoint_fingerprint=claim_fingerprint, + ) + try: + service_run_id = self.submit(state.url, unit) + except Exception as exc: # noqa: BLE001 - any submit failure is the host's + base.outcome = UnitOutcome.ENV_FAULT + base.detail = f"submit failed: {type(exc).__name__}: {exc}" + base.duration_s = time.monotonic() - started + return DispatchOutcome(result=base, terminal=False) + + base.service_run_id = service_run_id + try: + status = self.poll(state.url, service_run_id) + except Exception as exc: # noqa: BLE001 + base.outcome = UnitOutcome.ENV_FAULT + base.detail = f"poll failed: {type(exc).__name__}: {exc}" + base.duration_s = time.monotonic() - started + return DispatchOutcome(result=base, terminal=False) + + base.duration_s = time.monotonic() - started + if status.get("status") != "succeeded": + base.outcome = UnitOutcome.FAILED + base.detail = ( + f"service run ended {status.get('status')}: {status.get('error')}" + ) + return DispatchOutcome(result=base, terminal=False) + + report, output_dir = self.collect(state.url, service_run_id, unit, status) + # An engine restarted mid-unit yields a plausible run that scores near + # zero and exits successfully. Comparing the fingerprint is the only + # thing that distinguishes it from a genuinely bad model. + publish_fingerprint = self._fingerprint(state.url) + endpoint_changed = ( + claim_fingerprint is not None + and publish_fingerprint is not None + and claim_fingerprint != publish_fingerprint + ) + + accounted, resolved = accounted_and_resolved(report) + classification = classify_unit( + output_dir, + report.get("error_ids"), + killed_dir=self.killed_dir, + infrastructure_failure=bool(report.get("infrastructure_failure")), + endpoint_changed=endpoint_changed, + ) + base.accounted_instance_ids = accounted + base.resolved_instance_ids = resolved + base.infra_error_count = classification.infra_count + base.genuine_error_count = classification.genuine_count + base.error_kinds = classification.as_counts() + + if classification.should_retry: + # The agent phase succeeded and the service said so, but instances + # were lost to infrastructure. Publishing this as a success is how a + # run silently becomes unable to ever reach a full result. + base.outcome = UnitOutcome.INFRA + base.detail = ( + f"{classification.infra_count} instance(s) lost to infrastructure: " + + ", ".join(f"{k}={v}" for k, v in base.error_kinds.items()) + ) + return DispatchOutcome(result=base, terminal=False) + + missing = set(unit.instance_ids) - set(accounted) + if missing: + base.outcome = UnitOutcome.FAILED + base.detail = f"{len(missing)} instance(s) unaccounted for" + return DispatchOutcome(result=base, terminal=False) + + base.outcome = UnitOutcome.SUCCEEDED + return DispatchOutcome(result=base, terminal=True) + + def _settle( + self, unit: Unit, state: ServiceState, outcome: DispatchOutcome + ) -> None: + result = outcome.result + if outcome.terminal: + self.queue.publish(result) + state.completed_units += 1 + state.consecutive_env_faults = 0 + state.last_progress_at = time.monotonic() + return + + if self.unit_root is not None: + attempt = self.queue.attempts(unit.unit_id) + 1 + self.queue.snapshot_evidence( + unit.unit_id, self.unit_root / unit.unit_id, attempt + ) + + attempts = self.queue.record_attempt(result) + + if result.outcome is UnitOutcome.ENV_FAULT: + # The unit is fine; the host is not. Do not charge the unit, and + # withdraw the service if it keeps doing this. + state.consecutive_env_faults += 1 + if state.consecutive_env_faults >= self.max_consecutive_env_faults: + state.quarantined_reason = ( + f"{state.consecutive_env_faults} consecutive environment faults" + ) + logger.error( + "quarantining SWE-bench service %s: %s", + state.url, + state.quarantined_reason, + ) + self.queue.release(unit.unit_id) + return + + state.consecutive_env_faults = 0 + if attempts >= self.max_attempts: + # Stop burning slots. An abandoned unit is a loud, terminal record + # that the merge gate refuses, not a unit that spins forever. + self.queue.abandon(result) + state.last_progress_at = time.monotonic() + return + self.queue.release(unit.unit_id) + + def _check_stall(self, state: ServiceState) -> None: + """Withdraw a service that is healthy but not producing. + + Health is not progress. A service that answers ``/health`` while + completing nothing is the silent failure mode: verify the effect, never + the status. + """ + if not state.available: + return + idle = time.monotonic() - state.last_progress_at + if idle > self.stall_timeout_s: + state.quarantined_reason = ( + f"no unit completed in {idle:.0f}s despite a healthy service" + ) + logger.error( + "quarantining SWE-bench service %s: %s", + state.url, + state.quarantined_reason, + ) + + @property + def quarantined(self) -> dict[str, str]: + return { + url: state.quarantined_reason + for url, state in self.services.items() + if state.quarantined_reason is not None + } + + +def accounted_and_resolved( + report: dict[str, Any], +) -> tuple[tuple[str, ...], tuple[str, ...]]: + """Extract the accounted and resolved instance ids from a SWE-bench report. + + Ids, not counts. A shard with one duplicate and one missing id has the right + count and the wrong content, and only an id comparison catches it. + """ + accounted: list[str] = [] + seen: set[str] = set() + for key in ACCOUNTED_ID_KEYS: + for instance_id in report.get(key) or (): + text = str(instance_id) + if text in seen: + # Preserve the duplicate so the merge gate can refuse it rather + # than silently deduplicating a real accounting bug. + accounted.append(text) + continue + seen.add(text) + accounted.append(text) + resolved = tuple(str(x) for x in report.get("resolved_ids") or ()) + return tuple(accounted), resolved + + +def build_gates( + *, + expected_model: str | None, + tool_call_model: str | None, + min_prompt_tokens: int, + api_key: str | None = None, +) -> tuple[list[Gate], EndpointFingerprintGate]: + """Assemble the pre-dispatch gates. + + ``expected_model`` is optional only because not every deployment pins a + checkpoint path; when it is set the identity gate is mandatory. + """ + fingerprint_gate = EndpointFingerprintGate(api_key=api_key) + gates: list[Gate] = [] + if expected_model: + gates.append(CheckpointIdentityGate(expected_model, api_key=api_key)) + if tool_call_model: + gates.append( + ToolCallGate( + tool_call_model, + min_prompt_tokens=min_prompt_tokens, + api_key=api_key, + ) + ) + gates.append(fingerprint_gate) + return gates, fingerprint_gate + + +def load_benchmark_config(report_dir: Path) -> dict[str, Any]: + config_path = report_dir / "config.yaml" + if not config_path.exists(): + raise FileNotFoundError( + f"config.yaml not found at {config_path}. The fleet scorer must run " + "inside a benchmark that has already written its config." + ) + with config_path.open() as handle: + config = yaml.safe_load(handle) + if not isinstance(config, dict): + raise ValueError(f"benchmark config at {config_path} must be a YAML mapping") + return config + + +def write_merge_artifacts( + report_dir: Path, payload: dict[str, Any], name: str = "swe_bench_merge.json" +) -> Path: + path = report_dir / name + tmp = path.with_name(f".{path.name}.tmp") + tmp.write_bytes(msgspec.json.encode(payload)) + tmp.replace(path) + return path + + +__all__ = [ + "ACCOUNTED_ID_KEYS", + "QUEUE_DIRNAME", + "UNITS_DIRNAME", + "DispatchOutcome", + "FleetDispatcher", + "GateFailure", + "LocalProcessLiveness", + "MergeRefusal", + "ServiceQuarantined", + "ServiceState", + "accounted_and_resolved", + "build_gates", + "load_benchmark_config", + "merge_run", + "plan_units", + "reap", + "run_gates", + "urljoin", + "write_merge_artifacts", +] diff --git a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py new file mode 100644 index 000000000..2848d9213 --- /dev/null +++ b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py @@ -0,0 +1,392 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""SWE-bench accuracy scorer that fans out across a fleet of SWE-bench services. + +:class:`~inference_endpoint.evaluation.swe_bench_scorer.SWEBenchScorer` runs the +whole instance list as one service run. This scorer shards it, runs the shards +concurrently on several services, refuses to score a run whose instances are not +all accounted for, and retries the shards that lost instances to infrastructure +rather than the model. + +Configured entirely through ``accuracy_config.extras``:: + + accuracy_config: + eval_method: swe_bench_fleet + extras: + swebench_service_urls: + - http://swe-host-1:18080 + - http://swe-host-2:18080 + shard_size: 10 + max_attempts: 3 + expected_model: Org/Model-FP8 # optional; gates the checkpoint + min_prompt_tokens: 2000 +""" + +from __future__ import annotations + +import logging +import time +from pathlib import Path +from typing import Any, ClassVar +from urllib.parse import urljoin + +import msgspec + +from ..dataset_manager.dataset import Dataset +from ..exceptions import SetupError +from .extractor import Extractor +from .scoring import Scorer +from .swe_bench_distributed.fleet import ( + QUEUE_DIRNAME, + UNITS_DIRNAME, + FleetDispatcher, + accounted_and_resolved, + build_gates, + load_benchmark_config, + write_merge_artifacts, +) +from .swe_bench_distributed.gates import GateFailure, run_gates +from .swe_bench_distributed.merge import MergeRefusal, merge_run +from .swe_bench_distributed.queue import WorkQueue +from .swe_bench_distributed.units import Unit, plan_units +from .swe_bench_scorer import SWEBenchScorer + +logger = logging.getLogger(__name__) + + +class SWEBenchFleetScorer(Scorer, scorer_id="swe_bench_fleet"): + """Distributed SWE-bench scoring across N services.""" + + REQUIRES_EXTRACTOR: ClassVar[bool] = False + SKIP_ENDPOINT_PHASE: ClassVar[bool] = True + DEFAULT_SHARD_SIZE: ClassVar[int] = 10 + DEFAULT_MAX_ATTEMPTS: ClassVar[int] = 3 + DEFAULT_MIN_PROMPT_TOKENS: ClassVar[int] = 2000 + DEFAULT_STALL_TIMEOUT_S: ClassVar[int] = 3 * 60 * 60 + DEFAULT_SERVICE_TIMEOUT_S: ClassVar[int] = 24 * 60 * 60 + DEFAULT_POLL_INTERVAL_S: ClassVar[float] = 5.0 + + def __init__( + self, + dataset_name: str, + dataset: Dataset, + report_dir: Any, + extractor: type[Extractor] | None = None, + ground_truth_column: str | None = "instance_id", + **extras: Any, + ) -> None: + super().__init__( + dataset_name=dataset_name, + dataset=dataset, + report_dir=report_dir, + extractor=extractor, + ground_truth_column=ground_truth_column or "instance_id", + ) + self.report_dir = self.report_dir.resolve() + self.options = self._resolve_options(extras) + + # --------------------------------------------------------------- config -- + + @classmethod + def _service_urls(cls, extras: dict[str, Any]) -> list[str]: + raw = extras.get("swebench_service_urls") + if raw is None: + single = extras.get("swebench_service_url") + raw = [single] if single else [] + if isinstance(raw, str): + raw = [part.strip() for part in raw.split(",") if part.strip()] + urls = [SWEBenchScorer._normalize_service_url(url) for url in raw or []] + if not urls: + raise SetupError( + "accuracy_config.extras.swebench_service_urls is required for " + "swe_bench_fleet; list one URL per SWE-bench service host." + ) + duplicates = sorted({url for url in urls if urls.count(url) > 1}) + if duplicates: + # Two entries for one host is not extra capacity; it is two + # concurrent runs contending for the same Docker/Pyxis runtime. + raise SetupError( + "duplicate SWE-bench service URLs: " + ", ".join(duplicates) + ) + return urls + + @classmethod + def _resolve_options(cls, extras: dict[str, Any]) -> dict[str, Any]: + options = dict(SWEBenchScorer._resolve_dataset_options(extras)) + options["service_urls"] = cls._service_urls(extras) + options["auth_token"] = extras.get("swebench_service_auth_token") or None + options["num_instances"] = SWEBenchScorer._get_extra_int( + extras, + "num_instances", + default=SWEBenchScorer.DEFAULT_NUM_INSTANCES, + min_value=1, + ) + options["shard_size"] = SWEBenchScorer._get_extra_int( + extras, "shard_size", default=cls.DEFAULT_SHARD_SIZE, min_value=1 + ) + options["workers"] = SWEBenchScorer._get_extra_int( + extras, "workers", default=SWEBenchScorer.DEFAULT_WORKERS, min_value=1 + ) + options["max_eval_workers"] = SWEBenchScorer._get_extra_int( + extras, + "max_eval_workers", + default=SWEBenchScorer.DEFAULT_MAX_EVAL_WORKERS, + min_value=1, + ) + options["max_attempts"] = SWEBenchScorer._get_extra_int( + extras, "max_attempts", default=cls.DEFAULT_MAX_ATTEMPTS, min_value=1 + ) + options["min_prompt_tokens"] = SWEBenchScorer._get_extra_int( + extras, + "min_prompt_tokens", + default=cls.DEFAULT_MIN_PROMPT_TOKENS, + min_value=0, + ) + options["stall_timeout_s"] = SWEBenchScorer._get_extra_int( + extras, + "stall_timeout_s", + default=cls.DEFAULT_STALL_TIMEOUT_S, + min_value=1, + ) + options["service_timeout_s"] = SWEBenchScorer._get_extra_int( + extras, + "service_timeout_s", + default=cls.DEFAULT_SERVICE_TIMEOUT_S, + min_value=1, + ) + options["poll_interval_s"] = SWEBenchScorer._get_extra_float( + extras, + "poll_interval_s", + default=cls.DEFAULT_POLL_INTERVAL_S, + min_value=0, + ) + options["swebench_template"] = SWEBenchScorer._resolve_service_template(extras) + options["expected_model"] = extras.get("expected_model") or None + options["run_id"] = str(extras.get("run_id") or "swe_bench") + return options + + @classmethod + def dataset_loader_kwargs(cls, extras: dict[str, Any]) -> dict[str, Any]: + return SWEBenchScorer._resolve_dataset_options(extras) + + @classmethod + def external_sample_count(cls, extras: dict[str, Any]) -> int | None: + return SWEBenchScorer.external_sample_count(extras) + + # ------------------------------------------------------------ preflight -- + + @classmethod + def preflight( + cls, extras: dict[str, Any], *, loaded_sample_count: int | None = None + ) -> None: + """Health-check every service and run the pre-dispatch gates. + + Every problem is reported from one preflight. A run that starts against + a mis-served checkpoint, or against an endpoint that cannot emit a tool + call at SWE-bench prompt scale, produces a plausible-looking low score + hours later and costs the whole run. + """ + options = cls._resolve_options(extras) + for url in options["service_urls"]: + SWEBenchScorer._check_health(url, options["auth_token"]) + + endpoints = extras.get("endpoint_urls") or [] + if not endpoints: + logger.info( + "swe_bench_fleet: no endpoint URLs available at preflight; " + "checkpoint and tool-call gates run at dispatch instead" + ) + return + gates, _ = build_gates( + expected_model=options["expected_model"], + tool_call_model=extras.get("model_name"), + min_prompt_tokens=options["min_prompt_tokens"], + api_key=extras.get("endpoint_api_key"), + ) + try: + run_gates(gates, list(endpoints)) + except GateFailure as exc: + raise SetupError(str(exc)) from exc + + def score_single_sample(self, value: str, ground_truth: str) -> float: + raise RuntimeError( + "SWEBenchFleetScorer scores whole units through services; call score()." + ) + + # ---------------------------------------------------------------- score -- + + def score(self) -> tuple[float | None, int]: + self.complete = True + config = load_benchmark_config(self.report_dir) + model_params = config.get("model_params") or {} + model_name = model_params.get("name") + if not model_name: + raise ValueError("model_params.name is required in the benchmark config") + endpoint_config = config.get("endpoint_config") or {} + endpoint_urls = list(endpoint_config.get("endpoints") or []) + if not endpoint_urls: + raise SetupError("the benchmark config lists no endpoint URLs") + + instance_ids = self._instance_ids() + if not instance_ids: + logger.warning("swe_bench_fleet: no instances selected") + self.complete = False + return None, 1 + + gates, fingerprint_gate = build_gates( + expected_model=self.options["expected_model"], + tool_call_model=model_name, + min_prompt_tokens=self.options["min_prompt_tokens"], + api_key=endpoint_config.get("api_key"), + ) + try: + run_gates(gates, endpoint_urls) + except GateFailure as exc: + raise SetupError(str(exc)) from exc + + plan = plan_units( + self.options["run_id"], instance_ids, shard_size=self.options["shard_size"] + ) + queue = WorkQueue(self.report_dir / QUEUE_DIRNAME, plan) + unit_root = self.report_dir / UNITS_DIRNAME + unit_root.mkdir(parents=True, exist_ok=True) + + self._model_name = model_name + self._endpoint_urls = endpoint_urls + self._endpoint_api_key = endpoint_config.get("api_key") + self._generation_params = SWEBenchScorer._generation_params(model_params) + self._unit_root = unit_root + + def fingerprint() -> str | None: + values = [fingerprint_gate.fingerprint(url) for url in endpoint_urls] + if any(value is None for value in values): + return None + return "|".join(v for v in values if v is not None) + + dispatcher = FleetDispatcher( + queue=queue, + service_urls=self.options["service_urls"], + submit=self._submit_unit, + poll=self._poll_unit, + collect=self._collect_unit, + fingerprint=fingerprint, + max_attempts=self.options["max_attempts"], + stall_timeout_s=self.options["stall_timeout_s"], + unit_root=unit_root, + ) + dispatcher.run() + + payload: dict[str, Any] = { + "run_id": plan.run_id, + "plan_digest": plan.digest, + "services": self.options["service_urls"], + "quarantined": dispatcher.quarantined, + } + try: + merged = merge_run(queue, plan.run_id) + except MergeRefusal as exc: + payload["refused"] = exc.reasons + write_merge_artifacts(self.report_dir, payload) + logger.error("swe_bench_fleet: %s", exc) + self.complete = False + return None, 1 + + payload["merge"] = merged.to_dict() + write_merge_artifacts(self.report_dir, payload) + logger.info( + "swe_bench_fleet: resolved %d / %d (%.1f%%) across %d units", + merged.resolved_instances, + merged.total_instances, + merged.resolved_rate * 100, + merged.unit_count, + ) + return merged.resolved_rate, 1 + + # --------------------------------------------------------- service glue -- + + def _instance_ids(self) -> list[str]: + if self.dataset.dataframe is None: + raise RuntimeError( + "SWEBench dataset must be loaded before scoring; call dataset.load()." + ) + frame = self.dataset.dataframe + total = min(self.options["num_instances"], len(frame)) + return [ + str(instance_id) + for instance_id in frame.iloc[:total][self.ground_truth_column].tolist() + ] + + def _submit_unit(self, service_url: str, unit: Unit) -> str: + payload = { + "model_name": self._model_name, + # The service accepts exactly one endpoint URL per run; the fleet's + # parallelism comes from running many units, not many endpoints. + "endpoint_urls": self._endpoint_urls[:1], + "endpoint_api_key": self._endpoint_api_key, + "generation_params": self._generation_params, + "subset": self.options["subset"], + "split": self.options["split"], + "num_instances": len(unit.instance_ids), + "workers": self.options["workers"], + "max_eval_workers": self.options["max_eval_workers"], + "evaluated_instance_ids": list(unit.instance_ids), + "template": self.options["swebench_template"], + } + submitted = SWEBenchScorer._http_json( + urljoin(service_url, "v1/runs"), + method="POST", + payload=payload, + timeout_s=30.0, + auth_token=self.options["auth_token"], + ) + run_id = str(submitted.get("run_id") or "") + if not run_id: + raise SetupError(f"{service_url} did not return a run_id") + return run_id + + def _poll_unit(self, service_url: str, service_run_id: str) -> dict[str, Any]: + deadline = time.monotonic() + self.options["service_timeout_s"] + status: dict[str, Any] = {"status": "queued"} + while status.get("status") not in {"succeeded", "failed", "cancelled"}: + if time.monotonic() >= deadline: + SWEBenchScorer._cancel_service_run( + service_url, service_run_id, self.options["auth_token"] + ) + raise SetupError( + f"timed out waiting for {service_url} run {service_run_id}" + ) + time.sleep(self.options["poll_interval_s"]) + status = SWEBenchScorer._http_json( + urljoin(service_url, f"v1/runs/{service_run_id}"), + timeout_s=30.0, + auth_token=self.options["auth_token"], + ) + return status + + def _collect_unit( + self, + service_url: str, + service_run_id: str, + unit: Unit, + status: dict[str, Any], + ) -> tuple[dict[str, Any], Path]: + target = self._unit_root / unit.unit_id + target.mkdir(parents=True, exist_ok=True) + SWEBenchScorer._download_artifacts( + service_url, status, target, self.options["auth_token"] + ) + report = status.get("result") + if not isinstance(report, dict): + results_path = target / "swe_bench_results.json" + if results_path.exists(): + try: + report = msgspec.json.decode(results_path.read_bytes(), type=dict) + except msgspec.DecodeError: + report = {} + else: + report = {} + return report, target + + +__all__ = ["SWEBenchFleetScorer", "accounted_and_resolved"] diff --git a/tests/unit/evaluation/swe_bench_distributed/test_fleet.py b/tests/unit/evaluation/swe_bench_distributed/test_fleet.py new file mode 100644 index 000000000..02ee81d3b --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_fleet.py @@ -0,0 +1,293 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Fleet dispatch: fan-out, classification-driven retry, quarantine, merge.""" + +from __future__ import annotations + +import itertools + +import pytest + +from inference_endpoint.evaluation.swe_bench_distributed.fleet import ( + FleetDispatcher, + accounted_and_resolved, +) +from inference_endpoint.evaluation.swe_bench_distributed.merge import ( + MergeRefusal, + merge_run, +) +from inference_endpoint.evaluation.swe_bench_distributed.queue import ( + UnitOutcome, + WorkQueue, +) +from inference_endpoint.evaluation.swe_bench_distributed.units import plan_units + +pytestmark = pytest.mark.unit + +IDS = [f"repo__proj-{i:02d}" for i in range(30)] +SERVICES = ["http://svc-a:18080", "http://svc-b:18080"] + + +@pytest.fixture +def queue(tmp_path): + return WorkQueue(tmp_path / "wq", plan_units("run-a", IDS, shard_size=10)) + + +class FakeFleet: + """A scripted stand-in for the SWE-bench service HTTP protocol.""" + + def __init__(self, queue: WorkQueue, tmp_path, *, resolved_per_unit: int = 4): + self.queue = queue + self.tmp_path = tmp_path + self.resolved_per_unit = resolved_per_unit + self.counter = itertools.count() + self.submitted: list[tuple[str, str]] = [] + self.submit_errors: dict[str, Exception] = {} + self.status_for_unit: dict[str, str] = {} + self.error_ids_for_unit: dict[str, list[str]] = {} + self.fingerprints: list[str] = ["fp-1"] + + def submit(self, service_url, unit): + error = self.submit_errors.get(service_url) + if error is not None: + raise error + self.submitted.append((service_url, unit.unit_id)) + return f"svc-run-{next(self.counter)}" + + def poll(self, service_url, service_run_id): + unit_id = self.submitted[-1][1] + return {"status": self.status_for_unit.get(unit_id, "succeeded")} + + def collect(self, service_url, service_run_id, unit, status): + error_ids = self.error_ids_for_unit.get(unit.unit_id, []) + remaining = [i for i in unit.instance_ids if i not in error_ids] + resolved = remaining[: self.resolved_per_unit] + unresolved = remaining[self.resolved_per_unit :] + output_dir = self.tmp_path / "units" / unit.unit_id + output_dir.mkdir(parents=True, exist_ok=True) + report = { + "resolved_ids": resolved, + "unresolved_ids": unresolved, + "error_ids": error_ids, + "empty_patch_ids": [], + } + return report, output_dir + + def fingerprint(self): + return self.fingerprints[0] + + def write_eval_log(self, unit_id: str, instance_id: str, text: str) -> None: + log_dir = ( + self.tmp_path + / "units" + / unit_id + / "logs" + / "run_evaluation" + / "r" + / "m" + / instance_id + ) + log_dir.mkdir(parents=True, exist_ok=True) + (log_dir / "run_instance.log").write_text(text) + + +def dispatcher_for(queue, fleet, **overrides): + kwargs = { + "queue": queue, + "service_urls": SERVICES, + "submit": fleet.submit, + "poll": fleet.poll, + "collect": fleet.collect, + "fingerprint": fleet.fingerprint, + "max_attempts": 3, + } + kwargs.update(overrides) + return FleetDispatcher(**kwargs) + + +class TestAccounting: + def test_every_outcome_bucket_counts_as_accounted(self): + report = { + "resolved_ids": ["a"], + "unresolved_ids": ["b"], + "empty_patch_ids": ["c"], + "error_ids": ["d"], + } + accounted, resolved = accounted_and_resolved(report) + assert set(accounted) == {"a", "b", "c", "d"} + assert resolved == ("a",) + + def test_incomplete_instances_are_not_accounted(self): + # "Incomplete" is exactly what unaccounted means; counting it would let + # a partial shard through the merge gate. + accounted, _ = accounted_and_resolved( + {"resolved_ids": ["a"], "incomplete_ids": ["b"]} + ) + assert accounted == ("a",) + + def test_duplicates_are_preserved_for_the_gate_to_refuse(self): + accounted, _ = accounted_and_resolved( + {"resolved_ids": ["a"], "unresolved_ids": ["a"]} + ) + assert accounted == ("a", "a") + + +class TestHappyPath: + def test_all_units_complete_and_merge(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + dispatcher_for(queue, fleet).run() + + assert len(queue.completed_unit_ids()) == 3 + result = merge_run(queue, "run-a") + assert result.total_instances == 30 + assert result.resolved_instances == 12 + + def test_work_is_spread_across_services(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + dispatcher_for(queue, fleet).run() + assert len({service for service, _ in fleet.submitted}) >= 1 + assert len(fleet.submitted) == 3 + + def test_every_unit_runs_exactly_once(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + dispatcher_for(queue, fleet).run() + dispatched = [unit_id for _, unit_id in fleet.submitted] + assert sorted(dispatched) == sorted(queue.plan.unit_ids) + + +class TestInfraRetry: + def test_an_eval_infra_error_requeues_a_succeeded_run(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + fleet.error_ids_for_unit["run-a.s00"] = [IDS[0]] + fleet.write_eval_log("run-a.s00", IDS[0], "container state improper") + + dispatcher_for(queue, fleet, max_attempts=1).run() + + # The service said "succeeded" and every instance was accounted for, so + # nothing but classification distinguishes this from a real result. + stored = queue.result("run-a.s00") + assert stored is not None + assert stored.abandoned + assert stored.outcome is UnitOutcome.INFRA + assert stored.infra_error_count == 1 + with pytest.raises(MergeRefusal): + merge_run(queue, "run-a") + + def test_a_genuine_error_is_scored_not_retried(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + fleet.error_ids_for_unit["run-a.s00"] = [IDS[0]] + fleet.write_eval_log("run-a.s00", IDS[0], "Test timed out after 1800s") + + dispatcher_for(queue, fleet).run() + + stored = queue.result("run-a.s00") + assert stored is not None + assert stored.outcome is UnitOutcome.SUCCEEDED + assert stored.genuine_error_count == 1 + assert merge_run(queue, "run-a").total_instances == 30 + + def test_a_transient_infra_error_succeeds_on_retry(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + fleet.error_ids_for_unit["run-a.s00"] = [IDS[0]] + fleet.write_eval_log("run-a.s00", IDS[0], "container state improper") + + original_collect = fleet.collect + + def collect_once(service_url, service_run_id, unit, status): + result = original_collect(service_url, service_run_id, unit, status) + fleet.error_ids_for_unit.pop(unit.unit_id, None) + return result + + dispatcher_for(queue, fleet, collect=collect_once).run() + + stored = queue.result("run-a.s00") + assert stored is not None and stored.outcome is UnitOutcome.SUCCEEDED + assert queue.attempts("run-a.s00") == 1 + assert merge_run(queue, "run-a").total_instances == 30 + + def test_a_unit_is_abandoned_after_max_attempts(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + fleet.status_for_unit["run-a.s00"] = "failed" + + dispatcher_for(queue, fleet, max_attempts=2).run() + + stored = queue.result("run-a.s00") + assert stored is not None and stored.abandoned + assert queue.attempts("run-a.s00") == 2 + # An abandoned unit must be loud, not silent: it stops burning slots and + # the gate refuses the run. + assert queue.claimed_unit_ids() == set() + with pytest.raises(MergeRefusal, match="abandoned"): + merge_run(queue, "run-a") + + +class TestEndpointFingerprint: + def test_a_restarted_engine_requeues_the_unit(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + original_collect = fleet.collect + + def collect_then_restart(service_url, service_run_id, unit, status): + result = original_collect(service_url, service_run_id, unit, status) + if unit.unit_id == "run-a.s00" and fleet.fingerprints[0] == "fp-1": + fleet.fingerprints[0] = "fp-2" + return result + + dispatcher_for(queue, fleet, collect=collect_then_restart, max_attempts=1).run() + + stored = queue.result("run-a.s00") + assert stored is not None + # The run "succeeded" and every instance was accounted for; only the + # fingerprint says it was scored against a different engine. + assert stored.outcome is UnitOutcome.INFRA + assert "endpoint_changed" in stored.error_kinds + + +class TestEnvironmentFaults: + def test_a_submit_failure_does_not_charge_the_unit(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + fleet.submit_errors[SERVICES[0]] = OSError("service host is broken") + + dispatcher_for(queue, fleet).run() + + # A broken host is a property of the host. The units still complete, on + # the other service, with a clean attempt ledger. + assert len(queue.completed_unit_ids()) == 3 + assert all(queue.attempts(unit_id) == 0 for unit_id in queue.plan.unit_ids) + assert merge_run(queue, "run-a").total_instances == 30 + + def test_a_persistently_broken_service_is_quarantined(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + fleet.submit_errors[SERVICES[0]] = OSError("service host is broken") + + dispatcher = dispatcher_for(queue, fleet, max_consecutive_env_faults=2) + dispatcher.run() + + assert SERVICES[0] in dispatcher.quarantined + assert SERVICES[1] not in dispatcher.quarantined + + +class TestStallQuarantine: + def test_a_healthy_but_unproductive_service_is_withdrawn(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + dispatcher = dispatcher_for(queue, fleet, stall_timeout_s=-1) + dispatcher.run() + # Health is not progress. A service answering /health while completing + # nothing is the silent failure: verify the effect, never the status. + assert dispatcher.quarantined + assert all( + "no unit completed" in reason for reason in dispatcher.quarantined.values() + ) + + +class TestResume: + def test_a_restarted_client_does_not_redo_completed_units(self, queue, tmp_path): + fleet = FakeFleet(queue, tmp_path) + dispatcher_for(queue, fleet, service_urls=SERVICES[:1]).run() + first_pass = len(fleet.submitted) + + reopened = WorkQueue.open(queue.root) + dispatcher_for(reopened, fleet, service_urls=SERVICES[:1]).run() + + assert len(fleet.submitted) == first_pass + assert merge_run(reopened, "run-a").total_instances == 30 diff --git a/tests/unit/evaluation/swe_bench_distributed/test_fleet_scorer.py b/tests/unit/evaluation/swe_bench_distributed/test_fleet_scorer.py new file mode 100644 index 000000000..b96026227 --- /dev/null +++ b/tests/unit/evaluation/swe_bench_distributed/test_fleet_scorer.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Configuration and registration of the fleet scorer.""" + +from __future__ import annotations + +import pytest + +from inference_endpoint.config.schema import ScorerMethod +from inference_endpoint.evaluation.scoring import Scorer +from inference_endpoint.evaluation.swe_bench_fleet_scorer import SWEBenchFleetScorer +from inference_endpoint.exceptions import SetupError + +pytestmark = pytest.mark.unit + +URLS = ["http://svc-a:18080", "http://svc-b:18080"] + + +class TestRegistration: + def test_the_scorer_is_registered(self): + assert Scorer.get("swe_bench_fleet") is SWEBenchFleetScorer + + def test_the_scorer_method_enum_is_in_sync(self): + assert ScorerMethod.SWE_BENCH_FLEET.value in Scorer.available_scorers() + + def test_it_skips_the_endpoint_phase(self): + # Like the single-service scorer, this one drives the run itself rather + # than consuming responses collected by the load generator. + assert SWEBenchFleetScorer.SKIP_ENDPOINT_PHASE + assert not SWEBenchFleetScorer.REQUIRES_EXTRACTOR + + +class TestOptions: + def test_service_urls_are_normalised(self): + options = SWEBenchFleetScorer._resolve_options( + {"swebench_service_urls": ["http://svc-a:18080/"]} + ) + assert options["service_urls"] == ["http://svc-a:18080/"] + + def test_a_comma_separated_string_is_accepted(self): + options = SWEBenchFleetScorer._resolve_options( + {"swebench_service_urls": "http://svc-a:18080, http://svc-b:18080"} + ) + assert len(options["service_urls"]) == 2 + + def test_the_single_service_key_still_works(self): + options = SWEBenchFleetScorer._resolve_options( + {"swebench_service_url": "http://svc-a:18080"} + ) + assert options["service_urls"] == ["http://svc-a:18080/"] + + def test_no_service_urls_is_a_setup_error(self): + with pytest.raises(SetupError, match="swebench_service_urls is required"): + SWEBenchFleetScorer._resolve_options({}) + + def test_duplicate_service_urls_are_refused(self): + # Two entries for one host is not extra capacity; it is two concurrent + # runs contending for the same container runtime. + with pytest.raises(SetupError, match="duplicate"): + SWEBenchFleetScorer._resolve_options( + {"swebench_service_urls": ["http://svc-a:18080", "http://svc-a:18080/"]} + ) + + def test_defaults_are_sane(self): + options = SWEBenchFleetScorer._resolve_options( + {"swebench_service_urls": URLS} + ) + assert options["shard_size"] == 10 + assert options["max_attempts"] == 3 + # The tool-call gate's floor must stay at SWE-bench prompt scale. + assert options["min_prompt_tokens"] == 2000 + + def test_a_bad_shard_size_is_rejected(self): + with pytest.raises(SetupError, match="shard_size"): + SWEBenchFleetScorer._resolve_options( + {"swebench_service_urls": URLS, "shard_size": 0} + ) From 153f9ba91804a4b28d97fac170b2f78082e5bf88 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Mon, 10 Aug 2026 03:51:26 -0700 Subject: [PATCH 8/9] fix(swe-bench): validate model_params before reading generation settings score() reads the run's settings back from the report directory's config.yaml, which yaml.safe_load() returns as plain dictionaries. It then handed that mapping to SWEBenchScorer._generation_params(), which calls .model_dump() on it, so the fleet scorer raised AttributeError: 'dict' object has no attribute 'model_dump' on every run, after the plan and the work queue had been written but before a single unit was dispatched. Re-validate the mapping into ModelParams instead of re-implementing the field selection here, so the fleet path and the single-service path stay in agreement about which generation settings are forwarded to the service. --- .../evaluation/swe_bench_fleet_scorer.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py index 2848d9213..d3a932224 100644 --- a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py +++ b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py @@ -255,7 +255,15 @@ def score(self) -> tuple[float | None, int]: self._model_name = model_name self._endpoint_urls = endpoint_urls self._endpoint_api_key = endpoint_config.get("api_key") - self._generation_params = SWEBenchScorer._generation_params(model_params) + # load_benchmark_config() yaml.safe_load()s config.yaml, so model_params + # is a plain mapping here, while _generation_params() expects the + # pydantic ModelParams. Re-validate rather than re-implement the field + # selection, so the fleet path and the single-service path agree. + from ..config.schema import ModelParams + + self._generation_params = SWEBenchScorer._generation_params( + ModelParams.model_validate(model_params) + ) self._unit_root = unit_root def fingerprint() -> str | None: From 854f8549e90bfd70c7ff0e96cf94c085ae9945d2 Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Mon, 10 Aug 2026 03:51:45 -0700 Subject: [PATCH 9/9] feat(swe-bench): bind each unit to an endpoint by shard index Every unit was submitted with endpoint_urls[:1], so a fleet configured with N engines sent all of its work to the first one and left the other N-1 idle. The comment justified this by noting that the service accepts exactly one endpoint per run and that the fleet's parallelism comes from running many units -- true, but it does not follow that every unit must pick the same one. Two consequences. The obvious one is a throughput ceiling: concurrency is bounded by one engine no matter how much hardware the run was given. The serious one is a measurement hazard -- a single engine's behaviour decides the whole run's accuracy, so one degraded engine is indistinguishable from a degraded model, which is precisely the confusion the endpoint fingerprint exists to prevent. Bind unit -> endpoint by shard index instead. The mapping is deterministic, so a retried unit lands on the endpoint it was originally measured against and stays comparable to its first attempt, and a run with one endpoint behaves exactly as before. --- .../evaluation/swe_bench_fleet_scorer.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py index d3a932224..a946edfc2 100644 --- a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py +++ b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py @@ -326,11 +326,17 @@ def _instance_ids(self) -> list[str]: ] def _submit_unit(self, service_url: str, unit: Unit) -> str: + # The service accepts exactly one endpoint URL per run, so a unit is + # bound to exactly one endpoint. Sending every unit to endpoint 0 would + # funnel the whole fleet through a single engine while the rest idle, + # which is both a throughput ceiling and a measurement hazard: one + # engine's behaviour would decide the entire run's accuracy. Binding by + # shard index spreads units deterministically -- the same unit always + # gets the same endpoint, so a retry is comparable to its first attempt. + endpoint = self._endpoint_urls[unit.shard % len(self._endpoint_urls)] payload = { "model_name": self._model_name, - # The service accepts exactly one endpoint URL per run; the fleet's - # parallelism comes from running many units, not many endpoints. - "endpoint_urls": self._endpoint_urls[:1], + "endpoint_urls": [endpoint], "endpoint_api_key": self._endpoint_api_key, "generation_params": self._generation_params, "subset": self.options["subset"],