From 12921b3f73c8ea98c9fb7aacc30fe95bdc949c6b Mon Sep 17 00:00:00 2001 From: Stanley Phoong Date: Mon, 10 Aug 2026 03:51:45 -0700 Subject: [PATCH] feat(swe-bench): bind each unit to an endpoint by shard index Every unit was submitted with endpoint_urls[:1], so a fleet configured with N engines sent all of its work to the first one and left the other N-1 idle. The comment justified this by noting that the service accepts exactly one endpoint per run and that the fleet's parallelism comes from running many units -- true, but it does not follow that every unit must pick the same one. Two consequences. The obvious one is a throughput ceiling: concurrency is bounded by one engine no matter how much hardware the run was given. The serious one is a measurement hazard -- a single engine's behaviour decides the whole run's accuracy, so one degraded engine is indistinguishable from a degraded model, which is precisely the confusion the endpoint fingerprint exists to prevent. Bind unit -> endpoint by shard index instead. The mapping is deterministic, so a retried unit lands on the endpoint it was originally measured against and stays comparable to its first attempt, and a run with one endpoint behaves exactly as before. --- .../evaluation/swe_bench_fleet_scorer.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py index d3a932224..a946edfc2 100644 --- a/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py +++ b/src/inference_endpoint/evaluation/swe_bench_fleet_scorer.py @@ -326,11 +326,17 @@ def _instance_ids(self) -> list[str]: ] def _submit_unit(self, service_url: str, unit: Unit) -> str: + # The service accepts exactly one endpoint URL per run, so a unit is + # bound to exactly one endpoint. Sending every unit to endpoint 0 would + # funnel the whole fleet through a single engine while the rest idle, + # which is both a throughput ceiling and a measurement hazard: one + # engine's behaviour would decide the entire run's accuracy. Binding by + # shard index spreads units deterministically -- the same unit always + # gets the same endpoint, so a retry is comparable to its first attempt. + endpoint = self._endpoint_urls[unit.shard % len(self._endpoint_urls)] payload = { "model_name": self._model_name, - # The service accepts exactly one endpoint URL per run; the fleet's - # parallelism comes from running many units, not many endpoints. - "endpoint_urls": self._endpoint_urls[:1], + "endpoint_urls": [endpoint], "endpoint_api_key": self._endpoint_api_key, "generation_params": self._generation_params, "subset": self.options["subset"],