From b20aed9e89aae3c0d56a2c17ad22bd9f1359a993 Mon Sep 17 00:00:00 2001 From: Joe Blau Date: Mon, 27 Jul 2026 21:13:02 -0700 Subject: [PATCH] perf(tests): stabilize subscribe_user_trio so the gate stops flipping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `subscription/subscribe_user_trio` produced coin-flip regression verdicts on unrelated PRs. On #84 — which changes only two files under src/signing — it reported +39.6% with a band of [-29.7%, +177.2%], a band spanning zero because the three rounds disagreed in sign. The scenario built one transport per sample and awaited three subscriptions, so a sample measured async scheduling and first-transport warmup rather than the cost of subscribing. Every neighbouring scenario was stable across the same measurement positions, so this was the scenario's own instability, not runner noise. Two changes: - The scenario runs 20 iterations per sample instead of 1, so per-sample warmup and scheduling jitter amortize across 60 subscriptions rather than landing on 3. `unitsPerIteration` still normalizes to a per-subscription figure, so the reported unit is unchanged — it now reports the steady-state cost instead of a number dominated by the first transport in each sample. - `MockWebSocket` keeps only the most recently constructed instance rather than pushing every one into a static array. `lastMockWebSocket` only ever read `.at(-1)`, but the array kept every socket — and through the listeners on it, every transport and its keep-alive timers — reachable until the scenario ended. That mattered little at one transport per sample and matters a great deal at twenty. Measured over three full-suite runs on one machine: before 14.68 µs (rme 112.0%), 11.00 µs (9.6%), 13.22 µs (12.7%) after 9.90 µs (rme 6.6%), 9.61 µs (5.6%), 9.57 µs (6.8%) Run-to-run spread falls from 33% to 3.4% and worst-case rme from 112% to 6.8%. Baseline rme feeds the comparator's tolerance, so the tighter figure also stops this scenario inflating the gate's noise allowance for every other scenario in the report. Co-Authored-By: Claude Fable 5 --- tests/perf/_helpers.ts | 18 ++++++++++++++---- tests/perf/scenarios/user_account_channels.ts | 2 +- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/tests/perf/_helpers.ts b/tests/perf/_helpers.ts index ce1be098..8a619511 100644 --- a/tests/perf/_helpers.ts +++ b/tests/perf/_helpers.ts @@ -102,7 +102,17 @@ export class MockInfoTransport implements IRequestTransport { * Server-to-client frames are injected with {@linkcode MockWebSocket.serverSend}. */ export class MockWebSocket extends EventTarget { - static instances: MockWebSocket[] = []; + /** + * The most recently constructed instance, which is all {@linkcode lastMockWebSocket} ever needs. + * + * Deliberately a single reference rather than a list: scenarios that build a transport per + * sample would otherwise pile every socket — and, through the listeners registered on it, every + * transport and its keep-alive timers — into a static array that nothing releases until the + * scenario ends. That turns a per-sample measurement into a function of how many samples have + * already run, and leaves enough retained garbage that whether a major GC lands inside the + * measured window changes the result. + */ + static last: MockWebSocket | undefined; readonly url: string; binaryType: BinaryType = "blob"; @@ -114,7 +124,7 @@ export class MockWebSocket extends EventTarget { constructor(url: string | URL, _protocols?: string | string[]) { super(); this.url = String(url); - MockWebSocket.instances.push(this); + MockWebSocket.last = this; queueMicrotask(() => { if (this.readyState !== 0) return; this.readyState = 1; // OPEN @@ -170,7 +180,7 @@ const OriginalWebSocket: typeof globalThis.WebSocket = globalThis.WebSocket; /** Replaces `globalThis.WebSocket` with {@linkcode MockWebSocket} (picked up by `ReconnectingWebSocket`). */ export function installMockWebSocket(): void { - MockWebSocket.instances = []; + MockWebSocket.last = undefined; // The mock implements only what the transport touches, so it is not structurally a // `WebSocket`; the double assertion is the whole point of installing a stand-in. globalThis.WebSocket = MockWebSocket as unknown as typeof globalThis.WebSocket; @@ -183,7 +193,7 @@ export function restoreWebSocket(): void { /** Returns the most recently created {@linkcode MockWebSocket} (i.e. the one backing the transport). */ export function lastMockWebSocket(): MockWebSocket { - const socket = MockWebSocket.instances.at(-1); + const socket = MockWebSocket.last; if (!socket) throw new Error("No MockWebSocket instance was created"); return socket; } diff --git a/tests/perf/scenarios/user_account_channels.ts b/tests/perf/scenarios/user_account_channels.ts index 904a8600..1ff9a4e2 100644 --- a/tests/perf/scenarios/user_account_channels.ts +++ b/tests/perf/scenarios/user_account_channels.ts @@ -225,7 +225,7 @@ scenario({ "for one user — the calls a feed makes at session start and on every reconnect", unit: "subscription", unitsPerIteration: 3, - iterations: 1, + iterations: 20, samples: 25, warmupSamples: 3, setup: () => {