From b247d266e9aab7799a9c506ff064948104bc8429 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 27 Aug 2026 00:37:50 -0700 Subject: [PATCH 1/3] Attach a diagnosis to the dev-db socket wedge timeouts --- .../db/dev-db-socket-concurrency.node.test.ts | 56 ++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/apps/cloud/src/db/dev-db-socket-concurrency.node.test.ts b/apps/cloud/src/db/dev-db-socket-concurrency.node.test.ts index 980e5edb32..04494e4f1f 100644 --- a/apps/cloud/src/db/dev-db-socket-concurrency.node.test.ts +++ b/apps/cloud/src/db/dev-db-socket-concurrency.node.test.ts @@ -67,6 +67,54 @@ const openWireClient = async (port: number): Promise => { return socket; }; +/** + * Run a bystander's query with a bounded deadline and, on the deadline, fail + * with the server's internals instead of vitest's bare 30s timeout. + * + * The reap/ghost scenarios have each wedged ONCE in CI (runs 32933818134 and + * 33019527020: the bystander's startup was served, then its query hung until + * the test timeout) while ~800 replays of the isolated scenarios on macOS and + * Linux, idle and CPU-starved, never reproduced it. Until it fires again there + * is nothing to fix, so make the next occurrence carry its own diagnosis: + * the queue/handler stats at wedge time, plus whether a FRESH connection still + * completes startup (a latched queue serves nobody; per-handler affinity + * pinning still answers new startups). + */ +const diagnoseWedge = async ( + run: () => Promise, + context: { readonly server: PGLiteSocketServer; readonly port: number }, + deadlineMs = 20_000, +): Promise => { + let timer: NodeJS.Timeout | undefined; + const deadline = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + void (async () => { + const stats = JSON.stringify(context.server.getStats()); + // oxlint-disable-next-line executor/no-promise-catch -- test boundary: the probe outcome is diagnostic text, never a failure path + const freshStartup = await Promise.race([ + openWireClient(context.port).then((socket) => { + socket.destroy(); + return "completes"; + }), + sleep(3_000).then(() => "hangs"), + ]).catch(() => "errors"); + // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- test boundary: adapt the deadline to the assertion path with the diagnosis attached + reject( + new Error( + `bystander wedged for ${deadlineMs}ms; server stats=${stats}; fresh startup ${freshStartup}`, + ), + ); + })(); + }, deadlineMs); + }); + // oxlint-disable-next-line executor/no-try-catch-or-throw -- test boundary: the deadline timer must be cleared on every path + try { + return await Promise.race([run(), deadline]); + } finally { + clearTimeout(timer); + } +}; + // A Parse frame for an unnamed statement: opens an extended-protocol pipeline // that only a later Sync (or the server's recovery) closes. const parseFrame = (query: string): Buffer => { @@ -248,7 +296,9 @@ describe("dev-db PGlite socket under concurrent connections", () => { // oxlint-disable-next-line executor/no-try-catch-or-throw -- test boundary: sockets must be closed on every path try { // Connects and queries only once the staller is reaped (~250ms). - expect((await bystander.unsafe(`select 4 as four`))[0]).toEqual({ four: 4 }); + expect( + (await diagnoseWedge(() => bystander.unsafe(`select 4 as four`), { server, port }))[0], + ).toEqual({ four: 4 }); } finally { // oxlint-disable-next-line executor/no-promise-catch -- test boundary: a failed teardown must not mask the assertion await bystander.end({ timeout: 5 }).catch(() => {}); @@ -346,7 +396,9 @@ describe("dev-db PGlite socket under concurrent connections", () => { const bystander = makeClient(port); // oxlint-disable-next-line executor/no-try-catch-or-throw -- test boundary: sockets must be closed on every path try { - expect((await bystander.unsafe(`select 5 as five`))[0]).toEqual({ five: 5 }); + expect( + (await diagnoseWedge(() => bystander.unsafe(`select 5 as five`), { server, port }))[0], + ).toEqual({ five: 5 }); } finally { // oxlint-disable-next-line executor/no-promise-catch -- test boundary: a failed teardown must not mask the assertion await bystander.end({ timeout: 5 }).catch(() => {}); From 4cd58d4304839e600389a765b6b19e76d3f6ad2d Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 27 Aug 2026 05:39:39 -0700 Subject: [PATCH 2/3] Scope the lint suppressions to their lines --- apps/cloud/src/db/dev-db-socket-concurrency.node.test.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/cloud/src/db/dev-db-socket-concurrency.node.test.ts b/apps/cloud/src/db/dev-db-socket-concurrency.node.test.ts index 04494e4f1f..3a2d5d1675 100644 --- a/apps/cloud/src/db/dev-db-socket-concurrency.node.test.ts +++ b/apps/cloud/src/db/dev-db-socket-concurrency.node.test.ts @@ -98,8 +98,9 @@ const diagnoseWedge = async ( }), sleep(3_000).then(() => "hangs"), ]).catch(() => "errors"); - // oxlint-disable-next-line executor/no-promise-reject, executor/no-error-constructor -- test boundary: adapt the deadline to the assertion path with the diagnosis attached + // oxlint-disable-next-line executor/no-promise-reject -- test boundary: adapt the deadline to the assertion path with the diagnosis attached reject( + // oxlint-disable-next-line executor/no-error-constructor -- test boundary: the diagnosis rides the assertion failure new Error( `bystander wedged for ${deadlineMs}ms; server stats=${stats}; fresh startup ${freshStartup}`, ), From e3ff346283376f0a8863a752e5c3a832669dbce6 Mon Sep 17 00:00:00 2001 From: Rhys Sullivan <39114868+RhysSullivan@users.noreply.github.com> Date: Thu, 27 Aug 2026 20:11:09 -0700 Subject: [PATCH 3/3] Heartbeat the dev-db watchdog with connection stats --- apps/cloud/scripts/dev-db.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/apps/cloud/scripts/dev-db.ts b/apps/cloud/scripts/dev-db.ts index ec671a4d94..5b8ac7b91a 100644 --- a/apps/cloud/scripts/dev-db.ts +++ b/apps/cloud/scripts/dev-db.ts @@ -209,12 +209,29 @@ const probe = async (): Promise => { const watchdog = async () => { let consecutiveFailures = 0; let restarts = 0; + // Heartbeat: the CI e2e cascade of 2026-08-28 (run 33129376530) showed the + // app starving on CONNECT_TIMEOUT for 100+ seconds while every watchdog + // probe silently PASSED — the stall was on the app's side of the socket, + // not this server's. A silent-when-healthy watchdog cannot distinguish + // "healthy" from "not running", and it discards the one signal that would + // test the leading theory (workerd leaking dev-db connections until its + // socket layer starves): the active connection count over time. Log stats + // periodically and whenever the count jumps a bucket. + let lastHeartbeatAt = Date.now(); + let lastLoggedBucket = 0; for (;;) { await sleep(WATCHDOG_INTERVAL_MS); if (stopping) return; try { await probe(); consecutiveFailures = 0; + const stats = server.getStats(); + const bucket = Math.floor(stats.activeConnections / 50); + if (bucket !== lastLoggedBucket || Date.now() - lastHeartbeatAt >= 60_000) { + lastLoggedBucket = bucket; + lastHeartbeatAt = Date.now(); + console.log(`[dev-db][watchdog] healthy; stats: ${JSON.stringify(stats)}`); + } } catch (cause) { consecutiveFailures += 1; console.error(