Fix the dev-db socket test wedge: bind conflicts hung server.start()

apps/cloud/scripts/test-globalsetup.ts

@@ -13,12 +13,17 @@ import { fileURLToPath } from "node:url";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
+// 0 asks the OS for a free port — used by the globalsetup-exit fixture, whose
+// test never connects to the database. A suite whose tests DO connect (the
+// default 5434 path, matched by DATABASE_URL in vitest.config.ts) must pass a
+// real port. Fixed ports must stay below 32768: the Linux ephemeral range
+// (32768-60999) is contested by every concurrent suite's outbound sockets.
 const parsePort = (input: string | undefined): number => {
   if (input === undefined) return 5434;
   if (!/^\d+$/.test(input)) throw new Error("CLOUD_TEST_DB_PORT must be an integer");
   const port = Number(input);
-  if (!Number.isSafeInteger(port) || port < 1 || port > 65_535) {
-    throw new Error("CLOUD_TEST_DB_PORT must be between 1 and 65535");
+  if (!Number.isSafeInteger(port) || port > 65_535) {
+    throw new Error("CLOUD_TEST_DB_PORT must be between 0 and 65535");
   }
   return port;
 };

apps/cloud/src/db/dev-db-socket-concurrency.node.test.ts

@@ -22,16 +22,42 @@
 // one PGLiteSocketServer and asserts zero protocol corruption.
 
 import { setTimeout as sleep } from "node:timers/promises";
-import { connect, type Socket } from "node:net";
+import { connect, createServer, type Socket } from "node:net";
 import { describe, expect, it } from "@effect/vitest";
 import { PGlite } from "@electric-sql/pglite";
 import { PGLiteSocketServer } from "@electric-sql/pglite-socket";
 import postgres from "postgres";
 
-const PORT = 45998;
 const CLIENTS = 6;
 const QUERIES_PER_CLIENT = 40;
 
+/**
+ * Start a server bound to an OS-assigned port and return that port.
+ *
+ * Every server in this file binds port 0. The fixed ports this file used to
+ * bind (45993-45998) sit inside the default Linux ephemeral port range
+ * (32768-60999): on a busy CI runner any other socket — an outbound connection
+ * from a sibling suite in the same turbo shard, or a leaked e2e server, which
+ * squat exactly this block — can hold one of them at bind time. The stock
+ * server then swallowed the EADDRINUSE (start() rejected only when `active`
+ * was false, and start() sets `active` true before listen), so
+ * `await server.start()` never settled and the test died as a bare vitest
+ * timeout with zero diagnostics — the CI signature of every wedge in this
+ * family. macOS assigns ephemeral ports from 49152, which is why hundreds of
+ * local replays never reproduced it.
+ */
+const startOnOsPort = async (server: PGLiteSocketServer): Promise<number> => {
+  const listening = new Promise<number>((resolve) => {
+    server.addEventListener(
+      "listening",
+      (event) => resolve((event as CustomEvent<{ readonly port: number }>).detail.port),
+      { once: true },
+    );
+  });
+  await server.start();
+  return await listening;
+};
+
 const makeClient = (port: number, connectTimeout = 5) =>
   postgres(`postgres://postgres:postgres@127.0.0.1:${port}/postgres`, {
     max: 1,
@@ -71,14 +97,16 @@ const openWireClient = async (port: number): Promise<Socket> => {
  * Run a bystander's query with a bounded deadline and, on the deadline, fail
  * with the server's internals instead of vitest's bare 30s timeout.
  *
- * The reap/ghost scenarios have each wedged ONCE in CI (runs 32933818134 and
- * 33019527020: the bystander's startup was served, then its query hung until
- * the test timeout) while ~800 replays of the isolated scenarios on macOS and
- * Linux, idle and CPU-starved, never reproduced it. Until it fires again there
- * is nothing to fix, so make the next occurrence carry its own diagnosis:
- * the queue/handler stats at wedge time, plus whether a FRESH connection still
- * completes startup (a latched queue serves nobody; per-handler affinity
- * pinning still answers new startups).
+ * The reap/ghost scenarios each wedged in CI (runs 32933818134 and
+ * 33019527020) while ~800 replays of the isolated scenarios on macOS and
+ * Linux, idle and CPU-starved, never reproduced it. Those bare timeouts are
+ * now attributed: a bind conflict on this file's old fixed ephemeral-range
+ * ports left `server.start()` pending forever (see startOnOsPort), which a
+ * bare vitest timeout cannot distinguish from a queue wedge. The wrapper
+ * stays so that any FUTURE wedge that really is in the queue carries its own
+ * diagnosis: the queue/handler stats at wedge time, plus whether a FRESH
+ * connection still completes startup (a latched queue serves nobody;
+ * per-handler affinity pinning still answers new startups).
  */
 const diagnoseWedge = async <T>(
   run: () => Promise<T>,
@@ -133,17 +161,17 @@ describe("dev-db PGlite socket under concurrent connections", () => {
       const db = await PGlite.create();
       const server = new PGLiteSocketServer({
         db,
-        port: PORT,
+        port: 0,
         host: "127.0.0.1",
         maxConnections: 100,
       });
-      await server.start();
+      const port = await startOnOsPort(server);
 
       let ok = 0;
       const errors: string[] = [];
 
       const worker = async (id: number) => {
-        const sql = makeClient(PORT, 10);
+        const sql = makeClient(port, 10);
         // oxlint-disable-next-line executor/no-try-catch-or-throw -- test boundary: postgres.js is promise-native and the socket must be closed on every path
         try {
           for (let q = 0; q < QUERIES_PER_CLIENT; q++) {
@@ -192,10 +220,14 @@ describe("dev-db PGlite socket under concurrent connections", () => {
     "a rejected query fails one client, not the whole socket server",
     { timeout: 30_000 },
     async () => {
-      const port = 45997;
       const db = await PGlite.create();
-      const server = new PGLiteSocketServer({ db, port, host: "127.0.0.1", maxConnections: 100 });
-      await server.start();
+      const server = new PGLiteSocketServer({
+        db,
+        port: 0,
+        host: "127.0.0.1",
+        maxConnections: 100,
+      });
+      const port = await startOnOsPort(server);
 
       const first = makeClient(port);
       // oxlint-disable-next-line executor/no-try-catch-or-throw -- test boundary: sockets must be closed on every path
@@ -243,16 +275,15 @@ describe("dev-db PGlite socket under concurrent connections", () => {
   // only fires on a connection that is actually blocking the shared session —
   // an open pipeline or an open transaction.
   it("an idle-at-rest connection outlives the idle backstop", { timeout: 30_000 }, async () => {
-    const port = 45996;
     const db = await PGlite.create();
     const server = new PGLiteSocketServer({
       db,
-      port,
+      port: 0,
       host: "127.0.0.1",
       maxConnections: 100,
       idleTimeout: 250,
     });
-    await server.start();
+    const port = await startOnOsPort(server);
 
     const sql = makeClient(port);
     // oxlint-disable-next-line executor/no-try-catch-or-throw -- test boundary: sockets must be closed on every path
@@ -276,16 +307,15 @@ describe("dev-db PGlite socket under concurrent connections", () => {
     "a client stalled mid-pipeline is reaped and the queue recovers",
     { timeout: 30_000 },
     async () => {
-      const port = 45995;
       const db = await PGlite.create();
       const server = new PGLiteSocketServer({
         db,
-        port,
+        port: 0,
         host: "127.0.0.1",
         maxConnections: 100,
         idleTimeout: 250,
       });
-      await server.start();
+      const port = await startOnOsPort(server);
 
       // Hand-rolled wire client: complete the trust-auth startup, then send a
       // lone Parse. Its last frame type ('P') marks the pipeline open, so the
@@ -320,16 +350,15 @@ describe("dev-db PGlite socket under concurrent connections", () => {
   // as the same CONNECT_TIMEOUT cascade as the queue wedges. The server now
   // drops the handler when it dispatches its terminal error.
   it("reaped handlers release their connection slots", { timeout: 30_000 }, async () => {
-    const port = 45993;
     const db = await PGlite.create();
     const server = new PGLiteSocketServer({
       db,
-      port,
+      port: 0,
       host: "127.0.0.1",
       maxConnections: 2,
       idleTimeout: 250,
     });
-    await server.start();
+    const port = await startOnOsPort(server);
 
     // oxlint-disable-next-line executor/no-try-catch-or-throw -- test boundary: sockets must be closed on every path
     try {
@@ -372,7 +401,6 @@ describe("dev-db PGlite socket under concurrent connections", () => {
     "a client that dies mid-execution does not leave the queue pinned to its ghost",
     { timeout: 30_000 },
     async () => {
-      const port = 45994;
       const db = await PGlite.create();
 
       // Hold the marker query in flight long enough that the disconnect below
@@ -384,8 +412,13 @@ describe("dev-db PGlite socket under concurrent connections", () => {
         return real(...args);
       };
 
-      const server = new PGLiteSocketServer({ db, port, host: "127.0.0.1", maxConnections: 100 });
-      await server.start();
+      const server = new PGLiteSocketServer({
+        db,
+        port: 0,
+        host: "127.0.0.1",
+        maxConnections: 100,
+      });
+      const port = await startOnOsPort(server);
 
       const ghost = await openWireClient(port);
       ghost.write(parseFrame("select 'ghost_marker'"));
@@ -408,4 +441,38 @@ describe("dev-db PGlite socket under concurrent connections", () => {
       }
     },
   );
+
+  // Regression for the CI wedge behind every bare-timeout flake in this file:
+  // stock pglite-socket start() only rejected its promise while `active` was
+  // false, but start() sets `active` true BEFORE listen(), so a bind failure
+  // (EADDRINUSE — this file used to bind fixed ports inside the Linux
+  // ephemeral range, where any concurrent suite's outbound socket or a leaked
+  // e2e server can sit) dispatched an 'error' event nobody listened to and
+  // left `await server.start()` pending forever. The test then died as a raw
+  // vitest timeout with zero diagnostics. The patch rejects start() on server
+  // errors; a settled promise ignores later rejects, so post-listen errors
+  // still only surface through the 'error' event.
+  it(
+    "a bind conflict rejects start() instead of hanging forever",
+    { timeout: 30_000 },
+    async () => {
+      const squatter = createServer();
+      await new Promise<void>((resolve) => squatter.listen(0, "127.0.0.1", () => resolve()));
+      const address = squatter.address();
+      if (address === null || typeof address !== "object") {
+        expect.unreachable("squatter listener has no bound address");
+      }
+
+      const db = await PGlite.create();
+      const server = new PGLiteSocketServer({ db, port: address.port, host: "127.0.0.1" });
+      // oxlint-disable-next-line executor/no-try-catch-or-throw -- test boundary: the squatter and PGlite must be released on every path
+      try {
+        await expect(server.start()).rejects.toThrow(/EADDRINUSE/);
+      } finally {
+        await server.stop();
+        await db.close();
+        await new Promise<void>((resolve) => squatter.close(() => resolve()));
+      }
+    },
+  );
 });

apps/cloud/src/test-globalsetup-exit.node.test.ts

@@ -7,14 +7,19 @@ const appRoot = resolve(dirname(fileURLToPath(import.meta.url)), "..");
 const vitestBin = resolve(appRoot, "../../node_modules/vitest/vitest.mjs");
 const fixtureConfig = resolve(appRoot, "test-fixtures/test-globalsetup-exit/vitest.config.ts");
 
-const runFixture = (port: number, shouldPass: boolean) =>
+// The nested globalsetup binds an OS-assigned port (0): the fixture test never
+// connects to the database, and the fixed ports this file used to pass
+// (45435/45436) sat inside the Linux ephemeral range, where a concurrent
+// suite's outbound socket could hold them — the nested vitest then hung on the
+// swallowed bind failure until spawnSync's timeout killed it (signal !== null).
+const runFixture = (shouldPass: boolean) =>
   spawnSync(process.execPath, [vitestBin, "run", "--config", fixtureConfig], {
     cwd: appRoot,
     encoding: "utf8",
     timeout: 60_000,
     env: {
       ...process.env,
-      CLOUD_TEST_DB_PORT: String(port),
+      CLOUD_TEST_DB_PORT: "0",
       TEST_GLOBALSETUP_SHOULD_PASS: String(shouldPass),
     },
   });
@@ -24,15 +29,15 @@ const diagnostic = (result: ReturnType<typeof runFixture>): string =>
 
 describe("cloud test global setup", () => {
   it("does not let PGlite teardown turn a passed test red", { timeout: 60_000 }, () => {
-    const result = runFixture(45_435, true);
+    const result = runFixture(true);
 
     expect(result.error).toBeUndefined();
     expect(result.signal).toBeNull();
     expect(result.status, diagnostic(result)).toBe(0);
   });
 
   it("does not let PGlite teardown turn a failed test green", { timeout: 60_000 }, () => {
-    const result = runFixture(45_436, false);
+    const result = runFixture(false);
 
     expect(result.error).toBeUndefined();
     expect(result.signal).toBeNull();