Deflake the unit-test cluster: seed HNSW routing, signal-based waits, quarantine two unreproduced hangs

DESIGN.md

@@ -1094,6 +1094,15 @@ graphs being identical, though equal metrics do not prove it. It is the expected
 the upper layers are sparse enough that a greedy walk reaches the same entry point, which is why
 standard HNSW descends this way.
 
+Greedy-equals-full is statistical, not per-graph: rare level layouts route to a different layer-0
+entry point and displace the tail of the top-k (~2-3% of random 600-node graphs in the unit test's
+corpus). Tests that assert exact result-set equality across search strategies must therefore pin the
+graph: level assignment draws from the instance's `random` property (a test seam defaulting to
+`Math.random`), which the routing test replaces with a seeded PRNG. One pinned graph samples the
+property once, so that test sweeps a fixed list of seeds, each verified non-divergent when the list
+was written — a seed that starts diverging after an intentional index change is a re-pin, not
+necessarily a routing regression.
+
 ## `efConstruction` and the search-`ef` ceiling both auto-scale with the graph
 
 The connection-building pass selects each node's stored edges from a candidate list of

integrationTests/components/risk-query.test.ts

@@ -13,7 +13,12 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
 
 import { startHarper, teardownHarper, sendOperation, type ContextWithHarper } from '@harperfast/integration-testing';
 
-suite('Component: risk-query', (ctx: ContextWithHarper) => {
+// Quarantined on Windows: deploy_component (restart:true) hangs server-side after npm pack, which
+// stalls before() on undici's 300s headers timeout and cancels every child test. See
+// https://github.com/HarperFast/harper/issues/2273 — remove the skip when the deploy hang is fixed.
+const skipSuite = process.platform === 'win32';
+
+suite('Component: risk-query', { skip: skipSuite }, (ctx: ContextWithHarper) => {
 	before(async () => {
 		await startHarper(ctx);
 
@@ -27,11 +32,13 @@ suite('Component: risk-query', (ctx: ContextWithHarper) => {
 		strictEqual(body.message, 'Successfully deployed: risk-query, restarting Harper');
 		ok(typeof body.deployment_id === 'string', `expected deployment_id in deploy response, got ${body.deployment_id}`);
 
-		// Poll until the component is ready
+		// Poll until the component is ready. Each probe carries its own abort timeout: without it, a
+		// connection that opens but never sends headers holds fetch for undici's 300s default and
+		// blows past the deadline check (issue #2273's client-side half).
 		const deadline = Date.now() + 30_000;
 		while (true) {
 			try {
-				const check = await fetch(`${ctx.harper.httpURL}/RisqTable/`);
+				const check = await fetch(`${ctx.harper.httpURL}/RisqTable/`, { signal: AbortSignal.timeout(5_000) });
 				if (check.status === 200) break;
 			} catch {
 				// server not yet accepting connections

resources/indexes/HierarchicalNavigableSmallWorld.ts

@@ -209,6 +209,10 @@ export class HierarchicalNavigableSmallWorld {
 	// a value of 1 is extremely aggressive.
 	optimizeRouting = 0.5;
 	nodesVisitedCount = 0;
+	// Test seam: source of randomness for level assignment. Tests that assert exact result-set
+	// equality across search strategies replace this with a seeded PRNG so the graph shape is
+	// reproducible; greedy routing legitimately diverges on rare unlucky graphs otherwise.
+	random: () => number = Math.random;
 	// Visit-budget multiplier for predicate-aware traversal (#1241). Under-filled filtered searches
 	// stop after the resolved budget ef * filterExpansion visits; automatic search ef contributes at
 	// most AUTO_EF_MAX, while explicit schema/query ef remains authoritative. A filter that fills its
@@ -337,7 +341,7 @@ export class HierarchicalNavigableSmallWorld {
 			const storedScale = q ? q.scale : undefined;
 			let entryPoint = entryPointId && this.safeGetSync(entryPointId, options);
 			if (entryPoint == null) {
-				const level = Math.floor(-Math.log(Math.random()) * this.mL);
+				const level = Math.min(Math.floor(-Math.log(this.random()) * this.mL), MAX_LEVEL);
 				const node = {
 					vector: storedVector,
 					scale: storedScale,
@@ -358,7 +362,7 @@ export class HierarchicalNavigableSmallWorld {
 			}
 
 			// Generate random level for this new element
-			const level = oldNode.level ?? Math.min(Math.floor(-Math.log(Math.random()) * this.mL), MAX_LEVEL);
+			const level = oldNode.level ?? Math.min(Math.floor(-Math.log(this.random()) * this.mL), MAX_LEVEL);
 			let currentLevel = entryPoint.level;
 			if (level > currentLevel) {
 				// if we are at a higher level, make this the new entry point

unitTests/apiTests/mqtt-test.mjs

@@ -890,108 +890,117 @@ describe('test MQTT connections and commands', function () {
 		const granted = await subscribeAllowingSubackError(clientV5, '+/SimpleRecord/test');
 		assert.equal(granted[0].qos, 0x8f); // assert that the subscription was rejected
 	});
-	it('subscribe with QoS=1 and reconnect with non-clean session', async function () {
-		this.timeout(20000); // needs more than the suite-level 10 s on loaded runners
-		// this first connection is a tear down to remove any previous durable session with this id
-		let client = await connectAsync(mqttUrl, {
-			clean: true,
-			clientId: 'test-client1',
-			protocolVersion: 4,
-		});
-		await endDurableSession(client, 'test-client1');
-		client = await connectAsync(mqttUrl, {
-			clean: false,
-			clientId: 'test-client1',
-			protocolVersion: 4,
-		});
-		await client.subscribeAsync(['SimpleRecord/41', 'SimpleRecord/42'], { qos: 1 });
-		await endDurableSession(client, 'test-client1');
-		client = await connectAsync(mqttUrl, {
-			clean: false,
-			clientId: 'test-client1',
-			protocolVersion: 4,
-		});
-		await new Promise((resolve) => {
-			// Wait for the broker to finish processing (and durably persisting) our ack of this
-			// message, not just for the client to have sent it — see `session.acknowledge()`.
-			const acknowledged = waitForMqttSessionEvent('acknowledged', 'test-client1');
-			client.on('message', (topic, payload) => {
-				JSON.parse(payload);
-				resolve(acknowledged);
+	// Quarantined on lmdb only: hung silently to its 20s timeout on CI main (lmdb pass, Node 26)
+	// with no server-side log output; not reproduced in 30 contended local runs. The rocksdb pass
+	// keeps this durable-session coverage. Evidence, hypotheses, and the reinstatement path
+	// (bounded per-step waits) are in https://github.com/HarperFast/harper/issues/2274.
+	(process.env.HARPER_STORAGE_ENGINE === 'lmdb' ? it.skip : it)(
+		'subscribe with QoS=1 and reconnect with non-clean session',
+		async function () {
+			this.timeout(20000); // needs more than the suite-level 10 s on loaded runners
+			// this first connection is a tear down to remove any previous durable session with this id
+			let client = await connectAsync(mqttUrl, {
+				clean: true,
+				clientId: 'test-client1',
+				protocolVersion: 4,
 			});
+			await endDurableSession(client, 'test-client1');
+			client = await connectAsync(mqttUrl, {
+				clean: false,
+				clientId: 'test-client1',
+				protocolVersion: 4,
+			});
+			await client.subscribeAsync(['SimpleRecord/41', 'SimpleRecord/42'], { qos: 1 });
+			await endDurableSession(client, 'test-client1');
+			client = await connectAsync(mqttUrl, {
+				clean: false,
+				clientId: 'test-client1',
+				protocolVersion: 4,
+			});
+			await new Promise((resolve) => {
+				// Wait for the broker to finish processing (and durably persisting) our ack of this
+				// message, not just for the client to have sent it — see `session.acknowledge()`.
+				const acknowledged = waitForMqttSessionEvent('acknowledged', 'test-client1');
+				client.on('message', (topic, payload) => {
+					JSON.parse(payload);
+					resolve(acknowledged);
+				});
 
-			client.publish(
+				client.publish(
+					'SimpleRecord/41',
+					JSON.stringify({
+						name: 'This is a test of durable session with subscriptions restarting',
+					}),
+					{
+						qos: 1,
+					}
+				);
+			});
+			await endDurableSession(client, 'test-client1');
+			await clientV5.publishAsync(
 				'SimpleRecord/41',
 				JSON.stringify({
-					name: 'This is a test of durable session with subscriptions restarting',
+					name: 'This is a test of publishing to a disconnected durable session',
 				}),
 				{
 					qos: 1,
 				}
 			);
-		});
-		await endDurableSession(client, 'test-client1');
-		await clientV5.publishAsync(
-			'SimpleRecord/41',
-			JSON.stringify({
-				name: 'This is a test of publishing to a disconnected durable session',
-			}),
-			{
-				qos: 1,
-			}
-		);
-		await clientV5.publishAsync(
-			'SimpleRecord/42',
-			JSON.stringify({
-				name: 'This is a test of publishing to a disconnected durable session 2',
-			}),
-			{
-				qos: 1,
-			}
-		);
-		await clientV5.publishAsync(
-			'SimpleRecord/42',
-			JSON.stringify({
-				name: 'This is a test of publishing to a disconnected durable session 3',
-			}),
-			{
-				qos: 1,
-			}
-		);
-		let messages = [];
-		client = await connectWithMessageListener(
-			mqttUrl,
-			{
-				clean: false,
-				clientId: 'test-client1',
-				protocolVersion: 5,
-				properties: {
-					sessionExpiryInterval: 3600,
+			await clientV5.publishAsync(
+				'SimpleRecord/42',
+				JSON.stringify({
+					name: 'This is a test of publishing to a disconnected durable session 2',
+				}),
+				{
+					qos: 1,
+				}
+			);
+			await clientV5.publishAsync(
+				'SimpleRecord/42',
+				JSON.stringify({
+					name: 'This is a test of publishing to a disconnected durable session 3',
+				}),
+				{
+					qos: 1,
+				}
+			);
+			let messages = [];
+			client = await connectWithMessageListener(
+				mqttUrl,
+				{
+					clean: false,
+					clientId: 'test-client1',
+					protocolVersion: 5,
+					properties: {
+						sessionExpiryInterval: 3600,
+					},
 				},
-			},
-			(topic, message) => {
-				messages.push(message.toString());
-			}
-		);
-		await new Promise((resolve, reject) => {
-			const interval = setInterval(() => {
-				if (messages.length === 3) {
-					clearInterval(interval);
-					resolve();
+				(topic, message) => {
+					messages.push(message.toString());
 				}
-			}, 1);
-			setTimeout(() => {
-				clearInterval(interval);
-				reject(
-					new Error(`Expected 3 queued messages to be delivered to reconnected durable session, got ${messages.length}`)
-				);
-			}, 15000);
-		});
-		await delay(50);
-		await client.endAsync();
-		if (messages.length !== 3) console.error('Incorrect messages', { messages });
-		assert(messages.length === 3);
-	});
+			);
+			await new Promise((resolve, reject) => {
+				const interval = setInterval(() => {
+					if (messages.length === 3) {
+						clearInterval(interval);
+						resolve();
+					}
+				}, 1);
+				setTimeout(() => {
+					clearInterval(interval);
+					reject(
+						new Error(
+							`Expected 3 queued messages to be delivered to reconnected durable session, got ${messages.length}`
+						)
+					);
+				}, 15000);
+			});
+			await delay(50);
+			await client.endAsync();
+			if (messages.length !== 3) console.error('Incorrect messages', { messages });
+			assert(messages.length === 3);
+		}
+	);
 	it('subscribe with QoS=2', async function () {
 		// this first connection is a tear down to remove any previous durable session with this id
 		let client = await connectAsync(mqttUrl, {