fix(config): read root config synchronously and bound rename retries by wall clock

components/Scope.ts

@@ -69,6 +69,7 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 	#deployEndHandler: (name: string) => void;
 	#deployInFlight: boolean = false;
 	#restartRequestedDuringDeploy: boolean = false;
+	#optionsReady: boolean = false;
 	applicationScope?: ApplicationScope;
 
 	options: OptionsWatcher;
@@ -253,7 +254,21 @@ export class Scope extends EventEmitter<ScopeEventsMap> {
 		// We could make the user call `await scope.ready()` in their `handleApplication` function, but that could lead to the same issue and it'd
 		// be harder for the user to understand why.
 
+		// A second `ready` means the scope had no config of its own and now does — a config file
+		// that was unreadable when this worker booted, or one recreated after deletion. Re-emitting
+		// reaches nobody: componentLoader is long past its `await scope.ready`, so the component is
+		// running on the defaults until something restarts it. Same recovery as the `remove`
+		// listener, and the same convention — a plugin with its own `ready` handler owns it.
+		const started = this.#optionsReady;
+		this.#optionsReady = true;
+		const restartNeeded = started && this.listenerCount('ready') === 0;
+
 		this.emit('ready');
+
+		if (restartNeeded) {
+			this.#logger.debug?.('Options arrived after the scope started, requesting restart');
+			this.requestRestart();
+		}
 	}
 
 	#handleError(error: unknown): void {

config/RootConfigWatcher.ts

@@ -1,19 +1,30 @@
 import { FSWatcher } from 'chokidar';
-import { readFileSync } from 'node:fs';
 import { getConfigFilePath } from './configUtils.ts';
+import { readConfigFileSync } from './readConfigFileSync.ts';
 import { EventEmitter, once } from 'node:events';
-import { parse } from 'yaml';
+import { parseConfigFile } from './parseConfigFile.ts';
 import {
 	POLLING_FALLBACK_OPTIONS,
-	PartialReadRetry,
 	claimLostNativeWatchError,
 	guardedWatch,
-	isPartialReadError,
 	isWatcherExhaustionError,
 	warnWatcherFallback,
-	warnWatcherListenerError,
 } from '../utility/watcherFallback.ts';
 import { resolveWatchTarget } from '../utility/watchPath.ts';
+import { errorForLog, loggerWithTag } from '../utility/logging/harper_logger.ts';
+import { ConfigReadRetry } from './configReadRetry.ts';
+import { ArmGate } from './watcherArming.ts';
+
+function isMissingFile(error: unknown): boolean {
+	return !!error && typeof error === 'object' && (error as NodeJS.ErrnoException).code === 'ENOENT';
+}
+
+// `harper_logger` imports this module at its own bottom to break their cycle, so a tagged logger
+// built at module scope would run `loggerWithTag()` before `mainLogger` is initialized.
+let taggedLogger: ReturnType<typeof loggerWithTag> | undefined;
+function logger() {
+	return (taggedLogger ??= loggerWithTag('config-watcher'));
+}
 
 export class RootConfigWatcher extends EventEmitter {
 	#configFilePath: string;
@@ -23,30 +34,80 @@ export class RootConfigWatcher extends EventEmitter {
 	#usingPolling: boolean;
 	#closed: boolean;
 	#openCount: number = 0;
-	#partialRead: PartialReadRetry;
+	#readCount: number = 0;
+	#readRetry: ConfigReadRetry = new ConfigReadRetry();
+	#armGate: ArmGate = new ArmGate();
+	// The gate above is chokidar's scan finishing; this is the barrier's gate. A terminal outcome
+	// opens it without claiming the watch is armed, so the arming re-read still runs afterwards.
+	#barrierOpen: boolean = false;
+	#configLoaded: boolean = false;
+	#readyStaged: boolean = false;
+	#readyEmitted: boolean = false;
 	ready: Promise<any[]>;
 
 	constructor() {
 		super();
 		this.#configFilePath = getConfigFilePath();
 		const watchTarget = resolveWatchTarget(this.#configFilePath);
 		this.#watchPath = watchTarget.path;
-		this.#partialRead = new PartialReadRetry(this.#configFilePath);
 		this.#usingPolling = watchTarget.mustPoll;
 		this.#closed = false;
 		this.ready = once(this, 'ready');
 		this.#openWatcher();
 	}
 
 	#openWatcher() {
-		this.#openCount++;
+		const generation = ++this.#openCount;
 		this.#watcher = guardedWatch(this.#watchPath, {
 			persistent: false,
 			...(this.#usingPolling ? POLLING_FALLBACK_OPTIONS : {}),
 		})
 			.on('add', this.handleChange.bind(this))
 			.on('change', this.handleChange.bind(this))
-			.on('error', this.handleError.bind(this));
+			.on('error', this.handleError.bind(this))
+			// Generation-bound: `#armGate.reset()` runs before the failed watcher is closed, so a
+			// `ready` still queued on it would arm the gate on the replacement's behalf and the
+			// replacement's own `ready` would then be a no-op — leaving its scan window unre-read.
+			.on('ready', () => this.#handleArmed(generation));
+	}
+
+	#handleArmed(generation: number) {
+		if (this.#closed || generation !== this.#openCount) return;
+		this.#armGate.arm(() => this.#markArmed());
+	}
+
+	#markArmed() {
+		this.#barrierOpen = true;
+		// A write that landed while the watch was unarmed was never reported, and a scan that found
+		// no file at all reported nothing either, so arming always re-reads rather than publishing
+		// what an earlier read staged — or, with no file, staying pending forever.
+		this.#read(true);
+		// A read that armed the ladder settles `ready` itself, with the newer config.
+		if (!this.#readRetry.pending) this.#emitReady();
+	}
+
+	#emitReady() {
+		if (this.#readyEmitted || !this.#barrierOpen || !this.#readyStaged || this.#closed) return;
+		this.#readyEmitted = true;
+		try {
+			this.emit('ready', this.#config);
+		} catch (error) {
+			logger().warn('A Harper configuration listener failed', errorForLog(error));
+		}
+	}
+
+	// `harper_logger.start()` awaits `ready` with no timeout, so every terminal outcome has to
+	// settle the barrier. A failed read discards an earlier staged value, but a watcher error keeps
+	// it because no later read superseded it. No config is represented by `undefined`, not `{}`:
+	// an empty object is a configuration that turns logging off.
+	#stageBootFallback(discardStaged = true) {
+		if (this.#readyEmitted) return;
+		if (discardStaged) {
+			this.#config = undefined;
+			this.#configLoaded = false;
+		}
+		this.#readyStaged = true;
+		this.#emitReady();
 	}
 
 	// Test-only: simulate the underlying chokidar watcher emitting an error.
@@ -56,17 +117,26 @@ export class RootConfigWatcher extends EventEmitter {
 		this.handleError(error);
 	}
 
-	// Test-only: whether the watcher has fallen back to polling.
 	get _usingPollingForTests(): boolean {
 		return this.#usingPolling;
 	}
 
-	// Test-only: number of times the underlying watcher has been (re)opened.
 	get _openCountForTests(): number {
 		return this.#openCount;
 	}
 
+	// Distinguishes a ladder rung from a watcher event.
+	get _readCountForTests(): number {
+		return this.#readCount;
+	}
+
+	get _armedForTests(): boolean {
+		return this.#armGate.armed;
+	}
+
 	handleError(error: unknown) {
+		// A queued chokidar error can land after close(), which has dropped every listener.
+		if (this.#closed) return;
 		// See EntryHandler.#handleWatcherError: a lost native watch handle is benign
 		// and must not be surfaced to consumers as a config-watch failure.
 		if (claimLostNativeWatchError(error)) return;
@@ -77,6 +147,9 @@ export class RootConfigWatcher extends EventEmitter {
 			if (!this.#usingPolling) {
 				warnWatcherFallback(this.#configFilePath);
 				this.#usingPolling = true;
+				// The generation that just failed no longer speaks for the watch; the replacement
+				// arms on its own scan, and re-reads then as the first one did.
+				this.#armGate.reset();
 				// Start close() from a microtask, not directly here, so a synchronous throw
 				// can't escape this 'error' listener as an uncaught exception.
 				Promise.resolve()
@@ -87,55 +160,136 @@ export class RootConfigWatcher extends EventEmitter {
 					.then(() => {
 						if (!this.#closed) this.#openWatcher();
 					})
-					.catch((error) => console.error(`Could not reopen the ${this.#configFilePath} watch on polling:`, error));
+					.catch((error) =>
+						logger().warn(`Could not reopen the ${this.#configFilePath} watch on polling`, errorForLog(error))
+					);
+			} else {
+				// Already polling — the replacement failed too, or the watch was polling from
+				// construction (`mustPoll`) and never had a fallback to take. Either way the branch
+				// above reopens only once, so this is the watch's terminal outcome and the barrier
+				// has to settle or `harper_logger.start()` awaits it forever.
+				this.#barrierOpen = true;
+				this.#stageBootFallback(false);
 			}
 			return;
 		}
-		this.emit('error', error);
+		// chokidar may never reach its own `ready` after a scan error, and nothing else would
+		// settle the barrier: the error is this read's terminal outcome. The scan is not over
+		// though, so the arm gate stays closed and a later `ready` still takes the arming re-read.
+		this.#barrierOpen = true;
+		this.#stageBootFallback(false);
+		// Settling the barrier removed the `error` listener `once(this, 'ready')` attached, and an
+		// emit with none left throws the error back into chokidar's dispatch — as does a consumer
+		// that throws from its own handler.
+		if (this.listenerCount('error') === 0) {
+			logger().warn(`The Harper configuration watcher at ${this.#configFilePath} failed`, errorForLog(error));
+			return;
+		}
+		try {
+			this.emit('error', error);
+		} catch (listenerError) {
+			logger().warn('A Harper configuration error listener failed', errorForLog(listenerError));
+		}
 	}
 
-	// See the descriptor-lifetime invariant on atomicWriteFile (DESIGN.md).
 	handleChange() {
+		this.#read(true);
+	}
+
+	// `harper_logger.start()` awaits `ready` with no timeout and the ladder may be the only thing
+	// left to settle it, so until then its timer keeps the thread alive rather than letting it
+	// drain and exit mid-boot.
+	#schedule(): boolean {
+		return this.#readRetry.schedule(() => this.#read(false), !this.#readyEmitted);
+	}
+
+	#read(waitForLock: boolean) {
+		// A queued chokidar callback can still land after close(), which has already discarded the
+		// config and dropped every listener.
+		if (this.#closed) return;
+		this.#readCount++;
+		let data: string;
+		try {
+			data = readConfigFileSync(this.#configFilePath, waitForLock);
+		} catch (error) {
+			// A missing file is not a lock — `readConfigFileSync` does not retry it either, and
+			// `OptionsWatcher` settles it immediately as the install window. Taking the ladder here
+			// would disagree with that and cost `harper_logger.start()` the whole budget on every
+			// boot that has no config file (an env-var-only deployment, an empty mounted rootPath).
+			if (!isMissingFile(error) && this.#schedule()) return;
+			// A ladder armed by an earlier empty read is spent by the time a rung lands on ENOENT,
+			// and every other terminal path clears its deadline.
+			this.#readRetry.reset();
+			logger().warn(
+				`Unable to read the Harper configuration file at ${this.#configFilePath}` +
+					(this.#configLoaded ? ', continuing with the previously loaded configuration' : '; none has been loaded yet'),
+				errorForLog(error)
+			);
+			this.#stageBootFallback();
+			return;
+		}
+		// See DESIGN.md, "An empty read is a writer mid-write, not an empty config".
+		if (!data) {
+			if (this.#schedule()) return;
+			logger().warn(`The Harper configuration file at ${this.#configFilePath} is empty`);
+			this.#stageBootFallback();
+			return;
+		}
 		let config;
-		// Only the read and parse are guarded: a listener that throws must not be mistaken for a
-		// half-written file and replayed.
 		try {
-			config = parse(readFileSync(this.#configFilePath, 'utf-8'));
+			config = parseConfigFile(data, this.#configFilePath);
 		} catch (error) {
-			// A missing file needs no re-read; anything else may be the file being replaced.
-			if (isPartialReadError(error)) this.#scheduleReread(error);
+			// A read taken mid-write is untrustworthy, not only an empty one: the writer's first
+			// `write(2)` can land a prefix of the document, and the event carrying the rest is the one
+			// chokidar throttles away. So an unparseable read rides out the same ladder as an empty
+			// one, and only a read that parses releases it.
+			if (this.#schedule()) return;
+			logger().warn((error as Error).message);
+			this.#stageBootFallback();
 			return;
 		}
-		// A snapshot that does not parse to an object is the other shape a half-written file
-		// takes: `''`, `'\n'` and a truncated document all yield null, and adopting that would
-		// hand every consumer a config with nothing in it.
+		// The third shape a mid-write read takes, and the only one that parses: a truncated
+		// document, a lone `\n`, a file that is nothing but comments all yield `null` rather than
+		// throwing, and adopting one hands every consumer a config with nothing in it. Same ladder
+		// as the two above, and past it the file is empty rather than mid-write.
 		if (!config || typeof config !== 'object') {
-			this.#scheduleReread();
+			if (this.#schedule()) return;
+			logger().warn(`The Harper configuration file at ${this.#configFilePath} is empty`);
+			this.#stageBootFallback();
+			return;
+		}
+		this.#readRetry.reset();
+
+		// Before `ready` goes out there is no prior state to have changed *since*.
+		this.#configLoaded = true;
+		this.#readyStaged = true;
+		if (!this.#readyEmitted) {
+			this.#config = config;
+			this.#emitReady();
 			return;
 		}
-		this.#partialRead.settled();
 
 		try {
-			if (!this.#config) {
-				this.#config = config;
-				this.emit('ready', this.#config);
-				return;
-			}
 			this.emit('change', (this.#config = config));
 		} catch (error) {
-			warnWatcherListenerError(this.#configFilePath, error);
+			logger().warn('A Harper configuration change listener failed', errorForLog(error));
 		}
 	}
 
-	#scheduleReread(error?: unknown) {
-		if (this.#partialRead.schedule(() => this.handleChange())) return;
-		this.#partialRead.gaveUp(error);
-	}
-
 	close() {
+		// Closing is a terminal outcome too: leaving `ready` pending would hang anything still
+		// awaiting the barrier. Through `#emitReady`, so a listener that throws cannot skip the
+		// teardown below it and leave the watcher and its arm timer running.
+		this.#barrierOpen = true;
+		this.#readyStaged = true;
+		this.#emitReady();
 		this.#closed = true;
-		this.#partialRead.cancel();
-		this.#watcher.close();
+		this.#readRetry.cancel();
+		this.#armGate.cancel();
+		// chokidar's close() is a promise; an unhandled teardown rejection would reach Node as one,
+		// on the path whose whole job is to stop caring about this watcher. Same shape as the
+		// exhaustion-recovery close above, and as `OptionsWatcher.close`.
+		Promise.resolve(this.#watcher.close()).catch(() => {});
 		this.#config = undefined;
 		this.emit('close');
 		this.removeAllListeners();

config/configReadRetry.ts

@@ -0,0 +1,45 @@
+// Why a lock that outlives the reader's budget is retried from a timer, and why both the bound and
+// the backoff are wall clock rather than an attempt count: see "Root config watchers must read
+// synchronously" in DESIGN.md (harper#2191).
+const RETRY_BUDGET_MS = 3_100;
+const INITIAL_DELAY_MS = 100;
+const MAX_DELAY_MS = 1_600;
+
+export class ConfigReadRetry {
+	#timer?: NodeJS.Timeout;
+	#deadline?: number;
+
+	// `holdEventLoop` is for a caller whose boot barrier this ladder is the only thing left to
+	// settle: an unref'd timer would let the thread drain and exit mid-boot instead.
+	schedule(retry: () => void, holdEventLoop: boolean = false): boolean {
+		this.cancel();
+		const now = performance.now();
+		this.#deadline ??= now + RETRY_BUDGET_MS;
+		const remainingMs = this.#deadline - now;
+		if (remainingMs <= 0) {
+			this.reset();
+			return false;
+		}
+		const elapsedMs = RETRY_BUDGET_MS - remainingMs;
+		const delayMs = Math.min(Math.max(elapsedMs, INITIAL_DELAY_MS), MAX_DELAY_MS, remainingMs);
+		this.#timer = setTimeout(retry, delayMs);
+		if (!holdEventLoop) this.#timer.unref();
+		return true;
+	}
+
+	get pending(): boolean {
+		return this.#timer !== undefined;
+	}
+
+	reset(): void {
+		this.cancel();
+		this.#deadline = undefined;
+	}
+
+	cancel(): void {
+		if (this.#timer) {
+			clearTimeout(this.#timer);
+			this.#timer = undefined;
+		}
+	}
+}