From 19709dcc8055040c080b3a2b748811c1ca7e32d3 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 17 Jul 2026 15:20:24 -0400 Subject: [PATCH 01/94] feat(deploy): two-phase stage/activate for deploy_component MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split deploy_component into two replicated phases so a cluster deploy is all-or-nothing at go-live, and expose each phase as a first-class operation. - stage_component (phase 1): build the incoming version — download/npm pack (incl. git clone), extract, npm install — into a hidden `.deploy-staging` dir on every node. Never touches the live component dir, writes no config, never restarts, so it's safe to run cluster-wide and gate on. - activate_component (phase 2): atomically rename the staged copy into the live path and restart; persist root config for a `package` deploy at go-live. - deploy_component orchestrates stage -> (barrier: every node staged OK) -> activate. Request/response contract unchanged; SSE now emits stage/activate phases. `two_phase: false` forces the legacy one-shot path, preserved verbatim for opt-out, for peers replaying a one-shot deploy, and when `system` isn't replicated. Application.ts gains stageApplication/activateApplication/discardStagedApplication; extract/install now build into `buildDirPath` (defaults to the live dir, so the one-shot path, boot install, and direct extractApplication callers are unchanged). Staging lives under the components root — same filesystem — so go-live is an atomic rename (an os.tmpdir() location risks EXDEV and a slow copy at the worst moment). Adds unit tests for the stage/activate/discard primitives and the new validators; DESIGN.md documents the model and the atomic-rename/filesystem tradeoff. Co-Authored-By: Claude Opus 4.8 --- DESIGN.md | 43 + components/Application.ts | 323 +++++-- components/deploymentRecorder.ts | 12 +- components/operations.js | 846 +++++++++++++----- components/operationsValidation.js | 145 ++- server/serverHelpers/serverHandlers.js | 2 + server/serverHelpers/serverUtilities.ts | 8 + .../components/deployPhaseValidators.test.js | 86 ++ unitTests/components/deployStaging.test.js | 171 ++++ .../server/fastifyRoutes/operations.test.js | 41 +- utility/hdbTerms.ts | 6 + utility/operation_authorization.ts | 2 + 12 files changed, 1338 insertions(+), 347 deletions(-) create mode 100644 unitTests/components/deployPhaseValidators.test.js create mode 100644 unitTests/components/deployStaging.test.js diff --git a/DESIGN.md b/DESIGN.md index e26b359ff9..37fef08b92 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -305,3 +305,46 @@ guarded against (in the scan or in tar-fs's own pack walk); that's a pre-existin this fix doesn't attempt to solve. `deploy_component`/`package_component` still never validate that declared entry points (`jsResource`/`graphqlSchema`) survived extraction — a truncation from some other future cause would still report success silently; that's a deferred, separate fix. + +## Two-phase deploy: stage then activate (`components/Application.ts`, `components/operations.js`) + +`deploy_component` splits into two replicated phases so a cluster deploy is all-or-nothing at the +point of go-live. **Phase 1 (`stage_component`)** builds the incoming version — download/`npm pack` +(incl. a git clone), extract, `npm install` — into a hidden staging directory on every node. +**Phase 2 (`activate_component`)** atomically renames the staged copy into the live component path and +restarts. `deploy_component` orchestrates the two: it stages on the origin, replicates +`stage_component` to peers, **waits for every node to report a successful stage before any node +activates** (`ignore_replication_errors` opts out of the barrier), then replicates +`activate_component`. If a node can't fetch the package or fails `npm install`, it fails during +staging while the live component is still untouched _on every node_ — where the old one-shot path +could leave a peer half-installed after other peers had already restarted onto the new code. The +request/response contract is unchanged; only the SSE phase names differ (`stage`/`activate` vs the +old `prepare`/`replicate`). `two_phase: false` forces the legacy one-shot path. + +The staging directory (`.deploy-staging//`) lives **under the components root**, +not in `os.tmpdir()`, even though its contents are transient. This is deliberate and load-bearing: +the go-live step is `rename(stagingDir, liveDir)`, which is only atomic when both paths share a +filesystem. `os.tmpdir()` is frequently a different mount (tmpfs, a separate volume); a cross-device +rename throws `EXDEV` and Node has no atomic fallback — you'd be back to a slow recursive copy at the +exact moment you want the swap to be instantaneous, reintroducing the downtime window the split +exists to remove. The leading dot keeps `loadComponentDirectories` from loading it as a phantom +component, and it is **not** the watched base of any component's file watcher (those are rooted at +each live component dir, `EntryHandler`/`deriveCommonPatternBase`) — so building here fires no +restart-on-change events and needs no `deploy:start` watcher suppression. That suppression is now +scoped to `activateApplication`, the only phase that writes the live path. Staging is deterministic +from the deployment id precisely so `activate_component` (a separate replicated operation, and on +peers a separate invocation from `stage_component`) can reconstruct the same path the stage built — +peers build a fresh `Application` per sub-operation, so there is no shared in-memory handle to rely +on. `extractApplication`/`installApplication` build into `application.buildDirPath`, which defaults +to the live dir (`dirPath`) — this is what keeps the legacy one-shot path, boot-time +`installApplications`, and the direct `extractApplication` callers unchanged — and is repointed at +the staging dir only for the duration of a stage. + +Two-phase requires the `system` database to be replicated on the origin (`isSystemDatabaseReplicated`), +since the `hdb_deployment` row's `payload_blob` is how peers fetch the tarball and correlate the two +phases by deployment id. When `system` is excluded from a narrow `REPLICATION_DATABASES`, or the +caller passes `two_phase: false`, or the invocation is a peer replaying a one-shot deploy, +`deploy_component` falls back to `deployComponentOneShot` (the previous behavior, preserved verbatim). +Known gap for a rolling upgrade window: an origin on this version replicating `stage_component` to a +peer that predates these operations will see that peer fail the op; `two_phase: false` or +`ignore_replication_errors` is the escape hatch until capability negotiation lands. diff --git a/components/Application.ts b/components/Application.ts index 9e3e55297a..d5de3bfe69 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -401,10 +401,64 @@ async function packGitReferenceWithoutScripts( } // Hidden directory under the components root holding component versions renamed aside -// during a deploy swap (see extractApplication). The leading dot keeps +// during a deploy swap (see activateApplication). The leading dot keeps // loadComponentDirectories from loading its contents as components. export const ASIDE_STAGING_DIR = '.deploy-aside'; +// Hidden directory under the components root where the INCOMING version of a component is +// fully built (extracted + `npm install`) before it goes live — the counterpart to +// ASIDE_STAGING_DIR, which holds the OUTGOING version. Two-phase deploy stages here first +// (stage_component), then activateApplication renames the staged copy into the live +// component path in one atomic step (activate_component). +// +// It lives UNDER the components root on purpose, even though the bytes are "temporary": +// - Same filesystem as the live path, so the go-live rename() is atomic. An os.tmpdir() +// location is frequently a different mount (tmpfs / separate volume); a cross-device +// rename throws EXDEV and degrades to a slow recursive copy — reintroducing the very +// downtime window the two-phase split exists to remove. +// - The leading dot keeps loadComponentDirectories (componentLoader) from loading it as a +// phantom component, and it is not the watched base of any component's file watcher +// (those are rooted at each live component dir), so building here triggers no +// restart-on-change storm and needs no deploy:start watcher suppression. +export const DEPLOY_STAGING_DIR = '.deploy-staging'; + +/** + * Atomically move `targetDirPath` aside into a hidden, per-component staging directory if it + * exists, returning the aside staging directory (for best-effort cleanup) or null when there was + * nothing to move. + * + * Renaming the old directory aside — instead of clearing it in place — is immune to the race where + * a still-running worker keeps writing into the directory (e.g. a live Next.js app writing into + * `.next/cache`): an in-place recursive rm races that writer and fails with ENOTEMPTY, whereas the + * rename is atomic and the old worker harmlessly keeps writing into the renamed inode until it is + * replaced on restart. The aside lives on the same filesystem (sibling hidden dir under the same + * parent) so the rename stays atomic, and is per-component so a sibling deploy never collides with + * or sweeps another's aside. + */ +async function moveDirAside(targetDirPath: string): Promise { + const asideStagingDir = join(dirname(targetDirPath), ASIDE_STAGING_DIR, basename(targetDirPath)); + try { + await access(targetDirPath, constants.F_OK); + } catch (err) { + if (err.code === 'ENOENT') return null; // nothing there to move + throw err; + } + await mkdir(asideStagingDir, { recursive: true }); + await rename(targetDirPath, join(asideStagingDir, `${process.pid}-${Date.now()}-${randomUUID()}`)); + return asideStagingDir; +} + +// Best-effort removal of a per-component aside staging directory. The old worker may still hold +// files open in a renamed copy (the live writer that motivated the rename), so a failure here is +// expected in that case and logged at trace rather than as a warning — the survivor is swept by +// the next deploy. +function cleanupAsideDir(asideStagingDir: string | null, componentName: string): void { + if (!asideStagingDir) return; + rm(asideStagingDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }).catch((err) => + logger.trace?.(`Deferred cleanup of previous ${componentName} component directory: ${err.message}`) + ); +} + // The credential helper git executes for a private git-reference deploy. It ships alongside this // module (both in source and in dist), holds no secret, and is inert without a live session. export const GIT_CREDENTIAL_HELPER_PATH = join(__dirname, 'gitCredentialHelper.js'); @@ -414,7 +468,10 @@ export const GIT_CREDENTIAL_HELPER_PATH = join(__dirname, 'gitCredentialHelper.j * * Only one of `application.payload` or `application.package` should be specified; otherwise, an error is thrown. * - * Writes the application to the configured components root directory using the `application.name` and overwrites any existing directory. + * Writes the application into `application.buildDirPath`, overwriting any existing directory there. + * By default that is the live component directory (`application.dirPath`); during a two-phase deploy + * `stageApplication` points it at the hidden staging directory instead, so the live path is never + * touched until `activateApplication` swaps the staged copy into place. * * This method should only be called from the main thread */ @@ -447,8 +504,10 @@ export async function extractApplication(application: Application) { tarball = Readable.from(payload as Buffer); } } else { - // Given a package, there are a a couple options - const parentDirPath = dirname(application.dirPath); + // Given a package, there are a a couple options. The tarball is packed next to the build + // target (the staging dir during a two-phase deploy) so it lands on the same filesystem and + // is swept with the staging area rather than littering the live components root. + const parentDirPath = dirname(application.buildDirPath); // If the package identifier is a file path we need to check if its a tarball or a directory if (application.packageIdentifier.startsWith('file:')) { @@ -458,8 +517,11 @@ export async function extractApplication(application: Application) { const stats = await stat(packagePath); if (stats.isDirectory()) { - // If its a directory, symlink - await symlink(packagePath, application.dirPath, 'dir'); + // If its a directory, symlink. A stale build target (e.g. a retried stage) would + // make symlink() throw EEXIST, so clear it first. + await rm(application.buildDirPath, { recursive: true, force: true }); + await mkdir(dirname(application.buildDirPath), { recursive: true }); + await symlink(packagePath, application.buildDirPath, 'dir'); // And return early since we're done; no extraction needed return; } @@ -553,56 +615,38 @@ export async function extractApplication(application: Application) { } } - // Replace any existing component directory atomically instead of clearing it in - // place. A previous version's worker can still be running and actively writing - // into this directory — e.g. a live Next.js app writing into `.next/cache` — and - // an in-place recursive rm races that writer: rm empties `.next`, then its leaf - // `rmdir('.next')` fails with ENOTEMPTY because the worker just re-created a cache - // entry. (`force: true` only suppresses ENOENT; ENOTEMPTY is not retried unless - // `maxRetries` is set, and a continuously-writing app would outlast retries - // anyway.) Renaming the old directory aside is atomic and immune to the race: the - // still-running worker keeps writing into the renamed inode harmlessly until it's - // replaced on restart, and the aside copy is removed best-effort below. - // - // The aside lives under a hidden, component-scoped staging directory inside the - // components root: same filesystem as the source so the rename stays atomic, the - // leading dot keeps loadComponentDirectories from picking it up as a phantom - // component, and the per-component path means a sibling component never collides - // with (or sweeps) another's aside. - const asideStagingDir = join(dirname(application.dirPath), ASIDE_STAGING_DIR, basename(application.dirPath)); - let didRenameAside = false; - try { - await access(application.dirPath, constants.F_OK); - await mkdir(asideStagingDir, { recursive: true }); - await rename(application.dirPath, join(asideStagingDir, `${process.pid}-${Date.now()}-${randomUUID()}`)); - didRenameAside = true; - } catch (err) { - // Ignore does not exist error - if (err.code !== 'ENOENT') { - throw err; - } - } - // Finally, create the application directory fresh - await mkdir(application.dirPath, { recursive: true }); + const buildDirPath = application.buildDirPath; + + // Replace any existing build directory atomically instead of clearing it in place. When the + // build target IS the live directory (the legacy in-place path, and boot-time installs), a + // previous version's worker can still be running and actively writing into it — e.g. a live + // Next.js app writing into `.next/cache` — and an in-place recursive rm races that writer, + // failing with ENOTEMPTY. moveDirAside renames it aside atomically instead. When the build + // target is a fresh two-phase staging dir there is normally nothing to move (a retried stage + // is the exception), so this is a cheap no-op on the common path. + const asideStagingDir = await moveDirAside(buildDirPath); + + // Create the build directory fresh + await mkdir(buildDirPath, { recursive: true }); // Now pipeline the tarball into maybe-gunzip then tar-fs to reliably decompress and extract the contents - await pipeline(tarball, gunzip(), extract(application.dirPath)); + await pipeline(tarball, gunzip(), extract(buildDirPath)); // If the extracted directory contains a single folder, move the contents up one level // The `npm pack` command does this (the top-level folder is called "package") // Other packing tools may have similar behavior, but the directory name is not guaranteed. - const extracted = await readdir(application.dirPath, { withFileTypes: true }); + const extracted = await readdir(buildDirPath, { withFileTypes: true }); if (extracted.length === 1 && extracted[0].isDirectory()) { - const topLevelDirPath = join(application.dirPath, extracted[0].name); + const topLevelDirPath = join(buildDirPath, extracted[0].name); - const tempDirPath = await mkdtemp(application.dirPath); + const tempDirPath = await mkdtemp(buildDirPath); // Copy contents of top-level directory to temp directory (in order to avoid collisions of top-level directory name and one of the contents) await cp(topLevelDirPath, tempDirPath, { recursive: true }); // Remove top-level directory await rm(topLevelDirPath, { recursive: true, force: true }); // Copy contents of temp directory to application directory - await cp(tempDirPath, application.dirPath, { recursive: true }); + await cp(tempDirPath, buildDirPath, { recursive: true }); // Finally, remove the temp dir await rm(tempDirPath, { recursive: true, force: true }); } @@ -612,33 +656,29 @@ export async function extractApplication(application: Application) { await rm(tarballPath, { force: true }); } - // Remove this component's aside copies. The old worker may still hold files open - // in the just-renamed copy (the live writer that motivated the rename), so this is - // best-effort: removing the whole staging subdirectory also clears leftovers from - // earlier deploys whose workers have since exited, and a copy that survives because - // its worker is still live is swept by the next deploy. The failure is expected in - // the live-worker case, so it's logged at trace rather than as a warning. - if (didRenameAside) { - rm(asideStagingDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }).catch((err) => - logger.trace?.(`Deferred cleanup of previous ${application.name} component directory: ${err.message}`) - ); - } + // Remove this component's aside copies (best-effort; see cleanupAsideDir). + cleanupAsideDir(asideStagingDir, application.name); } /** - * Install an application to its relative `application.dirPath` using either a + * Install an application into `application.buildDirPath` using either a * configured `application.install` command, a derived package manager from the * application's `package.json#devEngines`, or falling back to the default * package manager, `npm`. * - * Will return early if `node_modules` already exists within the `application.dirPath` + * `buildDirPath` is the live component directory by default, or the hidden staging directory + * during a two-phase deploy (see stageApplication) — so `npm install`, the slowest and most + * failure-prone step, runs against the staged copy and never leaves the live path half-installed. + * + * Will return early if `node_modules` already exists within the build directory. * * This method should only be called from the main thread */ export async function installApplication(application: Application) { + const buildDirPath = application.buildDirPath; let packageJSON: any; try { - packageJSON = JSON.parse(await readFile(join(application.dirPath, 'package.json'), 'utf8')); + packageJSON = JSON.parse(await readFile(join(buildDirPath, 'package.json'), 'utf8')); } catch (err) { if (err.code !== 'ENOENT') throw err; // If no package.json, nothing to install @@ -647,7 +687,7 @@ export async function installApplication(application: Application) { } try { // Does node_modules exist? - await access(join(application.dirPath, 'node_modules'), constants.F_OK); + await access(join(buildDirPath, 'node_modules'), constants.F_OK); application.logger.info(`Application ${application.name} already has node_modules; skipping install`); return; } catch (err) { @@ -665,7 +705,7 @@ export async function installApplication(application: Application) { application.name, command, args, - application.dirPath, + buildDirPath, application.install?.timeout, customOnLine, application.npmUserconfigPath @@ -726,7 +766,7 @@ export async function installApplication(application: Application) { application.name, (application.packageManagerPrefix ? application.packageManagerPrefix + ' ' : '') + packageManager.name, application.install?.allowInstallScripts ? ['install'] : ['install', '--ignore-scripts'], // All of `npm`, `yarn`, and `pnpm` support the `install` command. If we need to configure options here we may have to use some other defaults though - application.dirPath, + buildDirPath, application.install?.timeout, pmOnLine, application.npmUserconfigPath @@ -783,7 +823,7 @@ export async function installApplication(application: Application) { application.name, (application.packageManagerPrefix ? application.packageManagerPrefix + ' ' : '') + 'npm', npmInstallArgs, - application.dirPath, + buildDirPath, application.install?.timeout, npmOnLine, application.npmUserconfigPath @@ -825,6 +865,11 @@ interface ApplicationOptions { // Deploy credentials already resolved to literal tokens, of any kind; partitioned by the // constructor into the npm and git halves, which are injected by entirely different mechanisms. credentials?: ResolvedCredential[]; + // Stable identifier for the hidden staging directory this deploy builds into, so a two-phase + // deploy's `activate_component` step can reconstruct the same staging path a prior + // `stage_component` built (both derive it from the deployment id). Defaults to a random UUID for + // callers that stage and activate against one in-memory Application instance. + stagingId?: string; } export class Application { @@ -846,10 +891,23 @@ export class Application { // Path to the per-deploy `.npmrc`, set by writeTransientNpmrc() during prepareApplication and // passed to the spawn calls; undefined when no registry credentials were provided. npmUserconfigPath?: string; + // Stable id for this deploy's staging directory (see ApplicationOptions.stagingId). + stagingId: string; + // When set, extract/install build here instead of the live `dirPath`. stageApplication() points + // it at `stagingDirPath`; activateApplication() clears it after swapping the staged copy live. + #buildDirPath?: string; #npmrcTempDir?: string; #gitCredentialSession?: GitCredentialSession; - constructor({ name, payload, packageIdentifier, install, onInstallLine, credentials }: ApplicationOptions) { + constructor({ + name, + payload, + packageIdentifier, + install, + onInstallLine, + credentials, + stagingId, + }: ApplicationOptions) { this.name = name; this.payload = payload; this.packageIdentifier = packageIdentifier && derivePackageIdentifier(packageIdentifier); @@ -872,10 +930,37 @@ export class Application { const componentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); if (!componentsRoot) throw new Error('componentsRoot is not configured'); this.dirPath = join(componentsRoot, name); + this.stagingId = stagingId ?? randomUUID(); this.logger = logger.loggerWithTag(name); this.packageManagerPrefix = getConfigValue(CONFIG_PARAMS.APPLICATIONS_PACKAGEMANAGERPREFIX); } + // Directory where extract/install currently build. Defaults to the live component directory + // (`dirPath`) — the legacy in-place path, still used by boot-time installApplications() — and is + // repointed at the hidden staging directory for the duration of a two-phase deploy. + get buildDirPath(): string { + return this.#buildDirPath ?? this.dirPath; + } + + // Hidden, per-deploy staging directory the incoming version is built into before it goes live. + // Deterministic from (component name, stagingId) so `activate_component` can find what + // `stage_component` built. Sits under the components root (dirname(dirPath)) so the go-live + // rename() into `dirPath` stays on one filesystem and is therefore atomic. See DEPLOY_STAGING_DIR. + get stagingDirPath(): string { + return join(dirname(this.dirPath), DEPLOY_STAGING_DIR, this.name, this.stagingId); + } + + // Route extract/install into the staging directory. Called by stageApplication(). + useStagingBuildDir(): void { + this.#buildDirPath = this.stagingDirPath; + } + + // Restore the live component directory as the build target. Called by activateApplication() + // once the staged copy has been swapped into place. + useLiveBuildDir(): void { + this.#buildDirPath = undefined; + } + // Write the transient `.npmrc` into a fresh 0700 temp dir (file mode 0600) and record its path // so the deploy's npm spawns authenticate against the private registry. No-op without registry // credentials. @@ -1014,6 +1099,122 @@ export async function prepareApplication(application: Application) { } } +/** + * Phase 1 of a two-phase deploy: build the INCOMING version of a component completely — download / + * `npm pack` (incl. a git clone), extract, and `npm install` — into the hidden staging directory, + * WITHOUT touching the live component directory. + * + * This is the slow, failure-prone half of a deploy, and doing it off to the side has two payoffs: + * - It is safe to run across the whole cluster and gate on: if a node can't fetch the package or + * `npm install` fails, that node reports the failure and NOTHING has changed anywhere — the live + * component is untouched on every node. Contrast the one-shot deploy, where a peer can fail + * mid-install with a half-written live directory while other peers have already gone live. + * - The staging directory is not the watched base of any component's file watcher and is ignored + * by the component loader (leading dot), so building here triggers no restart-on-change storm. + * No deploy:start/deploy:end watcher suppression is needed for this phase — that is reserved for + * activateApplication, which is the only phase that writes the live path. + * + * This method should only be called from the main thread. + * + * @param application The application to stage. + * @returns The absolute path of the staging directory the incoming version was built into. + */ +export async function stageApplication(application: Application): Promise { + application.useStagingBuildDir(); + // Start from a clean slate so a retried stage (same deployment id) can't inherit a half-built + // tree from a previous attempt. + await rm(application.stagingDirPath, { recursive: true, force: true }); + try { + await application.writeTransientNpmrc(); + try { + await application.startGitCredentialSession(); + await extractApplication(application); + } finally { + await application.cleanupGitCredentialSession(); + } + await installApplication(application); + } catch (err) { + // A failed stage leaves nothing live; remove the partial staging tree so it can't accumulate + // or be mistaken for a good build. Best-effort — never mask the original failure. + await rm(application.stagingDirPath, { recursive: true, force: true }).catch(() => {}); + application.useLiveBuildDir(); + throw err; + } finally { + await application.cleanupTransientNpmrc(); + } + return application.stagingDirPath; +} + +/** + * Phase 2 of a two-phase deploy: swap the already-staged incoming version into the live component + * directory in one atomic `rename()`, then let watchers restart onto it. + * + * This is the short, low-risk half — no network, no install, just a directory swap — so the window + * during which the component is being replaced is as small as the filesystem allows, and it is only + * entered once staging has succeeded (cluster-wide, when orchestrated by deploy_component). + * + * Bracketed with deploy:start/deploy:end so every thread's file watchers suppress restart-on-change + * while the live directory is replaced (harper#488) — the same suppression the one-shot deploy used + * to hold for the entire extract+install; here it wraps only the swap. + * + * This method should only be called from the main thread. + */ +export async function activateApplication(application: Application): Promise { + const stagingDirPath = application.stagingDirPath; + try { + await access(stagingDirPath, constants.F_OK); + } catch (err) { + if (err.code === 'ENOENT') { + throw new Error( + `Cannot activate ${application.name}: no staged build found at ${stagingDirPath}. ` + + `Stage the component (stage_component) before activating it.` + ); + } + throw err; + } + await broadcastDeployStart(application.name); + let asideStagingDir: string | null = null; + try { + // Move the current live version aside (atomic; tolerates a still-writing worker), then rename + // the staged copy into place. Both live under the components root, so the rename is same-fs and + // atomic — there is no interval where `dirPath` is a partially populated directory. + asideStagingDir = await moveDirAside(application.dirPath); + await mkdir(dirname(application.dirPath), { recursive: true }); + await rename(stagingDirPath, application.dirPath); + application.useLiveBuildDir(); + } finally { + broadcastDeployEnd(application.name); + } + // Best-effort cleanup of the outgoing version (see cleanupAsideDir) and the now-empty per-component + // staging parent. + cleanupAsideDir(asideStagingDir, application.name); + rm(dirname(stagingDirPath), { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }).catch((err) => + logger.trace?.(`Deferred cleanup of ${application.name} staging directory: ${err.message}`) + ); +} + +/** + * Discard a staged-but-not-activated build (an aborted two-phase deploy). Best-effort: removes the + * staging tree and tears down any transient credential state. The live component directory is never + * touched. Safe to call whether or not staging ever ran. + */ +export async function discardStagedApplication(application: Application): Promise { + try { + await application.cleanupGitCredentialSession(); + } catch { + /* best-effort */ + } + try { + await application.cleanupTransientNpmrc(); + } catch { + /* best-effort */ + } + application.useLiveBuildDir(); + await rm(application.stagingDirPath, { recursive: true, force: true }).catch((err) => + logger.trace?.(`Failed to discard ${application.name} staging directory: ${err.message}`) + ); +} + /** * Install all applications specified in the root config. * diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index be96eec483..73f499bbf1 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -49,8 +49,14 @@ type DeploymentStatus = | 'pending' | 'extracting' | 'installing' + // Two-phase deploy: building the incoming version into staging cluster-wide (stage phase), and the + // terminal resting state of a stage_component that has not yet been activated. + | 'staging' + | 'staged' | 'loading' | 'replicating' + // Two-phase deploy: swapping the staged build into the live path cluster-wide (activate phase). + | 'activating' | 'restarting' | 'success' | 'failed' @@ -395,7 +401,7 @@ export class DeploymentRecorder { this.sealed = true; } - async finish(status: 'success' | 'failed' | 'rolled_back', error?: unknown): Promise { + async finish(status: 'success' | 'failed' | 'rolled_back' | 'staged', error?: unknown): Promise { if (this.finished) return; // Send a terminal sentinel through the emitter (if any) BEFORE we unsubscribe and // remove it from the registry, so any SSE tail subscribers can resolve their wait @@ -564,10 +570,14 @@ function startStatusFor(phase: string | undefined): DeploymentStatus | null { return 'extracting'; case 'install': return 'installing'; + case 'stage': + return 'staging'; case 'load': return 'loading'; case 'replicate': return 'replicating'; + case 'activate': + return 'activating'; case 'restart': return 'restarting'; default: diff --git a/components/operations.js b/components/operations.js index 9231ad3ce4..303512a445 100644 --- a/components/operations.js +++ b/components/operations.js @@ -24,7 +24,15 @@ const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; const manageThreads = require('../server/threads/manageThreads.js'); const { packageDirectory } = require('../components/packageComponent.ts'); const { Resources } = require('../resources/Resources.ts'); -const { Application, prepareApplication, ASIDE_STAGING_DIR } = require('./Application.ts'); +const { + Application, + prepareApplication, + stageApplication, + activateApplication, + discardStagedApplication, + ASIDE_STAGING_DIR, + DEPLOY_STAGING_DIR, +} = require('./Application.ts'); const { server } = require('../server/Server.ts'); const { DeploymentRecorder, awaitDeploymentRow, DEFAULT_AWAIT_ROW_TIMEOUT_MS } = require('./deploymentRecorder.ts'); const { ProgressEmitter } = require('../server/serverHelpers/progressEmitter.ts'); @@ -352,11 +360,23 @@ async function packageComponent(req) { } /** - * Can deploy a component in multiple ways. If a 'package' is provided all it will do is write that package to - * harperdb-config, when HDB is restarted the package will be installed in hdb/nodeModules. If a base64 encoded string is passed it - * will write string to a temp tar file and extract that file into the deployed project in hdb/components. + * Deploy a component. Front door for the deploy family: derives the project name, validates, ingests + * any credential token into the secrets store (so it lives as a replicated reference, not embedded), + * then dispatches to the two-phase orchestrator (default) or the legacy one-shot path. + * + * Two-phase (stage → activate) builds the incoming version into a hidden staging directory on EVERY + * node first, verifies it landed everywhere, and only then swaps it live cluster-wide — so a node + * that can't fetch the package or fails `npm install` fails the deploy while the live component is + * still untouched on every node, and the go-live window shrinks to a fast atomic directory swap. + * See stageApplication/activateApplication in components/Application.ts. + * + * The request/response contract is unchanged: same inputs (`package`/payload, `restart`, + * `install_*`, `credentials`, `ignore_replication_errors`, `deployment_timeout`, …), same + * `deployment_id` in the response, same SSE progress stream (now emitting `stage`/`activate` phases + * instead of `prepare`/`replicate`). Pass `two_phase: false` to force the legacy one-shot path. + * * @param req - * @returns {Promise} + * @returns {Promise} */ async function deployComponent(req) { if (req.project) { @@ -374,53 +394,46 @@ async function deployComponent(req) { // replicated ciphertext (reference, not embed); already-reference entries pass through, and with // no custody a literal token stays as a transient, this-node-only fallback (#1158). Peers // re-running a replicated deploy already carry references and never re-ingest. - const { ingestCredentials, resolveCredentials } = require('./secretOperations.ts'); + const { ingestCredentials } = require('./secretOperations.ts'); req.credentials = await ingestCredentials(req, req.credentials, req.project); // References are safe to persist (config + deployment row) and replicate; a no-custody literal - // token is not — it is used only for this node's install below, then stripped before replication. + // token is not — it is used only for this node's install, then stripped before replication. const credentialReferences = (req.credentials ?? []).filter((entry) => entry && entry.secret !== undefined); - // Write to root config if the request contains a package identifier - if (req.package) { - // Check if trying to overwrite a core component (requires force) - // Lazy-load to avoid circular dependency with componentLoader - const { TRUSTED_RESOURCE_PLUGINS } = require('./componentLoader.ts'); - if (TRUSTED_RESOURCE_PLUGINS[req.project] && !req.force) { - throw handleHDBError( - new Error(), - `Cannot deploy component with name '${req.project}': this is a protected core component name. Use force: true to overwrite.`, - HTTP_STATUS_CODES.CONFLICT - ); - } - - const applicationConfig = { package: req.package }; - // Avoid writing an empty `install:` block - if (req.install_command || req.install_timeout || req.install_allow_scripts !== undefined) { - applicationConfig.install = { - command: req.install_command, - timeout: req.install_timeout, - allowInstallScripts: req.install_allow_scripts, - }; - } - if (req.urlPath !== undefined) applicationConfig.urlPath = req.urlPath; - // Persist credential references (never tokens) so every cold install of this component — - // reboot, new peer, rollback — re-resolves the credential from the store. - if (credentialReferences.length) applicationConfig.credentials = credentialReferences; - await configUtils.addConfig(req.project, applicationConfig); - } - - // Create a hdb_deployment row up front so the deploy is observable and auditable - // even if the CLI disconnects. The row also holds the payload in a Blob attribute, - // which doubles as the source for peer replication and (later) rollback. - // - // Only the origin node records — peers receiving a replicated deploy_component skip - // recording so we don't accumulate one row per node for the same deploy. The row - // reaches peers via the table's standard replication; the peer-side branch below - // reads payload_blob back from there. + // A peer replaying a replicated ONE-SHOT deploy_component arrives with `_deploymentId` set. In + // two-phase mode the origin never sends deploy_component to peers (it sends stage_component / + // activate_component), so `_deploymentId` here always means "legacy peer". const isReplicatedExecution = typeof req._deploymentId === 'string'; - // An SSE-bound caller already attached a ProgressEmitter (created in the server - // handler so it can also drive the response stream). Reuse it; otherwise spin up a - // fresh emitter so the recorder still gets phase events for non-SSE deploys. + + // Two-phase is the default, but it leans on the system table's replication channel to carry the + // payload and correlate the stage/activate steps across the cluster. Fall back to the legacy + // one-shot deploy when: the caller opted out (`two_phase: false`); this is a peer replaying a + // one-shot deploy; or `system` isn't replicated on this node (a narrow REPLICATION_DATABASES). + if (req.two_phase === false || isReplicatedExecution || !isSystemDatabaseReplicated()) { + return deployComponentOneShot(req, credentialReferences, isReplicatedExecution); + } + return deployComponentTwoPhase(req, credentialReferences); +} + +/** + * Legacy one-shot deploy: extract + `npm install` in place on the origin, then replicate the whole + * deploy_component operation to peers, which each do the same. Preserved verbatim (behavior-for- + * behavior) as the fallback path for `two_phase: false` and for peers replaying a one-shot deploy. + * The wrapper has already derived the project name, validated, and ingested credentials. + */ +async function deployComponentOneShot(req, credentialReferences, isReplicatedExecution) { + const { resolveCredentials } = require('./secretOperations.ts'); + + // Write to root config if the request contains a package identifier + if (req.package) await writeComponentRootConfig(req, credentialReferences); + + // Create a hdb_deployment row up front so the deploy is observable and auditable even if the CLI + // disconnects. The row also holds the payload in a Blob attribute, which doubles as the source for + // peer replication and (later) rollback. Only the origin node records — peers replaying the + // replicated deploy skip recording so we don't accumulate one row per node for the same deploy. + // An SSE-bound caller already attached a ProgressEmitter (created in the server handler so it can + // also drive the response stream). Reuse it; otherwise spin up a fresh emitter so the recorder + // still gets phase events for non-SSE deploys. const emitter = isReplicatedExecution ? null : (req.progress ?? new ProgressEmitter()); if (emitter && !req.progress) req.progress = emitter; const recorder = isReplicatedExecution @@ -438,76 +451,27 @@ async function deployComponent(req) { const emit = (event, data) => emitter?.emit(event, data); - // The new payload-via-replicated-row path depends on the `system` database actually - // being replicated on this node. If the cluster is configured with a narrower - // REPLICATION_DATABASES list that excludes `system`, peers won't see the - // hdb_deployment row and falling back to sending req.payload through the operation - // body is the only viable path. + // The payload-via-replicated-row path depends on `system` actually replicating on this node. const systemReplicated = isSystemDatabaseReplicated(); - let extractionPayload = req.payload; - // Bounded ring buffer of install stdout/stderr so a non-SSE caller sees the tail - // in the thrown error. SSE callers still stream every line live. + // Bounded ring buffer of install stdout/stderr so a non-SSE caller sees the tail in the thrown + // error. SSE callers still stream every line live. const installCapture = createInstallCapture(); try { - // On the origin, tee the tarball (Buffer or Readable from the multipart parser) - // through a hash-and-size tap into the row's payload_blob, then re-source extraction - // from the persisted blob. When `system` replicates, the blob becomes the channel - // peers read from; when it doesn't, the blob stays local for audit and rollback. - if (recorder && req.payload != null) { - await recorder.ingestPayload(req.payload); - extractionPayload = recorder.row.payload_blob.stream(); - } else if (isReplicatedExecution && req.payload == null && !req.package) { - // Peer received a replicated deploy without a payload — read the tarball from - // the replicated hdb_deployment row's payload_blob. Blob.stream() blocks on - // in-flight BLOB_CHUNK writes until the chunks land. If the row never arrives - // within the timeout, peer records a failure and origin sees it in peer_results. - // The wait budget defaults to 120s but is overridable per-deploy via - // `deployment_timeout` (ms) for clusters where the system-table channel is - // heavily backlogged (harper-pro#402). - const row = await awaitDeploymentRow(req._deploymentId, { timeoutMs: req.deployment_timeout }); - extractionPayload = row.payload_blob.stream(); - } - - // Resolve credential references into concrete tokens for this node's npm pack/install - // (a no-custody literal-token fallback passes through unchanged). On a peer running a - // replicated deploy, the referenced hdb_secret row may arrive just behind the deploy op, so - // allow a bounded grace period (same budget as the payload-row wait) for it to replicate in. - let credentialsWaitMs = 0; - if (isReplicatedExecution) { - const requested = Number(req.deployment_timeout); - credentialsWaitMs = Number.isFinite(requested) && requested >= 0 ? requested : DEFAULT_AWAIT_ROW_TIMEOUT_MS; - } - const resolvedCredentials = await resolveCredentials(req.credentials, req.project, { - waitMs: credentialsWaitMs, - }); - - const application = new Application({ - name: req.project, - payload: extractionPayload, - packageIdentifier: req.package, - install: { - command: req.install_command, - timeout: req.install_timeout, - allowInstallScripts: req.install_allow_scripts, - }, - // Tee each install line into both the capture buffer (for the thrown-error - // fallback) and the SSE channel (when a caller is streaming). Peers have no - // emitter, so their install output goes to the local logger and the buffer only. - onInstallLine: (manager, stream, line) => { - installCapture.push(manager, stream, line); - if (emitter) emit('install', { manager, stream, line }); - }, - // Deploy credentials (already resolved above), used here for this node's npm pack/install: - // registry entries via a transient .npmrc, git-host entries via the in-memory credential - // socket the clone spawn talks to. - credentials: resolvedCredentials, + const extractionPayload = await sourceExtractionPayload({ req, recorder, isReplicatedExecution }); + const resolvedCredentials = await resolveNodeCredentials({ req, resolveCredentials, isReplicatedExecution }); + const application = buildDeployApplication({ + req, + extractionPayload, + resolvedCredentials, + installCapture, + emitter, + emit, }); // Reduce req.credentials to references only (never a token) before it can reach an error/log // path or replication: references are what peers resolve from their own replicated hdb_secret // copy; a no-custody literal token is dropped entirely (peers fall back to their fabric-injected - // NPM_CONFIG_USERCONFIG, as before). This also fixes the prior success-only strip that leaked a - // literal token on a prepare/load failure. + // NPM_CONFIG_USERCONFIG, as before). if (credentialReferences.length) req.credentials = credentialReferences; else delete req.credentials; @@ -515,83 +479,33 @@ async function deployComponent(req) { await prepareApplication(application); emit('phase', { phase: 'prepare', status: 'done' }); - // now we attempt to actually load the component in case there is - // an error we can immediately detect and report, but app code should not run on the main thread - if (!isMainThread && !process.env.HARPER_SAFE_MODE) { - const pseudoResources = new Resources(); - pseudoResources.isWorker = true; - - const componentLoader = require('./componentLoader.ts').default || require('./componentLoader.ts'); - const { trackScopeClose } = require('./scopeShutdown.ts'); - let lastError; - componentLoader.setErrorReporter((error) => (lastError = error)); - emit('phase', { phase: 'load', status: 'start' }); - // This load exists only to surface load-time errors early; the Scopes it creates are - // throwaway. They are collected (instead of registered for worker-shutdown auto-close) so we - // can close them here once validation completes — otherwise each deploy leaks the Scope's - // deploy-lifecycle listeners on this worker, eventually tripping MaxListenersExceededWarning - // (#1462). - const validationScopes = new Set(); - const validation = (async () => { - try { - await componentLoader.loadComponent(application.dirPath, pseudoResources, undefined, { - collectScopes: validationScopes, - }); - } finally { - const closeResults = await Promise.allSettled(Array.from(validationScopes, (scope) => scope.close())); - for (const result of closeResults) { - if (result.status === 'rejected') log.warn('Failed to close a deploy-validation Scope', result.reason); - } - } - })(); - // Track the load+close so a concurrent worker shutdown waits for these scopes to finish - // disposing — a plugin may start a native runtime in handleApplication — before realExit. - trackScopeClose(validation); - await validation; - emit('phase', { phase: 'load', status: 'done' }); - - if (lastError) throw lastError; - } + // Load the component to surface load-time errors early (throwaway scopes; see loadValidateComponent). + await loadValidateComponent({ dirPath: application.dirPath, emit }); + const rollingRestart = req.restart === 'rolling'; // if doing a rolling restart set restart to false so that other nodes don't also restart. req.restart = rollingRestart ? false : req.restart; - // ProgressEmitter holds function listeners that can't survive the replication - // channel's serialization; strip it unconditionally. + // ProgressEmitter holds function listeners that can't survive the replication channel's + // serialization; strip it unconditionally. delete req.progress; - // req.credentials was already deleted immediately after the Application ctor (above) so the - // token never reaches the replication channel or a peer's operation log; peers authenticate - // against the private registry via their own fabric-injected NPM_CONFIG_USERCONFIG on reinstall. if (systemReplicated && recorder) { - // The hdb_deployment row + payload_blob will reach peers via table replication, - // so peers can look up the payload by deployment_id. Drop req.payload to keep - // the operation body small (the operations channel has frame-size limits the - // blob-replication channel doesn't share). _deploymentId is the handoff that - // lets peers find the replicated row. + // The hdb_deployment row + payload_blob reach peers via table replication, so peers look up + // the payload by deployment_id. Drop req.payload to keep the operation body small. delete req.payload; } - // As each peer settles, update the origin row so observers polling get_deployment - // see per-peer progress in real time rather than only at the aggregate end. - // replicateOperation in harper-pro accepts an optional onPeerResult callback that - // fires per peer; callers without the callback (older replicator) fall back to - // the aggregate response.replicated below. const onPeerResult = recorder ? (result) => { recorder.recordPeer(result); emit('peer', result); } : undefined; - // Seal the recorder before the replicate phase so the row's terminal write (finish()) - // isn't part of the tight put burst that can commit out of order on a peer and revert - // it (harperdb/harper#1170). onPeerResult/peer_results accumulate in memory and land in - // finish()'s single write; live SSE 'peer' events still fire below. + // Seal before the replicate phase so the row's terminal write (finish()) isn't part of the tight + // put burst that can commit out of order on a peer and revert it (harperdb/harper#1170). recorder?.seal(); emit('phase', { phase: 'replicate', status: 'start' }); let response = await server.replication.replicateOperation(req, { onPeerResult }); emit('phase', { phase: 'replicate', status: 'done' }); if (recorder && response?.replicated) { - // Fallback path for replicators that don't honor onPeerResult: re-record the - // aggregate. recordPeer's upsert-by-node-name semantics make this idempotent - // when the per-peer callback already fired for these. recorder.recordPeers(response.replicated); } if (req.restart === true) { @@ -613,21 +527,15 @@ async function deployComponent(req) { response.message = `Successfully deployed: ${application.name}, restarting Harper`; } else response.message = `Successfully deployed: ${application.name}`; - // Replication failures don't reject replicateOperation — they surface as 'failed' - // entries in peer_results. By default, treat any failed peer as an overall deploy - // failure so the operation returns a non-2xx status (and the CLI a non-zero exit - // code). The component is already deployed — and, if requested, restarted — on this - // origin node; the failure signals that one or more peers did not receive it. Pass - // ignore_replication_errors: true for best-effort deploys to partially-available clusters. + // Replication failures don't reject replicateOperation — they surface as 'failed' entries in + // peer_results. By default, treat any failed peer as an overall deploy failure so the operation + // returns a non-2xx status. Pass ignore_replication_errors: true for best-effort deploys. if (recorder && !req.ignore_replication_errors) { const failedPeers = recorder.getFailedPeers(); if (failedPeers.length > 0) { - const detail = failedPeers - .map((peer) => `${peer.node ?? 'unknown'} (${peer.error?.message ?? 'unknown error'})`) - .join(', '); throw new ServerError( `Component '${application.name}' was deployed on the origin node but failed to replicate to ` + - `${failedPeers.length} of ${recorder.row.peer_results.length} peer node(s): ${detail}. ` + + `${failedPeers.length} of ${recorder.row.peer_results.length} peer node(s): ${describePeers(failedPeers)}. ` + `See deployment ${recorder.deploymentId} (get_deployment) for details, or pass ` + `ignore_replication_errors: true to treat replication failures as non-fatal.` ); @@ -636,69 +544,541 @@ async function deployComponent(req) { if (recorder) { response.deployment_id = recorder.deploymentId; - // Reclaim the payload tarball for large deploys: every peer has now installed from - // the blob (replicateOperation resolved) and the origin no longer needs it. Dropping - // the reference before finish() folds the null into the single terminal write, which - // unlinks the file locally and replicates the null so peers drop their copies too. - // Metadata (size, hash, event_log) is retained for the audit trail. Two guards keep - // the tarball when it's still the artifact you'd debug or retry with: failed deploys - // don't reach this branch, and a deploy that reached here only because - // ignore_replication_errors masked failed peers keeps its payload for those peers. - const payloadSize = recorder.row.payload_size; - const retentionMaxSize = getPayloadRetentionMaxSize(); - if (typeof payloadSize === 'number' && payloadSize > retentionMaxSize && recorder.getFailedPeers().length === 0) { - const freed = recorder.dropPayload(); - if (freed > 0) emit('payload_dropped', { payload_size: freed, max_size: retentionMaxSize }); - } + maybeReclaimPayload(recorder, emit); emit('phase', { phase: 'success', status: 'done' }); await recorder.finish('success'); } return response; } catch (err) { - // Pack phase, install output tail, and deployment_id into http_resp_msg so the - // Fastify error handler forwards them verbatim (it does when http_resp_msg is an - // object). Non-SSE callers see structured failure detail; SSE callers already - // got the same data live via emit('error', ...) below. - const capture = installCapture.snapshot(); - const phase = recorder?.row.phase; - const baseMessage = err?.message ?? String(err); - const structured = { error: baseMessage }; - if (phase) structured.phase = phase; - if (capture.lines.length > 0) structured.install_output = capture; - if (recorder?.deploymentId) structured.deployment_id = recorder.deploymentId; - // Surface failed peer outcomes so callers see which nodes the deploy did not reach - // without a second get_deployment round-trip. Populated for replication failures (the - // throw above) and any other failure that occurred after peers reported. Carried on - // both the structured non-SSE body and the SSE 'error' event below so the two transports - // stay symmetric (the CLI uses SSE for deploy_component). - const failedPeers = recorder?.getFailedPeers() ?? []; - if (failedPeers.length > 0) structured.failed_peers = failedPeers; - - // Wrap as a ServerError so the Fastify error handler picks a 500 by default; preserve - // an upstream statusCode (e.g. a ClientError from payload validation) if present. - const outErr = new ServerError(baseMessage, err?.statusCode); - outErr.http_resp_msg = structured; - - emit('error', { - message: baseMessage, - code: outErr?.statusCode ?? err?.code, - phase, - install_output: capture.lines.length > 0 ? capture : undefined, - deployment_id: recorder?.deploymentId, - failed_peers: failedPeers.length > 0 ? failedPeers : undefined, + throw await finalizeDeployFailure({ err, recorder, installCapture, emit }); + } +} + +/** + * Two-phase deploy orchestrator (origin node). Builds the incoming version into staging on every + * node (phase 1, stage_component), gates on every node succeeding, then atomically swaps it live on + * every node (phase 2, activate_component). The live component on every node is untouched until the + * whole cluster has the bits in place, and the go-live window is just the swap + restart. + */ +async function deployComponentTwoPhase(req, credentialReferences) { + const { resolveCredentials } = require('./secretOperations.ts'); + // Fail fast on a protected core name before we create any state or touch the cluster. + if (req.package) assertNotProtectedCoreComponent(req.project, req.force); + + // The origin always records (a two-phase origin is never itself a replicated execution). + const emitter = req.progress ?? new ProgressEmitter(); + if (!req.progress) req.progress = emitter; + const recorder = await DeploymentRecorder.create({ + project: req.project, + package_identifier: req.package ?? null, + user: req.hdb_user?.username, + restart_mode: req.restart === 'rolling' ? 'rolling' : req.restart ? 'immediate' : null, + credentials: credentialReferences.length ? credentialReferences : null, + emitter, + }); + req._deploymentId = recorder.deploymentId; + const emit = (event, data) => emitter.emit(event, data); + const installCapture = createInstallCapture(); + const rollingRestart = req.restart === 'rolling'; + const recordPeer = (result) => { + recorder.recordPeer(result); + emit('peer', result); + }; + let application; + + try { + // Tee the payload into the row's blob (the replication channel peers read from) and re-source + // extraction from it. Two-phase requires systemReplicated, so peers always fetch from the row. + const extractionPayload = await sourceExtractionPayload({ req, recorder, isReplicatedExecution: false }); + const resolvedCredentials = await resolveNodeCredentials({ req, resolveCredentials, isReplicatedExecution: false }); + // stagingId = deployment id so peers (which build a fresh Application per sub-op) resolve the + // same staging path this deployment used. + application = buildDeployApplication({ + req, + extractionPayload, + resolvedCredentials, + stagingId: recorder.deploymentId, + installCapture, + emitter, + emit, + }); + // Strip tokens from req before any replication/log path; keep references (peers resolve those + // from their own hdb_secret copy). Strip the emitter and payload too — peers read the payload + // from the replicated row, keeping the sub-operation bodies small. + if (credentialReferences.length) req.credentials = credentialReferences; + else delete req.credentials; + delete req.progress; + delete req.payload; + + // ===== PHASE 1: STAGE — build on every node; nothing goes live. ===== + emit('phase', { phase: 'stage', status: 'start' }); + await stageApplication(application); + const stageOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.STAGE_COMPONENT); + const stageResp = await server.replication.replicateOperation(stageOp, { onPeerResult: recordPeer }); + if (stageResp?.replicated) recorder.recordPeers(stageResp.replicated); + emit('phase', { phase: 'stage', status: 'done' }); + + // ---- Cluster barrier: every node must have staged before ANY node activates. ---- + if (!req.ignore_replication_errors) { + const failed = recorder.getFailedPeers(); + if (failed.length > 0) { + await discardStagedApplication(application).catch(() => {}); + throw new ServerError( + `Component '${req.project}' failed to stage on ${failed.length} of ` + + `${recorder.row.peer_results.length} peer node(s): ${describePeers(failed)}. No node was activated — ` + + `the live component is unchanged everywhere. See deployment ${recorder.deploymentId} (get_deployment), ` + + `or pass ignore_replication_errors: true to activate the nodes that did stage.` + ); + } + } + + // Validate the staged build before go-live (loads from the staging dir; see loadValidateComponent). + await loadValidateComponent({ dirPath: application.buildDirPath, emit }); + + // ===== PHASE 2: ACTIVATE — atomic swap + restart, now the bits are in place everywhere. ===== + // Persist root config now (not before staging) so a `package` config never points at a version + // that failed to stage. + if (req.package) await writeComponentRootConfig(req, credentialReferences); + // if doing a rolling restart set restart to false so peers don't also immediately restart. + req.restart = rollingRestart ? false : req.restart; + + emit('phase', { phase: 'activate', status: 'start' }); + await activateApplication(application); + const activateOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.ACTIVATE_COMPONENT, { + restart: req.restart, + deploymentId: recorder.deploymentId, + }); + // Seal before the activate replicate burst (same #1170 rationale as one-shot). + recorder.seal(); + const activateResp = await server.replication.replicateOperation(activateOp, { onPeerResult: recordPeer }); + emit('phase', { phase: 'activate', status: 'done' }); + let response = activateResp && typeof activateResp === 'object' ? activateResp : { message: '' }; + if (activateResp?.replicated) recorder.recordPeers(activateResp.replicated); + + // ---- Restart on the origin. ---- + if (req.restart === true) { + emit('phase', { phase: 'restart', status: 'start' }); + manageThreads.restartWorkers('http'); + emit('phase', { phase: 'restart', status: 'done' }); + response.message = `Successfully deployed: ${application.name}, restarting Harper`; + } else if (rollingRestart) { + const serverUtilities = require('../server/serverHelpers/serverUtilities.ts'); + emit('phase', { phase: 'restart', status: 'start' }); + const jobResponse = await serverUtilities.executeJob({ + operation: 'restart_service', + service: 'http', + replicated: true, + }); + emit('phase', { phase: 'restart', status: 'done' }); + response.restartJobId = jobResponse.job_id; + response.message = `Successfully deployed: ${application.name}, restarting Harper`; + } else response.message = `Successfully deployed: ${application.name}`; + + // ---- Activate gate: rare, but a node can stage OK and then fail the swap. ---- + if (!req.ignore_replication_errors) { + const failed = recorder.getFailedPeers(); + if (failed.length > 0) { + throw new ServerError( + `Component '${application.name}' was activated on the origin but failed to activate on ${failed.length} ` + + `of ${recorder.row.peer_results.length} peer node(s): ${describePeers(failed)}. Those nodes have the staged ` + + `build but did not go live. See deployment ${recorder.deploymentId} (get_deployment), or pass ` + + `ignore_replication_errors: true.` + ); + } + } + + response.deployment_id = recorder.deploymentId; + maybeReclaimPayload(recorder, emit); + emit('phase', { phase: 'success', status: 'done' }); + await recorder.finish('success'); + return response; + } catch (err) { + // An aborted deploy leaves the live component untouched; drop any staged build so it can't leak. + if (application) await discardStagedApplication(application).catch(() => {}); + throw await finalizeDeployFailure({ err, recorder, installCapture, emit }); + } +} + +/** + * stage_component — phase 1 of a two-phase deploy, as a first-class operation. Builds the incoming + * version into the hidden staging directory on this node (and, when invoked directly rather than via + * replication, replicates the stage across the cluster). Never writes the live component directory, + * writes no root config, and never restarts — so it is safe to run cluster-wide and gate on. + * + * Reached three ways: directly by an operator (stage now, activate later), by a peer replaying a + * replicated stage (`_deploymentId` set), and indirectly — deploy_component drives staging itself, + * so it does not call this handler. + */ +async function stageComponent(req) { + if (req.project) { + req.project = path.parse(req.project).name; + } else if (req.package) { + req.project = getProjectNameFromPackage(req.package); + } + const validation = validator.stageComponentValidator(req); + if (validation) throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); + + const { ingestCredentials, resolveCredentials } = require('./secretOperations.ts'); + req.credentials = await ingestCredentials(req, req.credentials, req.project); + const credentialReferences = (req.credentials ?? []).filter((entry) => entry && entry.secret !== undefined); + if (req.package) assertNotProtectedCoreComponent(req.project, req.force); + + const isReplicatedExecution = typeof req._deploymentId === 'string'; + const emitter = isReplicatedExecution ? null : (req.progress ?? new ProgressEmitter()); + if (emitter && !req.progress) req.progress = emitter; + // Standalone stage records so it has a deployment_id + payload row for peers to fetch; a peer + // replaying the stage skips recording (the origin owns the row). + const recorder = isReplicatedExecution + ? null + : await DeploymentRecorder.create({ + project: req.project, + package_identifier: req.package ?? null, + user: req.hdb_user?.username, + restart_mode: null, + credentials: credentialReferences.length ? credentialReferences : null, + emitter, + }); + if (recorder) req._deploymentId = recorder.deploymentId; + const emit = (event, data) => emitter?.emit(event, data); + const installCapture = createInstallCapture(); + let application; + + try { + const extractionPayload = await sourceExtractionPayload({ req, recorder, isReplicatedExecution }); + const resolvedCredentials = await resolveNodeCredentials({ req, resolveCredentials, isReplicatedExecution }); + application = buildDeployApplication({ + req, + extractionPayload, + resolvedCredentials, + stagingId: req._deploymentId, + installCapture, + emitter, + emit, }); - // Record the terminal failure, but never let a finish() write error (full disk, lock, - // dropped system table) mask the actual deploy failure — outErr carries the phase, - // install output, and failed_peers the caller needs. + if (credentialReferences.length) req.credentials = credentialReferences; + else delete req.credentials; + delete req.progress; + + emit('phase', { phase: 'stage', status: 'start' }); + await stageApplication(application); + emit('phase', { phase: 'stage', status: 'done' }); + + const response = { message: `Staged component: ${req.project}`, project: req.project, staged: true }; if (recorder) { - try { - await recorder.finish('failed', err); - } catch (finishErr) { - log.warn('Failed to record deployment failure row', finishErr); + response.deployment_id = recorder.deploymentId; + // Replicate staging to peers so the bits land cluster-wide. Keep the payload in the body only + // when the row can't carry it (system not replicated on this node). + if (isSystemDatabaseReplicated()) delete req.payload; + const stageOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.STAGE_COMPONENT, { + includePayload: !isSystemDatabaseReplicated(), + }); + recorder.seal(); + const rep = await server.replication.replicateOperation(stageOp, { + onPeerResult: (result) => { + recorder.recordPeer(result); + emit('peer', result); + }, + }); + if (rep?.replicated) recorder.recordPeers(rep.replicated); + if (!req.ignore_replication_errors) { + const failed = recorder.getFailedPeers(); + if (failed.length > 0) { + throw new ServerError( + `Component '${req.project}' failed to stage on ${failed.length} of ` + + `${recorder.row.peer_results.length} peer node(s): ${describePeers(failed)}. ` + + `See deployment ${recorder.deploymentId} (get_deployment).` + ); + } + } + // Leave the row in a 'staged' resting state — the build exists cluster-wide but nothing is + // live yet; a subsequent activate_component (or the caller) takes it live. + emit('phase', { phase: 'staged', status: 'done' }); + await recorder.finish('staged'); + } + return response; + } catch (err) { + if (application) await discardStagedApplication(application).catch(() => {}); + throw await finalizeDeployFailure({ err, recorder, installCapture, emit }); + } +} + +/** + * activate_component — phase 2 of a two-phase deploy, as a first-class operation. Atomically swaps a + * previously-staged build (identified by `deployment_id`) into the live component directory, persists + * the component's root config for a `package` deploy, and restarts as requested. When invoked + * directly it also replicates the activation across the cluster. + * + * Reached two ways: directly by an operator to take a prior stage live, and by a peer replaying a + * replicated activate (`_deploymentId` set). deploy_component drives activation itself. + */ +async function activateComponent(req) { + if (req.project) req.project = path.parse(req.project).name; + const validation = validator.activateComponentValidator(req); + if (validation) throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); + + const isReplicatedExecution = typeof req._deploymentId === 'string'; + const stagingId = req._deploymentId ?? req.deployment_id; + if (!stagingId) { + throw handleHDBError( + new Error(), + `'deployment_id' is required to activate a staged component`, + HTTP_STATUS_CODES.BAD_REQUEST + ); + } + if (req.package) assertNotProtectedCoreComponent(req.project, req.force); + const credentialReferences = (req.credentials ?? []).filter((entry) => entry && entry.secret !== undefined); + + const emitter = isReplicatedExecution ? null : (req.progress ?? new ProgressEmitter()); + const emit = (event, data) => emitter?.emit(event, data); + const rollingRestart = req.restart === 'rolling'; + // Reconstruct the Application against the staged build. Only name + stagingId are needed to locate + // and swap it — the staged directory already holds the fully-installed incoming version. + const application = new Application({ name: req.project, packageIdentifier: req.package, stagingId }); + + emit('phase', { phase: 'activate', status: 'start' }); + await activateApplication(application); + emit('phase', { phase: 'activate', status: 'done' }); + + // Persist root config now that the component is live (package deploys, on every node). + if (req.package) await writeComponentRootConfig(req, credentialReferences); + + const response = { message: `Activated component: ${req.project}`, project: req.project, activated: true }; + + // Replicate the activation to peers (direct invocation only; a peer replaying an activate must not + // re-fan it out). + if (!isReplicatedExecution) { + delete req.progress; + const restartForPeers = rollingRestart ? false : req.restart; + const activateOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.ACTIVATE_COMPONENT, { + restart: restartForPeers, + deploymentId: stagingId, + }); + const rep = await server.replication.replicateOperation(activateOp, {}); + if (rep?.replicated) response.replicated = rep.replicated; + } + + // Restart on this node. A peer replaying an immediate-restart activate restarts locally; the + // rolling path is driven only by the direct invoker via a replicated restart_service job. + if (req.restart === true) { + emit('phase', { phase: 'restart', status: 'start' }); + manageThreads.restartWorkers('http'); + emit('phase', { phase: 'restart', status: 'done' }); + response.message = `Activated component: ${req.project}, restarting Harper`; + } else if (rollingRestart && !isReplicatedExecution) { + const serverUtilities = require('../server/serverHelpers/serverUtilities.ts'); + emit('phase', { phase: 'restart', status: 'start' }); + const jobResponse = await serverUtilities.executeJob({ + operation: 'restart_service', + service: 'http', + replicated: true, + }); + emit('phase', { phase: 'restart', status: 'done' }); + response.restartJobId = jobResponse.job_id; + response.message = `Activated component: ${req.project}, restarting Harper`; + } + return response; +} + +// ———————————————————————————————————————————————————————————————————————————— +// Shared deploy-family helpers (used by deploy_component, stage_component, activate_component). +// ———————————————————————————————————————————————————————————————————————————— + +// Reject deploying over a protected core component name unless force is set. Lazy-loads +// componentLoader to avoid a circular dependency. +function assertNotProtectedCoreComponent(project, force) { + const { TRUSTED_RESOURCE_PLUGINS } = require('./componentLoader.ts'); + if (TRUSTED_RESOURCE_PLUGINS[project] && !force) { + throw handleHDBError( + new Error(), + `Cannot deploy component with name '${project}': this is a protected core component name. Use force: true to overwrite.`, + HTTP_STATUS_CODES.CONFLICT + ); + } +} + +// Persist a `package` deploy's entry into root config so every cold install (reboot, new peer, +// rollback) reinstalls it. In two-phase this runs at activation, once the bits are staged everywhere. +async function writeComponentRootConfig(req, credentialReferences) { + assertNotProtectedCoreComponent(req.project, req.force); + const applicationConfig = { package: req.package }; + // Avoid writing an empty `install:` block + if (req.install_command || req.install_timeout || req.install_allow_scripts !== undefined) { + applicationConfig.install = { + command: req.install_command, + timeout: req.install_timeout, + allowInstallScripts: req.install_allow_scripts, + }; + } + if (req.urlPath !== undefined) applicationConfig.urlPath = req.urlPath; + // Persist credential references (never tokens) so every cold install re-resolves from the store. + if (credentialReferences.length) applicationConfig.credentials = credentialReferences; + await configUtils.addConfig(req.project, applicationConfig); +} + +// Resolve the tarball to extract from. On the origin, tee req.payload into the row's blob (the +// channel peers read from) and re-source extraction from the persisted blob. On a peer replaying a +// deploy without a payload, read the tarball from the replicated row's blob (bounded wait). +async function sourceExtractionPayload({ req, recorder, isReplicatedExecution }) { + if (recorder && req.payload != null) { + await recorder.ingestPayload(req.payload); + return recorder.row.payload_blob.stream(); + } + if (isReplicatedExecution && req.payload == null && !req.package) { + // Blob.stream() blocks on in-flight BLOB_CHUNK writes until the chunks land. If the row never + // arrives within the timeout, the peer records a failure and the origin sees it in peer_results. + const row = await awaitDeploymentRow(req._deploymentId, { timeoutMs: req.deployment_timeout }); + return row.payload_blob.stream(); + } + return req.payload; +} + +// Resolve credential references into concrete tokens for this node's npm pack/install. On a peer, the +// referenced hdb_secret row may arrive just behind the deploy op, so allow a bounded grace period. +async function resolveNodeCredentials({ req, resolveCredentials, isReplicatedExecution }) { + let credentialsWaitMs = 0; + if (isReplicatedExecution) { + const requested = Number(req.deployment_timeout); + credentialsWaitMs = Number.isFinite(requested) && requested >= 0 ? requested : DEFAULT_AWAIT_ROW_TIMEOUT_MS; + } + return resolveCredentials(req.credentials, req.project, { waitMs: credentialsWaitMs }); +} + +// Construct the Application for a deploy, teeing each install line into the capture buffer (for the +// thrown-error tail) and the SSE channel (when a caller is streaming). +function buildDeployApplication({ + req, + extractionPayload, + resolvedCredentials, + stagingId, + installCapture, + emitter, + emit, +}) { + return new Application({ + name: req.project, + payload: extractionPayload, + packageIdentifier: req.package, + install: { + command: req.install_command, + timeout: req.install_timeout, + allowInstallScripts: req.install_allow_scripts, + }, + onInstallLine: (manager, stream, line) => { + installCapture.push(manager, stream, line); + if (emitter) emit('install', { manager, stream, line }); + }, + credentials: resolvedCredentials, + stagingId, + }); +} + +// Load a component directory to surface load-time errors early (throwaway scopes). No-op on the main +// thread or in safe mode. In two-phase this loads the STAGED directory before go-live; in one-shot it +// loads the live directory after in-place prepare. +async function loadValidateComponent({ dirPath, emit }) { + if (isMainThread || process.env.HARPER_SAFE_MODE) return; + const pseudoResources = new Resources(); + pseudoResources.isWorker = true; + + const componentLoader = require('./componentLoader.ts').default || require('./componentLoader.ts'); + const { trackScopeClose } = require('./scopeShutdown.ts'); + let lastError; + componentLoader.setErrorReporter((error) => (lastError = error)); + emit('phase', { phase: 'load', status: 'start' }); + // The Scopes this load creates are throwaway. Collect them (instead of registering for + // worker-shutdown auto-close) so we can close them here once validation completes — otherwise each + // deploy leaks the Scope's deploy-lifecycle listeners on this worker (#1462). + const validationScopes = new Set(); + const validation = (async () => { + try { + await componentLoader.loadComponent(dirPath, pseudoResources, undefined, { collectScopes: validationScopes }); + } finally { + const closeResults = await Promise.allSettled(Array.from(validationScopes, (scope) => scope.close())); + for (const result of closeResults) { + if (result.status === 'rejected') log.warn('Failed to close a deploy-validation Scope', result.reason); } } - throw outErr; + })(); + // Track the load+close so a concurrent worker shutdown waits for these scopes to finish disposing. + trackScopeClose(validation); + await validation; + emit('phase', { phase: 'load', status: 'done' }); + if (lastError) throw lastError; +} + +// Build a replicated sub-operation body (stage_component / activate_component) from the deploy +// request. Carries only what a peer needs: project, the deployment id (correlation + payload lookup + +// staging id), the build/config inputs, and credential REFERENCES (tokens are already stripped). +function buildReplicatedSubOp(req, operation, { includePayload = false, restart, deploymentId } = {}) { + const op = { operation, project: req.project, _deploymentId: deploymentId ?? req._deploymentId }; + if (req.package) op.package = req.package; + if (req.install_command != null) op.install_command = req.install_command; + if (req.install_timeout != null) op.install_timeout = req.install_timeout; + if (req.install_allow_scripts !== undefined) op.install_allow_scripts = req.install_allow_scripts; + if (req.deployment_timeout != null) op.deployment_timeout = req.deployment_timeout; + if (req.urlPath !== undefined) op.urlPath = req.urlPath; + if (req.force !== undefined) op.force = req.force; + if (req.ignore_replication_errors !== undefined) op.ignore_replication_errors = req.ignore_replication_errors; + if (Array.isArray(req.credentials) && req.credentials.length) op.credentials = req.credentials; + if (includePayload && req.payload != null) op.payload = req.payload; + if (restart !== undefined) op.restart = restart; + return op; +} + +// Render failed peer outcomes as "node (error)" for an operator-facing error message. +function describePeers(failedPeers) { + return failedPeers.map((peer) => `${peer.node ?? 'unknown'} (${peer.error?.message ?? 'unknown error'})`).join(', '); +} + +// Reclaim the payload tarball for large deploys once every peer has installed from the blob. Dropping +// the reference before finish() folds the null into the single terminal write, which unlinks the file +// locally and replicates the null so peers drop their copies too. Metadata is retained. Kept when a +// peer failed (still the retry artifact) or the payload is small. +function maybeReclaimPayload(recorder, emit) { + const payloadSize = recorder.row.payload_size; + const retentionMaxSize = getPayloadRetentionMaxSize(); + if (typeof payloadSize === 'number' && payloadSize > retentionMaxSize && recorder.getFailedPeers().length === 0) { + const freed = recorder.dropPayload(); + if (freed > 0) emit('payload_dropped', { payload_size: freed, max_size: retentionMaxSize }); + } +} + +// Build the structured failure (phase, install-output tail, deployment_id, failed peers) that the +// Fastify error handler forwards verbatim, emit the matching SSE 'error' event so the two transports +// stay symmetric, and record the terminal failure row. Returns the ServerError to throw. +async function finalizeDeployFailure({ err, recorder, installCapture, emit }) { + const capture = installCapture.snapshot(); + const phase = recorder?.row.phase; + const baseMessage = err?.message ?? String(err); + const structured = { error: baseMessage }; + if (phase) structured.phase = phase; + if (capture.lines.length > 0) structured.install_output = capture; + if (recorder?.deploymentId) structured.deployment_id = recorder.deploymentId; + const failedPeers = recorder?.getFailedPeers() ?? []; + if (failedPeers.length > 0) structured.failed_peers = failedPeers; + + // Wrap as a ServerError so the Fastify error handler picks a 500 by default; preserve an upstream + // statusCode (e.g. a ClientError from payload validation) if present. + const outErr = new ServerError(baseMessage, err?.statusCode); + outErr.http_resp_msg = structured; + + emit('error', { + message: baseMessage, + code: outErr?.statusCode ?? err?.code, + phase, + install_output: capture.lines.length > 0 ? capture : undefined, + deployment_id: recorder?.deploymentId, + failed_peers: failedPeers.length > 0 ? failedPeers : undefined, + }); + // Record the terminal failure, but never let a finish() write error mask the actual deploy failure. + if (recorder) { + try { + await recorder.finish('failed', err); + } catch (finishErr) { + log.warn('Failed to record deployment failure row', finishErr); + } } + return outErr; } // Ring buffer of install stdout/stderr lines, capped by both line count and bytes so @@ -796,7 +1176,7 @@ async function getComponents() { const list = await fs.readdir(dir, { withFileTypes: true }); for (let item of list) { const itemName = item.name; - if (itemName === 'node_modules' || itemName === ASIDE_STAGING_DIR) continue; + if (itemName === 'node_modules' || itemName === ASIDE_STAGING_DIR || itemName === DEPLOY_STAGING_DIR) continue; const itemPath = path.join(dir, itemName); if (item.isDirectory() || item.isSymbolicLink()) { let res = { @@ -1129,6 +1509,8 @@ exports.addComponent = addComponent; exports.dropCustomFunctionProject = dropCustomFunctionProject; exports.packageComponent = packageComponent; exports.deployComponent = deployComponent; +exports.stageComponent = stageComponent; +exports.activateComponent = activateComponent; exports.getComponents = getComponents; exports.getComponentFile = getComponentFile; exports.setComponentFile = setComponentFile; diff --git a/components/operationsValidation.js b/components/operationsValidation.js index b14321d0ed..8a865d37bc 100644 --- a/components/operationsValidation.js +++ b/components/operationsValidation.js @@ -25,6 +25,8 @@ module.exports = { dropCustomFunctionProjectValidator, packageComponentValidator, deployComponentValidator, + stageComponentValidator, + activateComponentValidator, setComponentFileValidator, getComponentFileValidator, dropComponentFileValidator, @@ -434,6 +436,39 @@ const GIT_CREDENTIAL_ENTRY = Joi.object({ .xor('token', 'secret') .unknown(false); +// The kind-heterogeneous deploy credentials array, shared by deploy_component and its two-phase +// sub-operations (stage_component / activate_component) so the three stay in lockstep. An entry's +// kind is implied by its identifying key (`registry` → npm registry auth, otherwise git host auth), +// dispatched here so a malformed entry reports what is actually wrong with it rather than a generic +// "no alternative matched". +const CREDENTIALS_ARRAY_SCHEMA = Joi.array() + .items( + Joi.alternatives().conditional('.registry', { + is: Joi.exist(), + then: REGISTRY_CREDENTIAL_ENTRY, + otherwise: GIT_CREDENTIAL_ENTRY, + }) + ) + .optional(); + +// A component's URL mount path. Rejects `..` so a deploy can't mount a component outside its intended +// path. Shared across deploy_component and the two-phase sub-operations that persist component config. +const URL_PATH_SCHEMA = Joi.string() + .min(1) + .custom((value, helpers) => { + if (value.includes('..')) return helpers.error('any.invalid'); + return value; + }) + .optional() + .messages({ 'any.invalid': 'urlPath must not contain ".."' }); + +// `registryAuth` was the credentials field's name on the 5.2 dev line. Rejected rather than ignored: +// validation allows unknown keys, so a caller still sending it would otherwise get a deploy that +// silently installs with no credentials. Shared so every deploy-family op rejects it identically. +const FORBIDDEN_REGISTRY_AUTH = Joi.any().forbidden().messages({ + 'any.unknown': `'registryAuth' has been renamed to 'credentials'`, +}); + /** * Validate deployComponent requests. * @param req @@ -453,44 +488,80 @@ function deployComponentValidator(req) { deployment_timeout: Joi.number().min(0).optional(), force: Joi.boolean().optional(), ignore_replication_errors: Joi.boolean().optional(), - urlPath: Joi.string() - .min(1) - .custom((value, helpers) => { - if (value.includes('..')) return helpers.error('any.invalid'); - return value; - }) - .optional() - .messages({ 'any.invalid': 'urlPath must not contain ".."' }), - // Deploy credentials. The array is kind-heterogeneous: an entry's kind is implied by its - // identifying key rather than a separate discriminator field, so a new kind is added as - // another item alternative here without reshaping the field. Today: npm registry auth - // (`registry`) and git host auth for a git-reference package (`host`, #1792). - // - // Every kind supplies its credential exactly one of two ways: - // - `token`: a literal token, used only for this node's install and never persisted or - // replicated (stripped from req before replicateOperation). - // - `secret`: the name of an hdb_secret row (#1550); the token is resolved by decrypting - // that row on this node at deploy time, so the credential lives in the secrets store - // (reference, not embed) instead of travelling in the operation body. - // Dispatched on the presence of `registry` rather than tried as alternatives, so a malformed - // entry reports what is actually wrong with it (a newline in the token, an invalid scope) rather - // than a generic "no alternative matched". - credentials: Joi.array() - .items( - Joi.alternatives().conditional('.registry', { - is: Joi.exist(), - then: REGISTRY_CREDENTIAL_ENTRY, - otherwise: GIT_CREDENTIAL_ENTRY, - }) - ) - .optional(), - // `registryAuth` was this field's name on the 5.2 dev line before it grew to carry other - // credential kinds. Rejected rather than ignored: validation allows unknown keys, so a caller - // still sending it would otherwise get a deploy that silently installs with no credentials. - registryAuth: Joi.any().forbidden().messages({ - 'any.unknown': `'registryAuth' has been renamed to 'credentials'`, - }), + // Opt out of the two-phase (stage-then-activate) deploy and use the legacy one-shot path + // instead. Provided as an escape hatch for mixed-version clusters where a peer predates the + // stage_component/activate_component operations. Defaults to two-phase. + two_phase: Joi.boolean().optional(), + urlPath: URL_PATH_SCHEMA, + // Deploy credentials. Each entry is npm registry auth (`registry`) or git host auth (`host`, + // #1792), and supplies its secret either as a literal `token` (used only for this node's + // install, never persisted/replicated) or a `secret` reference to an hdb_secret row (#1550). + credentials: CREDENTIALS_ARRAY_SCHEMA, + registryAuth: FORBIDDEN_REGISTRY_AUTH, }).with('urlPath', 'package'); return validator.validateBySchema(req, deployProjSchema); } + +/** + * Validate stage_component requests — phase 1 of a two-phase deploy. Accepts the same build-time + * inputs as deploy_component (package/payload, install options, credentials) but no go-live controls + * (`restart`), since staging never restarts. `restart` is intentionally absent; a stray one is + * ignored (operations validate with allowUnknown). + * @param req + * @returns {*} + */ +function stageComponentValidator(req) { + const stageSchema = Joi.object({ + project: Joi.string() + .pattern(PROJECT_FILE_NAME_REGEX) + .required() + .messages({ 'string.pattern.base': HDB_ERROR_MSGS.BAD_PROJECT_NAME }), + package: Joi.string().optional(), + install_command: Joi.string().optional(), + install_timeout: Joi.number().optional(), + install_allow_scripts: Joi.boolean().optional(), + deployment_timeout: Joi.number().min(0).optional(), + force: Joi.boolean().optional(), + // urlPath is not applied at stage time (config is written at activate), but it is accepted and + // carried through so a single request body can flow stage → activate unchanged. + urlPath: URL_PATH_SCHEMA, + credentials: CREDENTIALS_ARRAY_SCHEMA, + registryAuth: FORBIDDEN_REGISTRY_AUTH, + }).with('urlPath', 'package'); + + return validator.validateBySchema(req, stageSchema); +} + +/** + * Validate activate_component requests — phase 2 of a two-phase deploy. Swaps an already-staged + * build (identified by `deployment_id`) into the live path and optionally restarts. Also carries the + * config-persistence inputs (package/install/credentials/urlPath) so a `package` deploy's root config + * is written at go-live rather than before the bits are in place. + * @param req + * @returns {*} + */ +function activateComponentValidator(req) { + const activateSchema = Joi.object({ + project: Joi.string() + .pattern(PROJECT_FILE_NAME_REGEX) + .required() + .messages({ 'string.pattern.base': HDB_ERROR_MSGS.BAD_PROJECT_NAME }), + // Identifies which staged build to activate. Required for a standalone activate; the + // deploy_component orchestrator supplies it as the deployment id it staged under. + deployment_id: Joi.string().optional(), + package: Joi.string().optional(), + install_command: Joi.string().optional(), + install_timeout: Joi.number().optional(), + install_allow_scripts: Joi.boolean().optional(), + deployment_timeout: Joi.number().min(0).optional(), + restart: Joi.alternatives().try(Joi.boolean(), Joi.string().valid('rolling')).optional(), + force: Joi.boolean().optional(), + ignore_replication_errors: Joi.boolean().optional(), + urlPath: URL_PATH_SCHEMA, + credentials: CREDENTIALS_ARRAY_SCHEMA, + registryAuth: FORBIDDEN_REGISTRY_AUTH, + }).with('urlPath', 'package'); + + return validator.validateBySchema(req, activateSchema); +} diff --git a/server/serverHelpers/serverHandlers.js b/server/serverHelpers/serverHandlers.js index c0f132ba40..3b52339b51 100644 --- a/server/serverHelpers/serverHandlers.js +++ b/server/serverHelpers/serverHandlers.js @@ -33,6 +33,8 @@ const { ProgressEmitter, createSSEResponseStream } = require('./progressEmitter. // the client disconnects (the emitter's abort signal), so subscribers see new lines live. const SSE_PROGRESS_OPERATIONS = new Set([ terms.OPERATIONS_ENUM.DEPLOY_COMPONENT, + terms.OPERATIONS_ENUM.STAGE_COMPONENT, + terms.OPERATIONS_ENUM.ACTIVATE_COMPONENT, terms.OPERATIONS_ENUM.GET_DEPLOYMENT, terms.OPERATIONS_ENUM.READ_LOG, ]); diff --git a/server/serverHelpers/serverUtilities.ts b/server/serverHelpers/serverUtilities.ts index 694185478e..8bc2d83b6e 100644 --- a/server/serverHelpers/serverUtilities.ts +++ b/server/serverHelpers/serverUtilities.ts @@ -534,6 +534,14 @@ function initializeOperationFunctionMap(): Map assert.strictEqual(result, undefined, `expected valid, got: ${result && result.message}`); +const rejected = (result) => assert.ok(result, 'expected a validation error'); + +describe('stageComponentValidator', () => { + it('accepts a project-only request', () => { + ok(validator.stageComponentValidator({ project: 'my_app' })); + }); + + it('accepts a package deploy with install options', () => { + ok( + validator.stageComponentValidator({ + project: 'my_app', + package: 'npm:@org/thing', + install_command: 'npm ci', + install_timeout: 60000, + install_allow_scripts: false, + deployment_timeout: 120000, + }) + ); + }); + + it('requires a project', () => { + rejected(validator.stageComponentValidator({ package: 'npm:@org/thing' })); + }); + + it('rejects an invalid project name', () => { + rejected(validator.stageComponentValidator({ project: 'bad/name' })); + }); + + it('rejects a urlPath containing ".."', () => { + rejected(validator.stageComponentValidator({ project: 'my_app', package: 'npm:x', urlPath: '/a/../b' })); + }); +}); + +describe('activateComponentValidator', () => { + it('accepts a project + deployment_id', () => { + ok(validator.activateComponentValidator({ project: 'my_app', deployment_id: 'abc-123' })); + }); + + it('accepts a rolling restart', () => { + ok(validator.activateComponentValidator({ project: 'my_app', deployment_id: 'abc-123', restart: 'rolling' })); + }); + + it('accepts a boolean restart and ignore_replication_errors', () => { + ok( + validator.activateComponentValidator({ + project: 'my_app', + deployment_id: 'abc-123', + restart: true, + ignore_replication_errors: true, + }) + ); + }); + + it('requires a project', () => { + rejected(validator.activateComponentValidator({ deployment_id: 'abc-123' })); + }); + + it('rejects an invalid restart value', () => { + rejected(validator.activateComponentValidator({ project: 'my_app', restart: 'sideways' })); + }); +}); + +describe('deployComponentValidator two_phase flag', () => { + it('accepts two_phase: false (legacy opt-out)', () => { + ok(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', two_phase: false })); + }); + + it('accepts two_phase: true', () => { + ok(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', two_phase: true })); + }); + + it('rejects a non-boolean two_phase', () => { + rejected(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', two_phase: 'yes' })); + }); +}); diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js new file mode 100644 index 0000000000..0998d2e70c --- /dev/null +++ b/unitTests/components/deployStaging.test.js @@ -0,0 +1,171 @@ +'use strict'; + +// Unit tests for the two-phase deploy primitives in components/Application.ts: +// stageApplication (build the incoming version into a hidden staging dir, never touching the live +// path), activateApplication (atomically swap the staged copy into the live path), and +// discardStagedApplication (drop an aborted stage). These exercise the real filesystem — no +// componentLoader, no network — so they run without the private agent dependency. + +const assert = require('node:assert'); +const fs = require('node:fs/promises'); +const { existsSync } = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const zlib = require('node:zlib'); +const tarfs = require('tar-fs'); + +const testUtils = require('../testUtils.js'); +testUtils.preTestPrep(); + +const { + Application, + stageApplication, + activateApplication, + discardStagedApplication, + DEPLOY_STAGING_DIR, + ASIDE_STAGING_DIR, +} = require('#src/components/Application'); +const { getConfigPath } = require('#src/config/configUtils'); +const { CONFIG_PARAMS } = require('#src/utility/hdbTerms'); + +const COMPONENTS_ROOT = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); + +// Pack a directory's CONTENTS into a gzipped tar Buffer, the shape a deploy payload takes. +function packDirectory(dir) { + return new Promise((resolve, reject) => { + const chunks = []; + tarfs + .pack(dir) + .pipe(zlib.createGzip()) + .on('data', (c) => chunks.push(c)) + .on('end', () => resolve(Buffer.concat(chunks))) + .on('error', reject); + }); +} + +// A minimal component source that already contains node_modules, so installApplication short-circuits +// ("already has node_modules; skipping install") and the test needs no npm/network. +async function makeComponentPayload(marker) { + const src = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-stage-src-')); + await fs.writeFile(path.join(src, 'package.json'), JSON.stringify({ name: 'stage-fixture', version: '1.0.0' })); + await fs.writeFile(path.join(src, 'index.js'), `module.exports = ${JSON.stringify(marker)};\n`); + await fs.mkdir(path.join(src, 'node_modules'), { recursive: true }); + await fs.writeFile(path.join(src, 'node_modules', '.marker'), marker); + const payload = await packDirectory(src); + await fs.rm(src, { recursive: true, force: true }); + return payload; +} + +async function readMarker(dir) { + return fs.readFile(path.join(dir, 'index.js'), 'utf8'); +} + +describe('two-phase deploy primitives (stage / activate / discard)', function () { + this.timeout(30_000); + + before(async () => { + await fs.mkdir(COMPONENTS_ROOT, { recursive: true }); + }); + + // Each case uses a unique component name so the tests are order-independent and don't collide. + let counter = 0; + function freshApp(payload) { + const name = `stage_test_${process.pid}_${counter++}`; + return new Application({ name, payload }); + } + + it('stageApplication builds into the hidden staging dir and does NOT touch the live path', async () => { + const app = freshApp(await makeComponentPayload('v1')); + + assert.ok(app.stagingDirPath.includes(DEPLOY_STAGING_DIR), 'staging path is under the staging dir'); + assert.strictEqual(app.buildDirPath, app.dirPath, 'build target defaults to the live dir before staging'); + + const stagedPath = await stageApplication(app); + + assert.strictEqual(stagedPath, app.stagingDirPath); + assert.ok(existsSync(path.join(app.stagingDirPath, 'index.js')), 'component extracted into staging'); + assert.ok(existsSync(path.join(app.stagingDirPath, 'node_modules')), 'node_modules present in staging'); + assert.strictEqual(existsSync(app.dirPath), false, 'live component dir was NOT created by staging'); + + await fs.rm(path.dirname(app.stagingDirPath), { recursive: true, force: true }); + }); + + it('activateApplication swaps the staged copy into the live path atomically', async () => { + const app = freshApp(await makeComponentPayload('v1')); + await stageApplication(app); + await activateApplication(app); + + assert.ok(existsSync(app.dirPath), 'live component dir now exists'); + assert.match(await readMarker(app.dirPath), /v1/, 'live dir holds the staged content'); + assert.strictEqual(existsSync(app.stagingDirPath), false, 'the staged copy was consumed by the swap'); + assert.strictEqual(app.buildDirPath, app.dirPath, 'build target reset to live after activation'); + + await fs.rm(app.dirPath, { recursive: true, force: true }); + }); + + it('stage → activate replaces an existing live version and moves the old one aside', async () => { + const name = `stage_test_${process.pid}_${counter++}`; + const dirPath = path.join(COMPONENTS_ROOT, name); + // Seed an existing live version. + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'index.js'), 'module.exports = "OLD";\n'); + await fs.writeFile(path.join(dirPath, 'leftover.txt'), 'from the old version'); + + const app = new Application({ name, payload: await makeComponentPayload('v2') }); + await stageApplication(app); + await activateApplication(app); + + assert.match(await readMarker(dirPath), /v2/, 'live dir now holds the new version'); + assert.strictEqual(existsSync(path.join(dirPath, 'leftover.txt')), false, 'old-version files are gone from live'); + + await fs.rm(dirPath, { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + }); + + it('discardStagedApplication removes the staging tree and leaves the live path untouched', async () => { + const name = `stage_test_${process.pid}_${counter++}`; + const dirPath = path.join(COMPONENTS_ROOT, name); + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'index.js'), 'module.exports = "LIVE";\n'); + + const app = new Application({ name, payload: await makeComponentPayload('v3') }); + await stageApplication(app); + assert.ok(existsSync(app.stagingDirPath), 'staged before discard'); + + await discardStagedApplication(app); + + assert.strictEqual(existsSync(app.stagingDirPath), false, 'staging tree removed'); + assert.match(await readMarker(dirPath), /LIVE/, 'live version untouched by discard'); + + await fs.rm(dirPath, { recursive: true, force: true }); + }); + + it('a failed stage leaves the live path untouched and cleans up its partial staging tree', async () => { + const name = `stage_test_${process.pid}_${counter++}`; + const dirPath = path.join(COMPONENTS_ROOT, name); + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'index.js'), 'module.exports = "LIVE";\n'); + + // No payload and no package identifier → extractApplication throws before anything is built. + const app = new Application({ name }); + await assert.rejects(() => stageApplication(app), /payload or package/i); + + assert.strictEqual(existsSync(app.stagingDirPath), false, 'partial staging tree removed on failure'); + assert.match(await readMarker(dirPath), /LIVE/, 'live version untouched by a failed stage'); + assert.strictEqual(app.buildDirPath, app.dirPath, 'build target reset to live after a failed stage'); + + await fs.rm(dirPath, { recursive: true, force: true }); + }); + + it('two independent components stage into non-colliding staging dirs', async () => { + const a = freshApp(await makeComponentPayload('A')); + const b = freshApp(await makeComponentPayload('B')); + await Promise.all([stageApplication(a), stageApplication(b)]); + + assert.notStrictEqual(a.stagingDirPath, b.stagingDirPath); + assert.match(await readMarker(a.stagingDirPath), /A/); + assert.match(await readMarker(b.stagingDirPath), /B/); + + await Promise.all([discardStagedApplication(a), discardStagedApplication(b)]); + }); +}); diff --git a/unitTests/server/fastifyRoutes/operations.test.js b/unitTests/server/fastifyRoutes/operations.test.js index 2643fc8e53..31075fa4bd 100644 --- a/unitTests/server/fastifyRoutes/operations.test.js +++ b/unitTests/server/fastifyRoutes/operations.test.js @@ -520,9 +520,11 @@ describe('Test custom functions operations', () => { // Mock addConfig to prevent actual file writes const addConfigStub = sandbox.stub(configUtils, 'addConfig').resolves(); - // Mock prepareApplication to prevent actual installation - const prepareApplicationStub = sandbox.stub(); - operations.__set__('prepareApplication', prepareApplicationStub); + // Mock the two-phase build/swap primitives to prevent actual install + filesystem work. + const stageApplicationStub = sandbox.stub().resolves('/tmp/staging'); + const activateApplicationStub = sandbox.stub().resolves(); + operations.__set__('stageApplication', stageApplicationStub); + operations.__set__('activateApplication', activateApplicationStub); // This should work - user components can be overwritten without force await operations.deployComponent({ @@ -530,13 +532,14 @@ describe('Test custom functions operations', () => { package: '@org/new-package', }); - // Verify addConfig was called + // Verify addConfig was called (at activation, once the bits are staged) expect(addConfigStub.calledOnce).to.be.true; expect(addConfigStub.firstCall.args[0]).to.equal('existing-component'); expect(addConfigStub.firstCall.args[1].package).to.equal('@org/new-package'); - // Verify prepareApplication was called - expect(prepareApplicationStub.calledOnce).to.be.true; + // Verify the component was staged then activated + expect(stageApplicationStub.calledOnce).to.be.true; + expect(activateApplicationStub.calledOnce).to.be.true; }); it('Test deployComponent allows deploying new component without force flag', async () => { @@ -546,9 +549,11 @@ describe('Test custom functions operations', () => { // Mock addConfig to prevent actual file writes const addConfigStub = sandbox.stub(configUtils, 'addConfig').resolves(); - // Mock prepareApplication to prevent actual installation - const prepareApplicationStub = sandbox.stub(); - operations.__set__('prepareApplication', prepareApplicationStub); + // Mock the two-phase build/swap primitives to prevent actual install + filesystem work. + const stageApplicationStub = sandbox.stub().resolves('/tmp/staging'); + const activateApplicationStub = sandbox.stub().resolves(); + operations.__set__('stageApplication', stageApplicationStub); + operations.__set__('activateApplication', activateApplicationStub); // This should work fine - no component exists yet await operations.deployComponent({ @@ -560,8 +565,9 @@ describe('Test custom functions operations', () => { expect(addConfigStub.calledOnce).to.be.true; expect(addConfigStub.firstCall.args[0]).to.equal('new-component'); - // Verify prepareApplication was called - expect(prepareApplicationStub.calledOnce).to.be.true; + // Verify the component was staged then activated + expect(stageApplicationStub.calledOnce).to.be.true; + expect(activateApplicationStub.calledOnce).to.be.true; }); it('Test deployComponent prevents overwriting core component without force flag', async () => { @@ -592,9 +598,11 @@ describe('Test custom functions operations', () => { // Mock addConfig to prevent actual file writes const addConfigStub = sandbox.stub(configUtils, 'addConfig').resolves(); - // Mock prepareApplication to prevent actual installation - const prepareApplicationStub = sandbox.stub(); - operations.__set__('prepareApplication', prepareApplicationStub); + // Mock the two-phase build/swap primitives to prevent actual install + filesystem work. + const stageApplicationStub = sandbox.stub().resolves('/tmp/staging'); + const activateApplicationStub = sandbox.stub().resolves(); + operations.__set__('stageApplication', stageApplicationStub); + operations.__set__('activateApplication', activateApplicationStub); // This should NOT throw an error because force is true await operations.deployComponent({ @@ -608,8 +616,9 @@ describe('Test custom functions operations', () => { expect(addConfigStub.firstCall.args[0]).to.equal('graphql'); expect(addConfigStub.firstCall.args[1].package).to.equal('@org/override-package'); - // Verify prepareApplication was called - expect(prepareApplicationStub.calledOnce).to.be.true; + // Verify the component was staged then activated + expect(stageApplicationStub.calledOnce).to.be.true; + expect(activateApplicationStub.calledOnce).to.be.true; }); it('Test deployComponent prevents overwriting multiple core component names', async () => { diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index cc708f15f7..c428264be1 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -291,6 +291,12 @@ export const OPERATIONS_ENUM = { DEPLOY_CUSTOM_FUNCTION_PROJECT: 'deploy_custom_function_project', PACKAGE_COMPONENT: 'package_component', DEPLOY_COMPONENT: 'deploy_component', + // Two-phase deploy sub-operations. stage_component builds the incoming version into a hidden + // staging directory cluster-wide (no go-live); activate_component atomically swaps the staged + // copy into the live path and restarts. deploy_component orchestrates the two so existing callers + // are unaffected. See components/Application.ts (stageApplication/activateApplication). + STAGE_COMPONENT: 'stage_component', + ACTIVATE_COMPONENT: 'activate_component', READ_TRANSACTION_LOG: 'read_transaction_log', DELETE_TRANSACTION_LOGS_BEFORE: 'delete_transaction_logs_before', INSTALL_NODE_MODULES: 'install_node_modules', diff --git a/utility/operation_authorization.ts b/utility/operation_authorization.ts index 112ab277e2..dcb70b6c26 100644 --- a/utility/operation_authorization.ts +++ b/utility/operation_authorization.ts @@ -286,6 +286,8 @@ requiredPermissions.set(functionsOperations.addComponent.name, new (permission a requiredPermissions.set(functionsOperations.dropCustomFunctionProject.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.packageComponent.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.deployComponent.name, new (permission as any)(true, [])); +requiredPermissions.set(functionsOperations.stageComponent.name, new (permission as any)(true, [])); +requiredPermissions.set(functionsOperations.activateComponent.name, new (permission as any)(true, [])); requiredPermissions.set( deploymentOperations.handleListDeployments.name, new (permission as any)(true, [], terms.OPERATIONS_ENUM.LIST_DEPLOYMENTS) From 909f384854d55b103234bcf07ac21855e5610384 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 17 Jul 2026 15:33:48 -0400 Subject: [PATCH 02/94] fix(deploy): staging parent for npm pack cwd, symlink-safe aside, sibling-safe cleanup Three fixes to the two-phase staging path: - Create the per-deploy staging parent before extraction. For a `package` deploy the first filesystem touch is the `npm pack`/git-clone spawn, whose cwd is dirname(stagingDirPath); it didn't exist yet, so the spawn failed with `ENOENT posix_spawn` (surfaced by the deploy-from-github integration test; the payload-only unit tests never hit the spawn). stageApplication now mkdirs it. - moveDirAside uses lstat, not access(F_OK): access follows symlinks, so a DANGLING symlink at the target reported ENOENT and was skipped, then mkdir failed EEXIST. lstat sees the link itself. (Gemini review.) - Reorder staging to .deploy-staging// (was /). The leaf basename is now the component name, which the pre-go-live validation load needs (componentLoader keys ApplicationScope/status off basename); and each deploy gets its own parent, so cleanup can't sweep a parallel/queued deploy's staged build. activate cleanup is a non-recursive rmdir of that parent (empty-only). (Gemini review.) Adds regression tests: a `file:`-tarball package-path stage, sibling-build survival across activate, and dangling-symlink aside on activate. Co-Authored-By: Claude Opus 4.8 --- DESIGN.md | 10 +++- components/Application.ts | 44 ++++++++++++---- unitTests/components/deployStaging.test.js | 60 ++++++++++++++++++++++ 3 files changed, 101 insertions(+), 13 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 37fef08b92..eb21179be5 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -321,7 +321,7 @@ could leave a peer half-installed after other peers had already restarted onto t request/response contract is unchanged; only the SSE phase names differ (`stage`/`activate` vs the old `prepare`/`replicate`). `two_phase: false` forces the legacy one-shot path. -The staging directory (`.deploy-staging//`) lives **under the components root**, +The staging directory (`.deploy-staging//`) lives **under the components root**, not in `os.tmpdir()`, even though its contents are transient. This is deliberate and load-bearing: the go-live step is `rename(stagingDir, liveDir)`, which is only atomic when both paths share a filesystem. `os.tmpdir()` is frequently a different mount (tmpfs, a separate volume); a cross-device @@ -335,7 +335,13 @@ scoped to `activateApplication`, the only phase that writes the live path. Stagi from the deployment id precisely so `activate_component` (a separate replicated operation, and on peers a separate invocation from `stage_component`) can reconstruct the same path the stage built — peers build a fresh `Application` per sub-operation, so there is no shared in-memory handle to rely -on. `extractApplication`/`installApplication` build into `application.buildDirPath`, which defaults +on. The deployment id sits ABOVE the component name (`…//`, not +`…//`) for two reasons: the leaf directory's basename is then the real component +name, which the pre-go-live validation load needs (`componentLoader` keys the `ApplicationScope` and +status registry off `basename(componentDirectory)`, so a UUID leaf would register the throwaway load +under a bogus name); and each deploy gets its own parent directory, so a parallel or queued deploy of +the same component can never share a directory or have its staged build swept by another's cleanup. +`extractApplication`/`installApplication` build into `application.buildDirPath`, which defaults to the live dir (`dirPath`) — this is what keeps the legacy one-shot path, boot-time `installApplications`, and the direct `extractApplication` callers unchanged — and is repointed at the staging dir only for the duration of a stage. diff --git a/components/Application.ts b/components/Application.ts index d5de3bfe69..dbeef9fdc6 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -18,12 +18,14 @@ import { access, constants, cp, + lstat, mkdir, mkdtemp, readdir, readFile, rename, rm, + rmdir, stat, symlink, writeFile, @@ -438,7 +440,11 @@ export const DEPLOY_STAGING_DIR = '.deploy-staging'; async function moveDirAside(targetDirPath: string): Promise { const asideStagingDir = join(dirname(targetDirPath), ASIDE_STAGING_DIR, basename(targetDirPath)); try { - await access(targetDirPath, constants.F_OK); + // lstat, not access(F_OK): access follows symlinks, so a DANGLING symlink at the path (left by a + // prior `file:`-directory deploy whose target was removed) would report ENOENT and be skipped + // here — then mkdir(targetDirPath) fails EEXIST because the dead link still occupies the path. + // lstat sees the link itself, so we move it aside like any other occupant. + await lstat(targetDirPath); } catch (err) { if (err.code === 'ENOENT') return null; // nothing there to move throw err; @@ -942,12 +948,19 @@ export class Application { return this.#buildDirPath ?? this.dirPath; } - // Hidden, per-deploy staging directory the incoming version is built into before it goes live. - // Deterministic from (component name, stagingId) so `activate_component` can find what - // `stage_component` built. Sits under the components root (dirname(dirPath)) so the go-live - // rename() into `dirPath` stays on one filesystem and is therefore atomic. See DEPLOY_STAGING_DIR. + // Hidden, per-deploy staging directory the incoming version is built into before it goes live: + // `/.deploy-staging//`. Deterministic from (stagingId, component + // name) so `activate_component` can find what `stage_component` built. Sits under the components + // root (dirname(dirPath)) so the go-live rename() into `dirPath` stays on one filesystem and is + // therefore atomic. Two properties fall out of putting the deployment id ABOVE the component name: + // - the leaf directory's basename IS the component name, so the pre-go-live validation load + // (componentLoader keys the ApplicationScope + status off basename) sees the real name, not a + // UUID; and + // - each deployment gets its OWN parent (.deploy-staging/), so a parallel or queued + // deploy of the same component never shares a directory — cleanup can't sweep a sibling. + // See DEPLOY_STAGING_DIR. get stagingDirPath(): string { - return join(dirname(this.dirPath), DEPLOY_STAGING_DIR, this.name, this.stagingId); + return join(dirname(this.dirPath), DEPLOY_STAGING_DIR, this.stagingId, this.name); } // Route extract/install into the staging directory. Called by stageApplication(). @@ -1124,6 +1137,11 @@ export async function stageApplication(application: Application): Promise) up front. extractApplication's + // own mkdir only covers the payload path — for a `package` deploy the FIRST filesystem touch is the + // `npm pack`/git-clone spawn, whose cwd is this parent directory; without it the spawn fails with + // ENOENT (posix_spawn) before any tarball is produced. + await mkdir(dirname(application.stagingDirPath), { recursive: true }); try { await application.writeTransientNpmrc(); try { @@ -1185,12 +1203,16 @@ export async function activateApplication(application: Application): Promise). Remove it with a NON-recursive rmdir, which succeeds only when it + // is empty — a belt-and-suspenders guard against ever recursively deleting a directory that could + // hold another deploy's build. ENOTEMPTY and ENOENT (already gone) are expected and ignored. cleanupAsideDir(asideStagingDir, application.name); - rm(dirname(stagingDirPath), { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }).catch((err) => - logger.trace?.(`Deferred cleanup of ${application.name} staging directory: ${err.message}`) - ); + rmdir(dirname(stagingDirPath)).catch((err) => { + if (err.code !== 'ENOTEMPTY' && err.code !== 'ENOENT') + logger.trace?.(`Deferred cleanup of ${application.name} staging directory: ${err.message}`); + }); } /** diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 0998d2e70c..283fb2f0a0 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -168,4 +168,64 @@ describe('two-phase deploy primitives (stage / activate / discard)', function () await Promise.all([discardStagedApplication(a), discardStagedApplication(b)]); }); + + it('stages a `file:` tarball package identifier (the package path, no payload)', async () => { + // Regression for the staging parent dir: extractApplication's `file:`-tarball branch (and the + // npm-pack branch) resolve paths relative to dirname(stagingDirPath), which must exist before + // extraction. A payload-only test never exercises that branch. + const name = `stage_test_${process.pid}_${counter++}`; + const tgzDir = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-stage-tgz-')); + const tgzPath = path.join(tgzDir, 'component.tgz'); + await fs.writeFile(tgzPath, await makeComponentPayload('from-tarball')); + + const app = new Application({ name, packageIdentifier: `file:${tgzPath}` }); + await stageApplication(app); + assert.match(await readMarker(app.stagingDirPath), /from-tarball/, 'tarball extracted into staging'); + assert.strictEqual(existsSync(app.dirPath), false, 'live dir untouched by staging a tarball'); + + await discardStagedApplication(app); + await fs.rm(tgzDir, { recursive: true, force: true }); + }); + + it('activate cleanup does NOT sweep a sibling staged build of the same component', async () => { + // Two deploys of the same component staged concurrently share the .deploy-staging/ parent. + // Activating one must not recursively delete that parent and destroy the other's staged build. + const name = `stage_test_${process.pid}_${counter++}`; + const first = new Application({ name, payload: await makeComponentPayload('first') }); + const second = new Application({ name, payload: await makeComponentPayload('second') }); + await stageApplication(first); + await stageApplication(second); + assert.notStrictEqual(first.stagingDirPath, second.stagingDirPath); + + await activateApplication(first); + + assert.match(await readMarker(first.dirPath), /first/, 'first went live'); + assert.ok(existsSync(second.stagingDirPath), 'the sibling staged build survived the activate cleanup'); + + await fs.rm(first.dirPath, { recursive: true, force: true }); + await discardStagedApplication(second); + await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + }); + + it('activate moves a DANGLING symlink at the live path aside instead of failing EEXIST', async () => { + // A prior `file:`-directory deploy leaves the live path as a symlink; if its target is later + // removed the link dangles. moveDirAside must detect it via lstat (access(F_OK) follows the link + // and reports ENOENT) so the swap replaces it cleanly. + const name = `stage_test_${process.pid}_${counter++}`; + const dirPath = path.join(COMPONENTS_ROOT, name); + await fs.symlink(path.join(os.tmpdir(), `does-not-exist-${process.pid}-${counter}`), dirPath); + assert.strictEqual(existsSync(dirPath), false, 'precondition: the symlink is dangling'); + + const app = new Application({ name, payload: await makeComponentPayload('replaced') }); + await stageApplication(app); + await activateApplication(app); + + const stat = await fs.lstat(dirPath); + assert.strictEqual(stat.isSymbolicLink(), false, 'live path is now a real directory, not the dead link'); + assert.match(await readMarker(dirPath), /replaced/); + + await fs.rm(dirPath, { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + }); }); From cb2020e60ddc36f04c25f538d50f5561e6c77279 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 17 Jul 2026 15:36:52 -0400 Subject: [PATCH 03/94] test(deploy): op-level coverage for stage_component, activate_component, one-shot Adds orchestration tests that call operations.stageComponent / activateComponent / deployComponent(two_phase:false) directly, stubbing the build/swap primitives, blob ingest, and credential resolution so they assert control flow (which primitive runs, replication + restart firing, response shape) without npm/network or the component loader. Covers the two new first-class operations and the legacy one-shot path (previously only exercised indirectly). Addresses the coverage gap flagged in review. Co-Authored-By: Claude Opus 4.8 --- .../components/deployPhaseOperations.test.js | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 unitTests/components/deployPhaseOperations.test.js diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js new file mode 100644 index 0000000000..3c7dc09c96 --- /dev/null +++ b/unitTests/components/deployPhaseOperations.test.js @@ -0,0 +1,98 @@ +'use strict'; + +// Operation-level orchestration tests for the two-phase deploy handlers: stage_component, +// activate_component, and the legacy one-shot deploy_component (two_phase: false). The heavy, +// environment-dependent internals (the actual build/swap, payload blob ingest, credential resolution) +// are rewired to stubs so these assert the ORCHESTRATION — which primitive runs, whether replication +// and restart fire, the response shape — without needing npm/network or the full component loader. + +const assert = require('node:assert'); +const rewire = require('rewire'); +const sinon = require('sinon'); + +const operations = rewire('#js/components/operations'); +const manageThreads = require('#src/server/threads/manageThreads'); + +// Neutralize the parts that touch the filesystem / datastore / cluster, leaving the control flow. +function stubInternals(sandbox) { + const stage = sandbox.stub().resolves('/staging'); + const activate = sandbox.stub().resolves(); + const prepare = sandbox.stub().resolves(); + operations.__set__('stageApplication', stage); + operations.__set__('activateApplication', activate); + operations.__set__('prepareApplication', prepare); + // Skip payload-blob ingest and credential resolution — not what these tests exercise. + operations.__set__('sourceExtractionPayload', sandbox.stub().resolves(Buffer.from('tarball'))); + operations.__set__('resolveNodeCredentials', sandbox.stub().resolves([])); + // Don't actually bounce workers. + sandbox.stub(manageThreads, 'restartWorkers'); + return { stage, activate, prepare }; +} + +describe('stage_component / activate_component / one-shot deploy orchestration', () => { + let sandbox; + beforeEach(() => { + sandbox = sinon.createSandbox(); + }); + afterEach(() => { + sandbox.restore(); + }); + + it('stageComponent stages, replicates, and returns a staged marker + deployment_id — no restart, no config write', async () => { + const { stage, activate, prepare } = stubInternals(sandbox); + const addConfig = sandbox.stub(require('#src/config/configUtils'), 'addConfig').resolves(); + + const res = await operations.stageComponent({ project: 'my_app', payload: Buffer.from('x') }); + + assert.strictEqual(stage.calledOnce, true, 'stageApplication was called'); + assert.strictEqual(activate.called, false, 'activate never runs during a stage'); + assert.strictEqual(prepare.called, false, 'the one-shot prepare path is not used'); + assert.strictEqual(res.staged, true); + assert.strictEqual(res.project, 'my_app'); + assert.ok(res.deployment_id, 'a deployment_id is returned'); + assert.strictEqual(addConfig.called, false, 'staging never writes root config'); + assert.strictEqual(manageThreads.restartWorkers.called, false, 'staging never restarts'); + }); + + it('activateComponent swaps live and restarts when restart:true; returns an activated marker', async () => { + const { activate } = stubInternals(sandbox); + + const res = await operations.activateComponent({ + project: 'my_app', + deployment_id: 'dep-123', + restart: true, + }); + + assert.strictEqual(activate.calledOnce, true, 'activateApplication was called'); + assert.strictEqual(res.activated, true); + assert.strictEqual(res.project, 'my_app'); + assert.strictEqual(manageThreads.restartWorkers.calledWith('http'), true, 'restarted the http workers'); + }); + + it('activateComponent does not restart when restart is omitted', async () => { + const { activate } = stubInternals(sandbox); + await operations.activateComponent({ project: 'my_app', deployment_id: 'dep-123' }); + assert.strictEqual(activate.calledOnce, true); + assert.strictEqual(manageThreads.restartWorkers.called, false); + }); + + it('activateComponent rejects when no deployment_id is supplied', async () => { + stubInternals(sandbox); + await assert.rejects(() => operations.activateComponent({ project: 'my_app' }), /deployment_id.*required/i); + }); + + it('deploy_component with two_phase:false takes the legacy one-shot path (prepareApplication, not stage/activate)', async () => { + const { stage, activate, prepare } = stubInternals(sandbox); + + const res = await operations.deployComponent({ + project: 'my_app', + payload: Buffer.from('x'), + two_phase: false, + }); + + assert.strictEqual(prepare.calledOnce, true, 'one-shot prepareApplication was called'); + assert.strictEqual(stage.called, false, 'two-phase stage not used on the one-shot path'); + assert.strictEqual(activate.called, false, 'two-phase activate not used on the one-shot path'); + assert.match(res.message, /Successfully deployed/); + }); +}); From 52718f6fde5dabfc434a54c4a6677179c23cd463 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 17 Jul 2026 15:44:42 -0400 Subject: [PATCH 04/94] test(deploy): update integration phase assertions to two-phase names MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two-phase deploy lifecycle emits stage/activate phases instead of the one-shot prepare/replicate. Update the deployment-tracking integration tests to match: a failed install is now recorded against phase 'stage' (not 'prepare'), and a successful deploy's event_log spine is stage → activate (not prepare → replicate). No behavior change — the recorded status='failed', error.message, install_output, deployment_id, error/payload_dropped events are all still asserted and preserved. Co-Authored-By: Claude Opus 4.8 --- integrationTests/deploy/deploy-tracking-events.test.ts | 7 ++++--- integrationTests/deploy/deploy-tracking.test.ts | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/integrationTests/deploy/deploy-tracking-events.test.ts b/integrationTests/deploy/deploy-tracking-events.test.ts index 0d80f2f600..49a290fc65 100644 --- a/integrationTests/deploy/deploy-tracking-events.test.ts +++ b/integrationTests/deploy/deploy-tracking-events.test.ts @@ -132,9 +132,10 @@ suite('Deployment tracking — events + SSE', (ctx: ContextWithHarper) => { ok(Array.isArray(row.event_log), 'event_log should be an array'); ok(row.event_log.length >= 2, `expected at least 2 events, got ${row.event_log.length}`); const phases = row.event_log.filter((e: any) => e.event === 'phase').map((e: any) => e.data?.phase); - // We emit prepare → (load) → replicate → success in the lifecycle. Verify the spine. - ok(phases.includes('prepare'), `event_log should include a prepare phase: ${phases.join(',')}`); - ok(phases.includes('replicate'), `event_log should include a replicate phase: ${phases.join(',')}`); + // A two-phase deploy emits stage → (load) → activate → success. Verify the spine (load is only + // emitted off the main thread, so it isn't asserted here). + ok(phases.includes('stage'), `event_log should include a stage phase: ${phases.join(',')}`); + ok(phases.includes('activate'), `event_log should include an activate phase: ${phases.join(',')}`); }); test('get_deployment with Accept: text/event-stream replays event_log and closes cleanly', async () => { diff --git a/integrationTests/deploy/deploy-tracking.test.ts b/integrationTests/deploy/deploy-tracking.test.ts index 57bc623c40..255e6d28b2 100644 --- a/integrationTests/deploy/deploy-tracking.test.ts +++ b/integrationTests/deploy/deploy-tracking.test.ts @@ -196,7 +196,9 @@ suite('Deployment tracking', (ctx: ContextWithHarper) => { // match what the install command actually printed; an empty array would pass the // pre-fix code, so assert on the line content too. const body = JSON.parse(response.body); - strictEqual(body.phase, 'prepare'); + // The install runs during the stage phase of a two-phase deploy, so a failed install is + // recorded against 'stage' (was 'prepare' in the one-shot flow). + strictEqual(body.phase, 'stage'); ok(Array.isArray(body.install_output?.lines) && body.install_output.lines.length > 0); ok( body.install_output.lines.some( From 64ad02b8f7b24629141dfeb4b8350d5dd5dd5ab6 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 17 Jul 2026 15:51:44 -0400 Subject: [PATCH 05/94] test(deploy): rewrite op-level tests as real-module tests (no sinon/rewire) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AGENTS.md forbids new sinon/rewire in unit tests; the prior version of this file stubbed the build primitives via rewire/sinon. Rewritten in the deployStaging.test.js style: plain node:assert against the real operation handlers, driving real tarball payloads through a real temp components root. Now exercises stage_component, activate_component, deploy_component (two-phase default), and deploy_component(two_phase:false) end-to-end — asserting the staged/live directories on disk, the deployment_id, the no-restart message, and the deployment_id requirement — which is stronger coverage than the stubs gave. Co-Authored-By: Claude Opus 4.8 --- .../components/deployPhaseOperations.test.js | 181 +++++++++++------- 1 file changed, 112 insertions(+), 69 deletions(-) diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index 3c7dc09c96..76945c031e 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -1,98 +1,141 @@ 'use strict'; -// Operation-level orchestration tests for the two-phase deploy handlers: stage_component, -// activate_component, and the legacy one-shot deploy_component (two_phase: false). The heavy, -// environment-dependent internals (the actual build/swap, payload blob ingest, credential resolution) -// are rewired to stubs so these assert the ORCHESTRATION — which primitive runs, whether replication -// and restart fire, the response shape — without needing npm/network or the full component loader. +// Operation-level tests for the two-phase deploy handlers: stage_component, activate_component, and +// deploy_component (both the two-phase default and the two_phase:false one-shot fallback). These run +// the real handlers against a real temp filesystem with a real tarball payload — no stubbing — in the +// deployStaging.test.js style (AGENTS.md: new tests use plain `assert` against real modules, no +// sinon/rewire). Payload deploys are used throughout so nothing reaches the component loader (a +// `package` deploy's protected-name guard would), and no test requests a restart. const assert = require('node:assert'); -const rewire = require('rewire'); -const sinon = require('sinon'); - -const operations = rewire('#js/components/operations'); -const manageThreads = require('#src/server/threads/manageThreads'); - -// Neutralize the parts that touch the filesystem / datastore / cluster, leaving the control flow. -function stubInternals(sandbox) { - const stage = sandbox.stub().resolves('/staging'); - const activate = sandbox.stub().resolves(); - const prepare = sandbox.stub().resolves(); - operations.__set__('stageApplication', stage); - operations.__set__('activateApplication', activate); - operations.__set__('prepareApplication', prepare); - // Skip payload-blob ingest and credential resolution — not what these tests exercise. - operations.__set__('sourceExtractionPayload', sandbox.stub().resolves(Buffer.from('tarball'))); - operations.__set__('resolveNodeCredentials', sandbox.stub().resolves([])); - // Don't actually bounce workers. - sandbox.stub(manageThreads, 'restartWorkers'); - return { stage, activate, prepare }; +const fs = require('node:fs/promises'); +const { existsSync } = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const zlib = require('node:zlib'); +const tarfs = require('tar-fs'); + +const testUtils = require('../testUtils.js'); +testUtils.preTestPrep(); + +const operations = require('#src/components/operations'); +const { DEPLOY_STAGING_DIR } = require('#src/components/Application'); +const { getConfigPath } = require('#src/config/configUtils'); +const { CONFIG_PARAMS } = require('#src/utility/hdbTerms'); + +const COMPONENTS_ROOT = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); + +// Pack a directory's CONTENTS into a gzipped tar Buffer, the shape a deploy payload takes. +function packDirectory(dir) { + return new Promise((resolve, reject) => { + const chunks = []; + tarfs + .pack(dir) + .pipe(zlib.createGzip()) + .on('data', (c) => chunks.push(c)) + .on('end', () => resolve(Buffer.concat(chunks))) + .on('error', reject); + }); } -describe('stage_component / activate_component / one-shot deploy orchestration', () => { - let sandbox; - beforeEach(() => { - sandbox = sinon.createSandbox(); - }); - afterEach(() => { - sandbox.restore(); +// A component source that already contains node_modules, so installApplication short-circuits and no +// npm/network is needed. +async function makeComponentPayload(marker) { + const src = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-op-src-')); + await fs.writeFile(path.join(src, 'package.json'), JSON.stringify({ name: 'op-fixture', version: '1.0.0' })); + await fs.writeFile(path.join(src, 'index.js'), `module.exports = ${JSON.stringify(marker)};\n`); + await fs.mkdir(path.join(src, 'node_modules'), { recursive: true }); + await fs.writeFile(path.join(src, 'node_modules', '.marker'), marker); + const payload = await packDirectory(src); + await fs.rm(src, { recursive: true, force: true }); + return payload; +} + +const readIndex = (dir) => fs.readFile(path.join(dir, 'index.js'), 'utf8'); + +describe('deploy operations: stage_component / activate_component / deploy_component', function () { + this.timeout(30_000); + + before(async () => { + await fs.mkdir(COMPONENTS_ROOT, { recursive: true }); }); - it('stageComponent stages, replicates, and returns a staged marker + deployment_id — no restart, no config write', async () => { - const { stage, activate, prepare } = stubInternals(sandbox); - const addConfig = sandbox.stub(require('#src/config/configUtils'), 'addConfig').resolves(); + let counter = 0; + const names = []; + function freshName() { + const name = `op_test_${process.pid}_${counter++}`; + names.push(name); + return name; + } + + // Sweep any live dirs, staging, and aside created by the suite. + after(async () => { + for (const name of names) await fs.rm(path.join(COMPONENTS_ROOT, name), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-aside'), { recursive: true, force: true }); + }); - const res = await operations.stageComponent({ project: 'my_app', payload: Buffer.from('x') }); + it('stage_component builds the incoming version into staging without going live, and returns a deployment_id', async () => { + const name = freshName(); + const res = await operations.stageComponent({ project: name, payload: await makeComponentPayload('op-staged') }); - assert.strictEqual(stage.calledOnce, true, 'stageApplication was called'); - assert.strictEqual(activate.called, false, 'activate never runs during a stage'); - assert.strictEqual(prepare.called, false, 'the one-shot prepare path is not used'); assert.strictEqual(res.staged, true); - assert.strictEqual(res.project, 'my_app'); - assert.ok(res.deployment_id, 'a deployment_id is returned'); - assert.strictEqual(addConfig.called, false, 'staging never writes root config'); - assert.strictEqual(manageThreads.restartWorkers.called, false, 'staging never restarts'); - }); + assert.strictEqual(res.project, name); + assert.strictEqual(typeof res.deployment_id, 'string'); - it('activateComponent swaps live and restarts when restart:true; returns an activated marker', async () => { - const { activate } = stubInternals(sandbox); + const stagedDir = path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, res.deployment_id, name); + assert.ok(existsSync(path.join(stagedDir, 'index.js')), 'component was built into the staging dir'); + assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, name)), false, 'staging did not touch the live path'); + }); - const res = await operations.activateComponent({ - project: 'my_app', - deployment_id: 'dep-123', - restart: true, + it('activate_component takes a prior stage live', async () => { + const name = freshName(); + const staged = await operations.stageComponent({ + project: name, + payload: await makeComponentPayload('op-activated'), }); + const res = await operations.activateComponent({ project: name, deployment_id: staged.deployment_id }); - assert.strictEqual(activate.calledOnce, true, 'activateApplication was called'); assert.strictEqual(res.activated, true); - assert.strictEqual(res.project, 'my_app'); - assert.strictEqual(manageThreads.restartWorkers.calledWith('http'), true, 'restarted the http workers'); + assert.strictEqual(res.project, name); + const liveDir = path.join(COMPONENTS_ROOT, name); + assert.ok(existsSync(liveDir), 'live component dir now exists'); + assert.match(await readIndex(liveDir), /op-activated/); + // A restart was not requested, so the message must not claim one. + assert.doesNotMatch(res.message, /restart/i); }); - it('activateComponent does not restart when restart is omitted', async () => { - const { activate } = stubInternals(sandbox); - await operations.activateComponent({ project: 'my_app', deployment_id: 'dep-123' }); - assert.strictEqual(activate.calledOnce, true); - assert.strictEqual(manageThreads.restartWorkers.called, false); + it('activate_component rejects when no deployment_id is supplied', async () => { + const name = freshName(); + await assert.rejects(() => operations.activateComponent({ project: name }), /deployment_id.*required/i); }); - it('activateComponent rejects when no deployment_id is supplied', async () => { - stubInternals(sandbox); - await assert.rejects(() => operations.activateComponent({ project: 'my_app' }), /deployment_id.*required/i); - }); + it('deploy_component (two-phase default) stages then activates end-to-end', async () => { + const name = freshName(); + const res = await operations.deployComponent({ project: name, payload: await makeComponentPayload('op-deployed') }); - it('deploy_component with two_phase:false takes the legacy one-shot path (prepareApplication, not stage/activate)', async () => { - const { stage, activate, prepare } = stubInternals(sandbox); + assert.match(res.message, /Successfully deployed/); + assert.strictEqual(typeof res.deployment_id, 'string'); + const liveDir = path.join(COMPONENTS_ROOT, name); + assert.match(await readIndex(liveDir), /op-deployed/, 'component is live after a two-phase deploy'); + // The staged copy was consumed by the swap; its per-deploy staging parent is cleaned up. + assert.strictEqual( + existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, res.deployment_id)), + false, + 'per-deploy staging parent removed after activation' + ); + }); + it('deploy_component with two_phase:false runs the legacy one-shot path', async () => { + const name = freshName(); const res = await operations.deployComponent({ - project: 'my_app', - payload: Buffer.from('x'), + project: name, + payload: await makeComponentPayload('op-oneshot'), two_phase: false, }); - assert.strictEqual(prepare.calledOnce, true, 'one-shot prepareApplication was called'); - assert.strictEqual(stage.called, false, 'two-phase stage not used on the one-shot path'); - assert.strictEqual(activate.called, false, 'two-phase activate not used on the one-shot path'); assert.match(res.message, /Successfully deployed/); + const liveDir = path.join(COMPONENTS_ROOT, name); + assert.match(await readIndex(liveDir), /op-oneshot/, 'component is live after a one-shot deploy'); }); }); From 26de01cac9a84f1b03e92da46f5224eb0d093dd4 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Sun, 19 Jul 2026 23:23:53 -0400 Subject: [PATCH 06/94] fix(types): cast commitResolution to Promise at recordCommitLatency call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unrelated to the two-phase deploy feature — fixes a pre-existing type error on main (introduced by #1688's commit-latency analytics) that main's other build steps swallow via `|| true`/`continue-on-error`, but the newly-added Next.js adapter integration workflow (#1385) runs the build without that tolerance and so fails on it (tsc exit 2) for every PR built on current main. `commitResolution` is declared with the wider `Promise | void` (the abort() branch reassigns it), but at this call site it is the `commit()` promise already cast to `Promise` on the line above. recordCommitLatency only awaits it for timing and never reads the resolved value, so the cast is type-only with no runtime effect — matching the author's existing cast and safety comment two lines up. Co-Authored-By: Claude Opus 4.8 --- resources/DatabaseTransaction.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/resources/DatabaseTransaction.ts b/resources/DatabaseTransaction.ts index fdc0bb2363..33626d19c0 100644 --- a/resources/DatabaseTransaction.ts +++ b/resources/DatabaseTransaction.ts @@ -364,7 +364,10 @@ export class DatabaseTransaction implements Transaction { // "Outstanding write transactions have too long of queue" (503) rejection. A transient- // conflict retry rejects this promise and issues a fresh commit(), and outstandingCommit // re-arms per attempt, so recording per attempt matches the overload semantics. - recordCommitLatency(commitResolution, performance.now()); + // `commitResolution` is declared with the wider `Promise | void` + // (the abort() branch below), but in this branch it is the `commit()` promise cast to + // `Promise` just above; recordCommitLatency only awaits it for timing. + recordCommitLatency(commitResolution as Promise, performance.now()); } else { try { commitResolution = transaction.abort(); From 6e4269c451a4745d2747a6469171f7899d8f6f75 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 20 Jul 2026 10:44:09 -0400 Subject: [PATCH 07/94] feat(deploy): revert_component + retained previous version, CLI commands MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the draft PR's open questions: - Reversibility (Q3): activate now RETAINS the outgoing live version as .deploy-previous/ (one per component, older evicted) instead of discarding it, and a new revert_component operation swaps live <-> previous cluster-wide (replicated like activate). The swap is bidirectional, so reverting a revert rolls forward again. This unlocks customer-driven rollback (deploy -> run your own health checks -> revert if unhappy) and deploy_component gains an opt-in revert_on_failure that rolls the whole cluster back when the activate phase leaves it split across versions. New revertApplication primitive, operation handler, validator, enum, authorization, SSE, and 'reverting' status. - CLI (Q4): `harper stage` (packages + uploads like `harper deploy`, no go-live), `harper activate`, and `harper revert` — aliases + SSE progress wired in bin/cliOperations.ts; stage shares deploy's cwd-packaging prep. - Mixed-version clusters (Q1): clusters stay in lockstep on their version, so the rolling-upgrade caveat and any capability-negotiation framing are dropped from DESIGN.md / validator comments. - Replicator contract (Q2): documented in DESIGN.md from harper-pro's replicator.ts — replicateOperation fans to server.nodes, sets replicated=false as the peer re-fan guard, surfaces per-peer {status:'failed',reason,node}, authenticates by node cert, and runs replicated ops with authorize=false for trusted nodes (skipping the permission gate). Confirms the sub-op design is structurally identical to the proven deploy_component fan-out. Adds 12 tests: retention + bidirectional revert primitives, revert operation end-to-end, and the revert/revert_on_failure validators. 39 deploy unit tests pass. Co-Authored-By: Claude Opus 4.8 --- DESIGN.md | 32 +++- bin/cliOperations.ts | 84 ++++++----- components/Application.ts | 125 +++++++++++++++- components/deploymentRecorder.ts | 4 + components/operations.js | 138 +++++++++++++++++- components/operationsValidation.js | 33 ++++- server/serverHelpers/serverHandlers.js | 1 + server/serverHelpers/serverUtilities.ts | 4 + .../components/deployPhaseOperations.test.js | 27 +++- .../components/deployPhaseValidators.test.js | 31 +++- unitTests/components/deployStaging.test.js | 70 +++++++++ utility/hdbTerms.ts | 4 + utility/operation_authorization.ts | 1 + 13 files changed, 504 insertions(+), 50 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 5b24f5264b..4674553135 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -351,9 +351,35 @@ since the `hdb_deployment` row's `payload_blob` is how peers fetch the tarball a phases by deployment id. When `system` is excluded from a narrow `REPLICATION_DATABASES`, or the caller passes `two_phase: false`, or the invocation is a peer replaying a one-shot deploy, `deploy_component` falls back to `deployComponentOneShot` (the previous behavior, preserved verbatim). -Known gap for a rolling upgrade window: an origin on this version replicating `stage_component` to a -peer that predates these operations will see that peer fail the op; `two_phase: false` or -`ignore_replication_errors` is the escape hatch until capability negotiation lands. +Cross-version skew is a non-issue by policy — a cluster stays in lockstep on its Harper version, so +every node understands `stage_component`/`activate_component` — which is why there is no capability +negotiation on the fan-out. + +**Replicator contract this rides on (`harper-pro/replication/replicator.ts`).** +`server.replication.replicateOperation(op, {onPeerResult})` fans `op` to every node in `server.nodes` +in parallel, setting `op.replicated = false` on the copy it sends so a peer never re-fans (the deploy +handlers instead detect a replicated execution by the presence of `_deploymentId`, which is always set +on the sub-operations). Per-peer failures never throw — `sendOperationToNode` rejections are caught and +surface as `{status:'failed', reason, node}` entries in the returned `replicated[]` array and via +`onPeerResult`, which is exactly the shape `DeploymentRecorder.normalizePeerResult` consumes. Peers +authenticate node-to-node by TLS certificate, and the receive side runs the op via +`server.operation(data, {user}, !isAuthorizedNode)` — for a trusted cluster node the authorize flag is +`false`, so a replicated super-user op skips the permission gate. That is why `stage_component` / +`activate_component` / `revert_component` (registered with the same `permission(true, [])` as +`deploy_component`, dispatched by `operation` name) replicate without an `hdb_user`, identically to the +long-proven `deploy_component` fan-out. + +**Reversibility: retained previous + `revert_component`.** `activateApplication` no longer discards the +outgoing live version — it retains it as `.deploy-previous/` (`retainAsPrevious`, evicting the +older one so exactly one previous is kept per component). `revert_component` swaps the live directory +with that retained previous via three same-filesystem renames through a hidden holding path, cluster- +wide and replicated like activate. The swap is bidirectional, so reverting a revert rolls forward +again. This backs two things: a customer can deploy, run their own health checks against the live +version, and `revert` if unhappy even when the cluster looks healthy; and `deploy_component`'s opt-in +`revert_on_failure` rolls the whole cluster back to the previous version when the activate phase leaves +some nodes live and some not, so the cluster reconverges on one version. The previous copy is retained +per-node (each node retains its own outgoing version during its own activate), so a replicated revert +has a local rollback source on every node. ## Scheduler: cluster-once execution without a consensus primitive (`resources/scheduler/`) diff --git a/bin/cliOperations.ts b/bin/cliOperations.ts index 019fa5ff20..c0f8daa67f 100644 --- a/bin/cliOperations.ts +++ b/bin/cliOperations.ts @@ -18,7 +18,16 @@ import { DeployRenderer } from './deployRenderer.ts'; import { getHdbPid } from '../utility/processManagement/processManagement.js'; import { initConfig, getConfigPath } from '../config/configUtils.ts'; -const OP_ALIASES = { deploy: 'deploy_component', package: 'package_component' }; +const OP_ALIASES = { + deploy: 'deploy_component', + package: 'package_component', + // Two-phase deploy: `harper stage` packages + uploads the incoming version to a hidden staging + // dir cluster-wide (no go-live); `harper activate` swaps a staged deployment live; `harper revert` + // swaps the live version back to its retained previous version. + stage: 'stage_component', + activate: 'activate_component', + revert: 'revert_component', +}; // Shown for any local-instance connection failure (missing pid, missing/stale domain // socket, or a refused/ENOENT connect against it) — they're all the same user-facing @@ -31,7 +40,7 @@ const LOCAL_NOT_RUNNING_MESSAGE = 'Harper is not running. Use `harperdb run` (or // deploy completes. Add an operation here only after wiring its server-side // SSE_PROGRESS_OPERATIONS entry — otherwise the server returns the buffered JSON path and // the SSE parser sees no events. -const SSE_OPERATIONS = new Set(['deploy_component']); +const SSE_OPERATIONS = new Set(['deploy_component', 'stage_component', 'activate_component', 'revert_component']); // Properties on `req` that the CLI itself uses for transport/UX, not the operations API. // They never get serialized into the request body. @@ -151,38 +160,47 @@ function operationFields(req: any): any { } export { cliOperations, buildRequest }; -const PREPARE_OPERATION: any = { - deploy_component: async (req) => { - if (req.package) { - return; - } - const projectPath = process.cwd(); - if (!req.project) req.project = path.basename(projectPath); - const packageOptions = { - skip_node_modules: req.skip_node_modules !== false, - skip_symlinks: req.skip_symlinks === true, - }; - // Store path + options for deferred stream creation after the renderer is set up, - // so the pre-gzip onBytes callback can be wired directly to renderer.countUploadBytes. - req._projectPath = projectPath; - req._packageOptions = packageOptions; - // Pre-walk the directory once for both the uncompressed-size estimate (progress bar - // total) and the dangling-symlink list — a dangling symlink would otherwise silently - // truncate the tarball (tar-fs finalizes early on the broken target). Packaging skips - // them; the list is reused below (no second walk) and warns the user which links were - // skipped so the omission is visible. - const scan = await scanPackageDirectory(projectPath, packageOptions); - req._uploadSizeEstimate = scan.totalSize; - req._danglingSymlinks = scan.danglingSymlinks; - if (scan.danglingSymlinks.length) { - process.stderr.write( - `warning: skipping ${scan.danglingSymlinks.length} broken symlink(s) — their linked content will NOT be deployed:\n` + - scan.danglingSymlinks.map((p) => ` ${p}\n`).join('') - ); - } - req._multipart = true; - }, +// Package the current working directory into a multipart tarball upload. Shared by `deploy` and +// `stage` — both send the incoming component version as a `payload` (unless a `package` identifier is +// given, in which case the server fetches it and there is nothing to upload). +const packageCwdForUpload = async (req) => { + if (req.package) { + return; + } + + const projectPath = process.cwd(); + if (!req.project) req.project = path.basename(projectPath); + const packageOptions = { + skip_node_modules: req.skip_node_modules !== false, + skip_symlinks: req.skip_symlinks === true, + }; + // Store path + options for deferred stream creation after the renderer is set up, + // so the pre-gzip onBytes callback can be wired directly to renderer.countUploadBytes. + req._projectPath = projectPath; + req._packageOptions = packageOptions; + // Pre-walk the directory once for both the uncompressed-size estimate (progress bar + // total) and the dangling-symlink list — a dangling symlink would otherwise silently + // truncate the tarball (tar-fs finalizes early on the broken target). Packaging skips + // them; the list is reused below (no second walk) and warns the user which links were + // skipped so the omission is visible. + const scan = await scanPackageDirectory(projectPath, packageOptions); + req._uploadSizeEstimate = scan.totalSize; + req._danglingSymlinks = scan.danglingSymlinks; + if (scan.danglingSymlinks.length) { + process.stderr.write( + `warning: skipping ${scan.danglingSymlinks.length} broken symlink(s) — their linked content will NOT be deployed:\n` + + scan.danglingSymlinks.map((p) => ` ${p}\n`).join('') + ); + } + req._multipart = true; +}; + +const PREPARE_OPERATION: any = { + deploy_component: packageCwdForUpload, + // `harper stage` uploads the same tarball as `harper deploy`. activate/revert take no payload + // (they operate on an already-staged / already-retained version), so they need no prep step. + stage_component: packageCwdForUpload, }; /** diff --git a/components/Application.ts b/components/Application.ts index dbeef9fdc6..8721ab91ff 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -424,6 +424,20 @@ export const ASIDE_STAGING_DIR = '.deploy-aside'; // restart-on-change storm and needs no deploy:start watcher suppression. export const DEPLOY_STAGING_DIR = '.deploy-staging'; +// Hidden directory under the components root that RETAINS the immediately-previous live version of a +// component (`.deploy-previous/`) after an activate swap, so it can be swapped back by +// revert_component. Exactly one previous version is kept per component — each activate evicts the +// older one — so the retention cost is bounded at one extra copy. Same filesystem as the live path +// (atomic swap on revert), leading-dot-hidden (loader/watchers ignore it). This is what turns the +// deploy swap into something reversible: a customer can activate, run their own health checks, and +// revert if unhappy; and a partially-failed activate can be swapped back cluster-wide. +export const DEPLOY_PREVIOUS_DIR = '.deploy-previous'; + +// Absolute path of the retained-previous copy for a component's live directory. +function previousDirPathFor(liveDirPath: string): string { + return join(dirname(liveDirPath), DEPLOY_PREVIOUS_DIR, basename(liveDirPath)); +} + /** * Atomically move `targetDirPath` aside into a hidden, per-component staging directory if it * exists, returning the aside staging directory (for best-effort cleanup) or null when there was @@ -465,6 +479,34 @@ function cleanupAsideDir(asideStagingDir: string | null, componentName: string): ); } +/** + * Retain the current live version of a component as its rollback source: rename it to + * `.deploy-previous/`, evicting any older retained-previous first. Returns the aside directory + * holding the evicted older-previous (for best-effort cleanup), or null when there was no live + * version to retain (a first-ever deploy). + * + * The eviction moves the older-previous ASIDE (an atomic rename that can't fail on a directory a + * lingering worker still holds open) rather than an in-place rm, so the subsequent rename onto + * `.deploy-previous/` never races an incomplete delete (ENOTEMPTY). Like the aside swap, the + * still-running worker of the version being retained keeps writing into the renamed inode harmlessly + * until it exits on restart. + */ +async function retainAsPrevious(liveDirPath: string): Promise { + const previousPath = previousDirPathFor(liveDirPath); + try { + await lstat(liveDirPath); // lstat, not access: see moveDirAside — a dangling symlink must still move + } catch (err) { + if (err.code === 'ENOENT') return null; // no live version yet — nothing to retain + throw err; + } + // Evict the older retained-previous (2 deploys ago; its worker exited on the last restart) by moving + // it aside atomically, clearing the target for the rename below. + const evictedAside = await moveDirAside(previousPath); + await mkdir(dirname(previousPath), { recursive: true }); + await rename(liveDirPath, previousPath); + return evictedAside; +} + // The credential helper git executes for a private git-reference deploy. It ships alongside this // module (both in source and in dist), holds no secret, and is inert without a live session. export const GIT_CREDENTIAL_HELPER_PATH = join(__dirname, 'gitCredentialHelper.js'); @@ -963,6 +1005,12 @@ export class Application { return join(dirname(this.dirPath), DEPLOY_STAGING_DIR, this.stagingId, this.name); } + // The retained-previous copy this component would revert to (`.deploy-previous/`). See + // DEPLOY_PREVIOUS_DIR / activateApplication / revertApplication. + get previousDirPath(): string { + return previousDirPathFor(this.dirPath); + } + // Route extract/install into the staging directory. Called by stageApplication(). useStagingBuildDir(): void { this.#buildDirPath = this.stagingDirPath; @@ -1175,6 +1223,9 @@ export async function stageApplication(application: Application): Promise` so + * revert_component can swap it back. See retainAsPrevious. + * * This method should only be called from the main thread. */ export async function activateApplication(application: Application): Promise { @@ -1191,30 +1242,90 @@ export async function activateApplication(application: Application): Promise), then rename // the staged copy into place. Both live under the components root, so the rename is same-fs and - // atomic — there is no interval where `dirPath` is a partially populated directory. - asideStagingDir = await moveDirAside(application.dirPath); + // atomic — there is no interval where `dirPath` is a partially populated directory. retainAsPrevious + // tolerates a still-writing worker exactly as the old aside swap did. + evictedAside = await retainAsPrevious(application.dirPath); await mkdir(dirname(application.dirPath), { recursive: true }); await rename(stagingDirPath, application.dirPath); application.useLiveBuildDir(); } finally { broadcastDeployEnd(application.name); } - // Best-effort cleanup of the outgoing version (see cleanupAsideDir). The rename already consumed - // stagingDirPath, so all that remains is this deploy's now-empty staging parent + // Best-effort cleanup of the EVICTED older-previous (the version from two deploys ago; see + // cleanupAsideDir) — NOT the retained previous, which is kept for revert. The rename already consumed + // stagingDirPath, so all that remains of staging is this deploy's now-empty parent // (.deploy-staging/). Remove it with a NON-recursive rmdir, which succeeds only when it // is empty — a belt-and-suspenders guard against ever recursively deleting a directory that could // hold another deploy's build. ENOTEMPTY and ENOENT (already gone) are expected and ignored. - cleanupAsideDir(asideStagingDir, application.name); + cleanupAsideDir(evictedAside, application.name); rmdir(dirname(stagingDirPath)).catch((err) => { if (err.code !== 'ENOTEMPTY' && err.code !== 'ENOENT') logger.trace?.(`Deferred cleanup of ${application.name} staging directory: ${err.message}`); }); } +/** + * Swap a component's live version with its retained previous version (`.deploy-previous/`), + * atomically, then let watchers restart onto it. This is what backs revert_component: a customer can + * activate a deploy, run their own health checks, and swap back if unhappy; and a partially-failed + * activate can be rolled back cluster-wide. + * + * The swap is bidirectional: the outgoing live becomes the new retained previous, so a second revert + * toggles forward again. Three same-filesystem renames via a hidden holding path — the only window + * where `dirPath` is momentarily absent is between two atomic renames, and deploy:start suppresses + * watchers across it (same as activate). + * + * Throws if there is no retained previous version (a component deployed only once, or never). + * + * This method should only be called from the main thread. + */ +export async function revertApplication(application: Application): Promise { + const liveDirPath = application.dirPath; + const previousPath = previousDirPathFor(liveDirPath); + try { + await lstat(previousPath); + } catch (err) { + if (err.code === 'ENOENT') { + throw new Error( + `Cannot revert ${application.name}: no previous version is retained. A component must have ` + + `been deployed over a prior version (which activate retains as .deploy-previous) to be reverted.` + ); + } + throw err; + } + await broadcastDeployStart(application.name); + try { + // Does a live version currently exist? (It always should after a deploy, but guard so a missing + // live dir degrades to "restore previous" rather than throwing mid-swap.) + let liveExists = true; + try { + await lstat(liveDirPath); + } catch (err) { + if (err.code === 'ENOENT') liveExists = false; + else throw err; + } + await mkdir(dirname(previousPath), { recursive: true }); + if (liveExists) { + // Three-way atomic swap: live → holding, previous → live, holding(old live) → previous. + const holding = join(dirname(previousPath), `.reverting-${basename(liveDirPath)}-${randomUUID()}`); + await rename(liveDirPath, holding); + await rename(previousPath, liveDirPath); + await rename(holding, previousPath); + } else { + // No live version to preserve; just restore the previous into place (nothing becomes the new + // previous, so the component can't be re-reverted until its next deploy). + await rename(previousPath, liveDirPath); + } + application.useLiveBuildDir(); + } finally { + broadcastDeployEnd(application.name); + } +} + /** * Discard a staged-but-not-activated build (an aborted two-phase deploy). Best-effort: removes the * staging tree and tears down any transient credential state. The live component directory is never diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index 73f499bbf1..fb09d4107d 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -57,6 +57,8 @@ type DeploymentStatus = | 'replicating' // Two-phase deploy: swapping the staged build into the live path cluster-wide (activate phase). | 'activating' + // revert_component: swapping the live version back to its retained previous version. + | 'reverting' | 'restarting' | 'success' | 'failed' @@ -578,6 +580,8 @@ function startStatusFor(phase: string | undefined): DeploymentStatus | null { return 'replicating'; case 'activate': return 'activating'; + case 'revert': + return 'reverting'; case 'restart': return 'restarting'; default: diff --git a/components/operations.js b/components/operations.js index 303512a445..5d47e7462f 100644 --- a/components/operations.js +++ b/components/operations.js @@ -29,9 +29,11 @@ const { prepareApplication, stageApplication, activateApplication, + revertApplication, discardStagedApplication, ASIDE_STAGING_DIR, DEPLOY_STAGING_DIR, + DEPLOY_PREVIOUS_DIR, } = require('./Application.ts'); const { server } = require('../server/Server.ts'); const { DeploymentRecorder, awaitDeploymentRow, DEFAULT_AWAIT_ROW_TIMEOUT_MS } = require('./deploymentRecorder.ts'); @@ -678,10 +680,31 @@ async function deployComponentTwoPhase(req, credentialReferences) { if (!req.ignore_replication_errors) { const failed = recorder.getFailedPeers(); if (failed.length > 0) { + let revertNote = ''; + // Opt-in swap-back: some nodes went live and some didn't, leaving the cluster split across + // versions. When revert_on_failure is set, roll the whole cluster (incl. this origin) back to + // the retained previous version so it converges on one version again. Best-effort — a revert + // failure must not mask the original activate failure. + if (req.revert_on_failure) { + try { + emit('phase', { phase: 'revert', status: 'start' }); + await revertApplication(application); + const revertOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.REVERT_COMPONENT, { + restart: req.restart, + deploymentId: recorder.deploymentId, + }); + await server.replication.replicateOperation(revertOp, {}); + emit('phase', { phase: 'revert', status: 'done' }); + revertNote = ` The cluster was rolled back to the previous version (revert_on_failure); verify with get_components.`; + } catch (revertErr) { + log.warn('revert_on_failure rollback failed', revertErr); + revertNote = ` An automatic rollback (revert_on_failure) was attempted but also failed: ${revertErr?.message ?? revertErr}.`; + } + } throw new ServerError( `Component '${application.name}' was activated on the origin but failed to activate on ${failed.length} ` + `of ${recorder.row.peer_results.length} peer node(s): ${describePeers(failed)}. Those nodes have the staged ` + - `build but did not go live. See deployment ${recorder.deploymentId} (get_deployment), or pass ` + + `build but did not go live.${revertNote} See deployment ${recorder.deploymentId} (get_deployment), or pass ` + `ignore_replication_errors: true.` ); } @@ -879,6 +902,110 @@ async function activateComponent(req) { return response; } +/** + * revert_component — swap a component's live version back to its retained previous version + * (`.deploy-previous/`, kept by the last activate), cluster-wide, then restart. Backs + * customer-driven rollback (deploy → run your own health checks → revert if unhappy) and a + * swap-back after a partially-failed activate. The swap is bidirectional, so reverting a revert + * rolls forward again. + * + * Reached two ways: directly by an operator, and by a peer replaying a replicated revert + * (`_deploymentId` set). deploy_component's `revert_on_failure` path drives it internally. + */ +async function revertComponent(req) { + if (req.project) req.project = path.parse(req.project).name; + const validation = validator.revertComponentValidator(req); + if (validation) throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); + + const isReplicatedExecution = typeof req._deploymentId === 'string'; + const emitter = isReplicatedExecution ? null : (req.progress ?? new ProgressEmitter()); + if (emitter && !req.progress) req.progress = emitter; + // The origin records a rollback row for observability; a peer replaying the revert does not. + const recorder = isReplicatedExecution + ? null + : await DeploymentRecorder.create({ + project: req.project, + package_identifier: null, + user: req.hdb_user?.username, + restart_mode: req.restart === 'rolling' ? 'rolling' : req.restart ? 'immediate' : null, + rollback_of: req.deployment_id ?? null, + emitter, + }); + if (recorder) req._deploymentId = recorder.deploymentId; + const emit = (event, data) => emitter?.emit(event, data); + const installCapture = createInstallCapture(); // revert has no install output, but finalizeDeployFailure expects one + const rollingRestart = req.restart === 'rolling'; + + try { + const application = new Application({ name: req.project }); + emit('phase', { phase: 'revert', status: 'start' }); + await revertApplication(application); + emit('phase', { phase: 'revert', status: 'done' }); + + const response = { message: `Reverted component: ${req.project}`, project: req.project, reverted: true }; + if (recorder) response.deployment_id = recorder.deploymentId; + + // Replicate the revert to peers (direct invocation only; a peer replaying must not re-fan). + req.restart = rollingRestart ? false : req.restart; + if (!isReplicatedExecution) { + delete req.progress; + const revertOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.REVERT_COMPONENT, { + restart: req.restart, + deploymentId: recorder?.deploymentId, + }); + recorder?.seal(); + const rep = await server.replication.replicateOperation(revertOp, { + onPeerResult: recorder + ? (result) => { + recorder.recordPeer(result); + emit('peer', result); + } + : undefined, + }); + if (recorder && rep?.replicated) recorder.recordPeers(rep.replicated); + } + + // Restart on this node (peers replaying an immediate-restart revert restart locally; the rolling + // path is driven only by the direct invoker via a replicated restart_service job). + if (req.restart === true) { + emit('phase', { phase: 'restart', status: 'start' }); + manageThreads.restartWorkers('http'); + emit('phase', { phase: 'restart', status: 'done' }); + response.message = `Reverted component: ${req.project}, restarting Harper`; + } else if (rollingRestart && !isReplicatedExecution) { + const serverUtilities = require('../server/serverHelpers/serverUtilities.ts'); + emit('phase', { phase: 'restart', status: 'start' }); + const jobResponse = await serverUtilities.executeJob({ + operation: 'restart_service', + service: 'http', + replicated: true, + }); + emit('phase', { phase: 'restart', status: 'done' }); + response.restartJobId = jobResponse.job_id; + response.message = `Reverted component: ${req.project}, restarting Harper`; + } + + if (recorder && !req.ignore_replication_errors) { + const failed = recorder.getFailedPeers(); + if (failed.length > 0) { + throw new ServerError( + `Component '${req.project}' was reverted on the origin but failed to revert on ${failed.length} ` + + `of ${recorder.row.peer_results.length} peer node(s): ${describePeers(failed)}. ` + + `See deployment ${recorder.deploymentId} (get_deployment), or pass ignore_replication_errors: true.` + ); + } + } + + if (recorder) { + emit('phase', { phase: 'success', status: 'done' }); + await recorder.finish('rolled_back'); + } + return response; + } catch (err) { + throw await finalizeDeployFailure({ err, recorder, installCapture, emit }); + } +} + // ———————————————————————————————————————————————————————————————————————————— // Shared deploy-family helpers (used by deploy_component, stage_component, activate_component). // ———————————————————————————————————————————————————————————————————————————— @@ -1176,7 +1303,13 @@ async function getComponents() { const list = await fs.readdir(dir, { withFileTypes: true }); for (let item of list) { const itemName = item.name; - if (itemName === 'node_modules' || itemName === ASIDE_STAGING_DIR || itemName === DEPLOY_STAGING_DIR) continue; + if ( + itemName === 'node_modules' || + itemName === ASIDE_STAGING_DIR || + itemName === DEPLOY_STAGING_DIR || + itemName === DEPLOY_PREVIOUS_DIR + ) + continue; const itemPath = path.join(dir, itemName); if (item.isDirectory() || item.isSymbolicLink()) { let res = { @@ -1511,6 +1644,7 @@ exports.packageComponent = packageComponent; exports.deployComponent = deployComponent; exports.stageComponent = stageComponent; exports.activateComponent = activateComponent; +exports.revertComponent = revertComponent; exports.getComponents = getComponents; exports.getComponentFile = getComponentFile; exports.setComponentFile = setComponentFile; diff --git a/components/operationsValidation.js b/components/operationsValidation.js index 8a865d37bc..b50f0304fc 100644 --- a/components/operationsValidation.js +++ b/components/operationsValidation.js @@ -27,6 +27,7 @@ module.exports = { deployComponentValidator, stageComponentValidator, activateComponentValidator, + revertComponentValidator, setComponentFileValidator, getComponentFileValidator, dropComponentFileValidator, @@ -488,9 +489,11 @@ function deployComponentValidator(req) { deployment_timeout: Joi.number().min(0).optional(), force: Joi.boolean().optional(), ignore_replication_errors: Joi.boolean().optional(), - // Opt out of the two-phase (stage-then-activate) deploy and use the legacy one-shot path - // instead. Provided as an escape hatch for mixed-version clusters where a peer predates the - // stage_component/activate_component operations. Defaults to two-phase. + // If the activate phase fails on some nodes (leaving the cluster split across versions), swap the + // whole cluster back to the retained previous version before reporting the failure. Off by default. + revert_on_failure: Joi.boolean().optional(), + // Opt out of the two-phase (stage-then-activate) deploy and use the legacy one-shot path instead. + // Defaults to two-phase. two_phase: Joi.boolean().optional(), urlPath: URL_PATH_SCHEMA, // Deploy credentials. Each entry is npm registry auth (`registry`) or git host auth (`host`, @@ -565,3 +568,27 @@ function activateComponentValidator(req) { return validator.validateBySchema(req, activateSchema); } + +/** + * Validate revert_component requests — swap a component's live version back to its retained previous + * version. No build inputs (nothing is fetched or installed); just the project, an optional restart, + * and the replication controls. + * @param req + * @returns {*} + */ +function revertComponentValidator(req) { + const revertSchema = Joi.object({ + project: Joi.string() + .pattern(PROJECT_FILE_NAME_REGEX) + .required() + .messages({ 'string.pattern.base': HDB_ERROR_MSGS.BAD_PROJECT_NAME }), + // The deployment being reverted, recorded as the rollback's `rollback_of` for the audit trail. + // Optional — revert operates on whatever version is currently live regardless. + deployment_id: Joi.string().optional(), + restart: Joi.alternatives().try(Joi.boolean(), Joi.string().valid('rolling')).optional(), + deployment_timeout: Joi.number().min(0).optional(), + ignore_replication_errors: Joi.boolean().optional(), + }); + + return validator.validateBySchema(req, revertSchema); +} diff --git a/server/serverHelpers/serverHandlers.js b/server/serverHelpers/serverHandlers.js index 3b52339b51..9acac09fb1 100644 --- a/server/serverHelpers/serverHandlers.js +++ b/server/serverHelpers/serverHandlers.js @@ -35,6 +35,7 @@ const SSE_PROGRESS_OPERATIONS = new Set([ terms.OPERATIONS_ENUM.DEPLOY_COMPONENT, terms.OPERATIONS_ENUM.STAGE_COMPONENT, terms.OPERATIONS_ENUM.ACTIVATE_COMPONENT, + terms.OPERATIONS_ENUM.REVERT_COMPONENT, terms.OPERATIONS_ENUM.GET_DEPLOYMENT, terms.OPERATIONS_ENUM.READ_LOG, ]); diff --git a/server/serverHelpers/serverUtilities.ts b/server/serverHelpers/serverUtilities.ts index 8bc2d83b6e..b0c6557949 100644 --- a/server/serverHelpers/serverUtilities.ts +++ b/server/serverHelpers/serverUtilities.ts @@ -542,6 +542,10 @@ function initializeOperationFunctionMap(): Map { for (const name of names) await fs.rm(path.join(COMPONENTS_ROOT, name), { recursive: true, force: true }); await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-previous'), { recursive: true, force: true }); await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-aside'), { recursive: true, force: true }); }); @@ -138,4 +139,28 @@ describe('deploy operations: stage_component / activate_component / deploy_compo const liveDir = path.join(COMPONENTS_ROOT, name); assert.match(await readIndex(liveDir), /op-oneshot/, 'component is live after a one-shot deploy'); }); + + it('revert_component swaps the live version back to the previous deployment', async () => { + const name = freshName(); + const liveDir = path.join(COMPONENTS_ROOT, name); + await operations.deployComponent({ project: name, payload: await makeComponentPayload('rev-v1') }); + await operations.deployComponent({ project: name, payload: await makeComponentPayload('rev-v2') }); + assert.match(await readIndex(liveDir), /rev-v2/, 'v2 is live before revert'); + + const res = await operations.revertComponent({ project: name }); + + assert.strictEqual(res.reverted, true); + assert.strictEqual(res.project, name); + assert.match(await readIndex(liveDir), /rev-v1/, 'revert restored v1 to live'); + }); + + it('revert_component rejects a component with no retained previous version', async () => { + const name = freshName(); + await operations.deployComponent({ project: name, payload: await makeComponentPayload('rev-once') }); + await assert.rejects(() => operations.revertComponent({ project: name }), /no previous version is retained/i); + }); + + it('revert_component requires a project', async () => { + await assert.rejects(() => operations.revertComponent({}), /project/i); + }); }); diff --git a/unitTests/components/deployPhaseValidators.test.js b/unitTests/components/deployPhaseValidators.test.js index d0ac511363..bc08a57bad 100644 --- a/unitTests/components/deployPhaseValidators.test.js +++ b/unitTests/components/deployPhaseValidators.test.js @@ -71,7 +71,36 @@ describe('activateComponentValidator', () => { }); }); -describe('deployComponentValidator two_phase flag', () => { +describe('revertComponentValidator', () => { + it('accepts a project-only revert', () => { + ok(validator.revertComponentValidator({ project: 'my_app' })); + }); + + it('accepts a deployment_id, restart, and ignore_replication_errors', () => { + ok( + validator.revertComponentValidator({ + project: 'my_app', + deployment_id: 'abc-123', + restart: 'rolling', + ignore_replication_errors: true, + }) + ); + }); + + it('requires a project', () => { + rejected(validator.revertComponentValidator({ deployment_id: 'abc-123' })); + }); + + it('rejects an invalid restart value', () => { + rejected(validator.revertComponentValidator({ project: 'my_app', restart: 'sideways' })); + }); +}); + +describe('deployComponentValidator two_phase + revert_on_failure flags', () => { + it('accepts revert_on_failure: true', () => { + ok(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', revert_on_failure: true })); + }); + it('accepts two_phase: false (legacy opt-out)', () => { ok(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', two_phase: false })); }); diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 283fb2f0a0..a9b3f36a8a 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -21,8 +21,10 @@ const { Application, stageApplication, activateApplication, + revertApplication, discardStagedApplication, DEPLOY_STAGING_DIR, + DEPLOY_PREVIOUS_DIR, ASIDE_STAGING_DIR, } = require('#src/components/Application'); const { getConfigPath } = require('#src/config/configUtils'); @@ -228,4 +230,72 @@ describe('two-phase deploy primitives (stage / activate / discard)', function () await fs.rm(dirPath, { recursive: true, force: true }); await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); }); + + // Deploy a version (stage + activate) through the primitives, returning the app. + async function deployVersion(name, marker) { + const app = new Application({ name, payload: await makeComponentPayload(marker) }); + await stageApplication(app); + await activateApplication(app); + return app; + } + + it('activate retains the outgoing version as .deploy-previous/', async () => { + const name = `stage_test_${process.pid}_${counter++}`; + await deployVersion(name, 'v1'); + await deployVersion(name, 'v2'); + + const liveDir = path.join(COMPONENTS_ROOT, name); + const previousDir = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); + assert.match(await readMarker(liveDir), /v2/, 'live is the newest version'); + assert.match(await readMarker(previousDir), /v1/, 'the outgoing version is retained as previous'); + + await fs.rm(liveDir, { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + }); + + it('revertApplication swaps live <-> previous, and a second revert rolls forward again', async () => { + const name = `stage_test_${process.pid}_${counter++}`; + await deployVersion(name, 'v1'); + const app = await deployVersion(name, 'v2'); + const liveDir = path.join(COMPONENTS_ROOT, name); + const previousDir = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); + + await revertApplication(app); + assert.match(await readMarker(liveDir), /v1/, 'reverted live back to v1'); + assert.match(await readMarker(previousDir), /v2/, 'the reverted-away v2 is now the previous'); + + await revertApplication(app); + assert.match(await readMarker(liveDir), /v2/, 'reverting the revert rolls forward to v2'); + assert.match(await readMarker(previousDir), /v1/, 'v1 is the previous again'); + + await fs.rm(liveDir, { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + }); + + it('revertApplication throws when there is no retained previous (deployed only once)', async () => { + const name = `stage_test_${process.pid}_${counter++}`; + const app = await deployVersion(name, 'only'); // first-ever deploy: nothing retained as previous + + await assert.rejects(() => revertApplication(app), /no previous version is retained/i); + + await fs.rm(path.join(COMPONENTS_ROOT, name), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + }); + + it('only one previous is retained across three deploys (older previous evicted)', async () => { + const name = `stage_test_${process.pid}_${counter++}`; + await deployVersion(name, 'v1'); + await deployVersion(name, 'v2'); + await deployVersion(name, 'v3'); + + const previousDir = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); + assert.match(await readMarker(path.join(COMPONENTS_ROOT, name)), /v3/, 'live is v3'); + assert.match(await readMarker(previousDir), /v2/, 'previous is v2; v1 was evicted'); + + await fs.rm(path.join(COMPONENTS_ROOT, name), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + }); }); diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index c428264be1..1b47d74f51 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -297,6 +297,10 @@ export const OPERATIONS_ENUM = { // are unaffected. See components/Application.ts (stageApplication/activateApplication). STAGE_COMPONENT: 'stage_component', ACTIVATE_COMPONENT: 'activate_component', + // Swap a component's live version back to its retained previous version, cluster-wide. Backs + // customer-driven rollback (activate → test → revert) and swap-back on a partially-failed activate. + // See components/Application.ts (revertApplication). + REVERT_COMPONENT: 'revert_component', READ_TRANSACTION_LOG: 'read_transaction_log', DELETE_TRANSACTION_LOGS_BEFORE: 'delete_transaction_logs_before', INSTALL_NODE_MODULES: 'install_node_modules', diff --git a/utility/operation_authorization.ts b/utility/operation_authorization.ts index dcb70b6c26..c48a4a6c3c 100644 --- a/utility/operation_authorization.ts +++ b/utility/operation_authorization.ts @@ -288,6 +288,7 @@ requiredPermissions.set(functionsOperations.packageComponent.name, new (permissi requiredPermissions.set(functionsOperations.deployComponent.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.stageComponent.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.activateComponent.name, new (permission as any)(true, [])); +requiredPermissions.set(functionsOperations.revertComponent.name, new (permission as any)(true, [])); requiredPermissions.set( deploymentOperations.handleListDeployments.name, new (permission as any)(true, [], terms.OPERATIONS_ENUM.LIST_DEPLOYMENTS) From 722158b8ee66f5fefc9a63180223d8646c04d878 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 20 Jul 2026 10:54:40 -0400 Subject: [PATCH 08/94] fix(deploy): revert_on_failure must skip peers that never activated MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit replicateOperation fans to every node with no subset targeting, so the prior revert_on_failure reverted failed-activate peers too. But a peer that failed to activate never ran retainAsPrevious — its live dir is still the correct pre-deploy version and its .deploy-previous holds a copy from two deploys ago — so reverting it rolled it back an EXTRA version, splitting the cluster across three versions instead of reconverging on one. Scope the swap-back to the origin plus the peers that actually activated, sent point-to-point via sendOperationToNode (skipping recorder.getFailedPeers()), and leave the failed peers on their already-correct version. (Review catch.) Co-Authored-By: Claude Opus 4.8 --- components/operations.js | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/components/operations.js b/components/operations.js index 5d47e7462f..aa0ec75efc 100644 --- a/components/operations.js +++ b/components/operations.js @@ -682,20 +682,40 @@ async function deployComponentTwoPhase(req, credentialReferences) { if (failed.length > 0) { let revertNote = ''; // Opt-in swap-back: some nodes went live and some didn't, leaving the cluster split across - // versions. When revert_on_failure is set, roll the whole cluster (incl. this origin) back to - // the retained previous version so it converges on one version again. Best-effort — a revert - // failure must not mask the original activate failure. + // versions. When revert_on_failure is set, roll the nodes that DID activate back to the + // retained previous version so the cluster reconverges. Best-effort — a revert failure must + // not mask the original activate failure. if (req.revert_on_failure) { try { emit('phase', { phase: 'revert', status: 'start' }); + // The origin activated, so revert it. await revertApplication(application); + // Revert ONLY the peers that successfully activated. A peer that FAILED to activate never + // swapped in the new version (validation/timeout/transport failures fire before + // activateApplication runs retainAsPrevious) — its live directory is still the correct + // pre-deploy version, and its `.deploy-previous` holds a copy from TWO deploys ago. Reverting + // it would roll it back an EXTRA version, splitting the cluster across three versions instead + // of reconverging on one. replicateOperation has no subset targeting (it fans to every + // server.node), so send point-to-point to the activated peers via sendOperationToNode, + // skipping recorder.getFailedPeers(). + const failedNodeNames = new Set(failed.map((peer) => peer.node).filter(Boolean)); + const activatedPeers = (server.nodes ?? []).filter((node) => !failedNodeNames.has(node.name)); const revertOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.REVERT_COMPONENT, { restart: req.restart, deploymentId: recorder.deploymentId, }); - await server.replication.replicateOperation(revertOp, {}); + revertOp.replicated = false; // point-to-point; the peer must not re-fan the revert + const revertResults = await Promise.allSettled( + activatedPeers.map((node) => server.replication.sendOperationToNode(node, revertOp)) + ); + const revertFailures = revertResults.filter((result) => result.status === 'rejected').length; emit('phase', { phase: 'revert', status: 'done' }); - revertNote = ` The cluster was rolled back to the previous version (revert_on_failure); verify with get_components.`; + revertNote = + ` Rolled the origin and ${activatedPeers.length - revertFailures} of ${activatedPeers.length} ` + + `activated peer(s) back to the previous version (revert_on_failure); the ${failed.length} peer(s) ` + + `that never activated were left on their current (correct) version.` + + (revertFailures > 0 ? ` ${revertFailures} peer revert(s) also failed.` : '') + + ` Verify with get_components.`; } catch (revertErr) { log.warn('revert_on_failure rollback failed', revertErr); revertNote = ` An automatic rollback (revert_on_failure) was attempted but also failed: ${revertErr?.message ?? revertErr}.`; From 2ac508f5ae80e6685b6cfe634ef908fc04847249 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 20 Jul 2026 11:01:59 -0400 Subject: [PATCH 09/94] fix(deploy): exclude this node from revert_on_failure point-to-point fan-out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The origin is reverted directly, then activatedPeers (server.nodes minus failed peers) was sent a point-to-point revert too. server.nodes normally excludes self (knownNodes populates it with a `!== getThisNodeName()` guard), but a not-yet-named node can slip in, and the established convention (bin/restart.ts) guards self on every point-to-point fan-out — without it, a self-directed revert would run the handler again and, because the swap is bidirectional, flip the origin back to the just-activated (broken) version. Filter out getThisNodeName() alongside the failed peers. (Review catch.) Co-Authored-By: Claude Opus 4.8 --- components/operations.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/components/operations.js b/components/operations.js index aa0ec75efc..88844d0967 100644 --- a/components/operations.js +++ b/components/operations.js @@ -697,9 +697,17 @@ async function deployComponentTwoPhase(req, credentialReferences) { // it would roll it back an EXTRA version, splitting the cluster across three versions instead // of reconverging on one. replicateOperation has no subset targeting (it fans to every // server.node), so send point-to-point to the activated peers via sendOperationToNode, - // skipping recorder.getFailedPeers(). + // skipping recorder.getFailedPeers() — and skipping THIS node, which was already reverted + // directly above and would otherwise be reverted a second time (a bidirectional swap that + // flips it back to the just-activated version). server.nodes normally excludes self, but a + // not-yet-named node can slip in (knownNodes) and every point-to-point fan-out in the code + // base guards self anyway (bin/restart.ts). + const { getThisNodeName } = require('../server/nodeName.ts'); + const thisNode = getThisNodeName(); const failedNodeNames = new Set(failed.map((peer) => peer.node).filter(Boolean)); - const activatedPeers = (server.nodes ?? []).filter((node) => !failedNodeNames.has(node.name)); + const activatedPeers = (server.nodes ?? []).filter( + (node) => node.name !== thisNode && !failedNodeNames.has(node.name) + ); const revertOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.REVERT_COMPONENT, { restart: req.restart, deploymentId: recorder.deploymentId, From baca6b7165c8427d772d6a0c2a6b2dad55d30586 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 20 Jul 2026 11:11:49 -0400 Subject: [PATCH 10/94] test(deploy): extract + cover revert_on_failure node targeting MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The revert_on_failure fan-out needs a live multi-node cluster to run end-to-end, so its node-targeting (skip failed peers, skip self) had no unit coverage — which is why both bugs there were caught by review rather than a test. Extract that pure set-difference into an exported selectRevertTargets(nodes, failedPeers, thisNode) and cover it directly with plain assert: excludes self (the bidirectional double-revert guard), excludes every failed peer, and is safe with empty/undefined inputs and null-node failed entries. deployComponentTwoPhase now calls the helper. Co-Authored-By: Claude Opus 4.8 --- components/operations.js | 38 ++++++++++--------- .../components/deployPhaseOperations.test.js | 35 +++++++++++++++++ 2 files changed, 56 insertions(+), 17 deletions(-) diff --git a/components/operations.js b/components/operations.js index 88844d0967..92673d3491 100644 --- a/components/operations.js +++ b/components/operations.js @@ -690,24 +690,12 @@ async function deployComponentTwoPhase(req, credentialReferences) { emit('phase', { phase: 'revert', status: 'start' }); // The origin activated, so revert it. await revertApplication(application); - // Revert ONLY the peers that successfully activated. A peer that FAILED to activate never - // swapped in the new version (validation/timeout/transport failures fire before - // activateApplication runs retainAsPrevious) — its live directory is still the correct - // pre-deploy version, and its `.deploy-previous` holds a copy from TWO deploys ago. Reverting - // it would roll it back an EXTRA version, splitting the cluster across three versions instead - // of reconverging on one. replicateOperation has no subset targeting (it fans to every - // server.node), so send point-to-point to the activated peers via sendOperationToNode, - // skipping recorder.getFailedPeers() — and skipping THIS node, which was already reverted - // directly above and would otherwise be reverted a second time (a bidirectional swap that - // flips it back to the just-activated version). server.nodes normally excludes self, but a - // not-yet-named node can slip in (knownNodes) and every point-to-point fan-out in the code - // base guards self anyway (bin/restart.ts). + // Revert ONLY the peers that successfully activated (see selectRevertTargets): every known + // node minus the ones that failed to activate (still on the correct version) and minus this + // node (already reverted directly above; a second bidirectional revert would flip it back). + // replicateOperation has no subset targeting, so send point-to-point via sendOperationToNode. const { getThisNodeName } = require('../server/nodeName.ts'); - const thisNode = getThisNodeName(); - const failedNodeNames = new Set(failed.map((peer) => peer.node).filter(Boolean)); - const activatedPeers = (server.nodes ?? []).filter( - (node) => node.name !== thisNode && !failedNodeNames.has(node.name) - ); + const activatedPeers = selectRevertTargets(server.nodes, failed, getThisNodeName()); const revertOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.REVERT_COMPONENT, { restart: req.restart, deploymentId: recorder.deploymentId, @@ -1185,6 +1173,21 @@ function describePeers(failedPeers) { return failedPeers.map((peer) => `${peer.node ?? 'unknown'} (${peer.error?.message ?? 'unknown error'})`).join(', '); } +// Choose which peers a revert_on_failure swap-back should target: every known node EXCEPT +// - `thisNodeName`: the origin, already reverted directly by the caller — a second (bidirectional) +// revert would flip it back to the just-activated version; and +// - any node in `failedPeers`: it never activated (its failure fired before activateApplication ran +// retainAsPrevious), so its live directory is still the correct pre-deploy version and reverting it +// would roll it back an EXTRA version onto a two-deploys-ago copy. +// Pure and exported so the node-targeting logic (which had two review-caught bugs — the failed-peer +// skip and the self-skip) is unit-testable without a live cluster. `nodes` is `server.nodes`, which +// normally already excludes self, but a not-yet-named node can slip in (knownNodes) so self is guarded +// here regardless — matching every other point-to-point fan-out in the code base (bin/restart.ts). +function selectRevertTargets(nodes, failedPeers, thisNodeName) { + const failedNodeNames = new Set((failedPeers ?? []).map((peer) => peer.node).filter(Boolean)); + return (nodes ?? []).filter((node) => node?.name !== thisNodeName && !failedNodeNames.has(node?.name)); +} + // Reclaim the payload tarball for large deploys once every peer has installed from the blob. Dropping // the reference before finish() folds the null into the single terminal write, which unlinks the file // locally and replicates the null so peers drop their copies too. Metadata is retained. Kept when a @@ -1673,6 +1676,7 @@ exports.deployComponent = deployComponent; exports.stageComponent = stageComponent; exports.activateComponent = activateComponent; exports.revertComponent = revertComponent; +exports.selectRevertTargets = selectRevertTargets; // exported for unit testing the revert_on_failure node-targeting exports.getComponents = getComponents; exports.getComponentFile = getComponentFile; exports.setComponentFile = setComponentFile; diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index 757f658209..fd93a84e13 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -163,4 +163,39 @@ describe('deploy operations: stage_component / activate_component / deploy_compo it('revert_component requires a project', async () => { await assert.rejects(() => operations.revertComponent({}), /project/i); }); + + // The revert_on_failure fan-out itself needs a live multi-node cluster (harper-pro's replicator) to + // run end-to-end, but its node-targeting is a pure function — and the exact spot that had two + // review-caught bugs (skip failed peers, skip self). Exercise it directly. + describe('selectRevertTargets (revert_on_failure node targeting)', () => { + const nodes = [{ name: 'origin' }, { name: 'peerA' }, { name: 'peerB' }, { name: 'peerC' }]; + + it('returns activated peers, excluding this node and failed peers', () => { + const failed = [{ node: 'peerB', status: 'failed' }]; + const targets = operations.selectRevertTargets(nodes, failed, 'origin').map((n) => n.name); + assert.deepStrictEqual(targets.sort(), ['peerA', 'peerC'], 'peerB (failed) and origin (self) excluded'); + }); + + it('excludes THIS node even when it is present in server.nodes (bidirectional double-revert guard)', () => { + const targets = operations.selectRevertTargets(nodes, [], 'origin').map((n) => n.name); + assert.ok(!targets.includes('origin'), 'self must never receive a self-directed revert'); + assert.deepStrictEqual(targets.sort(), ['peerA', 'peerB', 'peerC']); + }); + + it('excludes every failed peer (they never activated and are on the correct version)', () => { + const failed = [ + { node: 'peerA', status: 'failed' }, + { node: 'peerC', status: 'failed' }, + ]; + const targets = operations.selectRevertTargets(nodes, failed, 'origin').map((n) => n.name); + assert.deepStrictEqual(targets, ['peerB'], 'only the one activated peer is a revert target'); + }); + + it('is safe with empty/undefined nodes and failed lists, and ignores failed entries with no node name', () => { + assert.deepStrictEqual(operations.selectRevertTargets(undefined, undefined, 'origin'), []); + assert.deepStrictEqual(operations.selectRevertTargets([], [{ node: null }], 'origin'), []); + const targets = operations.selectRevertTargets(nodes, [{ node: null }], 'origin').map((n) => n.name); + assert.deepStrictEqual(targets.sort(), ['peerA', 'peerB', 'peerC'], 'a null-node failed entry drops nobody'); + }); + }); }); From 488d172047506fb67c341448b5bbee0e95e9c645 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 20 Jul 2026 11:35:08 -0400 Subject: [PATCH 11/94] fix(deploy): validate deployment_id charset; validate staged build during stage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings from kriszyp's review: - deployment_id becomes a staging-dir path segment (.deploy-staging//), but activate/revert validators accepted any string — a `../` value could resolve the staging source outside .deploy-staging. Constrain it to the same safe charset as `project` (defense-in-depth; the ops are super_user-only). +tests. - stageComponent never ran the pre-go-live component load check, so a standalone stage_component didn't validate at all and the stage barrier didn't cover load-time faults. Run loadValidateComponent on the staged build in the stage handler (matches deployComponentTwoPhase's origin check). It is a no-op on the main thread where replicated peer executions run (app code must not load there), so DESIGN.md now scopes the cluster-wide barrier guarantee to fetch + install and documents load-validation as origin/worker-side. Co-Authored-By: Claude Opus 4.8 --- DESIGN.md | 11 +++++++++++ components/operations.js | 7 +++++++ components/operationsValidation.js | 16 ++++++++++++---- .../components/deployPhaseValidators.test.js | 12 ++++++++++++ 4 files changed, 42 insertions(+), 4 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 4674553135..cb16318c20 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -321,6 +321,17 @@ could leave a peer half-installed after other peers had already restarted onto t request/response contract is unchanged; only the SSE phase names differ (`stage`/`activate` vs the old `prepare`/`replicate`). `two_phase: false` forces the legacy one-shot path. +**Scope of the barrier's guarantee: fetch + install, not load.** The cluster-wide "nobody activates +until everybody staged" guarantee covers the download/`npm pack` and `npm install` steps — the slow, +failure-prone work. The pre-go-live component _load_ check (`loadValidateComponent`, which surfaces a +component that installs cleanly but throws at load) runs during stage on the origin and on any node +whose stage executes on a worker (the op-API worker for a standalone `stage_component`), but it is a +no-op on the main thread — and replicated peer executions of `stage_component` run on the main thread +(`replicateOperation` → `sendOperationToNode` execute there), where app code deliberately isn't +loaded. So a load-time-only fault on a peer is not caught by the barrier; it surfaces at +activate/restart like any other. Gating load-time faults cluster-wide would require dispatching the +throwaway load to a worker on each peer during stage — a possible follow-up, not done here. + The staging directory (`.deploy-staging//`) lives **under the components root**, not in `os.tmpdir()`, even though its contents are transient. This is deliberate and load-bearing: the go-live step is `rename(stagingDir, liveDir)`, which is only atomic when both paths share a diff --git a/components/operations.js b/components/operations.js index 92673d3491..15cb44b498 100644 --- a/components/operations.js +++ b/components/operations.js @@ -800,6 +800,13 @@ async function stageComponent(req) { emit('phase', { phase: 'stage', status: 'start' }); await stageApplication(application); + // Surface load-time errors on the staged build before it can be activated, matching what + // deployComponentTwoPhase does on the origin (and closing the gap where a standalone + // stage_component never validated at all). Loads the STAGED dir (application.buildDirPath). This + // is a no-op on the main thread — where replicated peer executions run — because app code must + // not load there; so load-time faults are gated where the stage runs on a worker (origin op-API + // worker, standalone stage), while fetch + install remain gated cluster-wide by the barrier. + await loadValidateComponent({ dirPath: application.buildDirPath, emit }); emit('phase', { phase: 'stage', status: 'done' }); const response = { message: `Staged component: ${req.project}`, project: req.project, staged: true }; diff --git a/components/operationsValidation.js b/components/operationsValidation.js index b50f0304fc..a726f41b8d 100644 --- a/components/operationsValidation.js +++ b/components/operationsValidation.js @@ -551,8 +551,13 @@ function activateComponentValidator(req) { .required() .messages({ 'string.pattern.base': HDB_ERROR_MSGS.BAD_PROJECT_NAME }), // Identifies which staged build to activate. Required for a standalone activate; the - // deploy_component orchestrator supplies it as the deployment id it staged under. - deployment_id: Joi.string().optional(), + // deploy_component orchestrator supplies it as the deployment id it staged under. Constrained to + // the same safe charset as `project` because it becomes a path segment of the staging directory + // (`.deploy-staging//`) — without this a `../` value would resolve the staging + // source outside `.deploy-staging`. Defense-in-depth (the op is super_user-only). + deployment_id: Joi.string().pattern(PROJECT_FILE_NAME_REGEX).optional().messages({ + 'string.pattern.base': `'deployment_id' must only contain letters, numbers, dashes, and underscores`, + }), package: Joi.string().optional(), install_command: Joi.string().optional(), install_timeout: Joi.number().optional(), @@ -583,8 +588,11 @@ function revertComponentValidator(req) { .required() .messages({ 'string.pattern.base': HDB_ERROR_MSGS.BAD_PROJECT_NAME }), // The deployment being reverted, recorded as the rollback's `rollback_of` for the audit trail. - // Optional — revert operates on whatever version is currently live regardless. - deployment_id: Joi.string().optional(), + // Optional — revert operates on whatever version is currently live regardless. Same safe charset + // as elsewhere (it is an id, and this keeps the deploy family's `deployment_id` consistent). + deployment_id: Joi.string().pattern(PROJECT_FILE_NAME_REGEX).optional().messages({ + 'string.pattern.base': `'deployment_id' must only contain letters, numbers, dashes, and underscores`, + }), restart: Joi.alternatives().try(Joi.boolean(), Joi.string().valid('rolling')).optional(), deployment_timeout: Joi.number().min(0).optional(), ignore_replication_errors: Joi.boolean().optional(), diff --git a/unitTests/components/deployPhaseValidators.test.js b/unitTests/components/deployPhaseValidators.test.js index bc08a57bad..daa85573e6 100644 --- a/unitTests/components/deployPhaseValidators.test.js +++ b/unitTests/components/deployPhaseValidators.test.js @@ -69,6 +69,18 @@ describe('activateComponentValidator', () => { it('rejects an invalid restart value', () => { rejected(validator.activateComponentValidator({ project: 'my_app', restart: 'sideways' })); }); + + it('rejects a path-traversal deployment_id (it becomes a staging-dir path segment)', () => { + for (const bad of ['../evil', 'a/b', 'dep/../..', '.', '..']) { + rejected(validator.activateComponentValidator({ project: 'my_app', deployment_id: bad })); + } + }); + + it('accepts a normal UUID-shaped deployment_id', () => { + ok( + validator.activateComponentValidator({ project: 'my_app', deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa' }) + ); + }); }); describe('revertComponentValidator', () => { From d80e695316ebbdac3fb3661d117c92cf7e52dc43 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Tue, 21 Jul 2026 10:03:53 -0400 Subject: [PATCH 12/94] refactor(deploy): fold stage/activate into deploy_component (internal _phase) Per review (harper#1849): stage_component / activate_component are no longer public operations. Their two-phase cluster fan-out now rides deploy_component itself, tagged with an internal `_phase: 'stage' | 'activate'` marker, and the operator-facing capability is exposed as deploy_component properties: - deploy_component (default): full stage + activate (contract unchanged). - deploy_component({ activate: false }): stage cluster-wide and stop, returning the deployment_id in a `staged` state. - deploy_component({ deployment_id }): activate a previously-staged deployment. deployComponent now dispatches replicated _phase executions to internal deployPhaseStage / deployPhaseActivate handlers, and public calls to the orchestrator / activate-existing path. Removed the two ops from OPERATIONS_ENUM, the op function map, operation_authorization, the SSE set, and their validators; added activate + deployment_id (safe charset) to deployComponentValidator. Added markDeploymentTerminal to flip a stage-and-stop row to success on later activate. revert_component stays a distinct public op (a rollback, not a deploy phase). CLI: `harper stage` -> deploy_component activate=false (still packages the cwd); `harper activate deployment_id=` -> deploy_component deployment_id= (no upload). Tests + DESIGN.md updated to the single-public-op shape; 34 deploy unit tests pass. Co-Authored-By: Claude Opus 4.8 --- DESIGN.md | 63 +++-- bin/cliOperations.ts | 35 ++- components/deploymentRecorder.ts | 20 ++ components/operations.js | 261 ++++++++---------- components/operationsValidation.js | 83 +----- server/serverHelpers/serverHandlers.js | 2 - server/serverHelpers/serverUtilities.ts | 8 - .../components/deployPhaseOperations.test.js | 21 +- .../components/deployPhaseValidators.test.js | 95 ++----- utility/hdbTerms.ts | 14 +- utility/operation_authorization.ts | 2 - 11 files changed, 241 insertions(+), 363 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index cb16318c20..75a2921ad6 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -308,25 +308,38 @@ other future cause would still report success silently; that's a deferred, separ ## Two-phase deploy: stage then activate (`components/Application.ts`, `components/operations.js`) -`deploy_component` splits into two replicated phases so a cluster deploy is all-or-nothing at the -point of go-live. **Phase 1 (`stage_component`)** builds the incoming version — download/`npm pack` -(incl. a git clone), extract, `npm install` — into a hidden staging directory on every node. -**Phase 2 (`activate_component`)** atomically renames the staged copy into the live component path and -restarts. `deploy_component` orchestrates the two: it stages on the origin, replicates -`stage_component` to peers, **waits for every node to report a successful stage before any node -activates** (`ignore_replication_errors` opts out of the barrier), then replicates -`activate_component`. If a node can't fetch the package or fails `npm install`, it fails during -staging while the live component is still untouched _on every node_ — where the old one-shot path -could leave a peer half-installed after other peers had already restarted onto the new code. The -request/response contract is unchanged; only the SSE phase names differ (`stage`/`activate` vs the -old `prepare`/`replicate`). `two_phase: false` forces the legacy one-shot path. +`deploy_component` runs internally as two replicated phases so a cluster deploy is all-or-nothing at +the point of go-live. **Phase 1 (stage)** builds the incoming version — download/`npm pack` (incl. a +git clone), extract, `npm install` — into a hidden staging directory on every node. **Phase 2 +(activate)** atomically renames the staged copy into the live component path and restarts. The origin +stages locally, **waits for every node to report a successful stage before any node activates** +(`ignore_replication_errors` opts out of the barrier), then activates. If a node can't fetch the +package or fails `npm install`, it fails during staging while the live component is still untouched _on +every node_ — where the old one-shot path could leave a peer half-installed after other peers had +already restarted onto the new code. The request/response contract is unchanged; only the SSE phase +names differ (`stage`/`activate` vs the old `prepare`/`replicate`). `two_phase: false` forces the +legacy one-shot path. + +**There is only one public operation — `deploy_component`.** The two phases are NOT separate public +operations; the peer fan-out is `deploy_component` itself tagged with an internal `_phase: 'stage' | +'activate'` marker (the same `_`-prefixed internal-field convention peers already branch on, alongside +`_deploymentId`). `deployComponent` dispatches: a replicated execution with `_phase` runs the peer +stage/activate work (`deployPhaseStage` / `deployPhaseActivate`) and never re-fans; a public call runs +the origin orchestrator. Two public properties expose the phases when an operator wants them separated +(e.g. pre-stage the cluster now, flip later — or a CI-stages / approver-activates split): `activate: +false` stages cluster-wide and stops, returning the `deployment_id` in a `staged` state; passing that +`deployment_id` back to `deploy_component` (with no new payload) activates the already-staged build. +This was a deliberate API-surface choice (harper#1849 review): peer fan-out needs a wire format, not +two extra public ops, and folding the phases into `deploy_component` keeps the surface at one op while +the convergence properties cover the stage-now/activate-later use case. (`revert_component` stays a +distinct public op — it is a rollback, not a deploy phase.) **Scope of the barrier's guarantee: fetch + install, not load.** The cluster-wide "nobody activates until everybody staged" guarantee covers the download/`npm pack` and `npm install` steps — the slow, failure-prone work. The pre-go-live component _load_ check (`loadValidateComponent`, which surfaces a component that installs cleanly but throws at load) runs during stage on the origin and on any node -whose stage executes on a worker (the op-API worker for a standalone `stage_component`), but it is a -no-op on the main thread — and replicated peer executions of `stage_component` run on the main thread +whose stage executes on a worker (e.g. the op-API worker for an `activate: false` stage), but it is a +no-op on the main thread — and replicated peer stage executions run on the main thread (`replicateOperation` → `sendOperationToNode` execute there), where app code deliberately isn't loaded. So a load-time-only fault on a peer is not caught by the barrier; it surfaces at activate/restart like any other. Gating load-time faults cluster-wide would require dispatching the @@ -343,9 +356,9 @@ component, and it is **not** the watched base of any component's file watcher (t each live component dir, `EntryHandler`/`deriveCommonPatternBase`) — so building here fires no restart-on-change events and needs no `deploy:start` watcher suppression. That suppression is now scoped to `activateApplication`, the only phase that writes the live path. Staging is deterministic -from the deployment id precisely so `activate_component` (a separate replicated operation, and on -peers a separate invocation from `stage_component`) can reconstruct the same path the stage built — -peers build a fresh `Application` per sub-operation, so there is no shared in-memory handle to rely +from the deployment id precisely so the activate phase (a separate replicated `deploy_component` +invocation on peers, tagged `_phase: 'activate'`) can reconstruct the same path the stage built — +peers build a fresh `Application` per phase invocation, so there is no shared in-memory handle to rely on. The deployment id sits ABOVE the component name (`…//`, not `…//`) for two reasons: the leaf directory's basename is then the real component name, which the pre-go-live validation load needs (`componentLoader` keys the `ApplicationScope` and @@ -363,22 +376,22 @@ phases by deployment id. When `system` is excluded from a narrow `REPLICATION_DA caller passes `two_phase: false`, or the invocation is a peer replaying a one-shot deploy, `deploy_component` falls back to `deployComponentOneShot` (the previous behavior, preserved verbatim). Cross-version skew is a non-issue by policy — a cluster stays in lockstep on its Harper version, so -every node understands `stage_component`/`activate_component` — which is why there is no capability -negotiation on the fan-out. +every node understands the `_phase`-tagged `deploy_component` fan-out — which is why there is no +capability negotiation on it. **Replicator contract this rides on (`harper-pro/replication/replicator.ts`).** `server.replication.replicateOperation(op, {onPeerResult})` fans `op` to every node in `server.nodes` in parallel, setting `op.replicated = false` on the copy it sends so a peer never re-fans (the deploy -handlers instead detect a replicated execution by the presence of `_deploymentId`, which is always set -on the sub-operations). Per-peer failures never throw — `sendOperationToNode` rejections are caught and +handlers additionally detect a replicated execution by the presence of `_deploymentId` — always set on +the sub-operations — and run the peer stage/activate work off the `_phase` marker without re-fanning). Per-peer failures never throw — `sendOperationToNode` rejections are caught and surface as `{status:'failed', reason, node}` entries in the returned `replicated[]` array and via `onPeerResult`, which is exactly the shape `DeploymentRecorder.normalizePeerResult` consumes. Peers authenticate node-to-node by TLS certificate, and the receive side runs the op via `server.operation(data, {user}, !isAuthorizedNode)` — for a trusted cluster node the authorize flag is -`false`, so a replicated super-user op skips the permission gate. That is why `stage_component` / -`activate_component` / `revert_component` (registered with the same `permission(true, [])` as -`deploy_component`, dispatched by `operation` name) replicate without an `hdb_user`, identically to the -long-proven `deploy_component` fan-out. +`false`, so a replicated super-user op skips the permission gate. That is why the `_phase`-tagged +`deploy_component` fan-out and `revert_component` (registered with the same `permission(true, [])`, +dispatched by `operation` name) replicate without an `hdb_user`, identically to the long-proven +one-shot `deploy_component` fan-out. **Reversibility: retained previous + `revert_component`.** `activateApplication` no longer discards the outgoing live version — it retains it as `.deploy-previous/` (`retainAsPrevious`, evicting the diff --git a/bin/cliOperations.ts b/bin/cliOperations.ts index c0f8daa67f..51e022f0d9 100644 --- a/bin/cliOperations.ts +++ b/bin/cliOperations.ts @@ -21,14 +21,19 @@ import { initConfig, getConfigPath } from '../config/configUtils.ts'; const OP_ALIASES = { deploy: 'deploy_component', package: 'package_component', - // Two-phase deploy: `harper stage` packages + uploads the incoming version to a hidden staging - // dir cluster-wide (no go-live); `harper activate` swaps a staged deployment live; `harper revert` - // swaps the live version back to its retained previous version. - stage: 'stage_component', - activate: 'activate_component', revert: 'revert_component', }; +// CLI verbs that are sugar over `deploy_component` with preset properties (the stage/activate phases +// are folded into deploy_component; there are no separate stage/activate operations). `harper stage` +// packages + uploads the incoming version to a hidden staging dir cluster-wide and stops before +// go-live (`activate: false`), printing the staged deployment_id; `harper activate deployment_id=` +// takes that staged deployment live (no upload). +const OP_VERB_PROPS: Record> = { + stage: { operation: 'deploy_component', activate: false }, + activate: { operation: 'deploy_component' }, +}; + // Shown for any local-instance connection failure (missing pid, missing/stale domain // socket, or a refused/ENOENT connect against it) — they're all the same user-facing // scenario: Harper isn't running. Remote-target failures keep the detailed error instead, @@ -40,7 +45,7 @@ const LOCAL_NOT_RUNNING_MESSAGE = 'Harper is not running. Use `harperdb run` (or // deploy completes. Add an operation here only after wiring its server-side // SSE_PROGRESS_OPERATIONS entry — otherwise the server returns the buffered JSON path and // the SSE parser sees no events. -const SSE_OPERATIONS = new Set(['deploy_component', 'stage_component', 'activate_component', 'revert_component']); +const SSE_OPERATIONS = new Set(['deploy_component', 'revert_component']); // Properties on `req` that the CLI itself uses for transport/UX, not the operations API. // They never get serialized into the request body. @@ -161,11 +166,12 @@ function operationFields(req: any): any { export { cliOperations, buildRequest }; -// Package the current working directory into a multipart tarball upload. Shared by `deploy` and -// `stage` — both send the incoming component version as a `payload` (unless a `package` identifier is -// given, in which case the server fetches it and there is nothing to upload). +// Package the current working directory into a multipart tarball upload for deploy_component. Covers +// `harper deploy` and `harper stage` (deploy_component with activate:false) — both upload the incoming +// version. Nothing to package when a `package` identifier is given (the server fetches it) or when +// activating a previously-staged deployment (`deployment_id`, i.e. `harper activate`). const packageCwdForUpload = async (req) => { - if (req.package) { + if (req.package || req.deployment_id) { return; } @@ -197,10 +203,10 @@ const packageCwdForUpload = async (req) => { }; const PREPARE_OPERATION: any = { + // deploy_component covers `harper deploy` and `harper stage` (activate:false); packageCwdForUpload + // itself skips the upload for a `package` identifier or a `deployment_id` activate. revert takes no + // payload, so it needs no prep step. deploy_component: packageCwdForUpload, - // `harper stage` uploads the same tarball as `harper deploy`. activate/revert take no payload - // (they operate on an already-staged / already-retained version), so they need no prep step. - stage_component: packageCwdForUpload, }; /** @@ -211,6 +217,9 @@ function buildRequest(): any { for (const arg of process.argv.slice(2)) { if (OP_ALIASES.hasOwnProperty(arg)) { req.operation = OP_ALIASES[arg]; + } else if (OP_VERB_PROPS.hasOwnProperty(arg)) { + // Sugar verb (stage/activate) → deploy_component + preset props (e.g. activate:false). + Object.assign(req, OP_VERB_PROPS[arg]); } else if (arg.includes('=')) { let [first, ...rest] = arg.split('='); let restStr: any = rest.join('='); diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index fb09d4107d..92460cdc04 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -455,6 +455,26 @@ export class DeploymentRecorder { } } +/** + * Best-effort terminal-status update for an existing deployment row by id, used when a later operation + * finishes a deployment the current process didn't record with a live DeploymentRecorder — e.g. + * `deploy_component({ deployment_id })` activating a build that an earlier stage-and-stop left in the + * `staged` state. No-op when the row (or the table) is absent; observability only, so callers treat a + * failure here as non-fatal. + */ +export async function markDeploymentTerminal( + deploymentId: string, + status: 'success' | 'failed' | 'rolled_back' | 'staged' +): Promise { + const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; + if (!table) return; + const row = await table.get(deploymentId); + if (!row) return; + row.status = status; + row.completed_at = Date.now(); + await table.put(row); +} + // Default peer-wait budget for the hdb_deployment row to replicate. A deploy is a rare, // heavyweight, user-initiated operation, and the `system`-table replication channel can be // backlogged behind unrelated writes when several deploys land in succession, so the row can diff --git a/components/operations.js b/components/operations.js index 15cb44b498..f5ca5fabdf 100644 --- a/components/operations.js +++ b/components/operations.js @@ -36,7 +36,12 @@ const { DEPLOY_PREVIOUS_DIR, } = require('./Application.ts'); const { server } = require('../server/Server.ts'); -const { DeploymentRecorder, awaitDeploymentRow, DEFAULT_AWAIT_ROW_TIMEOUT_MS } = require('./deploymentRecorder.ts'); +const { + DeploymentRecorder, + awaitDeploymentRow, + markDeploymentTerminal, + DEFAULT_AWAIT_ROW_TIMEOUT_MS, +} = require('./deploymentRecorder.ts'); const { ProgressEmitter } = require('../server/serverHelpers/progressEmitter.ts'); /** @@ -392,6 +397,14 @@ async function deployComponent(req) { throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); } + const isReplicatedExecution = typeof req._deploymentId === 'string'; + + // Internal peer-phase execution. The two-phase cluster fan-out rides deploy_component itself, tagged + // with an internal `_phase` marker (`stage`/`activate`) rather than separate public operations — so + // the wire format is deploy_component + `_phase`, and only deploy_component is publicly exposed. + if (isReplicatedExecution && req._phase === 'stage') return deployPhaseStage(req); + if (isReplicatedExecution && req._phase === 'activate') return deployPhaseActivate(req); + // Ingest any provided credential token into the secrets store so the credential lives as // replicated ciphertext (reference, not embed); already-reference entries pass through, and with // no custody a literal token stays as a transient, this-node-only fallback (#1158). Peers @@ -402,18 +415,18 @@ async function deployComponent(req) { // token is not — it is used only for this node's install, then stripped before replication. const credentialReferences = (req.credentials ?? []).filter((entry) => entry && entry.secret !== undefined); - // A peer replaying a replicated ONE-SHOT deploy_component arrives with `_deploymentId` set. In - // two-phase mode the origin never sends deploy_component to peers (it sends stage_component / - // activate_component), so `_deploymentId` here always means "legacy peer". - const isReplicatedExecution = typeof req._deploymentId === 'string'; - // Two-phase is the default, but it leans on the system table's replication channel to carry the // payload and correlate the stage/activate steps across the cluster. Fall back to the legacy - // one-shot deploy when: the caller opted out (`two_phase: false`); this is a peer replaying a - // one-shot deploy; or `system` isn't replicated on this node (a narrow REPLICATION_DATABASES). + // one-shot deploy when: the caller opted out (`two_phase: false`); this is a legacy peer replaying a + // one-shot deploy (replicated, no `_phase`); or `system` isn't replicated on this node. if (req.two_phase === false || isReplicatedExecution || !isSystemDatabaseReplicated()) { return deployComponentOneShot(req, credentialReferences, isReplicatedExecution); } + + // `deployment_id` with no fresh payload → activate a previously-staged deployment (the second half of + // a stage-then-activate-later flow). Otherwise run the full stage+activate (which itself honors + // `activate: false` to stop after the cluster-wide staged barrier). + if (req.deployment_id) return deployComponentActivateExisting(req, credentialReferences); return deployComponentTwoPhase(req, credentialReferences); } @@ -615,7 +628,7 @@ async function deployComponentTwoPhase(req, credentialReferences) { // ===== PHASE 1: STAGE — build on every node; nothing goes live. ===== emit('phase', { phase: 'stage', status: 'start' }); await stageApplication(application); - const stageOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.STAGE_COMPONENT); + const stageOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.DEPLOY_COMPONENT, { phase: 'stage' }); const stageResp = await server.replication.replicateOperation(stageOp, { onPeerResult: recordPeer }); if (stageResp?.replicated) recorder.recordPeers(stageResp.replicated); emit('phase', { phase: 'stage', status: 'done' }); @@ -637,6 +650,20 @@ async function deployComponentTwoPhase(req, credentialReferences) { // Validate the staged build before go-live (loads from the staging dir; see loadValidateComponent). await loadValidateComponent({ dirPath: application.buildDirPath, emit }); + // `activate: false` — stage-and-stop. The build is verified on every node; leave the row in a + // `staged` state and return its deployment_id so a later deploy_component({deployment_id}) can + // take it live. Nothing has gone live anywhere. + if (req.activate === false) { + emit('phase', { phase: 'staged', status: 'done' }); + await recorder.finish('staged'); + return { + message: `Staged component: ${application.name}`, + project: application.name, + staged: true, + deployment_id: recorder.deploymentId, + }; + } + // ===== PHASE 2: ACTIVATE — atomic swap + restart, now the bits are in place everywhere. ===== // Persist root config now (not before staging) so a `package` config never points at a version // that failed to stage. @@ -646,7 +673,8 @@ async function deployComponentTwoPhase(req, credentialReferences) { emit('phase', { phase: 'activate', status: 'start' }); await activateApplication(application); - const activateOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.ACTIVATE_COMPONENT, { + const activateOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.DEPLOY_COMPONENT, { + phase: 'activate', restart: req.restart, deploymentId: recorder.deploymentId, }); @@ -739,52 +767,21 @@ async function deployComponentTwoPhase(req, credentialReferences) { } /** - * stage_component — phase 1 of a two-phase deploy, as a first-class operation. Builds the incoming - * version into the hidden staging directory on this node (and, when invoked directly rather than via - * replication, replicates the stage across the cluster). Never writes the live component directory, - * writes no root config, and never restarts — so it is safe to run cluster-wide and gate on. - * - * Reached three ways: directly by an operator (stage now, activate later), by a peer replaying a - * replicated stage (`_deploymentId` set), and indirectly — deploy_component drives staging itself, - * so it does not call this handler. + * Peer stage phase (internal — NOT a public operation). Runs on a peer when the origin fans out + * deploy_component tagged `_phase: 'stage'`: fetch the tarball from the replicated hdb_deployment row, + * build + `npm install` into the hidden staging directory, and load-validate — never touching the live + * path, writing config, or restarting. A failure here fails this peer's stage, which the origin's + * barrier catches. No recorder (the origin owns the row) and no re-replication. */ -async function stageComponent(req) { - if (req.project) { - req.project = path.parse(req.project).name; - } else if (req.package) { - req.project = getProjectNameFromPackage(req.package); - } - const validation = validator.stageComponentValidator(req); - if (validation) throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); - - const { ingestCredentials, resolveCredentials } = require('./secretOperations.ts'); - req.credentials = await ingestCredentials(req, req.credentials, req.project); - const credentialReferences = (req.credentials ?? []).filter((entry) => entry && entry.secret !== undefined); - if (req.package) assertNotProtectedCoreComponent(req.project, req.force); - - const isReplicatedExecution = typeof req._deploymentId === 'string'; - const emitter = isReplicatedExecution ? null : (req.progress ?? new ProgressEmitter()); - if (emitter && !req.progress) req.progress = emitter; - // Standalone stage records so it has a deployment_id + payload row for peers to fetch; a peer - // replaying the stage skips recording (the origin owns the row). - const recorder = isReplicatedExecution - ? null - : await DeploymentRecorder.create({ - project: req.project, - package_identifier: req.package ?? null, - user: req.hdb_user?.username, - restart_mode: null, - credentials: credentialReferences.length ? credentialReferences : null, - emitter, - }); - if (recorder) req._deploymentId = recorder.deploymentId; - const emit = (event, data) => emitter?.emit(event, data); +async function deployPhaseStage(req) { + const { resolveCredentials } = require('./secretOperations.ts'); + const emitter = null; // peers stream nothing back; the origin owns the emitter/recorder + const emit = () => {}; const installCapture = createInstallCapture(); let application; - try { - const extractionPayload = await sourceExtractionPayload({ req, recorder, isReplicatedExecution }); - const resolvedCredentials = await resolveNodeCredentials({ req, resolveCredentials, isReplicatedExecution }); + const extractionPayload = await sourceExtractionPayload({ req, recorder: null, isReplicatedExecution: true }); + const resolvedCredentials = await resolveNodeCredentials({ req, resolveCredentials, isReplicatedExecution: true }); application = buildDeployApplication({ req, extractionPayload, @@ -794,123 +791,84 @@ async function stageComponent(req) { emitter, emit, }); - if (credentialReferences.length) req.credentials = credentialReferences; - else delete req.credentials; - delete req.progress; - - emit('phase', { phase: 'stage', status: 'start' }); await stageApplication(application); - // Surface load-time errors on the staged build before it can be activated, matching what - // deployComponentTwoPhase does on the origin (and closing the gap where a standalone - // stage_component never validated at all). Loads the STAGED dir (application.buildDirPath). This - // is a no-op on the main thread — where replicated peer executions run — because app code must - // not load there; so load-time faults are gated where the stage runs on a worker (origin op-API - // worker, standalone stage), while fetch + install remain gated cluster-wide by the barrier. + // Surface load-time errors on the staged build (no-op on the main thread, where replicated peer + // executions run — app code must not load there; see loadValidateComponent + DESIGN.md). await loadValidateComponent({ dirPath: application.buildDirPath, emit }); - emit('phase', { phase: 'stage', status: 'done' }); - - const response = { message: `Staged component: ${req.project}`, project: req.project, staged: true }; - if (recorder) { - response.deployment_id = recorder.deploymentId; - // Replicate staging to peers so the bits land cluster-wide. Keep the payload in the body only - // when the row can't carry it (system not replicated on this node). - if (isSystemDatabaseReplicated()) delete req.payload; - const stageOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.STAGE_COMPONENT, { - includePayload: !isSystemDatabaseReplicated(), - }); - recorder.seal(); - const rep = await server.replication.replicateOperation(stageOp, { - onPeerResult: (result) => { - recorder.recordPeer(result); - emit('peer', result); - }, - }); - if (rep?.replicated) recorder.recordPeers(rep.replicated); - if (!req.ignore_replication_errors) { - const failed = recorder.getFailedPeers(); - if (failed.length > 0) { - throw new ServerError( - `Component '${req.project}' failed to stage on ${failed.length} of ` + - `${recorder.row.peer_results.length} peer node(s): ${describePeers(failed)}. ` + - `See deployment ${recorder.deploymentId} (get_deployment).` - ); - } - } - // Leave the row in a 'staged' resting state — the build exists cluster-wide but nothing is - // live yet; a subsequent activate_component (or the caller) takes it live. - emit('phase', { phase: 'staged', status: 'done' }); - await recorder.finish('staged'); - } - return response; + return { message: `Staged component: ${req.project}`, project: req.project, staged: true }; } catch (err) { if (application) await discardStagedApplication(application).catch(() => {}); - throw await finalizeDeployFailure({ err, recorder, installCapture, emit }); + throw await finalizeDeployFailure({ err, recorder: null, installCapture, emit }); } } /** - * activate_component — phase 2 of a two-phase deploy, as a first-class operation. Atomically swaps a - * previously-staged build (identified by `deployment_id`) into the live component directory, persists - * the component's root config for a `package` deploy, and restarts as requested. When invoked - * directly it also replicates the activation across the cluster. - * - * Reached two ways: directly by an operator to take a prior stage live, and by a peer replaying a - * replicated activate (`_deploymentId` set). deploy_component drives activation itself. + * Peer activate phase (internal — NOT a public operation). Runs on a peer when the origin fans out + * deploy_component tagged `_phase: 'activate'`: atomically swap the already-staged build (by deployment + * id) into the live path, persist root config for a package deploy, and restart if the origin asked + * for an immediate restart. No recorder, no re-replication. */ -async function activateComponent(req) { - if (req.project) req.project = path.parse(req.project).name; - const validation = validator.activateComponentValidator(req); - if (validation) throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); - - const isReplicatedExecution = typeof req._deploymentId === 'string'; - const stagingId = req._deploymentId ?? req.deployment_id; - if (!stagingId) { - throw handleHDBError( - new Error(), - `'deployment_id' is required to activate a staged component`, - HTTP_STATUS_CODES.BAD_REQUEST - ); - } +async function deployPhaseActivate(req) { if (req.package) assertNotProtectedCoreComponent(req.project, req.force); const credentialReferences = (req.credentials ?? []).filter((entry) => entry && entry.secret !== undefined); + const application = new Application({ + name: req.project, + packageIdentifier: req.package, + stagingId: req._deploymentId, + }); + await activateApplication(application); + if (req.package) await writeComponentRootConfig(req, credentialReferences); + // The origin sets restart=true on the sub-op only for an immediate restart; rolling restarts are + // driven separately by the origin via a replicated restart_service job. + if (req.restart === true) manageThreads.restartWorkers('http'); + return { message: `Activated component: ${req.project}`, project: req.project, activated: true }; +} - const emitter = isReplicatedExecution ? null : (req.progress ?? new ProgressEmitter()); - const emit = (event, data) => emitter?.emit(event, data); +/** + * Activate a previously-staged deployment cluster-wide — the second half of a stage-then-activate + * flow, reached as `deploy_component({ deployment_id })` with no fresh payload. Swaps the staged build + * into the live path on the origin, replicates the activate phase to peers (each activates its own + * staged copy of the same deployment id), restarts, and marks the deployment row success. + */ +async function deployComponentActivateExisting(req, credentialReferences) { + const stagingId = req.deployment_id; + if (req.package) assertNotProtectedCoreComponent(req.project, req.force); + const emitter = req.progress ?? new ProgressEmitter(); + if (!req.progress) req.progress = emitter; + const emit = (event, data) => emitter.emit(event, data); const rollingRestart = req.restart === 'rolling'; - // Reconstruct the Application against the staged build. Only name + stagingId are needed to locate - // and swap it — the staged directory already holds the fully-installed incoming version. const application = new Application({ name: req.project, packageIdentifier: req.package, stagingId }); emit('phase', { phase: 'activate', status: 'start' }); await activateApplication(application); emit('phase', { phase: 'activate', status: 'done' }); - - // Persist root config now that the component is live (package deploys, on every node). + // Persist root config now that the component is live (package deploys). if (req.package) await writeComponentRootConfig(req, credentialReferences); - const response = { message: `Activated component: ${req.project}`, project: req.project, activated: true }; + // Replicate the activate phase to peers (each activates its own staged copy of this deployment id). + req._deploymentId = stagingId; + delete req.progress; + const activateOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.DEPLOY_COMPONENT, { + phase: 'activate', + restart: rollingRestart ? false : req.restart, + deploymentId: stagingId, + }); + const rep = await server.replication.replicateOperation(activateOp, {}); - // Replicate the activation to peers (direct invocation only; a peer replaying an activate must not - // re-fan it out). - if (!isReplicatedExecution) { - delete req.progress; - const restartForPeers = rollingRestart ? false : req.restart; - const activateOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.ACTIVATE_COMPONENT, { - restart: restartForPeers, - deploymentId: stagingId, - }); - const rep = await server.replication.replicateOperation(activateOp, {}); - if (rep?.replicated) response.replicated = rep.replicated; - } + const response = { + message: `Activated component: ${req.project}`, + project: req.project, + activated: true, + deployment_id: stagingId, + }; + if (rep?.replicated) response.replicated = rep.replicated; - // Restart on this node. A peer replaying an immediate-restart activate restarts locally; the - // rolling path is driven only by the direct invoker via a replicated restart_service job. if (req.restart === true) { emit('phase', { phase: 'restart', status: 'start' }); manageThreads.restartWorkers('http'); emit('phase', { phase: 'restart', status: 'done' }); response.message = `Activated component: ${req.project}, restarting Harper`; - } else if (rollingRestart && !isReplicatedExecution) { + } else if (rollingRestart) { const serverUtilities = require('../server/serverHelpers/serverUtilities.ts'); emit('phase', { phase: 'restart', status: 'start' }); const jobResponse = await serverUtilities.executeJob({ @@ -922,6 +880,12 @@ async function activateComponent(req) { response.restartJobId = jobResponse.job_id; response.message = `Activated component: ${req.project}, restarting Harper`; } + + // Best-effort: flip the staged deployment row (left 'staged' by the stage-and-stop) to success now + // that it is live. Observability only — a tracking-write failure must not fail the activate. + await markDeploymentTerminal(stagingId, 'success').catch((err) => + log.warn('Failed to mark staged deployment as activated', err) + ); return response; } @@ -1156,11 +1120,14 @@ async function loadValidateComponent({ dirPath, emit }) { if (lastError) throw lastError; } -// Build a replicated sub-operation body (stage_component / activate_component) from the deploy -// request. Carries only what a peer needs: project, the deployment id (correlation + payload lookup + -// staging id), the build/config inputs, and credential REFERENCES (tokens are already stripped). -function buildReplicatedSubOp(req, operation, { includePayload = false, restart, deploymentId } = {}) { +// Build a replicated sub-operation body for the peer fan-out. For the two-phase peer phases this is +// `deploy_component` tagged with an internal `_phase` marker (`stage`/`activate`) — the wire format — +// so no separate public op is exposed; revert uses operation `revert_component`. Carries only what a +// peer needs: project, the deployment id (correlation + payload lookup + staging id), the internal +// `_phase`, the build/config inputs, and credential REFERENCES (tokens are already stripped). +function buildReplicatedSubOp(req, operation, { includePayload = false, restart, deploymentId, phase } = {}) { const op = { operation, project: req.project, _deploymentId: deploymentId ?? req._deploymentId }; + if (phase) op._phase = phase; if (req.package) op.package = req.package; if (req.install_command != null) op.install_command = req.install_command; if (req.install_timeout != null) op.install_timeout = req.install_timeout; @@ -1680,8 +1647,6 @@ exports.addComponent = addComponent; exports.dropCustomFunctionProject = dropCustomFunctionProject; exports.packageComponent = packageComponent; exports.deployComponent = deployComponent; -exports.stageComponent = stageComponent; -exports.activateComponent = activateComponent; exports.revertComponent = revertComponent; exports.selectRevertTargets = selectRevertTargets; // exported for unit testing the revert_on_failure node-targeting exports.getComponents = getComponents; diff --git a/components/operationsValidation.js b/components/operationsValidation.js index a726f41b8d..8964caba1d 100644 --- a/components/operationsValidation.js +++ b/components/operationsValidation.js @@ -25,8 +25,6 @@ module.exports = { dropCustomFunctionProjectValidator, packageComponentValidator, deployComponentValidator, - stageComponentValidator, - activateComponentValidator, revertComponentValidator, setComponentFileValidator, getComponentFileValidator, @@ -489,8 +487,19 @@ function deployComponentValidator(req) { deployment_timeout: Joi.number().min(0).optional(), force: Joi.boolean().optional(), ignore_replication_errors: Joi.boolean().optional(), + // Stop after the incoming version is staged and verified cluster-wide, without going live. Returns + // the staged deployment_id; a later deploy_component with that deployment_id activates it. Defaults + // to true (full stage + activate). + activate: Joi.boolean().optional(), + // Activate a previously-staged deployment (from an `activate: false` stage) cluster-wide. Same safe + // charset as `project` because it becomes a staging-dir path segment (`.deploy-staging//`) + // — a `../` value would otherwise resolve the staging source outside `.deploy-staging`. + deployment_id: Joi.string().pattern(PROJECT_FILE_NAME_REGEX).optional().messages({ + 'string.pattern.base': `'deployment_id' must only contain letters, numbers, dashes, and underscores`, + }), // If the activate phase fails on some nodes (leaving the cluster split across versions), swap the - // whole cluster back to the retained previous version before reporting the failure. Off by default. + // nodes that did activate back to the retained previous version before reporting the failure. Off + // by default. revert_on_failure: Joi.boolean().optional(), // Opt out of the two-phase (stage-then-activate) deploy and use the legacy one-shot path instead. // Defaults to two-phase. @@ -506,74 +515,6 @@ function deployComponentValidator(req) { return validator.validateBySchema(req, deployProjSchema); } -/** - * Validate stage_component requests — phase 1 of a two-phase deploy. Accepts the same build-time - * inputs as deploy_component (package/payload, install options, credentials) but no go-live controls - * (`restart`), since staging never restarts. `restart` is intentionally absent; a stray one is - * ignored (operations validate with allowUnknown). - * @param req - * @returns {*} - */ -function stageComponentValidator(req) { - const stageSchema = Joi.object({ - project: Joi.string() - .pattern(PROJECT_FILE_NAME_REGEX) - .required() - .messages({ 'string.pattern.base': HDB_ERROR_MSGS.BAD_PROJECT_NAME }), - package: Joi.string().optional(), - install_command: Joi.string().optional(), - install_timeout: Joi.number().optional(), - install_allow_scripts: Joi.boolean().optional(), - deployment_timeout: Joi.number().min(0).optional(), - force: Joi.boolean().optional(), - // urlPath is not applied at stage time (config is written at activate), but it is accepted and - // carried through so a single request body can flow stage → activate unchanged. - urlPath: URL_PATH_SCHEMA, - credentials: CREDENTIALS_ARRAY_SCHEMA, - registryAuth: FORBIDDEN_REGISTRY_AUTH, - }).with('urlPath', 'package'); - - return validator.validateBySchema(req, stageSchema); -} - -/** - * Validate activate_component requests — phase 2 of a two-phase deploy. Swaps an already-staged - * build (identified by `deployment_id`) into the live path and optionally restarts. Also carries the - * config-persistence inputs (package/install/credentials/urlPath) so a `package` deploy's root config - * is written at go-live rather than before the bits are in place. - * @param req - * @returns {*} - */ -function activateComponentValidator(req) { - const activateSchema = Joi.object({ - project: Joi.string() - .pattern(PROJECT_FILE_NAME_REGEX) - .required() - .messages({ 'string.pattern.base': HDB_ERROR_MSGS.BAD_PROJECT_NAME }), - // Identifies which staged build to activate. Required for a standalone activate; the - // deploy_component orchestrator supplies it as the deployment id it staged under. Constrained to - // the same safe charset as `project` because it becomes a path segment of the staging directory - // (`.deploy-staging//`) — without this a `../` value would resolve the staging - // source outside `.deploy-staging`. Defense-in-depth (the op is super_user-only). - deployment_id: Joi.string().pattern(PROJECT_FILE_NAME_REGEX).optional().messages({ - 'string.pattern.base': `'deployment_id' must only contain letters, numbers, dashes, and underscores`, - }), - package: Joi.string().optional(), - install_command: Joi.string().optional(), - install_timeout: Joi.number().optional(), - install_allow_scripts: Joi.boolean().optional(), - deployment_timeout: Joi.number().min(0).optional(), - restart: Joi.alternatives().try(Joi.boolean(), Joi.string().valid('rolling')).optional(), - force: Joi.boolean().optional(), - ignore_replication_errors: Joi.boolean().optional(), - urlPath: URL_PATH_SCHEMA, - credentials: CREDENTIALS_ARRAY_SCHEMA, - registryAuth: FORBIDDEN_REGISTRY_AUTH, - }).with('urlPath', 'package'); - - return validator.validateBySchema(req, activateSchema); -} - /** * Validate revert_component requests — swap a component's live version back to its retained previous * version. No build inputs (nothing is fetched or installed); just the project, an optional restart, diff --git a/server/serverHelpers/serverHandlers.js b/server/serverHelpers/serverHandlers.js index 9acac09fb1..fa71d7c140 100644 --- a/server/serverHelpers/serverHandlers.js +++ b/server/serverHelpers/serverHandlers.js @@ -33,8 +33,6 @@ const { ProgressEmitter, createSSEResponseStream } = require('./progressEmitter. // the client disconnects (the emitter's abort signal), so subscribers see new lines live. const SSE_PROGRESS_OPERATIONS = new Set([ terms.OPERATIONS_ENUM.DEPLOY_COMPONENT, - terms.OPERATIONS_ENUM.STAGE_COMPONENT, - terms.OPERATIONS_ENUM.ACTIVATE_COMPONENT, terms.OPERATIONS_ENUM.REVERT_COMPONENT, terms.OPERATIONS_ENUM.GET_DEPLOYMENT, terms.OPERATIONS_ENUM.READ_LOG, diff --git a/server/serverHelpers/serverUtilities.ts b/server/serverHelpers/serverUtilities.ts index b0c6557949..a4b631094a 100644 --- a/server/serverHelpers/serverUtilities.ts +++ b/server/serverHelpers/serverUtilities.ts @@ -534,14 +534,6 @@ function initializeOperationFunctionMap(): Map { + it('deploy_component({activate:false}) stages into the hidden dir without going live, and returns a deployment_id', async () => { const name = freshName(); - const res = await operations.stageComponent({ project: name, payload: await makeComponentPayload('op-staged') }); + const res = await operations.deployComponent({ + project: name, + payload: await makeComponentPayload('op-staged'), + activate: false, + }); assert.strictEqual(res.staged, true); assert.strictEqual(res.project, name); @@ -89,16 +93,18 @@ describe('deploy operations: stage_component / activate_component / deploy_compo assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, name)), false, 'staging did not touch the live path'); }); - it('activate_component takes a prior stage live', async () => { + it('deploy_component({deployment_id}) takes a prior stage live', async () => { const name = freshName(); - const staged = await operations.stageComponent({ + const staged = await operations.deployComponent({ project: name, payload: await makeComponentPayload('op-activated'), + activate: false, }); - const res = await operations.activateComponent({ project: name, deployment_id: staged.deployment_id }); + const res = await operations.deployComponent({ project: name, deployment_id: staged.deployment_id }); assert.strictEqual(res.activated, true); assert.strictEqual(res.project, name); + assert.strictEqual(res.deployment_id, staged.deployment_id); const liveDir = path.join(COMPONENTS_ROOT, name); assert.ok(existsSync(liveDir), 'live component dir now exists'); assert.match(await readIndex(liveDir), /op-activated/); @@ -106,11 +112,6 @@ describe('deploy operations: stage_component / activate_component / deploy_compo assert.doesNotMatch(res.message, /restart/i); }); - it('activate_component rejects when no deployment_id is supplied', async () => { - const name = freshName(); - await assert.rejects(() => operations.activateComponent({ project: name }), /deployment_id.*required/i); - }); - it('deploy_component (two-phase default) stages then activates end-to-end', async () => { const name = freshName(); const res = await operations.deployComponent({ project: name, payload: await makeComponentPayload('op-deployed') }); diff --git a/unitTests/components/deployPhaseValidators.test.js b/unitTests/components/deployPhaseValidators.test.js index daa85573e6..779cbfa32a 100644 --- a/unitTests/components/deployPhaseValidators.test.js +++ b/unitTests/components/deployPhaseValidators.test.js @@ -11,78 +11,6 @@ const validator = require('#js/components/operationsValidation'); const ok = (result) => assert.strictEqual(result, undefined, `expected valid, got: ${result && result.message}`); const rejected = (result) => assert.ok(result, 'expected a validation error'); -describe('stageComponentValidator', () => { - it('accepts a project-only request', () => { - ok(validator.stageComponentValidator({ project: 'my_app' })); - }); - - it('accepts a package deploy with install options', () => { - ok( - validator.stageComponentValidator({ - project: 'my_app', - package: 'npm:@org/thing', - install_command: 'npm ci', - install_timeout: 60000, - install_allow_scripts: false, - deployment_timeout: 120000, - }) - ); - }); - - it('requires a project', () => { - rejected(validator.stageComponentValidator({ package: 'npm:@org/thing' })); - }); - - it('rejects an invalid project name', () => { - rejected(validator.stageComponentValidator({ project: 'bad/name' })); - }); - - it('rejects a urlPath containing ".."', () => { - rejected(validator.stageComponentValidator({ project: 'my_app', package: 'npm:x', urlPath: '/a/../b' })); - }); -}); - -describe('activateComponentValidator', () => { - it('accepts a project + deployment_id', () => { - ok(validator.activateComponentValidator({ project: 'my_app', deployment_id: 'abc-123' })); - }); - - it('accepts a rolling restart', () => { - ok(validator.activateComponentValidator({ project: 'my_app', deployment_id: 'abc-123', restart: 'rolling' })); - }); - - it('accepts a boolean restart and ignore_replication_errors', () => { - ok( - validator.activateComponentValidator({ - project: 'my_app', - deployment_id: 'abc-123', - restart: true, - ignore_replication_errors: true, - }) - ); - }); - - it('requires a project', () => { - rejected(validator.activateComponentValidator({ deployment_id: 'abc-123' })); - }); - - it('rejects an invalid restart value', () => { - rejected(validator.activateComponentValidator({ project: 'my_app', restart: 'sideways' })); - }); - - it('rejects a path-traversal deployment_id (it becomes a staging-dir path segment)', () => { - for (const bad of ['../evil', 'a/b', 'dep/../..', '.', '..']) { - rejected(validator.activateComponentValidator({ project: 'my_app', deployment_id: bad })); - } - }); - - it('accepts a normal UUID-shaped deployment_id', () => { - ok( - validator.activateComponentValidator({ project: 'my_app', deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa' }) - ); - }); -}); - describe('revertComponentValidator', () => { it('accepts a project-only revert', () => { ok(validator.revertComponentValidator({ project: 'my_app' })); @@ -108,16 +36,29 @@ describe('revertComponentValidator', () => { }); }); -describe('deployComponentValidator two_phase + revert_on_failure flags', () => { +describe('deployComponentValidator two-phase props (activate / deployment_id / flags)', () => { + it('accepts activate: false (stage-and-stop)', () => { + ok(validator.deployComponentValidator({ project: 'my_app', activate: false })); + }); + + it('accepts a deployment_id (activate an existing stage)', () => { + ok( + validator.deployComponentValidator({ project: 'my_app', deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa' }) + ); + }); + + it('rejects a path-traversal deployment_id (it becomes a staging-dir path segment)', () => { + for (const bad of ['../evil', 'a/b', 'dep/../..', '.', '..']) { + rejected(validator.deployComponentValidator({ project: 'my_app', deployment_id: bad })); + } + }); + it('accepts revert_on_failure: true', () => { ok(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', revert_on_failure: true })); }); - it('accepts two_phase: false (legacy opt-out)', () => { + it('accepts two_phase: false (legacy opt-out) and true', () => { ok(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', two_phase: false })); - }); - - it('accepts two_phase: true', () => { ok(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', two_phase: true })); }); diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index 1b47d74f51..16e532aaa3 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -290,15 +290,15 @@ export const OPERATIONS_ENUM = { PACKAGE_CUSTOM_FUNCTION_PROJECT: 'package_custom_function_project', DEPLOY_CUSTOM_FUNCTION_PROJECT: 'deploy_custom_function_project', PACKAGE_COMPONENT: 'package_component', + // deploy_component runs a two-phase deploy internally (stage the incoming version into a hidden + // dir cluster-wide, gate on every node, then atomically swap it live). The two phases are fanned + // out to peers as deploy_component tagged with an internal `_phase` marker rather than separate + // public operations. Public knobs: `activate: false` (stage-and-stop, returns a staged + // deployment_id) and `deployment_id` (activate a previously-staged deployment). See + // components/Application.ts (stageApplication/activateApplication). DEPLOY_COMPONENT: 'deploy_component', - // Two-phase deploy sub-operations. stage_component builds the incoming version into a hidden - // staging directory cluster-wide (no go-live); activate_component atomically swaps the staged - // copy into the live path and restarts. deploy_component orchestrates the two so existing callers - // are unaffected. See components/Application.ts (stageApplication/activateApplication). - STAGE_COMPONENT: 'stage_component', - ACTIVATE_COMPONENT: 'activate_component', // Swap a component's live version back to its retained previous version, cluster-wide. Backs - // customer-driven rollback (activate → test → revert) and swap-back on a partially-failed activate. + // customer-driven rollback (deploy → test → revert) and swap-back on a partially-failed activate. // See components/Application.ts (revertApplication). REVERT_COMPONENT: 'revert_component', READ_TRANSACTION_LOG: 'read_transaction_log', diff --git a/utility/operation_authorization.ts b/utility/operation_authorization.ts index c48a4a6c3c..b24eb95786 100644 --- a/utility/operation_authorization.ts +++ b/utility/operation_authorization.ts @@ -286,8 +286,6 @@ requiredPermissions.set(functionsOperations.addComponent.name, new (permission a requiredPermissions.set(functionsOperations.dropCustomFunctionProject.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.packageComponent.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.deployComponent.name, new (permission as any)(true, [])); -requiredPermissions.set(functionsOperations.stageComponent.name, new (permission as any)(true, [])); -requiredPermissions.set(functionsOperations.activateComponent.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.revertComponent.name, new (permission as any)(true, [])); requiredPermissions.set( deploymentOperations.handleListDeployments.name, From 4197f1c285d9dcaeca52b9da4577097e79345883 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Tue, 21 Jul 2026 10:20:36 -0400 Subject: [PATCH 13/94] feat(deploy): bound staged-build retention per component (evict-on-stage) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With deploy_component({ activate: false }) leaving staged builds around for a later deploy_component({ deployment_id }), those not-yet-activated builds could accumulate without limit (a full deploy consumes its build on activate, so only stage-and-stops pile up). stageApplication now evicts the oldest such builds for a component beyond deployment_stagingRetention_maxCount (default 5) after each successful stage — always keeping the just-staged one plus the newest N-1 by mtime. Eviction is best-effort but awaited so the count is settled when the stage returns. Per the harper#1849 decision: retention is count-only and fully automatic (no new op); hdb_deployment rows stay as the audit trail (payload blobs already self-reclaim by size). Consequence: activating a deployment_id that has aged out of the window fails "no staged build found", as expected. DESIGN.md + a retention test added. Co-Authored-By: Claude Opus 4.8 --- DESIGN.md | 14 +++++ components/Application.ts | 71 ++++++++++++++++++++++ unitTests/components/deployStaging.test.js | 22 +++++++ utility/hdbTerms.ts | 3 + 4 files changed, 110 insertions(+) diff --git a/DESIGN.md b/DESIGN.md index 75a2921ad6..6523650681 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -405,6 +405,20 @@ some nodes live and some not, so the cluster reconverges on one version. The pre per-node (each node retains its own outgoing version during its own activate), so a replicated revert has a local rollback source on every node. +**Staged-build retention.** A full deploy consumes its staged build immediately (activate renames it +live), so the only builds that accumulate are `activate: false` stage-and-stops that are never +activated — each leaves `.deploy-staging//` in place so a later +`deploy_component({deployment_id})` can activate it. `stageApplication` bounds this: after a successful +stage it evicts the oldest not-yet-activated staged builds for that component beyond +`deployment_stagingRetention_maxCount` (default 5, `pruneStagedBuilds`), always keeping the just-staged +one and the newest N−1 by mtime. Eviction is best-effort (`allSettled`, trace-logged) but awaited so +the count is settled when the stage returns. Retention is deliberately count-only and automatic: +per the harper#1849 discussion, `hdb_deployment` rows stay as the audit trail (payload blobs already +self-reclaim by size, `deployment_payloadRetention_maxSize`), and no `delete_deployment` op was added — +eviction-on-stage keeps the surface at zero new operations. Consequence: activating a `deployment_id` +that has already aged out of the window fails with "no staged build found" — expected once more than +`maxCount` newer stages have landed for that component. + ## Scheduler: cluster-once execution without a consensus primitive (`resources/scheduler/`) The built-in `scheduler` plugin (#951) runs config-declared jobs "exactly once per cluster." The diff --git a/components/Application.ts b/components/Application.ts index 8721ab91ff..2ddaabfa40 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -438,6 +438,73 @@ function previousDirPathFor(liveDirPath: string): string { return join(dirname(liveDirPath), DEPLOY_PREVIOUS_DIR, basename(liveDirPath)); } +// Max not-yet-activated staged builds kept per component before the oldest are evicted on the next +// stage. A full deploy consumes its staged build immediately (activate renames it live), so this only +// bounds `activate: false` stage-and-stops that are never activated. Configurable via +// deployment_stagingRetention_maxCount. +export const DEFAULT_STAGING_RETENTION_MAX_COUNT = 5; + +function getStagingRetentionMaxCount(): number { + const configured = getConfigValue(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT); + // Only a number or numeric string is a valid count; reject everything else (unset, boolean, array, + // blank) and fall back to the default — mirroring getPayloadRetentionMaxSize's defensive coercion. + if (typeof configured !== 'number' && typeof configured !== 'string') return DEFAULT_STAGING_RETENTION_MAX_COUNT; + if (typeof configured === 'string' && configured.trim() === '') return DEFAULT_STAGING_RETENTION_MAX_COUNT; + const parsed = Number(configured); + return Number.isFinite(parsed) && parsed >= 1 ? Math.floor(parsed) : DEFAULT_STAGING_RETENTION_MAX_COUNT; +} + +/** + * Prune the oldest not-yet-activated staged builds for a component, keeping at most `maxCount` (newest + * by mtime) and ALWAYS retaining the just-built one (`keepStagingId`). Staged builds live at + * `.deploy-staging//`, and each stagingId parent holds exactly one component's build, + * so an evicted build's whole parent directory is removed. Entirely best-effort: any error (a racing + * concurrent stage, a busy dir) is logged at trace and never fails the stage that triggered it. + */ +async function pruneStagedBuilds(componentName: string, keepStagingId: string, maxCount: number): Promise { + try { + if (!(maxCount >= 1)) return; + const componentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); + if (!componentsRoot) return; + const stagingRoot = join(componentsRoot, DEPLOY_STAGING_DIR); + let parents: import('node:fs').Dirent[]; + try { + parents = await readdir(stagingRoot, { withFileTypes: true }); + } catch (err) { + if ((err as any).code === 'ENOENT') return; // nothing staged yet + throw err; + } + // Collect this component's staged builds: // that still exist. + const builds: Array<{ stagingId: string; parentPath: string; mtime: number }> = []; + for (const parent of parents) { + if (!parent.isDirectory()) continue; + const parentPath = join(stagingRoot, parent.name); + try { + const st = await stat(join(parentPath, componentName)); + builds.push({ stagingId: parent.name, parentPath, mtime: st.mtimeMs }); + } catch (err) { + if ((err as any).code !== 'ENOENT') throw err; // parent holds a different component; skip + } + } + // Always keep the build we just made, plus the newest (maxCount - 1) of the OTHERS by mtime; evict + // the rest. Computing it as "keep keepStagingId + top-(N-1) others" (rather than "evict everything + // past the top N") keeps the count exact even when mtimes tie and the just-built one would + // otherwise sort into the eviction window. Await the evictions (best-effort via allSettled) so the + // retention count is settled by the time the stage returns. + const others = builds.filter((build) => build.stagingId !== keepStagingId).sort((a, b) => b.mtime - a.mtime); + const evictions = others + .slice(Math.max(0, maxCount - 1)) + .map((build) => + rm(build.parentPath, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }).catch((err) => + logger.trace?.(`Deferred prune of staged ${componentName} build ${build.stagingId}: ${err.message}`) + ) + ); + await Promise.allSettled(evictions); + } catch (err) { + logger.trace?.(`Staged-build prune for ${componentName} skipped: ${(err as Error).message}`); + } +} + /** * Atomically move `targetDirPath` aside into a hidden, per-component staging directory if it * exists, returning the aside staging directory (for best-effort cleanup) or null when there was @@ -1208,6 +1275,10 @@ export async function stageApplication(application: Application): Promise { + // Simulate repeated `activate: false` stage-and-stops of the same component (distinct stagingIds, + // never activated). Each stage should prune older ones down to the retention count. + const name = `stage_test_${process.pid}_${counter++}`; + const stagingRoot = path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR); + const stagingIds = []; + for (let i = 0; i < 7; i++) { + const app = new Application({ name, payload: await makeComponentPayload(`v${i}`) }); + stagingIds.push(app.stagingId); + await stageApplication(app); + } + + const remaining = stagingIds.filter((id) => existsSync(path.join(stagingRoot, id, name))); + // Contract: at most `maxCount` staged builds are retained per component, and the just-staged one is + // always kept. (Which older builds are evicted is ordered by mtime — reliable in real use where + // stages are time-separated, but this tight loop can create ties, so it isn't asserted here.) + assert.strictEqual(remaining.length, 5, `expected 5 staged builds retained, got ${remaining.length}`); + assert.ok(existsSync(path.join(stagingRoot, stagingIds[6], name)), 'the most recent stage is always retained'); + + await fs.rm(stagingRoot, { recursive: true, force: true }); + }); + it('only one previous is retained across three deploys (older previous evicted)', async () => { const name = `stage_test_${process.pid}_${counter++}`; await deployVersion(name, 'v1'); diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index 16e532aaa3..e1e2dd81f9 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -555,6 +555,9 @@ export const CONFIG_PARAMS = { OPERATIONSAPI_NETWORK_MAXREQUESTBODYSIZE: 'operationsApi_network_maxRequestBodySize', OPERATIONSAPI_COMPONENTFILE_MAXSIZE: 'operationsApi_componentFile_maxSize', DEPLOYMENT_PAYLOADRETENTION_MAXSIZE: 'deployment_payloadRetention_maxSize', + // Max not-yet-activated staged builds kept per component (`activate: false` stage-and-stops). When a + // new stage lands, the oldest beyond this count are evicted. See components/Application.ts. + DEPLOYMENT_STAGINGRETENTION_MAXCOUNT: 'deployment_stagingRetention_maxCount', OPERATIONSAPI_TLS: 'operationsApi_tls', OPERATIONSAPI_TLS_CERTIFICATE: 'operationsApi_tls_certificate', OPERATIONSAPI_TLS_PRIVATEKEY: 'operationsApi_tls_privateKey', From 9f645bc29a490e817233505042c21e53b55d691a Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Tue, 21 Jul 2026 10:25:12 -0400 Subject: [PATCH 14/94] fix(cli): `harper activate` must carry a deployment_id (fail fast, not full deploy) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After folding activate into deploy_component, `harper activate` mapped to deploy_component with no marker. deployComponent only routes to activate-existing when deployment_id is truthy — otherwise it falls through to a full two-phase deploy, and packageCwdForUpload (skips only for package || deployment_id) would tar + upload the CWD. So `harper activate project=x` with a missing/mistyped deployment_id silently ran a brand-new deploy from local files. The `activate` verb now carries a CLI-internal `_verb` marker, and verbRequirementError (pure, exported) rejects it when deployment_id is absent — checked BEFORE packaging, with a clear message, so it fails fast instead of deploying. The marker is stripped before the request body. Adds CLI verb tests covering stage/activate mapping and the missing-deployment_id guard. (Review catch.) Co-Authored-By: Claude Opus 4.8 --- bin/cliOperations.ts | 25 +++++++++++++++++-- unitTests/bin/cliOperations.test.js | 37 +++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 2 deletions(-) diff --git a/bin/cliOperations.ts b/bin/cliOperations.ts index 51e022f0d9..c353bd9f57 100644 --- a/bin/cliOperations.ts +++ b/bin/cliOperations.ts @@ -31,9 +31,22 @@ const OP_ALIASES = { // takes that staged deployment live (no upload). const OP_VERB_PROPS: Record> = { stage: { operation: 'deploy_component', activate: false }, - activate: { operation: 'deploy_component' }, + // `_verb` is a CLI-internal marker (stripped before the request) so verbRequirementError can enforce + // that `harper activate` carries a deployment_id — without it, deploy_component's generic + // "no deployment_id → full deploy" fallback would silently build a brand-new deploy from the CWD. + activate: { operation: 'deploy_component', _verb: 'activate' }, }; +// Guard CLI-verb requirements that the operation itself can't enforce (the op has no notion of which +// verb invoked it). Returns an error message, or null when the request is fine. Pure + exported so it +// is unit-testable without the network/process-exit machinery in cliOperations. +function verbRequirementError(req: any): string | null { + if (req._verb === 'activate' && !req.deployment_id) { + return '`harper activate` requires a deployment_id from a prior `harper stage` — usage: harper activate project= deployment_id='; + } + return null; +} + // Shown for any local-instance connection failure (missing pid, missing/stale domain // socket, or a refused/ENOENT connect against it) — they're all the same user-facing // scenario: Harper isn't running. Remote-target failures keep the detailed error instead, @@ -164,7 +177,7 @@ function operationFields(req: any): any { return fields; } -export { cliOperations, buildRequest }; +export { cliOperations, buildRequest, verbRequirementError }; // Package the current working directory into a multipart tarball upload for deploy_component. Covers // `harper deploy` and `harper stage` (deploy_component with activate:false) — both upload the incoming @@ -303,6 +316,14 @@ async function cliOperations(req: any, skipResponseLog = false) { process.exit(1); } } + // Enforce CLI-verb requirements (e.g. `harper activate` needs a deployment_id) BEFORE packaging so a + // mistake fails fast instead of building + uploading a fresh deploy from the CWD. + const verbError = verbRequirementError(req); + if (verbError) { + console.error(verbError); + process.exit(1); + } + delete req._verb; // CLI-internal marker; never send it in the request body await PREPARE_OPERATION[req.operation]?.(req); try { let options = target ?? { diff --git a/unitTests/bin/cliOperations.test.js b/unitTests/bin/cliOperations.test.js index 69a5ae67ef..da0c76c92f 100644 --- a/unitTests/bin/cliOperations.test.js +++ b/unitTests/bin/cliOperations.test.js @@ -766,3 +766,40 @@ describe('cliOperations', () => { }); }); }); + +describe('deploy CLI verbs (stage / activate fold into deploy_component)', () => { + const { buildRequest, verbRequirementError } = cliOperationsModule; + let savedArgv; + beforeEach(() => { + savedArgv = process.argv; + }); + afterEach(() => { + process.argv = savedArgv; + }); + + it('`stage` maps to deploy_component with activate:false', () => { + process.argv = ['node', 'harper', 'stage', 'project=my_app']; + const req = buildRequest(); + assert.strictEqual(req.operation, 'deploy_component'); + assert.strictEqual(req.activate, false); + }); + + it('`activate` with a deployment_id maps to deploy_component and passes the verb guard', () => { + process.argv = ['node', 'harper', 'activate', 'project=my_app', 'deployment_id=abc-123']; + const req = buildRequest(); + assert.strictEqual(req.operation, 'deploy_component'); + assert.strictEqual(req.deployment_id, 'abc-123'); + assert.strictEqual(verbRequirementError(req), null); + }); + + it('`activate` WITHOUT a deployment_id is rejected (would otherwise become a full deploy from the CWD)', () => { + process.argv = ['node', 'harper', 'activate', 'project=my_app']; + const req = buildRequest(); + assert.match(verbRequirementError(req), /deployment_id/); + }); + + it('verbRequirementError ignores non-activate deploys', () => { + assert.strictEqual(verbRequirementError({ operation: 'deploy_component' }), null); + assert.strictEqual(verbRequirementError({ operation: 'deploy_component', activate: false }), null); + }); +}); From b0e33e3af601efcf6fe46c29b3570e3ae379458b Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Tue, 21 Jul 2026 19:22:04 -0400 Subject: [PATCH 15/94] test(deploy): reset process-wide restart buffer after two-phase op tests deployPhaseOperations deploys genuinely-new components, which post-merge trips deployComponent's requestRestart() (harper#674 new-component scoping) and sets the process-wide restart-needed shared buffer. That leaked into requestRestart.test.js's "pristine buffer" assertion, failing it on suite ordering alone. Restore the buffer in the existing after() hook. Co-Authored-By: Claude Opus 4.8 --- unitTests/components/deployPhaseOperations.test.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index 8979ca1bb4..5d26f95b2e 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -20,6 +20,7 @@ testUtils.preTestPrep(); const operations = require('#src/components/operations'); const { DEPLOY_STAGING_DIR } = require('#src/components/Application'); +const { resetRestartNeeded } = require('#src/components/requestRestart'); const { getConfigPath } = require('#src/config/configUtils'); const { CONFIG_PARAMS } = require('#src/utility/hdbTerms'); @@ -74,6 +75,11 @@ describe('deploy operations: stage_component / activate_component / deploy_compo await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR), { recursive: true, force: true }); await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-previous'), { recursive: true, force: true }); await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-aside'), { recursive: true, force: true }); + // Deploying a genuinely-new component flips the process-wide restart-needed buffer on + // (harper#674, via deployComponent's requestRestart() scoping). That buffer is shared across + // the whole mocha process, so restore it or a later test asserting a pristine buffer + // (e.g. requestRestart.test.js) fails on ordering alone. + resetRestartNeeded(); }); it('deploy_component({activate:false}) stages into the hidden dir without going live, and returns a deployment_id', async () => { From c907eb2a7432e08df26d94b3b213217a139c4192 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Tue, 21 Jul 2026 19:32:05 -0400 Subject: [PATCH 16/94] fix(deploy): two-phase deploy marks restart-required for new components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The merge with main brought harper#674/#1806: a deploy_component with restart:false must set get_status restartRequired for a genuinely-new (never-loaded) component, scoped so a redeploy of an already-active component stays quiet. Main implemented this only in the one-shot deploy path (requestRestart() gated on Application.isNewComponent). Two-phase deploy — the default whenever the system db is replicated — never carried that behavior, so inactive-component-404.test.ts failed: a fresh deploy left restartRequired false and the actionable 404 never surfaced. Two-phase also can't set isNewComponent the way the one-shot path does: stageApplication extracts into a fresh staging dir, so extractApplication never sees the live directory and isNewComponent stayed default-true for everything (which would wrongly mark a restart on a redeploy). Fix both: - activateApplication now sets isNewComponent from whether the live dir existed BEFORE the swap (the two-phase equivalent of extract's in-place check) — true for a first deploy, false for a redeploy. - Extract markRestartRequiredForNewComponent() and call it on every no-restart path: one-shot (unchanged behavior), two-phase origin, activate-existing, and the per-node peer activate — so a new component deployed cluster-wide with restart:false reports restartRequired on every node, matching the one-shot peer behavior. Unit coverage: deployStaging asserts activate sets isNewComponent true for a first-ever activate and false when replacing an existing live version. Co-Authored-By: Claude Opus 4.8 --- components/Application.ts | 13 ++++++++ components/operations.js | 39 +++++++++++++++++++--- unitTests/components/deployStaging.test.js | 7 ++++ 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 9730e80f21..9c6b3cf285 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -1383,6 +1383,19 @@ export async function activateApplication(application: Application): Promise), then rename // the staged copy into place. Both live under the components root, so the rename is same-fs and // atomic — there is no interval where `dirPath` is a partially populated directory. retainAsPrevious diff --git a/components/operations.js b/components/operations.js index f679d404e0..86d52e9f7c 100644 --- a/components/operations.js +++ b/components/operations.js @@ -430,6 +430,24 @@ async function deployComponent(req) { return deployComponentTwoPhase(req, credentialReferences); } +/** + * A genuinely-new (never-loaded) component deployed without an immediate restart can't serve its + * routes until Harper restarts, so mark a restart as needed (harper#674). This is the setter only; it + * does not itself restart — it makes get_status report restartRequired:true and lets the REST + * route-miss path surface the actionable "needs a restart" 404. Scoped to new components (harper#1806): + * an existing, already-loaded component's own file watcher independently requests a restart if a + * redeploy actually needs one, so a redeploy stays quiet. Runs per node — each node checks its own + * isNewComponent, since directory state (new vs. redeploy) can differ across the cluster. The one-shot + * path has extractApplication set isNewComponent in place; the two-phase path has activateApplication + * set it at swap time (staging is always fresh, so extract never sees the live dir). + */ +function markRestartRequiredForNewComponent(application) { + if (application.isNewComponent) { + const { requestRestart } = require('./requestRestart.ts'); + requestRestart(); + } +} + /** * Legacy one-shot deploy: extract + `npm install` in place on the origin, then replicate the whole * deploy_component operation to peers, which each do the same. Preserved verbatim (behavior-for- @@ -555,10 +573,7 @@ async function deployComponentOneShot(req, credentialReferences, isReplicatedExe // genuinely is needed, that component's already-running file watcher (Scope/ // EntryHandler, see deployLifecycle.ts) independently detects the post-deploy file // changes and requests the restart itself. - if (application.isNewComponent) { - const { requestRestart } = require('./requestRestart.ts'); - requestRestart(); - } + markRestartRequiredForNewComponent(application); response.message = `Successfully deployed: ${application.name}`; } @@ -722,7 +737,12 @@ async function deployComponentTwoPhase(req, credentialReferences) { emit('phase', { phase: 'restart', status: 'done' }); response.restartJobId = jobResponse.job_id; response.message = `Successfully deployed: ${application.name}, restarting Harper`; - } else response.message = `Successfully deployed: ${application.name}`; + } else { + // No restart requested: a genuinely-new component still needs one to serve its routes + // (harper#674). activateApplication set isNewComponent from the pre-swap live dir above. + markRestartRequiredForNewComponent(application); + response.message = `Successfully deployed: ${application.name}`; + } // ---- Activate gate: rare, but a node can stage OK and then fail the swap. ---- if (!req.ignore_replication_errors) { @@ -841,6 +861,11 @@ async function deployPhaseActivate(req) { // The origin sets restart=true on the sub-op only for an immediate restart; rolling restarts are // driven separately by the origin via a replicated restart_service job. if (req.restart === true) manageThreads.restartWorkers('http'); + // Not restarting now: mark restart-required per node for a genuinely-new component (harper#674), the + // same marking the one-shot peer path does — so a new component deployed cluster-wide with + // restart:false reports restartRequired on every node, not just the origin. A rolling restart, which + // also arrives here with restart:false, clears the flag when it reaches this node. + else markRestartRequiredForNewComponent(application); return { message: `Activated component: ${req.project}`, project: req.project, activated: true }; } @@ -899,6 +924,10 @@ async function deployComponentActivateExisting(req, credentialReferences) { emit('phase', { phase: 'restart', status: 'done' }); response.restartJobId = jobResponse.job_id; response.message = `Activated component: ${req.project}, restarting Harper`; + } else { + // No restart requested: activating a genuinely-new component still needs one to serve its routes + // (harper#674). activateApplication set isNewComponent from the pre-swap live dir above. + markRestartRequiredForNewComponent(application); } // Best-effort: flip the staged deployment row (left 'staged' by the stage-and-stop) to success now diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index a97a5c0607..696c8c3b64 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -101,6 +101,10 @@ describe('two-phase deploy primitives (stage / activate / discard)', function () assert.match(await readMarker(app.dirPath), /v1/, 'live dir holds the staged content'); assert.strictEqual(existsSync(app.stagingDirPath), false, 'the staged copy was consumed by the swap'); assert.strictEqual(app.buildDirPath, app.dirPath, 'build target reset to live after activation'); + // No live version existed before the swap, so this is a first deploy. deploy_component reads this + // to mark restartRequired for a never-loaded component (harper#674); staging is always fresh, so + // activate — not extract — is what establishes it on the two-phase path. + assert.strictEqual(app.isNewComponent, true, 'a first-ever activate marks the component new'); await fs.rm(app.dirPath, { recursive: true, force: true }); }); @@ -119,6 +123,9 @@ describe('two-phase deploy primitives (stage / activate / discard)', function () assert.match(await readMarker(dirPath), /v2/, 'live dir now holds the new version'); assert.strictEqual(existsSync(path.join(dirPath, 'leftover.txt')), false, 'old-version files are gone from live'); + // A live version existed before the swap, so this is a redeploy, not a first deploy — deploy_component + // must NOT self-request a restart here (harper#1806); the existing component's own watcher does. + assert.strictEqual(app.isNewComponent, false, 'activating over an existing live version marks it not-new'); await fs.rm(dirPath, { recursive: true, force: true }); await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); From 1ec722b4ad621cc0ed04f5841ec492de2168d4da Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Tue, 21 Jul 2026 19:43:22 -0400 Subject: [PATCH 17/94] test(deploy): assert restart-required directly on every two-phase leg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The activate-existing test only checked res.message, which never mentions "restart" on the no-restart path — so it passed whether or not the restart-required marking ran. Assert restartNeeded() directly instead, and cover the legs that had no fast test at all: - activate-existing (deployComponentActivateExisting): now asserts a never-live component marks a restart. - origin two-phase (deployComponentTwoPhase): asserts a fresh deploy marks a restart. - peer _phase:activate (deployPhaseActivate): new test driving the peer activate leg via the public op with internal markers, over a directly staged build — the per-node marking had zero coverage. - redeploy negative: a redeploy of an already-live component must NOT self-request a restart (harper#1806), guarding activateApplication's isNewComponent:false path at the operation level. beforeEach resets the process-wide restart buffer so each assertion reflects only its own deploy. Regressions on any leg now fail fast here instead of only in the inactive-component-404 integration test. Co-Authored-By: Claude Opus 4.8 --- .../components/deployPhaseOperations.test.js | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index 5d26f95b2e..d820a1f37a 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -19,8 +19,8 @@ const testUtils = require('../testUtils.js'); testUtils.preTestPrep(); const operations = require('#src/components/operations'); -const { DEPLOY_STAGING_DIR } = require('#src/components/Application'); -const { resetRestartNeeded } = require('#src/components/requestRestart'); +const { DEPLOY_STAGING_DIR, Application, stageApplication } = require('#src/components/Application'); +const { restartNeeded, resetRestartNeeded } = require('#src/components/requestRestart'); const { getConfigPath } = require('#src/config/configUtils'); const { CONFIG_PARAMS } = require('#src/utility/hdbTerms'); @@ -82,6 +82,11 @@ describe('deploy operations: stage_component / activate_component / deploy_compo resetRestartNeeded(); }); + // Each test below deploys fresh components, and deploying a never-live component flips the + // process-wide restart-needed buffer (harper#674). Start every test from a known-clean buffer so the + // restartNeeded() assertions below reflect only that test's own deploy, not a prior test's leak. + beforeEach(() => resetRestartNeeded()); + it('deploy_component({activate:false}) stages into the hidden dir without going live, and returns a deployment_id', async () => { const name = freshName(); const res = await operations.deployComponent({ @@ -116,6 +121,11 @@ describe('deploy operations: stage_component / activate_component / deploy_compo assert.match(await readIndex(liveDir), /op-activated/); // A restart was not requested, so the message must not claim one. assert.doesNotMatch(res.message, /restart/i); + // ...but this component was never live before, so activating it without a restart must mark one + // as required (harper#674) — the activate-existing leg of the two-phase restart-required fix. The + // message assertion above can't see this: the string never mentions "restart" on the no-restart + // path whether or not the marking runs, so assert the flag directly. + assert.strictEqual(restartNeeded(), true, 'activating a never-live component without restart marks a restart required'); }); it('deploy_component (two-phase default) stages then activates end-to-end', async () => { @@ -126,6 +136,8 @@ describe('deploy operations: stage_component / activate_component / deploy_compo assert.strictEqual(typeof res.deployment_id, 'string'); const liveDir = path.join(COMPONENTS_ROOT, name); assert.match(await readIndex(liveDir), /op-deployed/, 'component is live after a two-phase deploy'); + // New component deployed without a restart → restart required (harper#674), the origin two-phase leg. + assert.strictEqual(restartNeeded(), true, 'a fresh two-phase deploy without restart marks a restart required'); // The staged copy was consumed by the swap; its per-deploy staging parent is cleaned up. assert.strictEqual( existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, res.deployment_id)), @@ -134,6 +146,43 @@ describe('deploy operations: stage_component / activate_component / deploy_compo ); }); + it('redeploying an already-live component without restart does NOT mark a restart required', async () => { + // The negative direction of harper#674/#1806: an existing, already-active component's own watcher + // requests any restart a redeploy needs, so deploy_component must stay quiet. Two-phase can't lean + // on extractApplication's in-place check (it builds into a fresh staging dir), so this guards that + // activateApplication correctly reports isNewComponent:false when a live version already exists. + const name = freshName(); + await operations.deployComponent({ project: name, payload: await makeComponentPayload('redeploy-v1') }); + resetRestartNeeded(); // clear the flag the first (new-component) deploy legitimately set + await operations.deployComponent({ project: name, payload: await makeComponentPayload('redeploy-v2') }); + assert.strictEqual(restartNeeded(), false, 'a redeploy of an already-live component must not self-request a restart'); + }); + + it('peer _phase:activate takes a locally-staged build live and marks a restart for a new component', async () => { + // The peer leg of the fix: a peer applies the fanned-out deploy_component tagged _phase:'activate' + // (deployPhaseActivate), swapping its OWN locally-staged build live. It must mark restart-required + // per node for a genuinely-new component (harper#674) — otherwise a cluster-wide restart:false + // deploy reports restartRequired on the origin only. Stage the build directly (standing in for the + // peer's earlier stage phase), then drive the activate phase through the public op with the + // internal markers, exactly as the replicated fan-out does. + const name = freshName(); + const deploymentId = `peer-activate-${name}`; + const staged = new Application({ name, payload: await makeComponentPayload('peer-activated'), stagingId: deploymentId }); + await stageApplication(staged); + + const res = await operations.deployComponent({ + project: name, + _phase: 'activate', + _deploymentId: deploymentId, + restart: false, + }); + + assert.strictEqual(res.activated, true, 'peer activate reports the component activated'); + const liveDir = path.join(COMPONENTS_ROOT, name); + assert.match(await readIndex(liveDir), /peer-activated/, 'the locally-staged build is now live'); + assert.strictEqual(restartNeeded(), true, 'peer activate of a never-live component without restart marks a restart required'); + }); + it('deploy_component with two_phase:false runs the legacy one-shot path', async () => { const name = freshName(); const res = await operations.deployComponent({ From 911d617a2d19b9178b59db20f3a893356bf3505b Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Tue, 21 Jul 2026 19:45:46 -0400 Subject: [PATCH 18/94] style(test): prettier line-wrap for deployPhaseOperations assertions Co-Authored-By: Claude Opus 4.8 --- .../components/deployPhaseOperations.test.js | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index d820a1f37a..de5ab7e512 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -125,7 +125,11 @@ describe('deploy operations: stage_component / activate_component / deploy_compo // as required (harper#674) — the activate-existing leg of the two-phase restart-required fix. The // message assertion above can't see this: the string never mentions "restart" on the no-restart // path whether or not the marking runs, so assert the flag directly. - assert.strictEqual(restartNeeded(), true, 'activating a never-live component without restart marks a restart required'); + assert.strictEqual( + restartNeeded(), + true, + 'activating a never-live component without restart marks a restart required' + ); }); it('deploy_component (two-phase default) stages then activates end-to-end', async () => { @@ -155,7 +159,11 @@ describe('deploy operations: stage_component / activate_component / deploy_compo await operations.deployComponent({ project: name, payload: await makeComponentPayload('redeploy-v1') }); resetRestartNeeded(); // clear the flag the first (new-component) deploy legitimately set await operations.deployComponent({ project: name, payload: await makeComponentPayload('redeploy-v2') }); - assert.strictEqual(restartNeeded(), false, 'a redeploy of an already-live component must not self-request a restart'); + assert.strictEqual( + restartNeeded(), + false, + 'a redeploy of an already-live component must not self-request a restart' + ); }); it('peer _phase:activate takes a locally-staged build live and marks a restart for a new component', async () => { @@ -167,7 +175,11 @@ describe('deploy operations: stage_component / activate_component / deploy_compo // internal markers, exactly as the replicated fan-out does. const name = freshName(); const deploymentId = `peer-activate-${name}`; - const staged = new Application({ name, payload: await makeComponentPayload('peer-activated'), stagingId: deploymentId }); + const staged = new Application({ + name, + payload: await makeComponentPayload('peer-activated'), + stagingId: deploymentId, + }); await stageApplication(staged); const res = await operations.deployComponent({ @@ -180,7 +192,11 @@ describe('deploy operations: stage_component / activate_component / deploy_compo assert.strictEqual(res.activated, true, 'peer activate reports the component activated'); const liveDir = path.join(COMPONENTS_ROOT, name); assert.match(await readIndex(liveDir), /peer-activated/, 'the locally-staged build is now live'); - assert.strictEqual(restartNeeded(), true, 'peer activate of a never-live component without restart marks a restart required'); + assert.strictEqual( + restartNeeded(), + true, + 'peer activate of a never-live component without restart marks a restart required' + ); }); it('deploy_component with two_phase:false runs the legacy one-shot path', async () => { From 758047c3b3f37330fb67e0d67d125a973b71ead6 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Wed, 22 Jul 2026 07:40:03 -0400 Subject: [PATCH 19/94] fix(deploy): markDeploymentTerminal must patch, not mutate the fetched row MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit markDeploymentTerminal read the deployment row via table.get() and assigned onto it (row.status = …). table.get() returns a read-only record, so that throws "Cannot assign to read only property 'status'". The one caller (deploy_component({ deployment_id }) taking a stage-and-stop build live) treats the failure as best-effort, so it only surfaced as a caught warning — but the observability write never landed: get_deployment kept reporting 'staged' after the component had gone live. Use table.patch(id, { status, completed_at }) instead — the idiomatic partial update (see resources/dataLoader.ts). It avoids the read-only mutation and, unlike a spread-and-put, can't truncate the rest of the row. Regression test: deploymentRecorder mock now models the real table (read-only get() via an opt-in freezeGet, plus patch()), and a new markDeploymentTerminal suite asserts the row flips to a terminal status and is a no-op for an absent id. A revert to the row.status = … form throws against the frozen get(), so the regression can't return silently. Co-Authored-By: Claude Opus 4.8 --- components/deploymentRecorder.ts | 7 +-- .../components/deploymentRecorder.test.js | 44 +++++++++++++++++-- 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index a8b2ec0567..703dc94ce4 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -470,9 +470,10 @@ export async function markDeploymentTerminal( if (!table) return; const row = await table.get(deploymentId); if (!row) return; - row.status = status; - row.completed_at = Date.now(); - await table.put(row); + // Patch (partial update) rather than mutating the row: table.get() returns a read-only record, so + // `row.status = …` throws "Cannot assign to read only property". A patch also touches only these two + // fields, so it can't truncate the rest of the row the way a spread-and-put might. + await table.patch(deploymentId, { status, completed_at: Date.now() }); } // Default peer-wait budget for the hdb_deployment row to replicate. A deploy is a rare, diff --git a/unitTests/components/deploymentRecorder.test.js b/unitTests/components/deploymentRecorder.test.js index 0bd2b21c75..d78f5d28c1 100644 --- a/unitTests/components/deploymentRecorder.test.js +++ b/unitTests/components/deploymentRecorder.test.js @@ -18,23 +18,35 @@ const { DeploymentRecorder, awaitDeploymentRow, readPayloadBlobWithRetry, + markDeploymentTerminal, } = require('#src/components/deploymentRecorder'); const { databases } = require('#src/resources/databases'); const terms = require('#src/utility/hdbTerms'); const DEPLOYMENT_TABLE = terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME; -// Lightweight mock: keeps a Map of rows, exposes get(id) and put(row). -function installMockDeploymentTable() { +// Lightweight mock: keeps a Map of rows, exposes get(id), put(row), and patch(id, partial). With +// `freezeGet`, get() returns a FROZEN shallow copy to mirror the real table (table.get() yields a +// read-only record) so a caller that tries to mutate the fetched row — as the pre-fix +// markDeploymentTerminal did — throws here just as it would in production; the default returns the +// stored object so awaitDeploymentRow callers keep the real row (and its payload_blob). patch() +// applies a partial update to the stored row. +function installMockDeploymentTable({ freezeGet = false } = {}) { const rows = new Map(); const mock = { rows, async get(id) { - return rows.get(id); + const row = rows.get(id); + if (!row) return undefined; + return freezeGet ? Object.freeze({ ...row }) : row; }, async put(row) { rows.set(row.deployment_id, row); }, + async patch(id, partial) { + const existing = rows.get(id); + if (existing) rows.set(id, { ...existing, ...partial }); + }, }; if (!databases.system) databases.system = {}; const prior = databases.system[DEPLOYMENT_TABLE]; @@ -639,3 +651,29 @@ describe('readPayloadBlobWithRetry', () => { ); }); }); + +describe('markDeploymentTerminal', () => { + let installed; + beforeEach(() => { + // freezeGet mirrors the real table's read-only get() so a regression back to mutating the fetched + // row (row.status = …) throws here instead of silently passing against a mutable mock. + installed = installMockDeploymentTable({ freezeGet: true }); + }); + afterEach(() => installed.restore()); + + it('flips an existing row to the terminal status via patch (not a read-only mutation)', async () => { + // This is what deploy_component({ deployment_id }) does after taking a stage-and-stop build live. + // The row from table.get() is read-only; markDeploymentTerminal must patch rather than assign onto + // it (assigning threw "Cannot assign to read only property 'status'" and silently lost the update). + installed.mock.rows.set('dep-1', { deployment_id: 'dep-1', status: 'staged', completed_at: null }); + await markDeploymentTerminal('dep-1', 'success'); + const row = installed.mock.rows.get('dep-1'); + assert.strictEqual(row.status, 'success', 'the staged row was flipped to success'); + assert.strictEqual(typeof row.completed_at, 'number', 'completed_at was stamped'); + }); + + it('is a no-op when the row is absent (best-effort, never throws)', async () => { + await markDeploymentTerminal('missing', 'success'); + assert.strictEqual(installed.mock.rows.has('missing'), false, 'no row is fabricated for an unknown id'); + }); +}); From 3ebced77365e64006e79ea7b96f28953d755a53e Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Tue, 28 Jul 2026 19:36:21 -0400 Subject: [PATCH 20/94] fix(deploy): gate peer activation failures on the activate-existing path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deploy_component({ deployment_id }) fanned the activate out to peers but never inspected the result: replicateOperation doesn't reject on a per-peer failure (failures surface only as 'failed' peer entries), so a partially failed activate returned 2xx {activated:true} while part of the cluster stayed on the old version. revert_on_failure and ignore_replication_errors are accepted by deployComponentValidator for deploy_component uniformly, so both were silently no-ops whenever deployment_id was set. Extract the two-phase activate gate into enforceActivatePeerGate() and call it from both paths, so the flags behave identically whether the activate came from a full two-phase deploy or from an earlier stage-and-stop. The activate-existing path has no DeploymentRecorder (the row was created and finished as 'staged' by the stage), so createPeerResultCollector() stands in for recorder.recordPeer/getFailedPeers — same normalizePeerResult (now exported) and the same upsert-by-node dedup, so a peer reported via both onPeerResult and the final `replicated` aggregate counts once. A gated failure also marks the row 'failed' rather than leaving it 'staged'. Tests: the failed-peer scenario had no coverage on this path. Added two (replicateOperation swapped via the repo's property-swap pattern, no sinon/rewire) — a failed peer rejects, and ignore_replication_errors still resolves. Verified both fail with the gate disabled. Co-Authored-By: Claude Opus 4.8 --- components/deploymentRecorder.ts | 2 +- components/operations.js | 174 +++++++++++++----- .../components/deployPhaseOperations.test.js | 54 ++++++ 3 files changed, 180 insertions(+), 50 deletions(-) diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index 703dc94ce4..652de07f00 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -647,7 +647,7 @@ async function* readPayloadBlobChunks( } } -function normalizePeerResult(raw: unknown): Record { +export function normalizePeerResult(raw: unknown): Record { if (!raw || typeof raw !== 'object') { // Replication layer returned a primitive — preserve as a stringified marker so the // audit row at least records that something came back from a peer. diff --git a/components/operations.js b/components/operations.js index b21536918a..7b9a8171d9 100644 --- a/components/operations.js +++ b/components/operations.js @@ -40,6 +40,7 @@ const { DeploymentRecorder, awaitDeploymentRow, markDeploymentTerminal, + normalizePeerResult, readPayloadBlobWithRetry, coerceTimeoutMs, DEFAULT_AWAIT_ROW_TIMEOUT_MS, @@ -747,54 +748,14 @@ async function deployComponentTwoPhase(req, credentialReferences) { } // ---- Activate gate: rare, but a node can stage OK and then fail the swap. ---- - if (!req.ignore_replication_errors) { - const failed = recorder.getFailedPeers(); - if (failed.length > 0) { - let revertNote = ''; - // Opt-in swap-back: some nodes went live and some didn't, leaving the cluster split across - // versions. When revert_on_failure is set, roll the nodes that DID activate back to the - // retained previous version so the cluster reconverges. Best-effort — a revert failure must - // not mask the original activate failure. - if (req.revert_on_failure) { - try { - emit('phase', { phase: 'revert', status: 'start' }); - // The origin activated, so revert it. - await revertApplication(application); - // Revert ONLY the peers that successfully activated (see selectRevertTargets): every known - // node minus the ones that failed to activate (still on the correct version) and minus this - // node (already reverted directly above; a second bidirectional revert would flip it back). - // replicateOperation has no subset targeting, so send point-to-point via sendOperationToNode. - const { getThisNodeName } = require('../server/nodeName.ts'); - const activatedPeers = selectRevertTargets(server.nodes, failed, getThisNodeName()); - const revertOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.REVERT_COMPONENT, { - restart: req.restart, - deploymentId: recorder.deploymentId, - }); - revertOp.replicated = false; // point-to-point; the peer must not re-fan the revert - const revertResults = await Promise.allSettled( - activatedPeers.map((node) => server.replication.sendOperationToNode(node, revertOp)) - ); - const revertFailures = revertResults.filter((result) => result.status === 'rejected').length; - emit('phase', { phase: 'revert', status: 'done' }); - revertNote = - ` Rolled the origin and ${activatedPeers.length - revertFailures} of ${activatedPeers.length} ` + - `activated peer(s) back to the previous version (revert_on_failure); the ${failed.length} peer(s) ` + - `that never activated were left on their current (correct) version.` + - (revertFailures > 0 ? ` ${revertFailures} peer revert(s) also failed.` : '') + - ` Verify with get_components.`; - } catch (revertErr) { - log.warn('revert_on_failure rollback failed', revertErr); - revertNote = ` An automatic rollback (revert_on_failure) was attempted but also failed: ${revertErr?.message ?? revertErr}.`; - } - } - throw new ServerError( - `Component '${application.name}' was activated on the origin but failed to activate on ${failed.length} ` + - `of ${recorder.row.peer_results.length} peer node(s): ${describePeers(failed)}. Those nodes have the staged ` + - `build but did not go live.${revertNote} See deployment ${recorder.deploymentId} (get_deployment), or pass ` + - `ignore_replication_errors: true.` - ); - } - } + await enforceActivatePeerGate({ + req, + application, + emit, + failed: recorder.getFailedPeers(), + totalPeers: recorder.row.peer_results.length, + deploymentId: recorder.deploymentId, + }); response.deployment_id = recorder.deploymentId; maybeReclaimPayload(recorder, emit); @@ -900,7 +861,17 @@ async function deployComponentActivateExisting(req, credentialReferences) { restart: rollingRestart ? false : req.restart, deploymentId: stagingId, }); - const rep = await server.replication.replicateOperation(activateOp, {}); + // Collect per-peer outcomes so a partially-failed activate can be gated below. There is no + // DeploymentRecorder on this path (the row was created and finished as `staged` by the earlier + // stage-and-stop), so a local collector stands in for recorder.recordPeer/getFailedPeers. + const peers = createPeerResultCollector(); + const rep = await server.replication.replicateOperation(activateOp, { + onPeerResult: (result) => { + peers.record(result); + emit('peer', result); + }, + }); + if (rep?.replicated) peers.recordAll(rep.replicated); const response = { message: `Activated component: ${req.project}`, @@ -932,6 +903,27 @@ async function deployComponentActivateExisting(req, credentialReferences) { markRestartRequiredForNewComponent(application); } + // ---- Activate gate: a peer can hold a good staged build and still fail the swap. Same gate the + // two-phase activate phase uses, so revert_on_failure / ignore_replication_errors behave identically + // whether the activate came from a full deploy or from `deploy_component({ deployment_id })`. + try { + await enforceActivatePeerGate({ + req, + application, + emit, + failed: peers.getFailed(), + totalPeers: peers.total, + deploymentId: stagingId, + }); + } catch (err) { + // The origin went live but the cluster did not converge — record the terminal state before + // surfacing the failure, so get_deployment doesn't still read `staged`. + await markDeploymentTerminal(stagingId, 'failed').catch((markErr) => + log.warn('Failed to mark deployment as failed after a partial activate', markErr) + ); + throw err; + } + // Best-effort: flip the staged deployment row (left 'staged' by the stage-and-stop) to success now // that it is live. Observability only — a tracking-write failure must not fail the activate. await markDeploymentTerminal(stagingId, 'success').catch((err) => @@ -1215,6 +1207,90 @@ function describePeers(failedPeers) { return failedPeers.map((peer) => `${peer.node ?? 'unknown'} (${peer.error?.message ?? 'unknown error'})`).join(', '); } +/** + * Collect per-peer replication outcomes when there is no DeploymentRecorder to hold them — the + * activate-existing path, where the hdb_deployment row was already created (and finished as `staged`) + * by the earlier stage-and-stop. Mirrors DeploymentRecorder.recordPeer's semantics exactly: results are + * normalized by the same normalizePeerResult and upserted by node name, so a peer reported both through + * the streaming `onPeerResult` callback and again in replicateOperation's final `replicated` aggregate + * is counted once, not twice. + */ +function createPeerResultCollector() { + const list = []; + const record = (result) => { + const normalized = normalizePeerResult(result); + const nodeName = normalized.node; + const idx = nodeName ? list.findIndex((entry) => entry.node === nodeName) : -1; + if (idx >= 0) list[idx] = normalized; + else list.push(normalized); + }; + return { + record, + recordAll(results) { + if (Array.isArray(results)) for (const result of results) record(result); + }, + getFailed: () => list.filter((peer) => peer?.status === 'failed'), + get total() { + return list.length; + }, + }; +} + +/** + * Shared post-activate failure gate for every cluster-wide activate (the two-phase deploy's activate + * phase and `deploy_component({ deployment_id })`). replicateOperation never rejects on a per-peer + * failure — failures surface only as 'failed' peer entries — so without this gate a partially-failed + * activate returns 2xx and silently leaves the cluster split across versions. + * + * Unless `ignore_replication_errors` is set, throws when any peer failed to activate. When + * `revert_on_failure` is set, first rolls the origin and the peers that DID activate back to the + * retained previous version so the cluster reconverges — best-effort, since a revert failure must not + * mask the original activate failure. + */ +async function enforceActivatePeerGate({ req, application, emit, failed, totalPeers, deploymentId }) { + if (req.ignore_replication_errors) return; + if (!failed || failed.length === 0) return; + let revertNote = ''; + if (req.revert_on_failure) { + try { + emit('phase', { phase: 'revert', status: 'start' }); + // The origin activated, so revert it. + await revertApplication(application); + // Revert ONLY the peers that successfully activated (see selectRevertTargets): every known + // node minus the ones that failed to activate (still on the correct version) and minus this + // node (already reverted directly above; a second bidirectional revert would flip it back). + // replicateOperation has no subset targeting, so send point-to-point via sendOperationToNode. + const { getThisNodeName } = require('../server/nodeName.ts'); + const activatedPeers = selectRevertTargets(server.nodes, failed, getThisNodeName()); + const revertOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.REVERT_COMPONENT, { + restart: req.restart, + deploymentId, + }); + revertOp.replicated = false; // point-to-point; the peer must not re-fan the revert + const revertResults = await Promise.allSettled( + activatedPeers.map((node) => server.replication.sendOperationToNode(node, revertOp)) + ); + const revertFailures = revertResults.filter((result) => result.status === 'rejected').length; + emit('phase', { phase: 'revert', status: 'done' }); + revertNote = + ` Rolled the origin and ${activatedPeers.length - revertFailures} of ${activatedPeers.length} ` + + `activated peer(s) back to the previous version (revert_on_failure); the ${failed.length} peer(s) ` + + `that never activated were left on their current (correct) version.` + + (revertFailures > 0 ? ` ${revertFailures} peer revert(s) also failed.` : '') + + ` Verify with get_components.`; + } catch (revertErr) { + log.warn('revert_on_failure rollback failed', revertErr); + revertNote = ` An automatic rollback (revert_on_failure) was attempted but also failed: ${revertErr?.message ?? revertErr}.`; + } + } + throw new ServerError( + `Component '${application.name}' was activated on the origin but failed to activate on ${failed.length} ` + + `of ${totalPeers} peer node(s): ${describePeers(failed)}. Those nodes have the staged ` + + `build but did not go live.${revertNote} See deployment ${deploymentId} (get_deployment), or pass ` + + `ignore_replication_errors: true.` + ); +} + // Choose which peers a revert_on_failure swap-back should target: every known node EXCEPT // - `thisNodeName`: the origin, already reverted directly by the caller — a second (bidirectional) // revert would flip it back to the just-activated version; and diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index de5ab7e512..dd7484acd2 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -21,6 +21,7 @@ testUtils.preTestPrep(); const operations = require('#src/components/operations'); const { DEPLOY_STAGING_DIR, Application, stageApplication } = require('#src/components/Application'); const { restartNeeded, resetRestartNeeded } = require('#src/components/requestRestart'); +const { server } = require('#src/server/Server'); const { getConfigPath } = require('#src/config/configUtils'); const { CONFIG_PARAMS } = require('#src/utility/hdbTerms'); @@ -132,6 +133,59 @@ describe('deploy operations: stage_component / activate_component / deploy_compo ); }); + // The activate-existing path fans the activate out to peers just like the two-phase activate phase, + // and replicateOperation never rejects on a per-peer failure — failures surface only as 'failed' + // entries. Without the shared activate gate this path returned 2xx {activated:true} while part of the + // cluster stayed on the old version, making revert_on_failure/ignore_replication_errors no-ops here. + describe('deploy_component({deployment_id}) peer-failure gate', () => { + let priorReplicate; + // Swap in a replicator that reports one failed peer (the repo's property-swap pattern; no + // sinon/rewire per AGENTS.md). Restored after each test. + function stubFailedPeer() { + priorReplicate = server.replication.replicateOperation; + server.replication.replicateOperation = async () => ({ + replicated: [{ node: 'peer-a', status: 'failed', reason: 'no staged build found' }], + }); + } + afterEach(() => { + if (priorReplicate) server.replication.replicateOperation = priorReplicate; + priorReplicate = undefined; + }); + + async function stageOnly(name, marker) { + const staged = await operations.deployComponent({ + project: name, + payload: await makeComponentPayload(marker), + activate: false, + }); + return staged.deployment_id; + } + + it('rejects (does not report success) when a peer fails to activate', async () => { + const name = freshName(); + const deploymentId = await stageOnly(name, 'gate-fail'); + stubFailedPeer(); + await assert.rejects( + () => operations.deployComponent({ project: name, deployment_id: deploymentId }), + /failed to activate on 1 .*peer node\(s\).*peer-a/s, + 'a partially-failed activate must surface as an error, not a 2xx success' + ); + }); + + it('honors ignore_replication_errors on this path, resolving despite the failed peer', async () => { + const name = freshName(); + const deploymentId = await stageOnly(name, 'gate-ignore'); + stubFailedPeer(); + const res = await operations.deployComponent({ + project: name, + deployment_id: deploymentId, + ignore_replication_errors: true, + }); + assert.strictEqual(res.activated, true, 'the opt-out still returns success'); + assert.match(await readIndex(path.join(COMPONENTS_ROOT, name)), /gate-ignore/, 'the origin still went live'); + }); + }); + it('deploy_component (two-phase default) stages then activates end-to-end', async () => { const name = freshName(); const res = await operations.deployComponent({ project: name, payload: await makeComponentPayload('op-deployed') }); From 525dfd76d2b4a0757f09a3224cd1b0c768eeeec3 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Thu, 30 Jul 2026 09:25:08 -0400 Subject: [PATCH 21/94] fix(deploy): persist root config on activate-by-id; restart the reverted origin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings from cb1kenobi on #1849. Medium — a package deploy staged with `activate: false` and activated later by `deployment_id` never persisted its root-config entry. writeComponentRootConfig is gated on `req.package`, and an activate-by-id request carries none (`harper activate` sends only project + deployment_id). The fan-out copies `package` from the same request, so peers skipped it too: the package reference and the credential references that cold reinstalls and newly-joined peers rely on were silently lost, leaving the component recorded as a plain directory. deployComponentActivateExisting now recovers `package_identifier` — and the credential REFERENCES, which the row already stores — from the staged deployment row when the request omits them, so the origin persists config and the recovered identifier rides the sub-op out to every peer. Explicit request values still win. Recovery also means the protected-core-name guard now applies to this path. Added getDeploymentRow() for the lookup: a plain point-read, deliberately not awaitDeploymentRow, which polls for a row AND its payload_blob and so would never return a deployment whose payload retention already reclaimed — exactly the row this path still needs. Low — with `restart: true`, revert_on_failure left the origin diverged. The origin restart runs before the activate gate, so by the time the gate reverts the origin's directory its workers are already on the new version, while the peers' revert op carries `restart` and does come back on the previous one. The origin now restarts after its revert, so both ends of the reconvergence match. A rolling restart arrives with `restart` normalized to false on both sides, so it needs no second restart. Tests: 4 for getDeploymentRow (including the reclaimed-payload case that distinguishes it from awaitDeploymentRow). 146 deploy tests passing. --- components/deploymentRecorder.ts | 16 ++++++ components/operations.js | 32 +++++++++++ .../components/deploymentRecorder.test.js | 53 +++++++++++++++++++ 3 files changed, 101 insertions(+) diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index 64cf9b2c0b..c33bb59384 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -455,6 +455,22 @@ export class DeploymentRecorder { } } +/** + * Point-read a deployment row by id, or undefined when the row (or the table) is absent. + * + * Distinct from `awaitDeploymentRow`, which polls for a row to arrive by replication AND to carry a + * `payload_blob`: this is for reading a row the caller already owns locally — e.g. recovering a staged + * deployment's `package_identifier` when it is activated later by `deployment_id`. A row whose payload + * has been reclaimed by retention is a valid result here, which is exactly what awaitDeploymentRow + * would refuse to return. + */ +export async function getDeploymentRow(deploymentId: string): Promise | undefined> { + if (!deploymentId) return undefined; + const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; + if (!table) return undefined; + return table.get(deploymentId); +} + /** * Best-effort terminal-status update for an existing deployment row by id, used when a later operation * finishes a deployment the current process didn't record with a live DeploymentRecorder — e.g. diff --git a/components/operations.js b/components/operations.js index c00fdc4d44..6f2b0aac80 100644 --- a/components/operations.js +++ b/components/operations.js @@ -39,6 +39,7 @@ const { server } = require('../server/Server.ts'); const { DeploymentRecorder, awaitDeploymentRow, + getDeploymentRow, markDeploymentTerminal, normalizePeerResult, pruneProjectPayloads, @@ -845,6 +846,29 @@ async function deployPhaseActivate(req) { */ async function deployComponentActivateExisting(req, credentialReferences) { const stagingId = req.deployment_id; + // An activate-by-id call carries no `package` — `harper activate` sends only project + deployment_id, + // and the docs describe this path as fetching/installing nothing — so recover the staged deployment's + // package identifier and credential references from its row. Without this, a component staged as a + // `package` deploy and activated later would never persist its root-config entry: not on the origin + // (writeComponentRootConfig is gated on `req.package`) and not on any peer either, since the fanned-out + // sub-op copies `package`/`credentials` from this same `req`. The package reference and the credential + // references that cold reinstalls and newly-joined peers depend on would be silently lost, leaving the + // component recorded as a plain directory. Explicit values on the request always win. + if (!req.package) { + const stagedRow = await getDeploymentRow(stagingId).catch((err) => { + log.warn(`Could not read deployment ${stagingId} to recover its package identifier`, err); + return undefined; + }); + if (stagedRow?.package_identifier) { + req.package = stagedRow.package_identifier; + // The row stores credential REFERENCES (tokens were never persisted), which is exactly what + // root config should carry. Only fall back to them when the caller supplied none. + if (!req.credentials?.length && Array.isArray(stagedRow.credentials) && stagedRow.credentials.length) { + req.credentials = stagedRow.credentials; + } + credentialReferences = (req.credentials ?? []).filter((entry) => entry && entry.secret !== undefined); + } + } if (req.package) assertNotProtectedCoreComponent(req.project, req.force); const emitter = req.progress ?? new ProgressEmitter(); if (!req.progress) req.progress = emitter; @@ -1261,6 +1285,14 @@ async function enforceActivatePeerGate({ req, application, emit, failed, totalPe emit('phase', { phase: 'revert', status: 'start' }); // The origin activated, so revert it. await revertApplication(application); + // With `restart: true` the origin's workers already reloaded onto the new (failed-cluster) + // version — the origin restart runs before this gate — so the directory rollback above is not + // picked up on its own. The peers' revert op carries `restart`, so they DO come back on the + // previous version; without this the origin would be the one node left serving the new version, + // the exact opposite of the reconvergence revert_on_failure exists to provide. A rolling restart + // arrives here with `restart` already normalized to false and its peers likewise un-restarted, + // so origin and peers stay consistent in that case without a second restart. + if (req.restart === true) manageThreads.restartWorkers('http'); // Revert ONLY the peers that successfully activated (see selectRevertTargets): every known // node minus the ones that failed to activate (still on the correct version) and minus this // node (already reverted directly above; a second bidirectional revert would flip it back). diff --git a/unitTests/components/deploymentRecorder.test.js b/unitTests/components/deploymentRecorder.test.js index 7d234af481..b4ce153b36 100644 --- a/unitTests/components/deploymentRecorder.test.js +++ b/unitTests/components/deploymentRecorder.test.js @@ -20,6 +20,7 @@ const { readPayloadBlobWithRetry, markDeploymentTerminal, pruneProjectPayloads, + getDeploymentRow, } = require('#src/components/deploymentRecorder'); const { databases } = require('#src/resources/databases'); const terms = require('#src/utility/hdbTerms'); @@ -686,6 +687,58 @@ describe('markDeploymentTerminal', () => { }); }); +describe('getDeploymentRow', () => { + let installed; + beforeEach(() => { + installed = installMockDeploymentTable(); + }); + afterEach(() => installed.restore()); + + it('returns the row, including its package_identifier', async () => { + // deploy_component({deployment_id}) reads this to recover the package identifier of a deployment + // that was staged as a `package` deploy — an activate-by-id request carries no `package` of its own. + installed.mock.rows.set('dep-1', { + deployment_id: 'dep-1', + project: 'my-app', + package_identifier: 'npm:@my-org/my-app@1.2.3', + status: 'staged', + }); + const row = await getDeploymentRow('dep-1'); + assert.strictEqual(row?.package_identifier, 'npm:@my-org/my-app@1.2.3'); + }); + + it('returns a row whose payload has already been reclaimed', async () => { + // The point of this helper over awaitDeploymentRow: that one polls until the row carries a + // payload_blob, so it would never return a deployment whose payload retention already dropped — + // which is exactly the row an activate-by-id still needs to read. + installed.mock.rows.set('reclaimed', { + deployment_id: 'reclaimed', + package_identifier: 'npm:pkg@1', + payload_blob: null, + status: 'success', + }); + const row = await getDeploymentRow('reclaimed'); + assert.strictEqual(row?.package_identifier, 'npm:pkg@1', 'a payload-less row is still returned'); + }); + + it('returns undefined for an unknown id or a blank id', async () => { + assert.strictEqual(await getDeploymentRow('nope'), undefined); + assert.strictEqual(await getDeploymentRow(''), undefined); + }); + + it('returns undefined when the deployment table is not provisioned', async () => { + installed.restore(); + const prior = databases.system?.[DEPLOYMENT_TABLE]; + if (databases.system) delete databases.system[DEPLOYMENT_TABLE]; + try { + assert.strictEqual(await getDeploymentRow('dep-1'), undefined, 'a missing table is not an error'); + } finally { + if (databases.system && prior !== undefined) databases.system[DEPLOYMENT_TABLE] = prior; + installed = installMockDeploymentTable(); + } + }); +}); + describe('pruneProjectPayloads (deployment_payloadRetention_maxCount)', () => { let installed; beforeEach(() => { From 764ce6777becf072f2e0780d1b8ac87c660a68a5 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Thu, 30 Jul 2026 09:41:15 -0400 Subject: [PATCH 22/94] test(deploy): cover package-identifier recovery on activate-by-id end to end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recovery branch added in 525dfd76d was only covered at the getDeploymentRow unit level; nothing drove a `package` deploy through stage → activate-by-id to prove root config actually gets persisted and that the recovered identifier reaches peers — the exact bug that commit fixed. Added that test: stages a `file:` tarball package with `activate: false` (no network, and a package deploy needs no payload blob, so a mock deployment table is enough), then activates by deployment_id with NO `package` on the request — what `harper activate` sends — and asserts both that the origin wrote the package reference to root config and that the activate sub-op carries it so peers do the same. The config file is snapshotted and restored around the test. Verified it is a real regression test: with the recovery branch disabled it fails on the sub-op assertion, and passes with it restored. A `package` deploy runs the protected-core-name guard, which requires componentLoader and so pulls in the private @harperfast/skills dependency that some local checkouts lack (which is why every other test in this file uses payload deploys). The test probes for it and skips rather than failing on a missing dependency; CI has it and runs the assertions for real. --- .../components/deployPhaseOperations.test.js | 91 ++++++++++++++++++- 1 file changed, 90 insertions(+), 1 deletion(-) diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index dd7484acd2..7da26cb21a 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -22,9 +22,12 @@ const operations = require('#src/components/operations'); const { DEPLOY_STAGING_DIR, Application, stageApplication } = require('#src/components/Application'); const { restartNeeded, resetRestartNeeded } = require('#src/components/requestRestart'); const { server } = require('#src/server/Server'); -const { getConfigPath } = require('#src/config/configUtils'); +const { databases } = require('#src/resources/databases'); +const { SYSTEM_TABLE_NAMES } = require('#src/utility/hdbTerms'); +const { getConfigPath, getConfiguration, getConfigFilePath } = require('#src/config/configUtils'); const { CONFIG_PARAMS } = require('#src/utility/hdbTerms'); +const DEPLOYMENT_TABLE = SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME; const COMPONENTS_ROOT = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); // Pack a directory's CONTENTS into a gzipped tar Buffer, the shape a deploy payload takes. @@ -55,6 +58,18 @@ async function makeComponentPayload(marker) { const readIndex = (dir) => fs.readFile(path.join(dir, 'index.js'), 'utf8'); +// componentLoader reaches the private `@harperfast/skills` dependency, which is absent from some local +// checkouts. Only the `package`-deploy path touches it (via the protected-core-name guard), so probe +// once and let that one test skip rather than fail on a missing dependency. +function componentLoaderAvailable() { + try { + require('#src/components/componentLoader'); + return true; + } catch { + return false; + } +} + describe('deploy operations: stage_component / activate_component / deploy_component', function () { this.timeout(30_000); @@ -161,6 +176,80 @@ describe('deploy operations: stage_component / activate_component / deploy_compo return staged.deployment_id; } + // End-to-end cover for the package-identifier recovery: stage a `package` deploy with + // `activate: false`, then activate it by id with no `package` on the request (what `harper activate` + // sends) and assert root config is persisted AND the recovered identifier reaches peers. A `file:` + // tarball package needs no network and no payload blob — extraction reads the tarball directly — so + // a mock deployment table is enough to drive the whole path. + it('recovers a staged package deploy: persists root config and fans the package out to peers', async function () { + // Unlike the payload deploys used elsewhere in this file, a `package` deploy runs the + // protected-core-name guard, which requires componentLoader and so pulls in the private + // `@harperfast/skills` dependency. That isn't installed in every local checkout, so skip there + // instead of failing on the environment; CI installs it and runs this for real. + if (!componentLoaderAvailable()) return this.skip(); + const name = freshName(); + const tgzDir = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-op-pkg-')); + const tgzPath = path.join(tgzDir, 'component.tgz'); + await fs.writeFile(tgzPath, await makeComponentPayload('pkg-staged')); + const packageId = `file:${tgzPath}`; + + const rows = new Map(); + const priorTable = databases.system?.[DEPLOYMENT_TABLE]; + if (!databases.system) databases.system = {}; + databases.system[DEPLOYMENT_TABLE] = { + async get(id) { + return rows.get(id); + }, + async put(row) { + rows.set(row.deployment_id, { ...row }); + }, + async patch(id, partial) { + const existing = rows.get(id); + if (existing) rows.set(id, { ...existing, ...partial }); + }, + async *search(conditions = []) { + for (const row of rows.values()) if (conditions.every((c) => row[c.attribute] === c.value)) yield row; + }, + }; + const configBackup = await fs.readFile(getConfigFilePath(), 'utf8'); + + try { + const staged = await operations.deployComponent({ project: name, package: packageId, activate: false }); + assert.strictEqual(staged.staged, true, 'the package deploy staged without going live'); + assert.strictEqual( + rows.get(staged.deployment_id)?.package_identifier, + packageId, + 'the stage recorded the package identifier on the row — the source the activate recovers from' + ); + + let fannedOut; + priorReplicate = server.replication.replicateOperation; + server.replication.replicateOperation = async (op) => { + fannedOut = op; + return {}; + }; + + // No `package` here — exactly what `harper activate project=… deployment_id=…` sends. + await operations.deployComponent({ project: name, deployment_id: staged.deployment_id }); + + assert.strictEqual( + fannedOut?.package, + packageId, + 'the recovered identifier rides the activate sub-op, so peers persist root config too' + ); + assert.strictEqual( + getConfiguration()[name]?.package, + packageId, + 'the origin persisted the package reference to root config' + ); + } finally { + await fs.writeFile(getConfigFilePath(), configBackup); + if (priorTable === undefined) delete databases.system[DEPLOYMENT_TABLE]; + else databases.system[DEPLOYMENT_TABLE] = priorTable; + await fs.rm(tgzDir, { recursive: true, force: true }); + } + }); + it('rejects (does not report success) when a peer fails to activate', async () => { const name = freshName(); const deploymentId = await stageOnly(name, 'gate-fail'); From 2b189cc4fc4fe91987beffccc64c5d18801cba75 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 3 Aug 2026 16:07:13 -0600 Subject: [PATCH 23/94] Restore component directories after failed preparation --- DESIGN.md | 11 + components/Application.ts | 455 +++++++++++++++--- components/componentLoader.ts | 23 +- components/componentPreparationLock.ts | 4 +- components/operations.js | 82 ++-- .../deploy-tracking-peer-branch.test.ts | 83 +++- .../components/extractApplicationSwap.test.js | 279 ++++++++++- .../prepareApplicationSerialization.test.js | 102 +++- 8 files changed, 927 insertions(+), 112 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index b96cbd1ff5..d9859ed614 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -152,6 +152,17 @@ Extraction renames an existing component aside before writing the replacement an dependency installation and metadata verification complete. Any preparation failure atomically renames the partial tree into hidden staging before restoring the prior tree, so a live writer cannot wedge rollback with `ENOTEMPTY`; cleanup completes while the same-component lock is still held. +The aside name is itself the recovery record: `.in-progress-*` is recoverable after an interrupted +deploy unless a sibling `.retired-*` marker records that the replacement committed. Cleanup removes +the aside before its marker, so an interrupted cleanup cannot make an obsolete tree recoverable. +Component loading recovers marked interrupted deploys before scanning the component root, and +preparation repeats recovery under the same-component lock before reading runtime metadata. A full +`drop_component` writes retirement markers before deleting the live tree and keeps its filesystem, +configuration, and replication mutations under that lock, so cleanup residue cannot resurrect a +dropped component and a concurrent deploy cannot interleave with the drop. Full-component drops +rename the live tree into staging before best-effort cleanup, avoiding an in-place recursive-delete +race with the running worker. The marker protocol guarantees process-crash recovery; it does not +claim persistence ordering across a host power loss without filesystem-level durability guarantees. A package-manager timeout must not release this lock while npm descendants are still mutating `node_modules`. POSIX spawns therefore run in a dedicated process group; timeout sends the group `SIGTERM`, escalates to `SIGKILL`, and waits for exit before rejecting. Windows uses `taskkill /T /F` for the equivalent process-tree termination. `manageThreads` tracks each spawned process tree by its owning Harper thread and force-terminates it if that worker exits, preventing detached installers from surviving a worker restart or Harper shutdown. `SIGKILL`/`taskkill` only queue termination, so a worker's dead-owner reclamation (above) waits for that thread's tracked process groups to be confirmed gone, not merely signaled—otherwise a replacement preparation could start while the old writer might still be alive. A process group a dead worker's own event loop spawned is never reaped from another thread, so it persists as a zombie rather than fully disappearing; since a zombie can no longer touch the filesystem, confirmation treats a zombie the same as a fully reaped exit. diff --git a/components/Application.ts b/components/Application.ts index a6f269a0a2..09b5e6f11c 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -3,7 +3,7 @@ import { getConfigObj, getConfigValue, getConfigPath } from '../config/configUti import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import logger, { errorForLog } from '../utility/logging/harper_logger.ts'; import { broadcastDeployStart, broadcastDeployEnd } from './deployLifecycle.ts'; -import { withComponentPreparationLock } from './componentPreparationLock.ts'; +import { ComponentPreparationLockTimeoutError, withComponentPreparationLock } from './componentPreparationLock.ts'; import { isThreadRunning, registerProcessGroup, unregisterProcessGroup } from '../server/threads/manageThreads.js'; import type { CredentialReference, ResolvedCredential, ResolvedRegistryCredential } from './secretOperations.ts'; import { @@ -18,13 +18,15 @@ import { ENV_ENCRYPTED_PREFIX } from '../utility/envFile.ts'; import { basename, dirname, extname, join } from 'node:path'; import { access, + chmod, constants, - cp, + lstat, mkdir, mkdtemp, readdir, readFile, rename, + rmdir, rm, stat, symlink, @@ -478,6 +480,8 @@ async function runNpmPack( // during a deploy swap (see extractApplication). The leading dot keeps // loadComponentDirectories from loading its contents as components. export const ASIDE_STAGING_DIR = '.deploy-aside'; +const IN_PROGRESS_ASIDE_PREFIX = '.in-progress-'; +const RETIRED_ASIDE_PREFIX = '.retired-'; const DEFAULT_COMMAND_TIMEOUT_MS = 60 * 60 * 1000; const COMPONENT_PREPARATION_WAIT_MARGIN_MS = 30000; const MAX_GIT_EXTRACTION_COMMANDS = 4; @@ -488,6 +492,8 @@ type ExtractionTransaction = { rollback(): Promise; }; +type ExtractionContext = Pick; + // The credential helper git executes for a private git-reference deploy. It ships alongside this // module (both in source and in dist), holds no secret, and is inert without a live session. export const GIT_CREDENTIAL_HELPER_PATH = join(__dirname, 'gitCredentialHelper.js'); @@ -607,7 +613,7 @@ export async function extractApplication( throw new Error('Both payload and package cannot be provided'); } // Resolve the tarball from the input - let tarballPath: string; + let tarballPath: string | undefined; let tarball: Readable; let shouldDeleteTarball = false; @@ -728,131 +734,419 @@ export async function extractApplication( // leading dot keeps loadComponentDirectories from picking it up as a phantom // component, and the per-component path means a sibling component never collides // with (or sweeps) another's aside. - const asideStagingDir = join(dirname(application.dirPath), ASIDE_STAGING_DIR, basename(application.dirPath)); + const asideStagingDir = extractionStagingDirectory(application.dirPath); + const transactionPaths = new Set(); let asidePath: string | undefined; try { - await access(application.dirPath, constants.F_OK); - await mkdir(asideStagingDir, { recursive: true }); - const candidateAsidePath = join(asideStagingDir, `${process.pid}-${Date.now()}-${randomUUID()}`); - await rename(application.dirPath, candidateAsidePath); - asidePath = candidateAsidePath; - } catch (err) { - // Ignore does not exist error - if (err.code !== 'ENOENT') { - throw err; + await ensureExtractionStagingDirectory(asideStagingDir); + await recoverOrCleanupStaleExtractionPaths(application, asideStagingDir); + let componentExists = true; + try { + await access(application.dirPath, constants.F_OK); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + componentExists = false; } - } - // A directory existed for this component name prior to this deploy, so this is a redeploy of - // an already-active component rather than a first-time deploy. See `isNewComponent` above. - if (asidePath) application.isNewComponent = false; - try { - await mkdir(application.dirPath, { recursive: true }); - await pipeline(tarball, gunzip(), extract(application.dirPath)); - - const extracted = await readdir(application.dirPath, { withFileTypes: true }); - if (extracted.length === 1 && extracted[0].isDirectory()) { - const topLevelDirPath = join(application.dirPath, extracted[0].name); - await mkdir(asideStagingDir, { recursive: true }); - const tempDirPath = await mkdtemp(join(asideStagingDir, '.normalize-')); - await cp(topLevelDirPath, tempDirPath, { recursive: true }); - await rm(topLevelDirPath, { recursive: true, force: true }); - await cp(tempDirPath, application.dirPath, { recursive: true }); - await rm(tempDirPath, { recursive: true, force: true }); + if (componentExists) { + await ensureExtractionStagingDirectory(asideStagingDir); + asidePath = join(asideStagingDir, `${IN_PROGRESS_ASIDE_PREFIX}${Date.now()}-${process.pid}-${randomUUID()}`); + await rename(application.dirPath, asidePath); + transactionPaths.add(asidePath); } - } catch (error) { + if (asidePath) application.isNewComponent = false; + try { - await rollbackExtractedDirectory(application, asideStagingDir, asidePath); - } catch (rollbackError) { - throw new AggregateError( - [error, rollbackError], - `Failed to extract ${application.name} and restore its previous component directory` + await mkdir(application.dirPath, { recursive: true }); + await pipeline(tarball, gunzip(), extract(application.dirPath)); + + const extracted = await readdir(application.dirPath, { withFileTypes: true }); + if (extracted.length === 1 && extracted[0].isDirectory()) { + const topLevelDirPath = join(application.dirPath, extracted[0].name); + await ensureExtractionStagingDirectory(asideStagingDir); + const tempDirPath = join(asideStagingDir, `.normalize-${process.pid}-${Date.now()}-${randomUUID()}`); + transactionPaths.add(tempDirPath); + await rename(topLevelDirPath, tempDirPath); + await rmdir(application.dirPath); + await rename(tempDirPath, application.dirPath); + transactionPaths.delete(tempDirPath); + } + } catch (error) { + try { + await rollbackExtractedDirectory(application, asideStagingDir, asidePath, transactionPaths, false); + } catch (rollbackError) { + throw new AggregateError( + [error, rollbackError], + `Failed to extract ${application.name}: ${errorMessage(error)}; ` + + `also failed to restore its previous component directory: ${errorMessage(rollbackError)}` + ); + } + throw error; + } + } finally { + if (shouldDeleteTarball && tarballPath) { + await rm(tarballPath, { force: true }).catch((error) => + application.logger.warn(`Failed to remove temporary package ${tarballPath}:`, error) ); } - throw error; - } - // Clean up the original tarball - if (shouldDeleteTarball && tarballPath) { - await rm(tarballPath, { force: true }).catch((error) => - application.logger.warn(`Failed to remove temporary package ${tarballPath}:`, error) - ); } - // Remove this component's aside copies. The old worker may still hold files open - // in the just-renamed copy (the live writer that motivated the rename), so this is - // best-effort: removing the whole staging subdirectory also clears leftovers from - // earlier deploys whose workers have since exited, and a copy that survives because - // its worker is still live is swept by the next deploy. The failure is expected in - // the live-worker case, so it's logged at trace rather than as a warning. let settled = false; const transaction: ExtractionTransaction = { async commit() { if (settled) return; + if (asidePath) { + const retiredMarkerPath = retiredMarkerForAside(asidePath); + try { + await writeFile(retiredMarkerPath, '', { flag: 'wx', mode: 0o600 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + transactionPaths.add(retiredMarkerPath); + } settled = true; - await cleanupExtractionStaging(application, asideStagingDir); + await cleanupExtractionPaths(application, asideStagingDir, transactionPaths); }, async rollback() { if (settled) return; + await rollbackExtractedDirectory(application, asideStagingDir, asidePath, transactionPaths, true); settled = true; - await rollbackExtractedDirectory(application, asideStagingDir, asidePath); }, }; if (deferCommit) return transaction; await transaction.commit(); } -async function cleanupExtractionStaging(application: Application, asideStagingDir: string): Promise { +function extractionStagingDirectory(componentDirPath: string): string { + return join(dirname(componentDirPath), ASIDE_STAGING_DIR, basename(componentDirPath)); +} + +function retiredMarkerForAside(asidePath: string): string { + return join( + dirname(asidePath), + `${RETIRED_ASIDE_PREFIX}${basename(asidePath).slice(IN_PROGRESS_ASIDE_PREFIX.length)}` + ); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +async function ensureExtractionStagingDirectory(asideStagingDir: string): Promise { + for (const stagingDir of [dirname(asideStagingDir), asideStagingDir]) { + await mkdir(stagingDir, { recursive: true, mode: 0o700 }); + const stagingStat = await lstat(stagingDir); + if (!stagingStat.isDirectory() || stagingStat.isSymbolicLink()) { + throw new Error(`Component deploy staging path is not a directory: ${stagingDir}`); + } + await chmod(stagingDir, 0o700); + } +} + +async function recoverOrCleanupStaleExtractionPaths( + application: ExtractionContext, + asideStagingDir: string +): Promise { + const entries = await readdir(asideStagingDir, { withFileTypes: true }); + const entryNames = new Set(entries.map((entry) => entry.name)); + const paths = new Set(entries.map((entry) => join(asideStagingDir, entry.name))); + const restorable = entries + .filter( + (entry) => + entry.isDirectory() && + entry.name.startsWith(IN_PROGRESS_ASIDE_PREFIX) && + !entryNames.has(`${RETIRED_ASIDE_PREFIX}${entry.name.slice(IN_PROGRESS_ASIDE_PREFIX.length)}`) + ) + .sort((left, right) => extractionAsideTimestamp(right.name) - extractionAsideTimestamp(left.name)); + if (restorable.length > 0) { + const restoredPath = join(asideStagingDir, restorable[0].name); + await rollbackExtractedDirectory(application, asideStagingDir, restoredPath, paths, false); + application.logger.warn( + `Recovered the previous ${application.name} component directory after an interrupted deploy` + + (restorable.length > 1 ? `; discarded ${restorable.length - 1} older recovery candidates` : '') + ); + return; + } + await cleanupExtractionPaths(application, asideStagingDir, paths); +} + +function extractionAsideTimestamp(name: string): number { + const timestampEnd = name.indexOf('-', IN_PROGRESS_ASIDE_PREFIX.length); + return Number(name.slice(IN_PROGRESS_ASIDE_PREFIX.length, timestampEnd)); +} + +export async function recoverInterruptedComponentExtractions( + componentsRootDirPath: string +): Promise> { + const stagingRoot = join(componentsRootDirPath, ASIDE_STAGING_DIR); + let entries; try { - await rm(asideStagingDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }); + const stagingStat = await lstat(stagingRoot); + if (!stagingStat.isDirectory() || stagingStat.isSymbolicLink()) { + throw new Error(`Component deploy staging path is not a directory: ${stagingRoot}`); + } + entries = await readdir(stagingRoot, { withFileTypes: true }); } catch (error) { - logger.trace?.(`Cleanup of previous ${application.name} component directory deferred: ${error.message}`); + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return new Map(); + throw error; } + const failedComponents = new Map(); + await Promise.all( + entries + .filter((entry) => entry.isDirectory()) + .map(async (entry) => { + const componentDirPath = join(componentsRootDirPath, entry.name); + try { + await withComponentPreparationLock( + componentDirPath, + async () => { + const asideStagingDir = extractionStagingDirectory(componentDirPath); + try { + await lstat(asideStagingDir); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + throw error; + } + await ensureExtractionStagingDirectory(asideStagingDir); + await recoverOrCleanupStaleExtractionPaths( + { name: entry.name, dirPath: componentDirPath, logger }, + asideStagingDir + ); + }, + { + timeoutMs: 0, + onWait: (owner) => + logger.info( + `Waiting to recover an interrupted deployment of ${entry.name}` + + (owner ? ` held by process ${owner.pid}, thread ${owner.threadId}` : '') + ), + } + ); + } catch (error) { + const recoveryError = error instanceof Error ? error : new Error(String(error)); + failedComponents.set(entry.name, recoveryError); + const deferred = recoveryError instanceof ComponentPreparationLockTimeoutError; + logger[deferred ? 'warn' : 'error']( + `${deferred ? 'Deferring' : 'Not loading'} ${entry.name} because its interrupted component deployment ` + + `${deferred ? 'is still being prepared' : 'could not be recovered'}:`, + errorForLog(recoveryError) + ); + } + }) + ); + return failedComponents; +} + +export async function retireComponentExtractionStaging( + componentDirPath: string, + componentName = basename(componentDirPath), + componentLogger: Logger = logger +): Promise { + const asideStagingDir = extractionStagingDirectory(componentDirPath); + let entries; + try { + await lstat(asideStagingDir); + await ensureExtractionStagingDirectory(asideStagingDir); + entries = await readdir(asideStagingDir, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + throw error; + } + const paths = new Set(entries.map((entry) => join(asideStagingDir, entry.name))); + for (const entry of entries) { + if (!entry.isDirectory() || !entry.name.startsWith(IN_PROGRESS_ASIDE_PREFIX)) continue; + const markerPath = retiredMarkerForAside(join(asideStagingDir, entry.name)); + try { + await writeFile(markerPath, '', { flag: 'wx', mode: 0o600 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + paths.add(markerPath); + } + await cleanupExtractionPaths( + { name: componentName, dirPath: componentDirPath, logger: componentLogger }, + asideStagingDir, + paths + ); +} + +export async function dropComponentDirectory( + componentDirPath: string, + componentName = basename(componentDirPath), + componentLogger: Logger = logger +): Promise { + await retireComponentExtractionStaging(componentDirPath, componentName, componentLogger); + const asideStagingDir = extractionStagingDirectory(componentDirPath); + await ensureExtractionStagingDirectory(asideStagingDir); + const droppedPath = join(asideStagingDir, `.dropped-${process.pid}-${Date.now()}-${randomUUID()}`); + try { + await rename(componentDirPath, droppedPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + await cleanupExtractionPaths( + { name: componentName, dirPath: componentDirPath, logger: componentLogger }, + asideStagingDir, + new Set([droppedPath]) + ); +} + +async function cleanupExtractionPaths( + application: ExtractionContext, + asideStagingDir: string, + paths: Set +): Promise { + const retiredMarkers: string[] = []; + for (const path of paths) { + if (basename(path).startsWith(RETIRED_ASIDE_PREFIX)) { + retiredMarkers.push(path); + continue; + } + try { + await rm(path, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }); + } catch (error) { + application.logger.trace?.( + `Cleanup of previous ${application.name} component directory deferred: ${errorMessage(error)}` + ); + } + } + for (const markerPath of retiredMarkers) { + const asidePath = join( + asideStagingDir, + `${IN_PROGRESS_ASIDE_PREFIX}${basename(markerPath).slice(RETIRED_ASIDE_PREFIX.length)}` + ); + try { + await access(asidePath, constants.F_OK); + continue; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') continue; + } + await rm(markerPath, { force: true }).catch((error) => + application.logger.trace?.( + `Cleanup of previous ${application.name} component directory deferred: ${errorMessage(error)}` + ) + ); + } + await rmdir(asideStagingDir).catch((error) => { + if (!['ENOENT', 'ENOTEMPTY'].includes((error as NodeJS.ErrnoException).code ?? '')) { + application.logger.trace?.( + `Cleanup of ${application.name} deploy staging directory deferred: ${errorMessage(error)}` + ); + } + }); } async function rollbackExtractedDirectory( - application: Application, + application: ExtractionContext, asideStagingDir: string, - asidePath: string | undefined + asidePath: string | undefined, + transactionPaths: Set, + retainReplacement: boolean ): Promise { - await mkdir(asideStagingDir, { recursive: true }); + await ensureExtractionStagingDirectory(asideStagingDir); const retryableRenameCodes = new Set(['EEXIST', 'ENOTEMPTY', 'EPERM', 'EACCES', 'EBUSY']); - const displaceCurrentDirectory = async () => { - const displacedPath = join(asideStagingDir, `.failed-${process.pid}-${Date.now()}-${randomUUID()}`); + const retryDeadline = Date.now() + 5000; + const displaceCurrentDirectory = async (): Promise => { let lastError: unknown; - for (let attempt = 0; attempt < 100; attempt++) { + do { + const displacedPath = join(asideStagingDir, `.failed-${process.pid}-${Date.now()}-${randomUUID()}`); try { await rename(application.dirPath, displacedPath); - return; + transactionPaths.add(displacedPath); + return displacedPath; } catch (error) { const code = (error as NodeJS.ErrnoException).code; - if (code === 'ENOENT') return; + if (code === 'ENOENT') return undefined; if (!retryableRenameCodes.has(code ?? '')) throw error; lastError = error; await delay(10); } - } + } while (Date.now() < retryDeadline); throw lastError; }; - await displaceCurrentDirectory(); if (asidePath) { - let restored = false; let restoreError: unknown; - for (let attempt = 0; attempt < 100; attempt++) { + let fallbackDisplacedPath: string | undefined; + const failRestore = async (error: unknown): Promise => { + try { + if (retainReplacement) { + if (fallbackDisplacedPath) { + await rm(application.dirPath, { + recursive: true, + force: true, + maxRetries: 3, + retryDelay: 100, + }); + await rename(fallbackDisplacedPath, application.dirPath); + transactionPaths.delete(fallbackDisplacedPath); + } + } + const disposablePaths = new Set(transactionPaths); + disposablePaths.delete(asidePath); + await cleanupExtractionPaths(application, asideStagingDir, disposablePaths); + } catch (fallbackError) { + throw new AggregateError( + [error, fallbackError], + `Failed to restore either the previous or replacement ${application.name} component directory` + ); + } + throw new Error( + `Failed to restore ${asidePath} to the live component directory ${application.dirPath}: ${errorMessage(error)}`, + { cause: error } + ); + }; + do { try { await rename(asidePath, application.dirPath); - restored = true; - break; + transactionPaths.delete(asidePath); + await cleanupExtractionPaths(application, asideStagingDir, transactionPaths); + return; } catch (error) { restoreError = error; - if (!retryableRenameCodes.has((error as NodeJS.ErrnoException).code ?? '')) break; - await displaceCurrentDirectory(); + if (!retryableRenameCodes.has((error as NodeJS.ErrnoException).code ?? '')) { + return failRestore(error); + } + let displacedPath: string | undefined; + try { + displacedPath = await displaceCurrentDirectory(); + } catch (displaceError) { + return failRestore( + new AggregateError( + [error, displaceError], + `Failed to clear the live ${application.name} component directory for rollback` + ) + ); + } + fallbackDisplacedPath ??= displacedPath; + if (process.platform !== 'win32') { + try { + await mkdir(application.dirPath, { mode: 0o000 }); + } catch (placeholderError) { + if ((placeholderError as NodeJS.ErrnoException).code !== 'EEXIST') { + return failRestore(placeholderError); + } + } + } await delay(10); } + } while (Date.now() < retryDeadline); + return failRestore(restoreError); + } + try { + await displaceCurrentDirectory(); + } catch (displaceError) { + try { + await rm(application.dirPath, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }); + await cleanupExtractionPaths(application, asideStagingDir, transactionPaths); + return; + } catch (removeError) { + throw new AggregateError( + [displaceError, removeError], + `Failed to remove the partial ${application.name} component directory after extraction failed` + ); } - if (!restored) throw restoreError; } - - await cleanupExtractionStaging(application, asideStagingDir); + await cleanupExtractionPaths(application, asideStagingDir, transactionPaths); } /** @@ -1239,6 +1533,18 @@ export async function prepareApplication(application: Application) { await withComponentPreparationLock( application.dirPath, async () => { + const asideStagingDir = extractionStagingDirectory(application.dirPath); + let recoveryPending = true; + try { + await lstat(asideStagingDir); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + recoveryPending = false; + } + if (recoveryPending) { + await ensureExtractionStagingDirectory(asideStagingDir); + await recoverOrCleanupStaleExtractionPaths(application, asideStagingDir); + } const previousPackageMetadata = await readInstalledPackageMetadata(application.dirPath); let extraction: ExtractionTransaction | undefined; try { @@ -1271,7 +1577,8 @@ export async function prepareApplication(application: Application) { } catch (rollbackError) { throw new AggregateError( [error, rollbackError], - `Failed to prepare ${application.name} and restore its previous component directory` + `Failed to prepare ${application.name}: ${errorMessage(error)}; ` + + `also failed to restore its previous component directory: ${errorMessage(rollbackError)}` ); } throw error; diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 2dd9f87770..ce1fc953be 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -51,7 +51,8 @@ import { lifecycle as componentLifecycle } from './status/index.ts'; import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts'; import { materializeGlobalSecrets, processComponentEnv } from './componentSecrets.ts'; import { PluginModule } from './PluginModule.ts'; -import { getEnvBuiltInComponents } from './Application.ts'; +import { getEnvBuiltInComponents, recoverInterruptedComponentExtractions } from './Application.ts'; +import { ComponentPreparationLockTimeoutError } from './componentPreparationLock.ts'; import { pathToFileURL } from 'node:url'; const CF_ROUTES_DIR = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); @@ -113,6 +114,16 @@ function tryRootConfigMount(appName: string): { ok: true; mount: ScopeMount | un export async function loadComponentDirectories(loadedPluginModules?: Map, loadedResources?: Resources) { if (loadedResources) resources = loadedResources; if (loadedPluginModules) loadedComponents = loadedPluginModules; + let failedRecoveries = new Map(); + try { + failedRecoveries = await recoverInterruptedComponentExtractions(CF_ROUTES_DIR); + } catch (error) { + const recoveryError = error instanceof Error ? error : new Error(String(error)); + harperLogger.warn( + 'Loading existing filesystem components without deploy recovery because staging could not be inspected:', + errorForLog(recoveryError) + ); + } // Materialize hdb_secret global-tier rows into process.env and snapshot the scoped tier before // any application loads (root components — including the Pro custody registration — have // already loaded by this point). Re-runs on each reload cycle, which is how changed/late-custody @@ -127,6 +138,16 @@ export async function loadComponentDirectories(loadedPluginModules?: Map boolean | Promise; } +export class ComponentPreparationLockTimeoutError extends Error {} + export function componentPreparationLockIdentity( componentDirPath: string, platform: NodeJS.Platform = process.platform @@ -286,7 +288,7 @@ async function acquireComponentPreparationLock( // liveness cannot be positively established. deadline = performance.now() + timeoutMs; } else { - throw new Error( + throw new ComponentPreparationLockTimeoutError( `Timed out waiting for component preparation lock for ${canonicalPath}` + ` held by process ${blocker.pid}, thread ${blocker.threadId}` ); diff --git a/components/operations.js b/components/operations.js index 634052a8b6..b8a7853236 100644 --- a/components/operations.js +++ b/components/operations.js @@ -24,8 +24,8 @@ const { HDB_ERROR_MSGS, HTTP_STATUS_CODES } = hdbErrors; const manageThreads = require('../server/threads/manageThreads.js'); const { packageDirectory } = require('../components/packageComponent.ts'); const { Resources } = require('../resources/Resources.ts'); -const { Application, prepareApplication, ASIDE_STAGING_DIR } = require('./Application.ts'); -const { COMPONENT_PREPARATION_LOCK_DIR } = require('./componentPreparationLock.ts'); +const { Application, prepareApplication, ASIDE_STAGING_DIR, dropComponentDirectory } = require('./Application.ts'); +const { COMPONENT_PREPARATION_LOCK_DIR, withComponentPreparationLock } = require('./componentPreparationLock.ts'); const { server } = require('../server/Server.ts'); const { DeploymentRecorder, @@ -36,6 +36,21 @@ const { } = require('./deploymentRecorder.ts'); const { ProgressEmitter } = require('../server/serverHelpers/progressEmitter.ts'); +const DROP_COMPONENT_LOCK_TIMEOUT_MS = 5 * 60 * 1000; + +function componentDropLockOptions(project) { + return { + timeoutMs: DROP_COMPONENT_LOCK_TIMEOUT_MS, + onWait: (owner) => + log.info( + `Waiting to drop ${project} while component preparation is in progress` + + (owner ? ` in process ${owner.pid}, thread ${owner.threadId}` : '') + ), + onReleaseError: (error) => log.error(`Failed to release the component preparation lock for ${project}:`, error), + isOwnerAlive: (owner) => owner.pid !== process.pid || manageThreads.isThreadRunning(owner.threadId), + }; +} + /** * Read the settings.js file and return the * @@ -233,14 +248,12 @@ async function addComponent(req) { } log.trace(`adding component`); - const cfDir = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT); const { project, install_command, install_timeout, install_allow_scripts } = req; const template = req.template || 'https://github.com/harperdb/application-template'; try { - const projectDir = path.join(cfDir, project); - fs.mkdirSync(projectDir, { recursive: true }); + await fs.mkdir(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), { recursive: true }); const application = new Application({ name: project, packageIdentifier: template, @@ -305,8 +318,14 @@ async function dropCustomFunctionProject(req) { try { const projectDir = path.join(cfDir, project); - fs.rmSync(projectDir, { recursive: true }); - let response = await server.replication.replicateOperation(req); + await withComponentPreparationLock( + projectDir, + async () => { + await dropComponentDirectory(projectDir, project, log); + }, + componentDropLockOptions(project) + ); + const response = await server.replication.replicateOperation(req); response.message = `Successfully deleted project: ${project}`; return response; } catch (err) { @@ -1138,29 +1157,38 @@ async function dropComponent(req) { const { project, file } = req; const projectPath = req.file ? path.join(project, file) : project; - const pathToComponent = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), projectPath); - - const componentSymlink = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'node_modules', project); - if (await fs.pathExists(componentSymlink)) { - await fs.unlink(componentSymlink); - } + const componentsRoot = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT); + const componentPath = path.join(componentsRoot, project); + const pathToComponent = path.join(componentsRoot, projectPath); + + await withComponentPreparationLock( + componentPath, + async () => { + const componentSymlink = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'node_modules', project); + if (await fs.pathExists(componentSymlink)) { + await fs.unlink(componentSymlink); + } - if (await fs.pathExists(pathToComponent)) { - await fs.remove(pathToComponent); - } + if (!file) { + await dropComponentDirectory(componentPath, project, log); + } else if (await fs.pathExists(pathToComponent)) { + await fs.remove(pathToComponent); + } - // Remove the component from the package.json file - const packageJsonPath = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'package.json'); - if (await fs.pathExists(packageJsonPath)) { - const packageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf8')); - if (packageJson?.dependencies?.[project]) { - delete packageJson.dependencies[project]; - } - await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2), 'utf8'); - } + const packageJsonPath = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'package.json'); + if (await fs.pathExists(packageJsonPath)) { + const packageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf8')); + if (packageJson?.dependencies?.[project]) { + delete packageJson.dependencies[project]; + } + await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2), 'utf8'); + } - configUtils.deleteConfigFromFile([project]); - let response = await server.replication.replicateOperation(req); + configUtils.deleteConfigFromFile([project]); + }, + componentDropLockOptions(project) + ); + const response = await server.replication.replicateOperation(req); if (req.restart === true) { manageThreads.restartWorkers('http'); response.message = `Successfully dropped: ${projectPath}, restarting Harper`; diff --git a/integrationTests/deploy/deploy-tracking-peer-branch.test.ts b/integrationTests/deploy/deploy-tracking-peer-branch.test.ts index 81fc25e8ed..501363e4ce 100644 --- a/integrationTests/deploy/deploy-tracking-peer-branch.test.ts +++ b/integrationTests/deploy/deploy-tracking-peer-branch.test.ts @@ -20,8 +20,19 @@ import { suite, test, before, after } from 'node:test'; import { ok, strictEqual } from 'node:assert'; import { join } from 'node:path'; -import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { + existsSync, + mkdtempSync, + mkdirSync, + readdirSync, + readFileSync, + rmSync, + statSync, + truncateSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; +import { randomFillSync } from 'node:crypto'; import { setTimeout as sleep } from 'node:timers/promises'; import { request } from 'node:http'; import { Readable } from 'node:stream'; @@ -31,6 +42,25 @@ import { startHarper, teardownHarper, type ContextWithHarper } from '@harperfast import { streamPackagedDirectory } from '../../dist/components/packageComponent.js'; import { buildMultipartBody } from '../../dist/bin/multipartBuilder.js'; +const PEER_PROJECT = 'peer-branch-replay-application'; + +function filesUnder(directory: string): string[] { + if (!existsSync(directory)) return []; + return readdirSync(directory, { withFileTypes: true }).flatMap((entry) => { + const entryPath = join(directory, entry.name); + return entry.isDirectory() ? filesUnder(entryPath) : [entryPath]; + }); +} + +async function waitForMarkedAside(asidePath: string): Promise { + const deadline = Date.now() + 10_000; + while (Date.now() < deadline) { + if (existsSync(asidePath) && readdirSync(asidePath).some((entry) => entry.startsWith('.in-progress-'))) return; + await sleep(5); + } + throw new Error('Timed out waiting for peer extraction to move the previous component aside'); +} + function postMultipart( url: URL, contentType: string, @@ -89,14 +119,16 @@ async function callOperation( suite('Deployment tracking — peer-side branch', (ctx: ContextWithHarper) => { let fixtureDir: string; let seedDeploymentId: string; + let seedPayloadBlobPath: string; before(async () => { - await startHarper(ctx); + await startHarper(ctx, { config: { storage: { blobReadTimeout: 1000 } }, env: {} }); fixtureDir = mkdtempSync(join(tmpdir(), 'peer-branch-fixture-')); writeFileSync(join(fixtureDir, 'config.yaml'), 'graphqlSchema:\n files: schema.graphql\nrest: true\n'); writeFileSync(join(fixtureDir, 'schema.graphql'), 'type Query { hello: String }\n'); mkdirSync(join(fixtureDir, 'web'), { recursive: true }); writeFileSync(join(fixtureDir, 'web', 'index.html'), '

Hello, Peer Branch!

'); + writeFileSync(join(fixtureDir, 'web', 'blob.bin'), randomFillSync(Buffer.alloc(8 * 1024 * 1024))); }); after(async () => { @@ -110,6 +142,7 @@ suite('Deployment tracking — peer-side branch', (ctx: ContextWithHarper) => { test('seed: an initial deploy populates an hdb_deployment row with a payload_blob', async () => { const project = 'peer-branch-seed-application'; + const existingBlobFiles = new Set(filesUnder(join(ctx.harper.dataRootDir, 'blobs', 'system'))); const multipart = buildMultipartBody( { operation: 'deploy_component', project, restart: false }, { @@ -132,6 +165,13 @@ suite('Deployment tracking — peer-side branch', (ctx: ContextWithHarper) => { strictEqual(got.status, 200); ok(got.body.payload_blob_present, 'seed row should have a payload_blob attached'); ok(got.body.payload_hash, 'seed row should have a sha256 payload_hash'); + const blobFiles = filesUnder(join(ctx.harper.dataRootDir, 'blobs', 'system')) + .filter((path) => !existingBlobFiles.has(path)) + .map((path) => ({ path, size: statSync(path).size })) + .sort((left, right) => right.size - left.size); + const [payloadBlobFile] = blobFiles; + ok(payloadBlobFile?.size > 1024 * 1024, 'expected the retained deployment payload to be file-backed'); + seedPayloadBlobPath = payloadBlobFile.path; }); // On Bun the deploy hangs after extraction when reading a Web ReadableStream from a @@ -146,10 +186,9 @@ suite('Deployment tracking — peer-side branch', (ctx: ContextWithHarper) => { // Simulate the operation shape origin produces for peers via `replicateOperation`: // `_deploymentId` set, no `payload`, no multipart. The handler should detect this is a // replicated execution and source the tarball from the row's payload_blob. - const peerProject = 'peer-branch-replay-application'; const response = await callOperation(ctx, { operation: 'deploy_component', - project: peerProject, + project: PEER_PROJECT, restart: false, _deploymentId: seedDeploymentId, }); @@ -157,7 +196,7 @@ suite('Deployment tracking — peer-side branch', (ctx: ContextWithHarper) => { // Confirm the component was actually written on disk (peer code path ran extraction // from the row's payload_blob and not from a missing req.payload). - const fetched = await fetch(`${ctx.harper.operationsAPIURL}/${peerProject}/`, { + const fetched = await fetch(`${ctx.harper.operationsAPIURL}/${PEER_PROJECT}/`, { headers: { Authorization: 'Basic ' + Buffer.from(`${ctx.harper.admin.username}:${ctx.harper.admin.password}`).toString('base64'), @@ -172,6 +211,40 @@ suite('Deployment tracking — peer-side branch', (ctx: ContextWithHarper) => { } ); + // This test intentionally corrupts the seed deployment blob and must remain the suite's final deployment test. + test( + 'peer-side branch: a payload blob failure after the swap restores the previous tree', + { skip: skipOnBun }, + async () => { + ok(seedPayloadBlobPath, 'seed payload blob path should be recorded before the failure test'); + const componentPath = join(ctx.harper.dataRootDir, 'components', PEER_PROJECT); + const asidePath = join(ctx.harper.dataRootDir, 'components', '.deploy-aside', PEER_PROJECT); + const oldOnlyPath = join(componentPath, 'old-only.txt'); + writeFileSync(oldOnlyPath, 'previous bytes\n'); + truncateSync(seedPayloadBlobPath, 128 * 1024); + + const responsePromise = callOperation(ctx, { + operation: 'deploy_component', + project: PEER_PROJECT, + restart: false, + _deploymentId: seedDeploymentId, + deployment_timeout: 5000, + }); + responsePromise.catch(() => {}); + await waitForMarkedAside(asidePath); + const response = await responsePromise; + + strictEqual(response.status, 500, `peer deploy should fail after payload truncation: ${response.rawText}`); + ok( + /blob|payload|incomplete|stall/i.test(response.rawText), + `peer failure should report payload delivery, got: ${response.rawText}` + ); + strictEqual(readFileSync(oldOnlyPath, 'utf8'), 'previous bytes\n'); + strictEqual(readFileSync(join(componentPath, 'web', 'index.html'), 'utf8'), '

Hello, Peer Branch!

'); + strictEqual(existsSync(asidePath), false, 'rollback should remove the component deploy staging directory'); + } + ); + // Note: the bogus-_deploymentId-id timeout case isn't covered here because the // awaitDeploymentRow 120s default would balloon test time. The timeout path (and the // per-deploy deployment_timeout override) is exercised by the unit tests for diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index d09c877a4f..7b7f65aa07 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -4,12 +4,22 @@ const assert = require('node:assert'); const path = require('node:path'); const fs = require('node:fs/promises'); const os = require('node:os'); +const { Readable } = require('node:stream'); const testUtils = require('../testUtils.js'); testUtils.preTestPrep(); -const { extractApplication, Application } = require('#src/components/Application'); +const { + extractApplication, + recoverInterruptedComponentExtractions, + dropComponentDirectory, + Application, +} = require('#src/components/Application'); const { packageDirectory } = require('#src/components/packageComponent'); +const { + ComponentPreparationLockTimeoutError, + withComponentPreparationLock, +} = require('#src/components/componentPreparationLock'); async function makeFixture(files) { const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-src-')); @@ -22,6 +32,273 @@ async function makeFixture(files) { } describe('extractApplication directory swap', () => { + it('restores the exact previous tree when payload extraction fails', async function () { + this.timeout(20000); + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-rollback-')); + const dirPath = path.join(componentsRoot, 'web'); + await fs.mkdir(path.join(dirPath, 'nested'), { recursive: true }); + await fs.writeFile(path.join(dirPath, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + await fs.writeFile(path.join(dirPath, '.env'), 'OLD_ONLY=true\n'); + await fs.writeFile(path.join(dirPath, 'nested', 'old-only.txt'), 'previous bytes\n'); + const sourceDir = await makeFixture({ + 'package.json': '{"name":"web","version":"2.0.0"}\n', + 'index.js': 'module.exports = () => 2;\n', + }); + const archive = await packageDirectory(sourceDir, { skip_node_modules: true }); + const extractionError = new Error('payload delivery failed'); + const payload = Readable.from( + (async function* () { + yield archive.subarray(0, Math.floor(archive.length / 2)); + throw extractionError; + })() + ); + const app = new Application({ name: 'web', payload }); + app.dirPath = dirPath; + + try { + await assert.rejects( + () => extractApplication(app), + (error) => error === extractionError + ); + assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); + assert.strictEqual(await fs.readFile(path.join(dirPath, '.env'), 'utf8'), 'OLD_ONLY=true\n'); + assert.strictEqual(await fs.readFile(path.join(dirPath, 'nested', 'old-only.txt'), 'utf8'), 'previous bytes\n'); + await assert.rejects(fs.access(path.join(componentsRoot, '.deploy-aside', 'web'))); + } finally { + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + await fs.rm(sourceDir, { recursive: true, force: true }); + } + }); + + it('removes a partial directory when a first deploy fails', async function () { + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-new-failure-')); + const dirPath = path.join(componentsRoot, 'web'); + const extractionError = new Error('payload delivery failed'); + const payload = new Readable({ + read() { + this.destroy(extractionError); + }, + }); + const app = new Application({ name: 'web', payload }); + app.dirPath = dirPath; + + try { + await assert.rejects(() => extractApplication(app), extractionError); + await assert.rejects(fs.access(dirPath)); + await assert.rejects(fs.access(path.join(componentsRoot, '.deploy-aside', 'web'))); + } finally { + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + + it('settles a deferred extraction transaction only once', async function () { + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-idempotent-')); + const dirPath = path.join(componentsRoot, 'web'); + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + const sourceDir = await makeFixture({ 'package.json': '{"name":"web","version":"2.0.0"}\n' }); + const app = new Application({ + name: 'web', + payload: await packageDirectory(sourceDir, { skip_node_modules: true }), + }); + app.dirPath = dirPath; + + try { + const extraction = await extractApplication(app, true); + await extraction.rollback(); + await extraction.rollback(); + await extraction.commit(); + assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); + } finally { + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + await fs.rm(sourceDir, { recursive: true, force: true }); + } + }); + + it('recovers the newest marked aside over a partial replacement before retrying', async function () { + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-recover-')); + const dirPath = path.join(componentsRoot, 'web'); + const olderAside = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-100-previous'); + const staleAside = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-200-previous'); + await fs.mkdir(olderAside, { recursive: true }); + await fs.writeFile(path.join(olderAside, 'package.json'), '{"name":"web","version":"0.9.0"}\n'); + await fs.mkdir(staleAside, { recursive: true }); + await fs.writeFile(path.join(staleAside, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + await fs.writeFile(path.join(staleAside, 'old-only.txt'), 'previous bytes\n'); + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'package.json'), '{"name":"web","version":"2.0.0"}\n'); + await fs.writeFile(path.join(dirPath, 'partial-only.txt'), 'incomplete bytes\n'); + const extractionError = new Error('payload delivery failed again'); + const payload = new Readable({ + read() { + this.destroy(extractionError); + }, + }); + const app = new Application({ name: 'web', payload }); + app.dirPath = dirPath; + + try { + await assert.rejects(() => extractApplication(app), extractionError); + assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); + assert.strictEqual(await fs.readFile(path.join(dirPath, 'old-only.txt'), 'utf8'), 'previous bytes\n'); + await assert.rejects(fs.access(path.join(dirPath, 'partial-only.txt'))); + await assert.rejects(fs.access(path.join(componentsRoot, '.deploy-aside', 'web'))); + } finally { + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + + it('does not resurrect a committed cleanup leftover after the component was dropped', async function () { + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-dropped-')); + const dirPath = path.join(componentsRoot, 'web'); + const stagingDir = path.join(componentsRoot, '.deploy-aside', 'web'); + const retiredAside = path.join(stagingDir, '.in-progress-123-previous'); + await fs.mkdir(retiredAside, { recursive: true }); + await fs.writeFile(path.join(retiredAside, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + await fs.writeFile(path.join(stagingDir, '.retired-123-previous'), ''); + const extractionError = new Error('new deployment failed'); + const app = new Application({ + name: 'web', + payload: new Readable({ + read() { + this.destroy(extractionError); + }, + }), + }); + app.dirPath = dirPath; + + try { + await assert.rejects(() => extractApplication(app), extractionError); + await assert.rejects(fs.access(dirPath)); + await assert.rejects(fs.access(path.join(componentsRoot, '.deploy-aside', 'web'))); + } finally { + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + + it('recovers an interrupted deploy before component loading', async function () { + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-startup-recovery-')); + const dirPath = path.join(componentsRoot, 'web'); + const asidePath = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-123-previous'); + await fs.mkdir(asidePath, { recursive: true }); + await fs.writeFile(path.join(asidePath, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + await fs.writeFile(path.join(asidePath, 'old-only.txt'), 'previous bytes\n'); + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'package.json'), '{"name":"web","version":"2.0.0"}\n'); + await fs.writeFile(path.join(dirPath, 'partial-only.txt'), 'incomplete bytes\n'); + + try { + await recoverInterruptedComponentExtractions(componentsRoot); + assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); + assert.strictEqual(await fs.readFile(path.join(dirPath, 'old-only.txt'), 'utf8'), 'previous bytes\n'); + await assert.rejects(fs.access(path.join(dirPath, 'partial-only.txt'))); + await assert.rejects(fs.access(path.join(componentsRoot, '.deploy-aside', 'web'))); + } finally { + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + + it('defers startup recovery while the component is actively being prepared', async function () { + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-startup-active-')); + const dirPath = path.join(componentsRoot, 'web'); + const asidePath = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-123-previous'); + await fs.mkdir(asidePath, { recursive: true }); + await fs.mkdir(dirPath, { recursive: true }); + let releasePreparation; + let preparationStarted; + const started = new Promise((resolve) => (preparationStarted = resolve)); + const preparation = withComponentPreparationLock(dirPath, async () => { + preparationStarted(); + await new Promise((resolve) => (releasePreparation = resolve)); + }); + + try { + await started; + const failures = await recoverInterruptedComponentExtractions(componentsRoot); + assert(failures.get('web') instanceof ComponentPreparationLockTimeoutError); + await fs.access(asidePath); + } finally { + releasePreparation?.(); + await preparation; + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + + it('retires interrupted deploy state before a component is dropped', async function () { + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-drop-retire-')); + const dirPath = path.join(componentsRoot, 'web'); + const asidePath = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-123-previous'); + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'package.json'), '{"name":"web","version":"2.0.0"}\n'); + await fs.mkdir(asidePath, { recursive: true }); + await fs.writeFile(path.join(asidePath, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + + try { + await dropComponentDirectory(dirPath); + await recoverInterruptedComponentExtractions(componentsRoot); + await assert.rejects(fs.access(dirPath)); + await assert.rejects(fs.access(path.join(componentsRoot, '.deploy-aside', 'web'))); + } finally { + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + + it('cleans only paths owned by the settled transaction', async function () { + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-owned-cleanup-')); + const dirPath = path.join(componentsRoot, 'web'); + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + const sourceDir = await makeFixture({ 'package.json': '{"name":"web","version":"2.0.0"}\n' }); + const app = new Application({ + name: 'web', + payload: await packageDirectory(sourceDir, { skip_node_modules: true }), + }); + app.dirPath = dirPath; + const unrelatedPath = path.join(componentsRoot, '.deploy-aside', 'web', 'unrelated-transaction'); + + try { + const extraction = await extractApplication(app, true); + await fs.mkdir(unrelatedPath, { recursive: true }); + await fs.writeFile(path.join(unrelatedPath, 'marker'), 'keep\n'); + await extraction.commit(); + assert.strictEqual(await fs.readFile(path.join(unrelatedPath, 'marker'), 'utf8'), 'keep\n'); + assert.strictEqual((await fs.stat(path.join(componentsRoot, '.deploy-aside'))).mode & 0o777, 0o700); + assert.strictEqual((await fs.stat(path.dirname(unrelatedPath))).mode & 0o777, 0o700); + } finally { + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + await fs.rm(sourceDir, { recursive: true, force: true }); + } + }); + + it('rejects a symlinked deploy staging directory', async function () { + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-symlink-')); + const externalDir = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-external-')); + const dirPath = path.join(componentsRoot, 'web'); + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + try { + await fs.symlink(externalDir, path.join(componentsRoot, '.deploy-aside'), 'dir'); + } catch (error) { + if (error.code === 'EPERM') { + await fs.rm(componentsRoot, { recursive: true, force: true }); + await fs.rm(externalDir, { recursive: true, force: true }); + this.skip(); + } + throw error; + } + const app = new Application({ name: 'web', payload: Buffer.from('not an archive') }); + app.dirPath = dirPath; + + try { + await assert.rejects(() => extractApplication(app), /deploy staging path is not a directory/); + assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); + assert.deepStrictEqual(await fs.readdir(externalDir), []); + } finally { + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + await fs.rm(externalDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + // Regression for the replicate-phase failure on a live Next.js peer: // ENOTEMPTY: directory not empty, rmdir '/.next' // The old worker keeps writing into .next/cache while the deploy replaces the diff --git a/unitTests/components/prepareApplicationSerialization.test.js b/unitTests/components/prepareApplicationSerialization.test.js index 4def5d2547..723b520ac2 100644 --- a/unitTests/components/prepareApplicationSerialization.test.js +++ b/unitTests/components/prepareApplicationSerialization.test.js @@ -20,14 +20,41 @@ async function makePayload(rootDir, name, version, installScript) { } describe('prepareApplication serialization', () => { - it('restores the loaded component tree when installation fails', async function () { + it('compares runtime metadata after recovering an interrupted deploy', async function () { this.timeout(10000); - const rootDir = await mkdtemp(join(tmpdir(), 'prepare-application-rollback-')); + const rootDir = await mkdtemp(join(tmpdir(), 'prepare-application-recovery-metadata-')); const componentDirPath = join(rootDir, 'shared'); + const asidePath = join(rootDir, '.deploy-aside', 'shared', '.in-progress-123-previous'); await mkdir(componentDirPath, { recursive: true }); + await writeFile(join(componentDirPath, 'package.json'), JSON.stringify({ name: 'shared', version: '2.0.0' })); + await mkdir(asidePath, { recursive: true }); + await writeFile(join(asidePath, 'package.json'), JSON.stringify({ name: 'shared', version: '1.0.0' })); + const sourceDir = await mkdtemp(join(rootDir, 'shared-2.0.0-')); + await writeFile(join(sourceDir, 'package.json'), JSON.stringify({ name: 'shared', version: '2.0.0' })); + const application = new Application({ + name: 'shared', + payload: await packageDirectory(sourceDir, { skip_node_modules: true }), + }); + application.dirPath = componentDirPath; + + try { + await prepareApplication(application); + assert.equal(application.packageMetadataChanged, true); + assert.equal(JSON.parse(await readFile(join(componentDirPath, 'package.json'), 'utf8')).version, '2.0.0'); + } finally { + await rm(rootDir, { recursive: true, force: true }); + } + }); + + it('restores the previous component when installation fails', async function () { + this.timeout(10000); + const rootDir = await mkdtemp(join(tmpdir(), 'prepare-application-rollback-')); + const componentDirPath = join(rootDir, 'shared'); + await mkdir(join(componentDirPath, 'nested'), { recursive: true }); await writeFile(join(componentDirPath, 'package.json'), JSON.stringify({ name: 'shared', version: '1.0.0' })); await writeFile(join(componentDirPath, 'index.js'), 'module.exports = 1;\n'); - + await writeFile(join(componentDirPath, '.env'), 'OLD_ONLY=true\n'); + await writeFile(join(componentDirPath, 'nested', 'old-only.txt'), 'previous bytes\n'); const failedApplication = new Application({ name: 'shared', payload: await makePayload(rootDir, 'shared', '2.0.0', 'process.exit(2);'), @@ -39,6 +66,75 @@ describe('prepareApplication serialization', () => { await assert.rejects(() => prepareApplication(failedApplication), /Failed to install dependencies/); assert.equal(JSON.parse(await readFile(join(componentDirPath, 'package.json'), 'utf8')).version, '1.0.0'); assert.equal(await readFile(join(componentDirPath, 'index.js'), 'utf8'), 'module.exports = 1;\n'); + assert.equal(await readFile(join(componentDirPath, '.env'), 'utf8'), 'OLD_ONLY=true\n'); + assert.equal(await readFile(join(componentDirPath, 'nested', 'old-only.txt'), 'utf8'), 'previous bytes\n'); + await assert.rejects(access(join(rootDir, '.deploy-aside', 'shared'))); + } finally { + await rm(rootDir, { recursive: true, force: true }); + } + }); + + it('reports both errors when installation and restoration fail', async function () { + this.timeout(10000); + const rootDir = await mkdtemp(join(tmpdir(), 'prepare-application-aggregate-')); + const componentDirPath = join(rootDir, 'shared'); + await mkdir(componentDirPath, { recursive: true }); + await writeFile(join(componentDirPath, 'package.json'), JSON.stringify({ name: 'shared', version: '1.0.0' })); + const installScript = ` + const fs = require('node:fs'); + const path = require('node:path'); + const asideDir = path.resolve('..', '.deploy-aside', 'shared'); + const asidePath = path.join(asideDir, fs.readdirSync(asideDir)[0]); + fs.renameSync(asidePath, path.resolve('..', 'moved-aside')); + process.exit(2); + `; + const application = new Application({ + name: 'shared', + payload: await makePayload(rootDir, 'shared', '2.0.0', installScript), + install: { command: 'node install.js', timeout: 5000 }, + }); + application.dirPath = componentDirPath; + + try { + await assert.rejects( + () => prepareApplication(application), + (error) => + error instanceof AggregateError && + error.errors.length === 2 && + error.message.includes('Failed to install dependencies') && + error.message.includes('failed to restore its previous component directory') && + error.message.includes('ENOENT') + ); + assert.equal(JSON.parse(await readFile(join(componentDirPath, 'package.json'), 'utf8')).version, '2.0.0'); + } finally { + await rm(rootDir, { recursive: true, force: true }); + } + }); + + it('keeps a successful replacement when transient credential cleanup fails', async function () { + this.timeout(10000); + const rootDir = await mkdtemp(join(tmpdir(), 'prepare-application-cleanup-')); + const componentDirPath = join(rootDir, 'shared'); + await mkdir(componentDirPath, { recursive: true }); + await writeFile(join(componentDirPath, 'package.json'), JSON.stringify({ name: 'shared', version: '1.0.0' })); + const application = new Application({ + name: 'shared', + payload: await makePayload(rootDir, 'shared', '2.0.0', 'process.exit(0);'), + install: { command: 'node install.js', timeout: 5000 }, + }); + application.dirPath = componentDirPath; + const cleanupError = new Error('credential cleanup failed'); + application.cleanupTransientNpmrc = async () => { + throw cleanupError; + }; + + try { + await assert.rejects( + () => prepareApplication(application), + (error) => error === cleanupError + ); + assert.equal(JSON.parse(await readFile(join(componentDirPath, 'package.json'), 'utf8')).version, '2.0.0'); + await assert.rejects(access(join(rootDir, '.deploy-aside', 'shared'))); } finally { await rm(rootDir, { recursive: true, force: true }); } From befbd166e12900876135e1fefd227d9364de7c34 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 3 Aug 2026 18:53:26 -0600 Subject: [PATCH 24/94] Handle concurrent component recovery during startup --- components/Application.ts | 26 +++++++++++---- components/componentPreparationLock.ts | 6 +++- components/operations.js | 2 ++ .../deploy-tracking-peer-branch.test.ts | 13 +++++--- .../components/extractApplicationSwap.test.js | 33 +++++++++++++++++++ 5 files changed, 68 insertions(+), 12 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 09b5e6f11c..ac70548ff1 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -484,6 +484,8 @@ const IN_PROGRESS_ASIDE_PREFIX = '.in-progress-'; const RETIRED_ASIDE_PREFIX = '.retired-'; const DEFAULT_COMMAND_TIMEOUT_MS = 60 * 60 * 1000; const COMPONENT_PREPARATION_WAIT_MARGIN_MS = 30000; +const COMPONENT_RECOVERY_WAIT_TIMEOUT_MS = 30000; +const COMPONENT_RECOVERY_LOCK_PURPOSE = 'component-recovery'; const MAX_GIT_EXTRACTION_COMMANDS = 4; const MAX_INSTALL_COMMANDS = 2; @@ -727,7 +729,8 @@ export async function extractApplication( // `maxRetries` is set, and a continuously-writing app would outlast retries // anyway.) Renaming the old directory aside is atomic and immune to the race: the // still-running worker keeps writing into the renamed inode harmlessly until it's - // replaced on restart, and the aside copy is removed best-effort below. + // replaced on restart. The aside remains the rollback/recovery record until commit + // marks it retired and cleanup removes it. // // The aside lives under a hidden, component-scoped staging directory inside the // components root: same filesystem as the source so the rename stays atomic, the @@ -913,12 +916,21 @@ export async function recoverInterruptedComponentExtractions( ); }, { - timeoutMs: 0, - onWait: (owner) => + timeoutMs: COMPONENT_RECOVERY_WAIT_TIMEOUT_MS, + purpose: COMPONENT_RECOVERY_LOCK_PURPOSE, + renewTimeoutWhileOwnerAlive: false, + onWait: (owner) => { + if (owner?.purpose !== COMPONENT_RECOVERY_LOCK_PURPOSE) { + throw new ComponentPreparationLockTimeoutError( + `Component preparation is still in progress for ${componentDirPath}` + ); + } logger.info( `Waiting to recover an interrupted deployment of ${entry.name}` + (owner ? ` held by process ${owner.pid}, thread ${owner.threadId}` : '') - ), + ); + }, + isOwnerAlive: (owner) => owner.pid !== process.pid || isThreadRunning(owner.threadId), } ); } catch (error) { @@ -1044,8 +1056,8 @@ async function rollbackExtractedDirectory( ): Promise { await ensureExtractionStagingDirectory(asideStagingDir); const retryableRenameCodes = new Set(['EEXIST', 'ENOTEMPTY', 'EPERM', 'EACCES', 'EBUSY']); - const retryDeadline = Date.now() + 5000; const displaceCurrentDirectory = async (): Promise => { + const retryDeadline = Date.now() + 5000; let lastError: unknown; do { const displacedPath = join(asideStagingDir, `.failed-${process.pid}-${Date.now()}-${randomUUID()}`); @@ -1066,6 +1078,7 @@ async function rollbackExtractedDirectory( if (asidePath) { let restoreError: unknown; + let restoreRetryDeadline: number | undefined; let fallbackDisplacedPath: string | undefined; const failRestore = async (error: unknown): Promise => { try { @@ -1118,6 +1131,7 @@ async function rollbackExtractedDirectory( ); } fallbackDisplacedPath ??= displacedPath; + restoreRetryDeadline ??= Date.now() + 5000; if (process.platform !== 'win32') { try { await mkdir(application.dirPath, { mode: 0o000 }); @@ -1129,7 +1143,7 @@ async function rollbackExtractedDirectory( } await delay(10); } - } while (Date.now() < retryDeadline); + } while (Date.now() < (restoreRetryDeadline ?? Date.now() + 5000)); return failRestore(restoreError); } try { diff --git a/components/componentPreparationLock.ts b/components/componentPreparationLock.ts index cba24323e8..cee45d58ed 100644 --- a/components/componentPreparationLock.ts +++ b/components/componentPreparationLock.ts @@ -23,6 +23,7 @@ export interface ComponentPreparationLockOwner { processInstanceId: string; token: string; ticket?: number; + purpose?: string; } export interface ComponentPreparationLockOptions { @@ -30,6 +31,8 @@ export interface ComponentPreparationLockOptions { onWait?: (owner: ComponentPreparationLockOwner | null) => void; onReleaseError?: (error: unknown) => void; isOwnerAlive?: (owner: ComponentPreparationLockOwner) => boolean | Promise; + purpose?: string; + renewTimeoutWhileOwnerAlive?: boolean; } export class ComponentPreparationLockTimeoutError extends Error {} @@ -246,6 +249,7 @@ async function acquireComponentPreparationLock( threadId, processInstanceId: COMPONENT_PREPARATION_PROCESS_INSTANCE_ID, token: randomUUID(), + purpose: options.purpose, }; const choosingPath = join(lockRoot, `${lockName}.choosing.${owner.token}.json`); let ticketPath: string | undefined; @@ -283,7 +287,7 @@ async function acquireComponentPreparationLock( options.onWait?.(blocker); } if (performance.now() >= deadline) { - if (await ownerLivenessConfirmed(blocker, options)) { + if (options.renewTimeoutWhileOwnerAlive !== false && (await ownerLivenessConfirmed(blocker, options))) { // A confirmed-live holder is allowed to finish; the deadline only bounds owners whose // liveness cannot be positively established. deadline = performance.now() + timeoutMs; diff --git a/components/operations.js b/components/operations.js index b8a7853236..dfd15bd7c5 100644 --- a/components/operations.js +++ b/components/operations.js @@ -318,6 +318,8 @@ async function dropCustomFunctionProject(req) { try { const projectDir = path.join(cfDir, project); + const stagingDir = path.join(cfDir, ASIDE_STAGING_DIR, project); + if (!(await fs.pathExists(projectDir)) && !(await fs.pathExists(stagingDir))) await fs.stat(projectDir); await withComponentPreparationLock( projectDir, async () => { diff --git a/integrationTests/deploy/deploy-tracking-peer-branch.test.ts b/integrationTests/deploy/deploy-tracking-peer-branch.test.ts index 501363e4ce..9964d05432 100644 --- a/integrationTests/deploy/deploy-tracking-peer-branch.test.ts +++ b/integrationTests/deploy/deploy-tracking-peer-branch.test.ts @@ -122,13 +122,13 @@ suite('Deployment tracking — peer-side branch', (ctx: ContextWithHarper) => { let seedPayloadBlobPath: string; before(async () => { - await startHarper(ctx, { config: { storage: { blobReadTimeout: 1000 } }, env: {} }); + await startHarper(ctx, { config: { storage: { blobReadTimeout: 2000 } }, env: {} }); fixtureDir = mkdtempSync(join(tmpdir(), 'peer-branch-fixture-')); writeFileSync(join(fixtureDir, 'config.yaml'), 'graphqlSchema:\n files: schema.graphql\nrest: true\n'); writeFileSync(join(fixtureDir, 'schema.graphql'), 'type Query { hello: String }\n'); mkdirSync(join(fixtureDir, 'web'), { recursive: true }); writeFileSync(join(fixtureDir, 'web', 'index.html'), '

Hello, Peer Branch!

'); - writeFileSync(join(fixtureDir, 'web', 'blob.bin'), randomFillSync(Buffer.alloc(8 * 1024 * 1024))); + writeFileSync(join(fixtureDir, 'web', 'blob.bin'), randomFillSync(Buffer.alloc(2 * 1024 * 1024))); }); after(async () => { @@ -230,9 +230,12 @@ suite('Deployment tracking — peer-side branch', (ctx: ContextWithHarper) => { _deploymentId: seedDeploymentId, deployment_timeout: 5000, }); - responsePromise.catch(() => {}); - await waitForMarkedAside(asidePath); - const response = await responsePromise; + const [asideResult, responseResult] = await Promise.allSettled([waitForMarkedAside(asidePath), responsePromise]); + if (responseResult.status === 'rejected') throw responseResult.reason; + if (asideResult.status === 'rejected') { + throw new Error(`${asideResult.reason}; deploy response: ${responseResult.value.rawText}`); + } + const response = responseResult.value; strictEqual(response.status, 500, `peer deploy should fail after payload truncation: ${response.rawText}`); ok( diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index 7b7f65aa07..929b5424ca 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -224,6 +224,39 @@ describe('extractApplication directory swap', () => { } }); + it('waits for another startup recovery before loading the component', async function () { + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-startup-race-')); + const dirPath = path.join(componentsRoot, 'web'); + const asidePath = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-123-previous'); + await fs.mkdir(asidePath, { recursive: true }); + await fs.writeFile(path.join(asidePath, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'package.json'), '{"name":"web","version":"2.0.0"}\n'); + let releaseRecovery; + let recoveryStarted; + const started = new Promise((resolve) => (recoveryStarted = resolve)); + const recovery = withComponentPreparationLock( + dirPath, + async () => { + recoveryStarted(); + await new Promise((resolve) => (releaseRecovery = resolve)); + }, + { purpose: 'component-recovery' } + ); + + try { + await started; + setTimeout(() => releaseRecovery(), 25); + const failures = await recoverInterruptedComponentExtractions(componentsRoot); + assert.strictEqual(failures.size, 0); + assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); + } finally { + releaseRecovery?.(); + await recovery; + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + it('retires interrupted deploy state before a component is dropped', async function () { const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-drop-retire-')); const dirPath = path.join(componentsRoot, 'web'); From 9f4918dcaf040e700ca1ed083edfa03b670f3a69 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 3 Aug 2026 19:00:42 -0600 Subject: [PATCH 25/94] Bound preparation waits without racing recovery --- components/Application.ts | 37 +++++++++++++------ components/componentLoader.ts | 9 ++++- components/componentPreparationLock.ts | 12 +++--- .../components/extractApplicationSwap.test.js | 2 +- 4 files changed, 42 insertions(+), 18 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index ac70548ff1..d3476243dc 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -878,7 +878,8 @@ function extractionAsideTimestamp(name: string): number { } export async function recoverInterruptedComponentExtractions( - componentsRootDirPath: string + componentsRootDirPath: string, + waitTimeoutMs = COMPONENT_RECOVERY_WAIT_TIMEOUT_MS ): Promise> { const stagingRoot = join(componentsRootDirPath, ASIDE_STAGING_DIR); let entries; @@ -916,17 +917,12 @@ export async function recoverInterruptedComponentExtractions( ); }, { - timeoutMs: COMPONENT_RECOVERY_WAIT_TIMEOUT_MS, + timeoutMs: waitTimeoutMs, purpose: COMPONENT_RECOVERY_LOCK_PURPOSE, - renewTimeoutWhileOwnerAlive: false, + renewTimeoutWhileOwnerAlive: (owner) => owner.purpose === COMPONENT_RECOVERY_LOCK_PURPOSE, onWait: (owner) => { - if (owner?.purpose !== COMPONENT_RECOVERY_LOCK_PURPOSE) { - throw new ComponentPreparationLockTimeoutError( - `Component preparation is still in progress for ${componentDirPath}` - ); - } logger.info( - `Waiting to recover an interrupted deployment of ${entry.name}` + + `Waiting to settle component deployment state for ${entry.name}` + (owner ? ` held by process ${owner.pid}, thread ${owner.threadId}` : '') ); }, @@ -1080,6 +1076,7 @@ async function rollbackExtractedDirectory( let restoreError: unknown; let restoreRetryDeadline: number | undefined; let fallbackDisplacedPath: string | undefined; + let placeholderIdentity: { dev: bigint; ino: bigint } | undefined; const failRestore = async (error: unknown): Promise => { try { if (retainReplacement) { @@ -1093,6 +1090,21 @@ async function rollbackExtractedDirectory( await rename(fallbackDisplacedPath, application.dirPath); transactionPaths.delete(fallbackDisplacedPath); } + } else if (placeholderIdentity) { + try { + const current = await lstat(application.dirPath, { bigint: true }); + if (current.dev === placeholderIdentity.dev && current.ino === placeholderIdentity.ino) { + await rm(application.dirPath, { + recursive: true, + force: true, + maxRetries: 3, + retryDelay: 100, + }); + placeholderIdentity = undefined; + } + } catch (placeholderError) { + if ((placeholderError as NodeJS.ErrnoException).code !== 'ENOENT') throw placeholderError; + } } const disposablePaths = new Set(transactionPaths); disposablePaths.delete(asidePath); @@ -1122,6 +1134,7 @@ async function rollbackExtractedDirectory( let displacedPath: string | undefined; try { displacedPath = await displaceCurrentDirectory(); + placeholderIdentity = undefined; } catch (displaceError) { return failRestore( new AggregateError( @@ -1132,9 +1145,11 @@ async function rollbackExtractedDirectory( } fallbackDisplacedPath ??= displacedPath; restoreRetryDeadline ??= Date.now() + 5000; - if (process.platform !== 'win32') { + if (process.platform !== 'win32' && process.getuid?.() !== 0) { try { await mkdir(application.dirPath, { mode: 0o000 }); + const placeholder = await lstat(application.dirPath, { bigint: true }); + placeholderIdentity = { dev: placeholder.dev, ino: placeholder.ino }; } catch (placeholderError) { if ((placeholderError as NodeJS.ErrnoException).code !== 'EEXIST') { return failRestore(placeholderError); @@ -1143,7 +1158,7 @@ async function rollbackExtractedDirectory( } await delay(10); } - } while (Date.now() < (restoreRetryDeadline ?? Date.now() + 5000)); + } while (restoreRetryDeadline !== undefined && Date.now() < restoreRetryDeadline); return failRestore(restoreError); } try { diff --git a/components/componentLoader.ts b/components/componentLoader.ts index ce1fc953be..0e5be0fb03 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -140,7 +140,14 @@ export async function loadComponentDirectories(loadedPluginModules?: Map void; isOwnerAlive?: (owner: ComponentPreparationLockOwner) => boolean | Promise; purpose?: string; - renewTimeoutWhileOwnerAlive?: boolean; + renewTimeoutWhileOwnerAlive?: boolean | ((owner: ComponentPreparationLockOwner) => boolean | Promise); } export class ComponentPreparationLockTimeoutError extends Error {} @@ -270,7 +270,7 @@ async function acquireComponentPreparationLock( await rm(choosingPath, { force: true }).catch(() => {}); } - let waitingReported = false; + let waitingReportedForToken: string | undefined; try { for (;;) { const claims = await scanLiveClaims(lockRoot, lockName, options, owner.token); @@ -282,12 +282,14 @@ async function acquireComponentPreparationLock( })[0]; const blocker = claims.choosing[0] ?? precedingTicket; if (!blocker) break; - if (!waitingReported) { - waitingReported = true; + if (waitingReportedForToken !== blocker.token) { + waitingReportedForToken = blocker.token; options.onWait?.(blocker); } if (performance.now() >= deadline) { - if (options.renewTimeoutWhileOwnerAlive !== false && (await ownerLivenessConfirmed(blocker, options))) { + const renewOption = options.renewTimeoutWhileOwnerAlive; + const renewForOwner = typeof renewOption === 'function' ? await renewOption(blocker) : renewOption !== false; + if (renewForOwner && (await ownerLivenessConfirmed(blocker, options))) { // A confirmed-live holder is allowed to finish; the deadline only bounds owners whose // liveness cannot be positively established. deadline = performance.now() + timeoutMs; diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index 929b5424ca..66f59c7043 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -214,7 +214,7 @@ describe('extractApplication directory swap', () => { try { await started; - const failures = await recoverInterruptedComponentExtractions(componentsRoot); + const failures = await recoverInterruptedComponentExtractions(componentsRoot, 10); assert(failures.get('web') instanceof ComponentPreparationLockTimeoutError); await fs.access(asidePath); } finally { From ee64f6af14e608b717654b2135df4f4079083ed6 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 3 Aug 2026 19:10:19 -0600 Subject: [PATCH 26/94] Wait for live component preparation during recovery --- components/Application.ts | 28 +++++++++--- components/componentPreparationLock.ts | 8 +--- .../components/extractApplicationSwap.test.js | 45 +++---------------- 3 files changed, 28 insertions(+), 53 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index d3476243dc..dd18af5003 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -485,7 +485,6 @@ const RETIRED_ASIDE_PREFIX = '.retired-'; const DEFAULT_COMMAND_TIMEOUT_MS = 60 * 60 * 1000; const COMPONENT_PREPARATION_WAIT_MARGIN_MS = 30000; const COMPONENT_RECOVERY_WAIT_TIMEOUT_MS = 30000; -const COMPONENT_RECOVERY_LOCK_PURPOSE = 'component-recovery'; const MAX_GIT_EXTRACTION_COMMANDS = 4; const MAX_INSTALL_COMMANDS = 2; @@ -878,8 +877,7 @@ function extractionAsideTimestamp(name: string): number { } export async function recoverInterruptedComponentExtractions( - componentsRootDirPath: string, - waitTimeoutMs = COMPONENT_RECOVERY_WAIT_TIMEOUT_MS + componentsRootDirPath: string ): Promise> { const stagingRoot = join(componentsRootDirPath, ASIDE_STAGING_DIR); let entries; @@ -917,9 +915,7 @@ export async function recoverInterruptedComponentExtractions( ); }, { - timeoutMs: waitTimeoutMs, - purpose: COMPONENT_RECOVERY_LOCK_PURPOSE, - renewTimeoutWhileOwnerAlive: (owner) => owner.purpose === COMPONENT_RECOVERY_LOCK_PURPOSE, + timeoutMs: COMPONENT_RECOVERY_WAIT_TIMEOUT_MS, onWait: (owner) => { logger.info( `Waiting to settle component deployment state for ${entry.name}` + @@ -1090,7 +1086,8 @@ async function rollbackExtractedDirectory( await rename(fallbackDisplacedPath, application.dirPath); transactionPaths.delete(fallbackDisplacedPath); } - } else if (placeholderIdentity) { + } + if (placeholderIdentity) { try { const current = await lstat(application.dirPath, { bigint: true }); if (current.dev === placeholderIdentity.dev && current.ino === placeholderIdentity.ino) { @@ -1132,9 +1129,26 @@ async function rollbackExtractedDirectory( return failRestore(error); } let displacedPath: string | undefined; + const displacedPlaceholderIdentity = placeholderIdentity; try { displacedPath = await displaceCurrentDirectory(); placeholderIdentity = undefined; + if (displacedPath && displacedPlaceholderIdentity) { + const displaced = await lstat(displacedPath, { bigint: true }); + if ( + displaced.dev === displacedPlaceholderIdentity.dev && + displaced.ino === displacedPlaceholderIdentity.ino + ) { + await rm(displacedPath, { + recursive: true, + force: true, + maxRetries: 3, + retryDelay: 100, + }); + transactionPaths.delete(displacedPath); + displacedPath = undefined; + } + } } catch (displaceError) { return failRestore( new AggregateError( diff --git a/components/componentPreparationLock.ts b/components/componentPreparationLock.ts index 41e6728181..5c2fa22093 100644 --- a/components/componentPreparationLock.ts +++ b/components/componentPreparationLock.ts @@ -23,7 +23,6 @@ export interface ComponentPreparationLockOwner { processInstanceId: string; token: string; ticket?: number; - purpose?: string; } export interface ComponentPreparationLockOptions { @@ -31,8 +30,6 @@ export interface ComponentPreparationLockOptions { onWait?: (owner: ComponentPreparationLockOwner | null) => void; onReleaseError?: (error: unknown) => void; isOwnerAlive?: (owner: ComponentPreparationLockOwner) => boolean | Promise; - purpose?: string; - renewTimeoutWhileOwnerAlive?: boolean | ((owner: ComponentPreparationLockOwner) => boolean | Promise); } export class ComponentPreparationLockTimeoutError extends Error {} @@ -249,7 +246,6 @@ async function acquireComponentPreparationLock( threadId, processInstanceId: COMPONENT_PREPARATION_PROCESS_INSTANCE_ID, token: randomUUID(), - purpose: options.purpose, }; const choosingPath = join(lockRoot, `${lockName}.choosing.${owner.token}.json`); let ticketPath: string | undefined; @@ -287,9 +283,7 @@ async function acquireComponentPreparationLock( options.onWait?.(blocker); } if (performance.now() >= deadline) { - const renewOption = options.renewTimeoutWhileOwnerAlive; - const renewForOwner = typeof renewOption === 'function' ? await renewOption(blocker) : renewOption !== false; - if (renewForOwner && (await ownerLivenessConfirmed(blocker, options))) { + if (await ownerLivenessConfirmed(blocker, options)) { // A confirmed-live holder is allowed to finish; the deadline only bounds owners whose // liveness cannot be positively established. deadline = performance.now() + timeoutMs; diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index 66f59c7043..30aeb1038c 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -16,10 +16,7 @@ const { Application, } = require('#src/components/Application'); const { packageDirectory } = require('#src/components/packageComponent'); -const { - ComponentPreparationLockTimeoutError, - withComponentPreparationLock, -} = require('#src/components/componentPreparationLock'); +const { withComponentPreparationLock } = require('#src/components/componentPreparationLock'); async function makeFixture(files) { const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-src-')); @@ -198,33 +195,7 @@ describe('extractApplication directory swap', () => { } }); - it('defers startup recovery while the component is actively being prepared', async function () { - const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-startup-active-')); - const dirPath = path.join(componentsRoot, 'web'); - const asidePath = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-123-previous'); - await fs.mkdir(asidePath, { recursive: true }); - await fs.mkdir(dirPath, { recursive: true }); - let releasePreparation; - let preparationStarted; - const started = new Promise((resolve) => (preparationStarted = resolve)); - const preparation = withComponentPreparationLock(dirPath, async () => { - preparationStarted(); - await new Promise((resolve) => (releasePreparation = resolve)); - }); - - try { - await started; - const failures = await recoverInterruptedComponentExtractions(componentsRoot, 10); - assert(failures.get('web') instanceof ComponentPreparationLockTimeoutError); - await fs.access(asidePath); - } finally { - releasePreparation?.(); - await preparation; - await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); - } - }); - - it('waits for another startup recovery before loading the component', async function () { + it('waits for active component preparation before recovering the component', async function () { const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-startup-race-')); const dirPath = path.join(componentsRoot, 'web'); const asidePath = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-123-previous'); @@ -235,14 +206,10 @@ describe('extractApplication directory swap', () => { let releaseRecovery; let recoveryStarted; const started = new Promise((resolve) => (recoveryStarted = resolve)); - const recovery = withComponentPreparationLock( - dirPath, - async () => { - recoveryStarted(); - await new Promise((resolve) => (releaseRecovery = resolve)); - }, - { purpose: 'component-recovery' } - ); + const recovery = withComponentPreparationLock(dirPath, async () => { + recoveryStarted(); + await new Promise((resolve) => (releaseRecovery = resolve)); + }); try { await started; From a215341c445add6b9816ba9805d4f44be98446a9 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 3 Aug 2026 19:20:40 -0600 Subject: [PATCH 27/94] Defer busy component recovery off startup path --- components/Application.ts | 65 +++++++++++-------- components/componentLoader.ts | 46 +++++++++++-- components/componentPreparationLock.ts | 3 +- .../components/extractApplicationSwap.test.js | 13 ++-- 4 files changed, 88 insertions(+), 39 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index dd18af5003..4101e3e8fe 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -896,35 +896,8 @@ export async function recoverInterruptedComponentExtractions( entries .filter((entry) => entry.isDirectory()) .map(async (entry) => { - const componentDirPath = join(componentsRootDirPath, entry.name); try { - await withComponentPreparationLock( - componentDirPath, - async () => { - const asideStagingDir = extractionStagingDirectory(componentDirPath); - try { - await lstat(asideStagingDir); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; - throw error; - } - await ensureExtractionStagingDirectory(asideStagingDir); - await recoverOrCleanupStaleExtractionPaths( - { name: entry.name, dirPath: componentDirPath, logger }, - asideStagingDir - ); - }, - { - timeoutMs: COMPONENT_RECOVERY_WAIT_TIMEOUT_MS, - onWait: (owner) => { - logger.info( - `Waiting to settle component deployment state for ${entry.name}` + - (owner ? ` held by process ${owner.pid}, thread ${owner.threadId}` : '') - ); - }, - isOwnerAlive: (owner) => owner.pid !== process.pid || isThreadRunning(owner.threadId), - } - ); + await recoverInterruptedComponentExtraction(componentsRootDirPath, entry.name, false); } catch (error) { const recoveryError = error instanceof Error ? error : new Error(String(error)); failedComponents.set(entry.name, recoveryError); @@ -940,6 +913,42 @@ export async function recoverInterruptedComponentExtractions( return failedComponents; } +export async function recoverInterruptedComponentExtraction( + componentsRootDirPath: string, + componentName: string, + waitForPreparation = true +): Promise { + const componentDirPath = join(componentsRootDirPath, componentName); + await withComponentPreparationLock( + componentDirPath, + async () => { + const asideStagingDir = extractionStagingDirectory(componentDirPath); + try { + await lstat(asideStagingDir); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + throw error; + } + await ensureExtractionStagingDirectory(asideStagingDir); + await recoverOrCleanupStaleExtractionPaths( + { name: componentName, dirPath: componentDirPath, logger }, + asideStagingDir + ); + }, + { + timeoutMs: waitForPreparation ? COMPONENT_RECOVERY_WAIT_TIMEOUT_MS : 0, + renewTimeoutWhileOwnerAlive: waitForPreparation, + onWait: (owner) => { + logger.info( + `Waiting to settle component deployment state for ${componentName}` + + (owner ? ` held by process ${owner.pid}, thread ${owner.threadId}` : '') + ); + }, + isOwnerAlive: (owner) => owner.pid !== process.pid || isThreadRunning(owner.threadId), + } + ); +} + export async function retireComponentExtractionStaging( componentDirPath: string, componentName = basename(componentDirPath), diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 0e5be0fb03..35c4239629 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -51,7 +51,11 @@ import { lifecycle as componentLifecycle } from './status/index.ts'; import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts'; import { materializeGlobalSecrets, processComponentEnv } from './componentSecrets.ts'; import { PluginModule } from './PluginModule.ts'; -import { getEnvBuiltInComponents, recoverInterruptedComponentExtractions } from './Application.ts'; +import { + getEnvBuiltInComponents, + recoverInterruptedComponentExtraction, + recoverInterruptedComponentExtractions, +} from './Application.ts'; import { ComponentPreparationLockTimeoutError } from './componentPreparationLock.ts'; import { pathToFileURL } from 'node:url'; @@ -59,6 +63,7 @@ const CF_ROUTES_DIR = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); let loadedComponents = new Map(); let watchesSetup; let resources; +let componentLoadGeneration = 0; /** * Load all the applications registered in Harper, those in the components directory as well as any directly @@ -112,6 +117,7 @@ function tryRootConfigMount(appName: string): { ok: true; mount: ScopeMount | un } export async function loadComponentDirectories(loadedPluginModules?: Map, loadedResources?: Resources) { + const loadGeneration = ++componentLoadGeneration; if (loadedResources) resources = loadedResources; if (loadedPluginModules) loadedComponents = loadedPluginModules; let failedRecoveries = new Map(); @@ -130,6 +136,34 @@ export async function loadComponentDirectories(loadedPluginModules?: Map[] = []; + const deferredRecoveries = new Map( + [...failedRecoveries].filter(([, error]) => error instanceof ComponentPreparationLockTimeoutError) + ); + const deferComponentLoad = (appName: string) => { + componentLifecycle.loading(appName, `Component '${appName}' is waiting for in-progress preparation to finish`); + void recoverInterruptedComponentExtraction(CF_ROUTES_DIR, appName) + .then(async () => { + if (loadGeneration !== componentLoadGeneration) return; + const appFolder = join(CF_ROUTES_DIR, appName); + if (!existsSync(appFolder)) return; + const mountResult = tryRootConfigMount(appName); + if (!mountResult.ok) return; + await loadComponent(appFolder, resources, HDB_ROOT_DIR_NAME, { + isRoot: false, + autoReload: false, + appName, + mount: mountResult.mount, + }); + }) + .catch((error) => { + const recoveryError = error instanceof Error ? error : new Error(String(error)); + componentLifecycle.failed( + appName, + recoveryError, + `Component '${appName}' failed to load after waiting for in-progress preparation` + ); + }); + }; if (existsSync(CF_ROUTES_DIR)) { const cfFolders = readdirSync(CF_ROUTES_DIR, { withFileTypes: true }); for (const appEntry of cfFolders) { @@ -141,11 +175,8 @@ export async function loadComponentDirectories(loadedPluginModules?: Map void; onReleaseError?: (error: unknown) => void; isOwnerAlive?: (owner: ComponentPreparationLockOwner) => boolean | Promise; + renewTimeoutWhileOwnerAlive?: boolean; } export class ComponentPreparationLockTimeoutError extends Error {} @@ -283,7 +284,7 @@ async function acquireComponentPreparationLock( options.onWait?.(blocker); } if (performance.now() >= deadline) { - if (await ownerLivenessConfirmed(blocker, options)) { + if (options.renewTimeoutWhileOwnerAlive !== false && (await ownerLivenessConfirmed(blocker, options))) { // A confirmed-live holder is allowed to finish; the deadline only bounds owners whose // liveness cannot be positively established. deadline = performance.now() + timeoutMs; diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index 30aeb1038c..dfc929b8da 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -11,12 +11,16 @@ testUtils.preTestPrep(); const { extractApplication, + recoverInterruptedComponentExtraction, recoverInterruptedComponentExtractions, dropComponentDirectory, Application, } = require('#src/components/Application'); const { packageDirectory } = require('#src/components/packageComponent'); -const { withComponentPreparationLock } = require('#src/components/componentPreparationLock'); +const { + ComponentPreparationLockTimeoutError, + withComponentPreparationLock, +} = require('#src/components/componentPreparationLock'); async function makeFixture(files) { const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-src-')); @@ -195,7 +199,7 @@ describe('extractApplication directory swap', () => { } }); - it('waits for active component preparation before recovering the component', async function () { + it('defers bulk recovery, then waits for active preparation before recovering the component', async function () { const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-startup-race-')); const dirPath = path.join(componentsRoot, 'web'); const asidePath = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-123-previous'); @@ -213,9 +217,10 @@ describe('extractApplication directory swap', () => { try { await started; - setTimeout(() => releaseRecovery(), 25); const failures = await recoverInterruptedComponentExtractions(componentsRoot); - assert.strictEqual(failures.size, 0); + assert(failures.get('web') instanceof ComponentPreparationLockTimeoutError); + setTimeout(() => releaseRecovery(), 25); + await recoverInterruptedComponentExtraction(componentsRoot, 'web'); assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); } finally { releaseRecovery?.(); From bd9c1fd1bb17d6ece10105c62cd237fb787bc163 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 3 Aug 2026 19:27:22 -0600 Subject: [PATCH 28/94] Harden deferred recovery state handling --- components/Application.ts | 33 +++++++++++-------- components/componentLoader.ts | 11 ++++--- components/componentPreparationLock.ts | 3 +- .../componentPreparationLock.test.js | 25 ++++++++++++++ 4 files changed, 53 insertions(+), 19 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 4101e3e8fe..40689a4f13 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -1143,19 +1143,26 @@ async function rollbackExtractedDirectory( displacedPath = await displaceCurrentDirectory(); placeholderIdentity = undefined; if (displacedPath && displacedPlaceholderIdentity) { - const displaced = await lstat(displacedPath, { bigint: true }); - if ( - displaced.dev === displacedPlaceholderIdentity.dev && - displaced.ino === displacedPlaceholderIdentity.ino - ) { - await rm(displacedPath, { - recursive: true, - force: true, - maxRetries: 3, - retryDelay: 100, - }); - transactionPaths.delete(displacedPath); - displacedPath = undefined; + try { + const displaced = await lstat(displacedPath, { bigint: true }); + if ( + displaced.dev === displacedPlaceholderIdentity.dev && + displaced.ino === displacedPlaceholderIdentity.ino + ) { + const displacedPlaceholderPath = displacedPath; + displacedPath = undefined; + await rm(displacedPlaceholderPath, { + recursive: true, + force: true, + maxRetries: 3, + retryDelay: 100, + }); + transactionPaths.delete(displacedPlaceholderPath); + } + } catch (placeholderCleanupError) { + application.logger.trace?.( + `Cleanup of the ${application.name} rollback placeholder deferred: ${errorMessage(placeholderCleanupError)}` + ); } } } catch (displaceError) { diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 35c4239629..34cd403eb5 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -47,7 +47,7 @@ import { ComponentV1, processResourceExtensionComponent } from './ComponentV1.ts import * as httpComponent from '../server/http.ts'; import * as mcpComponent from './mcp/index.ts'; import { Status } from '../server/status/index.ts'; -import { lifecycle as componentLifecycle } from './status/index.ts'; +import { lifecycle as componentLifecycle, statusForComponent } from './status/index.ts'; import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts'; import { materializeGlobalSecrets, processComponentEnv } from './componentSecrets.ts'; import { PluginModule } from './PluginModule.ts'; @@ -145,7 +145,10 @@ export async function loadComponentDirectories(loadedPluginModules?: Map { if (loadGeneration !== componentLoadGeneration) return; const appFolder = join(CF_ROUTES_DIR, appName); - if (!existsSync(appFolder)) return; + if (!existsSync(appFolder)) { + statusForComponent(appName).unknown('Component directory no longer exists after preparation settled'); + return; + } const mountResult = tryRootConfigMount(appName); if (!mountResult.ok) return; await loadComponent(appFolder, resources, HDB_ROOT_DIR_NAME, { @@ -199,9 +202,7 @@ export async function loadComponentDirectories(loadedPluginModules?: Map 0; let deadline = performance.now() + timeoutMs; await mkdir(lockRoot, { recursive: true, mode: 0o700 }); @@ -284,7 +285,7 @@ async function acquireComponentPreparationLock( options.onWait?.(blocker); } if (performance.now() >= deadline) { - if (options.renewTimeoutWhileOwnerAlive !== false && (await ownerLivenessConfirmed(blocker, options))) { + if (renewTimeoutWhileOwnerAlive && (await ownerLivenessConfirmed(blocker, options))) { // A confirmed-live holder is allowed to finish; the deadline only bounds owners whose // liveness cannot be positively established. deadline = performance.now() + timeoutMs; diff --git a/unitTests/components/componentPreparationLock.test.js b/unitTests/components/componentPreparationLock.test.js index 5edc689e22..c31c5d8b7a 100644 --- a/unitTests/components/componentPreparationLock.test.js +++ b/unitTests/components/componentPreparationLock.test.js @@ -264,6 +264,31 @@ describe('component preparation lock', () => { ); }); + it('treats a zero timeout as a non-blocking lock attempt', async () => { + const componentDirPath = join(rootDir, 'zero-timeout'); + let releaseHolder; + let holderStarted; + const started = new Promise((resolve) => (holderStarted = resolve)); + const holder = withComponentPreparationLock(componentDirPath, async () => { + holderStarted(); + await new Promise((resolve) => (releaseHolder = resolve)); + }); + + try { + await started; + await assert.rejects( + withComponentPreparationLock(componentDirPath, async () => {}, { + timeoutMs: 0, + isOwnerAlive: () => true, + }), + /Timed out waiting/ + ); + } finally { + releaseHolder(); + await holder; + } + }); + it('does not renew the wait deadline for a foreign-PID owner (the PID may have been recycled)', async () => { // A bare kill(pid, 0) on another process only proves *some* process holds that PID, not that // it is the original owner — after a hard crash the OS can recycle the PID to an unrelated From 0e74190733bcc8bf299f5817021c8ce00ff37cd2 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 3 Aug 2026 19:32:33 -0600 Subject: [PATCH 29/94] Finalize deferred component readiness --- components/componentLoader.ts | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 34cd403eb5..e407aeea7d 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -120,6 +120,8 @@ export async function loadComponentDirectories(loadedPluginModules?: Map(); try { failedRecoveries = await recoverInterruptedComponentExtractions(CF_ROUTES_DIR); @@ -151,12 +153,20 @@ export async function loadComponentDirectories(loadedPluginModules?: Map !modulesBeforeLoad.has(serverModule) && serverModule.ready) + .map((serverModule) => serverModule.ready()) + ); }) .catch((error) => { const recoveryError = error instanceof Error ? error : new Error(String(error)); From c6ea2806992ac2aab228d39f9f9d43bd4d1f287b Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 3 Aug 2026 19:38:03 -0600 Subject: [PATCH 30/94] Deduplicate deferred component readiness --- components/componentLoader.ts | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/components/componentLoader.ts b/components/componentLoader.ts index e407aeea7d..4ad66ce2d2 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -138,6 +138,7 @@ export async function loadComponentDirectories(loadedPluginModules?: Map[] = []; + const deferredReadyModules = new Set(); const deferredRecoveries = new Map( [...failedRecoveries].filter(([, error]) => error instanceof ComponentPreparationLockTimeoutError) ); @@ -159,14 +160,14 @@ export async function loadComponentDirectories(loadedPluginModules?: Map !modulesBeforeLoad.has(serverModule) && serverModule.ready) - .map((serverModule) => serverModule.ready()) + const modulesToReady = [...cycleLoadedComponents.keys()].filter( + (serverModule) => + !modulesBeforeLoad.has(serverModule) && !deferredReadyModules.has(serverModule) && serverModule.ready ); + for (const serverModule of modulesToReady) deferredReadyModules.add(serverModule); + await Promise.all(modulesToReady.map((serverModule) => serverModule.ready())); }) .catch((error) => { const recoveryError = error instanceof Error ? error : new Error(String(error)); From 302ba4cfd05d9af1e189829dec5a793d606aae66 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 3 Aug 2026 19:55:48 -0600 Subject: [PATCH 31/94] Gracefully settle sibling startup recovery --- components/Application.ts | 9 +++++++-- components/componentLoader.ts | 11 ++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 40689a4f13..7702ea87d0 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -485,6 +485,7 @@ const RETIRED_ASIDE_PREFIX = '.retired-'; const DEFAULT_COMMAND_TIMEOUT_MS = 60 * 60 * 1000; const COMPONENT_PREPARATION_WAIT_MARGIN_MS = 30000; const COMPONENT_RECOVERY_WAIT_TIMEOUT_MS = 30000; +const COMPONENT_RECOVERY_TRY_TIMEOUT_MS = 250; const MAX_GIT_EXTRACTION_COMMANDS = 4; const MAX_INSTALL_COMMANDS = 2; @@ -840,7 +841,11 @@ async function ensureExtractionStagingDirectory(asideStagingDir: string): Promis if (!stagingStat.isDirectory() || stagingStat.isSymbolicLink()) { throw new Error(`Component deploy staging path is not a directory: ${stagingDir}`); } - await chmod(stagingDir, 0o700); + if ((stagingStat.mode & 0o777) !== 0o700) { + await chmod(stagingDir, 0o700).catch((error) => + logger.warn(`Could not restrict component deploy staging permissions for ${stagingDir}:`, errorForLog(error)) + ); + } } } @@ -936,7 +941,7 @@ export async function recoverInterruptedComponentExtraction( ); }, { - timeoutMs: waitForPreparation ? COMPONENT_RECOVERY_WAIT_TIMEOUT_MS : 0, + timeoutMs: waitForPreparation ? COMPONENT_RECOVERY_WAIT_TIMEOUT_MS : COMPONENT_RECOVERY_TRY_TIMEOUT_MS, renewTimeoutWhileOwnerAlive: waitForPreparation, onWait: (owner) => { logger.info( diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 4ad66ce2d2..0fca3688bd 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -143,13 +143,18 @@ export async function loadComponentDirectories(loadedPluginModules?: Map error instanceof ComponentPreparationLockTimeoutError) ); const deferComponentLoad = (appName: string) => { - componentLifecycle.loading(appName, `Component '${appName}' is waiting for in-progress preparation to finish`); + const appFolder = join(CF_ROUTES_DIR, appName); + const appWasVisible = existsSync(appFolder); + if (appWasVisible) { + componentLifecycle.loading(appName, `Component '${appName}' is waiting for in-progress preparation to finish`); + } void recoverInterruptedComponentExtraction(CF_ROUTES_DIR, appName) .then(async () => { if (loadGeneration !== componentLoadGeneration) return; - const appFolder = join(CF_ROUTES_DIR, appName); if (!existsSync(appFolder)) { - statusForComponent(appName).unknown('Component directory no longer exists after preparation settled'); + if (appWasVisible) { + statusForComponent(appName).unknown('Component directory no longer exists after preparation settled'); + } return; } const mountResult = tryRootConfigMount(appName); From 1f91dfef8f1209323049a9082d6e9aad8671e81c Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 3 Aug 2026 20:00:28 -0600 Subject: [PATCH 32/94] Distinguish recovery lock ownership --- components/Application.ts | 8 +++-- components/componentLoader.ts | 12 ++++--- components/componentPreparationLock.ts | 11 +++++-- .../components/extractApplicationSwap.test.js | 33 +++++++++++++++++++ 4 files changed, 55 insertions(+), 9 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 7702ea87d0..8f86e36826 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -486,6 +486,7 @@ const DEFAULT_COMMAND_TIMEOUT_MS = 60 * 60 * 1000; const COMPONENT_PREPARATION_WAIT_MARGIN_MS = 30000; const COMPONENT_RECOVERY_WAIT_TIMEOUT_MS = 30000; const COMPONENT_RECOVERY_TRY_TIMEOUT_MS = 250; +const COMPONENT_RECOVERY_LOCK_PURPOSE = 'component-recovery'; const MAX_GIT_EXTRACTION_COMMANDS = 4; const MAX_INSTALL_COMMANDS = 2; @@ -841,7 +842,7 @@ async function ensureExtractionStagingDirectory(asideStagingDir: string): Promis if (!stagingStat.isDirectory() || stagingStat.isSymbolicLink()) { throw new Error(`Component deploy staging path is not a directory: ${stagingDir}`); } - if ((stagingStat.mode & 0o777) !== 0o700) { + if (process.platform !== 'win32' && (stagingStat.mode & 0o777) !== 0o700) { await chmod(stagingDir, 0o700).catch((error) => logger.warn(`Could not restrict component deploy staging permissions for ${stagingDir}:`, errorForLog(error)) ); @@ -942,7 +943,10 @@ export async function recoverInterruptedComponentExtraction( }, { timeoutMs: waitForPreparation ? COMPONENT_RECOVERY_WAIT_TIMEOUT_MS : COMPONENT_RECOVERY_TRY_TIMEOUT_MS, - renewTimeoutWhileOwnerAlive: waitForPreparation, + purpose: COMPONENT_RECOVERY_LOCK_PURPOSE, + renewTimeoutWhileOwnerAlive: waitForPreparation + ? true + : (owner) => owner.purpose === COMPONENT_RECOVERY_LOCK_PURPOSE, onWait: (owner) => { logger.info( `Waiting to settle component deployment state for ${componentName}` + diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 0fca3688bd..668dd6c7eb 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -176,11 +176,13 @@ export async function loadComponentDirectories(loadedPluginModules?: Map { const recoveryError = error instanceof Error ? error : new Error(String(error)); - componentLifecycle.failed( - appName, - recoveryError, - `Component '${appName}' failed to load after waiting for in-progress preparation` - ); + if (appWasVisible) { + componentLifecycle.failed( + appName, + recoveryError, + `Component '${appName}' failed to load after waiting for in-progress preparation` + ); + } }); }; if (existsSync(CF_ROUTES_DIR)) { diff --git a/components/componentPreparationLock.ts b/components/componentPreparationLock.ts index bfb50776ec..d1bfaf2880 100644 --- a/components/componentPreparationLock.ts +++ b/components/componentPreparationLock.ts @@ -23,6 +23,7 @@ export interface ComponentPreparationLockOwner { processInstanceId: string; token: string; ticket?: number; + purpose?: string; } export interface ComponentPreparationLockOptions { @@ -30,7 +31,8 @@ export interface ComponentPreparationLockOptions { onWait?: (owner: ComponentPreparationLockOwner | null) => void; onReleaseError?: (error: unknown) => void; isOwnerAlive?: (owner: ComponentPreparationLockOwner) => boolean | Promise; - renewTimeoutWhileOwnerAlive?: boolean; + purpose?: string; + renewTimeoutWhileOwnerAlive?: boolean | ((owner: ComponentPreparationLockOwner) => boolean | Promise); } export class ComponentPreparationLockTimeoutError extends Error {} @@ -247,6 +249,7 @@ async function acquireComponentPreparationLock( threadId, processInstanceId: COMPONENT_PREPARATION_PROCESS_INSTANCE_ID, token: randomUUID(), + purpose: options.purpose, }; const choosingPath = join(lockRoot, `${lockName}.choosing.${owner.token}.json`); let ticketPath: string | undefined; @@ -285,7 +288,11 @@ async function acquireComponentPreparationLock( options.onWait?.(blocker); } if (performance.now() >= deadline) { - if (renewTimeoutWhileOwnerAlive && (await ownerLivenessConfirmed(blocker, options))) { + const renewForOwner = + typeof renewTimeoutWhileOwnerAlive === 'function' + ? await renewTimeoutWhileOwnerAlive(blocker) + : renewTimeoutWhileOwnerAlive; + if (renewForOwner && (await ownerLivenessConfirmed(blocker, options))) { // A confirmed-live holder is allowed to finish; the deadline only bounds owners whose // liveness cannot be positively established. deadline = performance.now() + timeoutMs; diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index dfc929b8da..27ed64b04c 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -229,6 +229,39 @@ describe('extractApplication directory swap', () => { } }); + it('waits for a peer recovery that outlasts the bulk recovery grace period', async function () { + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-peer-recovery-')); + const dirPath = path.join(componentsRoot, 'web'); + const asidePath = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-123-previous'); + await fs.mkdir(asidePath, { recursive: true }); + await fs.writeFile(path.join(asidePath, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'package.json'), '{"name":"web","version":"2.0.0"}\n'); + let releaseRecovery; + let recoveryStarted; + const started = new Promise((resolve) => (recoveryStarted = resolve)); + const recovery = withComponentPreparationLock( + dirPath, + async () => { + recoveryStarted(); + await new Promise((resolve) => (releaseRecovery = resolve)); + }, + { purpose: 'component-recovery' } + ); + + try { + await started; + setTimeout(() => releaseRecovery(), 350); + const failures = await recoverInterruptedComponentExtractions(componentsRoot); + assert.strictEqual(failures.size, 0); + assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); + } finally { + releaseRecovery?.(); + await recovery; + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + it('retires interrupted deploy state before a component is dropped', async function () { const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-drop-retire-')); const dirPath = path.join(componentsRoot, 'web'); From b11822e203b3f97b2600f84517e5ea3f4eb6c975 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 3 Aug 2026 20:07:58 -0600 Subject: [PATCH 33/94] test(components): preserve first-deploy absence --- unitTests/server/fastifyRoutes/operations.test.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/unitTests/server/fastifyRoutes/operations.test.js b/unitTests/server/fastifyRoutes/operations.test.js index 2643fc8e53..c325ac6c40 100644 --- a/unitTests/server/fastifyRoutes/operations.test.js +++ b/unitTests/server/fastifyRoutes/operations.test.js @@ -54,12 +54,13 @@ describe('Test custom functions operations', () => { fs.removeSync(path.join(CF_DIR_ROOT, 'unit_test')); }); - it('Test addComponent creates the project folder with the correct name', async () => { + it('Test addComponent leaves the project folder creation to preparation', async () => { const response = await operations.addComponent({ project: 'unit_test' }); expect(response.message).to.equal('Successfully added project: unit_test'); expect(prepareApplicationStub.calledOnce).to.be.true; - expect(fs.existsSync(path.join(CF_DIR_ROOT, 'unit_test'))).to.be.true; + expect(fs.existsSync(CF_DIR_ROOT)).to.be.true; + expect(fs.existsSync(path.join(CF_DIR_ROOT, 'unit_test'))).to.be.false; }); it('Test getCustomFunctions returns object with proper length and content', async () => { From 4f46a43fe29cbfec5b60b1c23cf5304fe4db67ed Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 3 Aug 2026 20:23:01 -0600 Subject: [PATCH 34/94] fix(components): harden interrupted recovery discovery --- DESIGN.md | 11 ++++++----- components/Application.ts | 7 +++++-- components/componentLoader.ts | 11 +++++++++++ unitTests/components/extractApplicationSwap.test.js | 3 +++ 4 files changed, 25 insertions(+), 7 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index d9859ed614..ee80c0032a 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -155,14 +155,15 @@ wedge rollback with `ENOTEMPTY`; cleanup completes while the same-component lock The aside name is itself the recovery record: `.in-progress-*` is recoverable after an interrupted deploy unless a sibling `.retired-*` marker records that the replacement committed. Cleanup removes the aside before its marker, so an interrupted cleanup cannot make an obsolete tree recoverable. -Component loading recovers marked interrupted deploys before scanning the component root, and +Component loading recovers unretired interrupted deploys before scanning the component root, and preparation repeats recovery under the same-component lock before reading runtime metadata. A full `drop_component` writes retirement markers before deleting the live tree and keeps its filesystem, -configuration, and replication mutations under that lock, so cleanup residue cannot resurrect a -dropped component and a concurrent deploy cannot interleave with the drop. Full-component drops +and configuration mutations under that lock, so cleanup residue cannot resurrect a dropped +component and a concurrent deploy cannot interleave with the drop. Peer replication begins after +the local lock is released, and each peer serializes its own drop independently. Full-component drops rename the live tree into staging before best-effort cleanup, avoiding an in-place recursive-delete -race with the running worker. The marker protocol guarantees process-crash recovery; it does not -claim persistence ordering across a host power loss without filesystem-level durability guarantees. +race with the running worker. Recovery is durable across a process crash. It relies on rename/create +ordering rather than `fsync`, so a host power loss can lose the marker. A package-manager timeout must not release this lock while npm descendants are still mutating `node_modules`. POSIX spawns therefore run in a dedicated process group; timeout sends the group `SIGTERM`, escalates to `SIGKILL`, and waits for exit before rejecting. Windows uses `taskkill /T /F` for the equivalent process-tree termination. `manageThreads` tracks each spawned process tree by its owning Harper thread and force-terminates it if that worker exits, preventing detached installers from surviving a worker restart or Harper shutdown. `SIGKILL`/`taskkill` only queue termination, so a worker's dead-owner reclamation (above) waits for that thread's tracked process groups to be confirmed gone, not merely signaled—otherwise a replacement preparation could start while the old writer might still be alive. A process group a dead worker's own event loop spawned is never reaped from another thread, so it persists as a zombie rather than fully disappearing; since a zombie can no longer touch the filesystem, confirmation treats a zombie the same as a fully reaped exit. diff --git a/components/Application.ts b/components/Application.ts index 8f86e36826..b14b63ae88 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -864,9 +864,11 @@ async function recoverOrCleanupStaleExtractionPaths( entry.name.startsWith(IN_PROGRESS_ASIDE_PREFIX) && !entryNames.has(`${RETIRED_ASIDE_PREFIX}${entry.name.slice(IN_PROGRESS_ASIDE_PREFIX.length)}`) ) - .sort((left, right) => extractionAsideTimestamp(right.name) - extractionAsideTimestamp(left.name)); + .map((entry) => ({ entry, timestamp: extractionAsideTimestamp(entry.name) })) + .filter(({ timestamp }) => Number.isFinite(timestamp)) + .sort((left, right) => right.timestamp - left.timestamp); if (restorable.length > 0) { - const restoredPath = join(asideStagingDir, restorable[0].name); + const restoredPath = join(asideStagingDir, restorable[0].entry.name); await rollbackExtractedDirectory(application, asideStagingDir, restoredPath, paths, false); application.logger.warn( `Recovered the previous ${application.name} component directory after an interrupted deploy` + @@ -879,6 +881,7 @@ async function recoverOrCleanupStaleExtractionPaths( function extractionAsideTimestamp(name: string): number { const timestampEnd = name.indexOf('-', IN_PROGRESS_ASIDE_PREFIX.length); + if (timestampEnd < 0) return Number.NaN; return Number(name.slice(IN_PROGRESS_ASIDE_PREFIX.length, timestampEnd)); } diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 668dd6c7eb..9fa65d633f 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -142,6 +142,9 @@ export async function loadComponentDirectories(loadedPluginModules?: Map error instanceof ComponentPreparationLockTimeoutError) ); + const unreportedFailedRecoveries = new Map( + [...failedRecoveries].filter(([, error]) => !(error instanceof ComponentPreparationLockTimeoutError)) + ); const deferComponentLoad = (appName: string) => { const appFolder = join(CF_ROUTES_DIR, appName); const appWasVisible = existsSync(appFolder); @@ -200,6 +203,7 @@ export async function loadComponentDirectories(loadedPluginModules?: Map { const dirPath = path.join(componentsRoot, 'web'); const olderAside = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-100-previous'); const staleAside = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-200-previous'); + const malformedAside = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-999999'); await fs.mkdir(olderAside, { recursive: true }); await fs.writeFile(path.join(olderAside, 'package.json'), '{"name":"web","version":"0.9.0"}\n'); await fs.mkdir(staleAside, { recursive: true }); await fs.writeFile(path.join(staleAside, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); await fs.writeFile(path.join(staleAside, 'old-only.txt'), 'previous bytes\n'); + await fs.mkdir(malformedAside, { recursive: true }); + await fs.writeFile(path.join(malformedAside, 'package.json'), '{"name":"web","version":"9.9.9"}\n'); await fs.mkdir(dirPath, { recursive: true }); await fs.writeFile(path.join(dirPath, 'package.json'), '{"name":"web","version":"2.0.0"}\n'); await fs.writeFile(path.join(dirPath, 'partial-only.txt'), 'incomplete bytes\n'); From d6acffc5fe623daa2c71c9643d3470adf135d0fa Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Mon, 3 Aug 2026 20:47:11 -0600 Subject: [PATCH 35/94] fix(components): guard deferred ready hooks --- components/componentLoader.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 9fa65d633f..3600f3d35e 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -172,7 +172,9 @@ export async function loadComponentDirectories(loadedPluginModules?: Map - !modulesBeforeLoad.has(serverModule) && !deferredReadyModules.has(serverModule) && serverModule.ready + !modulesBeforeLoad.has(serverModule) && + !deferredReadyModules.has(serverModule) && + typeof serverModule.ready === 'function' ); for (const serverModule of modulesToReady) deferredReadyModules.add(serverModule); await Promise.all(modulesToReady.map((serverModule) => serverModule.ready())); From 1a6fa319115cf16c66856d8b9e6f882cc4bd7bdd Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 7 Aug 2026 10:39:07 -0600 Subject: [PATCH 36/94] Preserve retained component replacements Co-Authored-By: GPT-5 Codex --- components/Application.ts | 19 ++++++++++----- .../components/extractApplicationSwap.test.js | 24 ++++++++++++++++++- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index b14b63ae88..a751f508ef 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -787,6 +787,7 @@ export async function extractApplication( throw error; } } finally { + if (!tarball.destroyed) tarball.destroy(); if (shouldDeleteTarball && tarballPath) { await rm(tarballPath, { force: true }).catch((error) => application.logger.warn(`Failed to remove temporary package ${tarballPath}:`, error) @@ -799,12 +800,7 @@ export async function extractApplication( async commit() { if (settled) return; if (asidePath) { - const retiredMarkerPath = retiredMarkerForAside(asidePath); - try { - await writeFile(retiredMarkerPath, '', { flag: 'wx', mode: 0o600 }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; - } + const retiredMarkerPath = await retireExtractionAside(asidePath); transactionPaths.add(retiredMarkerPath); } settled = true; @@ -831,6 +827,16 @@ function retiredMarkerForAside(asidePath: string): string { ); } +async function retireExtractionAside(asidePath: string): Promise { + const retiredMarkerPath = retiredMarkerForAside(asidePath); + try { + await writeFile(retiredMarkerPath, '', { flag: 'wx', mode: 0o600 }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + return retiredMarkerPath; +} + function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } @@ -1107,6 +1113,7 @@ async function rollbackExtractedDirectory( await rename(fallbackDisplacedPath, application.dirPath); transactionPaths.delete(fallbackDisplacedPath); } + transactionPaths.add(await retireExtractionAside(asidePath)); } if (placeholderIdentity) { try { diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index e8dacc4528..663e110b65 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -180,6 +180,26 @@ describe('extractApplication directory swap', () => { } }); + it('keeps a retained replacement live when its previous tree is retired', async function () { + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-retained-replacement-')); + const dirPath = path.join(componentsRoot, 'web'); + const stagingDir = path.join(componentsRoot, '.deploy-aside', 'web'); + const retiredAside = path.join(stagingDir, '.in-progress-123-previous'); + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'package.json'), '{"name":"web","version":"2.0.0"}\n'); + await fs.mkdir(retiredAside, { recursive: true }); + await fs.writeFile(path.join(retiredAside, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + await fs.writeFile(path.join(stagingDir, '.retired-123-previous'), ''); + + try { + await recoverInterruptedComponentExtractions(componentsRoot); + assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '2.0.0'); + await assert.rejects(fs.access(path.join(componentsRoot, '.deploy-aside', 'web'))); + } finally { + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + it('recovers an interrupted deploy before component loading', async function () { const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-startup-recovery-')); const dirPath = path.join(componentsRoot, 'web'); @@ -327,11 +347,13 @@ describe('extractApplication directory swap', () => { } throw error; } - const app = new Application({ name: 'web', payload: Buffer.from('not an archive') }); + const payload = new Readable({ read() {} }); + const app = new Application({ name: 'web', payload }); app.dirPath = dirPath; try { await assert.rejects(() => extractApplication(app), /deploy staging path is not a directory/); + assert.strictEqual(payload.destroyed, true); assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); assert.deepStrictEqual(await fs.readdir(externalDir), []); } finally { From 96d90f36eb2e849ad23c209972b8bac40bc8abfa Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 7 Aug 2026 10:50:14 -0600 Subject: [PATCH 37/94] Avoid retiring an absent replacement Co-Authored-By: GPT-5 Codex --- components/Application.ts | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index a751f508ef..883122b3db 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -1102,17 +1102,15 @@ async function rollbackExtractedDirectory( let placeholderIdentity: { dev: bigint; ino: bigint } | undefined; const failRestore = async (error: unknown): Promise => { try { - if (retainReplacement) { - if (fallbackDisplacedPath) { - await rm(application.dirPath, { - recursive: true, - force: true, - maxRetries: 3, - retryDelay: 100, - }); - await rename(fallbackDisplacedPath, application.dirPath); - transactionPaths.delete(fallbackDisplacedPath); - } + if (retainReplacement && fallbackDisplacedPath) { + await rm(application.dirPath, { + recursive: true, + force: true, + maxRetries: 3, + retryDelay: 100, + }); + await rename(fallbackDisplacedPath, application.dirPath); + transactionPaths.delete(fallbackDisplacedPath); transactionPaths.add(await retireExtractionAside(asidePath)); } if (placeholderIdentity) { From 8afdfb7336b516cce20230ba98443952e56fe594 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 7 Aug 2026 10:55:39 -0600 Subject: [PATCH 38/94] Ready deferred component modules once Co-Authored-By: GPT-5 Codex --- components/componentLoader.ts | 28 +++++++++++++++------ server/loadRootComponents.js | 8 ++---- unitTests/components/componentReady.test.js | 24 ++++++++++++++++++ 3 files changed, 46 insertions(+), 14 deletions(-) create mode 100644 unitTests/components/componentReady.test.js diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 3600f3d35e..704105f35d 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -64,6 +64,24 @@ let loadedComponents = new Map(); let watchesSetup; let resources; let componentLoadGeneration = 0; +const readiedComponentModules = new WeakSet(); + +export async function readyComponentModules(serverModules: Iterable): Promise { + const modulesToReady: any[] = []; + for (const serverModule of serverModules) { + if ( + (typeof serverModule !== 'object' && typeof serverModule !== 'function') || + serverModule === null || + typeof serverModule.ready !== 'function' || + readiedComponentModules.has(serverModule) + ) { + continue; + } + readiedComponentModules.add(serverModule); + modulesToReady.push(serverModule); + } + await Promise.all(modulesToReady.map((serverModule) => serverModule.ready())); +} /** * Load all the applications registered in Harper, those in the components directory as well as any directly @@ -138,7 +156,6 @@ export async function loadComponentDirectories(loadedPluginModules?: Map[] = []; - const deferredReadyModules = new Set(); const deferredRecoveries = new Map( [...failedRecoveries].filter(([, error]) => error instanceof ComponentPreparationLockTimeoutError) ); @@ -170,14 +187,9 @@ export async function loadComponentDirectories(loadedPluginModules?: Map - !modulesBeforeLoad.has(serverModule) && - !deferredReadyModules.has(serverModule) && - typeof serverModule.ready === 'function' + await readyComponentModules( + [...cycleLoadedComponents.keys()].filter((serverModule) => !modulesBeforeLoad.has(serverModule)) ); - for (const serverModule of modulesToReady) deferredReadyModules.add(serverModule); - await Promise.all(modulesToReady.map((serverModule) => serverModule.ready())); }) .catch((error) => { const recoveryError = error instanceof Error ? error : new Error(String(error)); diff --git a/server/loadRootComponents.js b/server/loadRootComponents.js index 3b3720fc89..fbd7ee265b 100644 --- a/server/loadRootComponents.js +++ b/server/loadRootComponents.js @@ -1,6 +1,6 @@ const { isMainThread } = require('worker_threads'); const { getTables } = require('../resources/databases.ts'); -const { loadComponentDirectories, loadComponent } = require('../components/componentLoader.ts'); +const { loadComponentDirectories, loadComponent, readyComponentModules } = require('../components/componentLoader.ts'); const { resetResources } = require('../resources/Resources.ts'); const configUtils = require('../config/configUtils.ts'); const { dirname } = require('path'); @@ -35,11 +35,7 @@ async function loadRootComponents(isWorkerThread = false) { // once the global plugins are loaded, we now load all the CF and run applications (and their components) await loadComponentDirectories(loadedComponents, resources); } - let allReady = []; - for (let [serverModule] of loadedComponents) { - if (serverModule.ready) allReady.push(serverModule.ready()); - } - if (allReady.length > 0) await Promise.all(allReady); + await readyComponentModules(loadedComponents.keys()); } module.exports.loadRootComponents = loadRootComponents; diff --git a/unitTests/components/componentReady.test.js b/unitTests/components/componentReady.test.js new file mode 100644 index 0000000000..d0ac8afcf2 --- /dev/null +++ b/unitTests/components/componentReady.test.js @@ -0,0 +1,24 @@ +'use strict'; + +const assert = require('node:assert'); + +const testUtils = require('../testUtils.js'); +testUtils.preTestPrep(); + +const { readyComponentModules } = require('#src/components/componentLoader'); + +describe('component readiness', () => { + it('calls each component ready hook only once', async () => { + let calls = 0; + const component = { + ready() { + calls++; + }, + }; + + await readyComponentModules([component]); + await readyComponentModules([component]); + + assert.strictEqual(calls, 1); + }); +}); From cec013d94e950e4b0a8b75ca25fddd6dbd4c5412 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Fri, 7 Aug 2026 11:08:04 -0600 Subject: [PATCH 39/94] Await shared component readiness Co-Authored-By: GPT-5 Codex --- components/componentLoader.ts | 32 +++++++++++++------- server/loadRootComponents.js | 5 +++- unitTests/components/componentReady.test.js | 33 ++++++++++++++++++--- 3 files changed, 55 insertions(+), 15 deletions(-) diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 704105f35d..bfac78a25c 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -64,23 +64,30 @@ let loadedComponents = new Map(); let watchesSetup; let resources; let componentLoadGeneration = 0; -const readiedComponentModules = new WeakSet(); +type ComponentReadyPromises = WeakMap>; -export async function readyComponentModules(serverModules: Iterable): Promise { - const modulesToReady: any[] = []; +export async function readyComponentModules( + serverModules: Iterable, + readyComponentPromises: ComponentReadyPromises = new WeakMap() +): Promise { + const readyPromises: Promise[] = []; for (const serverModule of serverModules) { if ( (typeof serverModule !== 'object' && typeof serverModule !== 'function') || serverModule === null || - typeof serverModule.ready !== 'function' || - readiedComponentModules.has(serverModule) + typeof serverModule.ready !== 'function' ) { continue; } - readiedComponentModules.add(serverModule); - modulesToReady.push(serverModule); + let readyPromise = readyComponentPromises.get(serverModule); + if (!readyPromise) { + readyPromise = Promise.resolve().then(() => serverModule.ready()); + readyComponentPromises.set(serverModule, readyPromise); + void readyPromise.catch(() => readyComponentPromises.delete(serverModule)); + } + readyPromises.push(readyPromise); } - await Promise.all(modulesToReady.map((serverModule) => serverModule.ready())); + await Promise.all(readyPromises); } /** @@ -134,7 +141,11 @@ function tryRootConfigMount(appName: string): { ok: true; mount: ScopeMount | un } } -export async function loadComponentDirectories(loadedPluginModules?: Map, loadedResources?: Resources) { +export async function loadComponentDirectories( + loadedPluginModules?: Map, + loadedResources?: Resources, + readyComponentPromises: ComponentReadyPromises = new WeakMap() +) { const loadGeneration = ++componentLoadGeneration; if (loadedResources) resources = loadedResources; if (loadedPluginModules) loadedComponents = loadedPluginModules; @@ -188,7 +199,8 @@ export async function loadComponentDirectories(loadedPluginModules?: Map !modulesBeforeLoad.has(serverModule)) + [...cycleLoadedComponents.keys()].filter((serverModule) => !modulesBeforeLoad.has(serverModule)), + readyComponentPromises ); }) .catch((error) => { diff --git a/server/loadRootComponents.js b/server/loadRootComponents.js index fbd7ee265b..948e784cc9 100644 --- a/server/loadRootComponents.js +++ b/server/loadRootComponents.js @@ -33,7 +33,10 @@ async function loadRootComponents(isWorkerThread = false) { }); if (!process.env.HARPER_SAFE_MODE) { // once the global plugins are loaded, we now load all the CF and run applications (and their components) - await loadComponentDirectories(loadedComponents, resources); + const readyComponentPromises = new WeakMap(); + await loadComponentDirectories(loadedComponents, resources, readyComponentPromises); + await readyComponentModules(loadedComponents.keys(), readyComponentPromises); + return; } await readyComponentModules(loadedComponents.keys()); } diff --git a/unitTests/components/componentReady.test.js b/unitTests/components/componentReady.test.js index d0ac8afcf2..3dc9ec8992 100644 --- a/unitTests/components/componentReady.test.js +++ b/unitTests/components/componentReady.test.js @@ -8,17 +8,42 @@ testUtils.preTestPrep(); const { readyComponentModules } = require('#src/components/componentLoader'); describe('component readiness', () => { - it('calls each component ready hook only once', async () => { + it('shares an in-flight component ready hook', async () => { let calls = 0; + let releaseReady; + const readyStarted = new Promise((resolve) => { + releaseReady = resolve; + }); const component = { - ready() { + async ready() { calls++; + await readyStarted; }, }; + const readyComponentPromises = new WeakMap(); - await readyComponentModules([component]); - await readyComponentModules([component]); + const firstReady = readyComponentModules([component], readyComponentPromises); + const secondReady = readyComponentModules([component], readyComponentPromises); + await new Promise((resolve) => setImmediate(resolve)); + releaseReady(); + await Promise.all([firstReady, secondReady]); assert.strictEqual(calls, 1); }); + + it('retries a component ready hook after it rejects', async () => { + let calls = 0; + const component = { + ready() { + calls++; + if (calls === 1) throw new Error('not ready yet'); + }, + }; + const readyComponentPromises = new WeakMap(); + + await assert.rejects(() => readyComponentModules([component], readyComponentPromises), /not ready yet/); + await readyComponentModules([component], readyComponentPromises); + + assert.strictEqual(calls, 2); + }); }); From 58fe680aaae592508a899bc1c24a5e4fc90827e7 Mon Sep 17 00:00:00 2001 From: Nathan Heskew Date: Mon, 10 Aug 2026 13:30:16 -0700 Subject: [PATCH 40/94] Make two-phase component deploys crash-safe and fail closed --- DESIGN.md | 21 + bin/cliOperations.ts | 40 +- components/Application.ts | 488 +++++++++- components/componentLoader.ts | 40 +- components/deploymentOperations.ts | 14 +- components/deploymentRecorder.ts | 162 ++- components/operations.js | 665 ++++++++++++- components/operationsValidation.js | 39 +- config/configUtils.ts | 2 +- .../deploy/deploy-tracking-events.test.ts | 2 +- .../deploy-tracking-peer-branch.test.ts | 77 +- json/systemSchema.json | 3 + resources/registrationDeprecated.ts | 1 + .../operationAuthorizationState.ts | 11 + server/serverHelpers/serverHandlers.js | 1 - server/serverHelpers/serverUtilities.ts | 14 +- unitTests/bin/cliOperations.test.js | 136 +++ .../components/deployPhaseOperations.test.js | 919 ++++++++++++------ .../components/deployPhaseValidators.test.js | 80 +- unitTests/components/deployStaging.test.js | 541 ++++++----- .../components/deploymentOperations.test.js | 51 +- .../components/deploymentRecorder.test.js | 138 ++- .../server/fastifyRoutes/operations.test.js | 20 +- .../serverHelpers/serverUtilities.test.js | 25 +- utility/hdbTerms.ts | 13 +- utility/operation_authorization.ts | 2 +- 26 files changed, 2722 insertions(+), 783 deletions(-) create mode 100644 server/serverHelpers/operationAuthorizationState.ts diff --git a/DESIGN.md b/DESIGN.md index 0191990fb9..25f06804d9 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -133,6 +133,27 @@ A package-manager timeout must not release this lock while npm descendants are s Boot's `harper-application-lock.json` records an application configuration only after preparation fulfills. Recording at queue time would make a failed install look complete and suppress its retry on the next boot. +## Two-phase component deploys roll forward from a durable activation claim + +With `system` database replication enabled, `deploy_component` first prepares the candidate under +`.deploy-staging//` on every node. The deployment row carries the complete, +immutable activation specification (package/install settings, routing, credential references, and +`force`); activate-by-id never accepts replacements for those fields. After every stage response, the +origin durably checkpoints the row as `staged`. Activation claims that row as `activating` while holding +the same per-component filesystem lock, then swaps the candidate into the live path and commits root +config plus `harper-application-lock.json` as one compensating transaction. Peers make the same local +claim before their swap. This ordering is the recovery record: startup preserves `staged` candidates, +deletes terminal/orphan candidates, and rolls an `activating` candidate forward before loading apps. + +Peer stage/activate/restart messages use the distinct authenticated `component_deploy_phase` operation. +An older peer therefore rejects the unknown operation instead of ignoring a phase marker and deploying +the staged build live. Public `_phase`/`_deploymentId` fields are rejected; the latter remains accepted +only on the authenticated legacy one-shot replication path. Restart is gated until activation responses +have settled. A partial activation is reported as split-node state and recovered by staging and activating +a known-good build; there is deliberately no toggle-style automatic revert because retrying one after a +lost response can reverse the recovery. `deployment_stagingRetention_maxCount` bounds resting staged +trees per component and payload retention is pruned in the same row-aware lifecycle. + ## Peer-side deploy_component payload read: retryable blob stalls and `Readable.from()` cancellation `readPayloadBlobWithRetry` (`components/deploymentRecorder.ts`) wraps the peer's read of a replicated `hdb_deployment` row's `payload_blob` so a transient 503 `BlobReadError` (`BLOB_UNAVAILABLE_STATUS`, `resources/blob.ts`) — content bytes not arriving within `blobReadTimeout`, e.g. a parked blob send on the origin — retries instead of failing the whole deploy. Two non-obvious constraints shaped the design: diff --git a/bin/cliOperations.ts b/bin/cliOperations.ts index 796eca2776..627155ebee 100644 --- a/bin/cliOperations.ts +++ b/bin/cliOperations.ts @@ -153,6 +153,22 @@ async function targetSupportsStreamingDeploy(options: any): Promise { } } +async function targetSupportsStagedDeploy(options: any): Promise { + try { + const probeOptions = { + ...options, + headers: { ...options.headers, Accept: 'application/json' }, + timeout: CLI_OPERATION_TIMEOUT_MS, + }; + delete probeOptions.streamResponse; + const response = await httpRequest(probeOptions, { operation: 'registration_info' }); + if (response.statusCode !== 200 || !response.body) return false; + return JSON.parse(response.body)?.capabilities?.componentDeployTwoPhase === 1; + } catch { + return false; + } +} + // Wraps the local packaging stream so an fs error while tar'ing up the payload (e.g. a file // vanishing after the pre-deploy scan, or a permissions failure reading the project tree) // surfaces as a descriptive packaging error instead of a raw fs error code. Without this, an @@ -283,7 +299,11 @@ export { cliOperations, buildRequest, redactCredentials, refreshExpiredOperation // version. Nothing to package when a `package` identifier is given (the server fetches it) or when // activating a previously-staged deployment (`deployment_id`, i.e. `harper activate`). const packageCwdForUpload = async (req) => { - if (req.package || req.deployment_id) { + if (req.deployment_id) { + req.project ||= path.basename(process.cwd()); + return; + } + if (req.package) { return; } @@ -467,8 +487,6 @@ async function cliOperations(req: any, skipResponseLog = false) { console.error(verbError); process.exit(1); } - delete req._verb; // CLI-internal marker; never send it in the request body - await PREPARE_OPERATION[req.operation]?.(req); try { let options = target ?? { protocol: 'http:', @@ -506,6 +524,20 @@ async function cliOperations(req: any, skipResponseLog = false) { if (target && !options.headers.Authorization && req.username && req.password) { options.headers.Authorization = basicAuthHeader(req.username, req.password); } + const requestsStagedDeploy = + req._verb !== undefined || + req._cliVerb !== undefined || + req.activate === false || + req.deployment_id !== undefined || + req.two_phase === true; + if (target && requestsStagedDeploy && !(await targetSupportsStagedDeploy(options))) { + throw new Error( + `Target Harper does not advertise staged-deploy support; refusing the request because an older server could deploy it live` + ); + } + delete req._verb; + delete req._cliVerb; + await PREPARE_OPERATION[req.operation]?.(req); // Streaming deploy (multipart upload + SSE progress) only works against >= 5.1 servers. // When deploying to a remote target, probe its version first and downgrade to the // legacy JSON deploy if it predates 5.1. Local (domain-socket) deploys always @@ -540,7 +572,7 @@ async function cliOperations(req: any, skipResponseLog = false) { // One renderer owns the (future) upload bar and the SSE event rendering for a // multipart deploy. Created here so the upload-stream tap and the SSE consumer // below share the same instance. - const renderer = req._multipart ? new DeployRenderer({ uploadTotal: req._uploadSizeEstimate ?? 0 }) : null; + const renderer = useSse ? new DeployRenderer({ uploadTotal: req._uploadSizeEstimate ?? 0 }) : null; let body; if (req._multipart) { // Create the package stream here — after the renderer exists — so we can pass diff --git a/components/Application.ts b/components/Application.ts index 1256f2b989..ca69acdf06 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -1,5 +1,12 @@ import { type Logger } from '../utility/logging/logger.ts'; -import { getConfigObj, getConfigValue, getConfigPath } from '../config/configUtils.ts'; +import { + addConfig, + deleteConfigFromFile, + getConfigObj, + getConfigValue, + getConfigPath, + readConfigFile, +} from '../config/configUtils.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; import logger, { errorForLog } from '../utility/logging/harper_logger.ts'; import { broadcastDeployStart, broadcastDeployEnd } from './deployLifecycle.ts'; @@ -501,6 +508,11 @@ const MAX_INSTALL_COMMANDS = 2; // (those are rooted at each live component dir), so building here triggers no // restart-on-change storm and needs no deploy:start watcher suppression. export const DEPLOY_STAGING_DIR = '.deploy-staging'; +export const DEPLOY_ACTIVATION_DIR = '.deploy-activating'; +const STAGED_COMPLETE_MARKER = '.complete'; +const ACTIVATION_BACKUP_PREFIX = '.previous-'; +const ACTIVATION_NEW_PREFIX = '.new-'; +const DEPLOYMENT_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; // Hidden directory under the components root that RETAINS the immediately-previous live version of a // component (`.deploy-previous/`) after an activate swap, so it can be swapped back by @@ -1353,7 +1365,7 @@ export async function prepareApplication(application: Application) { * @param application The application to stage. * @returns The absolute path of the staging directory the incoming version was built into. */ -export async function stageApplication(application: Application): Promise { +async function _legacyStageApplication(application: Application): Promise { application.useStagingBuildDir(); // Start from a clean slate so a retried stage (same deployment id) can't inherit a half-built // tree from a previous attempt. @@ -1521,7 +1533,7 @@ export async function revertApplication(application: Application): Promise * staging tree and tears down any transient credential state. The live component directory is never * touched. Safe to call whether or not staging ever ran. */ -export async function discardStagedApplication(application: Application): Promise { +async function _legacyDiscardStagedApplication(application: Application): Promise { try { await application.cleanupGitCredentialSession(); } catch { @@ -1538,6 +1550,387 @@ export async function discardStagedApplication(application: Application): Promis ); } +export function stagedApplicationPath(componentDirPath: string, deploymentId: string): string { + if (!DEPLOYMENT_ID_PATTERN.test(deploymentId)) throw new Error(`Invalid deployment id '${deploymentId}'`); + return join(dirname(componentDirPath), DEPLOY_STAGING_DIR, deploymentId, basename(componentDirPath)); +} + +async function ensureSecureDirectory(directory: string, create: boolean, description: string): Promise { + let directoryStat; + try { + directoryStat = await lstat(directory); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + if (!create) return false; + await mkdir(directory, { recursive: true, mode: 0o700 }); + directoryStat = await lstat(directory); + } + if (!directoryStat.isDirectory() || directoryStat.isSymbolicLink()) { + throw new Error(`${description} is not a directory: ${directory}`); + } + return true; +} + +async function secureStagingDeploymentDirectory( + componentDirPath: string, + deploymentId: string, + create: boolean +): Promise { + const stagingDirPath = stagedApplicationPath(componentDirPath, deploymentId); + const deploymentDirPath = dirname(stagingDirPath); + const stagingRoot = dirname(deploymentDirPath); + if (!(await ensureSecureDirectory(stagingRoot, create, 'Component deploy staging path'))) return undefined; + if (!(await ensureSecureDirectory(deploymentDirPath, create, 'Component deploy staging path'))) return undefined; + return deploymentDirPath; +} + +/** Build and install a candidate under .deploy-staging without mutating the live component tree. */ +export async function stageApplication(application: Application, deploymentId: string): Promise { + const liveDirPath = application.dirPath; + const stagingDirPath = stagedApplicationPath(liveDirPath, deploymentId); + application.stagingId = deploymentId; + application.useStagingBuildDir(); + try { + await withComponentPreparationLock(liveDirPath, async () => { + const deploymentDirPath = await secureStagingDeploymentDirectory(liveDirPath, deploymentId, true); + await rm(stagingDirPath, { recursive: true, force: true }); + await rm(join(deploymentDirPath!, STAGED_COMPLETE_MARKER), { force: true }); + try { + await application.writeTransientNpmrc(); + try { + await application.startGitCredentialSession(); + await extractApplication(application); + } finally { + await application.cleanupGitCredentialSession(); + } + await installApplication(application); + await writeFile(join(deploymentDirPath!, STAGED_COMPLETE_MARKER), '', { flag: 'wx', mode: 0o600 }); + } catch (error) { + await rm(stagingDirPath, { recursive: true, force: true }).catch(() => {}); + await rm(join(deploymentDirPath!, STAGED_COMPLETE_MARKER), { force: true }).catch(() => {}); + throw error; + } finally { + await application.cleanupTransientNpmrc(); + } + }); + } finally { + application.useLiveBuildDir(); + } + await pruneStagedBuilds(application.name, deploymentId, getStagingRetentionMaxCount()); + return stagingDirPath; +} + +function activationStagingDirectory(componentDirPath: string): string { + return join(dirname(componentDirPath), DEPLOY_ACTIVATION_DIR, basename(componentDirPath)); +} + +async function activationArtifacts(componentDirPath: string, deploymentId: string): Promise { + const directory = activationStagingDirectory(componentDirPath); + let entries; + try { + entries = await readdir(directory, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw error; + } + return entries + .filter( + (entry) => + entry.name.startsWith(`${ACTIVATION_BACKUP_PREFIX}${deploymentId}`) || + entry.name === `${ACTIVATION_NEW_PREFIX}${deploymentId}` + ) + .map((entry) => join(directory, entry.name)); +} + +export async function hasCompleteStagedApplication(stagingDirPath: string): Promise { + const deploymentDirPath = dirname(stagingDirPath); + const [stagingRootStat, deploymentStat, stagedStat, stagedTargetStat, markerStat] = await Promise.all([ + lstat(dirname(deploymentDirPath)).catch(() => undefined), + lstat(deploymentDirPath).catch(() => undefined), + lstat(stagingDirPath).catch(() => undefined), + stat(stagingDirPath).catch(() => undefined), + lstat(join(deploymentDirPath, STAGED_COMPLETE_MARKER)).catch(() => undefined), + ]); + return ( + !!stagingRootStat?.isDirectory() && + !stagingRootStat.isSymbolicLink() && + !!deploymentStat?.isDirectory() && + !deploymentStat.isSymbolicLink() && + !!stagedStat && + (stagedStat.isDirectory() || stagedStat.isSymbolicLink()) && + !!stagedTargetStat?.isDirectory() && + !!markerStat?.isFile() && + !markerStat.isSymbolicLink() + ); +} + +/** Atomically replace the live component and compensate if persistent activation work fails. */ +export async function activateStagedApplication( + application: Application, + deploymentId: string, + hooks: { + beforeSwap?: () => Promise; + beforeCommit?: () => Promise; + onRollback?: () => Promise; + } = {} +): Promise { + const stagingDirPath = stagedApplicationPath(application.dirPath, deploymentId); + await withComponentPreparationLock(application.dirPath, async () => { + await secureStagingDeploymentDirectory(application.dirPath, deploymentId, false); + if (!(await hasCompleteStagedApplication(stagingDirPath))) { + const stagedStat = await lstat(stagingDirPath).catch(() => undefined); + if (!stagedStat) { + throw new Error(`Cannot activate ${application.name}: deployment '${deploymentId}' has no staged build`); + } + throw new Error(`Cannot activate ${application.name}: staged build is incomplete`); + } + + const activationDir = activationStagingDirectory(application.dirPath); + await ensureSecureDirectory(dirname(activationDir), true, 'Component activation staging path'); + await ensureSecureDirectory(activationDir, true, 'Component activation staging path'); + await broadcastDeployStart(application.name); + let backupPath: string | undefined; + let newMarkerPath: string | undefined; + let swapped = false; + try { + await hooks.beforeSwap?.(); + const existingArtifacts = await activationArtifacts(application.dirPath, deploymentId); + backupPath = existingArtifacts.find((candidate) => basename(candidate).startsWith(ACTIVATION_BACKUP_PREFIX)); + newMarkerPath = existingArtifacts.find((candidate) => basename(candidate).startsWith(ACTIVATION_NEW_PREFIX)); + if (!backupPath && !newMarkerPath) { + try { + await lstat(application.dirPath); + application.isNewComponent = false; + backupPath = join( + activationDir, + `${ACTIVATION_BACKUP_PREFIX}${deploymentId}-${Date.now()}-${process.pid}-${randomUUID()}` + ); + await rename(application.dirPath, backupPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + application.isNewComponent = true; + newMarkerPath = join(activationDir, `${ACTIVATION_NEW_PREFIX}${deploymentId}`); + try { + await writeFile(newMarkerPath, '', { flag: 'wx', mode: 0o600 }); + } catch (markerError) { + if ((markerError as NodeJS.ErrnoException).code !== 'EEXIST') throw markerError; + const markerStat = await lstat(newMarkerPath); + if (!markerStat.isFile() || markerStat.isSymbolicLink()) { + throw new Error(`Component activation marker is not a regular file: ${newMarkerPath}`); + } + } + } + } + await rename(stagingDirPath, application.dirPath); + swapped = true; + await hooks.beforeCommit?.(); + } catch (error) { + const rollbackErrors: unknown[] = []; + if (swapped) { + try { + await rename(application.dirPath, stagingDirPath); + } catch (rollbackError) { + rollbackErrors.push(rollbackError); + } + } + if (backupPath) { + try { + await rename(backupPath, application.dirPath); + } catch (rollbackError) { + rollbackErrors.push(rollbackError); + } + } + if (newMarkerPath) + await rm(newMarkerPath, { force: true }).catch((rollbackError) => rollbackErrors.push(rollbackError)); + try { + await hooks.onRollback?.(); + } catch (rollbackError) { + rollbackErrors.push(rollbackError); + } + if (rollbackErrors.length) { + throw new AggregateError( + [error, ...rollbackErrors], + `Failed to activate and fully restore ${application.name}` + ); + } + throw error; + } finally { + broadcastDeployEnd(application.name); + } + if (backupPath) await rm(backupPath, { recursive: true, force: true }); + if (newMarkerPath) await rm(newMarkerPath, { force: true }); + await rmdir(activationDir).catch((error) => { + if (!['ENOENT', 'ENOTEMPTY'].includes((error as NodeJS.ErrnoException).code ?? '')) throw error; + }); + }); + await rm(dirname(stagingDirPath), { recursive: true, force: true }).catch((error) => + logger.warn(`Failed to remove committed deploy staging for ${application.name}:`, errorForLog(error)) + ); +} + +export async function discardStagedApplication(componentDirPath: string, deploymentId: string): Promise { + const stagingDirPath = stagedApplicationPath(componentDirPath, deploymentId); + await withComponentPreparationLock(componentDirPath, async () => { + if (!(await secureStagingDeploymentDirectory(componentDirPath, deploymentId, false))) return; + await rm(dirname(stagingDirPath), { recursive: true, force: true }); + }); +} + +export async function discardProjectStagedApplications(componentDirPath: string): Promise { + const stagingRoot = join(dirname(componentDirPath), DEPLOY_STAGING_DIR); + if (!(await ensureSecureDirectory(stagingRoot, false, 'Component deploy staging path'))) return; + for (const entry of await readdir(stagingRoot, { withFileTypes: true })) { + if (!entry.isDirectory() || !DEPLOYMENT_ID_PATTERN.test(entry.name)) continue; + const deploymentDirPath = join(stagingRoot, entry.name); + if (existsSync(join(deploymentDirPath, basename(componentDirPath)))) { + await rm(deploymentDirPath, { recursive: true, force: true }); + } + } +} + +export async function discardProjectActivationArtifacts(componentDirPath: string): Promise { + const activationRoot = join(dirname(componentDirPath), DEPLOY_ACTIVATION_DIR); + if (!(await ensureSecureDirectory(activationRoot, false, 'Component activation staging path'))) return; + await rm(activationStagingDirectory(componentDirPath), { recursive: true, force: true }); + await rmdir(activationRoot).catch((error) => { + if (!['ENOENT', 'ENOTEMPTY'].includes((error as NodeJS.ErrnoException).code ?? '')) throw error; + }); +} + +type DeploymentLookup = (deploymentId: string) => Promise | undefined>; + +function safeComponentName(name: unknown): name is string { + return typeof name === 'string' && /^[a-zA-Z0-9_-]+$/.test(name); +} + +function activationArtifactDeploymentId(name: string): string | undefined { + const prefix = name.startsWith(ACTIVATION_BACKUP_PREFIX) + ? ACTIVATION_BACKUP_PREFIX + : name.startsWith(ACTIVATION_NEW_PREFIX) + ? ACTIVATION_NEW_PREFIX + : undefined; + if (!prefix) return undefined; + const deploymentId = name.slice(prefix.length, prefix.length + 36); + return DEPLOYMENT_ID_PATTERN.test(deploymentId) ? deploymentId : undefined; +} + +async function removeActivationArtifacts(componentDirPath: string, deploymentId: string): Promise { + for (const artifact of await activationArtifacts(componentDirPath, deploymentId)) { + await rm(artifact, { recursive: true, force: true }); + } + await rmdir(activationStagingDirectory(componentDirPath)).catch(() => {}); +} + +export async function reconcileStagedApplicationArtifacts( + componentsRootDirPath: string, + getDeployment: DeploymentLookup, + persistActivation: (row: Record) => Promise +): Promise<{ recovered: string[]; removed: string[]; errors: Map }> { + const recovered = new Set(); + const removed: string[] = []; + const errors = new Map(); + const stagingRoot = join(componentsRootDirPath, DEPLOY_STAGING_DIR); + let deploymentEntries = []; + try { + if (await ensureSecureDirectory(stagingRoot, false, 'Component deploy staging path')) { + deploymentEntries = await readdir(stagingRoot, { withFileTypes: true }); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + + for (const entry of deploymentEntries) { + const deploymentPath = join(stagingRoot, entry.name); + if (!entry.isDirectory() || !DEPLOYMENT_ID_PATTERN.test(entry.name)) { + await rm(deploymentPath, { recursive: true, force: true }); + removed.push(entry.name); + continue; + } + try { + let row = await getDeployment(entry.name); + if (!row || !safeComponentName(row.project)) { + await rm(deploymentPath, { recursive: true, force: true }); + removed.push(entry.name); + continue; + } + const componentDirPath = join(componentsRootDirPath, row.project); + if (!['staged', 'activating'].includes(row.status)) { + let shouldRemove = false; + await withComponentPreparationLock(componentDirPath, async () => { + row = await getDeployment(entry.name); + shouldRemove = !row || !safeComponentName(row.project) || !['staged', 'activating'].includes(row.status); + if (shouldRemove) await rm(deploymentPath, { recursive: true, force: true }); + }); + if (shouldRemove) { + removed.push(entry.name); + continue; + } + } + const stagedPath = stagedApplicationPath(componentDirPath, entry.name); + if (row.status === 'staged') { + if (!(await hasCompleteStagedApplication(stagedPath))) { + throw new Error(`Staged deployment '${entry.name}' has no valid component tree for '${row.project}'`); + } + continue; + } + if (await hasCompleteStagedApplication(stagedPath)) { + await activateStagedApplication(new Application({ name: row.project }), entry.name, { + beforeCommit: () => persistActivation(row), + }); + } else { + const liveStat = await lstat(componentDirPath).catch(() => undefined); + if (!liveStat?.isDirectory() || liveStat.isSymbolicLink()) { + throw new Error(`Interrupted activation '${entry.name}' has neither a staged nor live component tree`); + } + await persistActivation(row); + } + await removeActivationArtifacts(componentDirPath, entry.name); + recovered.add(entry.name); + } catch (error) { + errors.set(entry.name, error instanceof Error ? error : new Error(String(error))); + } + } + + const activationRoot = join(componentsRootDirPath, DEPLOY_ACTIVATION_DIR); + if (await ensureSecureDirectory(activationRoot, false, 'Component activation staging path')) { + for (const projectEntry of await readdir(activationRoot, { withFileTypes: true })) { + const projectPath = join(activationRoot, projectEntry.name); + if (!projectEntry.isDirectory() || !safeComponentName(projectEntry.name)) { + await rm(projectPath, { recursive: true, force: true }); + continue; + } + for (const artifact of await readdir(projectPath, { withFileTypes: true })) { + const deploymentId = activationArtifactDeploymentId(artifact.name); + const artifactPath = join(projectPath, artifact.name); + if (!deploymentId) { + await rm(artifactPath, { recursive: true, force: true }); + continue; + } + const row = await getDeployment(deploymentId).catch(() => undefined); + const livePath = join(componentsRootDirPath, projectEntry.name); + const liveStat = await lstat(livePath).catch(() => undefined); + if (row?.status === 'activating' && row.project === projectEntry.name && liveStat?.isDirectory()) { + try { + await persistActivation(row); + await rm(artifactPath, { recursive: true, force: true }); + recovered.add(deploymentId); + } catch (error) { + errors.set(deploymentId, error instanceof Error ? error : new Error(String(error))); + } + } else if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && !liveStat) { + await rename(artifactPath, livePath); + } else { + await rm(artifactPath, { recursive: true, force: true }); + } + } + await rmdir(projectPath).catch(() => {}); + } + await rmdir(activationRoot).catch(() => {}); + } + await rmdir(stagingRoot).catch(() => {}); + return { recovered: [...recovered], removed, errors }; +} + /** * Install all applications specified in the root config. * @@ -1674,6 +2067,95 @@ async function persistApplicationLock( await next; } +function applicationConfigFromActivationSpec(spec: Record): ApplicationConfig | undefined { + if (!spec.package) return undefined; + const applicationConfig: ApplicationConfig = { package: spec.package }; + if (spec.install_command !== null || spec.install_timeout !== null || spec.install_allow_scripts !== null) { + applicationConfig.install = {}; + if (spec.install_command !== null) applicationConfig.install.command = spec.install_command; + if (spec.install_timeout !== null) applicationConfig.install.timeout = spec.install_timeout; + if (spec.install_allow_scripts !== null) applicationConfig.install.allowInstallScripts = spec.install_allow_scripts; + } + if (spec.urlPath !== null) applicationConfig.urlPath = spec.urlPath; + if (spec.host !== null) applicationConfig.host = spec.host; + if (spec.credentials?.length) applicationConfig.credentials = spec.credentials; + return applicationConfig; +} + +async function readApplicationLock(lockPath: string): Promise<{ applications: Record }> { + try { + const lock = JSON.parse(await readFile(lockPath, 'utf8')); + if (!lock.applications || typeof lock.applications !== 'object') lock.applications = {}; + return lock; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return { applications: {} }; + throw error; + } +} + +/** Persist a runtime activation into the boot-time application lock, or remove it during compensation. */ +export async function updateApplicationLockEntry( + name: string, + applicationConfig: ApplicationConfig | undefined +): Promise { + const lockPath = join(getConfigValue(CONFIG_PARAMS.ROOTPATH), 'harper-application-lock.json'); + const previous = applicationLockWriteQueues.get(lockPath) ?? Promise.resolve(); + const next = previous + .catch(() => {}) + .then(async () => { + const lock = await readApplicationLock(lockPath); + if (applicationConfig === undefined) delete lock.applications[name]; + else lock.applications[name] = applicationConfig; + const tempPath = `${lockPath}.${process.pid}.${randomUUID()}.tmp`; + await writeFile(tempPath, JSON.stringify(lock, null, 2), 'utf8'); + await rename(tempPath, lockPath); + }); + applicationLockWriteQueues.set(lockPath, next); + await next; +} + +async function getApplicationLockEntry(name: string): Promise { + const lockPath = join(getConfigValue(CONFIG_PARAMS.ROOTPATH), 'harper-application-lock.json'); + const previous = applicationLockWriteQueues.get(lockPath) ?? Promise.resolve(); + let entry: ApplicationConfig | undefined; + const read = previous + .catch(() => {}) + .then(async () => { + entry = (await readApplicationLock(lockPath)).applications[name]; + }); + applicationLockWriteQueues.set(lockPath, read); + await read; + return entry; +} + +export async function createApplicationActivationTransaction( + project: string, + spec: Record +): Promise<{ commit(): Promise; rollback(): Promise }> { + const nextConfig = applicationConfigFromActivationSpec(spec); + if (!nextConfig) return { commit: async () => {}, rollback: async () => {} }; + let previousConfig: ApplicationConfig | undefined; + let previousLockConfig: ApplicationConfig | undefined; + let commitStarted = false; + return { + async commit() { + if (commitStarted) return; + previousConfig = readConfigFile()?.[project]; + previousLockConfig = await getApplicationLockEntry(project); + commitStarted = true; + await addConfig(project, nextConfig); + await updateApplicationLockEntry(project, nextConfig); + }, + async rollback() { + if (!commitStarted) return; + if (previousConfig === undefined) deleteConfigFromFile([project]); + else await addConfig(project, previousConfig); + await updateApplicationLockEntry(project, previousLockConfig); + commitStarted = false; + }, + }; +} + /** * Keep the boot-time application lock honest while a reinstall is in flight. Factored as an * explicit production seam so the failure transition can be tested without replacing module diff --git a/components/componentLoader.ts b/components/componentLoader.ts index bca7c04f88..9e9324e116 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -49,13 +49,19 @@ import { lifecycle as componentLifecycle } from './status/index.ts'; import { DEFAULT_CONFIG } from './DEFAULT_CONFIG.ts'; import { materializeGlobalSecrets, processComponentEnv } from './componentSecrets.ts'; import { PluginModule } from './PluginModule.ts'; -import { getEnvBuiltInComponents } from './Application.ts'; +import { + getEnvBuiltInComponents, + createApplicationActivationTransaction, + reconcileStagedApplicationArtifacts, +} from './Application.ts'; +import { getDeploymentRow } from './deploymentRecorder.ts'; import { pathToFileURL } from 'node:url'; const CF_ROUTES_DIR = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); let loadedComponents = new Map(); let watchesSetup; let resources; +let stagedArtifactsReconciled = false; /** * Load all the applications registered in Harper, those in the components directory as well as any directly @@ -66,6 +72,38 @@ let resources; export async function loadComponentDirectories(loadedPluginModules?: Map, loadedResources?: Resources) { if (loadedResources) resources = loadedResources; if (loadedPluginModules) loadedComponents = loadedPluginModules; + if (isMainThread && !stagedArtifactsReconciled) { + stagedArtifactsReconciled = true; + try { + const reconciliation = await reconcileStagedApplicationArtifacts(CF_ROUTES_DIR, getDeploymentRow, async (row) => { + const transaction = await createApplicationActivationTransaction(row.project, row.activation_spec); + try { + await transaction.commit(); + } catch (error) { + try { + await transaction.rollback(); + } catch (rollbackError) { + throw new AggregateError( + [error, rollbackError], + `Could not persist or roll back the recovered component activation for '${row.project}'` + ); + } + throw error; + } + }); + for (const deploymentId of reconciliation.recovered) { + harperLogger.warn( + `Rolled forward interrupted component activation '${deploymentId}' on this node; ` + + `the deployment remains activating until cluster state is reconciled` + ); + } + for (const [deploymentId, error] of reconciliation.errors) { + harperLogger.error(`Could not reconcile staged component deployment '${deploymentId}':`, errorForLog(error)); + } + } catch (error) { + harperLogger.error('Could not inspect staged component deployments during startup:', errorForLog(error as Error)); + } + } // Materialize hdb_secret global-tier rows into process.env and snapshot the scoped tier before // any application loads (root components — including the Pro custody registration — have // already loaded by this point). Re-runs on each reload cycle, which is how changed/late-custody diff --git a/components/deploymentOperations.ts b/components/deploymentOperations.ts index adf4af5170..fd53472735 100644 --- a/components/deploymentOperations.ts +++ b/components/deploymentOperations.ts @@ -101,8 +101,8 @@ export async function handleGetDeployment(req: GetRequest): Promise { // SSE content-negotiated branch — when serverHandlers.js detects // `Accept: text/event-stream` it attaches a ProgressEmitter as req.progress and wraps // our return as the operation's final SSE event. We replay event_log on connect, then - // tail the deployment's live emitter (if it's still running on this node) until it - // reaches a terminal status. The final return value becomes the SSE `done` event. + // tail the deployment's live emitter (if it's still running on this node) until its + // recorder finishes. The final return value becomes the SSE `done` event. if (req.progress && typeof (req.progress as any).emit === 'function') { const sse = req.progress; const liveEmitter = getActiveEmitter(req.deployment_id); @@ -150,7 +150,10 @@ export async function handleGetDeployment(req: GetRequest): Promise { if (typeof entry.t === 'number') lastReplayedTs = Math.max(lastReplayedTs, entry.t); } - if (liveEmitter && !TERMINAL_STATUSES.has(row.status)) { + // A full two-phase deploy checkpoints `staged` before immediately entering activation. + // The live emitter, not that transient row status, is authoritative for whether this + // origin is still running the deployment. + if (liveEmitter) { await new Promise((resolve) => { resolveLive = resolve; // Flush anything that arrived during replay, filtering to events the replay missed. @@ -170,7 +173,10 @@ export async function handleGetDeployment(req: GetRequest): Promise { if (!live || live !== liveEmitter) { clearInterval(pollTimer); const latest = await table.get(req.deployment_id); - if (latest && TERMINAL_STATUSES.has(latest.status) && !liveDone) { + // `staged` is a valid resting result and `activating` records + // uncertain cluster completion. If the original emitter is gone, + // there is no local work left to tail in either state. + if (latest && !liveDone) { liveDone = true; resolve(); } diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index c33bb59384..a167a4942a 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -6,8 +6,8 @@ // payload_blob (with sha256 + size), and writes the terminal status at the end. // Subscribes to a ProgressEmitter so phase transitions and install lines land in // event_log as they happen — making the deploy observable by Studio polling -// get_deployment without an attached CLI. The persisted payload_blob will also serve -// as the rollback source when that operation lands. +// get_deployment without an attached CLI. The persisted payload_blob also serves as +// the replication source for peer staging. import { randomUUID } from 'node:crypto'; import { createHash } from 'node:crypto'; @@ -50,14 +50,14 @@ type DeploymentStatus = | 'extracting' | 'installing' // Two-phase deploy: building the incoming version into staging cluster-wide (stage phase), and the - // terminal resting state of a stage_component that has not yet been activated. + // terminal resting state of a deploy_component stage that has not yet been activated. | 'staging' | 'staged' | 'loading' | 'replicating' // Two-phase deploy: swapping the staged build into the live path cluster-wide (activate phase). | 'activating' - // revert_component: swapping the live version back to its retained previous version. + // Retained for compatibility with deployment rows written by earlier preview builds. | 'reverting' | 'restarting' | 'success' @@ -71,10 +71,10 @@ interface CreateOptions { restart_mode?: 'immediate' | 'rolling' | null; rollback_of?: string | null; // Deploy credentials in reference form (`{ registry, secret, scope? }` / `{ host, secret, - // username? }`) — never a literal token. Kept so a rollback can re-resolve the credential from - // hdb_secret without the operator re-supplying it. Null when the deploy used no credentials or a - // no-custody transient token. + // username? }`) — never a literal token. Peers and later activation resolve the reference from + // hdb_secret. Null when the deploy used no credentials or a no-custody transient token. credentials?: CredentialReference[] | null; + activation_spec?: Record | null; emitter?: ProgressEmitter; } @@ -113,6 +113,7 @@ export class DeploymentRecorder { user: options.user ?? null, rollback_of: options.rollback_of ?? null, credentials: options.credentials ?? null, + activation_spec: options.activation_spec ?? null, error: null, }; const recorder = new DeploymentRecorder(deploymentId, record); @@ -403,17 +404,24 @@ export class DeploymentRecorder { this.sealed = true; } - async finish(status: 'success' | 'failed' | 'rolled_back' | 'staged', error?: unknown): Promise { + async checkpoint(status: DeploymentStatus, phase: string): Promise { if (this.finished) return; - // Send a terminal sentinel through the emitter (if any) BEFORE we unsubscribe and - // remove it from the registry, so any SSE tail subscribers can resolve their wait - // even on a code path that doesn't emit an explicit `error` event. + while (this.pendingPut) await this.pendingPut; + this.record.status = status; + this.record.phase = phase; + this.record.completed_at = null; + await this.put(); + } + + async finish(status: 'success' | 'failed' | 'rolled_back' | 'staged' | 'activating', error?: unknown): Promise { + if (this.finished) return; + // Keep the emitter registered until the terminal row write lands. An SSE tailer may + // otherwise observe the sentinel, re-read the preceding staged checkpoint, and + // incorrectly finish with a stale result. const emitter = activeEmitters.get(this.deploymentId); - emitter?.emit('_recorder_finished', { status }); this.finished = true; this.unsubscribe?.(); this.unsubscribe = null; - activeEmitters.delete(this.deploymentId); // Drain the ENTIRE coalesced-flush chain before mutating + persisting the terminal // state. Just awaiting `this.pendingPut` once isn't enough: its `.finally` may // re-schedule another put (when `dirty` was set during the in-flight put), and @@ -428,7 +436,7 @@ export class DeploymentRecorder { } } this.record.status = status; - this.record.completed_at = Date.now(); + this.record.completed_at = status === 'activating' ? null : Date.now(); if (error) { const e = error as { message?: string; code?: string | number; stack?: string }; this.record.error = { @@ -437,7 +445,15 @@ export class DeploymentRecorder { phase: this.record.phase, }; } - await this.put(); + try { + await this.put(); + } finally { + // Emit after persistence but before removing the registry entry so subscribers + // always have a durable terminal row to re-read. The sentinel also releases + // tailers on failure paths that did not emit an explicit error event. + emitter?.emit('_recorder_finished', { status }); + activeEmitters.delete(this.deploymentId); + } } get row(): Record { @@ -472,7 +488,7 @@ export async function getDeploymentRow(deploymentId: string): Promise { const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; if (!table) return; @@ -489,7 +506,64 @@ export async function markDeploymentTerminal( // Patch (partial update) rather than mutating the row: table.get() returns a read-only record, so // `row.status = …` throws "Cannot assign to read only property". A patch also touches only these two // fields, so it can't truncate the rest of the row the way a spread-and-put might. - await table.patch(deploymentId, { status, completed_at: Date.now() }); + const update: Record = { + status, + completed_at: status === 'activating' ? null : Date.now(), + }; + if (error !== undefined) { + update.error = { + message: error instanceof Error ? error.message : String(error), + }; + } + await table.patch(deploymentId, update); +} + +export async function recordDeploymentPeers(deploymentId: string, results: unknown): Promise { + if (!Array.isArray(results) || results.length === 0) return; + const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; + if (!table) return; + const row = await table.get(deploymentId); + if (!row) return; + const peers = Array.isArray(row.peer_results) + ? row.peer_results.map((entry: unknown) => ({ ...(entry as object) })) + : []; + for (const result of results) { + const normalized = normalizePeerResult(result); + const index = normalized.node ? peers.findIndex((entry: any) => entry.node === normalized.node) : -1; + if (index >= 0) peers[index] = normalized; + else peers.push(normalized); + } + await table.patch(deploymentId, { peer_results: peers }); +} + +/** Claim a staged deployment for activation while the component preparation lock is held. */ +export async function claimStagedDeployment( + deploymentId: string, + project: string, + options: { allowActivating?: boolean; waitForStagedMs?: number } = {} +): Promise> { + const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; + if (!table) throw new ClientError('Deployment tracking is unavailable; cannot activate a staged deployment'); + const deadline = Date.now() + coerceTimeoutMs(options.waitForStagedMs, 0); + let row = await table.get(deploymentId); + if (!row) throw new ClientError(`No deployment found with id '${deploymentId}'`); + if (row.project !== project) { + throw new ClientError(`Deployment '${deploymentId}' belongs to component '${row.project}', not '${project}'`); + } + while (['pending', 'staging'].includes(row.status) && Date.now() < deadline) { + await new Promise((resolve) => setTimeout(resolve, Math.min(25, deadline - Date.now()))); + row = await table.get(deploymentId); + if (!row) throw new ClientError(`No deployment found with id '${deploymentId}'`); + if (row.project !== project) { + throw new ClientError(`Deployment '${deploymentId}' belongs to component '${row.project}', not '${project}'`); + } + } + if (row.status === 'activating' && options.allowActivating) return row; + if (row.status !== 'staged') { + throw new ClientError(`Deployment '${deploymentId}' is '${row.status}', not staged and available for activation`); + } + await table.patch(deploymentId, { status: 'activating', phase: 'activate', completed_at: null, error: null }); + return row; } // Deployment statuses that are settled — a non-terminal deployment's payload_blob may still be the @@ -497,6 +571,48 @@ export async function markDeploymentTerminal( // deploymentOperations.ts's guard for the explicit delete_deployment_payload operation. const TERMINAL_STATUSES = new Set(['success', 'failed', 'rolled_back']); +async function settleStagedRows(project: string, keepCount: number): Promise { + const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; + if (!table) return []; + const staged: Array> = []; + for await (const row of table.search([{ attribute: 'project', value: project }])) { + if (row?.status === 'staged') staged.push(row); + } + staged.sort( + (a, b) => (b.started_at ?? 0) - (a.started_at ?? 0) || String(a.deployment_id).localeCompare(b.deployment_id) + ); + const expired = staged.slice(Math.max(0, keepCount)); + for (const row of expired) { + await table.patch(row.deployment_id, { + status: 'failed', + completed_at: Date.now(), + error: { message: 'Staged build expired by deployment_stagingRetention_maxCount', phase: 'staged' }, + }); + } + return expired.map((row) => row.deployment_id); +} + +export async function expireOldStagedDeployments(project: string, maxCount: number): Promise { + const count = Number.isFinite(maxCount) ? Math.max(1, Math.floor(maxCount)) : 1; + return settleStagedRows(project, count); +} + +export async function invalidateProjectStagedDeployments(project: string): Promise { + const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; + if (!table) return []; + const invalidated: string[] = []; + for await (const row of table.search([{ attribute: 'project', value: project }])) { + if (!['staged', 'activating'].includes(row?.status)) continue; + await table.patch(row.deployment_id, { + status: 'failed', + completed_at: Date.now(), + error: { message: 'Staged build invalidated because the component was dropped', phase: row.phase }, + }); + invalidated.push(row.deployment_id); + } + return invalidated; +} + /** * Count-based payload retention: keep at most `maxCount` stored payload tarballs for a project, * newest first, dropping the payload_blob of the rest. Rows are always retained — only the tarball @@ -579,7 +695,12 @@ export function coerceTimeoutMs(value: unknown, fallback: number): number { */ export async function awaitDeploymentRow( deploymentId: string, - options: { timeoutMs?: number; pollIntervalMs?: number; initialPollIntervalMs?: number } = {} + options: { + timeoutMs?: number; + pollIntervalMs?: number; + initialPollIntervalMs?: number; + requirePayload?: boolean; + } = {} ): Promise> { // Coerce defensively: the deploy operation's `deployment_timeout` reaches us via the // operation body, and the Joi validator's coerced number is discarded by validateBySchema @@ -594,6 +715,7 @@ export async function awaitDeploymentRow( // human-noticeable latency, then back off exponentially up to maxIntervalMs for the // rare case where the row is genuinely still replicating. let intervalMs = options.initialPollIntervalMs ?? 5; + const requirePayload = options.requirePayload !== false; const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; if (!table) { throw new Error( @@ -610,7 +732,7 @@ export async function awaitDeploymentRow( try { const row = await table.get(deploymentId); if (row) { - if (row.payload_blob != null) return row; + if (!requirePayload || row.payload_blob != null) return row; // Row replicated but its payload_blob write hasn't landed yet — replication is // alive, just mid-flight. Remember this so the timeout message points at the // payload write rather than a dead channel. diff --git a/components/operations.js b/components/operations.js index 1a9828cede..3e1f867175 100644 --- a/components/operations.js +++ b/components/operations.js @@ -1,6 +1,7 @@ 'use strict'; const path = require('node:path'); +const { isDeepStrictEqual } = require('node:util'); const { isMainThread } = require('node:worker_threads'); const fs = require('fs-extra'); const fg = require('fast-glob'); @@ -30,12 +31,20 @@ const { stageApplication, activateApplication, revertApplication, + stagedApplicationPath, + hasCompleteStagedApplication, + activateStagedApplication, discardStagedApplication, + discardProjectStagedApplications, + discardProjectActivationArtifacts, + updateApplicationLockEntry, + createApplicationActivationTransaction, ASIDE_STAGING_DIR, DEPLOY_STAGING_DIR, + DEPLOY_ACTIVATION_DIR, DEPLOY_PREVIOUS_DIR, } = require('./Application.ts'); -const { COMPONENT_PREPARATION_LOCK_DIR } = require('./componentPreparationLock.ts'); +const { COMPONENT_PREPARATION_LOCK_DIR, withComponentPreparationLock } = require('./componentPreparationLock.ts'); const { server } = require('../server/Server.ts'); const { DeploymentRecorder, @@ -43,12 +52,17 @@ const { getDeploymentRow, markDeploymentTerminal, normalizePeerResult, + recordDeploymentPeers, + claimStagedDeployment, + expireOldStagedDeployments, + invalidateProjectStagedDeployments, pruneProjectPayloads, readPayloadBlobWithRetry, coerceTimeoutMs, DEFAULT_AWAIT_ROW_TIMEOUT_MS, } = require('./deploymentRecorder.ts'); const { ProgressEmitter } = require('../server/serverHelpers/progressEmitter.ts'); +const { isOperationAuthorizationBypassed } = require('../server/serverHelpers/operationAuthorizationState.ts'); /** * Read the settings.js file and return the @@ -319,7 +333,11 @@ async function dropCustomFunctionProject(req) { try { const projectDir = path.join(cfDir, project); + await invalidateProjectStagedDeployments(project); + await discardProjectStagedApplications(projectDir); + await discardProjectActivationArtifacts(projectDir); fs.rmSync(projectDir, { recursive: true }); + await updateApplicationLockEntry(project, undefined); let response = await server.replication.replicateOperation(req); response.message = `Successfully deleted project: ${project}`; return response; @@ -398,18 +416,52 @@ async function deployComponent(req) { req.project = getProjectNameFromPackage(req.package); } - const validation = validator.deployComponentValidator(req); + const isReplicatedExecution = isTrustedReplicatedOperation(req); + let requestToValidate = req; + if (isReplicatedExecution) { + if (req._phase !== undefined) { + throw new ServerError( + `Unsupported legacy component deployment phase '${req._phase}'; upgrade the originating node before deploying` + ); + } + const { _deploymentId, ...publicRequest } = req; + requestToValidate = publicRequest; + } + const validation = validator.deployComponentValidator(requestToValidate); if (validation) { throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); } - - const isReplicatedExecution = typeof req._deploymentId === 'string'; - - // Internal peer-phase execution. The two-phase cluster fan-out rides deploy_component itself, tagged - // with an internal `_phase` marker (`stage`/`activate`) rather than separate public operations — so - // the wire format is deploy_component + `_phase`, and only deploy_component is publicly exposed. - if (isReplicatedExecution && req._phase === 'stage') return deployPhaseStage(req); - if (isReplicatedExecution && req._phase === 'activate') return deployPhaseActivate(req); + const systemReplicated = isSystemDatabaseReplicated(); + const requestedSeparatedPhase = req.activate === false || req.deployment_id !== undefined; + if (!isReplicatedExecution && req.two_phase === true && req.replicated === false) { + throw handleHDBError( + new Error(), + `two_phase:true requires operation replication to be enabled`, + HTTP_STATUS_CODES.BAD_REQUEST + ); + } + if (!isReplicatedExecution && req.two_phase === true && !systemReplicated) { + throw handleHDBError( + new Error(), + `two_phase:true requires system database replication to be enabled`, + HTTP_STATUS_CODES.BAD_REQUEST + ); + } + if ( + !isReplicatedExecution && + requestedSeparatedPhase && + (req.two_phase === false || req.replicated === false || !systemReplicated) + ) { + throw handleHDBError( + new Error(), + `activate:false and deployment_id require two-phase deploy with system database replication enabled`, + HTTP_STATUS_CODES.BAD_REQUEST + ); + } + if (req.replicated !== false && !isReplicatedExecution && req.two_phase !== false && systemReplicated) { + if (req.deployment_id) return deployComponentActivateExisting(req); + return deployComponentTwoPhase(req); + } // Ingest any provided credential token into the secrets store so the credential lives as // replicated ciphertext (reference, not embed); already-reference entries pass through, and with @@ -421,19 +473,7 @@ async function deployComponent(req) { // token is not — it is used only for this node's install, then stripped before replication. const credentialReferences = (req.credentials ?? []).filter((entry) => entry && entry.secret !== undefined); - // Two-phase is the default, but it leans on the system table's replication channel to carry the - // payload and correlate the stage/activate steps across the cluster. Fall back to the legacy - // one-shot deploy when: the caller opted out (`two_phase: false`); this is a legacy peer replaying a - // one-shot deploy (replicated, no `_phase`); or `system` isn't replicated on this node. - if (req.two_phase === false || isReplicatedExecution || !isSystemDatabaseReplicated()) { - return deployComponentOneShot(req, credentialReferences, isReplicatedExecution); - } - - // `deployment_id` with no fresh payload → activate a previously-staged deployment (the second half of - // a stage-then-activate-later flow). Otherwise run the full stage+activate (which itself honors - // `activate: false` to stop after the cluster-wide staged barrier). - if (req.deployment_id) return deployComponentActivateExisting(req, credentialReferences); - return deployComponentTwoPhase(req, credentialReferences); + return deployComponentOneShot(req, credentialReferences, isReplicatedExecution); } /** @@ -618,7 +658,7 @@ async function deployComponentOneShot(req, credentialReferences, isReplicatedExe * every node (phase 2, activate_component). The live component on every node is untouched until the * whole cluster has the bits in place, and the go-live window is just the swap + restart. */ -async function deployComponentTwoPhase(req, credentialReferences) { +async function _legacyDeployComponentTwoPhase(req, credentialReferences) { const { resolveCredentials } = require('./secretOperations.ts'); // Fail fast on a protected core name before we create any state or touch the cluster. if (req.package) assertNotProtectedCoreComponent(req.project, req.force); @@ -783,7 +823,7 @@ async function deployComponentTwoPhase(req, credentialReferences) { * path, writing config, or restarting. A failure here fails this peer's stage, which the origin's * barrier catches. No recorder (the origin owns the row) and no re-replication. */ -async function deployPhaseStage(req) { +async function _legacyDeployPhaseStage(req) { const { resolveCredentials } = require('./secretOperations.ts'); const emitter = null; // peers stream nothing back; the origin owns the emitter/recorder const emit = () => {}; @@ -818,7 +858,7 @@ async function deployPhaseStage(req) { * id) into the live path, persist root config for a package deploy, and restart if the origin asked * for an immediate restart. No recorder, no re-replication. */ -async function deployPhaseActivate(req) { +async function _legacyDeployPhaseActivate(req) { if (req.package) assertNotProtectedCoreComponent(req.project, req.force); const credentialReferences = (req.credentials ?? []).filter((entry) => entry && entry.secret !== undefined); const application = new Application({ @@ -845,7 +885,7 @@ async function deployPhaseActivate(req) { * into the live path on the origin, replicates the activate phase to peers (each activates its own * staged copy of the same deployment id), restarts, and marks the deployment row success. */ -async function deployComponentActivateExisting(req, credentialReferences) { +async function _legacyDeployComponentActivateExisting(req, credentialReferences) { const stagingId = req.deployment_id; // An activate-by-id call carries no `package` — `harper activate` sends only project + deployment_id, // and the docs describe this path as fetching/installing nothing — so recover the staged deployment's @@ -972,7 +1012,505 @@ async function deployComponentActivateExisting(req, credentialReferences) { * Reached two ways: directly by an operator, and by a peer replaying a replicated revert * (`_deploymentId` set). deploy_component's `revert_on_failure` path drives it internally. */ -async function revertComponent(req) { +function activationSpecFromRequest(req, credentialReferences) { + return { + project: req.project, + package: req.package ?? null, + install_command: req.install_command ?? null, + install_timeout: req.install_timeout ?? null, + install_allow_scripts: req.install_allow_scripts ?? null, + urlPath: req.urlPath ?? null, + host: req.host ?? null, + credentials: credentialReferences.length ? credentialReferences : null, + force: req.force === true, + }; +} + +function applicationFromSpec(spec, payload, resolvedCredentials, installCapture, emit) { + return new Application({ + name: spec.project, + payload, + packageIdentifier: spec.package ?? undefined, + install: { + command: spec.install_command ?? undefined, + timeout: spec.install_timeout ?? undefined, + allowInstallScripts: spec.install_allow_scripts ?? undefined, + }, + credentials: resolvedCredentials, + onInstallLine: (manager, stream, line) => { + installCapture?.push(manager, stream, line); + emit?.('install', { manager, stream, line }); + }, + }); +} + +function failedPeerResults(results) { + return (results ?? []).filter((result) => result?.status === 'failed' || result?.error || result?.reason); +} + +function describePeerFailures(failed) { + return failed + .map((peer) => `${peer.node ?? 'unknown'} (${peer.error?.message ?? peer.reason ?? 'unknown error'})`) + .join(', '); +} + +function buildPhaseOperation(phase, deploymentId, project, activationSpec, extra = {}) { + return { + operation: hdbTerms.OPERATIONS_ENUM.COMPONENT_DEPLOY_PHASE, + phase, + deployment_id: deploymentId, + project, + activation_spec: activationSpec, + ...extra, + }; +} + +async function resolveSpecCredentials(spec, waitMs = 0) { + const { resolveCredentials } = require('./secretOperations.ts'); + return resolveCredentials(spec.credentials ?? [], spec.project, { waitMs }); +} + +function assertStoredActivationSpec(row, deploymentId, project, spec, allowedStatuses) { + if ( + !row || + row.project !== project || + !allowedStatuses.includes(row.status) || + !isDeepStrictEqual(row.activation_spec, spec) + ) { + throw new ServerError( + `Deployment '${deploymentId}' does not have the expected immutable activation specification for '${project}'` + ); + } +} + +async function sourceStagedPayload(deploymentId, spec, timeoutMs) { + const deadline = Date.now() + timeoutMs; + const row = await awaitDeploymentRow(deploymentId, { timeoutMs, requirePayload: !spec.package }); + assertStoredActivationSpec(row, deploymentId, spec.project, spec, ['pending', 'staging', 'staged', 'activating']); + if (spec.package) return undefined; + return readPayloadBlobWithRetry(() => row.payload_blob.stream(), { + timeoutMs: Math.max(0, deadline - Date.now()), + }); +} + +async function discardDeploymentEverywhere(project, deploymentId, activationSpec) { + const componentPath = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), project); + await discardStagedApplication(componentPath, deploymentId).catch(() => {}); + await server.replication + .replicateOperation(buildPhaseOperation('discard', deploymentId, project, activationSpec)) + .catch(() => {}); +} + +function getStagingRetentionMaxCount() { + const value = Number(env.get(hdbTerms.CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT)); + return Number.isFinite(value) && value >= 1 ? Math.floor(value) : 5; +} + +function getPayloadRetentionMaxCount() { + const value = Number(env.get(hdbTerms.CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXCOUNT)); + return Number.isFinite(value) && value >= 0 ? Math.floor(value) : 1; +} + +async function pruneStagedDeploymentArtifacts(project, activationSpec) { + const expired = await expireOldStagedDeployments(project, getStagingRetentionMaxCount()); + for (const deploymentId of expired) await discardDeploymentEverywhere(project, deploymentId, activationSpec); +} + +async function restartActivatedComponent(req, deploymentId, project, activationSpec, emit) { + if (req.restart === true) { + emit('phase', { phase: 'restart', status: 'start' }); + const restartResponse = await server.replication.replicateOperation( + buildPhaseOperation('restart', deploymentId, project, activationSpec) + ); + const failed = failedPeerResults(restartResponse?.replicated); + manageThreads.restartWorkers('http'); + emit('phase', { phase: 'restart', status: 'done' }); + return { restartMessage: `, restarting Harper`, replicated: restartResponse?.replicated, failedPeers: failed }; + } + if (req.restart === 'rolling') { + const serverUtilities = require('../server/serverHelpers/serverUtilities.ts'); + emit('phase', { phase: 'restart', status: 'start' }); + const jobResponse = await serverUtilities.executeJob({ + operation: 'restart_service', + service: 'http', + replicated: true, + }); + emit('phase', { phase: 'restart', status: 'done' }); + return { restartMessage: `, restarting Harper`, restartJobId: jobResponse.job_id, failedPeers: [] }; + } + return { restartMessage: '', failedPeers: [] }; +} + +async function deployComponentTwoPhase(req) { + assertNotProtectedCoreComponent(req.project, req.force); + const { ingestCredentials, resolveCredentials } = require('./secretOperations.ts'); + req.credentials = await ingestCredentials(req, req.credentials, req.project); + const credentialReferences = (req.credentials ?? []).filter((entry) => entry?.secret !== undefined); + const activationSpec = activationSpecFromRequest(req, credentialReferences); + const emitter = req.progress ?? new ProgressEmitter(); + const emit = (event, data) => emitter.emit(event, data); + const installCapture = createInstallCapture(); + const recorder = await DeploymentRecorder.create({ + project: req.project, + package_identifier: req.package ?? null, + user: req.hdb_user?.username, + restart_mode: req.restart === 'rolling' ? 'rolling' : req.restart ? 'immediate' : null, + credentials: credentialReferences.length ? credentialReferences : null, + activation_spec: activationSpec, + emitter, + }); + let application; + let activationCommitted = false; + let activationBarrierPassed = false; + try { + let payload = req.payload; + if (req.payload != null) { + await recorder.ingestPayload(req.payload); + payload = recorder.row.payload_blob.stream(); + } + const resolvedCredentials = await resolveCredentials(req.credentials, req.project); + application = applicationFromSpec(activationSpec, payload, resolvedCredentials, installCapture, emit); + if (credentialReferences.length) req.credentials = credentialReferences; + else delete req.credentials; + delete req.progress; + delete req.payload; + + emit('phase', { phase: 'stage', status: 'start' }); + const stagedPath = await stageApplication(application, recorder.deploymentId); + await loadValidateComponent({ dirPath: stagedPath, emit }); + recorder.seal(); + const stageResponse = await server.replication.replicateOperation( + buildPhaseOperation('stage', recorder.deploymentId, req.project, activationSpec, { + deployment_timeout: req.deployment_timeout, + }), + { + onPeerResult: (result) => { + recorder.recordPeer(result); + emit('peer', result); + }, + } + ); + if (stageResponse?.replicated) recorder.recordPeers(stageResponse.replicated); + emit('phase', { phase: 'stage', status: 'done' }); + const stageFailures = recorder.getFailedPeers(); + if (stageFailures.length && !req.ignore_replication_errors) { + await discardDeploymentEverywhere(req.project, recorder.deploymentId, activationSpec); + throw new ServerError( + `Component '${req.project}' failed to stage on ${stageFailures.length} peer node(s): ` + + `${describePeerFailures(stageFailures)}. No node was activated and the live component is unchanged.` + ); + } + await recorder.checkpoint('staged', 'staged'); + + if (req.activate === false) { + emit('phase', { phase: 'staged', status: 'done' }); + await recorder.finish('staged'); + await pruneStagedDeploymentArtifacts(req.project, activationSpec); + await pruneProjectPayloads(req.project, getPayloadRetentionMaxCount()).catch((error) => + log.warn('Failed to prune staged deployment payloads', error) + ); + return { + message: `Staged component: ${req.project}`, + project: req.project, + staged: true, + deployment_id: recorder.deploymentId, + replicated: stageResponse?.replicated, + ...(stageFailures.length ? { failed_peers: stageFailures } : {}), + }; + } + + const configTransaction = await createApplicationActivationTransaction(req.project, activationSpec); + await activateStagedApplication(application, recorder.deploymentId, { + beforeSwap: async () => { + await claimStagedDeployment(recorder.deploymentId, req.project); + emit('phase', { phase: 'activate', status: 'start' }); + }, + beforeCommit: () => configTransaction.commit(), + onRollback: () => configTransaction.rollback(), + }); + activationCommitted = true; + const activateResponse = await server.replication.replicateOperation( + buildPhaseOperation('activate', recorder.deploymentId, req.project, activationSpec), + { + onPeerResult: (result) => { + recorder.recordPeer(result); + emit('peer', result); + }, + } + ); + if (activateResponse?.replicated) recorder.recordPeers(activateResponse.replicated); + emit('phase', { phase: 'activate', status: 'done' }); + const activateFailures = recorder.getFailedPeers(); + if (activateFailures.length && !req.ignore_replication_errors) { + throw new ServerError( + `Component '${req.project}' activated on only part of the cluster. Split nodes: ` + + `${describePeerFailures(activateFailures)}. Roll forward by staging and activating a known-good deployment.` + ); + } + activationBarrierPassed = activateFailures.length === 0; + if (!req.restart) markRestartRequiredForNewComponent(application); + const restart = await restartActivatedComponent(req, recorder.deploymentId, req.project, activationSpec, emit); + if (restart.failedPeers.length) recorder.recordPeers(restart.failedPeers); + if (restart.failedPeers.length && !req.ignore_replication_errors) { + throw new ServerError( + `Component '${req.project}' activated, but restart failed on: ${describePeerFailures(restart.failedPeers)}` + ); + } + emit('phase', { phase: 'success', status: 'done' }); + maybeReclaimPayload(recorder, emit); + await recorder.finish('success'); + await pruneProjectPayloads(req.project, getPayloadRetentionMaxCount()).catch((error) => + log.warn('Failed to prune deployment payloads', error) + ); + return { + message: `Successfully deployed: ${req.project}${restart.restartMessage}`, + project: req.project, + deployment_id: recorder.deploymentId, + replicated: activateResponse?.replicated, + ...(restart.restartJobId ? { restartJobId: restart.restartJobId } : {}), + ...(recorder.getFailedPeers().length ? { failed_peers: recorder.getFailedPeers() } : {}), + }; + } catch (error) { + if (application && !activationCommitted) { + await discardStagedApplication(application.dirPath, recorder.deploymentId).catch(() => {}); + } + const capture = installCapture.snapshot(); + const failedPeers = recorder.getFailedPeers(); + const message = error?.message ?? String(error); + const structured = { + error: message, + phase: recorder.row.phase, + deployment_id: recorder.deploymentId, + ...(capture.lines.length ? { install_output: capture } : {}), + ...(failedPeers.length ? { failed_peers: failedPeers } : {}), + }; + emit('error', { + message, + code: error?.statusCode ?? error?.code, + phase: recorder.row.phase, + deployment_id: recorder.deploymentId, + install_output: capture.lines.length ? capture : undefined, + failed_peers: failedPeers.length ? failedPeers : undefined, + }); + await recorder + .finish(activationBarrierPassed ? 'success' : activationCommitted ? 'activating' : 'failed', error) + .catch((finishError) => log.warn('Failed to record two-phase deployment failure', finishError)); + const outError = new ServerError(message, error?.statusCode); + outError.http_resp_msg = structured; + throw outError; + } +} + +const ACTIVATION_FRESH_FIELDS = [ + 'payload', + 'package', + 'install_command', + 'install_timeout', + 'install_allow_scripts', + 'urlPath', + 'host', + 'credentials', + 'force', + 'activate', + 'two_phase', +]; + +function assertActivationRequestIsReferenceOnly(req) { + const supplied = ACTIVATION_FRESH_FIELDS.filter((field) => req[field] !== undefined); + if (supplied.length) { + throw handleHDBError( + new Error(), + `deployment_id activation uses the immutable staged configuration; remove: ${supplied.join(', ')}`, + HTTP_STATUS_CODES.BAD_REQUEST + ); + } +} + +async function deployComponentActivateExisting(req) { + assertActivationRequestIsReferenceOnly(req); + const row = await getDeploymentRow(req.deployment_id); + if (!row) throw handleHDBError(new Error(), `No deployment found with id '${req.deployment_id}'`, 404); + if (row.project !== req.project || row.status !== 'staged' || !row.activation_spec) { + throw handleHDBError( + new Error(), + `Deployment '${req.deployment_id}' is not a staged deployment for component '${req.project}'`, + HTTP_STATUS_CODES.CONFLICT + ); + } + const spec = row.activation_spec; + assertNotProtectedCoreComponent(spec.project, spec.force); + const emitter = req.progress ?? new ProgressEmitter(); + const emit = (event, data) => emitter.emit(event, data); + const componentPath = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), req.project); + let application; + const stagedPath = stagedApplicationPath(componentPath, req.deployment_id); + if (await hasCompleteStagedApplication(stagedPath)) { + application = applicationFromSpec(spec, undefined, undefined, null, emit); + await loadValidateComponent({ dirPath: stagedPath, emit }); + } else { + const timeoutMs = coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS); + const payload = await sourceStagedPayload(req.deployment_id, spec, timeoutMs); + const credentials = await resolveSpecCredentials(spec, timeoutMs); + application = applicationFromSpec(spec, payload, credentials, createInstallCapture(), emit); + const rebuiltPath = await stageApplication(application, req.deployment_id); + await loadValidateComponent({ dirPath: rebuiltPath, emit }); + } + const configTransaction = await createApplicationActivationTransaction(req.project, spec); + emit('phase', { phase: 'activate', status: 'start' }); + let claimed = false; + try { + await activateStagedApplication(application, req.deployment_id, { + beforeSwap: async () => { + await claimStagedDeployment(req.deployment_id, req.project); + claimed = true; + }, + beforeCommit: () => configTransaction.commit(), + onRollback: () => configTransaction.rollback(), + }); + } catch (error) { + if (claimed) await markDeploymentTerminal(req.deployment_id, 'staged').catch(() => {}); + throw error; + } + let settledPeers = []; + let activationBarrierPassed = false; + try { + const peerResults = []; + const activateResponse = await server.replication.replicateOperation( + buildPhaseOperation('activate', req.deployment_id, req.project, spec), + { + onPeerResult: (result) => { + peerResults.push(result); + emit('peer', result); + }, + } + ); + settledPeers = Array.isArray(activateResponse?.replicated) ? activateResponse.replicated : peerResults; + await recordDeploymentPeers(req.deployment_id, settledPeers); + emit('phase', { phase: 'activate', status: 'done' }); + const failed = failedPeerResults(settledPeers); + if (failed.length && !req.ignore_replication_errors) { + throw new ServerError( + `Component '${req.project}' activated on only part of the cluster. Split nodes: ` + + `${describePeerFailures(failed)}. Roll forward by staging and activating a known-good deployment.` + ); + } + activationBarrierPassed = failed.length === 0; + if (!req.restart) markRestartRequiredForNewComponent(application); + const restart = await restartActivatedComponent(req, req.deployment_id, req.project, spec, emit); + if (restart.failedPeers.length) { + settledPeers = [...settledPeers, ...restart.failedPeers]; + await recordDeploymentPeers(req.deployment_id, restart.failedPeers); + } + if (restart.failedPeers.length && !req.ignore_replication_errors) { + throw new ServerError( + `Component '${req.project}' activated, but restart failed on: ${describePeerFailures(restart.failedPeers)}` + ); + } + await markDeploymentTerminal(req.deployment_id, 'success'); + await maybeReclaimFinishedPayload(req.deployment_id, emit); + await pruneProjectPayloads(req.project, getPayloadRetentionMaxCount()).catch((error) => + log.warn('Failed to prune activated deployment payloads', error) + ); + return { + message: `Activated component: ${req.project}${restart.restartMessage}`, + project: req.project, + activated: true, + deployment_id: req.deployment_id, + replicated: activateResponse?.replicated, + ...(restart.restartJobId ? { restartJobId: restart.restartJobId } : {}), + ...(failedPeerResults(settledPeers).length ? { failed_peers: failedPeerResults(settledPeers) } : {}), + }; + } catch (error) { + await markDeploymentTerminal(req.deployment_id, activationBarrierPassed ? 'success' : 'activating', error).catch( + () => {} + ); + throw error; + } +} + +async function componentDeployPhase(req) { + if (!isTrustedReplicatedOperation(req)) { + throw handleHDBError(new Error(), 'component_deploy_phase is restricted to authenticated cluster peers', 403); + } + const validation = validator.componentDeployPhaseValidator({ + phase: req.phase, + deployment_id: req.deployment_id, + project: req.project, + activation_spec: req.activation_spec, + deployment_timeout: req.deployment_timeout, + }); + if (validation) throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); + const spec = req.activation_spec; + if (!spec || spec.project !== req.project) { + throw handleHDBError(new Error(), 'Invalid immutable activation specification', HTTP_STATUS_CODES.BAD_REQUEST); + } + const componentPath = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), req.project); + if (req.phase === 'discard') { + await discardStagedApplication(componentPath, req.deployment_id); + return { message: `Discarded staged component: ${req.project}` }; + } + if (req.phase === 'restart') { + const row = await getDeploymentRow(req.deployment_id); + assertStoredActivationSpec(row, req.deployment_id, req.project, spec, [ + 'pending', + 'staging', + 'staged', + 'activating', + ]); + manageThreads.restartWorkers('http'); + return { message: `Restarting component runtime for: ${req.project}` }; + } + if (req.phase === 'stage') { + assertNotProtectedCoreComponent(req.project, spec.force); + const timeoutMs = coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS); + const payload = await sourceStagedPayload(req.deployment_id, spec, timeoutMs); + const credentials = await resolveSpecCredentials(spec, timeoutMs); + const application = applicationFromSpec(spec, payload, credentials, createInstallCapture()); + const stagedPath = await stageApplication(application, req.deployment_id); + await loadValidateComponent({ dirPath: stagedPath, emit: () => {} }); + return { message: `Staged component: ${req.project}`, project: req.project, staged: true }; + } + const row = await getDeploymentRow(req.deployment_id); + assertStoredActivationSpec(row, req.deployment_id, req.project, spec, ['pending', 'staging', 'staged', 'activating']); + let application; + const stagedPath = stagedApplicationPath(componentPath, req.deployment_id); + if (await hasCompleteStagedApplication(stagedPath)) { + application = applicationFromSpec(spec, undefined, undefined, null); + await loadValidateComponent({ dirPath: stagedPath, emit: () => {} }); + } else { + const timeoutMs = coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS); + const payload = await sourceStagedPayload(req.deployment_id, spec, timeoutMs); + const credentials = await resolveSpecCredentials(spec, timeoutMs); + application = applicationFromSpec(spec, payload, credentials, createInstallCapture()); + const rebuiltPath = await stageApplication(application, req.deployment_id); + await loadValidateComponent({ dirPath: rebuiltPath, emit: () => {} }); + } + const configTransaction = await createApplicationActivationTransaction(req.project, spec); + await activateStagedApplication(application, req.deployment_id, { + beforeSwap: async () => { + await claimStagedDeployment(req.deployment_id, req.project, { + allowActivating: true, + waitForStagedMs: coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS), + }); + }, + beforeCommit: () => configTransaction.commit(), + onRollback: () => configTransaction.rollback(), + }); + markRestartRequiredForNewComponent(application); + return { message: `Activated component: ${req.project}`, project: req.project, activated: true }; +} + +function isTrustedReplicatedOperation(req) { + const user = req.hdb_user; + return ( + isOperationAuthorizationBypassed() && + req.replicated === false && + !!user && + !!(user.name || user.replicates || user.subscribers) + ); +} + +async function _revertComponent(req) { if (req.project) req.project = path.parse(req.project).name; const validation = validator.revertComponentValidator(req); if (validation) throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); @@ -1357,6 +1895,29 @@ function maybeReclaimPayload(recorder, emit) { } } +async function maybeReclaimFinishedPayload(deploymentId, emit) { + try { + const row = await getDeploymentRow(deploymentId); + const payloadSize = row?.payload_size; + const retentionMaxSize = getPayloadRetentionMaxSize(); + if ( + typeof payloadSize !== 'number' || + payloadSize <= retentionMaxSize || + failedPeerResults(row.peer_results).length > 0 || + row.payload_blob == null + ) { + return; + } + const { handleDeleteDeploymentPayload } = require('./deploymentOperations.ts'); + const result = await handleDeleteDeploymentPayload({ deployment_id: deploymentId }); + if (result.freed_bytes > 0) { + emit('payload_dropped', { payload_size: result.freed_bytes, max_size: retentionMaxSize }); + } + } catch (error) { + log.warn(`Failed to reclaim payload for activated deployment '${deploymentId}'`, error); + } +} + /** * Count-based payload retention (deployment_payloadRetention_maxCount): after a successful deploy, keep * only the newest N stored payloads for this project and drop the rest. Where the size-based reclaim @@ -1513,6 +2074,7 @@ async function getComponents() { itemName === 'node_modules' || itemName === ASIDE_STAGING_DIR || itemName === DEPLOY_STAGING_DIR || + itemName === DEPLOY_ACTIVATION_DIR || itemName === DEPLOY_PREVIOUS_DIR || itemName === COMPONENT_PREPARATION_LOCK_DIR ) @@ -1830,28 +2392,34 @@ async function dropComponent(req) { const { project, file } = req; const projectPath = req.file ? path.join(project, file) : project; - const pathToComponent = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), projectPath); - - const componentSymlink = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'node_modules', project); - if (await fs.pathExists(componentSymlink)) { - await fs.unlink(componentSymlink); - } - - if (await fs.pathExists(pathToComponent)) { - await fs.remove(pathToComponent); - } + const componentsRoot = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT); + const componentPath = path.join(componentsRoot, project); + const pathToComponent = path.join(componentsRoot, projectPath); + + await withComponentPreparationLock(componentPath, async () => { + const componentSymlink = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'node_modules', project); + if (await fs.pathExists(componentSymlink)) { + await fs.unlink(componentSymlink); + } - // Remove the component from the package.json file - const packageJsonPath = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'package.json'); - if (await fs.pathExists(packageJsonPath)) { - const packageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf8')); - if (packageJson?.dependencies?.[project]) { - delete packageJson.dependencies[project]; + if (!file) { + await invalidateProjectStagedDeployments(project); + await discardProjectStagedApplications(componentPath); + await discardProjectActivationArtifacts(componentPath); } - await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2), 'utf8'); - } + if (await fs.pathExists(pathToComponent)) await fs.remove(pathToComponent); + if (!file) await updateApplicationLockEntry(project, undefined); + + const packageJsonPath = path.join(env.get(hdbTerms.CONFIG_PARAMS.ROOTPATH), 'package.json'); + if (await fs.pathExists(packageJsonPath)) { + const packageJson = JSON.parse(await fs.readFile(packageJsonPath, 'utf8')); + if (packageJson?.dependencies?.[project]) delete packageJson.dependencies[project]; + await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2), 'utf8'); + } + + configUtils.deleteConfigFromFile([project]); + }); - configUtils.deleteConfigFromFile([project]); let response = await server.replication.replicateOperation(req); if (req.restart === true) { manageThreads.restartWorkers('http'); @@ -1869,8 +2437,7 @@ exports.addComponent = addComponent; exports.dropCustomFunctionProject = dropCustomFunctionProject; exports.packageComponent = packageComponent; exports.deployComponent = deployComponent; -exports.revertComponent = revertComponent; -exports.selectRevertTargets = selectRevertTargets; // exported for unit testing the revert_on_failure node-targeting +exports.componentDeployPhase = componentDeployPhase; exports.getComponents = getComponents; exports.getComponentFile = getComponentFile; exports.setComponentFile = setComponentFile; diff --git a/components/operationsValidation.js b/components/operationsValidation.js index 8964caba1d..32a8a0913f 100644 --- a/components/operationsValidation.js +++ b/components/operationsValidation.js @@ -13,6 +13,7 @@ const { ENV_ENCRYPTED_PREFIX } = require('../utility/envFile.ts'); // File name can only be alphanumeric, dash and underscores const PROJECT_FILE_NAME_REGEX = /^[a-zA-Z0-9-_]+$/; +const DEPLOYMENT_ID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; // dotenv's accepted key character set. Restricting keys to this prevents a crafted key (e.g. one // containing `=` or a newline) from injecting extra assignments into a .env file. @@ -25,7 +26,7 @@ module.exports = { dropCustomFunctionProjectValidator, packageComponentValidator, deployComponentValidator, - revertComponentValidator, + componentDeployPhaseValidator, setComponentFileValidator, getComponentFileValidator, dropComponentFileValidator, @@ -456,10 +457,14 @@ const URL_PATH_SCHEMA = Joi.string() .min(1) .custom((value, helpers) => { if (value.includes('..')) return helpers.error('any.invalid'); + if (value.split('/').includes('.')) return helpers.error('string.dotSegment'); return value; }) .optional() - .messages({ 'any.invalid': 'urlPath must not contain ".."' }); + .messages({ + 'any.invalid': 'urlPath must not contain ".."', + 'string.dotSegment': 'urlPath must not contain "." path segments', + }); // `registryAuth` was the credentials field's name on the 5.2 dev line. Rejected rather than ignored: // validation allows unknown keys, so a caller still sending it would otherwise get a deploy that @@ -494,16 +499,20 @@ function deployComponentValidator(req) { // Activate a previously-staged deployment (from an `activate: false` stage) cluster-wide. Same safe // charset as `project` because it becomes a staging-dir path segment (`.deploy-staging//`) // — a `../` value would otherwise resolve the staging source outside `.deploy-staging`. - deployment_id: Joi.string().pattern(PROJECT_FILE_NAME_REGEX).optional().messages({ - 'string.pattern.base': `'deployment_id' must only contain letters, numbers, dashes, and underscores`, + deployment_id: Joi.string().pattern(DEPLOYMENT_ID_REGEX).optional().messages({ + 'string.pattern.base': `'deployment_id' must be a UUID`, }), // If the activate phase fails on some nodes (leaving the cluster split across versions), swap the // nodes that did activate back to the retained previous version before reporting the failure. Off // by default. - revert_on_failure: Joi.boolean().optional(), + revert_on_failure: Joi.any().forbidden().messages({ + 'any.unknown': `'revert_on_failure' is not supported; recover partial activation by rolling forward`, + }), // Opt out of the two-phase (stage-then-activate) deploy and use the legacy one-shot path instead. // Defaults to two-phase. two_phase: Joi.boolean().optional(), + _deploymentId: Joi.any().forbidden(), + _phase: Joi.any().forbidden(), urlPath: URL_PATH_SCHEMA, // Deploy credentials. Each entry is npm registry auth (`registry`) or git host auth (`host`, // #1792), and supplies its secret either as a literal `token` (used only for this node's @@ -515,6 +524,24 @@ function deployComponentValidator(req) { return validator.validateBySchema(req, deployProjSchema); } +/** Validate the path- and state-selecting fields on the authenticated peer-only deploy operation. */ +function componentDeployPhaseValidator(req) { + const phaseSchema = Joi.object({ + phase: Joi.string().valid('stage', 'activate', 'discard', 'restart').required(), + deployment_id: Joi.string().pattern(DEPLOYMENT_ID_REGEX).required().messages({ + 'string.pattern.base': `'deployment_id' must be a UUID`, + }), + project: Joi.string() + .pattern(PROJECT_FILE_NAME_REGEX) + .required() + .messages({ 'string.pattern.base': HDB_ERROR_MSGS.BAD_PROJECT_NAME }), + activation_spec: Joi.object().required(), + deployment_timeout: Joi.number().min(0).optional(), + }).unknown(false); + + return validator.validateBySchema(req, phaseSchema); +} + /** * Validate revert_component requests — swap a component's live version back to its retained previous * version. No build inputs (nothing is fetched or installed); just the project, an optional restart, @@ -522,7 +549,7 @@ function deployComponentValidator(req) { * @param req * @returns {*} */ -function revertComponentValidator(req) { +function _revertComponentValidator(req) { const revertSchema = Joi.object({ project: Joi.string() .pattern(PROJECT_FILE_NAME_REGEX) diff --git a/config/configUtils.ts b/config/configUtils.ts index 7fa1b18856..d4ba91906f 100644 --- a/config/configUtils.ts +++ b/config/configUtils.ts @@ -1202,7 +1202,7 @@ export async function addConfig(topLevelElement, values) { atomicWriteFile(getConfigFilePath(), String(configDoc)); } -export function deleteConfigFromFile(param: string) { +export function deleteConfigFromFile(param: Array) { const configFilePath = getConfigFilePath(hdbUtils.getPropsFilePath()); const configDoc = parseYamlDoc(configFilePath); configDoc.deleteIn(param); diff --git a/integrationTests/deploy/deploy-tracking-events.test.ts b/integrationTests/deploy/deploy-tracking-events.test.ts index 49a290fc65..7e86b14044 100644 --- a/integrationTests/deploy/deploy-tracking-events.test.ts +++ b/integrationTests/deploy/deploy-tracking-events.test.ts @@ -206,7 +206,7 @@ suite('Deployment tracking — events + SSE', (ctx: ContextWithHarper) => { // Poll list_deployments until we see the in-flight row. The recorder row is created // after the multipart body is fully received (which is fast for a tiny fixture). - const TERMINAL = new Set(['success', 'failed', 'rolled_back']); + const TERMINAL = new Set(['success', 'failed', 'rolled_back', 'staged']); let inFlightId: string | null = null; for (let i = 0; i < 15 && !inFlightId; i++) { await sleep(300); diff --git a/integrationTests/deploy/deploy-tracking-peer-branch.test.ts b/integrationTests/deploy/deploy-tracking-peer-branch.test.ts index 81fc25e8ed..214d15be96 100644 --- a/integrationTests/deploy/deploy-tracking-peer-branch.test.ts +++ b/integrationTests/deploy/deploy-tracking-peer-branch.test.ts @@ -1,21 +1,18 @@ /** - * Deployment tracking — peer-side branch. + * Deployment tracking — peer-operation authorization boundary. * - * In a real multi-node deploy, the origin strips `req.payload` before `replicateOperation` - * and the peer reads the tarball from the replicated `hdb_deployment.payload_blob` row - * attribute instead. This test exercises that **peer-side branch** in isolation on a - * single node by: + * In a real multi-node deploy, the origin sends a private `component_deploy_phase` + * operation and the peer reads the tarball from the replicated + * `hdb_deployment.payload_blob` row. The authorization-bypass context that admits that + * operation exists only around trusted replication dispatch, so an HTTP caller must not + * be able to impersonate a peer with the legacy `_deploymentId` marker. This test: * * 1. Doing a normal deploy to populate an `hdb_deployment` row with a `payload_blob`. - * 2. Submitting a second `deploy_component` operation with `_deploymentId` set to that - * row's id and **no** `payload` field — the same shape origin produces for peers. - * 3. Asserting the deploy completes successfully — meaning the peer-side branch in - * `deployComponent` found the row, streamed `payload_blob`, and ran prepare/install/load - * from the blob bytes. + * 2. Submitting a second public `deploy_component` operation with `_deploymentId` set. + * 3. Asserting validation rejects the caller-controlled internal marker. * - * The true 3-node test (verifies BLOB_CHUNK replication actually delivers the row to - * peers, and that `peer_results` is populated) lives in harper-pro, where the actual - * `replicateOperation` is implemented. This OSS test only verifies the handler wiring. + * The true 3-node test (including the trusted peer operation, BLOB_CHUNK delivery, and + * `peer_results`) lives in harper-pro, where `replicateOperation` is implemented. */ import { suite, test, before, after } from 'node:test'; import { ok, strictEqual } from 'node:assert'; @@ -86,7 +83,7 @@ async function callOperation( return { status: res.status, body: parsed, rawText: text }; } -suite('Deployment tracking — peer-side branch', (ctx: ContextWithHarper) => { +suite('Deployment tracking — peer-operation authorization boundary', (ctx: ContextWithHarper) => { let fixtureDir: string; let seedDeploymentId: string; @@ -134,46 +131,14 @@ suite('Deployment tracking — peer-side branch', (ctx: ContextWithHarper) => { ok(got.body.payload_hash, 'seed row should have a sha256 payload_hash'); }); - // On Bun the deploy hangs after extraction when reading a Web ReadableStream from a - // file-backed blob inside the same Harper process — same code passes on Node v22/v24 - // across Linux and Windows. Skipping for now; the harper-pro 3-node cluster test - // (HarperFast/harper-pro#221) covers the same code path end-to-end with real replication. - const skipOnBun = process.env.HARPER_RUNTIME === 'bun'; - test( - 'peer-side branch: deploy_component with _deploymentId + no payload uses the row blob', - { skip: skipOnBun }, - async () => { - // Simulate the operation shape origin produces for peers via `replicateOperation`: - // `_deploymentId` set, no `payload`, no multipart. The handler should detect this is a - // replicated execution and source the tarball from the row's payload_blob. - const peerProject = 'peer-branch-replay-application'; - const response = await callOperation(ctx, { - operation: 'deploy_component', - project: peerProject, - restart: false, - _deploymentId: seedDeploymentId, - }); - strictEqual(response.status, 200, `peer-side deploy should succeed; got ${response.status}: ${response.rawText}`); - - // Confirm the component was actually written on disk (peer code path ran extraction - // from the row's payload_blob and not from a missing req.payload). - const fetched = await fetch(`${ctx.harper.operationsAPIURL}/${peerProject}/`, { - headers: { - Authorization: - 'Basic ' + Buffer.from(`${ctx.harper.admin.username}:${ctx.harper.admin.password}`).toString('base64'), - }, - }); - // The component exposes nothing routable, but a 404 from the component (vs. a 503/connection - // failure) confirms it loaded — Harper only routes to deployed component names. - ok( - fetched.status === 404 || fetched.status === 200, - `expected component to be reachable (any 200/404 from the loaded component), got ${fetched.status}` - ); - } - ); - - // Note: the bogus-_deploymentId-id timeout case isn't covered here because the - // awaitDeploymentRow 120s default would balloon test time. The timeout path (and the - // per-deploy deployment_timeout override) is exercised by the unit tests for - // awaitDeploymentRow directly. + test('public deploy_component rejects the legacy _deploymentId peer marker', async () => { + const response = await callOperation(ctx, { + operation: 'deploy_component', + project: 'peer-branch-replay-application', + restart: false, + _deploymentId: seedDeploymentId, + }); + strictEqual(response.status, 400, `internal marker should be rejected; got: ${response.rawText}`); + strictEqual(response.body.error, "'_deploymentId' is not allowed"); + }); }); diff --git a/json/systemSchema.json b/json/systemSchema.json index 6f44989b5b..b37eff9001 100644 --- a/json/systemSchema.json +++ b/json/systemSchema.json @@ -424,6 +424,9 @@ { "attribute": "rollback_of" }, + { + "attribute": "activation_spec" + }, { "attribute": "error" }, diff --git a/resources/registrationDeprecated.ts b/resources/registrationDeprecated.ts index 39b140b176..173cbaf90f 100644 --- a/resources/registrationDeprecated.ts +++ b/resources/registrationDeprecated.ts @@ -4,5 +4,6 @@ export function getRegistrationInfo() { return { version: packageJson.version, deprecated: true, + capabilities: { componentDeployTwoPhase: 1 }, }; } diff --git a/server/serverHelpers/operationAuthorizationState.ts b/server/serverHelpers/operationAuthorizationState.ts new file mode 100644 index 0000000000..5ce0794667 --- /dev/null +++ b/server/serverHelpers/operationAuthorizationState.ts @@ -0,0 +1,11 @@ +import { AsyncLocalStorage } from 'node:async_hooks'; + +const operationAuthorizationState = new AsyncLocalStorage(); + +export function runWithOperationAuthorizationBypass(bypassAuth: boolean, callback: () => T): T { + return operationAuthorizationState.run(bypassAuth === true, callback); +} + +export function isOperationAuthorizationBypassed(): boolean { + return operationAuthorizationState.getStore() === true; +} diff --git a/server/serverHelpers/serverHandlers.js b/server/serverHelpers/serverHandlers.js index 2e112b5424..7f8c785a9f 100644 --- a/server/serverHelpers/serverHandlers.js +++ b/server/serverHelpers/serverHandlers.js @@ -33,7 +33,6 @@ const { ProgressEmitter, createSSEResponseStream } = require('./progressEmitter. // the client disconnects (the emitter's abort signal), so subscribers see new lines live. const SSE_PROGRESS_OPERATIONS = new Set([ terms.OPERATIONS_ENUM.DEPLOY_COMPONENT, - terms.OPERATIONS_ENUM.REVERT_COMPONENT, terms.OPERATIONS_ENUM.GET_DEPLOYMENT, terms.OPERATIONS_ENUM.READ_LOG, ]); diff --git a/server/serverHelpers/serverUtilities.ts b/server/serverHelpers/serverUtilities.ts index b7492d19de..3f932373d0 100644 --- a/server/serverHelpers/serverUtilities.ts +++ b/server/serverHelpers/serverUtilities.ts @@ -47,6 +47,7 @@ import { getRemoteOperationFunction, setLocalOperationDispatch, } from './registeredOperations.ts'; +import { runWithOperationAuthorizationBypass } from './operationAuthorizationState.ts'; const pSearchSearch = util.promisify(search.search); let pEvaluateSql: (sql: string) => Promise; @@ -309,9 +310,12 @@ _assignPackageExport('operation', operation); */ export function operation(operation: OperationRequestBody, context: Context, authorize: boolean) { operation.hdb_user = context?.user; - operation.bypass_auth = !authorize; - const operation_function = chooseOperation(operation); - return processLocalTransaction({ body: operation }, operation_function); + const bypassAuth = !authorize; + operation.bypass_auth = bypassAuth; + return runWithOperationAuthorizationBypass(bypassAuth, () => { + const operation_function = chooseOperation(operation); + return processLocalTransaction({ body: operation }, operation_function); + }); } interface Transaction { @@ -547,8 +551,8 @@ function initializeOperationFunctionMap(): Map { assert.strictEqual(deploy.options.headers.Accept, 'text/event-stream'); assert.strictEqual(result.success, true); }); + + it('fails closed on every staged-deploy control against a target without two-phase capability', async () => { + const originalExit = process.exit; + const originalConsoleError = console.error; + const errors = []; + const calls = []; + process.exit = (code) => { + throw new ProcessExitSignal(code); + }; + console.error = (...args) => errors.push(args.join(' ')); + commonUtilsModule.httpRequest = async (options, req) => { + calls.push({ options, req }); + return { statusCode: 200, body: JSON.stringify({ version: '5.1.7' }) }; + }; + try { + for (const request of [ + { package: '@scope/widget', activate: false, _cliVerb: 'stage' }, + { package: '@scope/widget', activate: false }, + { deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa' }, + { package: '@scope/widget', two_phase: true }, + ]) { + await assert.rejects( + cliOperationsModule.cliOperations( + { + operation: 'deploy_component', + project: 'widget', + target: 'example.com', + ...request, + }, + true + ), + ProcessExitSignal + ); + } + } finally { + process.exit = originalExit; + console.error = originalConsoleError; + } + + assert.deepStrictEqual( + calls.map(({ req }) => req.operation), + Array(4).fill('registration_info'), + 'only one capability probe per request reached the target' + ); + assert.match(errors.join('\n'), /does not advertise staged-deploy support/); + }); + + it('renders stage phase events and strips its CLI-only verb marker', async () => { + const calls = []; + const rendered = []; + const originalRenderEvent = DeployRenderer.prototype.renderEvent; + DeployRenderer.prototype.renderEvent = function (message) { + rendered.push(message.event); + }; + commonUtilsModule.httpRequest = async (options, req) => { + calls.push({ options, req }); + if (req.operation === 'registration_info') { + return { + statusCode: 200, + body: JSON.stringify({ + version: '5.2.0', + capabilities: { componentDeployTwoPhase: 1 }, + }), + }; + } + return Object.assign( + Readable.from([ + 'event: phase\ndata: {"phase":"stage","status":"start"}\n\n', + 'event: done\ndata: {"result":{"staged":true}}\n\n', + ]), + { statusCode: 200, headers: { 'content-type': 'text/event-stream' } } + ); + }; + let result; + try { + result = await cliOperationsModule.cliOperations( + { + operation: 'deploy_component', + project: 'widget', + package: '@scope/widget', + activate: false, + _cliVerb: 'stage', + target: 'example.com', + }, + true + ); + } finally { + DeployRenderer.prototype.renderEvent = originalRenderEvent; + } + + const deploy = calls.at(-1); + assert.strictEqual(deploy.req._cliVerb, undefined); + assert.strictEqual(deploy.req.activate, false); + assert.deepStrictEqual(rendered, ['phase', 'done']); + assert.strictEqual(result.staged, true); + }); + + it('defaults the activate project from the current directory', async () => { + const calls = []; + const projectDir = path.join(testDir, 'activate-project'); + fs.ensureDirSync(projectDir); + const priorCwd = process.cwd(); + commonUtilsModule.httpRequest = async (options, req) => { + calls.push({ options, req }); + if (req.operation === 'registration_info') { + return { + statusCode: 200, + body: JSON.stringify({ + version: '5.2.0', + capabilities: { componentDeployTwoPhase: 1 }, + }), + }; + } + return Object.assign(Readable.from(['event: done\ndata: {"result":{"activated":true}}\n\n']), { + statusCode: 200, + headers: { 'content-type': 'text/event-stream' }, + }); + }; + try { + process.chdir(projectDir); + await cliOperationsModule.cliOperations( + { + operation: 'deploy_component', + deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', + _cliVerb: 'activate', + target: 'example.com', + }, + true + ); + } finally { + process.chdir(priorCwd); + } + + assert.strictEqual(calls.at(-1).req.project, 'activate-project'); + }); }); describe('"Harper is not running" messaging (harper#658)', () => { diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index 7da26cb21a..6cc1ebcc37 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -1,13 +1,6 @@ 'use strict'; -// Operation-level tests for the two-phase deploy handlers: stage_component, activate_component, and -// deploy_component (both the two-phase default and the two_phase:false one-shot fallback). These run -// the real handlers against a real temp filesystem with a real tarball payload — no stubbing — in the -// deployStaging.test.js style (AGENTS.md: new tests use plain `assert` against real modules, no -// sinon/rewire). Payload deploys are used throughout so nothing reaches the component loader (a -// `package` deploy's protected-name guard would), and no test requests a restart. - -const assert = require('node:assert'); +const assert = require('node:assert/strict'); const fs = require('node:fs/promises'); const { existsSync } = require('node:fs'); const os = require('node:os'); @@ -19,398 +12,686 @@ const testUtils = require('../testUtils.js'); testUtils.preTestPrep(); const operations = require('#src/components/operations'); -const { DEPLOY_STAGING_DIR, Application, stageApplication } = require('#src/components/Application'); +const { DEPLOY_STAGING_DIR, DEPLOY_ACTIVATION_DIR, discardStagedApplication } = require('#src/components/Application'); const { restartNeeded, resetRestartNeeded } = require('#src/components/requestRestart'); const { server } = require('#src/server/Server'); const { databases } = require('#src/resources/databases'); -const { SYSTEM_TABLE_NAMES } = require('#src/utility/hdbTerms'); -const { getConfigPath, getConfiguration, getConfigFilePath } = require('#src/config/configUtils'); -const { CONFIG_PARAMS } = require('#src/utility/hdbTerms'); +const { SYSTEM_TABLE_NAMES, CONFIG_PARAMS } = require('#src/utility/hdbTerms'); +const { getConfigPath, readConfigFile } = require('#src/config/configUtils'); +const environment = require('#src/utility/environment/environmentManager'); +const { runWithOperationAuthorizationBypass } = require('#src/server/serverHelpers/operationAuthorizationState'); +const manageThreads = require('#src/server/threads/manageThreads'); -const DEPLOYMENT_TABLE = SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME; const COMPONENTS_ROOT = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); +const DEPLOYMENT_TABLE = SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME; -// Pack a directory's CONTENTS into a gzipped tar Buffer, the shape a deploy payload takes. -function packDirectory(dir) { +function packDirectory(directory) { return new Promise((resolve, reject) => { const chunks = []; tarfs - .pack(dir) + .pack(directory) .pipe(zlib.createGzip()) - .on('data', (c) => chunks.push(c)) + .on('data', (chunk) => chunks.push(chunk)) .on('end', () => resolve(Buffer.concat(chunks))) .on('error', reject); }); } -// A component source that already contains node_modules, so installApplication short-circuits and no -// npm/network is needed. -async function makeComponentPayload(marker) { - const src = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-op-src-')); - await fs.writeFile(path.join(src, 'package.json'), JSON.stringify({ name: 'op-fixture', version: '1.0.0' })); - await fs.writeFile(path.join(src, 'index.js'), `module.exports = ${JSON.stringify(marker)};\n`); - await fs.mkdir(path.join(src, 'node_modules'), { recursive: true }); - await fs.writeFile(path.join(src, 'node_modules', '.marker'), marker); - const payload = await packDirectory(src); - await fs.rm(src, { recursive: true, force: true }); +async function makePayload(marker, version = marker, withNodeModules = true) { + const source = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-op-')); + await fs.writeFile(path.join(source, 'package.json'), JSON.stringify({ name: 'phase-op', version })); + await fs.writeFile(path.join(source, 'index.js'), `module.exports = ${JSON.stringify(marker)};\n`); + if (withNodeModules) await fs.mkdir(path.join(source, 'node_modules'), { recursive: true }); + const payload = await packDirectory(source); + await fs.rm(source, { recursive: true, force: true }); return payload; } -const readIndex = (dir) => fs.readFile(path.join(dir, 'index.js'), 'utf8'); - -// componentLoader reaches the private `@harperfast/skills` dependency, which is absent from some local -// checkouts. Only the `package`-deploy path touches it (via the protected-core-name guard), so probe -// once and let that one test skip rather than fail on a missing dependency. -function componentLoaderAvailable() { - try { - require('#src/components/componentLoader'); - return true; - } catch { - return false; - } -} - -describe('deploy operations: stage_component / activate_component / deploy_component', function () { +describe('deploy_component two-phase orchestration', function () { this.timeout(30_000); + const rows = new Map(); + let priorTable; + let priorReplicate; + let priorSafeMode; + let sequence = 0; + const names = []; before(async () => { - await fs.mkdir(COMPONENTS_ROOT, { recursive: true }); + priorSafeMode = process.env.HARPER_SAFE_MODE; + process.env.HARPER_SAFE_MODE = 'true'; + // The first component operation completes lazy server initialization, which replaces + // databases.system. Run it before installing the table seam used by these tests. + const warmupProject = name(); + const warmup = await operations.deployComponent({ + project: warmupProject, + payload: await makePayload('warmup'), + activate: false, + }); + await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, warmup.deployment_id), { + recursive: true, + force: true, + }); + if (!databases.system) databases.system = {}; + priorTable = databases.system[DEPLOYMENT_TABLE]; }); - let counter = 0; - const names = []; - function freshName() { - const name = `op_test_${process.pid}_${counter++}`; - names.push(name); - return name; - } + beforeEach(() => { + resetRestartNeeded(); + rows.clear(); + databases.system[DEPLOYMENT_TABLE] = { + async get(id) { + return rows.get(id); + }, + async put(row) { + rows.set(row.deployment_id, { ...row }); + }, + async patch(id, partial) { + const row = rows.get(id); + if (row) rows.set(id, { ...row, ...partial }); + }, + async *search(conditions = []) { + for (const row of rows.values()) { + if (conditions.every((condition) => row[condition.attribute] === condition.value)) yield row; + } + }, + }; + priorReplicate = server.replication.replicateOperation; + server.replication.replicateOperation = async () => ({ replicated: [] }); + }); + + afterEach(() => { + server.replication.replicateOperation = priorReplicate; + resetRestartNeeded(); + }); - // Sweep any live dirs, staging, previous, and aside created by the suite. after(async () => { + if (priorSafeMode === undefined) delete process.env.HARPER_SAFE_MODE; + else process.env.HARPER_SAFE_MODE = priorSafeMode; + if (priorTable === undefined) delete databases.system[DEPLOYMENT_TABLE]; + else databases.system[DEPLOYMENT_TABLE] = priorTable; for (const name of names) await fs.rm(path.join(COMPONENTS_ROOT, name), { recursive: true, force: true }); await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR), { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-previous'), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR), { recursive: true, force: true }); await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-aside'), { recursive: true, force: true }); - // Deploying a genuinely-new component flips the process-wide restart-needed buffer on - // (harper#674, via deployComponent's requestRestart() scoping). That buffer is shared across - // the whole mocha process, so restore it or a later test asserting a pristine buffer - // (e.g. requestRestart.test.js) fails on ordering alone. - resetRestartNeeded(); }); - // Each test below deploys fresh components, and deploying a never-live component flips the - // process-wide restart-needed buffer (harper#674). Start every test from a known-clean buffer so the - // restartNeeded() assertions below reflect only that test's own deploy, not a prior test's leak. - beforeEach(() => resetRestartNeeded()); + function name() { + const value = `phase_op_${process.pid}_${sequence++}`; + names.push(value); + return value; + } - it('deploy_component({activate:false}) stages into the hidden dir without going live, and returns a deployment_id', async () => { - const name = freshName(); - const res = await operations.deployComponent({ - project: name, - payload: await makeComponentPayload('op-staged'), + it('stages without touching live and records an immutable activation specification', async () => { + const project = name(); + const result = await operations.deployComponent({ + project, + payload: await makePayload('1.0.0'), activate: false, }); - assert.strictEqual(res.staged, true); - assert.strictEqual(res.project, name); - assert.strictEqual(typeof res.deployment_id, 'string'); + assert.equal(result.staged, true); + assert.match(result.deployment_id, /^[0-9a-f-]{36}$/i); + assert.equal(existsSync(path.join(COMPONENTS_ROOT, project)), false); + assert.equal(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id, project)), true); + const row = rows.get(result.deployment_id); + assert.ok(row, `deployment row missing; present ids: ${Array.from(rows.keys()).join(', ')}`); + assert.equal(row.status, 'staged'); + assert.deepEqual(row.activation_spec, { + project, + package: null, + install_command: null, + install_timeout: null, + install_allow_scripts: null, + urlPath: null, + host: null, + credentials: null, + force: false, + }); + }); + + it('uses a no-custody literal registry token for the origin install without recording it', async () => { + const project = name(); + const token = 'transient-origin-token'; + const installCommand = + `node -e "const fs=require('fs');` + + `const value=fs.readFileSync(process.env.npm_config_userconfig||process.env.NPM_CONFIG_USERCONFIG,'utf8');` + + `if(!value.includes('//registry.example.com/:_authToken='))process.exit(7);` + + `fs.writeFileSync('credential-seen','yes')"`; + const result = await operations.deployComponent({ + project, + payload: await makePayload('credential-origin', '6.0.0', false), + install_command: installCommand, + credentials: [{ registry: 'https://registry.example.com', token }], + activate: false, + }); - const stagedDir = path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, res.deployment_id, name); - assert.ok(existsSync(path.join(stagedDir, 'index.js')), 'component was built into the staging dir'); - assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, name)), false, 'staging did not touch the live path'); + const stagedPath = path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id, project); + assert.equal(await fs.readFile(path.join(stagedPath, 'credential-seen'), 'utf8'), 'yes'); + assert.equal(rows.get(result.deployment_id).activation_spec.credentials, null); + assert.doesNotMatch(JSON.stringify(rows.get(result.deployment_id)), new RegExp(token)); }); - it('deploy_component({deployment_id}) takes a prior stage live', async () => { - const name = freshName(); + it('activates only a staged row owned by the requested project', async () => { + const project = name(); const staged = await operations.deployComponent({ - project: name, - payload: await makeComponentPayload('op-activated'), + project, + payload: await makePayload('2.0.0'), activate: false, }); - const res = await operations.deployComponent({ project: name, deployment_id: staged.deployment_id }); - - assert.strictEqual(res.activated, true); - assert.strictEqual(res.project, name); - assert.strictEqual(res.deployment_id, staged.deployment_id); - const liveDir = path.join(COMPONENTS_ROOT, name); - assert.ok(existsSync(liveDir), 'live component dir now exists'); - assert.match(await readIndex(liveDir), /op-activated/); - // A restart was not requested, so the message must not claim one. - assert.doesNotMatch(res.message, /restart/i); - // ...but this component was never live before, so activating it without a restart must mark one - // as required (harper#674) — the activate-existing leg of the two-phase restart-required fix. The - // message assertion above can't see this: the string never mentions "restart" on the no-restart - // path whether or not the marking runs, so assert the flag directly. - assert.strictEqual( - restartNeeded(), - true, - 'activating a never-live component without restart marks a restart required' + + await assert.rejects( + operations.deployComponent({ project: `${project}_other`, deployment_id: staged.deployment_id }), + /not a staged deployment/ ); + const activated = await operations.deployComponent({ project, deployment_id: staged.deployment_id }); + + assert.equal(activated.activated, true); + assert.equal(rows.get(staged.deployment_id).status, 'success'); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /2.0.0/); }); - // The activate-existing path fans the activate out to peers just like the two-phase activate phase, - // and replicateOperation never rejects on a per-peer failure — failures surface only as 'failed' - // entries. Without the shared activate gate this path returned 2xx {activated:true} while part of the - // cluster stayed on the old version, making revert_on_failure/ignore_replication_errors no-ops here. - describe('deploy_component({deployment_id}) peer-failure gate', () => { - let priorReplicate; - // Swap in a replicator that reports one failed peer (the repo's property-swap pattern; no - // sinon/rewire per AGENTS.md). Restored after each test. - function stubFailedPeer() { - priorReplicate = server.replication.replicateOperation; - server.replication.replicateOperation = async () => ({ - replicated: [{ node: 'peer-a', status: 'failed', reason: 'no staged build found' }], - }); - } - afterEach(() => { - if (priorReplicate) server.replication.replicateOperation = priorReplicate; - priorReplicate = undefined; + it('does not let a duplicate activation undo the winning activation state', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('duplicate-winner', '6.0.0'), + activate: false, }); - async function stageOnly(name, marker) { - const staged = await operations.deployComponent({ - project: name, - payload: await makeComponentPayload(marker), - activate: false, - }); - return staged.deployment_id; - } + const outcomes = await Promise.allSettled([ + operations.deployComponent({ project, deployment_id: staged.deployment_id }), + operations.deployComponent({ project, deployment_id: staged.deployment_id }), + ]); - // End-to-end cover for the package-identifier recovery: stage a `package` deploy with - // `activate: false`, then activate it by id with no `package` on the request (what `harper activate` - // sends) and assert root config is persisted AND the recovered identifier reaches peers. A `file:` - // tarball package needs no network and no payload blob — extraction reads the tarball directly — so - // a mock deployment table is enough to drive the whole path. - it('recovers a staged package deploy: persists root config and fans the package out to peers', async function () { - // Unlike the payload deploys used elsewhere in this file, a `package` deploy runs the - // protected-core-name guard, which requires componentLoader and so pulls in the private - // `@harperfast/skills` dependency. That isn't installed in every local checkout, so skip there - // instead of failing on the environment; CI installs it and runs this for real. - if (!componentLoaderAvailable()) return this.skip(); - const name = freshName(); - const tgzDir = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-op-pkg-')); - const tgzPath = path.join(tgzDir, 'component.tgz'); - await fs.writeFile(tgzPath, await makeComponentPayload('pkg-staged')); - const packageId = `file:${tgzPath}`; - - const rows = new Map(); - const priorTable = databases.system?.[DEPLOYMENT_TABLE]; - if (!databases.system) databases.system = {}; - databases.system[DEPLOYMENT_TABLE] = { - async get(id) { - return rows.get(id); - }, - async put(row) { - rows.set(row.deployment_id, { ...row }); - }, - async patch(id, partial) { - const existing = rows.get(id); - if (existing) rows.set(id, { ...existing, ...partial }); - }, - async *search(conditions = []) { - for (const row of rows.values()) if (conditions.every((c) => row[c.attribute] === c.value)) yield row; - }, - }; - const configBackup = await fs.readFile(getConfigFilePath(), 'utf8'); - - try { - const staged = await operations.deployComponent({ project: name, package: packageId, activate: false }); - assert.strictEqual(staged.staged, true, 'the package deploy staged without going live'); - assert.strictEqual( - rows.get(staged.deployment_id)?.package_identifier, - packageId, - 'the stage recorded the package identifier on the row — the source the activate recovers from' - ); - - let fannedOut; - priorReplicate = server.replication.replicateOperation; - server.replication.replicateOperation = async (op) => { - fannedOut = op; - return {}; - }; - - // No `package` here — exactly what `harper activate project=… deployment_id=…` sends. - await operations.deployComponent({ project: name, deployment_id: staged.deployment_id }); - - assert.strictEqual( - fannedOut?.package, - packageId, - 'the recovered identifier rides the activate sub-op, so peers persist root config too' - ); - assert.strictEqual( - getConfiguration()[name]?.package, - packageId, - 'the origin persisted the package reference to root config' - ); - } finally { - await fs.writeFile(getConfigFilePath(), configBackup); - if (priorTable === undefined) delete databases.system[DEPLOYMENT_TABLE]; - else databases.system[DEPLOYMENT_TABLE] = priorTable; - await fs.rm(tgzDir, { recursive: true, force: true }); - } + assert.equal(outcomes.filter((outcome) => outcome.status === 'fulfilled').length, 1); + assert.equal(outcomes.filter((outcome) => outcome.status === 'rejected').length, 1); + assert.equal(rows.get(staged.deployment_id).status, 'success'); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /duplicate-winner/); + }); + + it('rejects fresh build or routing input on activate-by-id', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('3.0.0'), + activate: false, }); - it('rejects (does not report success) when a peer fails to activate', async () => { - const name = freshName(); - const deploymentId = await stageOnly(name, 'gate-fail'); - stubFailedPeer(); - await assert.rejects( - () => operations.deployComponent({ project: name, deployment_id: deploymentId }), - /failed to activate on 1 .*peer node\(s\).*peer-a/s, - 'a partially-failed activate must surface as an error, not a 2xx success' - ); + await assert.rejects( + operations.deployComponent({ + project, + deployment_id: staged.deployment_id, + install_command: 'npm install --evil', + }), + /immutable staged configuration.*install_command/ + ); + }); + + it('stages and activates a full deploy before reporting success', async () => { + const project = name(); + const result = await operations.deployComponent({ + project, + payload: await makePayload('full-deploy', '6.0.0'), }); - it('honors ignore_replication_errors on this path, resolving despite the failed peer', async () => { - const name = freshName(); - const deploymentId = await stageOnly(name, 'gate-ignore'); - stubFailedPeer(); - const res = await operations.deployComponent({ - project: name, - deployment_id: deploymentId, - ignore_replication_errors: true, + assert.match(result.message, /Successfully deployed/); + assert.equal(rows.get(result.deployment_id).status, 'success'); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /full-deploy/); + assert.equal(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id)), false); + assert.equal(restartNeeded(), true, 'a new component activated without restart requires one'); + }); + + it('reclaims an oversized payload only after a full two-phase activation succeeds', async () => { + const project = name(); + const priorMaxSize = environment.get(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE); + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, 1); + try { + const result = await operations.deployComponent({ + project, + payload: await makePayload('reclaimed-full-deploy', '6.0.0'), }); - assert.strictEqual(res.activated, true, 'the opt-out still returns success'); - assert.match(await readIndex(path.join(COMPONENTS_ROOT, name)), /gate-ignore/, 'the origin still went live'); + + const row = rows.get(result.deployment_id); + assert.equal(row.status, 'success'); + assert.equal(row.payload_blob, null); + assert.ok(row.event_log.some((event) => event.event === 'payload_dropped')); + } finally { + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, priorMaxSize); + } + }); + + it('reclaims an oversized retained payload after activate-by-id succeeds', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('reclaimed-staged-deploy', '6.0.0'), + activate: false, + }); + assert.ok(rows.get(staged.deployment_id).payload_blob, 'staged deployment keeps its recovery payload'); + + const priorMaxSize = environment.get(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE); + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, 1); + try { + await operations.deployComponent({ project, deployment_id: staged.deployment_id }); + + const row = rows.get(staged.deployment_id); + assert.equal(row.status, 'success'); + assert.equal(row.payload_blob, null); + assert.ok(row.event_log.some((event) => event.event === 'payload_dropped')); + } finally { + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, priorMaxSize); + } + }); + + it('preserves the legacy one-shot path when explicitly requested', async () => { + const project = name(); + const result = await operations.deployComponent({ + project, + payload: await makePayload('one-shot', '6.0.0'), + two_phase: false, }); + + assert.match(result.message, /Successfully deployed/); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /one-shot/); }); - it('deploy_component (two-phase default) stages then activates end-to-end', async () => { - const name = freshName(); - const res = await operations.deployComponent({ project: name, payload: await makeComponentPayload('op-deployed') }); - - assert.match(res.message, /Successfully deployed/); - assert.strictEqual(typeof res.deployment_id, 'string'); - const liveDir = path.join(COMPONENTS_ROOT, name); - assert.match(await readIndex(liveDir), /op-deployed/, 'component is live after a two-phase deploy'); - // New component deployed without a restart → restart required (harper#674), the origin two-phase leg. - assert.strictEqual(restartNeeded(), true, 'a fresh two-phase deploy without restart marks a restart required'); - // The staged copy was consumed by the swap; its per-deploy staging parent is cleaned up. - assert.strictEqual( - existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, res.deployment_id)), - false, - 'per-deploy staging parent removed after activation' + it('accepts the legacy deployment row marker only from a trusted replicated operation', async () => { + const project = name(); + const payload = await makePayload('trusted-one-shot', '6.0.0'); + const result = await runWithOperationAuthorizationBypass(true, () => + operations.deployComponent({ + project, + payload, + two_phase: false, + _deploymentId: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', + replicated: false, + hdb_user: { name: 'cluster-peer' }, + }) ); + + assert.match(result.message, /Successfully deployed/); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /trusted-one-shot/); }); - it('redeploying an already-live component without restart does NOT mark a restart required', async () => { - // The negative direction of harper#674/#1806: an existing, already-active component's own watcher - // requests any restart a redeploy needs, so deploy_component must stay quiet. Two-phase can't lean - // on extractApplication's in-place check (it builds into a fresh staging dir), so this guards that - // activateApplication correctly reports isNewComponent:false when a live version already exists. - const name = freshName(); - await operations.deployComponent({ project: name, payload: await makeComponentPayload('redeploy-v1') }); - resetRestartNeeded(); // clear the flag the first (new-component) deploy legitimately set - await operations.deployComponent({ project: name, payload: await makeComponentPayload('redeploy-v2') }); - assert.strictEqual( - restartNeeded(), - false, - 'a redeploy of an already-live component must not self-request a restart' + it('fails closed on the preview phase marker even from a trusted peer', async () => { + await assert.rejects( + runWithOperationAuthorizationBypass(true, () => + operations.deployComponent({ + project: name(), + _phase: 'stage', + replicated: false, + hdb_user: { name: 'cluster-peer' }, + }) + ), + /Unsupported legacy component deployment phase/ ); }); - it('peer _phase:activate takes a locally-staged build live and marks a restart for a new component', async () => { - // The peer leg of the fix: a peer applies the fanned-out deploy_component tagged _phase:'activate' - // (deployPhaseActivate), swapping its OWN locally-staged build live. It must mark restart-required - // per node for a genuinely-new component (harper#674) — otherwise a cluster-wide restart:false - // deploy reports restartRequired on the origin only. Stage the build directly (standing in for the - // peer's earlier stage phase), then drive the activate phase through the public op with the - // internal markers, exactly as the replicated fan-out does. - const name = freshName(); - const deploymentId = `peer-activate-${name}`; - const staged = new Application({ - name, - payload: await makeComponentPayload('peer-activated'), - stagingId: deploymentId, + it('fails closed on an activate peer failure before scheduling a restart', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('4.0.0'), + activate: false, }); - await stageApplication(staged); + const phases = []; + server.replication.replicateOperation = async (operation) => { + phases.push(operation.phase); + if (operation.phase === 'activate') { + return { replicated: [{ node: 'peer-a', status: 'failed', reason: 'config write failed' }] }; + } + return { replicated: [] }; + }; - const res = await operations.deployComponent({ - project: name, - _phase: 'activate', - _deploymentId: deploymentId, - restart: false, + await assert.rejects( + operations.deployComponent({ project, deployment_id: staged.deployment_id, restart: true }), + /Split nodes: peer-a.*[Rr]oll forward/s + ); + assert.deepEqual(phases, ['activate'], 'restart phase was never sent after the activation gate failed'); + assert.equal(rows.get(staged.deployment_id).status, 'activating'); + assert.equal(rows.get(staged.deployment_id).completed_at, null); + assert.ok(rows.get(staged.deployment_id).payload_blob, 'payload remains available to repair a split cluster'); + assert.equal(rows.get(staged.deployment_id).peer_results[0].node, 'peer-a'); + }); + + it('records success when restart fails after the activation barrier', async () => { + const project = name(); + const phases = []; + server.replication.replicateOperation = async (operation) => { + phases.push(operation.phase); + return operation.phase === 'restart' + ? { replicated: [{ node: 'peer-a', status: 'failed', reason: 'restart unavailable' }] } + : { replicated: [] }; + }; + const priorRestartWorkers = manageThreads.restartWorkers; + manageThreads.restartWorkers = () => {}; + let deploymentId; + try { + await assert.rejects( + operations + .deployComponent({ + project, + payload: await makePayload('activated-before-restart-failure', '6.0.0'), + restart: true, + }) + .catch((error) => { + deploymentId = error.http_resp_msg?.deployment_id; + throw error; + }), + /restart failed/ + ); + } finally { + manageThreads.restartWorkers = priorRestartWorkers; + } + + assert.deepEqual(phases, ['stage', 'activate', 'restart']); + assert.equal(rows.get(deploymentId).status, 'success'); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /activated-before/); + }); + + it('records peer failures but honors ignore_replication_errors', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('ignored-peer', '6.0.0'), + activate: false, + }); + server.replication.replicateOperation = async () => ({ + replicated: [{ node: 'peer-a', status: 'failed', reason: 'offline' }], }); - assert.strictEqual(res.activated, true, 'peer activate reports the component activated'); - const liveDir = path.join(COMPONENTS_ROOT, name); - assert.match(await readIndex(liveDir), /peer-activated/, 'the locally-staged build is now live'); - assert.strictEqual( - restartNeeded(), - true, - 'peer activate of a never-live component without restart marks a restart required' - ); + const result = await operations.deployComponent({ + project, + deployment_id: staged.deployment_id, + ignore_replication_errors: true, + }); + + assert.equal(result.activated, true); + assert.equal(rows.get(staged.deployment_id).status, 'success'); + assert.equal(rows.get(staged.deployment_id).peer_results[0].status, 'failed'); }); - it('deploy_component with two_phase:false runs the legacy one-shot path', async () => { - const name = freshName(); - const res = await operations.deployComponent({ - project: name, - payload: await makeComponentPayload('op-oneshot'), - two_phase: false, + it('records and surfaces ignored restart failures after the activation gate', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('restart-failure', '6.0.0'), + activate: false, }); + const phases = []; + server.replication.replicateOperation = async (operation) => { + phases.push(operation.phase); + return operation.phase === 'restart' + ? { replicated: [{ node: 'peer-a', status: 'failed', reason: 'restart unavailable' }] } + : { replicated: [] }; + }; + const priorRestartWorkers = manageThreads.restartWorkers; + let localRestarts = 0; + manageThreads.restartWorkers = () => localRestarts++; + let result; + try { + result = await operations.deployComponent({ + project, + deployment_id: staged.deployment_id, + restart: true, + ignore_replication_errors: true, + }); + } finally { + manageThreads.restartWorkers = priorRestartWorkers; + } - assert.match(res.message, /Successfully deployed/); - const liveDir = path.join(COMPONENTS_ROOT, name); - assert.match(await readIndex(liveDir), /op-oneshot/, 'component is live after a one-shot deploy'); + assert.deepEqual(phases, ['activate', 'restart']); + assert.equal(localRestarts, 1); + assert.equal(result.activated, true); + assert.equal(result.failed_peers[0].node, 'peer-a'); + assert.equal(rows.get(staged.deployment_id).peer_results[0].status, 'failed'); }); - it('revert_component swaps the live version back to the previous deployment', async () => { - const name = freshName(); - const liveDir = path.join(COMPONENTS_ROOT, name); - await operations.deployComponent({ project: name, payload: await makeComponentPayload('rev-v1') }); - await operations.deployComponent({ project: name, payload: await makeComponentPayload('rev-v2') }); - assert.match(await readIndex(liveDir), /rev-v2/, 'v2 is live before revert'); + it('uses the row-backed immutable specification for trusted peer phases', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('peer-phase', '6.0.0'), + activate: false, + }); + const row = rows.get(staged.deployment_id); + const componentPath = path.join(COMPONENTS_ROOT, project); + await discardStagedApplication(componentPath, staged.deployment_id); + const executePeerPhase = (phase, activationSpec) => + runWithOperationAuthorizationBypass(true, () => + operations.componentDeployPhase({ + operation: 'component_deploy_phase', + phase, + project, + deployment_id: staged.deployment_id, + activation_spec: activationSpec, + replicated: false, + hdb_user: { name: 'cluster-peer' }, + }) + ); + + await assert.rejects( + executePeerPhase('stage', { ...row.activation_spec, host: 'tampered.example' }), + /immutable activation specification/ + ); + await executePeerPhase('stage', row.activation_spec); + assert.equal(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, staged.deployment_id, project)), true); + await executePeerPhase('activate', row.activation_spec); + assert.match(await fs.readFile(path.join(componentPath, 'index.js'), 'utf8'), /peer-phase/); + assert.equal(rows.get(staged.deployment_id).status, 'activating'); + assert.equal(restartNeeded(), true); + }); - const res = await operations.revertComponent({ project: name }); + it('rebuilds a missing peer stage from the durable deployment payload before activation', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('rebuilt-peer', '6.0.0'), + activate: false, + }); + const row = rows.get(staged.deployment_id); + const componentPath = path.join(COMPONENTS_ROOT, project); + await discardStagedApplication(componentPath, staged.deployment_id); + + await runWithOperationAuthorizationBypass(true, () => + operations.componentDeployPhase({ + operation: 'component_deploy_phase', + phase: 'activate', + project, + deployment_id: staged.deployment_id, + activation_spec: row.activation_spec, + replicated: false, + hdb_user: { name: 'cluster-peer' }, + }) + ); - assert.strictEqual(res.reverted, true); - assert.strictEqual(res.project, name); - assert.match(await readIndex(liveDir), /rev-v1/, 'revert restored v1 to live'); + assert.match(await fs.readFile(path.join(componentPath, 'index.js'), 'utf8'), /rebuilt-peer/); + assert.equal(rows.get(staged.deployment_id).status, 'activating'); }); - it('revert_component rejects a component with no retained previous version', async () => { - const name = freshName(); - await operations.deployComponent({ project: name, payload: await makeComponentPayload('rev-once') }); - await assert.rejects(() => operations.revertComponent({ project: name }), /no previous version is retained/i); + it('waits for the staged row checkpoint when peer activation arrives first', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('lagged-row', '6.0.0'), + activate: false, + }); + const row = rows.get(staged.deployment_id); + rows.set(staged.deployment_id, { ...row, status: 'staging' }); + setImmediate(() => rows.set(staged.deployment_id, { ...rows.get(staged.deployment_id), status: 'staged' })); + + await runWithOperationAuthorizationBypass(true, () => + operations.componentDeployPhase({ + operation: 'component_deploy_phase', + phase: 'activate', + project, + deployment_id: staged.deployment_id, + activation_spec: row.activation_spec, + deployment_timeout: 200, + replicated: false, + hdb_user: { name: 'cluster-peer' }, + }) + ); + + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /lagged-row/); + assert.equal(rows.get(staged.deployment_id).status, 'activating'); }); - it('revert_component requires a project', async () => { - await assert.rejects(() => operations.revertComponent({}), /project/i); + it('recovers a staged package specification for config and peer activation', async () => { + const project = name(); + const tarDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-package-')); + const tarPath = path.join(tarDirectory, 'component.tgz'); + await fs.writeFile(tarPath, await makePayload('package-stage', '6.0.0')); + const packageIdentifier = `file:${tarPath}`; + const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-config-')); + const configPath = path.join(configRoot, 'harper-config.yaml'); + await fs.writeFile(configPath, `rootPath: ${JSON.stringify(configRoot)}\n`); + const priorRootEnv = process.env.ROOTPATH; + const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); + process.env.ROOTPATH = configRoot; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); + try { + const staged = await operations.deployComponent({ + project, + package: packageIdentifier, + activate: false, + }); + let activationOperation; + server.replication.replicateOperation = async (operation) => { + activationOperation = operation; + return { replicated: [] }; + }; + + await operations.deployComponent({ project, deployment_id: staged.deployment_id }); + + assert.equal(readConfigFile()[project].package, packageIdentifier); + assert.equal(activationOperation.operation, 'component_deploy_phase'); + assert.equal(activationOperation.activation_spec.package, packageIdentifier); + } finally { + if (priorRootEnv === undefined) delete process.env.ROOTPATH; + else process.env.ROOTPATH = priorRootEnv; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); + await fs.rm(configRoot, { recursive: true, force: true }); + await fs.rm(tarDirectory, { recursive: true, force: true }); + } }); - // The revert_on_failure fan-out itself needs a live multi-node cluster (harper-pro's replicator) to - // run end-to-end, but its node-targeting is a pure function — and the exact spot that had two - // review-caught bugs (skip failed peers, skip self). Exercise it directly. - describe('selectRevertTargets (revert_on_failure node targeting)', () => { - const nodes = [{ name: 'origin' }, { name: 'peerA' }, { name: 'peerB' }, { name: 'peerC' }]; + it('drop_component invalidates staged rows and removes recovery artifacts', async () => { + const project = name(); + const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-drop-')); + const componentsRoot = path.join(configRoot, 'components'); + const configPath = path.join(configRoot, 'harper-config.yaml'); + const priorRootEnv = process.env.ROOTPATH; + const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); + const priorComponentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); + process.env.ROOTPATH = configRoot; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); + environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, componentsRoot); + await fs.writeFile(configPath, `rootPath: ${JSON.stringify(configRoot)}\n`); + try { + const staged = await operations.deployComponent({ + project, + payload: await makePayload('drop-stage', '6.0.0'), + activate: false, + }); + const activationPath = path.join(componentsRoot, DEPLOY_ACTIVATION_DIR, project, 'interrupted'); + await fs.mkdir(activationPath, { recursive: true }); + await fs.writeFile( + path.join(configRoot, 'harper-application-lock.json'), + JSON.stringify({ applications: { [project]: { package: 'stale-package' } } }) + ); - it('returns activated peers, excluding this node and failed peers', () => { - const failed = [{ node: 'peerB', status: 'failed' }]; - const targets = operations.selectRevertTargets(nodes, failed, 'origin').map((n) => n.name); - assert.deepStrictEqual(targets.sort(), ['peerA', 'peerC'], 'peerB (failed) and origin (self) excluded'); - }); + await operations.dropComponent({ project }); - it('excludes THIS node even when it is present in server.nodes (bidirectional double-revert guard)', () => { - const targets = operations.selectRevertTargets(nodes, [], 'origin').map((n) => n.name); - assert.ok(!targets.includes('origin'), 'self must never receive a self-directed revert'); - assert.deepStrictEqual(targets.sort(), ['peerA', 'peerB', 'peerC']); - }); + assert.equal(rows.get(staged.deployment_id).status, 'failed'); + const deploymentStagePath = path.join(componentsRoot, DEPLOY_STAGING_DIR, staged.deployment_id); + assert.equal( + existsSync(deploymentStagePath), + false, + `staged deployment directory still contains: ${await fs.readdir(deploymentStagePath).catch(() => [])}` + ); + assert.equal(existsSync(path.join(componentsRoot, DEPLOY_ACTIVATION_DIR, project)), false); + const applicationLock = JSON.parse(await fs.readFile(path.join(configRoot, 'harper-application-lock.json'))); + assert.equal(applicationLock.applications[project], undefined); + } finally { + if (priorRootEnv === undefined) delete process.env.ROOTPATH; + else process.env.ROOTPATH = priorRootEnv; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); + environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, priorComponentsRoot); + await fs.rm(configRoot, { recursive: true, force: true }); + } + }); - it('excludes every failed peer (they never activated and are on the correct version)', () => { - const failed = [ - { node: 'peerA', status: 'failed' }, - { node: 'peerC', status: 'failed' }, - ]; - const targets = operations.selectRevertTargets(nodes, failed, 'origin').map((n) => n.name); - assert.deepStrictEqual(targets, ['peerB'], 'only the one activated peer is a revert target'); - }); + it('rejects separated-phase controls on the one-shot fallback', async () => { + await assert.rejects( + operations.deployComponent({ + project: name(), + payload: await makePayload('5.0.0'), + activate: false, + two_phase: false, + }), + /require two-phase deploy/ + ); + }); - it('is safe with empty/undefined nodes and failed lists, and ignores failed entries with no node name', () => { - assert.deepStrictEqual(operations.selectRevertTargets(undefined, undefined, 'origin'), []); - assert.deepStrictEqual(operations.selectRevertTargets([], [{ node: null }], 'origin'), []); - const targets = operations.selectRevertTargets(nodes, [{ node: null }], 'origin').map((n) => n.name); - assert.deepStrictEqual(targets.sort(), ['peerA', 'peerB', 'peerC'], 'a null-node failed entry drops nobody'); - }); + it('rejects an explicit two-phase request when the system database is not replicated', async () => { + const priorReplications = environment.get(CONFIG_PARAMS.REPLICATION_DATABASES); + environment.setProperty(CONFIG_PARAMS.REPLICATION_DATABASES, ['data']); + try { + await assert.rejects( + operations.deployComponent({ + project: name(), + payload: await makePayload('requires-system-replication'), + two_phase: true, + }), + /requires system database replication/ + ); + } finally { + environment.setProperty(CONFIG_PARAMS.REPLICATION_DATABASES, priorReplications); + } + }); + + it('does not trust caller-supplied internal phase markers', async () => { + await assert.rejects( + operations.deployComponent({ + project: name(), + payload: await makePayload('untrusted-replication'), + replicated: false, + two_phase: true, + }), + /requires operation replication/ + ); + await assert.rejects( + operations.deployComponent({ + project: name(), + _deploymentId: '../../escape', + _phase: 'stage', + }), + /is not allowed/ + ); + await assert.rejects( + operations.componentDeployPhase({ + operation: 'component_deploy_phase', + phase: 'discard', + project: name(), + deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', + activation_spec: { project: 'anything' }, + }), + /restricted to authenticated cluster peers/ + ); + await assert.rejects( + runWithOperationAuthorizationBypass(true, () => + operations.componentDeployPhase({ + operation: 'component_deploy_phase', + phase: 'discard', + project: '../escape', + deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', + activation_spec: { project: '../escape' }, + replicated: false, + hdb_user: { name: 'cluster-peer' }, + }) + ), + /project name/i + ); }); }); diff --git a/unitTests/components/deployPhaseValidators.test.js b/unitTests/components/deployPhaseValidators.test.js index 779cbfa32a..e27609c743 100644 --- a/unitTests/components/deployPhaseValidators.test.js +++ b/unitTests/components/deployPhaseValidators.test.js @@ -1,68 +1,58 @@ 'use strict'; -// Validators for the two-phase deploy operations. Uses node:assert (not chai) and loads only -// operationsValidation (Joi + config helpers), so it runs without the private agent dependency that -// componentLoader pulls in. - -const assert = require('node:assert'); +const assert = require('node:assert/strict'); const validator = require('#js/components/operationsValidation'); -// validateBySchema returns a truthy validation error when invalid, undefined when valid. -const ok = (result) => assert.strictEqual(result, undefined, `expected valid, got: ${result && result.message}`); -const rejected = (result) => assert.ok(result, 'expected a validation error'); - -describe('revertComponentValidator', () => { - it('accepts a project-only revert', () => { - ok(validator.revertComponentValidator({ project: 'my_app' })); - }); +const valid = (result) => assert.equal(result, undefined, `expected valid, got: ${result?.message}`); +const invalid = (result) => assert.ok(result, 'expected a validation error'); - it('accepts a deployment_id, restart, and ignore_replication_errors', () => { - ok( - validator.revertComponentValidator({ +describe('deployComponentValidator two-phase controls', () => { + it('accepts stage-and-stop and UUID activation', () => { + valid(validator.deployComponentValidator({ project: 'my_app', activate: false })); + valid( + validator.deployComponentValidator({ project: 'my_app', - deployment_id: 'abc-123', - restart: 'rolling', - ignore_replication_errors: true, + deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', }) ); }); - it('requires a project', () => { - rejected(validator.revertComponentValidator({ deployment_id: 'abc-123' })); + it('requires deployment_id to be a UUID and rejects path traversal', () => { + for (const deploymentId of ['abc-123', '../evil', 'dep/../..', '.', '..']) { + invalid(validator.deployComponentValidator({ project: 'my_app', deployment_id: deploymentId })); + } }); - it('rejects an invalid restart value', () => { - rejected(validator.revertComponentValidator({ project: 'my_app', restart: 'sideways' })); + it('rejects caller-controlled internal phase markers', () => { + invalid(validator.deployComponentValidator({ project: 'my_app', _deploymentId: 'x' })); + invalid(validator.deployComponentValidator({ project: 'my_app', _phase: 'stage' })); }); -}); -describe('deployComponentValidator two-phase props (activate / deployment_id / flags)', () => { - it('accepts activate: false (stage-and-stop)', () => { - ok(validator.deployComponentValidator({ project: 'my_app', activate: false })); + it('rejects retry-unsafe automatic rollback', () => { + invalid(validator.deployComponentValidator({ project: 'my_app', revert_on_failure: true })); }); - it('accepts a deployment_id (activate an existing stage)', () => { - ok( - validator.deployComponentValidator({ project: 'my_app', deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa' }) - ); - }); - - it('rejects a path-traversal deployment_id (it becomes a staging-dir path segment)', () => { - for (const bad of ['../evil', 'a/b', 'dep/../..', '.', '..']) { - rejected(validator.deployComponentValidator({ project: 'my_app', deployment_id: bad })); - } + it('preserves routing validation', () => { + invalid(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', urlPath: '/a/./b' })); + invalid(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', urlPath: '/a/../b' })); }); +}); - it('accepts revert_on_failure: true', () => { - ok(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', revert_on_failure: true })); - }); +describe('componentDeployPhaseValidator', () => { + const validPhase = { + phase: 'stage', + deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', + project: 'my_app', + activation_spec: { project: 'my_app' }, + }; - it('accepts two_phase: false (legacy opt-out) and true', () => { - ok(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', two_phase: false })); - ok(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', two_phase: true })); + it('accepts a bounded internal phase request', () => { + valid(validator.componentDeployPhaseValidator(validPhase)); }); - it('rejects a non-boolean two_phase', () => { - rejected(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', two_phase: 'yes' })); + it('rejects invalid phases, project traversal, and non-UUID ids', () => { + invalid(validator.componentDeployPhaseValidator({ ...validPhase, phase: 'deploy' })); + invalid(validator.componentDeployPhaseValidator({ ...validPhase, project: '../escape' })); + invalid(validator.componentDeployPhaseValidator({ ...validPhase, deployment_id: '../../escape' })); }); }); diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 696c8c3b64..fa6d55b3a8 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -1,16 +1,11 @@ 'use strict'; -// Unit tests for the two-phase deploy primitives in components/Application.ts: -// stageApplication (build the incoming version into a hidden staging dir, never touching the live -// path), activateApplication (atomically swap the staged copy into the live path), and -// discardStagedApplication (drop an aborted stage). These exercise the real filesystem — no -// componentLoader, no network — so they run without the private agent dependency. - -const assert = require('node:assert'); +const assert = require('node:assert/strict'); const fs = require('node:fs/promises'); const { existsSync } = require('node:fs'); const os = require('node:os'); const path = require('node:path'); +const { randomUUID } = require('node:crypto'); const zlib = require('node:zlib'); const tarfs = require('tar-fs'); @@ -20,311 +15,375 @@ testUtils.preTestPrep(); const { Application, stageApplication, - activateApplication, - revertApplication, + activateStagedApplication, discardStagedApplication, + discardProjectActivationArtifacts, + reconcileStagedApplicationArtifacts, + createApplicationActivationTransaction, + stagedApplicationPath, DEPLOY_STAGING_DIR, - DEPLOY_PREVIOUS_DIR, + DEPLOY_ACTIVATION_DIR, ASIDE_STAGING_DIR, } = require('#src/components/Application'); -const { getConfigPath } = require('#src/config/configUtils'); +const { getConfigPath, readConfigFile } = require('#src/config/configUtils'); const { CONFIG_PARAMS } = require('#src/utility/hdbTerms'); +const environment = require('#src/utility/environment/environmentManager'); const COMPONENTS_ROOT = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); -// Pack a directory's CONTENTS into a gzipped tar Buffer, the shape a deploy payload takes. function packDirectory(dir) { return new Promise((resolve, reject) => { const chunks = []; tarfs .pack(dir) .pipe(zlib.createGzip()) - .on('data', (c) => chunks.push(c)) + .on('data', (chunk) => chunks.push(chunk)) .on('end', () => resolve(Buffer.concat(chunks))) .on('error', reject); }); } -// A minimal component source that already contains node_modules, so installApplication short-circuits -// ("already has node_modules; skipping install") and the test needs no npm/network. -async function makeComponentPayload(marker) { - const src = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-stage-src-')); - await fs.writeFile(path.join(src, 'package.json'), JSON.stringify({ name: 'stage-fixture', version: '1.0.0' })); - await fs.writeFile(path.join(src, 'index.js'), `module.exports = ${JSON.stringify(marker)};\n`); - await fs.mkdir(path.join(src, 'node_modules'), { recursive: true }); - await fs.writeFile(path.join(src, 'node_modules', '.marker'), marker); - const payload = await packDirectory(src); - await fs.rm(src, { recursive: true, force: true }); +async function makeComponentPayload(marker, version = '1.0.0') { + const source = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-stage-src-')); + await fs.writeFile(path.join(source, 'package.json'), JSON.stringify({ name: 'stage-fixture', version })); + await fs.writeFile(path.join(source, 'index.js'), `module.exports = ${JSON.stringify(marker)};\n`); + await fs.mkdir(path.join(source, 'node_modules'), { recursive: true }); + const payload = await packDirectory(source); + await fs.rm(source, { recursive: true, force: true }); return payload; } -async function readMarker(dir) { - return fs.readFile(path.join(dir, 'index.js'), 'utf8'); +async function readMarker(directory) { + return fs.readFile(path.join(directory, 'index.js'), 'utf8'); } -describe('two-phase deploy primitives (stage / activate / discard)', function () { +describe('two-phase component directory transaction', function () { this.timeout(30_000); + let sequence = 0; - before(async () => { - await fs.mkdir(COMPONENTS_ROOT, { recursive: true }); - }); + before(async () => fs.mkdir(COMPONENTS_ROOT, { recursive: true })); - // Each case uses a unique component name so the tests are order-independent and don't collide. - let counter = 0; - function freshApp(payload) { - const name = `stage_test_${process.pid}_${counter++}`; - return new Application({ name, payload }); + function fixtureName() { + return `stage_test_${process.pid}_${sequence++}`; } - it('stageApplication builds into the hidden staging dir and does NOT touch the live path', async () => { - const app = freshApp(await makeComponentPayload('v1')); - - assert.ok(app.stagingDirPath.includes(DEPLOY_STAGING_DIR), 'staging path is under the staging dir'); - assert.strictEqual(app.buildDirPath, app.dirPath, 'build target defaults to the live dir before staging'); - - const stagedPath = await stageApplication(app); + async function cleanup(name) { + await fs.rm(path.join(COMPONENTS_ROOT, name), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + } - assert.strictEqual(stagedPath, app.stagingDirPath); - assert.ok(existsSync(path.join(app.stagingDirPath, 'index.js')), 'component extracted into staging'); - assert.ok(existsSync(path.join(app.stagingDirPath, 'node_modules')), 'node_modules present in staging'); - assert.strictEqual(existsSync(app.dirPath), false, 'live component dir was NOT created by staging'); + it('builds a staged tree without touching the live component', async () => { + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('candidate') }); - await fs.rm(path.dirname(app.stagingDirPath), { recursive: true, force: true }); - }); + const stagedPath = await stageApplication(application, deploymentId); - it('activateApplication swaps the staged copy into the live path atomically', async () => { - const app = freshApp(await makeComponentPayload('v1')); - await stageApplication(app); - await activateApplication(app); - - assert.ok(existsSync(app.dirPath), 'live component dir now exists'); - assert.match(await readMarker(app.dirPath), /v1/, 'live dir holds the staged content'); - assert.strictEqual(existsSync(app.stagingDirPath), false, 'the staged copy was consumed by the swap'); - assert.strictEqual(app.buildDirPath, app.dirPath, 'build target reset to live after activation'); - // No live version existed before the swap, so this is a first deploy. deploy_component reads this - // to mark restartRequired for a never-loaded component (harper#674); staging is always fresh, so - // activate — not extract — is what establishes it on the two-phase path. - assert.strictEqual(app.isNewComponent, true, 'a first-ever activate marks the component new'); - - await fs.rm(app.dirPath, { recursive: true, force: true }); + assert.equal(stagedPath, stagedApplicationPath(application.dirPath, deploymentId)); + assert.match(await readMarker(stagedPath), /candidate/); + assert.equal(existsSync(application.dirPath), false); + await cleanup(name); }); - it('stage → activate replaces an existing live version and moves the old one aside', async () => { - const name = `stage_test_${process.pid}_${counter++}`; - const dirPath = path.join(COMPONENTS_ROOT, name); - // Seed an existing live version. - await fs.mkdir(dirPath, { recursive: true }); - await fs.writeFile(path.join(dirPath, 'index.js'), 'module.exports = "OLD";\n'); - await fs.writeFile(path.join(dirPath, 'leftover.txt'), 'from the old version'); - - const app = new Application({ name, payload: await makeComponentPayload('v2') }); - await stageApplication(app); - await activateApplication(app); - - assert.match(await readMarker(dirPath), /v2/, 'live dir now holds the new version'); - assert.strictEqual(existsSync(path.join(dirPath, 'leftover.txt')), false, 'old-version files are gone from live'); - // A live version existed before the swap, so this is a redeploy, not a first deploy — deploy_component - // must NOT self-request a restart here (harper#1806); the existing component's own watcher does. - assert.strictEqual(app.isNewComponent, false, 'activating over an existing live version marks it not-new'); - - await fs.rm(dirPath, { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + it('atomically activates a staged tree and consumes only that deployment', async () => { + const name = fixtureName(); + const deploymentId = randomUUID(); + const siblingId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('candidate') }); + await stageApplication(application, deploymentId); + application.payload = await makeComponentPayload('sibling'); + await stageApplication(application, siblingId); + + await activateStagedApplication(application, deploymentId); + + assert.match(await readMarker(application.dirPath), /candidate/); + assert.equal(existsSync(stagedApplicationPath(application.dirPath, deploymentId)), false); + assert.equal(existsSync(stagedApplicationPath(application.dirPath, siblingId)), true); + await cleanup(name); }); - it('discardStagedApplication removes the staging tree and leaves the live path untouched', async () => { - const name = `stage_test_${process.pid}_${counter++}`; - const dirPath = path.join(COMPONENTS_ROOT, name); - await fs.mkdir(dirPath, { recursive: true }); - await fs.writeFile(path.join(dirPath, 'index.js'), 'module.exports = "LIVE";\n'); - - const app = new Application({ name, payload: await makeComponentPayload('v3') }); - await stageApplication(app); - assert.ok(existsSync(app.stagingDirPath), 'staged before discard'); + it('rejects a candidate whose staging build did not complete', async () => { + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('incomplete') }); + const stagedPath = await stageApplication(application, deploymentId); + await fs.rm(path.join(path.dirname(stagedPath), '.complete')); - await discardStagedApplication(app); + await assert.rejects(activateStagedApplication(application, deploymentId), /staged build is incomplete/); - assert.strictEqual(existsSync(app.stagingDirPath), false, 'staging tree removed'); - assert.match(await readMarker(dirPath), /LIVE/, 'live version untouched by discard'); - - await fs.rm(dirPath, { recursive: true, force: true }); + assert.equal(existsSync(application.dirPath), false); + assert.match(await readMarker(stagedPath), /incomplete/); + await cleanup(name); }); - it('a failed stage leaves the live path untouched and cleans up its partial staging tree', async () => { - const name = `stage_test_${process.pid}_${counter++}`; - const dirPath = path.join(COMPONENTS_ROOT, name); - await fs.mkdir(dirPath, { recursive: true }); - await fs.writeFile(path.join(dirPath, 'index.js'), 'module.exports = "LIVE";\n'); - - // No payload and no package identifier → extractApplication throws before anything is built. - const app = new Application({ name }); - await assert.rejects(() => stageApplication(app), /payload or package/i); + it('resumes a new-component activation after its durable marker was already created', async () => { + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('resumed') }); + await stageApplication(application, deploymentId); + const activationPath = path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR, name); + await fs.mkdir(activationPath, { recursive: true }); + await fs.writeFile(path.join(activationPath, `.new-${deploymentId}`), '', { mode: 0o600 }); - assert.strictEqual(existsSync(app.stagingDirPath), false, 'partial staging tree removed on failure'); - assert.match(await readMarker(dirPath), /LIVE/, 'live version untouched by a failed stage'); - assert.strictEqual(app.buildDirPath, app.dirPath, 'build target reset to live after a failed stage'); + await activateStagedApplication(application, deploymentId); - await fs.rm(dirPath, { recursive: true, force: true }); + assert.match(await readMarker(application.dirPath), /resumed/); + assert.equal(existsSync(activationPath), false); + await cleanup(name); }); - it('two independent components stage into non-colliding staging dirs', async () => { - const a = freshApp(await makeComponentPayload('A')); - const b = freshApp(await makeComponentPayload('B')); - await Promise.all([stageApplication(a), stageApplication(b)]); - - assert.notStrictEqual(a.stagingDirPath, b.stagingDirPath); - assert.match(await readMarker(a.stagingDirPath), /A/); - assert.match(await readMarker(b.stagingDirPath), /B/); + it('does not report activation failure when only committed-stage cleanup is denied', async function () { + if (process.platform === 'win32' || process.getuid?.() === 0) this.skip(); + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('cleanup-deferred') }); + await stageApplication(application, deploymentId); + const stagingRoot = path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR); + await fs.chmod(stagingRoot, 0o500); + try { + await activateStagedApplication(application, deploymentId); + } finally { + await fs.chmod(stagingRoot, 0o700); + } - await Promise.all([discardStagedApplication(a), discardStagedApplication(b)]); + assert.match(await readMarker(application.dirPath), /cleanup-deferred/); + assert.equal(existsSync(path.join(stagingRoot, deploymentId)), true, 'cleanup remains retryable garbage'); + await cleanup(name); }); - it('stages a `file:` tarball package identifier (the package path, no payload)', async () => { - // Regression for the staging parent dir: extractApplication's `file:`-tarball branch (and the - // npm-pack branch) resolve paths relative to dirname(stagingDirPath), which must exist before - // extraction. A payload-only test never exercises that branch. - const name = `stage_test_${process.pid}_${counter++}`; - const tgzDir = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-stage-tgz-')); - const tgzPath = path.join(tgzDir, 'component.tgz'); - await fs.writeFile(tgzPath, await makeComponentPayload('from-tarball')); - - const app = new Application({ name, packageIdentifier: `file:${tgzPath}` }); - await stageApplication(app); - assert.match(await readMarker(app.stagingDirPath), /from-tarball/, 'tarball extracted into staging'); - assert.strictEqual(existsSync(app.dirPath), false, 'live dir untouched by staging a tarball'); - - await discardStagedApplication(app); - await fs.rm(tgzDir, { recursive: true, force: true }); + it('restores live and preserves the staged tree when persistent activation work fails', async () => { + const name = fixtureName(); + const livePath = path.join(COMPONENTS_ROOT, name); + const deploymentId = randomUUID(); + await fs.mkdir(livePath, { recursive: true }); + await fs.writeFile(path.join(livePath, 'index.js'), 'module.exports = "live";\n'); + const application = new Application({ name, payload: await makeComponentPayload('candidate', '2.0.0') }); + await stageApplication(application, deploymentId); + + await assert.rejects( + activateStagedApplication(application, deploymentId, { + beforeCommit: async () => { + throw new Error('config write failed'); + }, + }), + /config write failed/ + ); + + assert.match(await readMarker(livePath), /live/); + assert.match(await readMarker(stagedApplicationPath(livePath, deploymentId)), /candidate/); + await cleanup(name); }); - it('activate cleanup does NOT sweep a sibling staged build of the same component', async () => { - // Two deploys of the same component staged concurrently share the .deploy-staging/ parent. - // Activating one must not recursively delete that parent and destroy the other's staged build. - const name = `stage_test_${process.pid}_${counter++}`; - const first = new Application({ name, payload: await makeComponentPayload('first') }); - const second = new Application({ name, payload: await makeComponentPayload('second') }); - await stageApplication(first); - await stageApplication(second); - assert.notStrictEqual(first.stagingDirPath, second.stagingDirPath); - - await activateApplication(first); - - assert.match(await readMarker(first.dirPath), /first/, 'first went live'); - assert.ok(existsSync(second.stagingDirPath), 'the sibling staged build survived the activate cleanup'); - - await fs.rm(first.dirPath, { recursive: true, force: true }); - await discardStagedApplication(second); - await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR), { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + it('serializes duplicate activation without ever losing the live directory', async () => { + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('candidate') }); + await stageApplication(application, deploymentId); + + const outcomes = await Promise.allSettled([ + activateStagedApplication(application, deploymentId), + activateStagedApplication(application, deploymentId), + ]); + + assert.equal(outcomes.filter((outcome) => outcome.status === 'fulfilled').length, 1); + assert.equal(outcomes.filter((outcome) => outcome.status === 'rejected').length, 1); + assert.match(await readMarker(application.dirPath), /candidate/); + await cleanup(name); }); - it('activate moves a DANGLING symlink at the live path aside instead of failing EEXIST', async () => { - // A prior `file:`-directory deploy leaves the live path as a symlink; if its target is later - // removed the link dangles. moveDirAside must detect it via lstat (access(F_OK) follows the link - // and reports ENOENT) so the swap replaces it cleanly. - const name = `stage_test_${process.pid}_${counter++}`; - const dirPath = path.join(COMPONENTS_ROOT, name); - await fs.symlink(path.join(os.tmpdir(), `does-not-exist-${process.pid}-${counter}`), dirPath); - assert.strictEqual(existsSync(dirPath), false, 'precondition: the symlink is dangling'); - - const app = new Application({ name, payload: await makeComponentPayload('replaced') }); - await stageApplication(app); - await activateApplication(app); - - const stat = await fs.lstat(dirPath); - assert.strictEqual(stat.isSymbolicLink(), false, 'live path is now a real directory, not the dead link'); - assert.match(await readMarker(dirPath), /replaced/); - - await fs.rm(dirPath, { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + it('snapshots config at commit time so a queued rollback preserves the preceding winner', async () => { + const name = fixtureName(); + const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-activation-config-')); + const priorRootEnv = process.env.ROOTPATH; + const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); + process.env.ROOTPATH = configRoot; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); + await fs.writeFile(path.join(configRoot, 'harper-config.yaml'), `rootPath: ${JSON.stringify(configRoot)}\n`); + const spec = (version, urlPath) => ({ + project: name, + package: `example@${version}`, + install_command: null, + install_timeout: null, + install_allow_scripts: null, + urlPath, + host: null, + credentials: null, + }); + const first = await createApplicationActivationTransaction(name, spec('1.0.0', '/first')); + const second = await createApplicationActivationTransaction(name, spec('2.0.0', '/second')); + try { + await first.commit(); + await second.commit(); + await second.rollback(); + + assert.deepEqual(readConfigFile()[name], { package: 'example@1.0.0', urlPath: '/first' }); + const lock = JSON.parse( + await fs.readFile(path.join(configRoot, 'harper-application-lock.json'), { encoding: 'utf8' }) + ); + assert.deepEqual(lock.applications[name], { package: 'example@1.0.0', urlPath: '/first' }); + } finally { + await second.rollback(); + await first.rollback(); + if (priorRootEnv === undefined) delete process.env.ROOTPATH; + else process.env.ROOTPATH = priorRootEnv; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); + await fs.rm(configRoot, { recursive: true, force: true }); + } }); - // Deploy a version (stage + activate) through the primitives, returning the app. - async function deployVersion(name, marker) { - const app = new Application({ name, payload: await makeComponentPayload(marker) }); - await stageApplication(app); - await activateApplication(app); - return app; - } - - it('activate retains the outgoing version as .deploy-previous/', async () => { - const name = `stage_test_${process.pid}_${counter++}`; - await deployVersion(name, 'v1'); - await deployVersion(name, 'v2'); + it('discard removes only the selected staged tree and leaves live unchanged', async () => { + const name = fixtureName(); + const livePath = path.join(COMPONENTS_ROOT, name); + const deploymentId = randomUUID(); + await fs.mkdir(livePath, { recursive: true }); + await fs.writeFile(path.join(livePath, 'index.js'), 'module.exports = "live";\n'); + const application = new Application({ name, payload: await makeComponentPayload('candidate') }); + await stageApplication(application, deploymentId); - const liveDir = path.join(COMPONENTS_ROOT, name); - const previousDir = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); - assert.match(await readMarker(liveDir), /v2/, 'live is the newest version'); - assert.match(await readMarker(previousDir), /v1/, 'the outgoing version is retained as previous'); + await discardStagedApplication(livePath, deploymentId); - await fs.rm(liveDir, { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR), { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + assert.equal(existsSync(stagedApplicationPath(livePath, deploymentId)), false); + assert.match(await readMarker(livePath), /live/); + await cleanup(name); }); - it('revertApplication swaps live <-> previous, and a second revert rolls forward again', async () => { - const name = `stage_test_${process.pid}_${counter++}`; - await deployVersion(name, 'v1'); - const app = await deployVersion(name, 'v2'); - const liveDir = path.join(COMPONENTS_ROOT, name); - const previousDir = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); + it('drop cleanup removes only the selected component activation artifacts', async () => { + const name = fixtureName(); + const sibling = fixtureName(); + const activationRoot = path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR); + await fs.mkdir(path.join(activationRoot, name, 'old'), { recursive: true }); + await fs.mkdir(path.join(activationRoot, sibling, 'keep'), { recursive: true }); - await revertApplication(app); - assert.match(await readMarker(liveDir), /v1/, 'reverted live back to v1'); - assert.match(await readMarker(previousDir), /v2/, 'the reverted-away v2 is now the previous'); + await discardProjectActivationArtifacts(path.join(COMPONENTS_ROOT, name)); - await revertApplication(app); - assert.match(await readMarker(liveDir), /v2/, 'reverting the revert rolls forward to v2'); - assert.match(await readMarker(previousDir), /v1/, 'v1 is the previous again'); + assert.equal(existsSync(path.join(activationRoot, name)), false); + assert.equal(existsSync(path.join(activationRoot, sibling, 'keep')), true); + await cleanup(name); + }); - await fs.rm(liveDir, { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR), { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + it('startup reconciliation removes terminal stages and preserves valid staged rows', async () => { + const name = fixtureName(); + const keptId = randomUUID(); + const removedId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('kept') }); + await stageApplication(application, keptId); + application.payload = await makeComponentPayload('removed'); + await stageApplication(application, removedId); + const rows = new Map([ + [keptId, { deployment_id: keptId, project: name, status: 'staged' }], + [removedId, { deployment_id: removedId, project: name, status: 'failed' }], + ]); + + const result = await reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async (id) => rows.get(id), + async () => {} + ); + + assert.equal(existsSync(stagedApplicationPath(application.dirPath, keptId)), true); + assert.equal(existsSync(stagedApplicationPath(application.dirPath, removedId)), false); + assert.deepEqual(result.removed, [removedId]); + await cleanup(name); }); - it('revertApplication throws when there is no retained previous (deployed only once)', async () => { - const name = `stage_test_${process.pid}_${counter++}`; - const app = await deployVersion(name, 'only'); // first-ever deploy: nothing retained as previous + it('startup reconciliation rolls an activating staged tree forward', async () => { + const name = fixtureName(); + const deploymentId = randomUUID(); + const livePath = path.join(COMPONENTS_ROOT, name); + await fs.mkdir(livePath, { recursive: true }); + await fs.writeFile(path.join(livePath, 'index.js'), 'module.exports = "old";\n'); + const application = new Application({ name, payload: await makeComponentPayload('candidate') }); + await stageApplication(application, deploymentId); + const activationPath = path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR, name); + await fs.mkdir(activationPath, { recursive: true }); + await fs.rename(livePath, path.join(activationPath, `.previous-${deploymentId}-crash`)); + const row = { + deployment_id: deploymentId, + project: name, + status: 'activating', + activation_spec: { project: name }, + }; + const persisted = []; + + const result = await reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async (id) => (id === deploymentId ? row : undefined), + async (value) => persisted.push(value.deployment_id) + ); + + assert.match(await readMarker(livePath), /candidate/); + assert.deepEqual(persisted, [deploymentId]); + assert.deepEqual(result.recovered, [deploymentId]); + assert.equal(existsSync(activationPath), false); + await cleanup(name); + }); - await assert.rejects(() => revertApplication(app), /no previous version is retained/i); + it('startup reconciliation finishes an activation whose candidate is already live', async () => { + const name = fixtureName(); + const deploymentId = randomUUID(); + const livePath = path.join(COMPONENTS_ROOT, name); + const activationPath = path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR, name); + await fs.mkdir(livePath, { recursive: true }); + await fs.writeFile(path.join(livePath, 'index.js'), 'module.exports = "candidate";\n'); + await fs.mkdir(path.join(activationPath, `.previous-${deploymentId}-crash`), { recursive: true }); + const row = { + deployment_id: deploymentId, + project: name, + status: 'activating', + activation_spec: { project: name }, + }; + let persisted = 0; + + const result = await reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async (id) => (id === deploymentId ? row : undefined), + async () => persisted++ + ); + + assert.equal(persisted, 1); + assert.deepEqual(result.recovered, [deploymentId]); + assert.match(await readMarker(livePath), /candidate/); + assert.equal(existsSync(activationPath), false); + await cleanup(name); + }); - await fs.rm(path.join(COMPONENTS_ROOT, name), { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + it('rejects a non-UUID before it can become a filesystem path segment', () => { + assert.throws(() => stagedApplicationPath(path.join(COMPONENTS_ROOT, fixtureName()), '../../escape'), /Invalid/); }); - it('evicts the oldest not-yet-activated staged builds beyond the retention count (default 5)', async () => { - // Simulate repeated `activate: false` stage-and-stops of the same component (distinct stagingIds, - // never activated). Each stage should prune older ones down to the retention count. - const name = `stage_test_${process.pid}_${counter++}`; - const stagingRoot = path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR); - const stagingIds = []; - for (let i = 0; i < 7; i++) { - const app = new Application({ name, payload: await makeComponentPayload(`v${i}`) }); - stagingIds.push(app.stagingId); - await stageApplication(app); - } + it('rejects a symlinked staging root without touching its target', async () => { + const name = fixtureName(); + const deploymentId = randomUUID(); + const outside = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-stage-outside-')); + await fs.writeFile(path.join(outside, 'sentinel'), 'keep'); + await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR), { recursive: true, force: true }); + await fs.symlink(outside, path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR), 'dir'); + const application = new Application({ name, payload: await makeComponentPayload('candidate') }); - const remaining = stagingIds.filter((id) => existsSync(path.join(stagingRoot, id, name))); - // Contract: at most `maxCount` staged builds are retained per component, and the just-staged one is - // always kept. (Which older builds are evicted is ordered by mtime — reliable in real use where - // stages are time-separated, but this tight loop can create ties, so it isn't asserted here.) - assert.strictEqual(remaining.length, 5, `expected 5 staged builds retained, got ${remaining.length}`); - assert.ok(existsSync(path.join(stagingRoot, stagingIds[6], name)), 'the most recent stage is always retained'); + await assert.rejects(stageApplication(application, deploymentId), /staging path is not a directory/); + assert.equal(await fs.readFile(path.join(outside, 'sentinel'), 'utf8'), 'keep'); - await fs.rm(stagingRoot, { recursive: true, force: true }); + await cleanup(name); + await fs.rm(outside, { recursive: true, force: true }); }); - it('only one previous is retained across three deploys (older previous evicted)', async () => { - const name = `stage_test_${process.pid}_${counter++}`; - await deployVersion(name, 'v1'); - await deployVersion(name, 'v2'); - await deployVersion(name, 'v3'); + it('activates a completed local-directory package symlink with secure staging containers', async () => { + const name = fixtureName(); + const deploymentId = randomUUID(); + const packageDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-stage-package-')); + await fs.writeFile(path.join(packageDirectory, 'marker.txt'), 'directory-package'); + const application = new Application({ name, packageIdentifier: packageDirectory }); - const previousDir = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); - assert.match(await readMarker(path.join(COMPONENTS_ROOT, name)), /v3/, 'live is v3'); - assert.match(await readMarker(previousDir), /v2/, 'previous is v2; v1 was evicted'); + const stagedPath = await stageApplication(application, deploymentId); + assert.equal((await fs.lstat(stagedPath)).isSymbolicLink(), true); + await activateStagedApplication(application, deploymentId); - await fs.rm(path.join(COMPONENTS_ROOT, name), { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR), { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + assert.equal((await fs.lstat(application.dirPath)).isSymbolicLink(), true); + assert.equal(await fs.readFile(path.join(application.dirPath, 'marker.txt'), 'utf8'), 'directory-package'); + + await cleanup(name); + await fs.rm(packageDirectory, { recursive: true, force: true }); }); }); diff --git a/unitTests/components/deploymentOperations.test.js b/unitTests/components/deploymentOperations.test.js index 5748039f9c..e2b26bc1ca 100644 --- a/unitTests/components/deploymentOperations.test.js +++ b/unitTests/components/deploymentOperations.test.js @@ -14,7 +14,13 @@ const { Readable } = require('node:stream'); const testUtils = require('../testUtils.js'); testUtils.preTestPrep(); -const { handleGetDeploymentPayload, handleDeleteDeploymentPayload } = require('#src/components/deploymentOperations'); +const { + handleGetDeployment, + handleGetDeploymentPayload, + handleDeleteDeploymentPayload, +} = require('#src/components/deploymentOperations'); +const { DeploymentRecorder } = require('#src/components/deploymentRecorder'); +const { ProgressEmitter } = require('#src/server/serverHelpers/progressEmitter'); const { databases } = require('#src/resources/databases'); const terms = require('#src/utility/hdbTerms'); @@ -64,6 +70,33 @@ async function collect(stream) { return Buffer.concat(chunks); } +describe('handleGetDeployment SSE tail', () => { + let installed; + beforeEach(() => { + installed = installMockDeploymentTable(); + }); + afterEach(() => installed.restore()); + + it('keeps tailing a transient staged checkpoint while the origin recorder is active', async () => { + const lifecycle = new ProgressEmitter(); + const recorder = await DeploymentRecorder.create({ project: 'app', emitter: lifecycle }); + await recorder.checkpoint('staged', 'staged'); + let settled = false; + const resultPromise = handleGetDeployment({ + deployment_id: recorder.deploymentId, + progress: new ProgressEmitter(), + }).then((result) => { + settled = true; + return result; + }); + await new Promise((resolve) => setImmediate(resolve)); + assert.strictEqual(settled, false, 'staged is only terminal after its live recorder is gone'); + + await recorder.finish('success'); + assert.strictEqual((await resultPromise).status, 'success'); + }); +}); + describe('handleGetDeploymentPayload', () => { let installed; beforeEach(() => { @@ -138,13 +171,15 @@ describe('handleDeleteDeploymentPayload', () => { }); }); - it('409s on a non-terminal deployment (blob may still be replicating to peers)', async () => { - installed.mock.rows.set('d1', { deployment_id: 'd1', status: 'pending', payload_blob: mockBlob(Buffer.from('x')) }); - await assert.rejects(handleDeleteDeploymentPayload({ deployment_id: 'd1' }), (err) => { - assert.match(err.message, /not in a terminal state/); - assert.strictEqual(err.statusCode, 409); - return true; - }); + it('409s on pending and staged deployments whose blobs may still be needed by peers', async () => { + for (const status of ['pending', 'staged']) { + installed.mock.rows.set('d1', { deployment_id: 'd1', status, payload_blob: mockBlob(Buffer.from('x')) }); + await assert.rejects(handleDeleteDeploymentPayload({ deployment_id: 'd1' }), (err) => { + assert.match(err.message, /not in a terminal state/); + assert.strictEqual(err.statusCode, 409); + return true; + }); + } assert.strictEqual(installed.mock.puts.length, 0, 'must not write the row'); }); diff --git a/unitTests/components/deploymentRecorder.test.js b/unitTests/components/deploymentRecorder.test.js index b4ce153b36..f828f04a25 100644 --- a/unitTests/components/deploymentRecorder.test.js +++ b/unitTests/components/deploymentRecorder.test.js @@ -19,6 +19,10 @@ const { awaitDeploymentRow, readPayloadBlobWithRetry, markDeploymentTerminal, + recordDeploymentPeers, + claimStagedDeployment, + expireOldStagedDeployments, + invalidateProjectStagedDeployments, pruneProjectPayloads, getDeploymentRow, } = require('#src/components/deploymentRecorder'); @@ -386,6 +390,12 @@ describe('awaitDeploymentRow', () => { assert.ok(result.payload_blob); }); + it('can wait for package deployment metadata without requiring a payload blob', async () => { + const row = { deployment_id: 'package-row', package_identifier: 'npm:example', payload_blob: null }; + installed.mock.rows.set(row.deployment_id, row); + assert.strictEqual(await awaitDeploymentRow(row.deployment_id, { timeoutMs: 0, requirePayload: false }), row); + }); + it('rejects with a "did not replicate" timeout when the row never arrives within timeoutMs', async () => { await assert.rejects( () => awaitDeploymentRow('never-arrives', { timeoutMs: 100, pollIntervalMs: 25 }), @@ -681,6 +691,21 @@ describe('markDeploymentTerminal', () => { assert.strictEqual(typeof row.completed_at, 'number', 'completed_at was stamped'); }); + it('preserves an uncertain partial activation as non-terminal with its error', async () => { + installed.mock.rows.set('dep-activating', { + deployment_id: 'dep-activating', + status: 'staged', + completed_at: 100, + }); + + await markDeploymentTerminal('dep-activating', 'activating', new Error('peer did not acknowledge')); + + const row = installed.mock.rows.get('dep-activating'); + assert.strictEqual(row.status, 'activating'); + assert.strictEqual(row.completed_at, null); + assert.strictEqual(row.error.message, 'peer did not acknowledge'); + }); + it('is a no-op when the row is absent (best-effort, never throws)', async () => { await markDeploymentTerminal('missing', 'success'); assert.strictEqual(installed.mock.rows.has('missing'), false, 'no row is fabricated for an unknown id'); @@ -739,6 +764,115 @@ describe('getDeploymentRow', () => { }); }); +describe('staged deployment state', () => { + let installed; + beforeEach(() => { + installed = installMockDeploymentTable({ freezeGet: true }); + }); + afterEach(() => installed.restore()); + + it('claims only the matching staged row and durably marks it activating', async () => { + installed.mock.rows.set('staged-1', { + deployment_id: 'staged-1', + project: 'app', + status: 'staged', + completed_at: 100, + }); + + await assert.rejects(claimStagedDeployment('staged-1', 'other'), /belongs to component 'app'/); + await claimStagedDeployment('staged-1', 'app'); + + const row = installed.mock.rows.get('staged-1'); + assert.strictEqual(row.status, 'activating'); + assert.strictEqual(row.phase, 'activate'); + assert.strictEqual(row.completed_at, null); + }); + + it('accepts an already-activating row only for replicated activation ordering', async () => { + installed.mock.rows.set('activating-1', { + deployment_id: 'activating-1', + project: 'app', + status: 'activating', + }); + + await assert.rejects(claimStagedDeployment('activating-1', 'app'), /not staged/); + const row = await claimStagedDeployment('activating-1', 'app', { allowActivating: true }); + + assert.strictEqual(row.status, 'activating'); + }); + + it('waits for a replicated deployment row to reach staged before claiming it', async () => { + installed.mock.rows.set('lagging-1', { + deployment_id: 'lagging-1', + project: 'app', + status: 'staging', + }); + setImmediate(() => { + installed.mock.rows.set('lagging-1', { + ...installed.mock.rows.get('lagging-1'), + status: 'staged', + }); + }); + + await claimStagedDeployment('lagging-1', 'app', { waitForStagedMs: 200 }); + + assert.strictEqual(installed.mock.rows.get('lagging-1').status, 'activating'); + }); + + it('expires only staged rows beyond the per-project count', async () => { + for (const [id, startedAt, status = 'staged'] of [ + ['old', 100], + ['middle', 200], + ['new', 300], + ['active', 50, 'activating'], + ]) { + installed.mock.rows.set(id, { + deployment_id: id, + project: 'app', + status, + started_at: startedAt, + }); + } + + assert.deepStrictEqual(await expireOldStagedDeployments('app', 2), ['old']); + assert.strictEqual(installed.mock.rows.get('old').status, 'failed'); + assert.strictEqual(installed.mock.rows.get('middle').status, 'staged'); + assert.strictEqual(installed.mock.rows.get('new').status, 'staged'); + assert.strictEqual(installed.mock.rows.get('active').status, 'activating'); + }); + + it('invalidates staged and activating rows when their component is dropped', async () => { + for (const status of ['staged', 'activating', 'success']) { + installed.mock.rows.set(status, { deployment_id: status, project: 'app', status }); + } + + const invalidated = await invalidateProjectStagedDeployments('app'); + + assert.deepStrictEqual(invalidated.sort(), ['activating', 'staged']); + assert.strictEqual(installed.mock.rows.get('staged').status, 'failed'); + assert.strictEqual(installed.mock.rows.get('activating').status, 'failed'); + assert.strictEqual(installed.mock.rows.get('success').status, 'success'); + }); + + it('persists normalized peer outcomes on an existing staged row', async () => { + installed.mock.rows.set('dep', { + deployment_id: 'dep', + status: 'activating', + peer_results: [{ node: 'peer-a', status: 'success' }], + }); + + await recordDeploymentPeers('dep', [ + { node: 'peer-a', status: 'failed', reason: 'offline' }, + { node: 'peer-b', status: 'success' }, + ]); + + const peers = installed.mock.rows.get('dep').peer_results; + assert.strictEqual(peers.length, 2); + assert.strictEqual(peers.find((peer) => peer.node === 'peer-a').error.message, 'offline'); + assert.strictEqual(peers.find((peer) => peer.node === 'peer-b').status, 'success'); + }); +}); + describe('pruneProjectPayloads (deployment_payloadRetention_maxCount)', () => { let installed; beforeEach(() => { @@ -804,15 +938,17 @@ describe('pruneProjectPayloads (deployment_payloadRetention_maxCount)', () => { ); }); - it('never drops a non-terminal deployment (its blob may still be the replication channel)', async () => { + it('never drops staged or activating deployments whose blobs may still be the replication channel', async () => { seed('newest', { startedAt: 300 }); seed('in-flight', { startedAt: 200, status: 'activating' }); + seed('staged', { startedAt: 150, status: 'staged' }); seed('old', { startedAt: 100 }); await pruneProjectPayloads('app', 1); assert.strictEqual(hasPayload('newest'), true, 'newest retained'); assert.strictEqual(hasPayload('in-flight'), true, 'the in-flight deployment keeps its payload'); + assert.strictEqual(hasPayload('staged'), true, 'the staged deployment keeps its payload for later activation'); assert.strictEqual(hasPayload('old'), false, 'the settled older one is still dropped'); }); diff --git a/unitTests/server/fastifyRoutes/operations.test.js b/unitTests/server/fastifyRoutes/operations.test.js index 31075fa4bd..e8195613db 100644 --- a/unitTests/server/fastifyRoutes/operations.test.js +++ b/unitTests/server/fastifyRoutes/operations.test.js @@ -150,6 +150,7 @@ describe('Test custom functions operations', () => { await fs.ensureFile(path.join(CF_DIR_ROOT, 'my-cool-component', '.hidden')); await fs.ensureFile(path.join(CF_DIR_ROOT, 'my-cool-component', 'utils', 'utils.js')); await fs.outputFile(path.join(CF_DIR_ROOT, 'my-other-component', 'config.yaml'), test_yaml_string); + await fs.ensureFile(path.join(CF_DIR_ROOT, '.deploy-activating', 'my-cool-component', '.new-deployment')); sandbox.stub(configUtils, 'getConfiguration').returns({ 'my-other-component': { package: '@my-org/my-other-component', @@ -181,6 +182,7 @@ describe('Test custom functions operations', () => { expect(otherComponent.urlPath).to.equal('/other'); expect(otherComponent.host).to.equal('other.example.com'); expect(otherComponent.loadComponent).to.equal('if-installed'); + expect(result.entries.find((e) => e.name === '.deploy-activating')).to.be.undefined; }); it('Test getComponents includes status information when component status exists', async () => { @@ -522,9 +524,11 @@ describe('Test custom functions operations', () => { // Mock the two-phase build/swap primitives to prevent actual install + filesystem work. const stageApplicationStub = sandbox.stub().resolves('/tmp/staging'); - const activateApplicationStub = sandbox.stub().resolves(); + const activateApplicationStub = sandbox + .stub() + .callsFake(async (_application, _deploymentId, hooks) => hooks?.beforeCommit?.()); operations.__set__('stageApplication', stageApplicationStub); - operations.__set__('activateApplication', activateApplicationStub); + operations.__set__('activateStagedApplication', activateApplicationStub); // This should work - user components can be overwritten without force await operations.deployComponent({ @@ -551,9 +555,11 @@ describe('Test custom functions operations', () => { // Mock the two-phase build/swap primitives to prevent actual install + filesystem work. const stageApplicationStub = sandbox.stub().resolves('/tmp/staging'); - const activateApplicationStub = sandbox.stub().resolves(); + const activateApplicationStub = sandbox + .stub() + .callsFake(async (_application, _deploymentId, hooks) => hooks?.beforeCommit?.()); operations.__set__('stageApplication', stageApplicationStub); - operations.__set__('activateApplication', activateApplicationStub); + operations.__set__('activateStagedApplication', activateApplicationStub); // This should work fine - no component exists yet await operations.deployComponent({ @@ -600,9 +606,11 @@ describe('Test custom functions operations', () => { // Mock the two-phase build/swap primitives to prevent actual install + filesystem work. const stageApplicationStub = sandbox.stub().resolves('/tmp/staging'); - const activateApplicationStub = sandbox.stub().resolves(); + const activateApplicationStub = sandbox + .stub() + .callsFake(async (_application, _deploymentId, hooks) => hooks?.beforeCommit?.()); operations.__set__('stageApplication', stageApplicationStub); - operations.__set__('activateApplication', activateApplicationStub); + operations.__set__('activateStagedApplication', activateApplicationStub); // This should NOT throw an error because force is true await operations.deployComponent({ diff --git a/unitTests/server/serverHelpers/serverUtilities.test.js b/unitTests/server/serverHelpers/serverUtilities.test.js index 0527a3ebd3..83d3f0e2df 100644 --- a/unitTests/server/serverHelpers/serverUtilities.test.js +++ b/unitTests/server/serverHelpers/serverUtilities.test.js @@ -570,9 +570,11 @@ describe('Test serverUtilities.js module ', () => { describe('registerOperation permission seam', function () { const { server } = require('#src/server/Server'); const op_auth = require('#src/utility/operation_authorization'); + const { isOperationAuthorizationBypassed } = require('#src/server/serverHelpers/operationAuthorizationState'); const { validateOperations } = require('#src/utility/operationPermissions'); const SU_OP = 'test_registered_su_op'; const OPEN_OP = 'test_registered_open_op'; + const AUTH_STATE_OP = 'test_internal_authorization_state'; // A non-super_user request JSON for a given op, optionally carrying an `operations` grant. const nonSuRequest = (op, operations) => ({ @@ -596,7 +598,15 @@ describe('Test serverUtilities.js module ', () => { // Keep the process-global registries clean — these test-only ops shouldn't leak into other // suites. registerOperation touches three globals (the op-function map plus verifyPerms' // requiredPermissions and the grantable-ops set), so undo all three, not just the map. - for (const op of [SU_OP, OPEN_OP, 'test_name_pinning_op', 'shared_op_a', 'shared_op_b', 'dyn_grantable_op']) { + for (const op of [ + SU_OP, + OPEN_OP, + AUTH_STATE_OP, + 'test_name_pinning_op', + 'shared_op_a', + 'shared_op_b', + 'dyn_grantable_op', + ]) { serverUtilities.OPERATION_FUNCTION_MAP.delete(op); op_auth.unregisterOperationPermission(op); } @@ -660,6 +670,19 @@ describe('Test serverUtilities.js module ', () => { it('allows a non-super_user whose role grants the op via the operations allowlist (SU-bypass)', function () { assert.equal(op_auth.verifyPerms(nonSuRequest(SU_OP, [SU_OP]), SU_OP), null); }); + + it('exposes trusted authorization bypass through server.operation without leaking it afterward', async function () { + server.registerOperation({ + name: AUTH_STATE_OP, + execute: async () => ({ bypassed: isOperationAuthorizationBypassed() }), + requiresSuperUser: true, + }); + + const result = await server.operation({ operation: AUTH_STATE_OP }, { user: { name: 'cluster-peer' } }, false); + + assert.deepEqual(result, { bypassed: true }); + assert.equal(isOperationAuthorizationBypassed(), false); + }); }); // #1809 — process-wide server.* registrations must not leak from a throwaway deploy-validation load. diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index 126863e3ae..dbde9895be 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -318,17 +318,10 @@ export const OPERATIONS_ENUM = { PACKAGE_CUSTOM_FUNCTION_PROJECT: 'package_custom_function_project', DEPLOY_CUSTOM_FUNCTION_PROJECT: 'deploy_custom_function_project', PACKAGE_COMPONENT: 'package_component', - // deploy_component runs a two-phase deploy internally (stage the incoming version into a hidden - // dir cluster-wide, gate on every node, then atomically swap it live). The two phases are fanned - // out to peers as deploy_component tagged with an internal `_phase` marker rather than separate - // public operations. Public knobs: `activate: false` (stage-and-stop, returns a staged - // deployment_id) and `deployment_id` (activate a previously-staged deployment). See - // components/Application.ts (stageApplication/activateApplication). + // deploy_component runs a two-phase deploy internally. Peer phases use a distinct operation so + // older nodes fail closed instead of interpreting an unknown phase field as a one-shot deploy. DEPLOY_COMPONENT: 'deploy_component', - // Swap a component's live version back to its retained previous version, cluster-wide. Backs - // customer-driven rollback (deploy → test → revert) and swap-back on a partially-failed activate. - // See components/Application.ts (revertApplication). - REVERT_COMPONENT: 'revert_component', + COMPONENT_DEPLOY_PHASE: 'component_deploy_phase', READ_TRANSACTION_LOG: 'read_transaction_log', DELETE_TRANSACTION_LOGS_BEFORE: 'delete_transaction_logs_before', INSTALL_NODE_MODULES: 'install_node_modules', diff --git a/utility/operation_authorization.ts b/utility/operation_authorization.ts index 355f92f228..dc91a64598 100644 --- a/utility/operation_authorization.ts +++ b/utility/operation_authorization.ts @@ -286,7 +286,7 @@ requiredPermissions.set(functionsOperations.addComponent.name, new (permission a requiredPermissions.set(functionsOperations.dropCustomFunctionProject.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.packageComponent.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.deployComponent.name, new (permission as any)(true, [])); -requiredPermissions.set(functionsOperations.revertComponent.name, new (permission as any)(true, [])); +requiredPermissions.set(functionsOperations.componentDeployPhase.name, new (permission as any)(true, [])); requiredPermissions.set( deploymentOperations.handleListDeployments.name, new (permission as any)(true, [], terms.OPERATIONS_ENUM.LIST_DEPLOYMENTS) From 70e24b4d2bd6c1d26d7068c81819e50b03601117 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Wed, 12 Aug 2026 12:14:51 -0400 Subject: [PATCH 41/94] feat(deploy): restore revert_component as an addressed, idempotent rollback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #2135 removed `revert_component` entirely in favor of roll-forward-only, but left `harper revert` pointing at an operation that no longer existed, the DESIGN.md sections still describing it as shipping, and the whole `revertApplication`/`retainAsPrevious`/`.deploy-previous` machinery unreachable. This restores it in the shape @kriszyp actually asked for on harper#1849 — target-addressed and idempotent — rather than as the bidirectional toggle he objected to. Why keep it at all: the rollback customers need is the one bad rollout that just happened, and they need it fast. Every node already has the previous version on disk, so this resolves no package, decrypts no secret, downloads no artifact and runs no install — it is one atomic rename per node. **Addressed, not toggled.** `to_deployment_id` is required and names the version the caller expects live afterwards: - already live → no-op success, so an ordinary retry after a lost response cannot swap the rejected release back in (the retry hazard @kriszyp raised on components/Application.ts:1479); - matches the retained previous → swap, and the displaced tree becomes the new retained previous, so an explicitly-targeted revert-of-a-revert still rolls forward; - anything else → refused, naming what this component can actually revert to. Reaching an older version is a redeploy, not a revert; only one previous version is retained. **Persistent state moves with the directory.** A retained tree now carries a sidecar manifest recording which deployment produced it and the root-config entry it was activated with, and revert applies that entry to both the root config and the boot-time install lock via a new shared `createApplicationConfigTransaction` (which `createApplicationActivationTransaction` now delegates to). `null` config means REMOVE the entry — the case that matters when reverting away from a `package` deploy, where a stale `package:` reference would let `installApplications()` reinstall the reverted-away version over the restored directory on the next cold start. That was @kriszyp's components/operations.js:1540 thread; a config-write failure compensates by swapping back. **One contract on `.deploy-aside`.** The evicted two-deploys-ago tree is parked under a new `.discarded-` prefix. #2066's startup sweep restores the newest unretired `.in-progress-` directory OVER the live component, so a tree that is known garbage when parked must never look like a rollback record — otherwise a crash between parking and sweeping resurrects an ancient version over the current one. There is a regression test asserting no unretired `.in-progress-` entry is ever left behind by eviction. `revert_on_failure` stays forbidden. Automatic rollback after the activation barrier is unsound for the reason @kriszyp gave on operations.js:869: a peer can complete its swap and then fail the persistent work that follows, so it reports `failed` while running the new version. Auto-reverting "the failed peers" rolls an untouched node an extra version back and splits the cluster three ways. A partial activation stays visibly `activating` and is rolled forward — or rolled back explicitly, by target, with this operation. Also re-registers what #2135 orphaned (OPERATIONS_ENUM, the dispatch map, operation_authorization as super_user, SSE progress streaming) and moves `revert` out of OP_ALIASES into OP_VERB_PROPS so it can carry the `_cliVerb` marker its missing-target guard keys on — buildRequest checks OP_ALIASES first, so the alias would have shadowed it. Dead code #2135 left behind, now removed: `_legacyDeployComponentTwoPhase`, `_legacyDeployPhaseStage`, `_legacyDeployPhaseActivate`, `_legacyDeployComponentActivateExisting`, `_revertComponent`, `_revertComponentValidator`, `_legacyStageApplication`, `_legacyDiscardStagedApplication`, the legacy `activateApplication`, `enforceActivatePeerGate`, `selectRevertTargets`, `buildReplicatedSubOp`, `createPeerResultCollector`, and a duplicate `getPayloadRetentionMaxCount` declaration whose two copies had divergent input validation (function hoisting meant the later one silently won every call). `getStagingRetentionMaxCount` is now exported once from Application.ts instead of reimplemented in operations.js. Tests: 9 new cases covering retention addressing, the no-op retry, the refusal, one-previous retention across three activations, the `.discarded-` parking guarantee, and config reporting — plus the dangling-symlink activation regression test #2135 dropped. 284 deploy/component/CLI/server unit tests pass. Co-Authored-By: Claude Opus 5 --- bin/cliOperations.ts | 24 +- components/Application.ts | 551 ++++++++-------- components/operations.js | 694 ++++----------------- components/operationsValidation.js | 21 +- server/serverHelpers/serverHandlers.js | 1 + server/serverHelpers/serverUtilities.ts | 4 + unitTests/components/deployStaging.test.js | 202 ++++++ utility/hdbTerms.ts | 3 + utility/operation_authorization.ts | 1 + 9 files changed, 664 insertions(+), 837 deletions(-) diff --git a/bin/cliOperations.ts b/bin/cliOperations.ts index 5620a35cf4..4fc0a13faf 100644 --- a/bin/cliOperations.ts +++ b/bin/cliOperations.ts @@ -18,17 +18,21 @@ import { DeployRenderer } from './deployRenderer.ts'; import { getHdbPid } from '../utility/processManagement/processManagement.js'; import { initConfig, getConfigPath } from '../config/configUtils.ts'; +// Plain name aliases. `revert` is deliberately NOT here — it lives in OP_VERB_PROPS below so it can +// carry the `_cliVerb` marker its missing-target guard keys on, and buildRequest checks OP_ALIASES +// first, so an entry here would shadow that. const OP_ALIASES = { deploy: 'deploy_component', package: 'package_component', - revert: 'revert_component', }; -// CLI verbs that are sugar over `deploy_component` with preset properties (the stage/activate phases -// are folded into deploy_component; there are no separate stage/activate operations). `harper stage` -// packages + uploads the incoming version to a hidden staging dir cluster-wide and stops before -// go-live (`activate: false`), printing the staged deployment_id; `harper activate deployment_id=` -// takes that staged deployment live (no upload). +// CLI verbs that map to an operation plus preset properties. `harper stage` and `harper activate` are +// sugar over `deploy_component` (the stage/activate phases are folded into it; there are no separate +// stage/activate operations): `harper stage` packages + uploads the incoming version to a hidden +// staging dir cluster-wide and stops before go-live (`activate: false`), printing the staged +// deployment_id, and `harper activate deployment_id=` takes that staged deployment live (no +// upload). `harper revert` is its own operation — `revert_component` — because it is a rollback rather +// than a deploy phase, and uploads and installs nothing. const OP_VERB_PROPS: Record> = { stage: { operation: 'deploy_component', activate: false, _cliVerb: 'stage' }, // `_cliVerb` is a CLI-internal marker (stripped before the request is sent) so verbRequirementError @@ -36,6 +40,9 @@ const OP_VERB_PROPS: Record> = { // "no deployment_id → full deploy" fallback would silently build a brand-new deploy from the CWD. // It also tells the staged-deploy capability probe that this invocation needs two-phase support. activate: { operation: 'deploy_component', _cliVerb: 'activate' }, + // `harper revert` uploads nothing: the version it activates is already on every node. `_cliVerb` + // here only drives the missing-target guard below. + revert: { operation: 'revert_component', _cliVerb: 'revert' }, }; // Guard CLI-verb requirements that the operation itself can't enforce (the op has no notion of which @@ -45,6 +52,11 @@ function verbRequirementError(req: any): string | null { if (req._cliVerb === 'activate' && !req.deployment_id) { return '`harper activate` requires a deployment_id from a prior `harper stage` — usage: harper activate project= deployment_id='; } + // revert_component requires its target so a retry can't toggle the rejected release back in. Caught + // here too, so the CLI names the flag instead of surfacing a raw validation error. + if (req._cliVerb === 'revert' && !req.to_deployment_id) { + return '`harper revert` requires the deployment you want live again — usage: harper revert project= to_deployment_id= (list_deployments reports the id)'; + } return null; } diff --git a/components/Application.ts b/components/Application.ts index 0b5b74def2..1fd4396416 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -488,6 +488,9 @@ async function runNpmPack( // loadComponentDirectories from loading its contents as components. export const ASIDE_STAGING_DIR = '.deploy-aside'; const IN_PROGRESS_ASIDE_PREFIX = '.in-progress-'; +// Parked and known-disposable at park time (an evicted two-deploys-ago retained-previous). NEVER a +// recovery candidate — see discardDirAside for why the distinction has to be in the name. +const DISCARDED_ASIDE_PREFIX = '.discarded-'; const RETIRED_ASIDE_PREFIX = '.retired-'; const DEFAULT_COMMAND_TIMEOUT_MS = 60 * 60 * 1000; const COMPONENT_PREPARATION_WAIT_MARGIN_MS = 30000; @@ -528,18 +531,13 @@ const DEPLOYMENT_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][ // revert if unhappy; and a partially-failed activate can be swapped back cluster-wide. export const DEPLOY_PREVIOUS_DIR = '.deploy-previous'; -// Absolute path of the retained-previous copy for a component's live directory. -function previousDirPathFor(liveDirPath: string): string { - return join(dirname(liveDirPath), DEPLOY_PREVIOUS_DIR, basename(liveDirPath)); -} - // Max not-yet-activated staged builds kept per component before the oldest are evicted on the next // stage. A full deploy consumes its staged build immediately (activate renames it live), so this only // bounds `activate: false` stage-and-stops that are never activated. Configurable via // deployment_stagingRetention_maxCount. export const DEFAULT_STAGING_RETENTION_MAX_COUNT = 5; -function getStagingRetentionMaxCount(): number { +export function getStagingRetentionMaxCount(): number { const configured = getConfigValue(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT); // Only a number or numeric string is a valid count; reject everything else (unset, boolean, array, // blank) and fall back to the default — mirroring getPayloadRetentionMaxSize's defensive coercion. @@ -601,72 +599,255 @@ async function pruneStagedBuilds(componentName: string, keepStagingId: string, m } /** - * Atomically move `targetDirPath` aside into a hidden, per-component staging directory if it - * exists, returning the aside staging directory (for best-effort cleanup) or null when there was - * nothing to move. + * Park `targetDirPath` in this component's `.deploy-aside` directory under a name that marks it + * KNOWN-DISPOSABLE at the moment it is parked, and sweep it best-effort. Returns false when there + * was nothing there to park. + * + * Renaming aside — instead of clearing in place — is immune to the race where a still-running worker + * keeps writing into the directory (e.g. a live Next.js app writing into `.next/cache`): an in-place + * recursive rm races that writer and fails with ENOTEMPTY, whereas the rename is atomic and the old + * worker harmlessly keeps writing into the renamed inode until it exits on restart. * - * Renaming the old directory aside — instead of clearing it in place — is immune to the race where - * a still-running worker keeps writing into the directory (e.g. a live Next.js app writing into - * `.next/cache`): an in-place recursive rm races that writer and fails with ENOTEMPTY, whereas the - * rename is atomic and the old worker harmlessly keeps writing into the renamed inode until it is - * replaced on restart. The aside lives on the same filesystem (sibling hidden dir under the same - * parent) so the rename stays atomic, and is per-component so a sibling deploy never collides with - * or sweeps another's aside. + * The `.discarded-` prefix is load-bearing, and is the whole reason this exists alongside the + * extraction transaction's `.in-progress-` asides. `.deploy-aside` has exactly ONE contract + * (harper#1849 review, @kriszyp): an `.in-progress-` directory with no matching `.retired-` marker is + * a ROLLBACK RECORD, and `recoverInterruptedComponentExtractions` restores the newest such directory + * OVER the live component path at startup. A tree that is already known to be garbage when it is + * parked — the evicted two-deploys-ago retained-previous below — must therefore never carry that + * prefix, or a crash between parking and sweeping would make startup recovery resurrect an ancient + * version over the current one. `.discarded-` says "never restore this", and cleanupExtractionPaths + * already removes anything that is not an unretired `.in-progress-`. */ -async function moveDirAside(targetDirPath: string): Promise { - const asideStagingDir = join(dirname(targetDirPath), ASIDE_STAGING_DIR, basename(targetDirPath)); +async function discardDirAside(targetDirPath: string, componentName: string): Promise { + const asideStagingDir = extractionStagingDirectory(targetDirPath); try { // lstat, not access(F_OK): access follows symlinks, so a DANGLING symlink at the path (left by a // prior `file:`-directory deploy whose target was removed) would report ENOENT and be skipped - // here — then mkdir(targetDirPath) fails EEXIST because the dead link still occupies the path. - // lstat sees the link itself, so we move it aside like any other occupant. + // here — then a later mkdir fails EEXIST because the dead link still occupies the path. lstat + // sees the link itself, so we park it like any other occupant. await lstat(targetDirPath); } catch (err) { - if (err.code === 'ENOENT') return null; // nothing there to move + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; // nothing there to park throw err; } - await mkdir(asideStagingDir, { recursive: true }); - await rename(targetDirPath, join(asideStagingDir, `${process.pid}-${Date.now()}-${randomUUID()}`)); - return asideStagingDir; + await ensureExtractionStagingDirectory(asideStagingDir); + const discardedPath = join(asideStagingDir, `${DISCARDED_ASIDE_PREFIX}${Date.now()}-${process.pid}-${randomUUID()}`); + await rename(targetDirPath, discardedPath); + // Best-effort: the outgoing worker may still hold files open in the renamed copy, so a failure here + // is expected in that case and swept by the next deploy or the next startup recovery pass. + void cleanupExtractionPaths( + { name: componentName, dirPath: targetDirPath, logger }, + asideStagingDir, + new Set([discardedPath]) + ).catch((err) => logger.trace?.(`Deferred cleanup of discarded ${componentName} directory: ${errorMessage(err)}`)); + return true; } -// Best-effort removal of a per-component aside staging directory. The old worker may still hold -// files open in a renamed copy (the live writer that motivated the rename), so a failure here is -// expected in that case and logged at trace rather than as a warning — the survivor is swept by -// the next deploy. -function cleanupAsideDir(asideStagingDir: string | null, componentName: string): void { - if (!asideStagingDir) return; - rm(asideStagingDir, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }).catch((err) => - logger.trace?.(`Deferred cleanup of previous ${componentName} component directory: ${err.message}`) - ); +/** + * The retained-previous manifest for a component: which deployment produced the tree now sitting in + * `.deploy-previous/`, which produced the tree now LIVE, and the root-config entry each one was + * activated with. + * + * This is what makes `revert_component` addressable rather than a blind toggle (harper#1849 review, + * @kriszyp): the caller names the deployment it expects to end up live, so a retried request whose + * response was lost is a no-op instead of flipping the rejected release back in. It is also what lets + * a revert restore persistent state, not just the directory: `application_config` is the root-config + * entry (and install-lock entry) that belongs with each tree, so reverting away from a `package` + * deploy removes that package reference instead of leaving `installApplications()` free to reinstall + * the reverted-away version over the restored directory on the next cold start. + * + * `application_config: null` means "that version had no root-config entry" (a payload deploy) and is + * therefore an instruction to DELETE the entry on revert, not to leave it alone. + */ +type RetainedVersion = { + deployment_id: string | null; + application_config: ApplicationConfig | null; +}; + +type RetainedPreviousManifest = { + previous: RetainedVersion; + live: RetainedVersion; +}; + +// Absolute path of the retained-previous copy for a component's live directory, and of the sidecar +// manifest describing it. The manifest is a sibling FILE rather than something inside the retained +// tree, so the tree stays a byte-for-byte copy of what was live. +function previousDirPathFor(liveDirPath: string): string { + return join(dirname(liveDirPath), DEPLOY_PREVIOUS_DIR, basename(liveDirPath)); +} + +function previousManifestPathFor(liveDirPath: string): string { + return `${previousDirPathFor(liveDirPath)}.json`; +} + +async function readRetainedPreviousManifest(liveDirPath: string): Promise { + try { + const parsed = JSON.parse(await readFile(previousManifestPathFor(liveDirPath), 'utf8')); + if (!parsed?.previous || !parsed?.live) return undefined; + return parsed as RetainedPreviousManifest; + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw err; + } +} + +async function writeRetainedPreviousManifest( + liveDirPath: string, + manifest: RetainedPreviousManifest +): Promise { + const manifestPath = previousManifestPathFor(liveDirPath); + // Temp + rename so a crash mid-write can never leave a half-written manifest, which would make a + // retained tree unaddressable (and so unrevertable). + const tempPath = `${manifestPath}.${process.pid}.${randomUUID()}.tmp`; + await mkdir(dirname(manifestPath), { recursive: true }); + await writeFile(tempPath, JSON.stringify(manifest, null, 2), { mode: 0o600 }); + await rename(tempPath, manifestPath); } /** - * Retain the current live version of a component as its rollback source: rename it to - * `.deploy-previous/`, evicting any older retained-previous first. Returns the aside directory - * holding the evicted older-previous (for best-effort cleanup), or null when there was no live - * version to retain (a first-ever deploy). + * Retain the tree this activation displaced as the component's rollback source + * (`.deploy-previous/`), recording which deployment produced it and the root-config entry it + * was activated with. Called by activateStagedApplication after a committed swap, in place of simply + * deleting the displaced tree. + * + * Exactly one previous version is kept per component — this evicts the older one — so retention costs + * a bounded one extra copy per component rather than growing without limit. * - * The eviction moves the older-previous ASIDE (an atomic rename that can't fail on a directory a - * lingering worker still holds open) rather than an in-place rm, so the subsequent rename onto - * `.deploy-previous/` never races an incomplete delete (ENOTEMPTY). Like the aside swap, the - * still-running worker of the version being retained keeps writing into the renamed inode harmlessly - * until it exits on restart. + * Best-effort by design: a component that cannot retain its previous version is still successfully + * deployed, it just isn't revertable. Failing the deploy here would be strictly worse. */ -async function retainAsPrevious(liveDirPath: string): Promise { +async function retainActivatedPrevious( + application: Application, + displacedPath: string | undefined, + deploymentId: string, + activatedConfig: ApplicationConfig | undefined, + outgoing: RetainedVersion +): Promise { + const liveDirPath = application.dirPath; const previousPath = previousDirPathFor(liveDirPath); try { - await lstat(liveDirPath); // lstat, not access: see moveDirAside — a dangling symlink must still move + // Evict the older retained-previous (two deploys ago) before renaming this one into its place, so + // the rename never races an incomplete recursive delete (ENOTEMPTY). Also covers the first-deploy + // case, where any retained tree left over from a dropped-and-redeployed component is stale. + await discardDirAside(previousPath, application.name); + if (displacedPath) { + await mkdir(dirname(previousPath), { recursive: true }); + await rename(displacedPath, previousPath); + } + // Written unconditionally: even a first-ever deploy that retained nothing has to record which + // deployment is now live, or the NEXT activation cannot name the tree it displaces and the + // component stays unrevertable forever. + await writeRetainedPreviousManifest(liveDirPath, { + previous: displacedPath ? outgoing : { deployment_id: null, application_config: null }, + live: { deployment_id: deploymentId, application_config: activatedConfig ?? null }, + }); + } catch (err) { + logger.warn( + `Deployed ${application.name}, but could not retain its previous version for revert:`, + errorForLog(err as Error) + ); + } +} + +/** What a component can currently be reverted to, for reporting and for revert targeting. */ +export async function getRevertTarget( + componentDirPath: string +): Promise<{ live: RetainedVersion; previous: RetainedVersion } | undefined> { + const manifest = await readRetainedPreviousManifest(componentDirPath); + if (!manifest) return undefined; + try { + await lstat(previousDirPathFor(componentDirPath)); } catch (err) { - if (err.code === 'ENOENT') return null; // no live version yet — nothing to retain + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return undefined; // manifest without a tree throw err; } - // Evict the older retained-previous (2 deploys ago; its worker exited on the last restart) by moving - // it aside atomically, clearing the target for the rename below. - const evictedAside = await moveDirAside(previousPath); - await mkdir(dirname(previousPath), { recursive: true }); - await rename(liveDirPath, previousPath); - return evictedAside; + return { live: manifest.live, previous: manifest.previous }; +} + +/** + * Swap a component's live version with its retained previous version (`.deploy-previous/`), + * atomically, then let watchers restart onto it. This is what backs `revert_component`: a customer can + * deploy, run their own health checks, and swap back fast — with no package resolution, artifact + * download or install, because the bytes are already on disk. + * + * ADDRESSED, NOT TOGGLED (harper#1849 review, @kriszyp). The caller passes the deployment id it + * expects to be live when this returns: + * - already live → a no-op success, so an ordinary request retry whose first response was lost + * cannot swap the rejected release back in. + * - matches the retained previous → swap, and the displaced tree becomes the new retained previous + * (so a deliberate, explicitly-targeted revert-of-a-revert still rolls forward). + * - anything else → rejected, naming what this component can actually be reverted to. + * + * Returns the root-config entry the newly-live tree was originally activated with, so the caller can + * restore persistent config/install-lock state as part of the rollback, plus whether a swap happened. + * + * This method should only be called from the main thread. + */ +export async function revertApplication( + application: Application, + toDeploymentId: string +): Promise<{ swapped: boolean; activatedConfig: ApplicationConfig | null; fromDeploymentId: string | null }> { + const liveDirPath = application.dirPath; + return withComponentPreparationLock(liveDirPath, async () => { + const target = await getRevertTarget(liveDirPath); + if (!target) { + throw new Error( + `Cannot revert ${application.name}: no previous version is retained. A component must have been ` + + `deployed over a prior version (which activation retains as .deploy-previous) to be reverted.` + ); + } + if (target.live.deployment_id === toDeploymentId) { + // Already there. Idempotent so a retry is safe. + return { swapped: false, activatedConfig: target.live.application_config, fromDeploymentId: null }; + } + if (target.previous.deployment_id !== toDeploymentId) { + throw new Error( + `Cannot revert ${application.name} to deployment '${toDeploymentId}': it is neither the live version ` + + `('${target.live.deployment_id ?? 'unknown'}') nor the retained previous version ` + + `('${target.previous.deployment_id ?? 'unknown'}'). Only the immediately-previous version is retained ` + + `on disk; redeploy the version you want with deploy_component instead.` + ); + } + + const previousPath = previousDirPathFor(liveDirPath); + const deployLifecycleId = await broadcastDeployStart(application.name); + try { + let liveExists = true; + try { + await lstat(liveDirPath); + } catch (err) { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') liveExists = false; + else throw err; + } + await mkdir(dirname(previousPath), { recursive: true }); + if (liveExists) { + // Three-way atomic swap through a hidden holding path: live → holding, previous → live, + // holding(old live) → previous. The only window where `dirPath` is absent is between two + // atomic renames, and deploy:start suppresses watchers across it (same as activation). + const holding = join(dirname(previousPath), `.reverting-${basename(liveDirPath)}-${randomUUID()}`); + await rename(liveDirPath, holding); + await rename(previousPath, liveDirPath); + await rename(holding, previousPath); + } else { + // No live version to preserve; restore the previous into place. Nothing becomes the new + // retained previous, so the component can't be reverted again until its next deploy. + await rename(previousPath, liveDirPath); + } + application.useLiveBuildDir(); + // Roles exchange: what was live is now the retained previous, and vice versa. + await writeRetainedPreviousManifest(liveDirPath, { + previous: liveExists ? target.live : { deployment_id: null, application_config: null }, + live: target.previous, + }); + return { + swapped: true, + activatedConfig: target.previous.application_config, + fromDeploymentId: target.live.deployment_id, + }; + } finally { + broadcastDeployEnd(application.name, deployLifecycleId); + } + }); } type ExtractionTransaction = { commit(): Promise; @@ -1960,211 +2141,6 @@ export async function prepareApplication(application: Application) { } } -/** - * Phase 1 of a two-phase deploy: build the INCOMING version of a component completely — download / - * `npm pack` (incl. a git clone), extract, and `npm install` — into the hidden staging directory, - * WITHOUT touching the live component directory. - * - * This is the slow, failure-prone half of a deploy, and doing it off to the side has two payoffs: - * - It is safe to run across the whole cluster and gate on: if a node can't fetch the package or - * `npm install` fails, that node reports the failure and NOTHING has changed anywhere — the live - * component is untouched on every node. Contrast the one-shot deploy, where a peer can fail - * mid-install with a half-written live directory while other peers have already gone live. - * - The staging directory is not the watched base of any component's file watcher and is ignored - * by the component loader (leading dot), so building here triggers no restart-on-change storm. - * No deploy:start/deploy:end watcher suppression is needed for this phase — that is reserved for - * activateApplication, which is the only phase that writes the live path. - * - * This method should only be called from the main thread. - * - * @param application The application to stage. - * @returns The absolute path of the staging directory the incoming version was built into. - */ -async function _legacyStageApplication(application: Application): Promise { - application.useStagingBuildDir(); - // Start from a clean slate so a retried stage (same deployment id) can't inherit a half-built - // tree from a previous attempt. - await rm(application.stagingDirPath, { recursive: true, force: true }); - // Create the per-deploy staging parent (.deploy-staging/) up front. extractApplication's - // own mkdir only covers the payload path — for a `package` deploy the FIRST filesystem touch is the - // `npm pack`/git-clone spawn, whose cwd is this parent directory; without it the spawn fails with - // ENOENT (posix_spawn) before any tarball is produced. - await mkdir(dirname(application.stagingDirPath), { recursive: true }); - try { - await application.writeTransientNpmrc(); - try { - await application.startGitCredentialSession(); - await extractApplication(application); - } finally { - await application.cleanupGitCredentialSession(); - } - await installApplication(application); - } catch (err) { - // A failed stage leaves nothing live; remove the partial staging tree so it can't accumulate - // or be mistaken for a good build. Best-effort — never mask the original failure. - await rm(application.stagingDirPath, { recursive: true, force: true }).catch(() => {}); - application.useLiveBuildDir(); - throw err; - } finally { - await application.cleanupTransientNpmrc(); - } - // Now that this stage succeeded, evict the oldest not-yet-activated staged builds for this component - // beyond the retention count (a full deploy consumes this one on activate; `activate: false` - // stage-and-stops are what actually accumulate). Best-effort; never fails the stage. - await pruneStagedBuilds(application.name, application.stagingId, getStagingRetentionMaxCount()); - return application.stagingDirPath; -} - -/** - * Phase 2 of a two-phase deploy: swap the already-staged incoming version into the live component - * directory in one atomic `rename()`, then let watchers restart onto it. - * - * This is the short, low-risk half — no network, no install, just a directory swap — so the window - * during which the component is being replaced is as small as the filesystem allows, and it is only - * entered once staging has succeeded (cluster-wide, when orchestrated by deploy_component). - * - * Bracketed with deploy:start/deploy:end so every thread's file watchers suppress restart-on-change - * while the live directory is replaced (harper#488) — the same suppression the one-shot deploy used - * to hold for the entire extract+install; here it wraps only the swap. - * - * The outgoing live version is not discarded — it is retained as `.deploy-previous/` so - * revert_component can swap it back. See retainAsPrevious. - * - * This method should only be called from the main thread. - */ -export async function activateApplication(application: Application): Promise { - const stagingDirPath = application.stagingDirPath; - try { - await access(stagingDirPath, constants.F_OK); - } catch (err) { - if (err.code === 'ENOENT') { - throw new Error( - `Cannot activate ${application.name}: no staged build found at ${stagingDirPath}. ` + - `Stage the component (stage_component) before activating it.` - ); - } - throw err; - } - const deployLifecycleId = await broadcastDeployStart(application.name); - let evictedAside: string | null = null; - try { - // A live version already existing makes this a redeploy, not a first deploy. This is the two-phase - // equivalent of extractApplication's in-place isNewComponent check: in two-phase the build lands in - // a fresh staging dir, so extractApplication never sees the live dir — activate (the swap) is where - // we learn it. Gates deploy_component's restart-required marking (harper#674/#1806); must be read - // BEFORE retainAsPrevious renames the live dir away. lstat, not access: a dangling symlink still - // counts as an existing directory (see moveDirAside). - try { - await lstat(application.dirPath); - application.isNewComponent = false; - } catch (err) { - if (err.code === 'ENOENT') application.isNewComponent = true; - else throw err; - } - // Retain the current live version as the rollback source (.deploy-previous/), then rename - // the staged copy into place. Both live under the components root, so the rename is same-fs and - // atomic — there is no interval where `dirPath` is a partially populated directory. retainAsPrevious - // tolerates a still-writing worker exactly as the old aside swap did. - evictedAside = await retainAsPrevious(application.dirPath); - await mkdir(dirname(application.dirPath), { recursive: true }); - await rename(stagingDirPath, application.dirPath); - application.useLiveBuildDir(); - } finally { - broadcastDeployEnd(application.name, deployLifecycleId); - } - // Best-effort cleanup of the EVICTED older-previous (the version from two deploys ago; see - // cleanupAsideDir) — NOT the retained previous, which is kept for revert. The rename already consumed - // stagingDirPath, so all that remains of staging is this deploy's now-empty parent - // (.deploy-staging/). Remove it with a NON-recursive rmdir, which succeeds only when it - // is empty — a belt-and-suspenders guard against ever recursively deleting a directory that could - // hold another deploy's build. ENOTEMPTY and ENOENT (already gone) are expected and ignored. - cleanupAsideDir(evictedAside, application.name); - rmdir(dirname(stagingDirPath)).catch((err) => { - if (err.code !== 'ENOTEMPTY' && err.code !== 'ENOENT') - logger.trace?.(`Deferred cleanup of ${application.name} staging directory: ${err.message}`); - }); -} - -/** - * Swap a component's live version with its retained previous version (`.deploy-previous/`), - * atomically, then let watchers restart onto it. This is what backs revert_component: a customer can - * activate a deploy, run their own health checks, and swap back if unhappy; and a partially-failed - * activate can be rolled back cluster-wide. - * - * The swap is bidirectional: the outgoing live becomes the new retained previous, so a second revert - * toggles forward again. Three same-filesystem renames via a hidden holding path — the only window - * where `dirPath` is momentarily absent is between two atomic renames, and deploy:start suppresses - * watchers across it (same as activate). - * - * Throws if there is no retained previous version (a component deployed only once, or never). - * - * This method should only be called from the main thread. - */ -export async function revertApplication(application: Application): Promise { - const liveDirPath = application.dirPath; - const previousPath = previousDirPathFor(liveDirPath); - try { - await lstat(previousPath); - } catch (err) { - if (err.code === 'ENOENT') { - throw new Error( - `Cannot revert ${application.name}: no previous version is retained. A component must have ` + - `been deployed over a prior version (which activate retains as .deploy-previous) to be reverted.` - ); - } - throw err; - } - const deployLifecycleId = await broadcastDeployStart(application.name); - try { - // Does a live version currently exist? (It always should after a deploy, but guard so a missing - // live dir degrades to "restore previous" rather than throwing mid-swap.) - let liveExists = true; - try { - await lstat(liveDirPath); - } catch (err) { - if (err.code === 'ENOENT') liveExists = false; - else throw err; - } - await mkdir(dirname(previousPath), { recursive: true }); - if (liveExists) { - // Three-way atomic swap: live → holding, previous → live, holding(old live) → previous. - const holding = join(dirname(previousPath), `.reverting-${basename(liveDirPath)}-${randomUUID()}`); - await rename(liveDirPath, holding); - await rename(previousPath, liveDirPath); - await rename(holding, previousPath); - } else { - // No live version to preserve; just restore the previous into place (nothing becomes the new - // previous, so the component can't be re-reverted until its next deploy). - await rename(previousPath, liveDirPath); - } - application.useLiveBuildDir(); - } finally { - broadcastDeployEnd(application.name, deployLifecycleId); - } -} - -/** - * Discard a staged-but-not-activated build (an aborted two-phase deploy). Best-effort: removes the - * staging tree and tears down any transient credential state. The live component directory is never - * touched. Safe to call whether or not staging ever ran. - */ -async function _legacyDiscardStagedApplication(application: Application): Promise { - try { - await application.cleanupGitCredentialSession(); - } catch { - /* best-effort */ - } - try { - await application.cleanupTransientNpmrc(); - } catch { - /* best-effort */ - } - application.useLiveBuildDir(); - await rm(application.stagingDirPath, { recursive: true, force: true }).catch((err) => - logger.trace?.(`Failed to discard ${application.name} staging directory: ${err.message}`) - ); -} - export function stagedApplicationPath(componentDirPath: string, deploymentId: string): string { if (!DEPLOYMENT_ID_PATTERN.test(deploymentId)) throw new Error(`Invalid deployment id '${deploymentId}'`); return join(dirname(componentDirPath), DEPLOY_STAGING_DIR, deploymentId, basename(componentDirPath)); @@ -2287,6 +2263,13 @@ export async function activateStagedApplication( beforeSwap?: () => Promise; beforeCommit?: () => Promise; onRollback?: () => Promise; + /** + * The immutable activation specification this deployment is being activated with. Used only to + * derive the root-config entry recorded alongside the retained previous version, so a later + * `revert_component` can restore persistent config/install-lock state and not just the directory. + * Omit it and the swap still happens — the component just isn't revertable afterwards. + */ + activationSpec?: Record; } = {} ): Promise { const stagingDirPath = stagedApplicationPath(application.dirPath, deploymentId); @@ -2303,6 +2286,17 @@ export async function activateStagedApplication( const activationDir = activationStagingDirectory(application.dirPath); await ensureSecureDirectory(dirname(activationDir), true, 'Component activation staging path'); await ensureSecureDirectory(activationDir, true, 'Component activation staging path'); + // Who is live right now, so the tree this activation displaces stays addressable for revert. Read + // before the swap, since the swap is what makes it "previous". + const outgoing: RetainedVersion = (await readRetainedPreviousManifest(application.dirPath).catch( + () => undefined + ))?.live ?? { + // No manifest yet: either a first-ever deploy, or a component last activated by a Harper that + // predates retention. Record the config it is running so a revert can still restore that, even + // though its deployment id is unknowable and so cannot be a revert target. + deployment_id: null, + application_config: readConfigFile()?.[application.name] ?? null, + }; const deployLifecycleId = await broadcastDeployStart(application.name); let backupPath: string | undefined; let newMarkerPath: string | undefined; @@ -2372,7 +2366,16 @@ export async function activateStagedApplication( } finally { broadcastDeployEnd(application.name, deployLifecycleId); } - if (backupPath) await rm(backupPath, { recursive: true, force: true }); + // The displaced tree is the component's rollback source now, not garbage: retain it as + // `.deploy-previous/` with a manifest recording which deployment produced it. Best-effort — + // a component that can't retain its previous version is still deployed, just not revertable. + await retainActivatedPrevious( + application, + backupPath, + deploymentId, + hooks.activationSpec ? applicationConfigFromActivationSpec(hooks.activationSpec) : undefined, + outgoing + ); if (newMarkerPath) await rm(newMarkerPath, { force: true }); await rmdir(activationDir).catch((error) => { if (!['ENOENT', 'ENOTEMPTY'].includes((error as NodeJS.ErrnoException).code ?? '')) throw error; @@ -2491,6 +2494,9 @@ export async function reconcileStagedApplicationArtifacts( if (await hasCompleteStagedApplication(stagedPath)) { await activateStagedApplication(new Application({ name: row.project }), entry.name, { beforeCommit: () => persistActivation(row), + // A recovered roll-forward retains its displaced tree the same as a normal activation, so a + // deploy that crashed mid-swap is still revertable once the node is back up. + activationSpec: row.activation_spec, }); } else { const liveStat = await lstat(componentDirPath).catch(() => undefined); @@ -2743,12 +2749,21 @@ async function getApplicationLockEntry(name: string): Promise + nextConfig: ApplicationConfig | null | undefined ): Promise<{ commit(): Promise; rollback(): Promise }> { - const nextConfig = applicationConfigFromActivationSpec(spec); - if (!nextConfig) return { commit: async () => {}, rollback: async () => {} }; + if (nextConfig === undefined) return { commit: async () => {}, rollback: async () => {} }; let previousConfig: ApplicationConfig | undefined; let previousLockConfig: ApplicationConfig | undefined; let commitStarted = false; @@ -2758,8 +2773,9 @@ export async function createApplicationActivationTransaction( previousConfig = readConfigFile()?.[project]; previousLockConfig = await getApplicationLockEntry(project); commitStarted = true; - await addConfig(project, nextConfig); - await updateApplicationLockEntry(project, nextConfig); + if (nextConfig === null) deleteConfigFromFile([project]); + else await addConfig(project, nextConfig); + await updateApplicationLockEntry(project, nextConfig ?? undefined); }, async rollback() { if (!commitStarted) return; @@ -2771,6 +2787,15 @@ export async function createApplicationActivationTransaction( }; } +export async function createApplicationActivationTransaction( + project: string, + spec: Record +): Promise<{ commit(): Promise; rollback(): Promise }> { + // A payload deploy has no package reference to persist, so activating from it says nothing about + // root config and must leave whatever is there alone (undefined, not null). + return createApplicationConfigTransaction(project, applicationConfigFromActivationSpec(spec)); +} + /** * Keep the boot-time application lock honest while a reinstall is in flight. Factored as an * explicit production seam so the failure transition can be tested without replacing module diff --git a/components/operations.js b/components/operations.js index 366eea3900..c7bf67ca98 100644 --- a/components/operations.js +++ b/components/operations.js @@ -29,7 +29,6 @@ const { Application, prepareApplication, stageApplication, - activateApplication, revertApplication, stagedApplicationPath, hasCompleteStagedApplication, @@ -39,6 +38,9 @@ const { discardProjectActivationArtifacts, updateApplicationLockEntry, createApplicationActivationTransaction, + createApplicationConfigTransaction, + getRevertTarget, + getStagingRetentionMaxCount, dropComponentDirectory, ASIDE_STAGING_DIR, DEPLOY_STAGING_DIR, @@ -52,7 +54,6 @@ const { awaitDeploymentRow, getDeploymentRow, markDeploymentTerminal, - normalizePeerResult, recordDeploymentPeers, claimStagedDeployment, expireOldStagedDeployments, @@ -690,356 +691,6 @@ async function deployComponentOneShot(req, credentialReferences, isReplicatedExe } } -/** - * Two-phase deploy orchestrator (origin node). Builds the incoming version into staging on every - * node (phase 1, stage_component), gates on every node succeeding, then atomically swaps it live on - * every node (phase 2, activate_component). The live component on every node is untouched until the - * whole cluster has the bits in place, and the go-live window is just the swap + restart. - */ -async function _legacyDeployComponentTwoPhase(req, credentialReferences) { - const { resolveCredentials } = require('./secretOperations.ts'); - // Fail fast on a protected core name before we create any state or touch the cluster. - if (req.package) assertNotProtectedCoreComponent(req.project, req.force); - - // The origin always records (a two-phase origin is never itself a replicated execution). - const emitter = req.progress ?? new ProgressEmitter(); - if (!req.progress) req.progress = emitter; - const recorder = await DeploymentRecorder.create({ - project: req.project, - package_identifier: req.package ?? null, - user: req.hdb_user?.username, - restart_mode: req.restart === 'rolling' ? 'rolling' : req.restart ? 'immediate' : null, - credentials: credentialReferences.length ? credentialReferences : null, - emitter, - }); - req._deploymentId = recorder.deploymentId; - const emit = (event, data) => emitter.emit(event, data); - const installCapture = createInstallCapture(); - const rollingRestart = req.restart === 'rolling'; - const recordPeer = (result) => { - recorder.recordPeer(result); - emit('peer', result); - }; - let application; - - try { - // Tee the payload into the row's blob (the replication channel peers read from) and re-source - // extraction from it. Two-phase requires systemReplicated, so peers always fetch from the row. - const extractionPayload = await sourceExtractionPayload({ req, recorder, isReplicatedExecution: false }); - const resolvedCredentials = await resolveNodeCredentials({ req, resolveCredentials, isReplicatedExecution: false }); - // stagingId = deployment id so peers (which build a fresh Application per sub-op) resolve the - // same staging path this deployment used. - application = buildDeployApplication({ - req, - extractionPayload, - resolvedCredentials, - stagingId: recorder.deploymentId, - installCapture, - emitter, - emit, - }); - // Strip tokens from req before any replication/log path; keep references (peers resolve those - // from their own hdb_secret copy). Strip the emitter and payload too — peers read the payload - // from the replicated row, keeping the sub-operation bodies small. - if (credentialReferences.length) req.credentials = credentialReferences; - else delete req.credentials; - delete req.progress; - delete req.payload; - - // ===== PHASE 1: STAGE — build on every node; nothing goes live. ===== - emit('phase', { phase: 'stage', status: 'start' }); - await stageApplication(application); - const stageOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.DEPLOY_COMPONENT, { phase: 'stage' }); - const stageResp = await server.replication.replicateOperation(stageOp, { onPeerResult: recordPeer }); - if (stageResp?.replicated) recorder.recordPeers(stageResp.replicated); - emit('phase', { phase: 'stage', status: 'done' }); - - // ---- Cluster barrier: every node must have staged before ANY node activates. ---- - if (!req.ignore_replication_errors) { - const failed = recorder.getFailedPeers(); - if (failed.length > 0) { - await discardStagedApplication(application).catch(() => {}); - throw new ServerError( - `Component '${req.project}' failed to stage on ${failed.length} of ` + - `${recorder.row.peer_results.length} peer node(s): ${describePeers(failed)}. No node was activated — ` + - `the live component is unchanged everywhere. See deployment ${recorder.deploymentId} (get_deployment), ` + - `or pass ignore_replication_errors: true to activate the nodes that did stage.` - ); - } - } - - // Validate the staged build before go-live (loads from the staging dir; see loadValidateComponent). - await loadValidateComponent({ dirPath: application.buildDirPath, emit }); - - // `activate: false` — stage-and-stop. The build is verified on every node; leave the row in a - // `staged` state and return its deployment_id so a later deploy_component({deployment_id}) can - // take it live. Nothing has gone live anywhere. - if (req.activate === false) { - emit('phase', { phase: 'staged', status: 'done' }); - await recorder.finish('staged'); - return { - message: `Staged component: ${application.name}`, - project: application.name, - staged: true, - deployment_id: recorder.deploymentId, - }; - } - - // ===== PHASE 2: ACTIVATE — atomic swap + restart, now the bits are in place everywhere. ===== - // Persist root config now (not before staging) so a `package` config never points at a version - // that failed to stage. - if (req.package) await writeComponentRootConfig(req, credentialReferences); - // if doing a rolling restart set restart to false so peers don't also immediately restart. - req.restart = rollingRestart ? false : req.restart; - - emit('phase', { phase: 'activate', status: 'start' }); - await activateApplication(application); - const activateOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.DEPLOY_COMPONENT, { - phase: 'activate', - restart: req.restart, - deploymentId: recorder.deploymentId, - }); - // Seal before the activate replicate burst (same #1170 rationale as one-shot). - recorder.seal(); - const activateResp = await server.replication.replicateOperation(activateOp, { onPeerResult: recordPeer }); - emit('phase', { phase: 'activate', status: 'done' }); - let response = activateResp && typeof activateResp === 'object' ? activateResp : { message: '' }; - if (activateResp?.replicated) recorder.recordPeers(activateResp.replicated); - - // ---- Restart on the origin. ---- - if (req.restart === true) { - emit('phase', { phase: 'restart', status: 'start' }); - manageThreads.restartWorkers('http'); - emit('phase', { phase: 'restart', status: 'done' }); - response.message = `Successfully deployed: ${application.name}, restarting Harper`; - } else if (rollingRestart) { - const serverUtilities = require('../server/serverHelpers/serverUtilities.ts'); - emit('phase', { phase: 'restart', status: 'start' }); - const jobResponse = await serverUtilities.executeJob({ - operation: 'restart_service', - service: 'http', - replicated: true, - }); - emit('phase', { phase: 'restart', status: 'done' }); - response.restartJobId = jobResponse.job_id; - response.message = `Successfully deployed: ${application.name}, restarting Harper`; - } else { - // No restart requested: a genuinely-new component still needs one to serve its routes - // (harper#674). activateApplication set isNewComponent from the pre-swap live dir above. - markRestartRequiredForDeploy(application); - response.message = `Successfully deployed: ${application.name}`; - } - - // ---- Activate gate: rare, but a node can stage OK and then fail the swap. ---- - await enforceActivatePeerGate({ - req, - application, - emit, - failed: recorder.getFailedPeers(), - totalPeers: recorder.row.peer_results.length, - deploymentId: recorder.deploymentId, - }); - - response.deployment_id = recorder.deploymentId; - maybeReclaimPayload(recorder, emit); - emit('phase', { phase: 'success', status: 'done' }); - await recorder.finish('success'); - // After finish(), so this deploy's row is terminal and counts as the newest retained payload. - schedulePayloadRetentionPrune(recorder, req.project, emit); - return response; - } catch (err) { - // An aborted deploy leaves the live component untouched; drop any staged build so it can't leak. - if (application) await discardStagedApplication(application).catch(() => {}); - throw await finalizeDeployFailure({ err, recorder, installCapture, emit }); - } -} - -/** - * Peer stage phase (internal — NOT a public operation). Runs on a peer when the origin fans out - * deploy_component tagged `_phase: 'stage'`: fetch the tarball from the replicated hdb_deployment row, - * build + `npm install` into the hidden staging directory, and load-validate — never touching the live - * path, writing config, or restarting. A failure here fails this peer's stage, which the origin's - * barrier catches. No recorder (the origin owns the row) and no re-replication. - */ -async function _legacyDeployPhaseStage(req) { - const { resolveCredentials } = require('./secretOperations.ts'); - const emitter = null; // peers stream nothing back; the origin owns the emitter/recorder - const emit = () => {}; - const installCapture = createInstallCapture(); - let application; - try { - const extractionPayload = await sourceExtractionPayload({ req, recorder: null, isReplicatedExecution: true }); - const resolvedCredentials = await resolveNodeCredentials({ req, resolveCredentials, isReplicatedExecution: true }); - application = buildDeployApplication({ - req, - extractionPayload, - resolvedCredentials, - stagingId: req._deploymentId, - installCapture, - emitter, - emit, - }); - await stageApplication(application); - // Surface load-time errors on the staged build (no-op on the main thread, where replicated peer - // executions run — app code must not load there; see loadValidateComponent + DESIGN.md). - await loadValidateComponent({ dirPath: application.buildDirPath, emit }); - return { message: `Staged component: ${req.project}`, project: req.project, staged: true }; - } catch (err) { - if (application) await discardStagedApplication(application).catch(() => {}); - throw await finalizeDeployFailure({ err, recorder: null, installCapture, emit }); - } -} - -/** - * Peer activate phase (internal — NOT a public operation). Runs on a peer when the origin fans out - * deploy_component tagged `_phase: 'activate'`: atomically swap the already-staged build (by deployment - * id) into the live path, persist root config for a package deploy, and restart if the origin asked - * for an immediate restart. No recorder, no re-replication. - */ -async function _legacyDeployPhaseActivate(req) { - if (req.package) assertNotProtectedCoreComponent(req.project, req.force); - const credentialReferences = (req.credentials ?? []).filter((entry) => entry && entry.secret !== undefined); - const application = new Application({ - name: req.project, - packageIdentifier: req.package, - stagingId: req._deploymentId, - }); - await activateApplication(application); - if (req.package) await writeComponentRootConfig(req, credentialReferences); - // The origin sets restart=true on the sub-op only for an immediate restart; rolling restarts are - // driven separately by the origin via a replicated restart_service job. - if (req.restart === true) manageThreads.restartWorkers('http'); - // Not restarting now: mark restart-required per node for a genuinely-new component (harper#674), the - // same marking the one-shot peer path does — so a new component deployed cluster-wide with - // restart:false reports restartRequired on every node, not just the origin. A rolling restart, which - // also arrives here with restart:false, clears the flag when it reaches this node. - else markRestartRequiredForDeploy(application); - return { message: `Activated component: ${req.project}`, project: req.project, activated: true }; -} - -/** - * Activate a previously-staged deployment cluster-wide — the second half of a stage-then-activate - * flow, reached as `deploy_component({ deployment_id })` with no fresh payload. Swaps the staged build - * into the live path on the origin, replicates the activate phase to peers (each activates its own - * staged copy of the same deployment id), restarts, and marks the deployment row success. - */ -async function _legacyDeployComponentActivateExisting(req, credentialReferences) { - const stagingId = req.deployment_id; - // An activate-by-id call carries no `package` — `harper activate` sends only project + deployment_id, - // and the docs describe this path as fetching/installing nothing — so recover the staged deployment's - // package identifier and credential references from its row. Without this, a component staged as a - // `package` deploy and activated later would never persist its root-config entry: not on the origin - // (writeComponentRootConfig is gated on `req.package`) and not on any peer either, since the fanned-out - // sub-op copies `package`/`credentials` from this same `req`. The package reference and the credential - // references that cold reinstalls and newly-joined peers depend on would be silently lost, leaving the - // component recorded as a plain directory. Explicit values on the request always win. - if (!req.package) { - const stagedRow = await getDeploymentRow(stagingId).catch((err) => { - log.warn(`Could not read deployment ${stagingId} to recover its package identifier`, err); - return undefined; - }); - if (stagedRow?.package_identifier) { - req.package = stagedRow.package_identifier; - // The row stores credential REFERENCES (tokens were never persisted), which is exactly what - // root config should carry. Only fall back to them when the caller supplied none. - if (!req.credentials?.length && Array.isArray(stagedRow.credentials) && stagedRow.credentials.length) { - req.credentials = stagedRow.credentials; - } - credentialReferences = (req.credentials ?? []).filter((entry) => entry && entry.secret !== undefined); - } - } - if (req.package) assertNotProtectedCoreComponent(req.project, req.force); - const emitter = req.progress ?? new ProgressEmitter(); - if (!req.progress) req.progress = emitter; - const emit = (event, data) => emitter.emit(event, data); - const rollingRestart = req.restart === 'rolling'; - const application = new Application({ name: req.project, packageIdentifier: req.package, stagingId }); - - emit('phase', { phase: 'activate', status: 'start' }); - await activateApplication(application); - emit('phase', { phase: 'activate', status: 'done' }); - // Persist root config now that the component is live (package deploys). - if (req.package) await writeComponentRootConfig(req, credentialReferences); - - // Replicate the activate phase to peers (each activates its own staged copy of this deployment id). - req._deploymentId = stagingId; - delete req.progress; - const activateOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.DEPLOY_COMPONENT, { - phase: 'activate', - restart: rollingRestart ? false : req.restart, - deploymentId: stagingId, - }); - // Collect per-peer outcomes so a partially-failed activate can be gated below. There is no - // DeploymentRecorder on this path (the row was created and finished as `staged` by the earlier - // stage-and-stop), so a local collector stands in for recorder.recordPeer/getFailedPeers. - const peers = createPeerResultCollector(); - const rep = await server.replication.replicateOperation(activateOp, { - onPeerResult: (result) => { - peers.record(result); - emit('peer', result); - }, - }); - if (rep?.replicated) peers.recordAll(rep.replicated); - - const response = { - message: `Activated component: ${req.project}`, - project: req.project, - activated: true, - deployment_id: stagingId, - }; - if (rep?.replicated) response.replicated = rep.replicated; - - if (req.restart === true) { - emit('phase', { phase: 'restart', status: 'start' }); - manageThreads.restartWorkers('http'); - emit('phase', { phase: 'restart', status: 'done' }); - response.message = `Activated component: ${req.project}, restarting Harper`; - } else if (rollingRestart) { - const serverUtilities = require('../server/serverHelpers/serverUtilities.ts'); - emit('phase', { phase: 'restart', status: 'start' }); - const jobResponse = await serverUtilities.executeJob({ - operation: 'restart_service', - service: 'http', - replicated: true, - }); - emit('phase', { phase: 'restart', status: 'done' }); - response.restartJobId = jobResponse.job_id; - response.message = `Activated component: ${req.project}, restarting Harper`; - } else { - // No restart requested: activating a genuinely-new component still needs one to serve its routes - // (harper#674). activateApplication set isNewComponent from the pre-swap live dir above. - markRestartRequiredForDeploy(application); - } - - // ---- Activate gate: a peer can hold a good staged build and still fail the swap. Same gate the - // two-phase activate phase uses, so revert_on_failure / ignore_replication_errors behave identically - // whether the activate came from a full deploy or from `deploy_component({ deployment_id })`. - try { - await enforceActivatePeerGate({ - req, - application, - emit, - failed: peers.getFailed(), - totalPeers: peers.total, - deploymentId: stagingId, - }); - } catch (err) { - // The origin went live but the cluster did not converge — record the terminal state before - // surfacing the failure, so get_deployment doesn't still read `staged`. - await markDeploymentTerminal(stagingId, 'failed').catch((markErr) => - log.warn('Failed to mark deployment as failed after a partial activate', markErr) - ); - throw err; - } - - // Best-effort: flip the staged deployment row (left 'staged' by the stage-and-stop) to success now - // that it is live. Observability only — a tracking-write failure must not fail the activate. - await markDeploymentTerminal(stagingId, 'success').catch((err) => - log.warn('Failed to mark staged deployment as activated', err) - ); - return response; -} - /** * revert_component — swap a component's live version back to its retained previous version * (`.deploy-previous/`, kept by the last activate), cluster-wide, then restart. Backs @@ -1139,16 +790,6 @@ async function discardDeploymentEverywhere(project, deploymentId, activationSpec .catch(() => {}); } -function getStagingRetentionMaxCount() { - const value = Number(env.get(hdbTerms.CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT)); - return Number.isFinite(value) && value >= 1 ? Math.floor(value) : 5; -} - -function getPayloadRetentionMaxCount() { - const value = Number(env.get(hdbTerms.CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXCOUNT)); - return Number.isFinite(value) && value >= 0 ? Math.floor(value) : 1; -} - async function pruneStagedDeploymentArtifacts(project, activationSpec) { const expired = await expireOldStagedDeployments(project, getStagingRetentionMaxCount()); for (const deploymentId of expired) await discardDeploymentEverywhere(project, deploymentId, activationSpec); @@ -1265,6 +906,7 @@ async function deployComponentTwoPhase(req) { }, beforeCommit: () => configTransaction.commit(), onRollback: () => configTransaction.rollback(), + activationSpec, }); activationCommitted = true; const activateResponse = await server.replication.replicateOperation( @@ -1404,6 +1046,7 @@ async function deployComponentActivateExisting(req) { }, beforeCommit: () => configTransaction.commit(), onRollback: () => configTransaction.rollback(), + activationSpec: spec, }); } catch (error) { if (claimed) await markDeploymentTerminal(req.deployment_id, 'staged').catch(() => {}); @@ -1533,6 +1176,7 @@ async function componentDeployPhase(req) { }, beforeCommit: () => configTransaction.commit(), onRollback: () => configTransaction.rollback(), + activationSpec: spec, }); markRestartRequiredForDeploy(application); return { message: `Activated component: ${req.project}`, project: req.project, activated: true }; @@ -1548,98 +1192,156 @@ function isTrustedReplicatedOperation(req) { ); } -async function _revertComponent(req) { +/** + * revert_component — put a component's retained previous version back in service, cluster-wide. + * + * This is the fast-rollback half of the deploy story, and it is deliberately a separate public + * operation rather than a deploy phase: it fetches nothing, resolves no package or secret, downloads + * no artifact and runs no install. The bytes it activates are already on disk on every node — the tree + * each node's last activation displaced and retained as `.deploy-previous/` — so the whole + * cluster can go back to the version it was serving a minute ago at the cost of one atomic rename per + * node. It exists for the case operators actually hit: the one bad rollout that just happened. + * + * It is NOT a general "go to any past version" operation. Exactly one previous version is retained per + * component, so this reaches back exactly one activation. Returning to an older version is a redeploy + * (`deploy_component`), not a revert. + * + * `to_deployment_id` is required and names the deployment the caller expects to be live afterwards, + * which is what makes the operation idempotent under ordinary request retries (harper#1849 review): if + * that version is already live the call succeeds without touching anything, instead of toggling the + * rejected release back in. Targeting the retained previous swaps, and the displaced tree becomes the + * new retained previous — so an explicitly-targeted revert-of-a-revert still rolls forward. + * + * The swap is paired with persistent state. `revertApplication` reports the root-config entry the + * newly-live tree was originally activated with, and this applies it to both the root config and the + * boot-time install lock. Without that, reverting away from a `package` deploy would leave the config + * still naming the reverted-away package, and `installApplications()` would quietly reinstall it over + * the restored directory on the next cold start — undoing the rollback. + */ +async function revertComponent(req) { if (req.project) req.project = path.parse(req.project).name; const validation = validator.revertComponentValidator(req); if (validation) throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); + assertNotProtectedCoreComponent(req.project, req.force); - const isReplicatedExecution = typeof req._deploymentId === 'string'; + const isReplicatedExecution = isTrustedReplicatedOperation(req); const emitter = isReplicatedExecution ? null : (req.progress ?? new ProgressEmitter()); - if (emitter && !req.progress) req.progress = emitter; - // The origin records a rollback row for observability; a peer replaying the revert does not. + const emit = emitter ? (event, data) => emitter.emit(event, data) : () => {}; + delete req.progress; + + const application = new Application({ name: req.project }); + // Read the retained-previous state before anything is created or moved: it supplies the deployment + // this rollback takes out of service (recorded as `rollback_of`, which the recorder can only accept + // at create time), and it lets an unrevertable request fail before a deployment row exists. + const componentsRoot = configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT); + const revertTarget = await getRevertTarget(path.join(componentsRoot, req.project)); + // The origin records an auditable rollback row; a peer replaying the revert does not (the origin + // owns the row, exactly as in the deploy fan-out). const recorder = isReplicatedExecution ? null : await DeploymentRecorder.create({ project: req.project, - package_identifier: null, user: req.hdb_user?.username, restart_mode: req.restart === 'rolling' ? 'rolling' : req.restart ? 'immediate' : null, - rollback_of: req.deployment_id ?? null, + rollback_of: revertTarget?.live?.deployment_id ?? null, emitter, }); - if (recorder) req._deploymentId = recorder.deploymentId; - const emit = (event, data) => emitter?.emit(event, data); - const installCapture = createInstallCapture(); // revert has no install output, but finalizeDeployFailure expects one - const rollingRestart = req.restart === 'rolling'; - try { - const application = new Application({ name: req.project }); emit('phase', { phase: 'revert', status: 'start' }); - await revertApplication(application); - emit('phase', { phase: 'revert', status: 'done' }); - - const response = { message: `Reverted component: ${req.project}`, project: req.project, reverted: true }; - if (recorder) response.deployment_id = recorder.deploymentId; - - // Replicate the revert to peers (direct invocation only; a peer replaying must not re-fan). - req.restart = rollingRestart ? false : req.restart; - if (!isReplicatedExecution) { - delete req.progress; - const revertOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.REVERT_COMPONENT, { - restart: req.restart, - deploymentId: recorder?.deploymentId, - }); - recorder?.seal(); - const rep = await server.replication.replicateOperation(revertOp, { - onPeerResult: recorder - ? (result) => { - recorder.recordPeer(result); - emit('peer', result); - } - : undefined, - }); - if (recorder && rep?.replicated) recorder.recordPeers(rep.replicated); - } - - // Restart on this node (peers replaying an immediate-restart revert restart locally; the rolling - // path is driven only by the direct invoker via a replicated restart_service job). - if (req.restart === true) { - emit('phase', { phase: 'restart', status: 'start' }); - manageThreads.restartWorkers('http'); - emit('phase', { phase: 'restart', status: 'done' }); - response.message = `Reverted component: ${req.project}, restarting Harper`; - } else if (rollingRestart && !isReplicatedExecution) { - const serverUtilities = require('../server/serverHelpers/serverUtilities.ts'); - emit('phase', { phase: 'restart', status: 'start' }); - const jobResponse = await serverUtilities.executeJob({ - operation: 'restart_service', - service: 'http', - replicated: true, - }); - emit('phase', { phase: 'restart', status: 'done' }); - response.restartJobId = jobResponse.job_id; - response.message = `Reverted component: ${req.project}, restarting Harper`; + const result = await revertApplication(application, req.to_deployment_id); + if (result.swapped) { + // Persist the reverted-to version's config + install lock. Compensating by swapping back is + // deliberate: a half-reverted node (new tree live, old config persisted) would reinstall the + // wrong version on its next cold start, which is the failure this pairing exists to prevent. + try { + const configTransaction = await createApplicationConfigTransaction(req.project, result.activatedConfig); + await configTransaction.commit(); + } catch (configError) { + await revertApplication(application, result.fromDeploymentId).catch((swapBackError) => { + log.error(`Failed to undo the ${req.project} revert after its config write failed`, swapBackError); + }); + throw configError; + } } + emit('phase', { phase: 'revert', status: 'done' }); - if (recorder && !req.ignore_replication_errors) { - const failed = recorder.getFailedPeers(); - if (failed.length > 0) { + // Fan out to peers. The operation is idempotent and target-addressed, so a peer that already + // holds the target live converges to the same place without swapping — which is what makes a + // straight re-run of the same operation the right replication primitive here. + let replication; + if (req.replicated !== false && !isReplicatedExecution) { + replication = await server.replication.replicateOperation( + { + operation: hdbTerms.OPERATIONS_ENUM.REVERT_COMPONENT, + project: req.project, + to_deployment_id: req.to_deployment_id, + restart: req.restart === true, + ...(req.force ? { force: req.force } : {}), + ...(req.deployment_timeout !== undefined ? { deployment_timeout: req.deployment_timeout } : {}), + }, + { + onPeerResult: (peerResult) => { + recorder?.recordPeer(peerResult); + emit('peer', peerResult); + }, + } + ); + if (replication?.replicated) recorder?.recordPeers(replication.replicated); + const failed = recorder?.getFailedPeers() ?? []; + if (failed.length && !req.ignore_replication_errors) { throw new ServerError( - `Component '${req.project}' was reverted on the origin but failed to revert on ${failed.length} ` + - `of ${recorder.row.peer_results.length} peer node(s): ${describePeers(failed)}. ` + - `See deployment ${recorder.deploymentId} (get_deployment), or pass ignore_replication_errors: true.` + `Component '${req.project}' reverted on this node but failed on ${failed.length} peer node(s): ` + + `${describePeerFailures(failed)}. The cluster is split across versions; retry the revert (it is a ` + + `no-op on the nodes that already reverted) or roll forward with deploy_component.` ); } } - if (recorder) { - emit('phase', { phase: 'success', status: 'done' }); - await recorder.finish('rolled_back'); - } - return response; - } catch (err) { - throw await finalizeDeployFailure({ err, recorder, installCapture, emit }); + const restart = await restartRevertedComponent(req, emit); + emit('phase', { phase: 'success', status: 'done' }); + await recorder?.finish('rolled_back'); + return { + message: result.swapped + ? `Reverted component: ${req.project} to deployment ${req.to_deployment_id}${restart.restartMessage}` + : `Component ${req.project} is already running deployment ${req.to_deployment_id}; nothing to revert`, + project: req.project, + reverted: result.swapped, + to_deployment_id: req.to_deployment_id, + ...(result.fromDeploymentId ? { from_deployment_id: result.fromDeploymentId } : {}), + ...(recorder ? { deployment_id: recorder.deploymentId } : {}), + ...(replication?.replicated ? { replicated: replication.replicated } : {}), + ...(restart.restartJobId ? { restartJobId: restart.restartJobId } : {}), + }; + } catch (error) { + const message = error?.message ?? String(error); + emit('error', { message, code: error?.statusCode ?? error?.code }); + await recorder + ?.finish('failed', error) + .catch((finishError) => log.warn('Failed to record component revert failure', finishError)); + throw new ServerError(message, error?.statusCode); + } +} + +/** Restart after a revert, mirroring the deploy path's restart handling. */ +async function restartRevertedComponent(req, emit) { + if (req.restart === true) { + emit('phase', { phase: 'restart', status: 'start' }); + manageThreads.restartWorkers('http'); + emit('phase', { phase: 'restart', status: 'done' }); + return { restartMessage: ', restarting Harper' }; + } + if (req.restart === 'rolling') { + const serverUtilities = require('../server/serverHelpers/serverUtilities.ts'); + emit('phase', { phase: 'restart', status: 'start' }); + const jobResponse = await serverUtilities.executeJob({ + operation: 'restart_service', + service: 'http', + replicated: true, + }); + emit('phase', { phase: 'restart', status: 'done' }); + return { restartMessage: ', restarting Harper', restartJobId: jobResponse.job_id }; } + return { restartMessage: '' }; } // ———————————————————————————————————————————————————————————————————————————— @@ -1788,140 +1490,11 @@ async function loadValidateComponent({ dirPath, emit }) { if (lastError) throw lastError; } -// Build a replicated sub-operation body for the peer fan-out. For the two-phase peer phases this is -// `deploy_component` tagged with an internal `_phase` marker (`stage`/`activate`) — the wire format — -// so no separate public op is exposed; revert uses operation `revert_component`. Carries only what a -// peer needs: project, the deployment id (correlation + payload lookup + staging id), the internal -// `_phase`, the build/config inputs, and credential REFERENCES (tokens are already stripped). -function buildReplicatedSubOp(req, operation, { includePayload = false, restart, deploymentId, phase } = {}) { - const op = { operation, project: req.project, _deploymentId: deploymentId ?? req._deploymentId }; - if (phase) op._phase = phase; - if (req.package) op.package = req.package; - if (req.install_command != null) op.install_command = req.install_command; - if (req.install_timeout != null) op.install_timeout = req.install_timeout; - if (req.install_allow_scripts !== undefined) op.install_allow_scripts = req.install_allow_scripts; - if (req.deployment_timeout != null) op.deployment_timeout = req.deployment_timeout; - if (req.urlPath !== undefined) op.urlPath = req.urlPath; - if (req.force !== undefined) op.force = req.force; - if (req.ignore_replication_errors !== undefined) op.ignore_replication_errors = req.ignore_replication_errors; - if (Array.isArray(req.credentials) && req.credentials.length) op.credentials = req.credentials; - if (includePayload && req.payload != null) op.payload = req.payload; - if (restart !== undefined) op.restart = restart; - return op; -} - // Render failed peer outcomes as "node (error)" for an operator-facing error message. function describePeers(failedPeers) { return failedPeers.map((peer) => `${peer.node ?? 'unknown'} (${peer.error?.message ?? 'unknown error'})`).join(', '); } -/** - * Collect per-peer replication outcomes when there is no DeploymentRecorder to hold them — the - * activate-existing path, where the hdb_deployment row was already created (and finished as `staged`) - * by the earlier stage-and-stop. Mirrors DeploymentRecorder.recordPeer's semantics exactly: results are - * normalized by the same normalizePeerResult and upserted by node name, so a peer reported both through - * the streaming `onPeerResult` callback and again in replicateOperation's final `replicated` aggregate - * is counted once, not twice. - */ -function createPeerResultCollector() { - const list = []; - const record = (result) => { - const normalized = normalizePeerResult(result); - const nodeName = normalized.node; - const idx = nodeName ? list.findIndex((entry) => entry.node === nodeName) : -1; - if (idx >= 0) list[idx] = normalized; - else list.push(normalized); - }; - return { - record, - recordAll(results) { - if (Array.isArray(results)) for (const result of results) record(result); - }, - getFailed: () => list.filter((peer) => peer?.status === 'failed'), - get total() { - return list.length; - }, - }; -} - -/** - * Shared post-activate failure gate for every cluster-wide activate (the two-phase deploy's activate - * phase and `deploy_component({ deployment_id })`). replicateOperation never rejects on a per-peer - * failure — failures surface only as 'failed' peer entries — so without this gate a partially-failed - * activate returns 2xx and silently leaves the cluster split across versions. - * - * Unless `ignore_replication_errors` is set, throws when any peer failed to activate. When - * `revert_on_failure` is set, first rolls the origin and the peers that DID activate back to the - * retained previous version so the cluster reconverges — best-effort, since a revert failure must not - * mask the original activate failure. - */ -async function enforceActivatePeerGate({ req, application, emit, failed, totalPeers, deploymentId }) { - if (req.ignore_replication_errors) return; - if (!failed || failed.length === 0) return; - let revertNote = ''; - if (req.revert_on_failure) { - try { - emit('phase', { phase: 'revert', status: 'start' }); - // The origin activated, so revert it. - await revertApplication(application); - // With `restart: true` the origin's workers already reloaded onto the new (failed-cluster) - // version — the origin restart runs before this gate — so the directory rollback above is not - // picked up on its own. The peers' revert op carries `restart`, so they DO come back on the - // previous version; without this the origin would be the one node left serving the new version, - // the exact opposite of the reconvergence revert_on_failure exists to provide. A rolling restart - // arrives here with `restart` already normalized to false and its peers likewise un-restarted, - // so origin and peers stay consistent in that case without a second restart. - if (req.restart === true) manageThreads.restartWorkers('http'); - // Revert ONLY the peers that successfully activated (see selectRevertTargets): every known - // node minus the ones that failed to activate (still on the correct version) and minus this - // node (already reverted directly above; a second bidirectional revert would flip it back). - // replicateOperation has no subset targeting, so send point-to-point via sendOperationToNode. - const { getThisNodeName } = require('../server/nodeName.ts'); - const activatedPeers = selectRevertTargets(server.nodes, failed, getThisNodeName()); - const revertOp = buildReplicatedSubOp(req, hdbTerms.OPERATIONS_ENUM.REVERT_COMPONENT, { - restart: req.restart, - deploymentId, - }); - revertOp.replicated = false; // point-to-point; the peer must not re-fan the revert - const revertResults = await Promise.allSettled( - activatedPeers.map((node) => server.replication.sendOperationToNode(node, revertOp)) - ); - const revertFailures = revertResults.filter((result) => result.status === 'rejected').length; - emit('phase', { phase: 'revert', status: 'done' }); - revertNote = - ` Rolled the origin and ${activatedPeers.length - revertFailures} of ${activatedPeers.length} ` + - `activated peer(s) back to the previous version (revert_on_failure); the ${failed.length} peer(s) ` + - `that never activated were left on their current (correct) version.` + - (revertFailures > 0 ? ` ${revertFailures} peer revert(s) also failed.` : '') + - ` Verify with get_components.`; - } catch (revertErr) { - log.warn('revert_on_failure rollback failed', revertErr); - revertNote = ` An automatic rollback (revert_on_failure) was attempted but also failed: ${revertErr?.message ?? revertErr}.`; - } - } - throw new ServerError( - `Component '${application.name}' was activated on the origin but failed to activate on ${failed.length} ` + - `of ${totalPeers} peer node(s): ${describePeers(failed)}. Those nodes have the staged ` + - `build but did not go live.${revertNote} See deployment ${deploymentId} (get_deployment), or pass ` + - `ignore_replication_errors: true.` - ); -} - -// Choose which peers a revert_on_failure swap-back should target: every known node EXCEPT -// - `thisNodeName`: the origin, already reverted directly by the caller — a second (bidirectional) -// revert would flip it back to the just-activated version; and -// - any node in `failedPeers`: it never activated (its failure fired before activateApplication ran -// retainAsPrevious), so its live directory is still the correct pre-deploy version and reverting it -// would roll it back an EXTRA version onto a two-deploys-ago copy. -// Pure and exported so the node-targeting logic (which had two review-caught bugs — the failed-peer -// skip and the self-skip) is unit-testable without a live cluster. `nodes` is `server.nodes`, which -// normally already excludes self, but a not-yet-named node can slip in (knownNodes) so self is guarded -// here regardless — matching every other point-to-point fan-out in the code base (bin/restart.ts). -function selectRevertTargets(nodes, failedPeers, thisNodeName) { - const failedNodeNames = new Set((failedPeers ?? []).map((peer) => peer.node).filter(Boolean)); - return (nodes ?? []).filter((node) => node?.name !== thisNodeName && !failedNodeNames.has(node?.name)); -} - // Reclaim the payload tarball for large deploys once every peer has installed from the blob. Dropping // the reference before finish() folds the null into the single terminal write, which unlinks the file // locally and replicates the null so peers drop their copies too. Metadata is retained. Kept when a @@ -2489,6 +2062,7 @@ exports.dropCustomFunctionProject = dropCustomFunctionProject; exports.packageComponent = packageComponent; exports.deployComponent = deployComponent; exports.componentDeployPhase = componentDeployPhase; +exports.revertComponent = revertComponent; exports.getComponents = getComponents; exports.getComponentFile = getComponentFile; exports.setComponentFile = setComponentFile; diff --git a/components/operationsValidation.js b/components/operationsValidation.js index e7d7c157ff..406936d1f0 100644 --- a/components/operationsValidation.js +++ b/components/operationsValidation.js @@ -32,6 +32,7 @@ module.exports = { packageComponentValidator, deployComponentValidator, componentDeployPhaseValidator, + revertComponentValidator, setComponentFileValidator, getComponentFileValidator, dropComponentFileValidator, @@ -576,26 +577,30 @@ function componentDeployPhaseValidator(req) { /** * Validate revert_component requests — swap a component's live version back to its retained previous - * version. No build inputs (nothing is fetched or installed); just the project, an optional restart, - * and the replication controls. + * version. There are no build inputs: nothing is fetched, resolved or installed, because the bytes + * being reverted to are already on disk. Just the project, the version being targeted, an optional + * restart, and the replication controls. * @param req * @returns {*} */ -function _revertComponentValidator(req) { +function revertComponentValidator(req) { const revertSchema = Joi.object({ project: Joi.string() .pattern(PROJECT_FILE_NAME_REGEX) .required() .messages({ 'string.pattern.base': HDB_ERROR_MSGS.BAD_PROJECT_NAME }), - // The deployment being reverted, recorded as the rollback's `rollback_of` for the audit trail. - // Optional — revert operates on whatever version is currently live regardless. Same safe charset - // as elsewhere (it is an id, and this keeps the deploy family's `deployment_id` consistent). - deployment_id: Joi.string().pattern(PROJECT_FILE_NAME_REGEX).optional().messages({ - 'string.pattern.base': `'deployment_id' must only contain letters, numbers, dashes, and underscores`, + // The deployment the caller expects to be live once this returns. REQUIRED, and the whole reason + // revert is safe to retry: a revert that names its target is a no-op when that version is already + // live, where a bare "swap to the other one" toggle would flip the rejected release back in if a + // caller retried after losing the first response (harper#1849 review). + to_deployment_id: Joi.string().pattern(DEPLOYMENT_ID_REGEX).required().messages({ + 'string.pattern.base': `'to_deployment_id' must be a UUID`, + 'any.required': `'to_deployment_id' is required: name the deployment you expect to be live after the revert (list_deployments reports it, and deploy_component returns it)`, }), restart: Joi.alternatives().try(Joi.boolean(), Joi.string().valid('rolling')).optional(), deployment_timeout: Joi.number().min(0).optional(), ignore_replication_errors: Joi.boolean().optional(), + force: Joi.boolean().optional(), }); return validator.validateBySchema(req, revertSchema); diff --git a/server/serverHelpers/serverHandlers.js b/server/serverHelpers/serverHandlers.js index f38dd65f5f..d75baee1dd 100644 --- a/server/serverHelpers/serverHandlers.js +++ b/server/serverHelpers/serverHandlers.js @@ -33,6 +33,7 @@ const { ProgressEmitter, createSSEResponseStream } = require('./progressEmitter. // the client disconnects (the emitter's abort signal), so subscribers see new lines live. const SSE_PROGRESS_OPERATIONS = new Set([ terms.OPERATIONS_ENUM.DEPLOY_COMPONENT, + terms.OPERATIONS_ENUM.REVERT_COMPONENT, terms.OPERATIONS_ENUM.GET_DEPLOYMENT, terms.OPERATIONS_ENUM.READ_LOG, ]); diff --git a/server/serverHelpers/serverUtilities.ts b/server/serverHelpers/serverUtilities.ts index 0c1e27dea6..719cd10045 100644 --- a/server/serverHelpers/serverUtilities.ts +++ b/server/serverHelpers/serverUtilities.ts @@ -560,6 +560,10 @@ function initializeOperationFunctionMap(): Map { @@ -386,4 +391,201 @@ describe('two-phase component directory transaction', function () { await cleanup(name); await fs.rm(packageDirectory, { recursive: true, force: true }); }); + // ———————————————————————————————————————————————————————————————————————————— + // Retained previous + addressed revert (harper#1849 review, @kriszyp) + // ———————————————————————————————————————————————————————————————————————————— + + it('retains the tree an activation displaced, addressed by the deployment that produced it', async () => { + const name = fixtureName(); + const firstId = randomUUID(); + const secondId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('v1') }); + await stageApplication(application, firstId); + await activateStagedApplication(application, firstId, { activationSpec: { package: null } }); + application.payload = await makeComponentPayload('v2'); + await stageApplication(application, secondId); + await activateStagedApplication(application, secondId, { activationSpec: { package: null } }); + + assert.match(await readMarker(application.dirPath), /v2/); + const retained = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); + assert.match(await readMarker(retained), /v1/); + + const target = await getRevertTarget(application.dirPath); + assert.equal(target.live.deployment_id, secondId); + assert.equal(target.previous.deployment_id, firstId); + await cleanup(name); + }); + + it('a first-ever deploy retains nothing, so there is no version to revert to', async () => { + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('only') }); + await stageApplication(application, deploymentId); + await activateStagedApplication(application, deploymentId, { activationSpec: { package: null } }); + + assert.equal(await getRevertTarget(application.dirPath), undefined); + await assert.rejects( + () => revertApplication(application, randomUUID()), + /no previous version is retained/, + 'a component deployed once cannot be reverted' + ); + await cleanup(name); + }); + + it('reverts to the named previous deployment and exchanges the retained roles', async () => { + const name = fixtureName(); + const firstId = randomUUID(); + const secondId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('v1') }); + await stageApplication(application, firstId); + await activateStagedApplication(application, firstId, { activationSpec: { package: null } }); + application.payload = await makeComponentPayload('v2'); + await stageApplication(application, secondId); + await activateStagedApplication(application, secondId, { activationSpec: { package: null } }); + + const result = await revertApplication(application, firstId); + + assert.equal(result.swapped, true); + assert.equal(result.fromDeploymentId, secondId); + assert.match(await readMarker(application.dirPath), /v1/, 'live is the reverted-to version'); + assert.match( + await readMarker(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name)), + /v2/, + 'the displaced version becomes the new retained previous' + ); + + // Explicitly targeting the other direction rolls forward again. + const forward = await revertApplication(application, secondId); + assert.equal(forward.swapped, true); + assert.match(await readMarker(application.dirPath), /v2/); + await cleanup(name); + }); + + it('is a no-op when the named deployment is already live, so a retry cannot toggle it back', async () => { + const name = fixtureName(); + const firstId = randomUUID(); + const secondId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('v1') }); + await stageApplication(application, firstId); + await activateStagedApplication(application, firstId, { activationSpec: { package: null } }); + application.payload = await makeComponentPayload('v2'); + await stageApplication(application, secondId); + await activateStagedApplication(application, secondId, { activationSpec: { package: null } }); + await revertApplication(application, firstId); + + // The delivery of the first response is lost and the caller retries the identical request. A + // bidirectional toggle would put the rejected v2 back live; an addressed revert must not. + const retry = await revertApplication(application, firstId); + + assert.equal(retry.swapped, false, 'a repeated revert to the live version does nothing'); + assert.match(await readMarker(application.dirPath), /v1/, 'still on the reverted-to version'); + await cleanup(name); + }); + + it('refuses a deployment that is neither live nor the retained previous', async () => { + const name = fixtureName(); + const firstId = randomUUID(); + const secondId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('v1') }); + await stageApplication(application, firstId); + await activateStagedApplication(application, firstId, { activationSpec: { package: null } }); + application.payload = await makeComponentPayload('v2'); + await stageApplication(application, secondId); + await activateStagedApplication(application, secondId, { activationSpec: { package: null } }); + + await assert.rejects( + () => revertApplication(application, randomUUID()), + /neither the live version .* nor the retained previous version/s, + 'only one previous version is retained, so anything else is a redeploy' + ); + assert.match(await readMarker(application.dirPath), /v2/, 'a refused revert changes nothing'); + await cleanup(name); + }); + + it('retains exactly one previous version across three activations', async () => { + const name = fixtureName(); + const ids = [randomUUID(), randomUUID(), randomUUID()]; + const application = new Application({ name, payload: await makeComponentPayload('v1') }); + for (const [index, id] of ids.entries()) { + if (index > 0) application.payload = await makeComponentPayload(`v${index + 1}`); + await stageApplication(application, id); + await activateStagedApplication(application, id, { activationSpec: { package: null } }); + } + + assert.match(await readMarker(application.dirPath), /v3/); + assert.match(await readMarker(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name)), /v2/); + const target = await getRevertTarget(application.dirPath); + assert.equal(target.previous.deployment_id, ids[1], 'v1 is evicted; only v2 stays revertable'); + await cleanup(name); + }); + + it('parks the evicted retained-previous where startup recovery will never restore it', async () => { + // The single `.deploy-aside` contract: an `.in-progress-` directory with no `.retired-` marker is + // a rollback record that recoverInterruptedComponentExtractions restores OVER the live component. + // An evicted two-deploys-ago tree is known garbage when parked, so it must not carry that prefix, + // or a crash before the sweep would resurrect an ancient version over the current one. + const name = fixtureName(); + const ids = [randomUUID(), randomUUID(), randomUUID()]; + const application = new Application({ name, payload: await makeComponentPayload('v1') }); + for (const [index, id] of ids.entries()) { + if (index > 0) application.payload = await makeComponentPayload(`v${index + 1}`); + await stageApplication(application, id); + await activateStagedApplication(application, id, { activationSpec: { package: null } }); + } + + const asideDir = path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR, name); + const parked = existsSync(asideDir) ? await fs.readdir(asideDir) : []; + const unretired = parked.filter( + (entry) => entry.startsWith('.in-progress-') && !parked.includes(`.retired-${entry.slice('.in-progress-'.length)}`) + ); + assert.deepEqual(unretired, [], 'an evicted previous is never left looking like a rollback record'); + await cleanup(name); + }); + + it('reports the config each retained version was activated with, so a revert can restore it', async () => { + // This is what makes a revert durable across a cold restart: reverting away from a `package` + // deploy has to take the package reference out of root config too, or installApplications() + // reinstalls the reverted-away version over the restored directory on the next boot. + const name = fixtureName(); + const packagedId = randomUUID(); + const payloadId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('packaged') }); + await stageApplication(application, packagedId); + await activateStagedApplication(application, packagedId, { + activationSpec: { + package: 'stage-fixture@1.0.0', + install_command: null, + install_timeout: null, + install_allow_scripts: null, + urlPath: null, + host: null, + }, + }); + application.payload = await makeComponentPayload('plain'); + await stageApplication(application, payloadId); + await activateStagedApplication(application, payloadId, { activationSpec: { package: null } }); + + const target = await getRevertTarget(application.dirPath); + assert.equal(target.previous.application_config.package, 'stage-fixture@1.0.0'); + const back = await revertApplication(application, packagedId); + assert.equal(back.activatedConfig.package, 'stage-fixture@1.0.0'); + await cleanup(name); + }); + + it('moves a dangling symlink at the live path aside instead of failing EEXIST', async () => { + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('candidate') }); + const missingTarget = path.join(os.tmpdir(), `harper-missing-${randomUUID()}`); + await fs.mkdir(COMPONENTS_ROOT, { recursive: true }); + await fs.symlink(missingTarget, application.dirPath, 'dir'); + + await stageApplication(application, deploymentId); + await activateStagedApplication(application, deploymentId, { activationSpec: { package: null } }); + + assert.equal((await fs.lstat(application.dirPath)).isSymbolicLink(), false); + assert.match(await readMarker(application.dirPath), /candidate/); + await cleanup(name); + }); + }); diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index cfcd7f0625..215619bd61 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -325,6 +325,9 @@ export const OPERATIONS_ENUM = { // older nodes fail closed instead of interpreting an unknown phase field as a one-shot deploy. DEPLOY_COMPONENT: 'deploy_component', COMPONENT_DEPLOY_PHASE: 'component_deploy_phase', + // Put a component's retained previous version back in service cluster-wide. A distinct public + // operation rather than a deploy phase: it fetches, resolves and installs nothing. + REVERT_COMPONENT: 'revert_component', READ_TRANSACTION_LOG: 'read_transaction_log', DELETE_TRANSACTION_LOGS_BEFORE: 'delete_transaction_logs_before', INSTALL_NODE_MODULES: 'install_node_modules', diff --git a/utility/operation_authorization.ts b/utility/operation_authorization.ts index c839bed14e..9e8c7318f0 100644 --- a/utility/operation_authorization.ts +++ b/utility/operation_authorization.ts @@ -312,6 +312,7 @@ requiredPermissions.set(functionsOperations.dropCustomFunctionProject.name, new requiredPermissions.set(functionsOperations.packageComponent.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.deployComponent.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.componentDeployPhase.name, new (permission as any)(true, [])); +requiredPermissions.set(functionsOperations.revertComponent.name, new (permission as any)(true, [])); requiredPermissions.set( deploymentOperations.handleListDeployments.name, new (permission as any)(true, [], terms.OPERATIONS_ENUM.LIST_DEPLOYMENTS) From 07fc92107ce554b80d12986512bee43b525f4a16 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Wed, 12 Aug 2026 12:18:12 -0400 Subject: [PATCH 42/94] fix(deploy): compare package metadata across the two-phase swap for the restart gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses @heskew's finding 2 on harper#1849. Main's restart gate is `isNewComponent || packageMetadataChanged` (since 84201118e): installed package metadata sits outside most plugin watch globs, so no file watcher notices a dependency or module-entry change — but it does invalidate already-loaded code. `prepareApplication` computes that by reading metadata before extraction and after install, both in place. The two-phase path never touches the live directory until the swap, so it bypassed that comparison entirely and gated only on `isNewComponent`. A redeploy with changed dependency metadata and `restart: false` would have silently skipped `restartRequired` on the new default path. `activateStagedApplication` now performs the equivalent comparison at the only point where both trees exist: the outgoing live tree against the staged tree, just before the swap, feeding `application.packageMetadataChanged` into the shared `markRestartRequiredForDeploy` gate. The opaque-install flag needed somewhere durable to live. `installationIsOpaque` is set during the stage's install, but on a peer the stage and the activation are separate operation invocations with separate Application objects, so an in-memory flag is gone by the time that peer activates — and an opaque install (bundled node_modules, a custom install command) is exactly the case where comparing metadata proves nothing. The stage now records it inside the completion marker it already writes, and activation reads it back. An empty or unparseable marker (a stage written by an older build) is treated as opaque, so the gate errs toward requiring a restart rather than silently skipping one. Tests: four cases pinning the gate — an unchanged redeploy with a comparable install stays quiet (the harper#1806 guarantee), a changed package.json version requires a restart, installable dependencies with no lockfile require one, and a bundled-node_modules redeploy requires one because its install can't be compared. The first of these restores the negative-case coverage #2135 dropped, corrected to main's current semantics. 288 deploy/component/CLI/server unit tests pass. Co-Authored-By: Claude Opus 5 --- components/Application.ts | 41 +++++++++--- .../components/deployPhaseOperations.test.js | 67 +++++++++++++++++++ unitTests/components/deployStaging.test.js | 4 +- 3 files changed, 102 insertions(+), 10 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 1fd4396416..4647faf079 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -691,10 +691,7 @@ async function readRetainedPreviousManifest(liveDirPath: string): Promise { +async function writeRetainedPreviousManifest(liveDirPath: string, manifest: RetainedPreviousManifest): Promise { const manifestPath = previousManifestPathFor(liveDirPath); // Temp + rename so a crash mid-write can never leave a half-written manifest, which would make a // retained tree unaddressable (and so unrevertable). @@ -2195,7 +2192,11 @@ export async function stageApplication(application: Application, deploymentId: s await application.cleanupGitCredentialSession(); } await installApplication(application); - await writeFile(join(deploymentDirPath!, STAGED_COMPLETE_MARKER), '', { flag: 'wx', mode: 0o600 }); + await writeFile( + join(deploymentDirPath!, STAGED_COMPLETE_MARKER), + JSON.stringify({ installationIsOpaque: application.installationIsOpaque }), + { flag: 'wx', mode: 0o600 } + ); } catch (error) { await rm(stagingDirPath, { recursive: true, force: true }).catch(() => {}); await rm(join(deploymentDirPath!, STAGED_COMPLETE_MARKER), { force: true }).catch(() => {}); @@ -2255,6 +2256,21 @@ export async function hasCompleteStagedApplication(stagingDirPath: string): Prom ); } +/** + * What the stage recorded about its own install. Falls back to "opaque" when the marker carries no + * usable content: an unknown install has to be treated as one whose result can't be compared, so the + * restart gate errs toward requiring a restart rather than silently skipping one. + */ +async function readStagedCompletion(stagingDirPath: string): Promise<{ installationIsOpaque: boolean }> { + try { + const raw = await readFile(join(dirname(stagingDirPath), STAGED_COMPLETE_MARKER), 'utf8'); + if (!raw.trim()) return { installationIsOpaque: true }; + return { installationIsOpaque: JSON.parse(raw)?.installationIsOpaque !== false }; + } catch { + return { installationIsOpaque: true }; + } +} + /** Atomically replace the live component and compensate if persistent activation work fails. */ export async function activateStagedApplication( application: Application, @@ -2288,9 +2304,8 @@ export async function activateStagedApplication( await ensureSecureDirectory(activationDir, true, 'Component activation staging path'); // Who is live right now, so the tree this activation displaces stays addressable for revert. Read // before the swap, since the swap is what makes it "previous". - const outgoing: RetainedVersion = (await readRetainedPreviousManifest(application.dirPath).catch( - () => undefined - ))?.live ?? { + const outgoing: RetainedVersion = (await readRetainedPreviousManifest(application.dirPath).catch(() => undefined)) + ?.live ?? { // No manifest yet: either a first-ever deploy, or a component last activated by a Harper that // predates retention. Record the config it is running so a revert can still restore that, even // though its deployment id is unknowable and so cannot be a revert target. @@ -2310,6 +2325,16 @@ export async function activateStagedApplication( try { await lstat(application.dirPath); application.isNewComponent = false; + // Installed package metadata sits outside most plugin watch globs, so no file watcher sees a + // dependency or module-entry change — but it does invalidate already-loaded code. The one-shot + // path compares it across its in-place install (prepareApplication); the two-phase path has to + // compare the outgoing live tree against the staged one, which is only possible here, while + // both still exist. Feeds markRestartRequiredForDeploy (harper#674, harper#1849 @heskew). + application.packageMetadataChanged = installedRuntimeChanged( + await readInstalledPackageMetadata(application.dirPath), + await readInstalledPackageMetadata(stagingDirPath), + (await readStagedCompletion(stagingDirPath)).installationIsOpaque + ); backupPath = join( activationDir, `${ACTIVATION_BACKUP_PREFIX}${deploymentId}-${Date.now()}-${process.pid}-${randomUUID()}` diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index 6cc1ebcc37..80bba03bc8 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -241,6 +241,73 @@ describe('deploy_component two-phase orchestration', function () { assert.equal(restartNeeded(), true, 'a new component activated without restart requires one'); }); + // ———————————————————————————————————————————————————————————————————————————— + // Restart gate across the swap (harper#674 / harper#1849 @heskew finding 2) + // ———————————————————————————————————————————————————————————————————————————— + + it('redeploying an unchanged component with a comparable install does NOT require a restart', async () => { + // The harper#1806 guarantee: a redeploy of an already-loaded component stays quiet, because that + // component's own file watcher (Scope/EntryHandler) requests a restart if the changed files + // actually need one. It holds only when the install is COMPARABLE — no bundled node_modules, so + // the staged tree's metadata can be checked against the outgoing one. + const project = name(); + await operations.deployComponent({ project, payload: await makePayload('gate-same', '1.0.0', false) }); + resetRestartNeeded(); // clear the flag the first (new-component) deploy legitimately set + + await operations.deployComponent({ project, payload: await makePayload('gate-same', '1.0.0', false) }); + + assert.equal(restartNeeded(), false, 'identical package metadata across the swap must stay quiet'); + }); + + it('requires a restart when a redeploy bundles node_modules, whose install cannot be compared', async () => { + // A payload that ships its own node_modules skips the install entirely, so nothing about the + // resulting tree can be compared against a fresh install — the runtime is opaque and has to be + // assumed changed. Same conclusion the one-shot path reaches for the same payload shape. + const project = name(); + await operations.deployComponent({ project, payload: await makePayload('gate-opaque-bundle', '1.0.0') }); + resetRestartNeeded(); + + await operations.deployComponent({ project, payload: await makePayload('gate-opaque-bundle', '1.0.0') }); + + assert.equal(restartNeeded(), true, 'a bundled node_modules redeploy is opaque, so it requires a restart'); + }); + + it('requires a restart when a redeploy changes package metadata the watchers cannot see', async () => { + // Installed package metadata is deliberately outside most plugin watch globs, so nothing else + // notices a dependency or module-entry change — but it invalidates already-loaded code. The + // one-shot path compares it across its in-place install; two-phase has to compare the outgoing + // live tree against the staged one at swap time, which is what this pins. + const project = name(); + await operations.deployComponent({ project, payload: await makePayload('gate-change', '1.0.0') }); + resetRestartNeeded(); + + await operations.deployComponent({ project, payload: await makePayload('gate-change', '2.0.0') }); + + assert.equal(restartNeeded(), true, 'a changed package.json version must still force a restart'); + }); + + it('requires a restart when a redeploy ships no lockfile for its dependencies', async () => { + // An install whose result can't be reproduced or compared (installable dependencies, no lockfile) + // is treated as opaque, and an opaque install always requires a restart. + const project = name(); + await operations.deployComponent({ project, payload: await makePayload('gate-opaque', '1.0.0') }); + resetRestartNeeded(); + + const source = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-op-opaque-')); + await fs.writeFile( + path.join(source, 'package.json'), + JSON.stringify({ name: 'phase-op', version: '1.0.0', dependencies: { 'some-dep': '1.0.0' } }) + ); + await fs.writeFile(path.join(source, 'index.js'), "module.exports = 'gate-opaque';\n"); + await fs.mkdir(path.join(source, 'node_modules'), { recursive: true }); + const payload = await packDirectory(source); + await fs.rm(source, { recursive: true, force: true }); + + await operations.deployComponent({ project, payload }); + + assert.equal(restartNeeded(), true, 'dependencies with no lockfile make the install opaque'); + }); + it('reclaims an oversized payload only after a full two-phase activation succeeds', async () => { const project = name(); const priorMaxSize = environment.get(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE); diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index a5cefa4060..b2a4a3c068 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -536,7 +536,8 @@ describe('two-phase component directory transaction', function () { const asideDir = path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR, name); const parked = existsSync(asideDir) ? await fs.readdir(asideDir) : []; const unretired = parked.filter( - (entry) => entry.startsWith('.in-progress-') && !parked.includes(`.retired-${entry.slice('.in-progress-'.length)}`) + (entry) => + entry.startsWith('.in-progress-') && !parked.includes(`.retired-${entry.slice('.in-progress-'.length)}`) ); assert.deepEqual(unretired, [], 'an evicted previous is never left looking like a rollback record'); await cleanup(name); @@ -587,5 +588,4 @@ describe('two-phase component directory transaction', function () { assert.match(await readMarker(application.dirPath), /candidate/); await cleanup(name); }); - }); From ed3294f4058b681fe3110add5d940ca80cf1fab4 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Wed, 12 Aug 2026 12:20:13 -0400 Subject: [PATCH 43/94] fix(deploy): use lstat, not access, to detect an occupied extraction target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Re-fixes the dangling-symlink finding from the Gemini review on harper#1849 (thread 3605636958), which resurfaced through #2066's rewritten extractApplication. `access(path, F_OK)` follows symlinks, so a DANGLING symlink at the extraction target — left behind by a prior `file:`-directory deploy whose target was removed — reports ENOENT. The transaction then concluded nothing occupied the path, skipped moving it aside, and the subsequent `mkdir` failed EEXIST because the dead link was still sitting there. `lstat` sees the link itself, so it is parked aside like any other occupant. The original fix was on the pre-#2066 `moveDirAside`, which this branch no longer has; it survives on the two-phase swap path (activateStagedApplication already used lstat) but had been lost on the in-place path. Both are now covered by regression tests, so neither can silently regress again. The 6 remaining failures in extractApplicationSwap.test.js are pre-existing on kris/deploy-peer-rollback and unaffected by this change. Co-Authored-By: Claude Opus 5 --- components/Application.ts | 6 +++++- unitTests/components/deployStaging.test.js | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/components/Application.ts b/components/Application.ts index 4647faf079..0e8fd5cea1 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -1124,7 +1124,11 @@ export async function extractApplication( await recoverOrCleanupStaleExtractionPaths(extractionContext, asideStagingDir); let componentExists = true; try { - await access(buildDirPath, constants.F_OK); + // lstat, not access(F_OK): access FOLLOWS symlinks, so a DANGLING symlink at the target (left by + // a prior `file:`-directory deploy whose target was removed) reports ENOENT and would be treated + // as "nothing here" — then the mkdir below fails EEXIST because the dead link still occupies the + // path. lstat sees the link itself, so it gets moved aside like any other occupant. + await lstat(buildDirPath); } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; componentExists = false; diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index b2a4a3c068..0f66e72359 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -22,6 +22,7 @@ const { createApplicationActivationTransaction, revertApplication, getRevertTarget, + extractApplication, stagedApplicationPath, DEPLOY_STAGING_DIR, DEPLOY_ACTIVATION_DIR, @@ -588,4 +589,20 @@ describe('two-phase component directory transaction', function () { assert.match(await readMarker(application.dirPath), /candidate/); await cleanup(name); }); + it('extracts in place over a DANGLING symlink at the component path instead of failing EEXIST', async () => { + // access(F_OK) follows symlinks, so a dead link at the target reports ENOENT and would be treated + // as "nothing here" — then mkdir fails EEXIST because the link still occupies the path. lstat sees + // the link itself. Left by a prior `file:`-directory deploy whose target was removed. + const name = fixtureName(); + const application = new Application({ name, payload: await makeComponentPayload('over-dead-link') }); + await fs.mkdir(COMPONENTS_ROOT, { recursive: true }); + await fs.symlink(path.join(os.tmpdir(), `harper-missing-${randomUUID()}`), application.dirPath, 'dir'); + + await extractApplication(application); + + assert.equal((await fs.lstat(application.dirPath)).isSymbolicLink(), false); + assert.match(await readMarker(application.dirPath), /over-dead-link/); + await cleanup(name); + }); + }); From fdc6682dc9f0dec4cf820a8a70f3119fe93528d9 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Wed, 12 Aug 2026 12:20:31 -0400 Subject: [PATCH 44/94] style: prettier formatting for the new revert regression tests Co-Authored-By: Claude Opus 5 --- unitTests/components/deployStaging.test.js | 1 - 1 file changed, 1 deletion(-) diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 0f66e72359..a7fa641fb6 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -604,5 +604,4 @@ describe('two-phase component directory transaction', function () { assert.match(await readMarker(application.dirPath), /over-dead-link/); await cleanup(name); }); - }); From 4d2a854a21ec571ee127fefd79f184d3c5573ca8 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Wed, 12 Aug 2026 12:22:27 -0400 Subject: [PATCH 45/94] docs(deploy): make DESIGN.md match the code again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DESIGN.md had gone internally inconsistent: one section said there is "deliberately no toggle-style automatic revert", while the Reversibility section still described `revert_component` as a bidirectional toggle and `revert_on_failure` as a shipping opt-in — neither of which is true after #2135 removed them and this branch restored revert in addressed form. Rewritten to describe what the code does: revert is addressed (a required `to_deployment_id`, a no-op when already live, refused for anything but the retained previous), it carries root config and the install lock with the directory swap, and `revert_on_failure` is rejected — with the reason stated, since it is the non-obvious part: past the barrier, "peer reported failed" does not imply "peer did not activate", so auto-reverting failed peers splits the cluster three ways instead of reconverging it. Also documents the single `.deploy-aside` contract (`.in-progress-` = rollback record restored at startup, `.retired-` = commit marker, `.discarded-` = parked disposable and never restored) and why the two startup recovery passes cannot claim each other's artifacts. Stale symbol references cleaned up across DESIGN.md, Application.ts, operations.js and operationsValidation.js: `activateApplication` (removed with the legacy path — now `activateStagedApplication`), `deployPhaseStage`/ `deployPhaseActivate` (now the `component_deploy_phase` operation), and `stage_component`/`activate_component`, which have not existed as public operations since d80e69531 folded the phases into `deploy_component`. Co-Authored-By: Claude Opus 5 --- DESIGN.md | 59 ++++++++++++++++++++++-------- components/Application.ts | 20 +++++----- components/operations.js | 5 ++- components/operationsValidation.js | 2 +- 4 files changed, 58 insertions(+), 28 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 3e1cef8020..28d70e622d 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -186,8 +186,12 @@ An older peer therefore rejects the unknown operation instead of ignoring a phas the staged build live. Public `_phase`/`_deploymentId` fields are rejected; the latter remains accepted only on the authenticated legacy one-shot replication path. Restart is gated until activation responses have settled. A partial activation is reported as split-node state and recovered by staging and activating -a known-good build; there is deliberately no toggle-style automatic revert because retrying one after a -lost response can reverse the recovery. `deployment_stagingRetention_maxCount` bounds resting staged +a known-good build, or rolled back explicitly with `revert_component`, which is addressed rather than a +toggle (see "Reversibility" below) so a retry after a lost response cannot reverse the recovery. There is +deliberately no AUTOMATIC rollback (`revert_on_failure` is rejected): once any node is past the barrier, +"this peer reported failed" does not mean "this peer did not activate" — a peer can complete its swap and +then fail the persistent work that follows — so auto-reverting the failed peers would roll an untouched +node an extra version back and split the cluster three ways. `deployment_stagingRetention_maxCount` bounds resting staged trees per component and payload retention is pruned in the same row-aware lifecycle. ## Peer-side deploy_component payload read: retryable blob stalls and `Readable.from()` cancellation @@ -413,7 +417,7 @@ legacy one-shot path. operations; the peer fan-out is `deploy_component` itself tagged with an internal `_phase: 'stage' | 'activate'` marker (the same `_`-prefixed internal-field convention peers already branch on, alongside `_deploymentId`). `deployComponent` dispatches: a replicated execution with `_phase` runs the peer -stage/activate work (`deployPhaseStage` / `deployPhaseActivate`) and never re-fans; a public call runs +stage/activate work (the `component_deploy_phase` operation) and never re-fans; a public call runs the origin orchestrator. Two public properties expose the phases when an operator wants them separated (e.g. pre-stage the cluster now, flip later — or a CI-stages / approver-activates split): `activate: false` stages cluster-wide and stops, returning the `deployment_id` in a `staged` state; passing that @@ -444,7 +448,7 @@ exists to remove. The leading dot keeps `loadComponentDirectories` from loading component, and it is **not** the watched base of any component's file watcher (those are rooted at each live component dir, `EntryHandler`/`deriveCommonPatternBase`) — so building here fires no restart-on-change events and needs no `deploy:start` watcher suppression. That suppression is now -scoped to `activateApplication`, the only phase that writes the live path. Staging is deterministic +scoped to `activateStagedApplication`, the only phase that writes the live path. Staging is deterministic from the deployment id precisely so the activate phase (a separate replicated `deploy_component` invocation on peers, tagged `_phase: 'activate'`) can reconstruct the same path the stage built — peers build a fresh `Application` per phase invocation, so there is no shared in-memory handle to rely @@ -482,17 +486,42 @@ authenticate node-to-node by TLS certificate, and the receive side runs the op v dispatched by `operation` name) replicate without an `hdb_user`, identically to the long-proven one-shot `deploy_component` fan-out. -**Reversibility: retained previous + `revert_component`.** `activateApplication` no longer discards the -outgoing live version — it retains it as `.deploy-previous/` (`retainAsPrevious`, evicting the -older one so exactly one previous is kept per component). `revert_component` swaps the live directory -with that retained previous via three same-filesystem renames through a hidden holding path, cluster- -wide and replicated like activate. The swap is bidirectional, so reverting a revert rolls forward -again. This backs two things: a customer can deploy, run their own health checks against the live -version, and `revert` if unhappy even when the cluster looks healthy; and `deploy_component`'s opt-in -`revert_on_failure` rolls the whole cluster back to the previous version when the activate phase leaves -some nodes live and some not, so the cluster reconverges on one version. The previous copy is retained -per-node (each node retains its own outgoing version during its own activate), so a replicated revert -has a local rollback source on every node. +**Reversibility: retained previous + `revert_component`.** `activateStagedApplication` does not discard +the tree its swap displaced — it retains it as `.deploy-previous/`, evicting the older one so +exactly one previous is kept per component (a bounded one extra copy). `revert_component` swaps the live +directory with that retained previous via three same-filesystem renames through a hidden holding path, +cluster-wide, and fetches nothing: no package resolution, no secret decryption, no artifact download, no +install. That is the point — the rollback operators actually need is the bad rollout that just happened, +and every node already has those bytes. + +**It is addressed, not toggled.** `to_deployment_id` is required and names the version the caller expects +live afterwards. If that version is already live the call is a no-op success; if it matches the retained +previous, the swap happens and the displaced tree becomes the new retained previous. Anything else is +refused, naming what the component can actually revert to. A bare "swap to the other one" toggle is +unsafe under ordinary request retries — a caller that loses the response and retries would flip the +rejected release back in — which is why the target is mandatory. Reaching an older version is a redeploy, +not a revert. + +**The swap carries persistent state with it.** Each retained tree has a sidecar manifest recording the +deployment that produced it and the root-config entry it was activated with, and revert applies that +entry to both the root config and `harper-application-lock.json` +(`createApplicationConfigTransaction`, shared with activation). A null entry means REMOVE it: reverting +away from a `package` deploy has to drop the package reference, or `installApplications()` would +reinstall the reverted-away version over the restored directory on the next cold start and silently undo +the rollback. A config-write failure compensates by swapping the directory back. The previous copy and +its manifest are per-node (each node retains its own outgoing tree during its own activate), so a +replicated revert has a local rollback source on every node. + +**One contract on `.deploy-aside`.** The directory has a single protocol, shared with the in-place +extraction transaction: an `.in-progress---` directory with no matching +`.retired-<...>` marker is a ROLLBACK RECORD, and `recoverInterruptedComponentExtractions` restores the +newest such directory OVER the live component path at startup. Anything else there is residue and is +swept. A tree that is already known to be garbage when it is parked — the evicted two-deploys-ago +retained-previous — therefore carries a distinct `.discarded-` prefix, because a crash between parking +and sweeping would otherwise make startup recovery resurrect an ancient version over the current one. +The two startup passes read disjoint directories (`.deploy-aside` for interrupted in-place preparation, +`.deploy-staging`/`.deploy-activating` for an interrupted two-phase activation), and extraction recovery +runs first so the staged reconciliation decides roll-forward against a settled live directory. **Staged-build retention.** A full deploy consumes its staged build immediately (activate renames it live), so the only builds that accumulate are `activate: false` stage-and-stops that are never diff --git a/components/Application.ts b/components/Application.ts index 0e8fd5cea1..5bdee835a1 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -484,7 +484,7 @@ async function runNpmPack( } // Hidden directory under the components root holding component versions renamed aside -// during a deploy swap (see activateApplication). The leading dot keeps +// during a deploy swap (see activateStagedApplication). The leading dot keeps // loadComponentDirectories from loading its contents as components. export const ASIDE_STAGING_DIR = '.deploy-aside'; const IN_PROGRESS_ASIDE_PREFIX = '.in-progress-'; @@ -503,8 +503,8 @@ const MAX_INSTALL_COMMANDS = 2; // Hidden directory under the components root where the INCOMING version of a component is // fully built (extracted + `npm install`) before it goes live — the counterpart to // ASIDE_STAGING_DIR, which holds the OUTGOING version. Two-phase deploy stages here first -// (stage_component), then activateApplication renames the staged copy into the live -// component path in one atomic step (activate_component). +// (the stage phase), then activateStagedApplication renames the staged copy into the live +// component path in one atomic step (the activate phase). // // It lives UNDER the components root on purpose, even though the bytes are "temporary": // - Same filesystem as the live path, so the go-live rename() is atomic. An os.tmpdir() @@ -956,7 +956,7 @@ function canonicalizeJSON(value: any): any { * Writes the application into `application.buildDirPath`, overwriting any existing directory there. * By default that is the live component directory (`application.dirPath`); during a two-phase deploy * `stageApplication` points it at the hidden staging directory instead, so the live path is never - * touched until `activateApplication` swaps the staged copy into place. + * touched until `activateStagedApplication` swaps the staged copy into place. * * This method may be called from any Harper thread. Same-component calls are serialized across * threads by the preparation lock below. @@ -1828,8 +1828,8 @@ interface ApplicationOptions { // constructor into the npm and git halves, which are injected by entirely different mechanisms. credentials?: ResolvedCredential[]; // Stable identifier for the hidden staging directory this deploy builds into, so a two-phase - // deploy's `activate_component` step can reconstruct the same staging path a prior - // `stage_component` built (both derive it from the deployment id). Defaults to a random UUID for + // deploy's activate phase can reconstruct the same staging path the prior stage phase + // built (both derive it from the deployment id). Defaults to a random UUID for // callers that stage and activate against one in-memory Application instance. stagingId?: string; } @@ -1856,7 +1856,7 @@ export class Application { // Stable id for this deploy's staging directory (see ApplicationOptions.stagingId). stagingId: string; // When set, extract/install build here instead of the live `dirPath`. stageApplication() points - // it at `stagingDirPath`; activateApplication() clears it after swapping the staged copy live. + // it at `stagingDirPath`; activateStagedApplication() clears it after swapping the staged copy live. #buildDirPath?: string; #npmrcTempDir?: string; #gitCredentialSession?: GitCredentialSession; @@ -1910,7 +1910,7 @@ export class Application { // Hidden, per-deploy staging directory the incoming version is built into before it goes live: // `/.deploy-staging//`. Deterministic from (stagingId, component - // name) so `activate_component` can find what `stage_component` built. Sits under the components + // name) so the activate phase can find what the stage phase built. Sits under the components // root (dirname(dirPath)) so the go-live rename() into `dirPath` stays on one filesystem and is // therefore atomic. Two properties fall out of putting the deployment id ABOVE the component name: // - the leaf directory's basename IS the component name, so the pre-go-live validation load @@ -1924,7 +1924,7 @@ export class Application { } // The retained-previous copy this component would revert to (`.deploy-previous/`). See - // DEPLOY_PREVIOUS_DIR / activateApplication / revertApplication. + // DEPLOY_PREVIOUS_DIR / activateStagedApplication / revertApplication. get previousDirPath(): string { return previousDirPathFor(this.dirPath); } @@ -1934,7 +1934,7 @@ export class Application { this.#buildDirPath = this.stagingDirPath; } - // Restore the live component directory as the build target. Called by activateApplication() + // Restore the live component directory as the build target. Called by activateStagedApplication() // once the staged copy has been swapped into place. useLiveBuildDir(): void { this.#buildDirPath = undefined; diff --git a/components/operations.js b/components/operations.js index c7bf67ca98..7b5513a0a7 100644 --- a/components/operations.js +++ b/components/operations.js @@ -429,7 +429,7 @@ async function packageComponent(req) { * node first, verifies it landed everywhere, and only then swaps it live cluster-wide — so a node * that can't fetch the package or fails `npm install` fails the deploy while the live component is * still untouched on every node, and the go-live window shrinks to a fast atomic directory swap. - * See stageApplication/activateApplication in components/Application.ts. + * See stageApplication/activateStagedApplication in components/Application.ts. * * The request/response contract is unchanged: same inputs (`package`/payload, `restart`, * `install_*`, `credentials`, `ignore_replication_errors`, `deployment_timeout`, …), same @@ -1345,7 +1345,8 @@ async function restartRevertedComponent(req, emit) { } // ———————————————————————————————————————————————————————————————————————————— -// Shared deploy-family helpers (used by deploy_component, stage_component, activate_component). +// Shared deploy-family helpers (used by deploy_component, its component_deploy_phase fan-out, and +// revert_component). // ———————————————————————————————————————————————————————————————————————————— // Reject deploying over a protected core component name unless force is set. Lazy-loads diff --git a/components/operationsValidation.js b/components/operationsValidation.js index 406936d1f0..6c84c1f78a 100644 --- a/components/operationsValidation.js +++ b/components/operationsValidation.js @@ -443,7 +443,7 @@ const GIT_CREDENTIAL_ENTRY = Joi.object({ .unknown(false); // The kind-heterogeneous deploy credentials array, shared by deploy_component and its two-phase -// sub-operations (stage_component / activate_component) so the three stay in lockstep. An entry's +// peer fan-out (component_deploy_phase) so both stay in lockstep. An entry's // kind is implied by its identifying key (`registry` → npm registry auth, otherwise git host auth), // dispatched here so a malformed entry reports what is actually wrong with it rather than a generic // "no alternative matched". From 1497fed88a9f0695892f21dfa01112f094911cbd Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Wed, 12 Aug 2026 12:24:17 -0400 Subject: [PATCH 46/94] test(deploy): cover revert_component at the operation level MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The primitives had direct coverage, but the public operation — a super_user-gated operation reachable over the operations API — did not, which is the bar the earlier review on harper#1849 set for the stage/activate handlers. Four cases through `operations.revertComponent`: - a cluster revert to a named previous deployment: the live tree is the reverted-to version, the audit row lands `rolled_back` with `rollback_of` naming the deployment taken out of service, and the peer fan-out carries the TARGET id (which is what makes the fan-out idempotent per node rather than a broadcast toggle); - a retried identical request is a no-op that leaves the reverted-to version live, the retry hazard @kriszyp raised; - a revert with no target is rejected, and one naming a version that is neither live nor retained is rejected without changing anything; - reverting away from a package deploy removes the stale `package:` reference from root config, so a cold restart's installApplications() cannot reinstall the reverted-away version over the restored directory. 293 deploy/component/CLI/server unit tests pass. Co-Authored-By: Claude Opus 5 --- .../components/deployPhaseOperations.test.js | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index 80bba03bc8..36877354d4 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -113,6 +113,7 @@ describe('deploy_component two-phase orchestration', function () { await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR), { recursive: true, force: true }); await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR), { recursive: true, force: true }); await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-aside'), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-previous'), { recursive: true, force: true }); }); function name() { @@ -308,6 +309,105 @@ describe('deploy_component two-phase orchestration', function () { assert.equal(restartNeeded(), true, 'dependencies with no lockfile make the install opaque'); }); + // ———————————————————————————————————————————————————————————————————————————— + // revert_component (harper#1849 review, @kriszyp) + // ———————————————————————————————————————————————————————————————————————————— + + it('reverts the cluster to a named previous deployment and fans the target out to peers', async () => { + const project = name(); + const first = await operations.deployComponent({ project, payload: await makePayload('rev-v1', '1.0.0') }); + const second = await operations.deployComponent({ project, payload: await makePayload('rev-v2', '2.0.0') }); + const fanout = []; + server.replication.replicateOperation = async (op) => { + fanout.push(op); + return { replicated: [] }; + }; + + const result = await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); + + assert.equal(result.reverted, true); + assert.equal(result.to_deployment_id, first.deployment_id); + assert.equal(result.from_deployment_id, second.deployment_id); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /rev-v1/); + assert.equal(rows.get(result.deployment_id).status, 'rolled_back'); + assert.equal( + rows.get(result.deployment_id).rollback_of, + second.deployment_id, + 'the audit row records which deployment the rollback took out of service' + ); + assert.equal(fanout.length, 1, 'peers get the revert'); + assert.equal(fanout[0].operation, 'revert_component'); + assert.equal( + fanout[0].to_deployment_id, + first.deployment_id, + 'peers are told WHICH version to end on, so the fan-out is idempotent per node' + ); + }); + + it('is a no-op when the requested deployment is already live, so a retry is safe', async () => { + const project = name(); + const first = await operations.deployComponent({ project, payload: await makePayload('retry-v1', '1.0.0') }); + await operations.deployComponent({ project, payload: await makePayload('retry-v2', '2.0.0') }); + await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); + + // The caller lost the first response and retried the identical request. + const retry = await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); + + assert.equal(retry.reverted, false); + assert.match(retry.message, /already running/); + assert.match( + await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), + /retry-v1/, + 'a retried revert must not toggle the rejected version back in' + ); + }); + + it('rejects a revert with no target, and one whose target is not retained', async () => { + const project = name(); + await operations.deployComponent({ project, payload: await makePayload('target-v1', '1.0.0') }); + await operations.deployComponent({ project, payload: await makePayload('target-v2', '2.0.0') }); + + await assert.rejects( + () => operations.revertComponent({ project }), + /to_deployment_id/, + 'the target is mandatory — that is what makes a retry safe' + ); + await assert.rejects( + () => operations.revertComponent({ project, to_deployment_id: '00000000-0000-4000-8000-000000000000' }), + /neither the live version/, + 'only the immediately-previous version is retained' + ); + assert.match( + await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), + /target-v2/, + 'a refused revert changes nothing' + ); + }); + + it('takes the package reference out of root config when reverting away from a package deploy', async () => { + // Without this, installApplications() would reinstall the reverted-away package over the restored + // directory on the next cold start and silently undo the rollback. + const project = name(); + const packaged = await operations.deployComponent({ + project, + payload: await makePayload('cfg-packaged', '1.0.0'), + }); + // Stamp root config as a package deploy would have, then activate a payload version over it. + const { addConfig } = require('#src/config/configUtils'); + await addConfig(project, { package: 'some-pkg@1.0.0' }); + const plain = await operations.deployComponent({ project, payload: await makePayload('cfg-plain', '2.0.0') }); + assert.ok(plain.deployment_id); + + await operations.revertComponent({ project, to_deployment_id: packaged.deployment_id }); + + const entry = readConfigFile()?.[project]; + assert.equal( + entry?.package, + undefined, + 'the reverted-to version had no package reference, so the stale one must be gone' + ); + }); + it('reclaims an oversized payload only after a full two-phase activation succeeds', async () => { const project = name(); const priorMaxSize = environment.get(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE); From 4d1f7393aa65462dd8316741a0d1bd77e757f82a Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Wed, 12 Aug 2026 12:47:29 -0400 Subject: [PATCH 47/94] test(deploy): assert the restart gate on the swap primitive, not through the op MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI (Node 22/26 `test:unit:main`) failed one of the restart-gate tests I added in 07fc92107 with `No deployment found with id '…'` from `claimStagedDeployment`, though it passes in isolation and with its whole file on both Node 22 and 24. The row is created and checkpointed `staged` before activation, so the only way that lookup misses is if `databases.system[hdb_deployment]` is no longer the mock the suite installed in `beforeEach` — `claimStagedDeployment` re-resolves the table from that global at activate time. This file's own `before` hook already documents that hazard ("the first component operation completes lazy server initialization, which replaces databases.system"), so the seam is fragile in full-suite ordering and my tests were the ones that happened to trip it. Rather than paper over it, the four gate tests move to `deployStaging.test.js` and assert `application.packageMetadataChanged` / `isNewComponent` on `activateStagedApplication` directly — which is where the comparison is actually computed, and which has no deployment-table seam to depend on. Added a fifth case for a first-ever deploy. The pre-existing `restartNeeded()` assertions in the operation-level suite are untouched, so the wiring from the flag into `markRestartRequiredForDeploy` is still covered there. Verified these are genuine regression tests, not passing bystanders: with the metadata comparison in `activateStagedApplication` disabled, 3 of the 5 fail; with it restored, all 31 in the file pass. Green on Node 22 and 24. I have NOT root-caused what replaces `databases.system` mid-suite. That fragility is pre-existing, affects every test in `deployPhaseOperations.test.js` that reaches activation, and is worth a separate look. Co-Authored-By: Claude Opus 5 --- .../components/deployPhaseOperations.test.js | 67 ----------------- unitTests/components/deployStaging.test.js | 75 +++++++++++++++++++ 2 files changed, 75 insertions(+), 67 deletions(-) diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index 36877354d4..f1f3f395e8 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -242,73 +242,6 @@ describe('deploy_component two-phase orchestration', function () { assert.equal(restartNeeded(), true, 'a new component activated without restart requires one'); }); - // ———————————————————————————————————————————————————————————————————————————— - // Restart gate across the swap (harper#674 / harper#1849 @heskew finding 2) - // ———————————————————————————————————————————————————————————————————————————— - - it('redeploying an unchanged component with a comparable install does NOT require a restart', async () => { - // The harper#1806 guarantee: a redeploy of an already-loaded component stays quiet, because that - // component's own file watcher (Scope/EntryHandler) requests a restart if the changed files - // actually need one. It holds only when the install is COMPARABLE — no bundled node_modules, so - // the staged tree's metadata can be checked against the outgoing one. - const project = name(); - await operations.deployComponent({ project, payload: await makePayload('gate-same', '1.0.0', false) }); - resetRestartNeeded(); // clear the flag the first (new-component) deploy legitimately set - - await operations.deployComponent({ project, payload: await makePayload('gate-same', '1.0.0', false) }); - - assert.equal(restartNeeded(), false, 'identical package metadata across the swap must stay quiet'); - }); - - it('requires a restart when a redeploy bundles node_modules, whose install cannot be compared', async () => { - // A payload that ships its own node_modules skips the install entirely, so nothing about the - // resulting tree can be compared against a fresh install — the runtime is opaque and has to be - // assumed changed. Same conclusion the one-shot path reaches for the same payload shape. - const project = name(); - await operations.deployComponent({ project, payload: await makePayload('gate-opaque-bundle', '1.0.0') }); - resetRestartNeeded(); - - await operations.deployComponent({ project, payload: await makePayload('gate-opaque-bundle', '1.0.0') }); - - assert.equal(restartNeeded(), true, 'a bundled node_modules redeploy is opaque, so it requires a restart'); - }); - - it('requires a restart when a redeploy changes package metadata the watchers cannot see', async () => { - // Installed package metadata is deliberately outside most plugin watch globs, so nothing else - // notices a dependency or module-entry change — but it invalidates already-loaded code. The - // one-shot path compares it across its in-place install; two-phase has to compare the outgoing - // live tree against the staged one at swap time, which is what this pins. - const project = name(); - await operations.deployComponent({ project, payload: await makePayload('gate-change', '1.0.0') }); - resetRestartNeeded(); - - await operations.deployComponent({ project, payload: await makePayload('gate-change', '2.0.0') }); - - assert.equal(restartNeeded(), true, 'a changed package.json version must still force a restart'); - }); - - it('requires a restart when a redeploy ships no lockfile for its dependencies', async () => { - // An install whose result can't be reproduced or compared (installable dependencies, no lockfile) - // is treated as opaque, and an opaque install always requires a restart. - const project = name(); - await operations.deployComponent({ project, payload: await makePayload('gate-opaque', '1.0.0') }); - resetRestartNeeded(); - - const source = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-op-opaque-')); - await fs.writeFile( - path.join(source, 'package.json'), - JSON.stringify({ name: 'phase-op', version: '1.0.0', dependencies: { 'some-dep': '1.0.0' } }) - ); - await fs.writeFile(path.join(source, 'index.js'), "module.exports = 'gate-opaque';\n"); - await fs.mkdir(path.join(source, 'node_modules'), { recursive: true }); - const payload = await packDirectory(source); - await fs.rm(source, { recursive: true, force: true }); - - await operations.deployComponent({ project, payload }); - - assert.equal(restartNeeded(), true, 'dependencies with no lockfile make the install opaque'); - }); - // ———————————————————————————————————————————————————————————————————————————— // revert_component (harper#1849 review, @kriszyp) // ———————————————————————————————————————————————————————————————————————————— diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index a7fa641fb6..4c73f51bd3 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -604,4 +604,79 @@ describe('two-phase component directory transaction', function () { assert.match(await readMarker(application.dirPath), /over-dead-link/); await cleanup(name); }); + // ———————————————————————————————————————————————————————————————————————————— + // Restart gate: package metadata compared across the swap + // (harper#674 / harper#1849 @heskew finding 2) + // ———————————————————————————————————————————————————————————————————————————— + // + // main's in-place prepareApplication compares installed package metadata before extraction against + // after install; the two-phase path never touches the live directory until the swap, so the + // equivalent comparison is outgoing-live vs staged, taken inside activateStagedApplication. These + // assert the flag it sets, which operations.js feeds into markRestartRequiredForDeploy. + + async function activateFrom(name, marker, version, { withNodeModules = true, dependencies } = {}) { + const source = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-gate-src-')); + const manifest = { name: 'stage-fixture', version }; + if (dependencies) manifest.dependencies = dependencies; + await fs.writeFile(path.join(source, 'package.json'), JSON.stringify(manifest)); + await fs.writeFile(path.join(source, 'index.js'), `module.exports = ${JSON.stringify(marker)};\n`); + if (withNodeModules) await fs.mkdir(path.join(source, 'node_modules'), { recursive: true }); + const payload = await packDirectory(source); + await fs.rm(source, { recursive: true, force: true }); + + const application = new Application({ name, payload }); + const deploymentId = randomUUID(); + await stageApplication(application, deploymentId); + await activateStagedApplication(application, deploymentId, { activationSpec: { package: null } }); + return application; + } + + it('does not flag a metadata change when a redeploy is identical and comparable', async () => { + // The harper#1806 guarantee: a redeploy of an already-loaded component stays quiet, because that + // component's own file watcher requests a restart if the changed files actually need one. It holds + // only for a COMPARABLE install — no bundled node_modules to make the result opaque. + const name = fixtureName(); + await activateFrom(name, 'quiet', '1.0.0', { withNodeModules: false }); + const second = await activateFrom(name, 'quiet', '1.0.0', { withNodeModules: false }); + + assert.equal(second.isNewComponent, false, 'the second activation is a redeploy'); + assert.equal(second.packageMetadataChanged, false, 'identical metadata across the swap must stay quiet'); + await cleanup(name); + }); + + it('flags a metadata change the watchers cannot see', async () => { + const name = fixtureName(); + await activateFrom(name, 'changed', '1.0.0', { withNodeModules: false }); + const second = await activateFrom(name, 'changed', '2.0.0', { withNodeModules: false }); + + assert.equal(second.packageMetadataChanged, true, 'a changed package.json version invalidates loaded code'); + await cleanup(name); + }); + + it('treats a bundled node_modules redeploy as opaque, since its install cannot be compared', async () => { + const name = fixtureName(); + await activateFrom(name, 'opaque', '1.0.0'); + const second = await activateFrom(name, 'opaque', '1.0.0'); + + assert.equal(second.packageMetadataChanged, true, 'a skipped install leaves nothing to compare'); + await cleanup(name); + }); + + it('treats installable dependencies with no lockfile as opaque', async () => { + const name = fixtureName(); + await activateFrom(name, 'nolock', '1.0.0'); + const second = await activateFrom(name, 'nolock', '1.0.0', { dependencies: { 'some-dep': '1.0.0' } }); + + assert.equal(second.packageMetadataChanged, true, 'dependencies with no lockfile are not reproducible'); + await cleanup(name); + }); + + it('does not flag a metadata change on a first-ever deploy', async () => { + const name = fixtureName(); + const first = await activateFrom(name, 'brand-new', '1.0.0', { withNodeModules: false }); + + assert.equal(first.isNewComponent, true); + assert.equal(first.packageMetadataChanged, false, 'nothing to compare against; isNewComponent carries it'); + await cleanup(name); + }); }); From 9430ad035f7e1b3e4c1835e5b1bdca0c7b7cb001 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Wed, 12 Aug 2026 12:51:27 -0400 Subject: [PATCH 48/94] test(cli): cover the harper revert verb and its required target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `revert` verb had no CLI coverage — the same gap an earlier review on harper#1849 flagged for `harper activate`. Two cases: the verb maps to revert_component and carries its `_cliVerb` marker (which is why `revert` lives in OP_VERB_PROPS rather than OP_ALIASES — buildRequest checks the alias table first, so an alias entry would set the operation and never attach the marker, silently disabling the guard), and a revert with no `to_deployment_id` is rejected before anything is sent. Co-Authored-By: Claude Opus 5 --- unitTests/bin/cliOperations.test.js | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/unitTests/bin/cliOperations.test.js b/unitTests/bin/cliOperations.test.js index 31125fb040..6ce3044394 100644 --- a/unitTests/bin/cliOperations.test.js +++ b/unitTests/bin/cliOperations.test.js @@ -1546,4 +1546,23 @@ describe('deploy CLI verbs (stage / activate fold into deploy_component)', () => assert.strictEqual(verbRequirementError({ operation: 'deploy_component' }), null); assert.strictEqual(verbRequirementError({ operation: 'deploy_component', activate: false }), null); }); + + it('`revert` maps to revert_component and carries the verb marker', () => { + // The marker has to survive buildRequest for the guard below to fire at all. `revert` deliberately + // lives in OP_VERB_PROPS rather than OP_ALIASES: buildRequest checks the alias table FIRST, so an + // alias entry would set the operation and never attach `_cliVerb`. + process.argv = ['node', 'harper', 'revert', 'project=my_app', 'to_deployment_id=abc-123']; + const req = buildRequest(); + assert.strictEqual(req.operation, 'revert_component'); + assert.strictEqual(req.to_deployment_id, 'abc-123'); + assert.strictEqual(verbRequirementError(req), null); + }); + + it('`revert` WITHOUT a to_deployment_id is rejected before anything is sent', () => { + // A revert with no target would be a blind toggle, which is unsafe to retry. + process.argv = ['node', 'harper', 'revert', 'project=my_app']; + const req = buildRequest(); + assert.strictEqual(req.operation, 'revert_component'); + assert.match(verbRequirementError(req), /to_deployment_id/); + }); }); From 9eb3a41b8de36e8cd71438e07cd4715d3b5a0d18 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Wed, 12 Aug 2026 12:58:47 -0400 Subject: [PATCH 49/94] fix(deploy): compensate and recover an interrupted revert swap MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the review finding on components/Application.ts:827: `revertApplication`'s three-way swap was the only swap primitive in the file with no failure compensation and no startup recovery. After `rename(live, holding)` the component has NO live directory, so an I/O error on the next rename left it unservable with its bytes stranded under `.reverting-*` and the manifest describing directories that no longer matched reality. Both halves are now covered, because they cover different failures: - **In process**: each rename is compensated. A failure of `rename(previous, live)` puts the outgoing tree straight back, so the component keeps serving what it was serving. A failure of the final retain step undoes the whole swap rather than leave the manifest claiming a retained previous that isn't there. A compensation that itself fails throws an AggregateError naming the holding path, so the operator knows exactly which directory holds the live tree. - **Across a crash**: compensation inherently cannot run if the process dies between renames, which is the more dangerous case. The holding directory is now named `.reverting--` and `recoverInterruptedReverts` sweeps `.deploy-previous` at startup, restoring the tree into whichever slot is empty — the live path when the swap had not yet placed the reverted-to version (the revert is undone and can be retried), or the retained-previous path when the swap completed and only the retain step was lost. Both slots occupied means the swap finished, so the holding tree is residue and is discarded. Wired into componentLoader alongside the extraction and staged-artifact passes, before anything loads; the three passes read disjoint directories. Writing the tests caught a real bug in the recovery I had just added: parsing the component name by cutting at the last dash left most of the trailing UUID glued to the name, so recovery would have restored into a nonexistent directory. It now matches the UUID explicitly, which also keeps component names containing dashes intact. Tests: three startup-recovery cases (died before the swap, died before the retain, residue from a completed swap). The in-process compensation is deliberately NOT tested and says so in place: once the live path is gone, `rename(previous, live)` has no existing target to conflict with and succeeds for every filesystem state a test can set up, so inducing it would need a fault-injection seam in production code. 317 deploy/component/CLI/server unit tests pass. Co-Authored-By: Claude Opus 5 --- DESIGN.md | 16 ++- components/Application.ts | 133 ++++++++++++++++++++- components/componentLoader.ts | 31 ++++- unitTests/components/deployStaging.test.js | 74 ++++++++++++ 4 files changed, 240 insertions(+), 14 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 28d70e622d..8a29f24a7b 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -519,9 +519,19 @@ newest such directory OVER the live component path at startup. Anything else the swept. A tree that is already known to be garbage when it is parked — the evicted two-deploys-ago retained-previous — therefore carries a distinct `.discarded-` prefix, because a crash between parking and sweeping would otherwise make startup recovery resurrect an ancient version over the current one. -The two startup passes read disjoint directories (`.deploy-aside` for interrupted in-place preparation, -`.deploy-staging`/`.deploy-activating` for an interrupted two-phase activation), and extraction recovery -runs first so the staged reconciliation decides roll-forward against a settled live directory. +The three startup passes read disjoint directories — `.deploy-aside` for an interrupted in-place +preparation, `.deploy-previous` for an interrupted revert, and `.deploy-staging`/`.deploy-activating` +for an interrupted two-phase activation — and the two directory repairs run before the staged +reconciliation so it decides roll-forward against a settled live directory. + +**An interrupted revert self-heals.** The three-way swap compensates in process (each rename is undone +if a later one fails, so an exceptional I/O error leaves the component serving what it was serving), +but compensation cannot cover the process dying between renames — and after the first rename the +component has NO live directory. So the holding path is named `.reverting--`, and +`recoverInterruptedReverts` restores it at startup into whichever slot is empty: the live path when the +swap had not yet placed the reverted-to version (the revert is undone and can be retried), or the +retained-previous path when the swap completed and only the retain step was lost. With both slots +occupied the swap finished and the holding tree is residue, so it is discarded. **Staged-build retention.** A full deploy consumes its staged build immediately (activate renames it live), so the only builds that accumulate are `activate: false` stage-and-stops that are never diff --git a/components/Application.ts b/components/Application.ts index 5bdee835a1..a3a8d4cc7e 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -491,6 +491,12 @@ const IN_PROGRESS_ASIDE_PREFIX = '.in-progress-'; // Parked and known-disposable at park time (an evicted two-deploys-ago retained-previous). NEVER a // recovery candidate — see discardDirAside for why the distinction has to be in the name. const DISCARDED_ASIDE_PREFIX = '.discarded-'; +// The holding path a revert's three-way swap parks the outgoing live tree in. Recoverable: if the +// process dies mid-swap, recoverInterruptedReverts puts it back. See revertApplication. +const REVERTING_PREFIX = '.reverting-'; +// Splits `-` off a revert holding directory name, anchored on the UUID so a +// component name containing dashes is preserved intact. +const REVERTING_NAME_PATTERN = /^(.+)-[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; const RETIRED_ASIDE_PREFIX = '.retired-'; const DEFAULT_COMMAND_TIMEOUT_MS = 60 * 60 * 1000; const COMPONENT_PREPARATION_WAIT_MARGIN_MS = 30000; @@ -746,6 +752,87 @@ async function retainActivatedPrevious( } } +/** + * Repair a revert that died between renames. The holding directory (`.reverting--`, a + * sibling of the retained previous) only exists while a swap is in flight, so finding one at startup + * means the process was killed mid-swap and the component may have no live directory at all. + * + * Recovery restores the holding tree to whichever slot is empty: the live path when the swap had not + * yet placed the reverted-to version (so the component comes back on the version it was serving), or + * the retained-previous path when the swap DID complete and only the retain step was lost. If both + * slots are occupied the swap finished and the holding tree is residue, so it is discarded. + * + * Best-effort per component: one component's unrecoverable state must not stop the rest from loading. + */ +export async function recoverInterruptedReverts(componentsRootDirPath: string): Promise> { + const previousRoot = join(componentsRootDirPath, DEPLOY_PREVIOUS_DIR); + const failures = new Map(); + let entries; + try { + entries = await readdir(previousRoot, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return failures; + throw error; + } + for (const entry of entries) { + if (!entry.isDirectory() || !entry.name.startsWith(REVERTING_PREFIX)) continue; + const holding = join(previousRoot, entry.name); + // `.reverting--`. The component name may itself contain dashes, so match the + // trailing UUID explicitly rather than cutting at the last dash — which would leave most of the + // UUID glued to the name and recover into the wrong (or a nonexistent) component directory. + const componentName = REVERTING_NAME_PATTERN.exec(entry.name.slice(REVERTING_PREFIX.length))?.[1]; + if (!componentName || !safeComponentName(componentName)) { + await rm(holding, { recursive: true, force: true }).catch(() => {}); + continue; + } + const liveDirPath = join(componentsRootDirPath, componentName); + try { + await withComponentPreparationLock(liveDirPath, async () => { + const liveExists = await lstat(liveDirPath).then( + () => true, + (err) => { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw err; + } + ); + if (!liveExists) { + await mkdir(dirname(liveDirPath), { recursive: true }); + await rename(holding, liveDirPath); + logger.warn( + `Restored the live ${componentName} component directory after an interrupted revert; ` + + `the revert did not take effect and can be retried` + ); + return; + } + const previousPath = previousDirPathFor(liveDirPath); + const previousExists = await lstat(previousPath).then( + () => true, + (err) => { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw err; + } + ); + if (!previousExists) { + await mkdir(dirname(previousPath), { recursive: true }); + await rename(holding, previousPath); + logger.warn( + `Completed an interrupted ${componentName} revert: the reverted-to version is live and the ` + + `version it displaced is retained again` + ); + return; + } + // Both slots occupied: the swap completed, so this is residue. + await rm(holding, { recursive: true, force: true }); + }); + } catch (error) { + const recoveryError = error instanceof Error ? error : new Error(String(error)); + failures.set(componentName, recoveryError); + logger.error(`Could not recover the interrupted ${componentName} revert:`, errorForLog(recoveryError)); + } + } + return failures; +} + /** What a component can currently be reverted to, for reporting and for revert targeting. */ export async function getRevertTarget( componentDirPath: string @@ -819,12 +906,48 @@ export async function revertApplication( await mkdir(dirname(previousPath), { recursive: true }); if (liveExists) { // Three-way atomic swap through a hidden holding path: live → holding, previous → live, - // holding(old live) → previous. The only window where `dirPath` is absent is between two - // atomic renames, and deploy:start suppresses watchers across it (same as activation). - const holding = join(dirname(previousPath), `.reverting-${basename(liveDirPath)}-${randomUUID()}`); + // holding(old live) → previous. The only window where `dirPath` is absent is between the + // first two renames, and deploy:start suppresses watchers across it (same as activation). + // + // Each step is compensated, because a failure here is the one that hurts most: after the + // first rename the component has NO live directory, so an uncompensated I/O error (disk + // full, a permission change, an unexpected fault on previousPath) would leave the component + // unservable with its bytes stranded under the holding path. The holding name is also + // recoverable by name at startup (recoverInterruptedReverts), which covers the case + // compensation cannot: the process dying between renames. + const holding = join(dirname(previousPath), `${REVERTING_PREFIX}${basename(liveDirPath)}-${randomUUID()}`); await rename(liveDirPath, holding); - await rename(previousPath, liveDirPath); - await rename(holding, previousPath); + try { + await rename(previousPath, liveDirPath); + } catch (swapError) { + // Nothing is live: put the outgoing tree straight back and fail with the component intact. + await rename(holding, liveDirPath).catch((restoreError) => { + throw new AggregateError( + [swapError, restoreError], + `Failed to revert ${application.name} and could not restore its live directory from ` + + `${holding}; the component has no live version until that directory is restored` + ); + }); + throw swapError; + } + try { + await rename(holding, previousPath); + } catch (retainError) { + // The revert itself succeeded (the requested version IS live); only retaining the tree it + // displaced failed. Undo the whole swap rather than leave the manifest describing a + // retained previous that is not where it says it is. + try { + await rename(liveDirPath, previousPath); + await rename(holding, liveDirPath); + } catch (restoreError) { + throw new AggregateError( + [retainError, restoreError], + `Reverted ${application.name} but could not retain the displaced version, and could not ` + + `undo the swap; ${holding} still holds the previously-live tree` + ); + } + throw retainError; + } } else { // No live version to preserve; restore the previous into place. Nothing becomes the new // retained previous, so the component can't be reverted again until its next deploy. diff --git a/components/componentLoader.ts b/components/componentLoader.ts index bc012764f5..44ffcb275a 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -57,6 +57,7 @@ import { reconcileStagedApplicationArtifacts, recoverInterruptedComponentExtraction, recoverInterruptedComponentExtractions, + recoverInterruptedReverts, } from './Application.ts'; import { getDeploymentRow } from './deploymentRecorder.ts'; import { ComponentPreparationLockTimeoutError } from './componentPreparationLock.ts'; @@ -155,18 +156,20 @@ export async function loadComponentDirectories( if (loadedPluginModules) loadedComponents = loadedPluginModules; const cycleResources = resources; const cycleLoadedComponents = loadedComponents; - // Two independent crash-recovery passes, in this order: + // Three independent crash-recovery passes, in this order: // // 1. recoverInterruptedComponentExtractions — an IN-PLACE preparation (the one-shot deploy path) // that died mid-swap left the prior tree parked in `.deploy-aside` with no retired marker; // restore it so the live directory is a known-good tree again. - // 2. reconcileStagedApplicationArtifacts — a TWO-PHASE activation that died in its swap window + // 2. recoverInterruptedReverts — a revert that died between renames left the outgoing tree parked + // under `.deploy-previous/.reverting-*`, possibly with no live directory at all. + // 3. reconcileStagedApplicationArtifacts — a TWO-PHASE activation that died in its swap window // is rolled forward from its durable deployment row + activation marker. // - // Extraction recovery runs first so the staged reconciliation makes its roll-forward decision - // against a settled live directory rather than a half-swapped one. The two passes read disjoint - // directories (`.deploy-aside` vs `.deploy-staging`/`.deploy-activating`), so neither can claim the - // other's artifacts — see DESIGN.md, "One contract on .deploy-aside". + // The two directory repairs run before the staged reconciliation so its roll-forward decision sees a + // settled live directory rather than a half-swapped one. The passes read disjoint directories + // (`.deploy-aside`, `.deploy-previous`, `.deploy-staging`/`.deploy-activating`), so none can claim + // another's artifacts — see DESIGN.md, "One contract on .deploy-aside". let failedRecoveries = new Map(); try { failedRecoveries = await recoverInterruptedComponentExtractions(CF_ROUTES_DIR); @@ -177,6 +180,22 @@ export async function loadComponentDirectories( errorForLog(recoveryError) ); } + if (isMainThread) { + // A revert that died between its renames can leave a component with no live directory at all, with + // the bytes parked under `.deploy-previous/.reverting-*`. Repaired before anything loads, and + // before the staged reconciliation below, for the same reason extraction recovery runs first: the + // roll-forward decision should see a settled live directory. + try { + for (const [component, error] of await recoverInterruptedReverts(CF_ROUTES_DIR)) { + if (!failedRecoveries.has(component)) failedRecoveries.set(component, error); + } + } catch (error) { + harperLogger.error( + 'Could not inspect retained component versions for interrupted reverts:', + errorForLog(error as Error) + ); + } + } if (isMainThread && !stagedArtifactsReconciled) { stagedArtifactsReconciled = true; try { diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 4c73f51bd3..1737035b78 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -22,6 +22,7 @@ const { createApplicationActivationTransaction, revertApplication, getRevertTarget, + recoverInterruptedReverts, extractApplication, stagedApplicationPath, DEPLOY_STAGING_DIR, @@ -679,4 +680,77 @@ describe('two-phase component directory transaction', function () { assert.equal(first.packageMetadataChanged, false, 'nothing to compare against; isNewComponent carries it'); await cleanup(name); }); + // ———————————————————————————————————————————————————————————————————————————— + // Interrupted revert: compensation and startup recovery + // ———————————————————————————————————————————————————————————————————————————— + + async function twoActivations(name) { + const first = randomUUID(); + const second = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('v1') }); + await stageApplication(application, first); + await activateStagedApplication(application, first, { activationSpec: { package: null } }); + application.payload = await makeComponentPayload('v2'); + await stageApplication(application, second); + await activateStagedApplication(application, second, { activationSpec: { package: null } }); + return { application, first, second }; + } + + // NOT covered here: an in-process failure of the middle rename. Once `rename(live, holding)` has run, + // the live path is gone, so `rename(previous, live)` has no existing target to conflict with and + // succeeds for every filesystem state reachable from a test (a file, a dangling symlink, a directory + // all rename cleanly onto a free path). Inducing it would need a fault-injection seam in production + // code. The compensation is still there — it is what turns an exceptional I/O error into "the + // component keeps serving what it was serving" — but the durable half below is what actually covers + // the reviewer's scenario: a process that dies mid-swap, which compensation inherently cannot handle. + + it('startup recovery restores the live tree from a revert that died before the swap', async () => { + const name = fixtureName(); + const { application } = await twoActivations(name); + // Simulate a crash right after `rename(live, holding)`: live is gone, holding holds its tree. + const holding = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, `.reverting-${name}-${randomUUID()}`); + await fs.rename(application.dirPath, holding); + + const failures = await recoverInterruptedReverts(COMPONENTS_ROOT); + + assert.equal(failures.size, 0); + assert.match(await readMarker(application.dirPath), /v2/, 'the interrupted revert is undone'); + assert.equal(existsSync(holding), false); + await cleanup(name); + }); + + it('startup recovery re-retains the displaced tree when only the retain step was lost', async () => { + const name = fixtureName(); + const { application } = await twoActivations(name); + // Simulate a crash after `rename(previous, live)` but before `rename(holding, previous)`: the + // reverted-to version is live, and the displaced tree is still parked. + const previousPath = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); + const holding = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, `.reverting-${name}-${randomUUID()}`); + await fs.rm(application.dirPath, { recursive: true, force: true }); + await fs.rename(previousPath, application.dirPath); + await fs.mkdir(holding, { recursive: true }); + await fs.writeFile(path.join(holding, 'index.js'), "module.exports = 'displaced';\n"); + + const failures = await recoverInterruptedReverts(COMPONENTS_ROOT); + + assert.equal(failures.size, 0); + assert.match(await readMarker(application.dirPath), /v1/, 'the reverted-to version stays live'); + assert.match(await readMarker(previousPath), /displaced/, 'the displaced tree is retained again'); + assert.equal(existsSync(holding), false); + await cleanup(name); + }); + + it('startup recovery discards a holding tree left by a revert that had already completed', async () => { + const name = fixtureName(); + const { application } = await twoActivations(name); + const holding = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, `.reverting-${name}-${randomUUID()}`); + await fs.mkdir(holding, { recursive: true }); + await fs.writeFile(path.join(holding, 'index.js'), "module.exports = 'residue';\n"); + + await recoverInterruptedReverts(COMPONENTS_ROOT); + + assert.equal(existsSync(holding), false, 'both slots were occupied, so the holding tree is residue'); + assert.match(await readMarker(application.dirPath), /v2/, 'live is untouched'); + await cleanup(name); + }); }); From cc4e72050211b9ac6a308a4646290b50477001db Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Wed, 12 Aug 2026 13:16:40 -0400 Subject: [PATCH 50/94] fix(deploy): park a displaced directory-package tree aside instead of deleting it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A regression I introduced in the #2066 reconciliation. Retargeting `extractApplication` at `buildDirPath` meant the local-directory package branch needed the build target cleared before `symlink()` (it throws EEXIST otherwise), and I used `rm(buildDirPath, { recursive: true, force: true })`. That branch **returns early, before the extraction transaction**, so on the in-place path — a redeploy of a `file:`-directory package — this recursively deleted the LIVE component tree: no aside, no rollback record, nothing for startup recovery to restore, and it races a still-running worker writing into the directory being removed. That is precisely the ENOTEMPTY/EPERM hazard this file documents as the reason every other path renames aside instead of removing in place. Before the reconciliation the code called `symlink()` straight onto `application.dirPath`, which would fail EEXIST on redeploy — a bug, but one that failed SAFE. Now parked with `discardDirAside` (atomic rename, best-effort sweep after), marked `.discarded-` so startup recovery never restores it over the new version. Regression test asserts a directory-package redeploy links the new version and leaves no recoverable `.in-progress-` aside behind. This is NOT confirmed to be the cause of the Windows Integration Tests 1/6 failure — that suite deploys by payload and never reaches this branch, so the Windows root cause is still open. Fixed on its own merits. Co-Authored-By: Claude Opus 5 --- components/Application.ts | 14 ++++++++--- unitTests/components/deployStaging.test.js | 28 ++++++++++++++++++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index a3a8d4cc7e..9c9ddca2db 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -1128,9 +1128,17 @@ export async function extractApplication( const stats = await stat(packagePath); if (stats.isDirectory()) { - // If its a directory, symlink. A stale build target (e.g. a retried stage) would - // make symlink() throw EEXIST, so clear it first. - await rm(application.buildDirPath, { recursive: true, force: true }); + // If its a directory, symlink. Anything already at the build target has to go first, or + // symlink() throws EEXIST — which on the in-place path means a redeploy of a + // directory-package component. + // + // Parked aside with an atomic rename, NOT removed in place. This returns early, before + // the extraction transaction below, so an in-place recursive rm here would delete the + // LIVE component tree outright: no aside, no rollback record, nothing for startup + // recovery to restore, and it races a still-running worker writing into the directory it + // is deleting (the ENOTEMPTY/EPERM hazard documented on discardDirAside). The rename is + // atomic and the sweep is best-effort afterwards. + await discardDirAside(application.buildDirPath, application.name); await mkdir(dirname(application.buildDirPath), { recursive: true }); await symlink(packagePath, application.buildDirPath, 'dir'); // And return early since we're done; no extraction needed diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 1737035b78..49cd391384 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -753,4 +753,32 @@ describe('two-phase component directory transaction', function () { assert.match(await readMarker(application.dirPath), /v2/, 'live is untouched'); await cleanup(name); }); + it('redeploys a local-directory package in place without destroying the live tree', async () => { + // This path returns early, BEFORE the extraction transaction, so clearing the build target with an + // in-place recursive rm would delete the LIVE tree outright: no aside, no rollback record, nothing + // for startup recovery, and it races a worker still writing into the directory being removed. It + // must be parked with an atomic rename instead. + const name = fixtureName(); + const packageDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-dirpkg-')); + await fs.writeFile(path.join(packageDirectory, 'marker.txt'), 'v2'); + const application = new Application({ name, packageIdentifier: `file:${packageDirectory}` }); + + // A live tree is already there (the component was deployed before, by any means). + await fs.mkdir(application.dirPath, { recursive: true }); + await fs.writeFile(path.join(application.dirPath, 'marker.txt'), 'v1'); + + await extractApplication(application); + + assert.equal((await fs.lstat(application.dirPath)).isSymbolicLink(), true, 'the new version is linked'); + assert.equal(await fs.readFile(path.join(application.dirPath, 'marker.txt'), 'utf8'), 'v2'); + // Whatever was displaced must have been parked, not recursively removed in place. It is parked as + // disposable (`.discarded-`), so startup recovery will never restore it over the new version. + const asideDir = path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR, name); + const parked = existsSync(asideDir) ? await fs.readdir(asideDir) : []; + const recoverable = parked.filter((entry) => entry.startsWith('.in-progress-')); + assert.deepEqual(recoverable, [], 'the displaced tree is never left looking like a rollback record'); + + await cleanup(name); + await fs.rm(packageDirectory, { recursive: true, force: true }); + }); }); From f60feec4425f82e61e7d5037480b4e4feee738fb Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 14 Aug 2026 10:02:06 -0400 Subject: [PATCH 51/94] fix(deploy): fail closed on startup reconciliation, and finish an interrupted revert completely MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three findings from @kriszyp's Codex review on harper#1849. All three were real. **1. Startup reconciliation was fail-open (componentLoader).** A `reconcileStagedApplicationArtifacts` failure — an activation whose root-config / application-lock commit could not be completed, or inconsistent staged/live trees — was logged and then execution fell through into normal component loading. That loads a swapped-in tree whose durable configuration still describes the previous release (or loads nothing at all after an interrupted rename) while the deployment row still claims the activation is incomplete. Worse, `stagedArtifactsReconciled` was set BEFORE the attempt, so no later reload cycle ever retried. Now: the guard is only retired after a clean pass, so reconciliation is retried; and the affected component is failed closed rather than logged. Failing it closed needed the reconciler to say WHICH component failed, which a deployment id cannot — so it now also returns `failedProjects` (project → error), which the loader merges into `failedRecoveries` so that component does not load. If the scan ITSELF throws we cannot tell which components are affected, so that aborts startup instead of loading everything over possibly-unreconciled state. **2. Revert's config transaction was never rolled back (operations.js).** `commit()` makes two persistent writes; if the root-config write succeeded and the application-lock write then failed, the catch swapped the directories back but left root config naming the reverted-to release — the exact mismatch the pairing exists to prevent, and something a cold start can act on. The transaction is now held outside the try, rolled back before the directory swap-back, and compensation failures are aggregated into one error that says the node may be serving a version its persisted configuration does not name. **3. Completing an interrupted revert left the manifest reversed (Application.ts).** Consider A live / B previous. A crash after `previous → live` leaves B live, A in holding, and the pre-swap manifest still saying A live / B previous — `revertApplication` writes the manifest only after all three renames. Recovery moved A back to `previous` but left the manifest untouched, so retrying the same addressed revert to B matched B against the reversed `previous` entry and swapped the successful revert straight back out, destroying the idempotency the addressed target exists to provide. Recovery now exchanges the manifest roles exactly as `revertApplication` would, and commits the newly-live version's configuration — which was never committed either, since revertComponent commits after the swap returns. Tests: the crash-point test now asserts `getRevertTarget` roles and that a retried revert is a no-op rather than only checking directory contents (verified it fails without the manifest exchange); reconciliation reports the failing component; and the config transaction restores both writes on rollback while a never-committed transaction rolls back as a no-op. One honest note on coverage: a genuinely PARTIAL commit is not injectable from a test, because the application lock is read before the config write, so any corruption that would break the lock write breaks that read first and fails before mutating anything. The test covers the property the compensation depends on (rollback restores the exact pre-commit state of both writes) rather than faking the partial case. 302 deploy/component/CLI/server unit tests pass. Co-Authored-By: Claude Opus 5 --- components/Application.ts | 55 +++++++++-- components/componentLoader.ts | 16 +++- components/operations.js | 24 ++++- unitTests/components/deployStaging.test.js | 103 ++++++++++++++++++++- 4 files changed, 188 insertions(+), 10 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 9c9ddca2db..1f868a1237 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -813,11 +813,36 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): } ); if (!previousExists) { + // The swap's second rename landed (the reverted-to version is live) and only the retain step + // was lost, so finish it — directory, manifest and durable config together. + // + // The manifest is still the PRE-swap one: revertApplication writes it only after all three + // renames, so reaching here means it was never updated. Leaving it alone would be actively + // harmful rather than merely stale: it names the version that is now RETAINED as live and the + // version that is now LIVE as previous, so a retry of the same addressed revert would match + // its target against the reversed `previous` entry and swap the successful revert back out — + // destroying the idempotency the addressed target exists to provide. Roles are exchanged here + // exactly as revertApplication would have. + const staleManifest = await readRetainedPreviousManifest(liveDirPath); await mkdir(dirname(previousPath), { recursive: true }); await rename(holding, previousPath); + if (staleManifest) { + await writeRetainedPreviousManifest(liveDirPath, { + previous: staleManifest.live, + live: staleManifest.previous, + }); + // Config was never committed either (revertComponent commits after the swap returns), so it + // still describes the version this revert moved away from. Bring it in line with what is + // actually live, or a cold start reinstalls the reverted-away release over it. + const configTransaction = await createApplicationConfigTransaction( + componentName, + staleManifest.previous.application_config + ); + await configTransaction.commit(); + } logger.warn( - `Completed an interrupted ${componentName} revert: the reverted-to version is live and the ` + - `version it displaced is retained again` + `Completed an interrupted ${componentName} revert: the reverted-to version is live, the ` + + `version it displaced is retained again, and its configuration has been reconciled` ); return; } @@ -2603,10 +2628,20 @@ export async function reconcileStagedApplicationArtifacts( componentsRootDirPath: string, getDeployment: DeploymentLookup, persistActivation: (row: Record) => Promise -): Promise<{ recovered: string[]; removed: string[]; errors: Map }> { +): Promise<{ + recovered: string[]; + removed: string[]; + errors: Map; + failedProjects: Map; +}> { const recovered = new Set(); const removed: string[] = []; const errors = new Map(); + // Same failures as `errors`, keyed by COMPONENT rather than deployment id. An activation whose + // persistent work could not be completed leaves the live tree and its durable configuration + // disagreeing, so the caller has to be able to keep that specific component from loading — which it + // cannot do from a deployment id. + const failedProjects = new Map(); const stagingRoot = join(componentsRootDirPath, DEPLOY_STAGING_DIR); let deploymentEntries = []; try { @@ -2624,8 +2659,10 @@ export async function reconcileStagedApplicationArtifacts( removed.push(entry.name); continue; } + // Declared outside the try so the catch can attribute a reconciliation failure to its component. + let row: Record | undefined; try { - let row = await getDeployment(entry.name); + row = await getDeployment(entry.name); if (!row || !safeComponentName(row.project)) { await rm(deploymentPath, { recursive: true, force: true }); removed.push(entry.name); @@ -2668,7 +2705,9 @@ export async function reconcileStagedApplicationArtifacts( await removeActivationArtifacts(componentDirPath, entry.name); recovered.add(entry.name); } catch (error) { - errors.set(entry.name, error instanceof Error ? error : new Error(String(error))); + const reconcileError = error instanceof Error ? error : new Error(String(error)); + errors.set(entry.name, reconcileError); + if (safeComponentName(row?.project)) failedProjects.set(row.project, reconcileError); } } @@ -2696,7 +2735,9 @@ export async function reconcileStagedApplicationArtifacts( await rm(artifactPath, { recursive: true, force: true }); recovered.add(deploymentId); } catch (error) { - errors.set(deploymentId, error instanceof Error ? error : new Error(String(error))); + const reconcileError = error instanceof Error ? error : new Error(String(error)); + errors.set(deploymentId, reconcileError); + failedProjects.set(projectEntry.name, reconcileError); } } else if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && !liveStat) { await rename(artifactPath, livePath); @@ -2709,7 +2750,7 @@ export async function reconcileStagedApplicationArtifacts( await rmdir(activationRoot).catch(() => {}); } await rmdir(stagingRoot).catch(() => {}); - return { recovered: [...recovered], removed, errors }; + return { recovered: [...recovered], removed, errors, failedProjects }; } /** diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 44ffcb275a..7680ed786c 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -197,7 +197,6 @@ export async function loadComponentDirectories( } } if (isMainThread && !stagedArtifactsReconciled) { - stagedArtifactsReconciled = true; try { const reconciliation = await reconcileStagedApplicationArtifacts(CF_ROUTES_DIR, getDeploymentRow, async (row) => { const transaction = await createApplicationActivationTransaction(row.project, row.activation_spec); @@ -224,8 +223,23 @@ export async function loadComponentDirectories( for (const [deploymentId, error] of reconciliation.errors) { harperLogger.error(`Could not reconcile staged component deployment '${deploymentId}':`, errorForLog(error)); } + // FAIL CLOSED. An activation we could not finish leaves the live tree and its durable + // configuration disagreeing — the swapped-in code with the previous release's root config, or no + // live directory at all after an interrupted rename — while the deployment row still claims the + // activation is incomplete. Loading that is worse than not loading it: the component serves + // requests whose behavior nobody can predict from either the code or the config, and the next + // cold start may reinstall over it. So the affected component does not load. + for (const [project, error] of reconciliation.failedProjects) { + if (!failedRecoveries.has(project)) failedRecoveries.set(project, error); + } + // Only a clean pass retires the one-shot guard, so a later reload cycle retries instead of + // leaving a component permanently unrecovered and permanently unloadable. + if (reconciliation.errors.size === 0) stagedArtifactsReconciled = true; } catch (error) { + // The scan itself failed, so we cannot tell WHICH components are affected and cannot fail just + // those closed. Abort startup rather than load every component over possibly-unreconciled state. harperLogger.error('Could not inspect staged component deployments during startup:', errorForLog(error as Error)); + throw error; } } // Materialize hdb_secret global-tier rows into process.env and snapshot the scoped tier before diff --git a/components/operations.js b/components/operations.js index 7b5513a0a7..ae159cc9bf 100644 --- a/components/operations.js +++ b/components/operations.js @@ -1253,13 +1253,35 @@ async function revertComponent(req) { // Persist the reverted-to version's config + install lock. Compensating by swapping back is // deliberate: a half-reverted node (new tree live, old config persisted) would reinstall the // wrong version on its next cold start, which is the failure this pairing exists to prevent. + // + // The transaction is held OUTSIDE the try so the compensation can roll it back. `commit()` makes + // two persistent writes (root config, then the boot-time application lock); if the first + // succeeds and the second throws, swapping the directories back is not enough on its own — + // root config would still name the reverted-to release while the original tree is live again, + // which is the very mismatch this pairing exists to prevent, and a cold start could act on it. + const configTransaction = await createApplicationConfigTransaction(req.project, result.activatedConfig); try { - const configTransaction = await createApplicationConfigTransaction(req.project, result.activatedConfig); await configTransaction.commit(); } catch (configError) { + const compensationErrors = []; + // Undo the persistent writes first, then the directory swap, so the node is never left with + // the original tree live and the reverted-to configuration persisted. + await configTransaction.rollback().catch((rollbackError) => { + compensationErrors.push(rollbackError); + log.error(`Failed to roll back the ${req.project} revert configuration write`, rollbackError); + }); await revertApplication(application, result.fromDeploymentId).catch((swapBackError) => { + compensationErrors.push(swapBackError); log.error(`Failed to undo the ${req.project} revert after its config write failed`, swapBackError); }); + if (compensationErrors.length) { + throw new ServerError( + `Failed to revert ${req.project}: ${configError?.message ?? configError}. Compensation also ` + + `failed (${compensationErrors.map((error) => error?.message ?? error).join('; ')}), so this ` + + `node may be serving a version its persisted configuration does not name.`, + configError?.statusCode + ); + } throw configError; } } diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 49cd391384..f45701ec58 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -23,6 +23,7 @@ const { revertApplication, getRevertTarget, recoverInterruptedReverts, + createApplicationConfigTransaction, extractApplication, stagedApplicationPath, DEPLOY_STAGING_DIR, @@ -721,7 +722,7 @@ describe('two-phase component directory transaction', function () { it('startup recovery re-retains the displaced tree when only the retain step was lost', async () => { const name = fixtureName(); - const { application } = await twoActivations(name); + const { application, first, second } = await twoActivations(name); // Simulate a crash after `rename(previous, live)` but before `rename(holding, previous)`: the // reverted-to version is live, and the displaced tree is still parked. const previousPath = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); @@ -737,6 +738,19 @@ describe('two-phase component directory transaction', function () { assert.match(await readMarker(application.dirPath), /v1/, 'the reverted-to version stays live'); assert.match(await readMarker(previousPath), /displaced/, 'the displaced tree is retained again'); assert.equal(existsSync(holding), false); + + // The directories are only half the state. The manifest was written before the swap, so completing + // the recovery has to exchange its roles too — otherwise it names the retained tree as live and the + // live tree as retained, and the retry below matches its target against the reversed `previous` + // entry and swaps the successful revert straight back out. + const target = await getRevertTarget(application.dirPath); + assert.equal(target.live.deployment_id, first, 'the manifest names the reverted-to version as live'); + assert.equal(target.previous.deployment_id, second, 'and the displaced version as the retained one'); + + // Idempotency has to survive the recovery: re-issuing the same addressed revert changes nothing. + const retry = await revertApplication(application, first); + assert.equal(retry.swapped, false, 'a retry after recovery is a no-op, not a swap back'); + assert.match(await readMarker(application.dirPath), /v1/, 'still on the reverted-to version'); await cleanup(name); }); @@ -781,4 +795,91 @@ describe('two-phase component directory transaction', function () { await cleanup(name); await fs.rm(packageDirectory, { recursive: true, force: true }); }); + it('rolls both persistent writes back, so a failed commit cannot leave config half-applied', async () => { + // commit() makes TWO persistent writes — root config, then the boot-time application lock — and + // arms rollback (`commitStarted`) BEFORE either. That is what lets a caller whose commit threw + // partway undo the write that did land: without it, compensating by swapping the directories back + // still left root config naming the version just rolled away from, for a cold start to act on. + // + // A genuinely partial commit is not injectable from a test: the lock is READ (getApplicationLockEntry) + // before the config write, so any corruption that would break the lock write breaks that read first + // and fails before mutating anything. What is verifiable is the property the compensation depends + // on — that rollback restores the exact pre-commit state of both writes. + const name = fixtureName(); + const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-cfg-rollback-')); + const priorRootEnv = process.env.ROOTPATH; + const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); + process.env.ROOTPATH = configRoot; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); + await fs.writeFile(path.join(configRoot, 'harper-config.yaml'), `rootPath: ${JSON.stringify(configRoot)}\n`); + const lockPath = path.join(configRoot, 'harper-application-lock.json'); + const readLock = async () => JSON.parse(await fs.readFile(lockPath, { encoding: 'utf8' })).applications[name]; + try { + // The version that is live now, and that a revert would be rolling back to. + const original = await createApplicationConfigTransaction(name, { package: 'example@1.0.0' }); + await original.commit(); + assert.deepEqual(readConfigFile()[name], { package: 'example@1.0.0' }); + assert.deepEqual(await readLock(), { package: 'example@1.0.0' }); + + const reverting = await createApplicationConfigTransaction(name, { package: 'example@2.0.0' }); + await reverting.commit(); + assert.deepEqual(readConfigFile()[name], { package: 'example@2.0.0' }, 'both writes moved forward'); + assert.deepEqual(await readLock(), { package: 'example@2.0.0' }); + + await reverting.rollback(); + + assert.deepEqual( + readConfigFile()[name], + { package: 'example@1.0.0' }, + 'rollback restores the root config the commit replaced' + ); + assert.deepEqual(await readLock(), { package: 'example@1.0.0' }, 'and the application-lock entry'); + + // A transaction that never committed must not touch anything on rollback, so compensation on an + // early failure cannot clobber a live config. + const untouched = await createApplicationConfigTransaction(name, { package: 'example@3.0.0' }); + await untouched.rollback(); + assert.deepEqual(readConfigFile()[name], { package: 'example@1.0.0' }, 'no-op rollback changes nothing'); + } finally { + if (priorRootEnv === undefined) delete process.env.ROOTPATH; + else process.env.ROOTPATH = priorRootEnv; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); + await fs.rm(configRoot, { recursive: true, force: true }); + } + }); + + it('reports the component whose activation persistence failed, so it can be kept from loading', async () => { + // Startup reconciliation used to only LOG a failure to finish an `activating` deployment, then carry + // on into normal component loading — serving a swapped-in tree whose durable configuration still + // described the previous release. The caller can only fail that component closed if it is told which + // component failed, which a deployment id alone does not give it. + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('candidate') }); + await stageApplication(application, deploymentId); + await fs.mkdir(application.dirPath, { recursive: true }); + + const row = { + deployment_id: deploymentId, + project: name, + status: 'activating', + activation_spec: { package: null }, + }; + const reconciliation = await reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async () => row, + async () => { + throw new Error('simulated activation-persistence failure'); + } + ); + + assert.equal(reconciliation.errors.has(deploymentId), true, 'the failure is reported by deployment'); + assert.equal( + reconciliation.failedProjects.get(name)?.message, + 'simulated activation-persistence failure', + 'and attributed to the component, so the loader can fail it closed' + ); + assert.equal(reconciliation.recovered.includes(deploymentId), false, 'a failed reconciliation is not "recovered"'); + await cleanup(name); + }); }); From ed1f8eb76ac6da6cc2f47f18a819021a9c5939f4 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 14 Aug 2026 10:10:08 -0400 Subject: [PATCH 52/94] test(deploy): stop racing the async aside sweep when reading parked entries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit My own test broke CI on Node 24 and 26 (`ENOENT: scandir …/.deploy-aside/`), and it is a race in the test rather than the code: `discardDirAside` sweeps fire-and-forget (`void cleanupExtractionPaths(...)`), and that sweep rmdir's the directory once it is empty — so `existsSync(asideDir)` followed by `fs.readdir(asideDir)` loses whenever the sweep lands between the two. It passed locally on macOS purely on timing. Replaced both call sites with a `parkedAsideEntries()` helper that treats ENOENT as an empty listing, which is the honest reading: the assertions ask whether anything recoverable was left parked, and a directory the sweep already removed satisfies that as well as an empty one does. Ran the suite repeatedly to confirm it is stable. This also explains the `review / review` failure on the same commit — that action hit its own 48-turn ceiling while chasing this test failure, so it emitted no review. Co-Authored-By: Claude Opus 5 --- unitTests/components/deployStaging.test.js | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index f45701ec58..3b49cc3101 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -63,6 +63,19 @@ async function readMarker(directory) { return fs.readFile(path.join(directory, 'index.js'), 'utf8'); } +// Entries currently parked in a component's `.deploy-aside`. discardDirAside sweeps asynchronously +// (`void cleanupExtractionPaths(...)`) and that sweep rmdir's the directory once it is empty, so an +// existsSync-then-readdir races it. An absent directory means everything was already swept, which +// satisfies every assertion below just as well as an empty one. +async function parkedAsideEntries(componentName) { + try { + return await fs.readdir(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR, componentName)); + } catch (error) { + if (error.code === 'ENOENT') return []; + throw error; + } +} + describe('two-phase component directory transaction', function () { this.timeout(30_000); let sequence = 0; @@ -536,8 +549,7 @@ describe('two-phase component directory transaction', function () { await activateStagedApplication(application, id, { activationSpec: { package: null } }); } - const asideDir = path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR, name); - const parked = existsSync(asideDir) ? await fs.readdir(asideDir) : []; + const parked = await parkedAsideEntries(name); const unretired = parked.filter( (entry) => entry.startsWith('.in-progress-') && !parked.includes(`.retired-${entry.slice('.in-progress-'.length)}`) @@ -787,8 +799,7 @@ describe('two-phase component directory transaction', function () { assert.equal(await fs.readFile(path.join(application.dirPath, 'marker.txt'), 'utf8'), 'v2'); // Whatever was displaced must have been parked, not recursively removed in place. It is parked as // disposable (`.discarded-`), so startup recovery will never restore it over the new version. - const asideDir = path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR, name); - const parked = existsSync(asideDir) ? await fs.readdir(asideDir) : []; + const parked = await parkedAsideEntries(name); const recoverable = parked.filter((entry) => entry.startsWith('.in-progress-')); assert.deepEqual(recoverable, [], 'the displaced tree is never left looking like a rollback record'); From f6d2291347e29239b4eac67c96f00212dcd55934 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 14 Aug 2026 10:23:45 -0400 Subject: [PATCH 53/94] fix(deploy): re-point dependency links the activation swap invalidates MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of the Windows `redeploy-runtime-equivalence` failures, and a real defect in the two-phase design rather than a test or CI problem. `npm install` runs against the STAGING directory (`components/.deploy-staging//`, confirmed in the CI logs), and activation then renames that directory to the live path. Any dependency npm materialized with an ABSOLUTE target inside the staging directory dangles the instant that rename lands, because the path it names no longer exists. That is the normal shape of a `file:` dependency on Windows: npm creates a directory JUNCTION, and junctions are always absolute. On Linux npm writes a relative symlink (`../vendor/probe`) which survives the move untouched — which is exactly why every Linux, Bun and uWS shard passed while Windows failed on the same commit. The Windows symptom is a bare `Cannot find module ''` at component load, well after a deploy that reported success: ResourceLoadError: Failed to load resource module …/redeploy-pure-esm-runtime-app/resources.js caused by: Error: Cannot find module 'pure-esm-probe' code=MODULE_NOT_FOUND `activateStagedApplication` now walks the newly-live `node_modules` (including `@scope/`) and recreates any link whose absolute target was inside the consumed staging directory, pointing it at the same relative location under the live directory — as a junction on Windows, a dir symlink elsewhere. Links with relative targets, and absolute links pointing outside the staging tree (a dependency deliberately linked elsewhere on the machine), are left exactly as they are. A link that cannot be repointed is logged rather than failing the deploy, which has already succeeded by that point. Absolute links exist on every platform, so this is testable without Windows: the new test stands in for what npm leaves behind for `file:vendor/probe`, and asserts the dependency resolves from the live tree afterwards, that scoped packages are handled, and that an out-of-staging link is untouched. Verified it has teeth — with the repointing disabled it fails with `ENOENT … node_modules/probe/index.js`, the same failure class Windows reports. I have not been able to run Windows CI to completion on this branch (the `uWebSockets.js` tarball fetch keeps failing at install), so this is verified by mechanism and by a platform-independent regression test rather than by a green Windows shard. 303 deploy/component/CLI/server unit tests pass. Co-Authored-By: Claude Opus 5 --- components/Application.ts | 76 +++++++++++++++++++++- unitTests/components/deployStaging.test.js | 54 +++++++++++++++ 2 files changed, 129 insertions(+), 1 deletion(-) diff --git a/components/Application.ts b/components/Application.ts index 1f868a1237..d2530afe1f 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -22,7 +22,7 @@ import { import { getSecretDecryptor } from '../resources/secretDecryptor.ts'; import { ENV_ENCRYPTED_PREFIX } from '../utility/envFile.ts'; -import { basename, dirname, extname, join } from 'node:path'; +import { basename, dirname, extname, isAbsolute, join, relative } from 'node:path'; import { access, chmod, @@ -32,6 +32,7 @@ import { mkdtemp, readdir, readFile, + readlink, rename, rm, rmdir, @@ -2431,6 +2432,70 @@ async function readStagedCompletion(stagingDirPath: string): Promise<{ installat } } +/** + * Re-point dependency links that the activation swap invalidated. + * + * `npm install` runs against the STAGING directory, and activation then renames that directory to the + * live path. Any dependency npm materialized as a link with an ABSOLUTE target inside the staging + * directory therefore dangles the moment the rename happens — the path it names no longer exists. + * + * This is the normal case for a `file:` dependency on Windows, where npm creates a directory JUNCTION + * and junctions are always absolute. On Linux npm writes a relative symlink (`../vendor/probe`), which + * survives the move untouched, which is why this only ever bites on Windows — the failure there is a + * bare `Cannot find module ''` at component load, well after a deploy that reported success. + * + * Each such link is recreated pointing at the same relative location under the LIVE directory. Links + * whose targets are relative, or absolute but outside the staging tree (a dependency deliberately + * linked elsewhere on the machine), are left exactly as they are. + */ +async function repointStagedDependencyLinks(liveDirPath: string, stagingDirPath: string): Promise { + const nodeModulesPath = join(liveDirPath, 'node_modules'); + let topLevel; + try { + topLevel = await readdir(nodeModulesPath, { withFileTypes: true }); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return 0; + throw error; + } + // Package links live at `node_modules/` or, for a scoped package, `node_modules/@scope/`. + const candidates: string[] = []; + for (const entry of topLevel) { + if (entry.name.startsWith('.')) continue; + if (entry.name.startsWith('@')) { + const scopePath = join(nodeModulesPath, entry.name); + for (const scoped of await readdir(scopePath, { withFileTypes: true }).catch(() => [])) { + candidates.push(join(scopePath, scoped.name)); + } + continue; + } + candidates.push(join(nodeModulesPath, entry.name)); + } + + let repointed = 0; + for (const linkPath of candidates) { + const linkStat = await lstat(linkPath).catch(() => undefined); + if (!linkStat?.isSymbolicLink()) continue; + const target = await readlink(linkPath).catch(() => undefined); + if (!target || !isAbsolute(target)) continue; + const withinStaging = relative(stagingDirPath, target); + // `..` or an absolute result means the target is outside the staging tree — not ours to touch. + if (!withinStaging || withinStaging.startsWith('..') || isAbsolute(withinStaging)) continue; + try { + await rm(linkPath, { force: true }); + await symlink(join(liveDirPath, withinStaging), linkPath, process.platform === 'win32' ? 'junction' : 'dir'); + repointed++; + } catch (error) { + // Best-effort: a link we cannot repoint is reported, not fatal. The deploy already succeeded, + // and failing it here would leave the component live but the operation reporting failure. + logger.warn( + `Could not re-point the ${basename(linkPath)} dependency link after activating ${basename(liveDirPath)}:`, + errorForLog(error as Error) + ); + } + } + return repointed; +} + /** Atomically replace the live component and compensate if persistent activation work fails. */ export async function activateStagedApplication( application: Application, @@ -2551,6 +2616,15 @@ export async function activateStagedApplication( } finally { broadcastDeployEnd(application.name, deployLifecycleId); } + // npm installed against the staging path; the rename above just invalidated any absolute link it + // created inside it (a `file:` dependency junction on Windows). Repoint before anything loads. + const repointed = await repointStagedDependencyLinks(application.dirPath, stagingDirPath); + if (repointed) { + logger.debug?.( + `Re-pointed ${repointed} dependency link(s) in ${application.name} from the staging path to the live path` + ); + } + // The displaced tree is the component's rollback source now, not garbage: retain it as // `.deploy-previous/` with a manifest recording which deployment produced it. Best-effort — // a component that can't retain its previous version is still deployed, just not revertable. diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 3b49cc3101..9fe3691170 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -893,4 +893,58 @@ describe('two-phase component directory transaction', function () { assert.equal(reconciliation.recovered.includes(deploymentId), false, 'a failed reconciliation is not "recovered"'); await cleanup(name); }); + it('re-points a dependency link whose absolute target the activation swap invalidated', async () => { + // npm installs against the STAGING directory, and activation renames that directory to the live + // path — so any dependency npm materialized with an ABSOLUTE target inside staging dangles the + // instant the rename lands. That is the normal shape of a `file:` dependency on Windows, where npm + // creates a junction and junctions are always absolute; on Linux it writes a relative symlink that + // survives the move. The Windows symptom is a bare `Cannot find module ''` at component load, + // well after a deploy that reported success. + // + // Absolute links exist on every platform, so the behavior is reproducible here regardless of OS. + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('with-dep') }); + const stagedPath = await stageApplication(application, deploymentId); + + // Stand in for what `npm install --force` leaves behind for `file:vendor/probe`: a real vendored + // directory, plus an ABSOLUTE link to it from node_modules. + await fs.mkdir(path.join(stagedPath, 'vendor', 'probe'), { recursive: true }); + await fs.writeFile(path.join(stagedPath, 'vendor', 'probe', 'index.js'), "module.exports = 'probe';\n"); + await fs.mkdir(path.join(stagedPath, 'node_modules'), { recursive: true }); + await fs.symlink(path.join(stagedPath, 'vendor', 'probe'), path.join(stagedPath, 'node_modules', 'probe'), 'dir'); + // A scoped package, and a link deliberately pointing OUTSIDE staging, which must be left alone. + const external = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-external-dep-')); + await fs.writeFile(path.join(external, 'index.js'), "module.exports = 'external';\n"); + await fs.mkdir(path.join(stagedPath, 'node_modules', '@scope'), { recursive: true }); + await fs.symlink( + path.join(stagedPath, 'vendor', 'probe'), + path.join(stagedPath, 'node_modules', '@scope', 'scoped'), + 'dir' + ); + await fs.symlink(external, path.join(stagedPath, 'node_modules', 'external'), 'dir'); + + await activateStagedApplication(application, deploymentId, { activationSpec: { package: null } }); + + // The dependency has to be resolvable from the LIVE tree, which is the whole point. + assert.equal( + await fs.readFile(path.join(application.dirPath, 'node_modules', 'probe', 'index.js'), 'utf8'), + "module.exports = 'probe';\n", + 'the staged-path link was re-pointed at the live path' + ); + assert.equal( + await fs.readFile(path.join(application.dirPath, 'node_modules', '@scope', 'scoped', 'index.js'), 'utf8'), + "module.exports = 'probe';\n", + 'scoped packages are re-pointed too' + ); + // Untouched: an absolute link to somewhere outside the staging tree is a deliberate choice. + assert.equal( + await fs.readlink(path.join(application.dirPath, 'node_modules', 'external')), + external, + 'a link outside staging is left exactly as it was' + ); + + await cleanup(name); + await fs.rm(external, { recursive: true, force: true }); + }); }); From 19389140c42a03a54e93c1a18f882b7fa43de518 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 14 Aug 2026 16:59:35 -0400 Subject: [PATCH 54/94] fix(deploy): repair dependency links inside the activation transaction, and keep revert-recovery evidence MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three more findings from @kriszyp's Codex review, all real. The third one proposed a better design than what I had, and I took it. **1. Dependency-link repair now runs inside the transaction, before the swap (Application.ts:2487).** It was post-swap and best-effort: a failure logged a warning and the deploy reported success with a dangling dependency — which pre-swap load validation structurally cannot catch, because the link was perfectly valid in staging. The repair now runs against the staging tree before the rename and throws on failure, so the existing compensation restores the previous release and the operation returns an error. Per the reviewer's suggestion, links are rewritten to their FUTURE live targets while still in staging, so after the rename they are already correct. I did consider creating them post-swap instead and kept the pre-swap form deliberately: it is the only variant where a failure is still compensable. **2. The walk is contained to the component's own node_modules (Application.ts:2466).** `readdir` follows symlinks, so a staged payload shipping `node_modules` — or a `node_modules/@scope` — as a link to somewhere else on the machine had that directory's children enumerated as candidates, and a link in there whose target happened to point into staging would be removed and recreated. That is a deployment writing outside the component tree. Both levels now require a real, non-symlink directory before descending, and every candidate is re-checked to be beneath the real node_modules root before it is mutated. **3. Revert recovery no longer destroys its own evidence (Application.ts:829).** The holding directory is the only durable sign that recovery is unfinished, and it was consumed by `rename(holding, previous)` before the manifest and config writes. A failure after that point left nothing for the next start to find: recovery saw a complete-looking tree and the component loaded with whichever write half-landed, with no retry. Recovery now writes a marker recording the intended end state before mutating anything, commits config and the manifest first, moves the tree back last, and clears the marker only when all three are durable. On re-entry the marker — not the manifest — is the source of truth, because an earlier attempt may already have exchanged the manifest and exchanging it twice flips it back. Tests, each covering the failure case rather than the happy path, and each mutation-verified: - a symlinked `node_modules/@scope` leaves the external directory's contents untouched (fails without the containment guards) - an unrepairable dependency link keeps the previous release live and rejects - a recovery whose first pass fails before the tree is moved back preserves the holding tree and marker, and a second pass finishes it (fails with the old ordering, on "the holding tree survives") 320 passing. The 6 failures in extractApplicationSwap.test.js remain pre-existing on kris/deploy-peer-rollback and are unaffected. Co-Authored-By: Claude Opus 5 --- components/Application.ts | 117 ++++++++++++++------ unitTests/components/deployStaging.test.js | 122 +++++++++++++++++++++ 2 files changed, 204 insertions(+), 35 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index d2530afe1f..1115febcc6 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -495,6 +495,10 @@ const DISCARDED_ASIDE_PREFIX = '.discarded-'; // The holding path a revert's three-way swap parks the outgoing live tree in. Recoverable: if the // process dies mid-swap, recoverInterruptedReverts puts it back. See revertApplication. const REVERTING_PREFIX = '.reverting-'; +// Records the manifest state a revert recovery is working toward. Written before recovery mutates +// anything and removed only once the directory, manifest and config are all durable, so a recovery +// interrupted part-way is resumable rather than losing its own evidence. See recoverInterruptedReverts. +const REVERT_RECOVERY_SUFFIX = '.recovering.json'; // Splits `-` off a revert holding directory name, anchored on the UUID so a // component name containing dashes is preserved intact. const REVERTING_NAME_PATTERN = /^(.+)-[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; @@ -698,6 +702,24 @@ async function readRetainedPreviousManifest(liveDirPath: string): Promise { + try { + const parsed = JSON.parse(await readFile(markerPath, 'utf8')); + if (!parsed?.previous || !parsed?.live) return undefined; + return parsed as RetainedPreviousManifest; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } +} + +async function writeJsonAtomically(targetPath: string, value: unknown): Promise { + const tempPath = `${targetPath}.${process.pid}.${randomUUID()}.tmp`; + await mkdir(dirname(targetPath), { recursive: true }); + await writeFile(tempPath, JSON.stringify(value, null, 2), { mode: 0o600 }); + await rename(tempPath, targetPath); +} + async function writeRetainedPreviousManifest(liveDirPath: string, manifest: RetainedPreviousManifest): Promise { const manifestPath = previousManifestPathFor(liveDirPath); // Temp + rename so a crash mid-write can never leave a half-written manifest, which would make a @@ -824,23 +846,36 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): // its target against the reversed `previous` entry and swap the successful revert back out — // destroying the idempotency the addressed target exists to provide. Roles are exchanged here // exactly as revertApplication would have. - const staleManifest = await readRetainedPreviousManifest(liveDirPath); - await mkdir(dirname(previousPath), { recursive: true }); - await rename(holding, previousPath); - if (staleManifest) { - await writeRetainedPreviousManifest(liveDirPath, { - previous: staleManifest.live, - live: staleManifest.previous, - }); - // Config was never committed either (revertComponent commits after the swap returns), so it - // still describes the version this revert moved away from. Bring it in line with what is - // actually live, or a cold start reinstalls the reverted-away release over it. + // The holding directory is the ONLY durable evidence that this recovery is unfinished, so it + // must not be consumed until the manifest and config writes are durable too. Otherwise a + // failure after the rename leaves nothing for the next start to find: recovery sees a + // complete-looking tree and the component loads with whichever write half-landed. + // + // A recovery marker carries the intended end state across restarts. On re-entry it — not the + // manifest — is the source of truth, because a previous attempt may already have exchanged the + // manifest, and exchanging an exchanged manifest would flip it back. + const recoveryMarkerPath = `${previousPath}${REVERT_RECOVERY_SUFFIX}`; + let intended = await readRevertRecoveryMarker(recoveryMarkerPath); + if (!intended) { + const staleManifest = await readRetainedPreviousManifest(liveDirPath); + if (staleManifest) { + intended = { previous: staleManifest.live, live: staleManifest.previous }; + await writeJsonAtomically(recoveryMarkerPath, intended); + } + } + if (intended) { + // Config first, then manifest, then the directory — so the holding tree survives until + // everything else is durable. Each step is idempotent on a repeat pass. const configTransaction = await createApplicationConfigTransaction( componentName, - staleManifest.previous.application_config + intended.live.application_config ); await configTransaction.commit(); + await writeRetainedPreviousManifest(liveDirPath, intended); } + await mkdir(dirname(previousPath), { recursive: true }); + await rename(holding, previousPath); + await rm(recoveryMarkerPath, { force: true }); logger.warn( `Completed an interrupted ${componentName} revert: the reverted-to version is live, the ` + `version it displaced is retained again, and its configuration has been reconciled` @@ -2448,8 +2483,15 @@ async function readStagedCompletion(stagingDirPath: string): Promise<{ installat * whose targets are relative, or absolute but outside the staging tree (a dependency deliberately * linked elsewhere on the machine), are left exactly as they are. */ -async function repointStagedDependencyLinks(liveDirPath: string, stagingDirPath: string): Promise { - const nodeModulesPath = join(liveDirPath, 'node_modules'); +async function repointStagedDependencyLinks(stagingDirPath: string, futureLiveDirPath: string): Promise { + const nodeModulesPath = join(stagingDirPath, 'node_modules'); + // The walk must stay inside the component's own node_modules. `readdir` FOLLOWS symlinks, so a staged + // payload shipping `node_modules` (or a `node_modules/@scope`) as a link to somewhere else on the + // machine would otherwise have its target's contents enumerated — and a link in there whose target + // happened to point into staging would be removed and recreated, writing outside the component tree. + // Requiring a real directory at each level we descend keeps every candidate under the real root. + const nodeModulesStat = await lstat(nodeModulesPath).catch(() => undefined); + if (!nodeModulesStat?.isDirectory() || nodeModulesStat.isSymbolicLink()) return 0; let topLevel; try { topLevel = await readdir(nodeModulesPath, { withFileTypes: true }); @@ -2463,6 +2505,10 @@ async function repointStagedDependencyLinks(liveDirPath: string, stagingDirPath: if (entry.name.startsWith('.')) continue; if (entry.name.startsWith('@')) { const scopePath = join(nodeModulesPath, entry.name); + // Only descend into a REAL directory: a symlinked scope directory belongs to whatever it points + // at, not to this component (see the note on nodeModulesStat above). + const scopeStat = await lstat(scopePath).catch(() => undefined); + if (!scopeStat?.isDirectory() || scopeStat.isSymbolicLink()) continue; for (const scoped of await readdir(scopePath, { withFileTypes: true }).catch(() => [])) { candidates.push(join(scopePath, scoped.name)); } @@ -2477,21 +2523,20 @@ async function repointStagedDependencyLinks(liveDirPath: string, stagingDirPath: if (!linkStat?.isSymbolicLink()) continue; const target = await readlink(linkPath).catch(() => undefined); if (!target || !isAbsolute(target)) continue; + // Belt-and-braces containment: never mutate a path that is not under the real node_modules root. + const withinNodeModules = relative(nodeModulesPath, linkPath); + if (withinNodeModules.startsWith('..') || isAbsolute(withinNodeModules)) continue; const withinStaging = relative(stagingDirPath, target); // `..` or an absolute result means the target is outside the staging tree — not ours to touch. if (!withinStaging || withinStaging.startsWith('..') || isAbsolute(withinStaging)) continue; - try { - await rm(linkPath, { force: true }); - await symlink(join(liveDirPath, withinStaging), linkPath, process.platform === 'win32' ? 'junction' : 'dir'); - repointed++; - } catch (error) { - // Best-effort: a link we cannot repoint is reported, not fatal. The deploy already succeeded, - // and failing it here would leave the component live but the operation reporting failure. - logger.warn( - `Could not re-point the ${basename(linkPath)} dependency link after activating ${basename(liveDirPath)}:`, - errorForLog(error as Error) - ); - } + // NOT best-effort. This link is only being touched because activation is about to invalidate its + // target, so a failure here means the component goes live with a dangling dependency — which the + // pre-swap load validation cannot catch, because the link was perfectly valid in staging. Throwing + // keeps the old release live and returns an error, instead of reporting a successful deploy of a + // component that cannot resolve its dependencies. + await rm(linkPath, { force: true }); + await symlink(join(futureLiveDirPath, withinStaging), linkPath, process.platform === 'win32' ? 'junction' : 'dir'); + repointed++; } return repointed; } @@ -2580,6 +2625,17 @@ export async function activateStagedApplication( } } } + // Re-point dependency links BEFORE the swap and inside the transaction. npm installed against the + // staging path, so any link it created with an absolute target inside staging is about to become + // dangling; rewriting them to their future live targets now means a failure lands in the catch + // below, which restores the previous release rather than leaving a live component that cannot + // resolve its dependencies. Done pre-swap for the compensation, not post-swap for convenience. + const repointed = await repointStagedDependencyLinks(stagingDirPath, application.dirPath); + if (repointed) { + logger.debug?.( + `Re-pointed ${repointed} dependency link(s) in ${application.name} from the staging path to the live path` + ); + } await rename(stagingDirPath, application.dirPath); swapped = true; await hooks.beforeCommit?.(); @@ -2616,15 +2672,6 @@ export async function activateStagedApplication( } finally { broadcastDeployEnd(application.name, deployLifecycleId); } - // npm installed against the staging path; the rename above just invalidated any absolute link it - // created inside it (a `file:` dependency junction on Windows). Repoint before anything loads. - const repointed = await repointStagedDependencyLinks(application.dirPath, stagingDirPath); - if (repointed) { - logger.debug?.( - `Re-pointed ${repointed} dependency link(s) in ${application.name} from the staging path to the live path` - ); - } - // The displaced tree is the component's rollback source now, not garbage: retain it as // `.deploy-previous/` with a manifest recording which deployment produced it. Best-effort — // a component that can't retain its previous version is still deployed, just not revertable. diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 9fe3691170..f9817020ad 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -947,4 +947,126 @@ describe('two-phase component directory transaction', function () { await cleanup(name); await fs.rm(external, { recursive: true, force: true }); }); + it('leaves external contents untouched when node_modules/@scope is a symlink', async () => { + // `readdir` follows symlinks, so a staged payload shipping `node_modules/@scope` as a link to + // somewhere else on the machine would otherwise have that directory's children enumerated as + // candidates — and a link in there whose target happened to point into staging would be removed and + // recreated, writing outside the component tree. + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('scoped') }); + const stagedPath = await stageApplication(application, deploymentId); + + // An external directory holding a link that DOES point into staging — the dangerous shape. + const external = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-external-scope-')); + await fs.mkdir(path.join(stagedPath, 'vendor', 'probe'), { recursive: true }); + await fs.writeFile(path.join(stagedPath, 'vendor', 'probe', 'index.js'), "module.exports = 'probe';\n"); + await fs.symlink(path.join(stagedPath, 'vendor', 'probe'), path.join(external, 'bait'), 'dir'); + const externalBaitTarget = await fs.readlink(path.join(external, 'bait')); + + // node_modules/@scope is a LINK to that external directory. + await fs.mkdir(path.join(stagedPath, 'node_modules'), { recursive: true }); + await fs.symlink(external, path.join(stagedPath, 'node_modules', '@scope'), 'dir'); + + await activateStagedApplication(application, deploymentId, { activationSpec: { package: null } }); + + assert.equal( + await fs.readlink(path.join(external, 'bait')), + externalBaitTarget, + 'a link inside a symlinked scope directory is never rewritten' + ); + assert.equal( + (await fs.lstat(path.join(application.dirPath, 'node_modules', '@scope'))).isSymbolicLink(), + true, + 'the scope link itself is left as a link' + ); + await cleanup(name); + await fs.rm(external, { recursive: true, force: true }); + }); + + it('keeps the previous release live when a dependency link cannot be repaired', async () => { + // The repair is not best-effort: the link is only being touched because activation is about to + // invalidate its target, so a failure means the component would go live unable to resolve a + // dependency — which pre-swap load validation cannot catch, since the link was valid in staging. + const name = fixtureName(); + const first = randomUUID(); + const second = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('live-v1') }); + await stageApplication(application, first); + await activateStagedApplication(application, first, { activationSpec: { package: null } }); + + application.payload = await makeComponentPayload('candidate-v2'); + const stagedPath = await stageApplication(application, second); + await fs.mkdir(path.join(stagedPath, 'vendor', 'probe'), { recursive: true }); + await fs.mkdir(path.join(stagedPath, 'node_modules'), { recursive: true }); + await fs.symlink(path.join(stagedPath, 'vendor', 'probe'), path.join(stagedPath, 'node_modules', 'probe'), 'dir'); + // Make recreating the link impossible: replace its parent with a read-only directory so the unlink + // and re-symlink cannot succeed. + await fs.chmod(path.join(stagedPath, 'node_modules'), 0o500); + + try { + await assert.rejects( + () => activateStagedApplication(application, second, { activationSpec: { package: null } }), + 'an unrepairable dependency link must fail the activation' + ); + assert.match( + await readMarker(application.dirPath), + /live-v1/, + 'the previous release is still live — the swap was compensated' + ); + } finally { + await fs.chmod(path.join(stagedPath, 'node_modules'), 0o700).catch(() => {}); + } + await cleanup(name); + }); + + it('resumes revert recovery when a first pass fails before the tree is moved back', async () => { + // The holding directory is the only durable evidence that recovery is unfinished, so it must not be + // consumed until the manifest and config writes are durable. A recovery marker carries the intended + // end state across restarts, and on re-entry it — not the manifest — is the source of truth, because + // an earlier attempt may already have exchanged the manifest and exchanging it twice flips it back. + const name = fixtureName(); + const { application, first, second } = await twoActivations(name); + const previousPath = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); + const manifestPath = `${previousPath}.json`; + const markerPath = `${previousPath}.recovering.json`; + const holding = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, `.reverting-${name}-${randomUUID()}`); + + // Crash state: the reverted-to version is live, the displaced tree is still parked. + const staleManifest = JSON.parse(await fs.readFile(manifestPath, 'utf8')); + await fs.rm(application.dirPath, { recursive: true, force: true }); + await fs.rename(previousPath, application.dirPath); + await fs.mkdir(holding, { recursive: true }); + await fs.writeFile(path.join(holding, 'index.js'), "module.exports = 'displaced';\n"); + // Stand in for a first pass that recorded its intent and then died: the marker is present. + await fs.writeFile( + markerPath, + JSON.stringify({ previous: staleManifest.live, live: staleManifest.previous }, null, 2) + ); + // Make the manifest WRITE fail (its read still succeeds, and the marker is what recovery reads + // anyway) by putting a directory where the manifest file belongs. + await fs.rm(manifestPath, { recursive: true, force: true }); + await fs.mkdir(manifestPath, { recursive: true }); + + const firstPass = await recoverInterruptedReverts(COMPONENTS_ROOT); + + assert.equal(firstPass.has(name), true, 'the failed pass is reported, so the component is failed closed'); + assert.equal(existsSync(holding), true, 'the holding tree survives, so a later pass can still finish'); + assert.equal(existsSync(markerPath), true, 'and the recovery marker survives with it'); + + // Clear the injected fault and run recovery again, as the next process start would. + await fs.rm(manifestPath, { recursive: true, force: true }); + const secondPass = await recoverInterruptedReverts(COMPONENTS_ROOT); + + assert.equal(secondPass.size, 0, 'the second pass completes'); + assert.match(await readMarker(application.dirPath), /v1/, 'the reverted-to version is live'); + assert.match(await readMarker(previousPath), /displaced/, 'the displaced tree is retained again'); + assert.equal(existsSync(holding), false, 'the holding tree is consumed only once everything is durable'); + assert.equal(existsSync(markerPath), false, 'and the marker is cleared'); + // The marker, not the twice-read manifest, decided the end state — so roles are exchanged once. + const target = await getRevertTarget(application.dirPath); + assert.equal(target.live.deployment_id, first); + assert.equal(target.previous.deployment_id, second); + await cleanup(name); + }); }); From dea4788aa8d2e6513d43eeae4070bd7c086c4fff Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 19:28:17 -0600 Subject: [PATCH 55/94] Make rollback placeholders movable Co-Authored-By: GPT-5 Codex --- DESIGN.md | 5 ++++ components/Application.ts | 24 ++++++++++++++++++ .../components/extractApplicationSwap.test.js | 25 +++++++++++++++++++ 3 files changed, 54 insertions(+) diff --git a/DESIGN.md b/DESIGN.md index ee80c0032a..cd8f4555e8 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -152,6 +152,11 @@ Extraction renames an existing component aside before writing the replacement an dependency installation and metadata verification complete. Any preparation failure atomically renames the partial tree into hidden staging before restoring the prior tree, so a live writer cannot wedge rollback with `ENOTEMPTY`; cleanup completes while the same-component lock is still held. +On non-root POSIX systems, rollback uses a mode-`000` placeholder to keep that writer out between +retries. Before moving or removing it, rollback verifies the placeholder's device/inode identity and +restores owner permissions because a cross-parent directory move updates `..` and requires write +permission on the moved directory. Recovery also recognizes an owner-owned mode-`000` directory as +an orphaned placeholder so a crash during rollback cannot leave the component wedged. The aside name is itself the recovery record: `.in-progress-*` is recoverable after an interrupted deploy unless a sibling `.retired-*` marker records that the replacement committed. Cleanup removes the aside before its marker, so an interrupted cleanup cannot make an obsolete tree recoverable. diff --git a/components/Application.ts b/components/Application.ts index 883122b3db..1024fe4816 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -1100,8 +1100,30 @@ async function rollbackExtractedDirectory( let restoreRetryDeadline: number | undefined; let fallbackDisplacedPath: string | undefined; let placeholderIdentity: { dev: bigint; ino: bigint } | undefined; + const makeRollbackPlaceholderMovable = async (): Promise => { + try { + const current = await lstat(application.dirPath, { bigint: true }); + const processUid = process.getuid?.(); + const isTrackedPlaceholder = + placeholderIdentity && current.dev === placeholderIdentity.dev && current.ino === placeholderIdentity.ino; + const isOrphanedPlaceholder = + !placeholderIdentity && + process.platform !== 'win32' && + processUid !== undefined && + processUid !== 0 && + current.isDirectory() && + current.uid === BigInt(processUid) && + (current.mode & 0o777n) === 0n; + if (isTrackedPlaceholder || isOrphanedPlaceholder) { + await chmod(application.dirPath, 0o700); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + }; const failRestore = async (error: unknown): Promise => { try { + await makeRollbackPlaceholderMovable(); if (retainReplacement && fallbackDisplacedPath) { await rm(application.dirPath, { recursive: true, @@ -1145,6 +1167,7 @@ async function rollbackExtractedDirectory( }; do { try { + await makeRollbackPlaceholderMovable(); await rename(asidePath, application.dirPath); transactionPaths.delete(asidePath); await cleanupExtractionPaths(application, asideStagingDir, transactionPaths); @@ -1157,6 +1180,7 @@ async function rollbackExtractedDirectory( let displacedPath: string | undefined; const displacedPlaceholderIdentity = placeholderIdentity; try { + await makeRollbackPlaceholderMovable(); displacedPath = await displaceCurrentDirectory(); placeholderIdentity = undefined; if (displacedPath && displacedPlaceholderIdentity) { diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index 663e110b65..13fe86c77e 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -484,6 +484,31 @@ describe('extractApplication directory swap', () => { await fs.rm(sourceDir, { recursive: true, force: true }); }); + it('recovers after a crash leaves an occupied rollback placeholder', async function () { + if (process.platform === 'win32' || process.getuid?.() === 0) this.skip(); + this.timeout(20000); + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-placeholder-')); + const dirPath = path.join(componentsRoot, 'web'); + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'occupied'), 'writer raced rollback\n'); + await fs.chmod(dirPath, 0o000); + const stagingDir = path.join(componentsRoot, '.deploy-aside', 'web'); + const asidePath = path.join(stagingDir, `.in-progress-${Date.now()}-crashed`); + await fs.mkdir(asidePath, { recursive: true }); + await fs.writeFile(path.join(asidePath, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + + try { + const failures = await recoverInterruptedComponentExtractions(componentsRoot); + assert.deepStrictEqual([...failures], []); + assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); + await assert.rejects(fs.access(path.join(dirPath, 'occupied'))); + await assert.rejects(fs.access(stagingDir)); + } finally { + await fs.chmod(dirPath, 0o700).catch(() => {}); + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + it('reclaims a stale aside copy left by an earlier deploy', async function () { this.timeout(20000); From 66cd942428c6293e15cfe942b1d9fc845ae99600 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Sun, 16 Aug 2026 20:05:04 -0600 Subject: [PATCH 56/94] Keep placeholder repair identity-gated Co-Authored-By: GPT-5 Codex --- DESIGN.md | 3 +- components/Application.ts | 42 ++++++++----------- .../components/extractApplicationSwap.test.js | 23 +++++----- 3 files changed, 29 insertions(+), 39 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index cd8f4555e8..d8e81ac0c6 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -155,8 +155,7 @@ wedge rollback with `ENOTEMPTY`; cleanup completes while the same-component lock On non-root POSIX systems, rollback uses a mode-`000` placeholder to keep that writer out between retries. Before moving or removing it, rollback verifies the placeholder's device/inode identity and restores owner permissions because a cross-parent directory move updates `..` and requires write -permission on the moved directory. Recovery also recognizes an owner-owned mode-`000` directory as -an orphaned placeholder so a crash during rollback cannot leave the component wedged. +permission on the moved directory. The aside name is itself the recovery record: `.in-progress-*` is recoverable after an interrupted deploy unless a sibling `.retired-*` marker records that the replacement committed. Cleanup removes the aside before its marker, so an interrupted cleanup cannot make an obsolete tree recoverable. diff --git a/components/Application.ts b/components/Application.ts index 1024fe4816..f7e527be88 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -841,6 +841,21 @@ function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } +export async function makeRollbackPlaceholderMovable( + applicationDirPath: string, + placeholderIdentity: { dev: bigint; ino: bigint } | undefined +): Promise { + if (!placeholderIdentity) return; + try { + const current = await lstat(applicationDirPath, { bigint: true }); + if (current.dev === placeholderIdentity.dev && current.ino === placeholderIdentity.ino) { + await chmod(applicationDirPath, 0o700); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } +} + async function ensureExtractionStagingDirectory(asideStagingDir: string): Promise { for (const stagingDir of [dirname(asideStagingDir), asideStagingDir]) { await mkdir(stagingDir, { recursive: true, mode: 0o700 }); @@ -1100,30 +1115,9 @@ async function rollbackExtractedDirectory( let restoreRetryDeadline: number | undefined; let fallbackDisplacedPath: string | undefined; let placeholderIdentity: { dev: bigint; ino: bigint } | undefined; - const makeRollbackPlaceholderMovable = async (): Promise => { - try { - const current = await lstat(application.dirPath, { bigint: true }); - const processUid = process.getuid?.(); - const isTrackedPlaceholder = - placeholderIdentity && current.dev === placeholderIdentity.dev && current.ino === placeholderIdentity.ino; - const isOrphanedPlaceholder = - !placeholderIdentity && - process.platform !== 'win32' && - processUid !== undefined && - processUid !== 0 && - current.isDirectory() && - current.uid === BigInt(processUid) && - (current.mode & 0o777n) === 0n; - if (isTrackedPlaceholder || isOrphanedPlaceholder) { - await chmod(application.dirPath, 0o700); - } - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - } - }; const failRestore = async (error: unknown): Promise => { try { - await makeRollbackPlaceholderMovable(); + await makeRollbackPlaceholderMovable(application.dirPath, placeholderIdentity); if (retainReplacement && fallbackDisplacedPath) { await rm(application.dirPath, { recursive: true, @@ -1167,7 +1161,7 @@ async function rollbackExtractedDirectory( }; do { try { - await makeRollbackPlaceholderMovable(); + await makeRollbackPlaceholderMovable(application.dirPath, placeholderIdentity); await rename(asidePath, application.dirPath); transactionPaths.delete(asidePath); await cleanupExtractionPaths(application, asideStagingDir, transactionPaths); @@ -1180,7 +1174,7 @@ async function rollbackExtractedDirectory( let displacedPath: string | undefined; const displacedPlaceholderIdentity = placeholderIdentity; try { - await makeRollbackPlaceholderMovable(); + await makeRollbackPlaceholderMovable(application.dirPath, placeholderIdentity); displacedPath = await displaceCurrentDirectory(); placeholderIdentity = undefined; if (displacedPath && displacedPlaceholderIdentity) { diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index 13fe86c77e..9385e6085e 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -14,6 +14,7 @@ const { recoverInterruptedComponentExtraction, recoverInterruptedComponentExtractions, dropComponentDirectory, + makeRollbackPlaceholderMovable, Application, } = require('#src/components/Application'); const { packageDirectory } = require('#src/components/packageComponent'); @@ -484,25 +485,21 @@ describe('extractApplication directory swap', () => { await fs.rm(sourceDir, { recursive: true, force: true }); }); - it('recovers after a crash leaves an occupied rollback placeholder', async function () { + it('makes only the tracked rollback placeholder movable', async function () { if (process.platform === 'win32' || process.getuid?.() === 0) this.skip(); - this.timeout(20000); const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-placeholder-')); const dirPath = path.join(componentsRoot, 'web'); await fs.mkdir(dirPath, { recursive: true }); - await fs.writeFile(path.join(dirPath, 'occupied'), 'writer raced rollback\n'); - await fs.chmod(dirPath, 0o000); - const stagingDir = path.join(componentsRoot, '.deploy-aside', 'web'); - const asidePath = path.join(stagingDir, `.in-progress-${Date.now()}-crashed`); - await fs.mkdir(asidePath, { recursive: true }); - await fs.writeFile(path.join(asidePath, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + const placeholder = await fs.lstat(dirPath, { bigint: true }); try { - const failures = await recoverInterruptedComponentExtractions(componentsRoot); - assert.deepStrictEqual([...failures], []); - assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); - await assert.rejects(fs.access(path.join(dirPath, 'occupied'))); - await assert.rejects(fs.access(stagingDir)); + await fs.chmod(dirPath, 0o000); + await makeRollbackPlaceholderMovable(dirPath, { dev: placeholder.dev, ino: placeholder.ino }); + assert.strictEqual((await fs.lstat(dirPath)).mode & 0o777, 0o700); + + await fs.chmod(dirPath, 0o000); + await makeRollbackPlaceholderMovable(dirPath, { dev: placeholder.dev, ino: placeholder.ino + 1n }); + assert.strictEqual((await fs.lstat(dirPath)).mode & 0o777, 0o000); } finally { await fs.chmod(dirPath, 0o700).catch(() => {}); await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); From dcbc09454122780b283c6a7729212f49bbd72def Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 18:41:53 -0600 Subject: [PATCH 57/94] Complete interrupted component recovery Co-Authored-By: GPT-5 Codex --- DESIGN.md | 8 +- components/Application.ts | 115 +++++++-- components/componentLoader.ts | 238 +++++++++--------- unitTests/components/componentLoader.test.js | 85 ++++++- .../components/extractApplicationSwap.test.js | 97 ++++++- 5 files changed, 398 insertions(+), 145 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index d8e81ac0c6..1ad76ebe40 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -156,9 +156,11 @@ On non-root POSIX systems, rollback uses a mode-`000` placeholder to keep that w retries. Before moving or removing it, rollback verifies the placeholder's device/inode identity and restores owner permissions because a cross-parent directory move updates `..` and requires write permission on the moved directory. -The aside name is itself the recovery record: `.in-progress-*` is recoverable after an interrupted -deploy unless a sibling `.retired-*` marker records that the replacement committed. Cleanup removes -the aside before its marker, so an interrupted cleanup cannot make an obsolete tree recoverable. +The aside name is itself the recovery record: an `.in-progress-*` directory or symlink preserves a +previous tree, while an `.in-progress-*-prior-absent` file records that a first deploy must remove a +partial live tree after a crash. A sibling `.retired-*` marker records that the replacement committed. +Cleanup removes the recovery record before its marker, so an interrupted cleanup cannot make obsolete +state recoverable. Component loading recovers unretired interrupted deploys before scanning the component root, and preparation repeats recovery under the same-component lock before reading runtime metadata. A full `drop_component` writes retirement markers before deleting the live tree and keeps its filesystem, diff --git a/components/Application.ts b/components/Application.ts index f7e527be88..ac9f743d8c 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -482,6 +482,7 @@ async function runNpmPack( export const ASIDE_STAGING_DIR = '.deploy-aside'; const IN_PROGRESS_ASIDE_PREFIX = '.in-progress-'; const RETIRED_ASIDE_PREFIX = '.retired-'; +const PRIOR_ABSENT_RECORD_SUFFIX = '-prior-absent'; const DEFAULT_COMMAND_TIMEOUT_MS = 60 * 60 * 1000; const COMPONENT_PREPARATION_WAIT_MARGIN_MS = 30000; const COMPONENT_RECOVERY_WAIT_TIMEOUT_MS = 30000; @@ -741,6 +742,7 @@ export async function extractApplication( const asideStagingDir = extractionStagingDirectory(application.dirPath); const transactionPaths = new Set(); let asidePath: string | undefined; + let recoveryRecordPath: string; try { await ensureExtractionStagingDirectory(asideStagingDir); await recoverOrCleanupStaleExtractionPaths(application, asideStagingDir); @@ -756,6 +758,15 @@ export async function extractApplication( asidePath = join(asideStagingDir, `${IN_PROGRESS_ASIDE_PREFIX}${Date.now()}-${process.pid}-${randomUUID()}`); await rename(application.dirPath, asidePath); transactionPaths.add(asidePath); + recoveryRecordPath = asidePath; + } else { + await ensureExtractionStagingDirectory(asideStagingDir); + recoveryRecordPath = join( + asideStagingDir, + `${IN_PROGRESS_ASIDE_PREFIX}${Date.now()}-${process.pid}-${randomUUID()}${PRIOR_ABSENT_RECORD_SUFFIX}` + ); + await writeFile(recoveryRecordPath, '', { flag: 'wx', mode: 0o600 }); + transactionPaths.add(recoveryRecordPath); } if (asidePath) application.isNewComponent = false; @@ -766,13 +777,20 @@ export async function extractApplication( const extracted = await readdir(application.dirPath, { withFileTypes: true }); if (extracted.length === 1 && extracted[0].isDirectory()) { const topLevelDirPath = join(application.dirPath, extracted[0].name); - await ensureExtractionStagingDirectory(asideStagingDir); - const tempDirPath = join(asideStagingDir, `.normalize-${process.pid}-${Date.now()}-${randomUUID()}`); - transactionPaths.add(tempDirPath); - await rename(topLevelDirPath, tempDirPath); - await rmdir(application.dirPath); - await rename(tempDirPath, application.dirPath); - transactionPaths.delete(tempDirPath); + if (process.platform === 'win32') { + for (const childName of await readdir(topLevelDirPath)) { + await rename(join(topLevelDirPath, childName), join(application.dirPath, childName)); + } + await rmdir(topLevelDirPath); + } else { + await ensureExtractionStagingDirectory(asideStagingDir); + const tempDirPath = join(asideStagingDir, `.normalize-${process.pid}-${Date.now()}-${randomUUID()}`); + transactionPaths.add(tempDirPath); + await rename(topLevelDirPath, tempDirPath); + await rmdir(application.dirPath); + await rename(tempDirPath, application.dirPath); + transactionPaths.delete(tempDirPath); + } } } catch (error) { try { @@ -799,10 +817,8 @@ export async function extractApplication( const transaction: ExtractionTransaction = { async commit() { if (settled) return; - if (asidePath) { - const retiredMarkerPath = await retireExtractionAside(asidePath); - transactionPaths.add(retiredMarkerPath); - } + const retiredMarkerPath = await retireExtractionAside(recoveryRecordPath); + transactionPaths.add(retiredMarkerPath); settled = true; await cleanupExtractionPaths(application, asideStagingDir, transactionPaths); }, @@ -856,6 +872,26 @@ export async function makeRollbackPlaceholderMovable( } } +async function identifyRollbackPlaceholder( + applicationDirPath: string +): Promise<{ dev: bigint; ino: bigint } | undefined> { + const userId = process.getuid?.(); + if (process.platform === 'win32' || userId === undefined || userId === 0) return undefined; + try { + const current = await lstat(applicationDirPath, { bigint: true }); + if ( + (current.isDirectory() || current.isFile()) && + current.uid === BigInt(userId) && + (Number(current.mode) & 0o777) === 0 + ) { + return { dev: current.dev, ino: current.ino }; + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + } + return undefined; +} + async function ensureExtractionStagingDirectory(asideStagingDir: string): Promise { for (const stagingDir of [dirname(asideStagingDir), asideStagingDir]) { await mkdir(stagingDir, { recursive: true, mode: 0o700 }); @@ -878,28 +914,56 @@ async function recoverOrCleanupStaleExtractionPaths( const entries = await readdir(asideStagingDir, { withFileTypes: true }); const entryNames = new Set(entries.map((entry) => entry.name)); const paths = new Set(entries.map((entry) => join(asideStagingDir, entry.name))); - const restorable = entries + const recoveryRecords = entries .filter( (entry) => - entry.isDirectory() && + isExtractionRecoveryRecord(entry) && entry.name.startsWith(IN_PROGRESS_ASIDE_PREFIX) && !entryNames.has(`${RETIRED_ASIDE_PREFIX}${entry.name.slice(IN_PROGRESS_ASIDE_PREFIX.length)}`) ) - .map((entry) => ({ entry, timestamp: extractionAsideTimestamp(entry.name) })) + .map((entry) => ({ + entry, + priorStateAbsent: isPriorAbsentRecoveryRecord(entry), + timestamp: extractionAsideTimestamp(entry.name), + })) .filter(({ timestamp }) => Number.isFinite(timestamp)) .sort((left, right) => right.timestamp - left.timestamp); - if (restorable.length > 0) { - const restoredPath = join(asideStagingDir, restorable[0].entry.name); - await rollbackExtractedDirectory(application, asideStagingDir, restoredPath, paths, false); + const recoveryRecord = + recoveryRecords.find(({ priorStateAbsent }) => !priorStateAbsent) ?? + recoveryRecords.find(({ priorStateAbsent }) => priorStateAbsent); + if (recoveryRecord) { + const recoveryPath = join(asideStagingDir, recoveryRecord.entry.name); + await rollbackExtractedDirectory( + application, + asideStagingDir, + recoveryRecord.priorStateAbsent ? undefined : recoveryPath, + paths, + false + ); application.logger.warn( - `Recovered the previous ${application.name} component directory after an interrupted deploy` + - (restorable.length > 1 ? `; discarded ${restorable.length - 1} older recovery candidates` : '') + (recoveryRecord.priorStateAbsent + ? `Removed the partial ${application.name} component directory after an interrupted first deploy` + : `Recovered the previous ${application.name} component directory after an interrupted deploy`) + + (recoveryRecords.length > 1 ? `; discarded ${recoveryRecords.length - 1} older recovery candidates` : '') ); return; } await cleanupExtractionPaths(application, asideStagingDir, paths); } +function isPriorAbsentRecoveryRecord(entry: { isFile(): boolean; name: string }): boolean { + return entry.isFile() && entry.name.endsWith(PRIOR_ABSENT_RECORD_SUFFIX); +} + +function isExtractionRecoveryRecord(entry: { + isDirectory(): boolean; + isFile(): boolean; + isSymbolicLink(): boolean; + name: string; +}): boolean { + return entry.isDirectory() || entry.isSymbolicLink() || isPriorAbsentRecoveryRecord(entry); +} + function extractionAsideTimestamp(name: string): number { const timestampEnd = name.indexOf('-', IN_PROGRESS_ASIDE_PREFIX.length); if (timestampEnd < 0) return Number.NaN; @@ -999,7 +1063,7 @@ export async function retireComponentExtractionStaging( } const paths = new Set(entries.map((entry) => join(asideStagingDir, entry.name))); for (const entry of entries) { - if (!entry.isDirectory() || !entry.name.startsWith(IN_PROGRESS_ASIDE_PREFIX)) continue; + if (!isExtractionRecoveryRecord(entry) || !entry.name.startsWith(IN_PROGRESS_ASIDE_PREFIX)) continue; const markerPath = retiredMarkerForAside(join(asideStagingDir, entry.name)); try { await writeFile(markerPath, '', { flag: 'wx', mode: 0o600 }); @@ -1089,7 +1153,7 @@ async function rollbackExtractedDirectory( retainReplacement: boolean ): Promise { await ensureExtractionStagingDirectory(asideStagingDir); - const retryableRenameCodes = new Set(['EEXIST', 'ENOTEMPTY', 'EPERM', 'EACCES', 'EBUSY']); + const retryableRenameCodes = new Set(['EEXIST', 'ENOTEMPTY', 'ENOTDIR', 'EISDIR', 'EPERM', 'EACCES', 'EBUSY']); const displaceCurrentDirectory = async (): Promise => { const retryDeadline = Date.now() + 5000; let lastError: unknown; @@ -1111,10 +1175,11 @@ async function rollbackExtractedDirectory( }; if (asidePath) { + const asideIsSymbolicLink = (await lstat(asidePath)).isSymbolicLink(); let restoreError: unknown; let restoreRetryDeadline: number | undefined; let fallbackDisplacedPath: string | undefined; - let placeholderIdentity: { dev: bigint; ino: bigint } | undefined; + let placeholderIdentity = await identifyRollbackPlaceholder(application.dirPath); const failRestore = async (error: unknown): Promise => { try { await makeRollbackPlaceholderMovable(application.dirPath, placeholderIdentity); @@ -1212,7 +1277,11 @@ async function rollbackExtractedDirectory( restoreRetryDeadline ??= Date.now() + 5000; if (process.platform !== 'win32' && process.getuid?.() !== 0) { try { - await mkdir(application.dirPath, { mode: 0o000 }); + if (asideIsSymbolicLink) { + await writeFile(application.dirPath, '', { flag: 'wx', mode: 0o000 }); + } else { + await mkdir(application.dirPath, { mode: 0o000 }); + } const placeholder = await lstat(application.dirPath, { bigint: true }); placeholderIdentity = { dev: placeholder.dev, ino: placeholder.ino }; } catch (placeholderError) { diff --git a/components/componentLoader.ts b/components/componentLoader.ts index bfac78a25c..d8b81c3f4a 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -64,6 +64,7 @@ let loadedComponents = new Map(); let watchesSetup; let resources; let componentLoadGeneration = 0; +let componentLoadGenerationTail = Promise.resolve(); type ComponentReadyPromises = WeakMap>; export async function readyComponentModules( @@ -146,136 +147,147 @@ export async function loadComponentDirectories( loadedResources?: Resources, readyComponentPromises: ComponentReadyPromises = new WeakMap() ) { - const loadGeneration = ++componentLoadGeneration; - if (loadedResources) resources = loadedResources; - if (loadedPluginModules) loadedComponents = loadedPluginModules; - const cycleResources = resources; - const cycleLoadedComponents = loadedComponents; - let failedRecoveries = new Map(); + const previousGeneration = componentLoadGenerationTail; + let settleGeneration: () => void; + const generationSettled = new Promise((resolve) => (settleGeneration = resolve)); + componentLoadGenerationTail = previousGeneration.then(() => generationSettled); + await previousGeneration; + const deferredLoads: Promise[] = []; try { - failedRecoveries = await recoverInterruptedComponentExtractions(CF_ROUTES_DIR); - } catch (error) { - const recoveryError = error instanceof Error ? error : new Error(String(error)); - harperLogger.warn( - 'Loading existing filesystem components without deploy recovery because staging could not be inspected:', - errorForLog(recoveryError) - ); - } - // Materialize hdb_secret global-tier rows into process.env and snapshot the scoped tier before - // any application loads (root components — including the Pro custody registration — have - // already loaded by this point). Re-runs on each reload cycle, which is how changed/late-custody - // secrets heal. Never throws. - await materializeGlobalSecrets(); - const cfsLoaded: Promise[] = []; - const deferredRecoveries = new Map( - [...failedRecoveries].filter(([, error]) => error instanceof ComponentPreparationLockTimeoutError) - ); - const unreportedFailedRecoveries = new Map( - [...failedRecoveries].filter(([, error]) => !(error instanceof ComponentPreparationLockTimeoutError)) - ); - const deferComponentLoad = (appName: string) => { - const appFolder = join(CF_ROUTES_DIR, appName); - const appWasVisible = existsSync(appFolder); - if (appWasVisible) { - componentLifecycle.loading(appName, `Component '${appName}' is waiting for in-progress preparation to finish`); + const loadGeneration = ++componentLoadGeneration; + if (loadedResources) resources = loadedResources; + if (loadedPluginModules) loadedComponents = loadedPluginModules; + const cycleResources = resources; + const cycleLoadedComponents = loadedComponents; + let failedRecoveries = new Map(); + try { + failedRecoveries = await recoverInterruptedComponentExtractions(CF_ROUTES_DIR); + } catch (error) { + const recoveryError = error instanceof Error ? error : new Error(String(error)); + harperLogger.warn( + 'Loading existing filesystem components without deploy recovery because staging could not be inspected:', + errorForLog(recoveryError) + ); } - void recoverInterruptedComponentExtraction(CF_ROUTES_DIR, appName) - .then(async () => { - if (loadGeneration !== componentLoadGeneration) return; - if (!existsSync(appFolder)) { + // Materialize hdb_secret global-tier rows into process.env and snapshot the scoped tier before + // any application loads (root components — including the Pro custody registration — have + // already loaded by this point). Re-runs on each reload cycle, which is how changed/late-custody + // secrets heal. Never throws. + await materializeGlobalSecrets(); + const cfsLoaded: Promise[] = []; + const deferredRecoveries = new Map( + [...failedRecoveries].filter(([, error]) => error instanceof ComponentPreparationLockTimeoutError) + ); + const unreportedFailedRecoveries = new Map( + [...failedRecoveries].filter(([, error]) => !(error instanceof ComponentPreparationLockTimeoutError)) + ); + const deferComponentLoad = (appName: string) => { + const appFolder = join(CF_ROUTES_DIR, appName); + const appWasVisible = existsSync(appFolder); + if (appWasVisible) { + componentLifecycle.loading(appName, `Component '${appName}' is waiting for in-progress preparation to finish`); + } + const deferredLoad = recoverInterruptedComponentExtraction(CF_ROUTES_DIR, appName) + .then(async () => { + if (loadGeneration !== componentLoadGeneration) return; + if (!existsSync(appFolder)) { + if (appWasVisible) { + statusForComponent(appName).unknown('Component directory no longer exists after preparation settled'); + } + return; + } + const mountResult = tryRootConfigMount(appName); + if (!mountResult.ok) return; + const modulesBeforeLoad = new Set(cycleLoadedComponents.keys()); + await loadComponent(appFolder, cycleResources, HDB_ROOT_DIR_NAME, { + isRoot: false, + autoReload: false, + appName, + mount: mountResult.mount, + }); + if (loadGeneration !== componentLoadGeneration) return; + await readyComponentModules( + [...cycleLoadedComponents.keys()].filter((serverModule) => !modulesBeforeLoad.has(serverModule)), + readyComponentPromises + ); + }) + .catch((error) => { + const recoveryError = error instanceof Error ? error : new Error(String(error)); if (appWasVisible) { - statusForComponent(appName).unknown('Component directory no longer exists after preparation settled'); + componentLifecycle.failed( + appName, + recoveryError, + `Component '${appName}' failed to load after waiting for in-progress preparation` + ); } - return; - } - const mountResult = tryRootConfigMount(appName); - if (!mountResult.ok) return; - const modulesBeforeLoad = new Set(cycleLoadedComponents.keys()); - await loadComponent(appFolder, cycleResources, HDB_ROOT_DIR_NAME, { - isRoot: false, - autoReload: false, - appName, - mount: mountResult.mount, }); - if (loadGeneration !== componentLoadGeneration) return; - await readyComponentModules( - [...cycleLoadedComponents.keys()].filter((serverModule) => !modulesBeforeLoad.has(serverModule)), - readyComponentPromises - ); - }) - .catch((error) => { - const recoveryError = error instanceof Error ? error : new Error(String(error)); - if (appWasVisible) { + deferredLoads.push(deferredLoad); + }; + if (existsSync(CF_ROUTES_DIR)) { + const cfFolders = readdirSync(CF_ROUTES_DIR, { withFileTypes: true }); + for (const appEntry of cfFolders) { + if (!appEntry.isDirectory() && !appEntry.isSymbolicLink()) continue; + // Skip hidden entries: component names are never dot-prefixed, and this keeps + // Harper's own staging dirs (e.g. deploy aside copies) from loading as components. + if (appEntry.name.startsWith('.')) continue; + const appName = appEntry.name; + const recoveryError = failedRecoveries.get(appName); + if (recoveryError) { + if (recoveryError instanceof ComponentPreparationLockTimeoutError) { + deferredRecoveries.delete(appName); + deferComponentLoad(appName); + continue; + } + unreportedFailedRecoveries.delete(appName); componentLifecycle.failed( appName, recoveryError, - `Component '${appName}' failed to load after waiting for in-progress preparation` + `Component '${appName}' failed to load because its interrupted deployment could not be recovered` ); - } - }); - }; - if (existsSync(CF_ROUTES_DIR)) { - const cfFolders = readdirSync(CF_ROUTES_DIR, { withFileTypes: true }); - for (const appEntry of cfFolders) { - if (!appEntry.isDirectory() && !appEntry.isSymbolicLink()) continue; - // Skip hidden entries: component names are never dot-prefixed, and this keeps - // Harper's own staging dirs (e.g. deploy aside copies) from loading as components. - if (appEntry.name.startsWith('.')) continue; - const appName = appEntry.name; - const recoveryError = failedRecoveries.get(appName); - if (recoveryError) { - if (recoveryError instanceof ComponentPreparationLockTimeoutError) { - deferredRecoveries.delete(appName); - deferComponentLoad(appName); continue; } - unreportedFailedRecoveries.delete(appName); - componentLifecycle.failed( - appName, - recoveryError, - `Component '${appName}' failed to load because its interrupted deployment could not be recovered` + const appFolder = join(CF_ROUTES_DIR, appName); + const mountResult = tryRootConfigMount(appName); + if (!mountResult.ok) continue; + cfsLoaded.push( + loadComponent(appFolder, resources, HDB_ROOT_DIR_NAME, { + isRoot: false, + autoReload: false, + appName, + mount: mountResult.mount, + }) ); - continue; } - const appFolder = join(CF_ROUTES_DIR, appName); - const mountResult = tryRootConfigMount(appName); - if (!mountResult.ok) continue; - cfsLoaded.push( - loadComponent(appFolder, resources, HDB_ROOT_DIR_NAME, { - isRoot: false, - autoReload: false, - appName, - mount: mountResult.mount, - }) - ); } - } - for (const [appName, recoveryError] of unreportedFailedRecoveries) { - componentLifecycle.failed( - appName, - recoveryError, - `Component '${appName}' failed to load because its interrupted deployment could not be recovered` - ); - } - for (const appName of deferredRecoveries.keys()) deferComponentLoad(appName); - const hdbAppFolder = process.env.RUN_HDB_APP; - if (hdbAppFolder) { - if (getWorkerIndex() === 0) harperLogger.info?.('Loading application from ' + hdbAppFolder); - const mountResult = tryRootConfigMount(basename(hdbAppFolder)); - if (mountResult.ok) { - cfsLoaded.push( - loadComponent(hdbAppFolder, resources, hdbAppFolder, { - isRoot: false, - autoReload: Boolean(process.env.DEV_MODE), - appName: hdbAppFolder, - mount: mountResult.mount, - }) + for (const [appName, recoveryError] of unreportedFailedRecoveries) { + componentLifecycle.failed( + appName, + recoveryError, + `Component '${appName}' failed to load because its interrupted deployment could not be recovered` ); } + for (const appName of deferredRecoveries.keys()) deferComponentLoad(appName); + const hdbAppFolder = process.env.RUN_HDB_APP; + if (hdbAppFolder) { + if (getWorkerIndex() === 0) harperLogger.info?.('Loading application from ' + hdbAppFolder); + const mountResult = tryRootConfigMount(basename(hdbAppFolder)); + if (mountResult.ok) { + cfsLoaded.push( + loadComponent(hdbAppFolder, resources, hdbAppFolder, { + isRoot: false, + autoReload: Boolean(process.env.DEV_MODE), + appName: hdbAppFolder, + mount: mountResult.mount, + }) + ); + } + } + return await Promise.all(cfsLoaded).then(() => { + watchesSetup = true; + }); + } finally { + void Promise.allSettled(deferredLoads).then(() => settleGeneration()); } - return Promise.all(cfsLoaded).then(() => { - watchesSetup = true; - }); } export const TRUSTED_RESOURCE_PLUGINS: any = { diff --git a/unitTests/components/componentLoader.test.js b/unitTests/components/componentLoader.test.js index 0b958c0130..c394bc03c6 100644 --- a/unitTests/components/componentLoader.test.js +++ b/unitTests/components/componentLoader.test.js @@ -3,11 +3,14 @@ const sinon = require('sinon'); const path = require('path'); const { tmpdir } = require('os'); const { mkdtempSync, mkdirSync, writeFileSync, rmSync, existsSync } = require('fs'); +const fs = require('node:fs/promises'); +const { waitFor } = require('../waitFor.js'); describe('ComponentLoader Status Integration', function () { let componentStatusRegistry; let tempDir; let componentLoader; + let withComponentPreparationLock; let lifecycle; let sandbox; @@ -20,7 +23,7 @@ describe('ComponentLoader Status Integration', function () { // Mock environment to use our temp directory const env = require('#src/utility/environment/environmentManager'); sandbox.stub(env, 'get').callsFake((key) => { - if (key === 'COMPONENTSROOT') { + if (key === 'componentsRoot') { return tempDir; } // Return some default values for other config @@ -60,6 +63,7 @@ describe('ComponentLoader Status Integration', function () { // Load componentLoader after setting up spies componentLoader = require('#src/components/componentLoader'); + ({ withComponentPreparationLock } = require('#src/components/componentPreparationLock')); }); after(function () { @@ -402,6 +406,85 @@ describe('ComponentLoader Status Integration', function () { }); }); + it('serializes a superseding load generation through deferred readiness', async function () { + this.timeout(15000); + const appName = 'deferred-generation-probe'; + const pluginName = 'deferredGenerationProbe'; + const componentDir = path.join(tempDir, appName); + const asidePath = path.join(tempDir, '.deploy-aside', appName, '.in-progress-123-previous'); + await fs.mkdir(asidePath, { recursive: true }); + await fs.writeFile(path.join(asidePath, 'config.yaml'), `${pluginName}: {}\n`); + await fs.mkdir(componentDir, { recursive: true }); + await fs.writeFile(path.join(componentDir, 'partial'), 'partial'); + + let releasePreparation; + let preparationHeld; + const preparationStarted = new Promise((resolve) => (preparationHeld = resolve)); + const preparation = withComponentPreparationLock(componentDir, async () => { + preparationHeld(); + await new Promise((resolve) => (releasePreparation = resolve)); + }); + + let releaseStart; + let startEntered = false; + let readyCalls = 0; + componentLoader.TRUSTED_RESOURCE_PLUGINS[pluginName] = { + async start() { + startEntered = true; + await new Promise((resolve) => (releaseStart = resolve)); + return { ready: () => readyCalls++ }; + }, + }; + const loadedComponents = new Map(); + const resources = { isWorker: true, set() {} }; + + try { + await preparationStarted; + await componentLoader.loadComponentDirectories(loadedComponents, resources, new WeakMap()); + releasePreparation(); + await preparation; + try { + await waitFor(() => startEntered, { timeout: 5000, message: 'deferred component load did not start' }); + } catch (error) { + error.message += `: loading=${lifecycle.loading + .getCalls() + .map((call) => String(call.args[0])) + .join(',')} failed=${lifecycle.failed + .getCalls() + .map((call) => String(call.args[1])) + .join('; ')} live=${existsSync(componentDir)} staging=${existsSync(path.dirname(asidePath))} loaded=${[ + ...componentLoader.loadedPaths.keys(), + ].join(',')}`; + throw error; + } + + let supersedingLoadSettled = false; + const supersedingLoad = componentLoader + .loadComponentDirectories(loadedComponents, resources, new WeakMap()) + .then(() => (supersedingLoadSettled = true)); + await new Promise((resolve) => setImmediate(resolve)); + assert.strictEqual(supersedingLoadSettled, false); + + releaseStart(); + await supersedingLoad; + await waitFor(() => readyCalls === 1); + assert.strictEqual(readyCalls, 1); + } finally { + releasePreparation?.(); + releaseStart?.(); + await preparation; + delete componentLoader.TRUSTED_RESOURCE_PLUGINS[pluginName]; + componentLoader.loadedPaths.clear(); + await fs.rm(path.join(tempDir, '.deploy-aside', appName), { + recursive: true, + force: true, + maxRetries: 5, + retryDelay: 100, + }); + await fs.rm(componentDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + // A mounted application's deprecated `start`/`startOnMainThread` hooks receive the raw, // unmounted server (unlike the new Plugin API's `handleApplication(scope)`), so routes they // register would silently escape the mount. Loading must fail closed instead (review finding). diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index 9385e6085e..815907305b 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -5,6 +5,7 @@ const path = require('node:path'); const fs = require('node:fs/promises'); const os = require('node:os'); const { Readable } = require('node:stream'); +const { waitFor } = require('../waitFor.js'); const testUtils = require('../testUtils.js'); testUtils.preTestPrep(); @@ -93,6 +94,26 @@ describe('extractApplication directory swap', () => { } }); + it('removes a partial first deploy after an interrupted process restarts', async function () { + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-first-deploy-crash-')); + const dirPath = path.join(componentsRoot, 'web'); + const stagingDir = path.join(componentsRoot, '.deploy-aside', 'web'); + const absentMarker = path.join(stagingDir, '.in-progress-123-1-record-prior-absent'); + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'package.json'), '{"name":"web"'); + await fs.mkdir(stagingDir, { recursive: true }); + await fs.writeFile(absentMarker, ''); + + try { + const failures = await recoverInterruptedComponentExtractions(componentsRoot); + assert.strictEqual(failures.size, 0, failures.get('web')?.stack); + await assert.rejects(fs.access(dirPath)); + await assert.rejects(fs.access(stagingDir)); + } finally { + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + it('settles a deferred extraction transaction only once', async function () { const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-idempotent-')); const dirPath = path.join(componentsRoot, 'web'); @@ -223,6 +244,29 @@ describe('extractApplication directory swap', () => { } }); + it('recovers a symlinked predecessor after an interrupted deploy', async function () { + if (process.platform === 'win32') this.skip(); + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-symlink-recovery-')); + const previousTarget = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-symlink-target-')); + const dirPath = path.join(componentsRoot, 'web'); + const asidePath = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-123-previous'); + await fs.mkdir(path.dirname(asidePath), { recursive: true }); + await fs.symlink(previousTarget, asidePath, 'dir'); + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'package.json'), '{"name":"web","version":"2.0.0"}\n'); + + try { + const failures = await recoverInterruptedComponentExtractions(componentsRoot); + assert.strictEqual(failures.size, 0, failures.get('web')?.stack); + assert.strictEqual((await fs.lstat(dirPath)).isSymbolicLink(), true); + assert.strictEqual(await fs.readlink(dirPath), previousTarget); + await assert.rejects(fs.access(path.join(componentsRoot, '.deploy-aside', 'web'))); + } finally { + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + await fs.rm(previousTarget, { recursive: true, force: true }); + } + }); + it('defers bulk recovery, then waits for active preparation before recovering the component', async function () { const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-startup-race-')); const dirPath = path.join(componentsRoot, 'web'); @@ -506,6 +550,29 @@ describe('extractApplication directory swap', () => { } }); + it('adopts an owner-owned rollback placeholder during interrupted recovery', async function () { + if (process.platform === 'win32' || process.getuid?.() === 0) this.skip(); + this.timeout(15000); + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-placeholder-recovery-')); + const dirPath = path.join(componentsRoot, 'web'); + const asidePath = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-123-previous'); + await fs.mkdir(asidePath, { recursive: true }); + await fs.writeFile(path.join(asidePath, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + await fs.mkdir(dirPath); + await fs.writeFile(path.join(dirPath, 'writer-created'), 'occupied'); + await fs.chmod(dirPath, 0o000); + + try { + const failures = await recoverInterruptedComponentExtractions(componentsRoot); + assert.strictEqual(failures.size, 0); + assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); + await assert.rejects(fs.access(path.join(componentsRoot, '.deploy-aside', 'web'))); + } finally { + await fs.chmod(dirPath, 0o700).catch(() => {}); + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + it('reclaims a stale aside copy left by an earlier deploy', async function () { this.timeout(20000); @@ -563,16 +630,35 @@ describe('extractApplication directory swap', () => { 'wrapper/package.json': '{"name":"web","version":"2.0.0"}\n', 'wrapper/index.js': 'module.exports = () => 2;\n', }); - const app = new Application({ - name: 'web', - payload: await packageDirectory(sourceDir, { skip_node_modules: true }), - }); + const archive = await packageDirectory(sourceDir, { skip_node_modules: true }); + let finishPayload; + const payload = Readable.from( + (async function* () { + yield archive; + await new Promise((resolve) => (finishPayload = resolve)); + })() + ); + const app = new Application({ name: 'web', payload }); app.dirPath = dirPath; - await extractApplication(app); + const extraction = extractApplication(app); + await waitFor(() => + fs.access(path.join(dirPath, 'wrapper', 'package.json')).then( + () => true, + () => false + ) + ); + const liveDirectoryIdentity = await fs.lstat(dirPath, { bigint: true }); + finishPayload(); + await extraction; assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '2.0.0'); assert.strictEqual(await fs.readFile(path.join(dirPath, 'index.js'), 'utf8'), 'module.exports = () => 2;\n'); + if (process.platform === 'win32') { + const normalizedDirectoryIdentity = await fs.lstat(dirPath, { bigint: true }); + assert.strictEqual(normalizedDirectoryIdentity.dev, liveDirectoryIdentity.dev); + assert.strictEqual(normalizedDirectoryIdentity.ino, liveDirectoryIdentity.ino); + } assert.deepStrictEqual( (await fs.readdir(componentsRoot)).filter((entry) => !entry.startsWith('.')), ['web'], @@ -599,6 +685,7 @@ describe('extractApplication directory swap', () => { assert.strictEqual(app.isNewComponent, true, 'defaults to true before extraction runs'); await extractApplication(app); assert.strictEqual(app.isNewComponent, true, 'no prior directory existed, so this is a new component'); + await assert.rejects(fs.access(path.join(componentsRoot, '.deploy-aside', 'web'))); await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); await fs.rm(sourceDir, { recursive: true, force: true }); From 44d91ed2c1f2880929bedf15f44f23fb46d1e6a6 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 20:16:31 -0600 Subject: [PATCH 58/94] Resolve component recovery review feedback Co-Authored-By: GPT-5 Codex --- components/Application.ts | 124 ++++++++++++++++--- components/componentLoader.ts | 115 +++++++++-------- unitTests/components/componentLoader.test.js | 20 ++- 3 files changed, 188 insertions(+), 71 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index ac9f743d8c..5262c8f9d5 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -35,7 +35,7 @@ import { import { spawn, type ChildProcess } from 'node:child_process'; import { tmpdir } from 'node:os'; import { randomUUID } from 'node:crypto'; -import { createReadStream, existsSync } from 'node:fs'; +import { chmodSync, createReadStream, existsSync, lstatSync, renameSync } from 'node:fs'; import { Readable } from 'node:stream'; import { pipeline } from 'node:stream/promises'; import { StringDecoder } from 'node:string_decoder'; @@ -879,10 +879,11 @@ async function identifyRollbackPlaceholder( if (process.platform === 'win32' || userId === undefined || userId === 0) return undefined; try { const current = await lstat(applicationDirPath, { bigint: true }); + const permissions = Number(current.mode) & 0o777; if ( (current.isDirectory() || current.isFile()) && current.uid === BigInt(userId) && - (Number(current.mode) & 0o777) === 0 + (permissions === 0 || (current.isDirectory() && permissions === 0o100)) ) { return { dev: current.dev, ino: current.ino }; } @@ -1184,13 +1185,33 @@ async function rollbackExtractedDirectory( try { await makeRollbackPlaceholderMovable(application.dirPath, placeholderIdentity); if (retainReplacement && fallbackDisplacedPath) { - await rm(application.dirPath, { - recursive: true, - force: true, - maxRetries: 3, - retryDelay: 100, - }); - await rename(fallbackDisplacedPath, application.dirPath); + const fallbackRetryDeadline = Date.now() + 5000; + let fallbackRestoreError: unknown; + do { + try { + const writerDisplacedPath = await displaceCurrentDirectory(); + placeholderIdentity = undefined; + if (writerDisplacedPath) { + await rm(writerDisplacedPath, { + recursive: true, + force: true, + maxRetries: 3, + retryDelay: 100, + }); + transactionPaths.delete(writerDisplacedPath); + } + await rename(fallbackDisplacedPath, application.dirPath); + fallbackRestoreError = undefined; + break; + } catch (restoreFallbackError) { + fallbackRestoreError = restoreFallbackError; + if (!retryableRenameCodes.has((restoreFallbackError as NodeJS.ErrnoException).code ?? '')) { + throw restoreFallbackError; + } + await delay(10); + } + } while (Date.now() < fallbackRetryDeadline); + if (fallbackRestoreError) throw fallbackRestoreError; transactionPaths.delete(fallbackDisplacedPath); transactionPaths.add(await retireExtractionAside(asidePath)); } @@ -1226,8 +1247,18 @@ async function rollbackExtractedDirectory( }; do { try { - await makeRollbackPlaceholderMovable(application.dirPath, placeholderIdentity); - await rename(asidePath, application.dirPath); + if (placeholderIdentity) { + const current = lstatSync(application.dirPath, { bigint: true }); + if (current.dev === placeholderIdentity.dev && current.ino === placeholderIdentity.ino) { + chmodSync(application.dirPath, 0o700); + renameSync(asidePath, application.dirPath); + } else { + placeholderIdentity = undefined; + await rename(asidePath, application.dirPath); + } + } else { + await rename(asidePath, application.dirPath); + } transactionPaths.delete(asidePath); await cleanupExtractionPaths(application, asideStagingDir, transactionPaths); return; @@ -1273,21 +1304,80 @@ async function rollbackExtractedDirectory( ) ); } - fallbackDisplacedPath ??= displacedPath; + if (displacedPath) { + if (fallbackDisplacedPath) { + try { + await rm(displacedPath, { + recursive: true, + force: true, + maxRetries: 3, + retryDelay: 100, + }); + transactionPaths.delete(displacedPath); + } catch (cleanupError) { + return failRestore( + new AggregateError( + [error, cleanupError], + `Failed to discard a displaced ${application.name} writer directory during rollback` + ) + ); + } + } else { + fallbackDisplacedPath = displacedPath; + } + } restoreRetryDeadline ??= Date.now() + 5000; if (process.platform !== 'win32' && process.getuid?.() !== 0) { + const stagedPlaceholderPath = join( + asideStagingDir, + `.rollback-placeholder-${process.pid}-${Date.now()}-${randomUUID()}` + ); try { if (asideIsSymbolicLink) { - await writeFile(application.dirPath, '', { flag: 'wx', mode: 0o000 }); + await writeFile(stagedPlaceholderPath, '', { flag: 'wx', mode: 0o000 }); } else { - await mkdir(application.dirPath, { mode: 0o000 }); + await mkdir(stagedPlaceholderPath, { mode: 0o300 }); + } + transactionPaths.add(stagedPlaceholderPath); + let placeholderPlacementError: unknown; + do { + try { + renameSync(stagedPlaceholderPath, application.dirPath); + if (!asideIsSymbolicLink) chmodSync(application.dirPath, 0o100); + transactionPaths.delete(stagedPlaceholderPath); + break; + } catch (placeholderError) { + placeholderPlacementError = placeholderError; + if (!retryableRenameCodes.has((placeholderError as NodeJS.ErrnoException).code ?? '')) { + throw placeholderError; + } + const writerDisplacedPath = await displaceCurrentDirectory(); + if (writerDisplacedPath) { + await rm(writerDisplacedPath, { + recursive: true, + force: true, + maxRetries: 3, + retryDelay: 100, + }); + transactionPaths.delete(writerDisplacedPath); + } + } + } while (Date.now() < restoreRetryDeadline); + if (transactionPaths.has(stagedPlaceholderPath)) { + throw new Error( + `Failed to place the ${application.name} rollback placeholder before the deadline: ${errorMessage(placeholderPlacementError)}`, + { cause: placeholderPlacementError } + ); } const placeholder = await lstat(application.dirPath, { bigint: true }); placeholderIdentity = { dev: placeholder.dev, ino: placeholder.ino }; } catch (placeholderError) { - if ((placeholderError as NodeJS.ErrnoException).code !== 'EEXIST') { - return failRestore(placeholderError); - } + return failRestore( + new AggregateError( + [error, placeholderError], + `Failed to block a live ${application.name} writer during rollback` + ) + ); } } await delay(10); diff --git a/components/componentLoader.ts b/components/componentLoader.ts index d8b81c3f4a..64d6dc4225 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -63,10 +63,23 @@ const CF_ROUTES_DIR = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); let loadedComponents = new Map(); let watchesSetup; let resources; -let componentLoadGeneration = 0; -let componentLoadGenerationTail = Promise.resolve(); +const componentLoadTails = new Map>(); type ComponentReadyPromises = WeakMap>; +function serializeComponentLoad(appName: string, load: () => Promise): Promise { + const previousLoad = componentLoadTails.get(appName); + const currentLoad = previousLoad ? previousLoad.then(load) : load(); + const loadTail = currentLoad.then( + () => undefined, + () => undefined + ); + componentLoadTails.set(appName, loadTail); + void loadTail.then(() => { + if (componentLoadTails.get(appName) === loadTail) componentLoadTails.delete(appName); + }); + return currentLoad; +} + export async function readyComponentModules( serverModules: Iterable, readyComponentPromises: ComponentReadyPromises = new WeakMap() @@ -147,14 +160,8 @@ export async function loadComponentDirectories( loadedResources?: Resources, readyComponentPromises: ComponentReadyPromises = new WeakMap() ) { - const previousGeneration = componentLoadGenerationTail; - let settleGeneration: () => void; - const generationSettled = new Promise((resolve) => (settleGeneration = resolve)); - componentLoadGenerationTail = previousGeneration.then(() => generationSettled); - await previousGeneration; const deferredLoads: Promise[] = []; try { - const loadGeneration = ++componentLoadGeneration; if (loadedResources) resources = loadedResources; if (loadedPluginModules) loadedComponents = loadedPluginModules; const cycleResources = resources; @@ -187,40 +194,40 @@ export async function loadComponentDirectories( if (appWasVisible) { componentLifecycle.loading(appName, `Component '${appName}' is waiting for in-progress preparation to finish`); } - const deferredLoad = recoverInterruptedComponentExtraction(CF_ROUTES_DIR, appName) - .then(async () => { - if (loadGeneration !== componentLoadGeneration) return; - if (!existsSync(appFolder)) { - if (appWasVisible) { - statusForComponent(appName).unknown('Component directory no longer exists after preparation settled'); + const deferredLoad = serializeComponentLoad(appName, () => + recoverInterruptedComponentExtraction(CF_ROUTES_DIR, appName) + .then(async () => { + if (!existsSync(appFolder)) { + if (appWasVisible) { + statusForComponent(appName).unknown('Component directory no longer exists after preparation settled'); + } + return; } - return; - } - const mountResult = tryRootConfigMount(appName); - if (!mountResult.ok) return; - const modulesBeforeLoad = new Set(cycleLoadedComponents.keys()); - await loadComponent(appFolder, cycleResources, HDB_ROOT_DIR_NAME, { - isRoot: false, - autoReload: false, - appName, - mount: mountResult.mount, - }); - if (loadGeneration !== componentLoadGeneration) return; - await readyComponentModules( - [...cycleLoadedComponents.keys()].filter((serverModule) => !modulesBeforeLoad.has(serverModule)), - readyComponentPromises - ); - }) - .catch((error) => { - const recoveryError = error instanceof Error ? error : new Error(String(error)); - if (appWasVisible) { - componentLifecycle.failed( + const mountResult = tryRootConfigMount(appName); + if (!mountResult.ok) return; + const modulesBeforeLoad = new Set(cycleLoadedComponents.keys()); + await loadComponent(appFolder, cycleResources, HDB_ROOT_DIR_NAME, { + isRoot: false, + autoReload: false, appName, - recoveryError, - `Component '${appName}' failed to load after waiting for in-progress preparation` + mount: mountResult.mount, + }); + await readyComponentModules( + [...cycleLoadedComponents.keys()].filter((serverModule) => !modulesBeforeLoad.has(serverModule)), + readyComponentPromises ); - } - }); + }) + .catch((error) => { + const recoveryError = error instanceof Error ? error : new Error(String(error)); + if (appWasVisible) { + componentLifecycle.failed( + appName, + recoveryError, + `Component '${appName}' failed to load after waiting for in-progress preparation` + ); + } + }) + ); deferredLoads.push(deferredLoad); }; if (existsSync(CF_ROUTES_DIR)) { @@ -250,12 +257,14 @@ export async function loadComponentDirectories( const mountResult = tryRootConfigMount(appName); if (!mountResult.ok) continue; cfsLoaded.push( - loadComponent(appFolder, resources, HDB_ROOT_DIR_NAME, { - isRoot: false, - autoReload: false, - appName, - mount: mountResult.mount, - }) + serializeComponentLoad(appName, () => + loadComponent(appFolder, cycleResources, HDB_ROOT_DIR_NAME, { + isRoot: false, + autoReload: false, + appName, + mount: mountResult.mount, + }) + ) ); } } @@ -273,12 +282,14 @@ export async function loadComponentDirectories( const mountResult = tryRootConfigMount(basename(hdbAppFolder)); if (mountResult.ok) { cfsLoaded.push( - loadComponent(hdbAppFolder, resources, hdbAppFolder, { - isRoot: false, - autoReload: Boolean(process.env.DEV_MODE), - appName: hdbAppFolder, - mount: mountResult.mount, - }) + serializeComponentLoad(hdbAppFolder, () => + loadComponent(hdbAppFolder, cycleResources, hdbAppFolder, { + isRoot: false, + autoReload: Boolean(process.env.DEV_MODE), + appName: hdbAppFolder, + mount: mountResult.mount, + }) + ) ); } } @@ -286,7 +297,7 @@ export async function loadComponentDirectories( watchesSetup = true; }); } finally { - void Promise.allSettled(deferredLoads).then(() => settleGeneration()); + void Promise.allSettled(deferredLoads); } } diff --git a/unitTests/components/componentLoader.test.js b/unitTests/components/componentLoader.test.js index c394bc03c6..cc1ce12d22 100644 --- a/unitTests/components/componentLoader.test.js +++ b/unitTests/components/componentLoader.test.js @@ -406,11 +406,14 @@ describe('ComponentLoader Status Integration', function () { }); }); - it('serializes a superseding load generation through deferred readiness', async function () { + it('serializes deferred readiness per component without blocking unrelated loads', async function () { this.timeout(15000); const appName = 'deferred-generation-probe'; const pluginName = 'deferredGenerationProbe'; + const independentAppName = 'independent-generation-probe'; + const independentPluginName = 'independentGenerationProbe'; const componentDir = path.join(tempDir, appName); + const independentComponentDir = path.join(tempDir, independentAppName); const asidePath = path.join(tempDir, '.deploy-aside', appName, '.in-progress-123-previous'); await fs.mkdir(asidePath, { recursive: true }); await fs.writeFile(path.join(asidePath, 'config.yaml'), `${pluginName}: {}\n`); @@ -428,6 +431,7 @@ describe('ComponentLoader Status Integration', function () { let releaseStart; let startEntered = false; let readyCalls = 0; + let independentStartCalls = 0; componentLoader.TRUSTED_RESOURCE_PLUGINS[pluginName] = { async start() { startEntered = true; @@ -457,12 +461,22 @@ describe('ComponentLoader Status Integration', function () { ].join(',')}`; throw error; } + componentLoader.TRUSTED_RESOURCE_PLUGINS[independentPluginName] = { + start() { + independentStartCalls++; + }, + }; + await fs.mkdir(independentComponentDir); + await fs.writeFile(path.join(independentComponentDir, 'config.yaml'), `${independentPluginName}: {}\n`); let supersedingLoadSettled = false; const supersedingLoad = componentLoader .loadComponentDirectories(loadedComponents, resources, new WeakMap()) .then(() => (supersedingLoadSettled = true)); - await new Promise((resolve) => setImmediate(resolve)); + await waitFor(() => independentStartCalls === 1, { + timeout: 5000, + message: 'unrelated component load was blocked by deferred readiness', + }); assert.strictEqual(supersedingLoadSettled, false); releaseStart(); @@ -474,6 +488,7 @@ describe('ComponentLoader Status Integration', function () { releaseStart?.(); await preparation; delete componentLoader.TRUSTED_RESOURCE_PLUGINS[pluginName]; + delete componentLoader.TRUSTED_RESOURCE_PLUGINS[independentPluginName]; componentLoader.loadedPaths.clear(); await fs.rm(path.join(tempDir, '.deploy-aside', appName), { recursive: true, @@ -482,6 +497,7 @@ describe('ComponentLoader Status Integration', function () { retryDelay: 100, }); await fs.rm(componentDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + await fs.rm(independentComponentDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); } }); From 807352e0195476b682779e5ae5912607ee32c513 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 21:58:25 -0600 Subject: [PATCH 59/94] Close live-writer rollback race Co-Authored-By: GPT-5 Codex --- components/Application.ts | 81 +++--- components/componentLoader.ts | 242 +++++++++--------- .../components/extractApplicationSwap.test.js | 2 +- 3 files changed, 171 insertions(+), 154 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 5262c8f9d5..2a305f6a0e 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -1155,6 +1155,17 @@ async function rollbackExtractedDirectory( ): Promise { await ensureExtractionStagingDirectory(asideStagingDir); const retryableRenameCodes = new Set(['EEXIST', 'ENOTEMPTY', 'ENOTDIR', 'EISDIR', 'EPERM', 'EACCES', 'EBUSY']); + const displaceCurrentDirectorySync = (): string | undefined => { + const displacedPath = join(asideStagingDir, `.failed-${process.pid}-${Date.now()}-${randomUUID()}`); + try { + renameSync(application.dirPath, displacedPath); + transactionPaths.add(displacedPath); + return displacedPath; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } + }; const displaceCurrentDirectory = async (): Promise => { const retryDeadline = Date.now() + 5000; let lastError: unknown; @@ -1188,27 +1199,34 @@ async function rollbackExtractedDirectory( const fallbackRetryDeadline = Date.now() + 5000; let fallbackRestoreError: unknown; do { + let writerDisplacedPath: string | undefined; try { - const writerDisplacedPath = await displaceCurrentDirectory(); + writerDisplacedPath = displaceCurrentDirectorySync(); placeholderIdentity = undefined; - if (writerDisplacedPath) { - await rm(writerDisplacedPath, { - recursive: true, - force: true, - maxRetries: 3, - retryDelay: 100, - }); - transactionPaths.delete(writerDisplacedPath); - } - await rename(fallbackDisplacedPath, application.dirPath); + renameSync(fallbackDisplacedPath, application.dirPath); fallbackRestoreError = undefined; - break; } catch (restoreFallbackError) { fallbackRestoreError = restoreFallbackError; - if (!retryableRenameCodes.has((restoreFallbackError as NodeJS.ErrnoException).code ?? '')) { - throw restoreFallbackError; - } + } + if (writerDisplacedPath) { + await rm(writerDisplacedPath, { + recursive: true, + force: true, + maxRetries: 3, + retryDelay: 100, + }); + transactionPaths.delete(writerDisplacedPath); + } + if ( + fallbackRestoreError && + !retryableRenameCodes.has((fallbackRestoreError as NodeJS.ErrnoException).code ?? '') + ) { + throw fallbackRestoreError; + } + if (fallbackRestoreError) { await delay(10); + } else { + break; } } while (Date.now() < fallbackRetryDeadline); if (fallbackRestoreError) throw fallbackRestoreError; @@ -1341,27 +1359,32 @@ async function rollbackExtractedDirectory( transactionPaths.add(stagedPlaceholderPath); let placeholderPlacementError: unknown; do { + let writerDisplacedPath: string | undefined; try { + writerDisplacedPath = displaceCurrentDirectorySync(); renameSync(stagedPlaceholderPath, application.dirPath); if (!asideIsSymbolicLink) chmodSync(application.dirPath, 0o100); transactionPaths.delete(stagedPlaceholderPath); - break; + placeholderPlacementError = undefined; } catch (placeholderError) { placeholderPlacementError = placeholderError; - if (!retryableRenameCodes.has((placeholderError as NodeJS.ErrnoException).code ?? '')) { - throw placeholderError; - } - const writerDisplacedPath = await displaceCurrentDirectory(); - if (writerDisplacedPath) { - await rm(writerDisplacedPath, { - recursive: true, - force: true, - maxRetries: 3, - retryDelay: 100, - }); - transactionPaths.delete(writerDisplacedPath); - } } + if (writerDisplacedPath) { + await rm(writerDisplacedPath, { + recursive: true, + force: true, + maxRetries: 3, + retryDelay: 100, + }); + transactionPaths.delete(writerDisplacedPath); + } + if ( + placeholderPlacementError && + !retryableRenameCodes.has((placeholderPlacementError as NodeJS.ErrnoException).code ?? '') + ) { + throw placeholderPlacementError; + } + if (!placeholderPlacementError) break; } while (Date.now() < restoreRetryDeadline); if (transactionPaths.has(stagedPlaceholderPath)) { throw new Error( diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 64d6dc4225..fdd5bbdabd 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -160,145 +160,139 @@ export async function loadComponentDirectories( loadedResources?: Resources, readyComponentPromises: ComponentReadyPromises = new WeakMap() ) { - const deferredLoads: Promise[] = []; + if (loadedResources) resources = loadedResources; + if (loadedPluginModules) loadedComponents = loadedPluginModules; + const cycleResources = resources; + const cycleLoadedComponents = loadedComponents; + let failedRecoveries = new Map(); try { - if (loadedResources) resources = loadedResources; - if (loadedPluginModules) loadedComponents = loadedPluginModules; - const cycleResources = resources; - const cycleLoadedComponents = loadedComponents; - let failedRecoveries = new Map(); - try { - failedRecoveries = await recoverInterruptedComponentExtractions(CF_ROUTES_DIR); - } catch (error) { - const recoveryError = error instanceof Error ? error : new Error(String(error)); - harperLogger.warn( - 'Loading existing filesystem components without deploy recovery because staging could not be inspected:', - errorForLog(recoveryError) - ); - } - // Materialize hdb_secret global-tier rows into process.env and snapshot the scoped tier before - // any application loads (root components — including the Pro custody registration — have - // already loaded by this point). Re-runs on each reload cycle, which is how changed/late-custody - // secrets heal. Never throws. - await materializeGlobalSecrets(); - const cfsLoaded: Promise[] = []; - const deferredRecoveries = new Map( - [...failedRecoveries].filter(([, error]) => error instanceof ComponentPreparationLockTimeoutError) - ); - const unreportedFailedRecoveries = new Map( - [...failedRecoveries].filter(([, error]) => !(error instanceof ComponentPreparationLockTimeoutError)) + failedRecoveries = await recoverInterruptedComponentExtractions(CF_ROUTES_DIR); + } catch (error) { + const recoveryError = error instanceof Error ? error : new Error(String(error)); + harperLogger.warn( + 'Loading existing filesystem components without deploy recovery because staging could not be inspected:', + errorForLog(recoveryError) ); - const deferComponentLoad = (appName: string) => { - const appFolder = join(CF_ROUTES_DIR, appName); - const appWasVisible = existsSync(appFolder); - if (appWasVisible) { - componentLifecycle.loading(appName, `Component '${appName}' is waiting for in-progress preparation to finish`); - } - const deferredLoad = serializeComponentLoad(appName, () => - recoverInterruptedComponentExtraction(CF_ROUTES_DIR, appName) - .then(async () => { - if (!existsSync(appFolder)) { - if (appWasVisible) { - statusForComponent(appName).unknown('Component directory no longer exists after preparation settled'); - } - return; - } - const mountResult = tryRootConfigMount(appName); - if (!mountResult.ok) return; - const modulesBeforeLoad = new Set(cycleLoadedComponents.keys()); - await loadComponent(appFolder, cycleResources, HDB_ROOT_DIR_NAME, { - isRoot: false, - autoReload: false, - appName, - mount: mountResult.mount, - }); - await readyComponentModules( - [...cycleLoadedComponents.keys()].filter((serverModule) => !modulesBeforeLoad.has(serverModule)), - readyComponentPromises - ); - }) - .catch((error) => { - const recoveryError = error instanceof Error ? error : new Error(String(error)); + } + // Materialize hdb_secret global-tier rows into process.env and snapshot the scoped tier before + // any application loads (root components — including the Pro custody registration — have + // already loaded by this point). Re-runs on each reload cycle, which is how changed/late-custody + // secrets heal. Never throws. + await materializeGlobalSecrets(); + const cfsLoaded: Promise[] = []; + const deferredRecoveries = new Map( + [...failedRecoveries].filter(([, error]) => error instanceof ComponentPreparationLockTimeoutError) + ); + const unreportedFailedRecoveries = new Map( + [...failedRecoveries].filter(([, error]) => !(error instanceof ComponentPreparationLockTimeoutError)) + ); + const deferComponentLoad = (appName: string) => { + const appFolder = join(CF_ROUTES_DIR, appName); + const appWasVisible = existsSync(appFolder); + if (appWasVisible) { + componentLifecycle.loading(appName, `Component '${appName}' is waiting for in-progress preparation to finish`); + } + void serializeComponentLoad(appName, () => + recoverInterruptedComponentExtraction(CF_ROUTES_DIR, appName) + .then(async () => { + if (!existsSync(appFolder)) { if (appWasVisible) { - componentLifecycle.failed( - appName, - recoveryError, - `Component '${appName}' failed to load after waiting for in-progress preparation` - ); + statusForComponent(appName).unknown('Component directory no longer exists after preparation settled'); } - }) - ); - deferredLoads.push(deferredLoad); - }; - if (existsSync(CF_ROUTES_DIR)) { - const cfFolders = readdirSync(CF_ROUTES_DIR, { withFileTypes: true }); - for (const appEntry of cfFolders) { - if (!appEntry.isDirectory() && !appEntry.isSymbolicLink()) continue; - // Skip hidden entries: component names are never dot-prefixed, and this keeps - // Harper's own staging dirs (e.g. deploy aside copies) from loading as components. - if (appEntry.name.startsWith('.')) continue; - const appName = appEntry.name; - const recoveryError = failedRecoveries.get(appName); - if (recoveryError) { - if (recoveryError instanceof ComponentPreparationLockTimeoutError) { - deferredRecoveries.delete(appName); - deferComponentLoad(appName); - continue; + return; } - unreportedFailedRecoveries.delete(appName); - componentLifecycle.failed( + const mountResult = tryRootConfigMount(appName); + if (!mountResult.ok) return; + const modulesBeforeLoad = new Set(cycleLoadedComponents.keys()); + await loadComponent(appFolder, cycleResources, HDB_ROOT_DIR_NAME, { + isRoot: false, + autoReload: false, appName, - recoveryError, - `Component '${appName}' failed to load because its interrupted deployment could not be recovered` + mount: mountResult.mount, + }); + await readyComponentModules( + [...cycleLoadedComponents.keys()].filter((serverModule) => !modulesBeforeLoad.has(serverModule)), + readyComponentPromises ); + }) + .catch((error) => { + const recoveryError = error instanceof Error ? error : new Error(String(error)); + if (appWasVisible) { + componentLifecycle.failed( + appName, + recoveryError, + `Component '${appName}' failed to load after waiting for in-progress preparation` + ); + } + }) + ); + }; + if (existsSync(CF_ROUTES_DIR)) { + const cfFolders = readdirSync(CF_ROUTES_DIR, { withFileTypes: true }); + for (const appEntry of cfFolders) { + if (!appEntry.isDirectory() && !appEntry.isSymbolicLink()) continue; + // Skip hidden entries: component names are never dot-prefixed, and this keeps + // Harper's own staging dirs (e.g. deploy aside copies) from loading as components. + if (appEntry.name.startsWith('.')) continue; + const appName = appEntry.name; + const recoveryError = failedRecoveries.get(appName); + if (recoveryError) { + if (recoveryError instanceof ComponentPreparationLockTimeoutError) { + deferredRecoveries.delete(appName); + deferComponentLoad(appName); continue; } - const appFolder = join(CF_ROUTES_DIR, appName); - const mountResult = tryRootConfigMount(appName); - if (!mountResult.ok) continue; - cfsLoaded.push( - serializeComponentLoad(appName, () => - loadComponent(appFolder, cycleResources, HDB_ROOT_DIR_NAME, { - isRoot: false, - autoReload: false, - appName, - mount: mountResult.mount, - }) - ) + unreportedFailedRecoveries.delete(appName); + componentLifecycle.failed( + appName, + recoveryError, + `Component '${appName}' failed to load because its interrupted deployment could not be recovered` ); + continue; } - } - for (const [appName, recoveryError] of unreportedFailedRecoveries) { - componentLifecycle.failed( - appName, - recoveryError, - `Component '${appName}' failed to load because its interrupted deployment could not be recovered` + const appFolder = join(CF_ROUTES_DIR, appName); + const mountResult = tryRootConfigMount(appName); + if (!mountResult.ok) continue; + cfsLoaded.push( + serializeComponentLoad(appName, () => + loadComponent(appFolder, cycleResources, HDB_ROOT_DIR_NAME, { + isRoot: false, + autoReload: false, + appName, + mount: mountResult.mount, + }) + ) ); } - for (const appName of deferredRecoveries.keys()) deferComponentLoad(appName); - const hdbAppFolder = process.env.RUN_HDB_APP; - if (hdbAppFolder) { - if (getWorkerIndex() === 0) harperLogger.info?.('Loading application from ' + hdbAppFolder); - const mountResult = tryRootConfigMount(basename(hdbAppFolder)); - if (mountResult.ok) { - cfsLoaded.push( - serializeComponentLoad(hdbAppFolder, () => - loadComponent(hdbAppFolder, cycleResources, hdbAppFolder, { - isRoot: false, - autoReload: Boolean(process.env.DEV_MODE), - appName: hdbAppFolder, - mount: mountResult.mount, - }) - ) - ); - } + } + for (const [appName, recoveryError] of unreportedFailedRecoveries) { + componentLifecycle.failed( + appName, + recoveryError, + `Component '${appName}' failed to load because its interrupted deployment could not be recovered` + ); + } + for (const appName of deferredRecoveries.keys()) deferComponentLoad(appName); + const hdbAppFolder = process.env.RUN_HDB_APP; + if (hdbAppFolder) { + if (getWorkerIndex() === 0) harperLogger.info?.('Loading application from ' + hdbAppFolder); + const mountResult = tryRootConfigMount(basename(hdbAppFolder)); + if (mountResult.ok) { + cfsLoaded.push( + serializeComponentLoad(hdbAppFolder, () => + loadComponent(hdbAppFolder, cycleResources, hdbAppFolder, { + isRoot: false, + autoReload: Boolean(process.env.DEV_MODE), + appName: hdbAppFolder, + mount: mountResult.mount, + }) + ) + ); } - return await Promise.all(cfsLoaded).then(() => { - watchesSetup = true; - }); - } finally { - void Promise.allSettled(deferredLoads); } + return await Promise.all(cfsLoaded).then(() => { + watchesSetup = true; + }); } export const TRUSTED_RESOURCE_PLUGINS: any = { diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index 815907305b..316b85bdb1 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -560,7 +560,7 @@ describe('extractApplication directory swap', () => { await fs.writeFile(path.join(asidePath, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); await fs.mkdir(dirPath); await fs.writeFile(path.join(dirPath, 'writer-created'), 'occupied'); - await fs.chmod(dirPath, 0o000); + await fs.chmod(dirPath, 0o100); try { const failures = await recoverInterruptedComponentExtractions(componentsRoot); From 8c706772890e39498134951bd592a490b0c0060a Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 22:03:57 -0600 Subject: [PATCH 60/94] Preserve placeholder on mode failure Co-Authored-By: GPT-5 Codex --- components/Application.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/components/Application.ts b/components/Application.ts index 2a305f6a0e..61d2012c83 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -1363,7 +1363,14 @@ async function rollbackExtractedDirectory( try { writerDisplacedPath = displaceCurrentDirectorySync(); renameSync(stagedPlaceholderPath, application.dirPath); - if (!asideIsSymbolicLink) chmodSync(application.dirPath, 0o100); + if (!asideIsSymbolicLink) { + try { + chmodSync(application.dirPath, 0o100); + } catch (chmodError) { + renameSync(application.dirPath, stagedPlaceholderPath); + throw chmodError; + } + } transactionPaths.delete(stagedPlaceholderPath); placeholderPlacementError = undefined; } catch (placeholderError) { From b3f510c54c26db9fc9f0fd45b6e0b1012c6622ed Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 22:07:24 -0600 Subject: [PATCH 61/94] Recover pre-chmod rollback placeholders Co-Authored-By: GPT-5 Codex --- components/Application.ts | 2 +- .../components/extractApplicationSwap.test.js | 21 +++++++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/components/Application.ts b/components/Application.ts index 61d2012c83..07e025c588 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -883,7 +883,7 @@ async function identifyRollbackPlaceholder( if ( (current.isDirectory() || current.isFile()) && current.uid === BigInt(userId) && - (permissions === 0 || (current.isDirectory() && permissions === 0o100)) + (permissions === 0 || (current.isDirectory() && (permissions === 0o100 || permissions === 0o300))) ) { return { dev: current.dev, ino: current.ino }; } diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index 316b85bdb1..cbdd0deca3 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -573,6 +573,27 @@ describe('extractApplication directory swap', () => { } }); + it('adopts a rollback placeholder interrupted before its restrictive chmod', async function () { + if (process.platform === 'win32' || process.getuid?.() === 0) this.skip(); + this.timeout(15000); + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-placeholder-chmod-')); + const dirPath = path.join(componentsRoot, 'web'); + const asidePath = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-123-previous'); + await fs.mkdir(asidePath, { recursive: true }); + await fs.writeFile(path.join(asidePath, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + await fs.mkdir(dirPath, { mode: 0o300 }); + + try { + const failures = await recoverInterruptedComponentExtractions(componentsRoot); + assert.strictEqual(failures.size, 0); + assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); + await assert.rejects(fs.access(path.join(componentsRoot, '.deploy-aside', 'web'))); + } finally { + await fs.chmod(dirPath, 0o700).catch(() => {}); + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + it('reclaims a stale aside copy left by an earlier deploy', async function () { this.timeout(20000); From d41190051044f3cd8eb7d978590e8a27a2abff42 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Tue, 18 Aug 2026 22:57:57 -0600 Subject: [PATCH 62/94] Retire losing recovery candidates and unblock rollback retries Address review feedback on the component rollback path: - the placeholder placement retry loop now yields between attempts, so a retryable failure from the synchronous displacement no longer spins the event loop for the full 5s deadline - a failed compensating rename after a failed chmod reports both errors instead of discarding the chmod failure it was compensating for - non-chosen `.in-progress-` recovery candidates are retired durably before rollback, so a cleanup that fails cannot let a later pass re-litigate them - the pre-chmod placeholder test now creates a non-empty placeholder, which is the state a live writer actually leaves and the one the 0o300 adoption clause is needed for Co-Authored-By: Claude Opus --- components/Application.ts | 16 ++++++++- .../components/extractApplicationSwap.test.js | 34 ++++++++++++++++++- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 07e025c588..9a46623eb0 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -934,6 +934,12 @@ async function recoverOrCleanupStaleExtractionPaths( recoveryRecords.find(({ priorStateAbsent }) => priorStateAbsent); if (recoveryRecord) { const recoveryPath = join(asideStagingDir, recoveryRecord.entry.name); + // Retire the losing candidates durably; a cleanup that fails must not let a later + // pass adopt one of them and restore an older tree over the one recovered here. + for (const { entry } of recoveryRecords) { + if (entry === recoveryRecord.entry) continue; + paths.add(await retireExtractionAside(join(asideStagingDir, entry.name))); + } await rollbackExtractedDirectory( application, asideStagingDir, @@ -1367,7 +1373,14 @@ async function rollbackExtractedDirectory( try { chmodSync(application.dirPath, 0o100); } catch (chmodError) { - renameSync(application.dirPath, stagedPlaceholderPath); + try { + renameSync(application.dirPath, stagedPlaceholderPath); + } catch (compensationError) { + throw new AggregateError( + [chmodError, compensationError], + `Failed to restrict and then restore the ${application.name} rollback placeholder` + ); + } throw chmodError; } } @@ -1392,6 +1405,7 @@ async function rollbackExtractedDirectory( throw placeholderPlacementError; } if (!placeholderPlacementError) break; + await delay(10); } while (Date.now() < restoreRetryDeadline); if (transactionPaths.has(stagedPlaceholderPath)) { throw new Error( diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index cbdd0deca3..a98ecb43a8 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -174,6 +174,36 @@ describe('extractApplication directory swap', () => { } }); + it('retires losing recovery candidates so a failed cleanup cannot resurrect them', async function () { + if (process.platform === 'win32' || process.getuid?.() === 0) this.skip(); + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-retire-losers-')); + const dirPath = path.join(componentsRoot, 'web'); + const stagingDir = path.join(componentsRoot, '.deploy-aside', 'web'); + const losingAside = path.join(stagingDir, '.in-progress-100-previous'); + const winningAside = path.join(stagingDir, '.in-progress-200-previous'); + await fs.mkdir(losingAside, { recursive: true }); + await fs.writeFile(path.join(losingAside, 'package.json'), '{"name":"web","version":"0.9.0"}\n'); + await fs.mkdir(winningAside, { recursive: true }); + await fs.writeFile(path.join(winningAside, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(path.join(dirPath, 'package.json'), '{"name":"web","version":"2.0.0"}\n'); + // Deny the best-effort cleanup permission to empty the losing aside, so it outlives recovery. + await fs.chmod(losingAside, 0o500); + + try { + assert.strictEqual((await recoverInterruptedComponentExtractions(componentsRoot)).size, 0); + assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); + await fs.access(losingAside); + await fs.access(path.join(stagingDir, '.retired-100-previous')); + + assert.strictEqual((await recoverInterruptedComponentExtractions(componentsRoot)).size, 0); + assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); + } finally { + await fs.chmod(losingAside, 0o700).catch(() => {}); + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + it('does not resurrect a committed cleanup leftover after the component was dropped', async function () { const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-dropped-')); const dirPath = path.join(componentsRoot, 'web'); @@ -581,7 +611,9 @@ describe('extractApplication directory swap', () => { const asidePath = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-123-previous'); await fs.mkdir(asidePath, { recursive: true }); await fs.writeFile(path.join(asidePath, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); - await fs.mkdir(dirPath, { mode: 0o300 }); + await fs.mkdir(dirPath, { mode: 0o700 }); + await fs.writeFile(path.join(dirPath, 'writer-created'), 'occupied'); + await fs.chmod(dirPath, 0o300); try { const failures = await recoverInterruptedComponentExtractions(componentsRoot); From a99785ca7ba89faa318951890ada19034b371352 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 19 Aug 2026 01:30:30 -0600 Subject: [PATCH 63/94] Pin mode-000 directory placeholder adoption Cover the permissions === 0 half of identifyRollbackPlaceholder: a mode-000 directory is the shape a build predating the 0o300/0o100 placeholder modes leaves behind, and only a directory (not the symlink-aside file placeholder) needs adoption to be renamed over. Co-Authored-By: Claude Opus --- .../components/extractApplicationSwap.test.js | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index a98ecb43a8..d04a40137a 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -626,6 +626,30 @@ describe('extractApplication directory swap', () => { } }); + it('adopts a mode-000 directory placeholder left by an interrupted rollback', async function () { + if (process.platform === 'win32' || process.getuid?.() === 0) this.skip(); + this.timeout(15000); + // Builds before the placeholder gained its 0o300/0o100 modes left mode-000 directories behind. + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-placeholder-zero-')); + const dirPath = path.join(componentsRoot, 'web'); + const asidePath = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-123-previous'); + await fs.mkdir(asidePath, { recursive: true }); + await fs.writeFile(path.join(asidePath, 'package.json'), '{"name":"web","version":"1.0.0"}\n'); + await fs.mkdir(dirPath, { mode: 0o700 }); + await fs.writeFile(path.join(dirPath, 'writer-created'), 'occupied'); + await fs.chmod(dirPath, 0o000); + + try { + const failures = await recoverInterruptedComponentExtractions(componentsRoot); + assert.strictEqual(failures.size, 0); + assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); + await assert.rejects(fs.access(path.join(componentsRoot, '.deploy-aside', 'web'))); + } finally { + await fs.chmod(dirPath, 0o700).catch(() => {}); + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + it('reclaims a stale aside copy left by an earlier deploy', async function () { this.timeout(20000); From 2488435100d38313e9de731ee93aa7d2a3259e29 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 19 Aug 2026 18:11:32 -0600 Subject: [PATCH 64/94] Scope deferred readiness to each component load Co-Authored-By: GPT-5 Codex --- components/componentLoader.ts | 13 ++- unitTests/components/componentLoader.test.js | 114 +++++++++++++++++++ 2 files changed, 121 insertions(+), 6 deletions(-) diff --git a/components/componentLoader.ts b/components/componentLoader.ts index fdd5bbdabd..9c6bcf06f6 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -163,7 +163,6 @@ export async function loadComponentDirectories( if (loadedResources) resources = loadedResources; if (loadedPluginModules) loadedComponents = loadedPluginModules; const cycleResources = resources; - const cycleLoadedComponents = loadedComponents; let failedRecoveries = new Map(); try { failedRecoveries = await recoverInterruptedComponentExtractions(CF_ROUTES_DIR); @@ -203,17 +202,15 @@ export async function loadComponentDirectories( } const mountResult = tryRootConfigMount(appName); if (!mountResult.ok) return; - const modulesBeforeLoad = new Set(cycleLoadedComponents.keys()); + const loadedModules = new Set(); await loadComponent(appFolder, cycleResources, HDB_ROOT_DIR_NAME, { isRoot: false, autoReload: false, appName, mount: mountResult.mount, + collectLoadedModules: loadedModules, }); - await readyComponentModules( - [...cycleLoadedComponents.keys()].filter((serverModule) => !modulesBeforeLoad.has(serverModule)), - readyComponentPromises - ); + await readyComponentModules(loadedModules, readyComponentPromises); }) .catch((error) => { const recoveryError = error instanceof Error ? error : new Error(String(error)); @@ -561,6 +558,7 @@ export interface LoadComponentOptions { // (e.g. the deploy pre-flight validation) so their deploy-lifecycle listeners don't accumulate // across deploys (#1462). collectScopes?: Set; + collectLoadedModules?: Set; // Routing the operator declared for this application in the root config (`host`/`urlPath` on // the application's entry). Applied to every plugin scope this load creates, and inherited by // components the application itself declares, so the whole subtree moves together. @@ -592,6 +590,7 @@ export async function loadComponent( autoReload, appName, mount, + collectLoadedModules, } = options; applicationScope.runtimeRoot ??= resolvedFolder; applicationScope.allowedPath ??= realpathSync(componentDirectory); @@ -736,6 +735,7 @@ export async function loadComponent( autoReload: false, appName: appName || componentName, collectScopes: options.collectScopes, + collectLoadedModules, // `host`/`urlPath` on this entry route the component being loaded. For an // application (no plugin module of its own) that entry is the only place an // operator can say where the app is served — its own config.yaml declares the @@ -928,6 +928,7 @@ export async function loadComponent( ...componentConfig, })) || extensionModule; loadedComponents.set(extensionModule, true); + collectLoadedModules?.add(extensionModule); if ( (extensionModule.handleFile || diff --git a/unitTests/components/componentLoader.test.js b/unitTests/components/componentLoader.test.js index cc1ce12d22..b76fe8e439 100644 --- a/unitTests/components/componentLoader.test.js +++ b/unitTests/components/componentLoader.test.js @@ -501,6 +501,120 @@ describe('ComponentLoader Status Integration', function () { } }); + it('keeps readiness scoped to each concurrent deferred component load', async function () { + this.timeout(15000); + const firstAppName = 'first-deferred-readiness-probe'; + const firstPluginName = 'firstDeferredReadinessProbe'; + const secondAppName = 'second-deferred-readiness-probe'; + const secondPluginName = 'secondDeferredReadinessProbe'; + const secondBlockerName = 'secondDeferredReadinessBlocker'; + const firstComponentDir = path.join(tempDir, firstAppName); + const secondComponentDir = path.join(tempDir, secondAppName); + const firstAsideDir = path.join(tempDir, '.deploy-aside', firstAppName, '.in-progress-123-previous'); + const secondAsideDir = path.join(tempDir, '.deploy-aside', secondAppName, '.in-progress-123-previous'); + await fs.mkdir(firstAsideDir, { recursive: true }); + await fs.writeFile(path.join(firstAsideDir, 'config.yaml'), `${firstPluginName}: {}\n`); + await fs.mkdir(secondAsideDir, { recursive: true }); + await fs.writeFile(path.join(secondAsideDir, 'config.yaml'), `${secondPluginName}: {}\n${secondBlockerName}: {}\n`); + await fs.mkdir(firstComponentDir, { recursive: true }); + await fs.writeFile(path.join(firstComponentDir, 'partial'), 'partial'); + await fs.mkdir(secondComponentDir, { recursive: true }); + await fs.writeFile(path.join(secondComponentDir, 'partial'), 'partial'); + + let releaseFirstPreparation; + let releaseSecondPreparation; + let firstPreparationHeld; + let secondPreparationHeld; + const firstPreparationStarted = new Promise((resolve) => (firstPreparationHeld = resolve)); + const secondPreparationStarted = new Promise((resolve) => (secondPreparationHeld = resolve)); + const firstPreparation = withComponentPreparationLock(firstComponentDir, async () => { + firstPreparationHeld(); + await new Promise((resolve) => (releaseFirstPreparation = resolve)); + }); + const secondPreparation = withComponentPreparationLock(secondComponentDir, async () => { + secondPreparationHeld(); + await new Promise((resolve) => (releaseSecondPreparation = resolve)); + }); + + let releaseFirstStart; + let releaseSecondStart; + let firstStartEntered; + let secondBlockerEntered = false; + let firstReadyCalls = 0; + let secondReadyCalls = 0; + const firstStartReached = new Promise((resolve) => (firstStartEntered = resolve)); + componentLoader.TRUSTED_RESOURCE_PLUGINS[firstPluginName] = { + async start() { + firstStartEntered(); + await new Promise((resolve) => (releaseFirstStart = resolve)); + return { ready: () => firstReadyCalls++ }; + }, + }; + componentLoader.TRUSTED_RESOURCE_PLUGINS[secondPluginName] = { + async start() { + await firstStartReached; + return { ready: () => secondReadyCalls++ }; + }, + }; + componentLoader.TRUSTED_RESOURCE_PLUGINS[secondBlockerName] = { + async start() { + secondBlockerEntered = true; + await new Promise((resolve) => (releaseSecondStart = resolve)); + }, + }; + const loadedComponents = new Map(); + const resources = { isWorker: true, set() {} }; + + try { + await Promise.all([firstPreparationStarted, secondPreparationStarted]); + await componentLoader.loadComponentDirectories(loadedComponents, resources, new WeakMap()); + releaseFirstPreparation(); + releaseSecondPreparation(); + await Promise.all([firstPreparation, secondPreparation]); + await waitFor(() => secondBlockerEntered, { + timeout: 5000, + message: 'concurrent deferred loads did not reach the controlled interleaving', + }); + + releaseFirstStart(); + await waitFor(() => firstReadyCalls === 1, { + timeout: 5000, + message: 'first deferred component did not become ready', + }); + assert.strictEqual(secondReadyCalls, 0, 'second component became ready before its load completed'); + + releaseSecondStart(); + await waitFor(() => secondReadyCalls === 1, { + timeout: 5000, + message: 'second deferred component did not become ready after its load completed', + }); + } finally { + releaseFirstPreparation?.(); + releaseSecondPreparation?.(); + releaseFirstStart?.(); + releaseSecondStart?.(); + await Promise.all([firstPreparation, secondPreparation]); + delete componentLoader.TRUSTED_RESOURCE_PLUGINS[firstPluginName]; + delete componentLoader.TRUSTED_RESOURCE_PLUGINS[secondPluginName]; + delete componentLoader.TRUSTED_RESOURCE_PLUGINS[secondBlockerName]; + componentLoader.loadedPaths.clear(); + await fs.rm(path.join(tempDir, '.deploy-aside', firstAppName), { + recursive: true, + force: true, + maxRetries: 5, + retryDelay: 100, + }); + await fs.rm(path.join(tempDir, '.deploy-aside', secondAppName), { + recursive: true, + force: true, + maxRetries: 5, + retryDelay: 100, + }); + await fs.rm(firstComponentDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + await fs.rm(secondComponentDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + // A mounted application's deprecated `start`/`startOnMainThread` hooks receive the raw, // unmounted server (unlike the new Plugin API's `handleApplication(scope)`), so routes they // register would silently escape the mount. Loading must fail closed instead (review finding). From 1074309cb9e5a0186af907aa35fccad053d3cb97 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 19 Aug 2026 19:29:03 -0600 Subject: [PATCH 65/94] Bound startup component recovery waits Keep boot-time recovery probes bounded even when another live recovery owns the component lock, allowing workers to defer that component and bind their listeners. Cover installed-package readiness propagation and the live-recovery timeout with mutation-killing tests. Co-Authored-By: GPT-5 Codex --- DESIGN.md | 2 +- components/Application.ts | 4 +--- unitTests/components/componentLoader.test.js | 11 +++++++++-- .../components/extractApplicationSwap.test.js | 19 +++++++++++++++---- 4 files changed, 26 insertions(+), 10 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 1ad76ebe40..766f195a21 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -141,7 +141,7 @@ Future agents touching `components/deploymentRecorder.ts` for Slice B's streamin `prepareApplication()` performs one destructive transaction against a component directory: extract the incoming payload, then run its dependency installer. Deploy operations can execute on worker threads as well as main, so a module-local promise queue is insufficient—each worker has its own module registry. `withComponentPreparationLock()` (`components/componentPreparationLock.ts`) instead acquires an atomic filesystem lock keyed by the absolute component path. The deprecated `install_node_modules` operation uses the same lock, so it cannot run npm concurrently with a deploy. -The deploy lifecycle broadcast deliberately sits _outside_ the lock. Overlapping requests therefore increment the existing per-component lifecycle refcount before queueing; watchers remain suppressed continuously until the final queued preparation ends. The lock itself covers credential materialization, extraction, and installation. Its fully-written owner record is published with an atomic rename, so contenders never observe a partially initialized lock. A lock is never stolen from a known-live owner based on elapsed wall time: installs can be long-running and clocks can jump. Locks from a dead process are reclaimed, and a same-process contender asks the main thread whether the owning worker still exists so a worker crash does not wedge that component until Harper restarts. The bounded wait remains a backstop when owner liveness cannot be established. +The deploy lifecycle broadcast deliberately sits _outside_ the lock. Overlapping requests therefore increment the existing per-component lifecycle refcount before queueing; watchers remain suppressed continuously until the final queued preparation ends. The lock itself covers credential materialization, extraction, and installation. Its fully-written owner record is published with an atomic rename, so contenders never observe a partially initialized lock. A preparation caller never steals a lock from a known-live owner based on elapsed wall time: installs can be long-running and clocks can jump. Locks from a dead process are reclaimed, and a same-process contender asks the main thread whether the owning worker still exists so a worker crash does not wedge that component until Harper restarts. The boot-time bulk-recovery probe is deliberately different: it never renews its 250 ms deadline, even behind another live recovery, so it can defer that component and let the worker bind its listener. A plugin load that begins while its component is being deployed waits for that lifecycle to end before starting `handleApplication`; if a deploy begins during the load, the plugin timeout counts only active, diff --git a/components/Application.ts b/components/Application.ts index 9a46623eb0..cc972bd46f 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -1039,9 +1039,7 @@ export async function recoverInterruptedComponentExtraction( { timeoutMs: waitForPreparation ? COMPONENT_RECOVERY_WAIT_TIMEOUT_MS : COMPONENT_RECOVERY_TRY_TIMEOUT_MS, purpose: COMPONENT_RECOVERY_LOCK_PURPOSE, - renewTimeoutWhileOwnerAlive: waitForPreparation - ? true - : (owner) => owner.purpose === COMPONENT_RECOVERY_LOCK_PURPOSE, + renewTimeoutWhileOwnerAlive: waitForPreparation, onWait: (owner) => { logger.info( `Waiting to settle component deployment state for ${componentName}` + diff --git a/unitTests/components/componentLoader.test.js b/unitTests/components/componentLoader.test.js index b76fe8e439..f6ec8d7b81 100644 --- a/unitTests/components/componentLoader.test.js +++ b/unitTests/components/componentLoader.test.js @@ -501,9 +501,10 @@ describe('ComponentLoader Status Integration', function () { } }); - it('keeps readiness scoped to each concurrent deferred component load', async function () { + it('keeps installed-package readiness scoped to each concurrent deferred component load', async function () { this.timeout(15000); const firstAppName = 'first-deferred-readiness-probe'; + const firstPackageName = 'first-deferred-readiness-package'; const firstPluginName = 'firstDeferredReadinessProbe'; const secondAppName = 'second-deferred-readiness-probe'; const secondPluginName = 'secondDeferredReadinessProbe'; @@ -513,7 +514,13 @@ describe('ComponentLoader Status Integration', function () { const firstAsideDir = path.join(tempDir, '.deploy-aside', firstAppName, '.in-progress-123-previous'); const secondAsideDir = path.join(tempDir, '.deploy-aside', secondAppName, '.in-progress-123-previous'); await fs.mkdir(firstAsideDir, { recursive: true }); - await fs.writeFile(path.join(firstAsideDir, 'config.yaml'), `${firstPluginName}: {}\n`); + await fs.writeFile( + path.join(firstAsideDir, 'config.yaml'), + `${firstPackageName}:\n package: ${firstPackageName}\n` + ); + const firstPackageDir = path.join(firstAsideDir, 'node_modules', firstPackageName); + await fs.mkdir(firstPackageDir, { recursive: true }); + await fs.writeFile(path.join(firstPackageDir, 'config.yaml'), `${firstPluginName}: {}\n`); await fs.mkdir(secondAsideDir, { recursive: true }); await fs.writeFile(path.join(secondAsideDir, 'config.yaml'), `${secondPluginName}: {}\n${secondBlockerName}: {}\n`); await fs.mkdir(firstComponentDir, { recursive: true }); diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index d04a40137a..8ec146396c 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -327,7 +327,7 @@ describe('extractApplication directory swap', () => { } }); - it('waits for a peer recovery that outlasts the bulk recovery grace period', async function () { + it('bounds bulk recovery while a peer recovery remains live', async function () { const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-peer-recovery-')); const dirPath = path.join(componentsRoot, 'web'); const asidePath = path.join(componentsRoot, '.deploy-aside', 'web', '.in-progress-123-previous'); @@ -349,9 +349,20 @@ describe('extractApplication directory swap', () => { try { await started; - setTimeout(() => releaseRecovery(), 350); - const failures = await recoverInterruptedComponentExtractions(componentsRoot); - assert.strictEqual(failures.size, 0); + let failIfUnbounded; + const failures = await Promise.race([ + recoverInterruptedComponentExtractions(componentsRoot), + new Promise((_, reject) => { + failIfUnbounded = setTimeout( + () => reject(new Error('bulk recovery waited past its bounded lock timeout')), + 1000 + ); + }), + ]).finally(() => clearTimeout(failIfUnbounded)); + assert(failures.get('web') instanceof ComponentPreparationLockTimeoutError); + releaseRecovery(); + await recovery; + await recoverInterruptedComponentExtraction(componentsRoot, 'web'); assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); } finally { releaseRecovery?.(); From 40fc6faf0c4c6681de58a176d8a28783e9505d63 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 19 Aug 2026 19:57:58 -0600 Subject: [PATCH 66/94] Pin deferred recovery lock renewal Allow the internal recovery wait timeout to be shortened in tests, then hold a live owner across repeated deadlines to prove the deferred full-recovery path renews while the bounded boot probe does not. Co-Authored-By: GPT-5 Codex --- components/Application.ts | 5 +++-- unitTests/components/extractApplicationSwap.test.js | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index cc972bd46f..bc4b9f9e0b 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -1017,7 +1017,8 @@ export async function recoverInterruptedComponentExtractions( export async function recoverInterruptedComponentExtraction( componentsRootDirPath: string, componentName: string, - waitForPreparation = true + waitForPreparation = true, + waitTimeoutMs = COMPONENT_RECOVERY_WAIT_TIMEOUT_MS ): Promise { const componentDirPath = join(componentsRootDirPath, componentName); await withComponentPreparationLock( @@ -1037,7 +1038,7 @@ export async function recoverInterruptedComponentExtraction( ); }, { - timeoutMs: waitForPreparation ? COMPONENT_RECOVERY_WAIT_TIMEOUT_MS : COMPONENT_RECOVERY_TRY_TIMEOUT_MS, + timeoutMs: waitForPreparation ? waitTimeoutMs : COMPONENT_RECOVERY_TRY_TIMEOUT_MS, purpose: COMPONENT_RECOVERY_LOCK_PURPOSE, renewTimeoutWhileOwnerAlive: waitForPreparation, onWait: (owner) => { diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index 8ec146396c..28c09cd863 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -317,8 +317,8 @@ describe('extractApplication directory swap', () => { await started; const failures = await recoverInterruptedComponentExtractions(componentsRoot); assert(failures.get('web') instanceof ComponentPreparationLockTimeoutError); - setTimeout(() => releaseRecovery(), 25); - await recoverInterruptedComponentExtraction(componentsRoot, 'web'); + setTimeout(() => releaseRecovery(), 350); + await recoverInterruptedComponentExtraction(componentsRoot, 'web', true, 100); assert.strictEqual(JSON.parse(await fs.readFile(path.join(dirPath, 'package.json'), 'utf8')).version, '1.0.0'); } finally { releaseRecovery?.(); From fb77b5150138ca72001a289ac2ebdda6d4b96eb3 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 19 Aug 2026 20:04:38 -0600 Subject: [PATCH 67/94] Preserve dangling component symlinks on rollback Use lstat for the pre-swap existence probe so a dangling component symlink is renamed aside and restored if payload extraction fails. Add a mutation-killing regression for the configured-link case. Co-Authored-By: GPT-5 Codex --- components/Application.ts | 2 +- .../components/extractApplicationSwap.test.js | 27 +++++++++++++++++++ 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/components/Application.ts b/components/Application.ts index bc4b9f9e0b..8a483fe643 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -748,7 +748,7 @@ export async function extractApplication( await recoverOrCleanupStaleExtractionPaths(application, asideStagingDir); let componentExists = true; try { - await access(application.dirPath, constants.F_OK); + await lstat(application.dirPath); } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; componentExists = false; diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index 28c09cd863..4081f81822 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -73,6 +73,33 @@ describe('extractApplication directory swap', () => { } }); + it('restores a dangling symlink when payload extraction fails', async function () { + if (process.platform === 'win32') this.skip(); + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-dangling-symlink-')); + const dirPath = path.join(componentsRoot, 'web'); + const missingTarget = path.join(componentsRoot, 'missing-target'); + const extractionError = new Error('payload delivery failed'); + await fs.symlink(missingTarget, dirPath, 'dir'); + const app = new Application({ + name: 'web', + payload: new Readable({ + read() { + this.destroy(extractionError); + }, + }), + }); + app.dirPath = dirPath; + + try { + await assert.rejects(() => extractApplication(app), extractionError); + assert.strictEqual((await fs.lstat(dirPath)).isSymbolicLink(), true); + assert.strictEqual(await fs.readlink(dirPath), missingTarget); + await assert.rejects(fs.access(path.join(componentsRoot, '.deploy-aside', 'web'))); + } finally { + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + } + }); + it('removes a partial directory when a first deploy fails', async function () { const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-new-failure-')); const dirPath = path.join(componentsRoot, 'web'); From 88cd258aa70b310ac2c1dbe8bcfbcafbe0beee40 Mon Sep 17 00:00:00 2001 From: Kris Zyp Date: Wed, 19 Aug 2026 21:17:20 -0600 Subject: [PATCH 68/94] Contain dangling component load failures Co-Authored-By: GPT-5 Codex --- components/componentLoader.ts | 5 ++- unitTests/components/componentLoader.test.js | 33 ++++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 9c6bcf06f6..250b5f3d64 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -258,7 +258,10 @@ export async function loadComponentDirectories( appName, mount: mountResult.mount, }) - ) + ).catch((error) => { + const loadError = error instanceof Error ? error : new Error(String(error)); + componentLifecycle.failed(appName, loadError, `Component '${appName}' failed to load`); + }) ); } } diff --git a/unitTests/components/componentLoader.test.js b/unitTests/components/componentLoader.test.js index f6ec8d7b81..223764c5b1 100644 --- a/unitTests/components/componentLoader.test.js +++ b/unitTests/components/componentLoader.test.js @@ -322,6 +322,39 @@ describe('ComponentLoader Status Integration', function () { }); }); + it('isolates a dangling component symlink while loading healthy siblings', async function () { + if (process.platform === 'win32') this.skip(); + const danglingAppName = 'dangling-component-probe'; + const healthyAppName = 'healthy-sibling-probe'; + const healthyPluginName = 'healthySiblingProbe'; + const danglingAppPath = path.join(tempDir, danglingAppName); + const healthyAppPath = path.join(tempDir, healthyAppName); + let healthyLoadCalls = 0; + + await fs.symlink(path.join(tempDir, 'missing-component-target'), danglingAppPath, 'dir'); + await fs.mkdir(healthyAppPath); + await fs.writeFile(path.join(healthyAppPath, 'config.yaml'), `${healthyPluginName}: {}\n`); + componentLoader.TRUSTED_RESOURCE_PLUGINS[healthyPluginName] = { + start() { + healthyLoadCalls++; + }, + }; + + try { + await componentLoader.loadComponentDirectories(new Map(), { isWorker: true, set() {} }, new WeakMap()); + + assert.strictEqual(healthyLoadCalls, 1, 'healthy sibling did not load'); + const danglingFailure = lifecycle.failed.getCalls().find((call) => call.args[0] === danglingAppName); + assert.ok(danglingFailure, 'dangling component was not marked failed'); + assert.strictEqual(danglingFailure.args[1].code, 'ENOENT'); + } finally { + delete componentLoader.TRUSTED_RESOURCE_PLUGINS[healthyPluginName]; + componentLoader.loadedPaths.clear(); + await fs.rm(danglingAppPath, { force: true }); + await fs.rm(healthyAppPath, { recursive: true, force: true }); + } + }); + describe('deploy lifecycle listener lifecycle (#1462)', function () { const { deployLifecycle } = require('#src/components/deployLifecycle'); From 8664805e7afc5fb7d3346efdefc8b0a728f2cefc Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 11:47:58 -0400 Subject: [PATCH 69/94] fix(deploy): fence revert persistence, contain reconciliation, and complete the link walk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Works through the Codex/Barber review round on harper#1849. Twelve findings; the substantive ones cluster into four root causes. **Revert now commits its persistent state inside the swap.** Three findings (operations.js:1262, Application.ts:995, and a local concurrency review) were the same defect from different angles: the config transaction committed after `revertApplication` released the component lock, and the third rename consumed the only recovery evidence before the manifest and config were durable. So a queued activation could swap and commit in the gap — leaving its bytes live under the revert's config — and a crash after the final rename left exchanged directories under a stale manifest with nothing for startup to reconcile. `revertApplication` now takes a `commitPersistentState` hook, writes a revert-intent marker before the first rename, and orders the tail commit → manifest → final rename → clear marker, so the holding tree survives until everything is durable. **The revert marker is bound to its holding directory.** A fixed per-component path could be orphaned by a crash between the final rename and the marker removal, then trusted by a later unrelated revert of the same component — silently committing an old config and manifest. The marker is now named after the holding directory it describes, and `recoverInterruptedReverts` sweeps markers whose holding directory is gone before it does anything else. **Reconciliation no longer fails a healthy component closed.** A broken *staged* candidate says nothing about the live tree or its config, but it was thrown, folded into `failedProjects`, and kept the component from loading — permanently, since nothing sweeps a staging directory whose subtree is missing. Only an interrupted *activation* now attributes to the component; a bad staged candidate is settled `failed`, its residue removed, and the live component loads untouched. Settling the row also lets payload retention reclaim its blob, which a non-terminal row blocks forever. **The dependency-link walk follows nested trees.** It stopped at `node_modules/` and `node_modules/@scope/`, but npm nests a `file:` link under `node_modules//node_modules/` whenever hoisting is blocked by a version conflict, and routinely under workspaces. Those links dangle after the swap exactly like top-level ones, and the hard-fail could not help because they were never enumerated. Also fixed: - `retainActivatedPrevious` clears the manifest before it moves anything, so every intermediate state reads as "not revertable" rather than revertable-to-the-wrong bytes. The failure path drops it too, which is what its contract already claimed. - `drop_component` / `dropCustomFunctionProject` clear `.deploy-previous` and its manifest, so a dropped component can no longer be resurrected by `revert_component` re-adding its root-config and lock entries. Staged-deployment invalidation moved inside the preparation lock, closing a window where a concurrent stage completed and was then activated over the drop. - `revert_component` and `component_deploy_phase` pass their `api_name` to the permission constructor; without it Gate 1 compares a role grant against the internal function name and denies it. - Staging retention pins the just-staged deployment id, so clock skew across nodes cannot rank a peer's row above the one this call is about to return. - `deployment_timeout` rides the activate and restart fan-outs, not just stage — a peer rebuilding at activate time was always using the 120s default. - Staging retention prune is `.catch`-guarded, so disk hygiene cannot 500 a stage that already reached `finish('staged')`. - `harper revert` defaults `project` from the CWD like every other deploy verb, via a prepare step rather than the pure verb guard. - Deleted a stale `revert_component` JSDoc left stranded above `activationSpecFromRequest`. Tests: the two aside assertions that passed vacuously whenever the detached sweep won the race are replaced by a deterministic test against `discardDirAside` itself, plus staging-retention eviction and a table-driven `getStagingRetentionMaxCount` coercion test — all three previously uncovered. New cases for the config fence (observed from inside the lock), its compensation, the orphan-marker sweep, and a nested-node_modules link. 346 deploy/component/CLI/ server unit tests pass. --- bin/cliOperations.ts | 10 +- components/Application.ts | 183 ++++++++++++----- components/componentLoader.ts | 38 ++-- components/deploymentRecorder.ts | 18 +- components/operations.js | 97 ++++----- unitTests/components/deployStaging.test.js | 225 ++++++++++++++++++--- utility/operation_authorization.ts | 10 +- 7 files changed, 424 insertions(+), 157 deletions(-) diff --git a/bin/cliOperations.ts b/bin/cliOperations.ts index 4fc0a13faf..3513d8f1a9 100644 --- a/bin/cliOperations.ts +++ b/bin/cliOperations.ts @@ -347,11 +347,17 @@ const packageCwdForUpload = async (req) => { req._multipart = true; }; +// `harper revert` uploads nothing, so it has no packaging step — but it still needs the CWD project +// default every other deploy-family verb gets, or it fails server-side on a missing `project`. +const prepareRevert = async (req) => { + req.project ||= path.basename(process.cwd()); +}; + const PREPARE_OPERATION: any = { // deploy_component covers `harper deploy` and `harper stage` (activate:false); packageCwdForUpload - // itself skips the upload for a `package` identifier or a `deployment_id` activate. revert takes no - // payload, so it needs no prep step. + // itself skips the upload for a `package` identifier or a `deployment_id` activate. deploy_component: packageCwdForUpload, + revert_component: prepareRevert, }; /** diff --git a/components/Application.ts b/components/Application.ts index 0d92461941..b08f8c3cfe 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -491,7 +491,7 @@ export const ASIDE_STAGING_DIR = '.deploy-aside'; const IN_PROGRESS_ASIDE_PREFIX = '.in-progress-'; // Parked and known-disposable at park time (an evicted two-deploys-ago retained-previous). NEVER a // recovery candidate — see discardDirAside for why the distinction has to be in the name. -const DISCARDED_ASIDE_PREFIX = '.discarded-'; +export const DISCARDED_ASIDE_PREFIX = '.discarded-'; // The holding path a revert's three-way swap parks the outgoing live tree in. Recoverable: if the // process dies mid-swap, recoverInterruptedReverts puts it back. See revertApplication. const REVERTING_PREFIX = '.reverting-'; @@ -499,6 +499,13 @@ const REVERTING_PREFIX = '.reverting-'; // anything and removed only once the directory, manifest and config are all durable, so a recovery // interrupted part-way is resumable rather than losing its own evidence. See recoverInterruptedReverts. const REVERT_RECOVERY_SUFFIX = '.recovering.json'; + +// The revert-intent marker is named after the holding directory it describes. A fixed per-component +// path would be reused by every future revert of that component, so an orphan left by one attempt +// could be trusted by a later unrelated one. +function revertRecoveryMarkerFor(holdingPath: string): string { + return `${holdingPath}${REVERT_RECOVERY_SUFFIX}`; +} // Splits `-` off a revert holding directory name, anchored on the UUID so a // component name containing dashes is preserved intact. const REVERTING_NAME_PATTERN = /^(.+)-[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i; @@ -630,7 +637,7 @@ async function pruneStagedBuilds(componentName: string, keepStagingId: string, m * version over the current one. `.discarded-` says "never restore this", and cleanupExtractionPaths * already removes anything that is not an unretired `.in-progress-`. */ -async function discardDirAside(targetDirPath: string, componentName: string): Promise { +export async function discardDirAside(targetDirPath: string, componentName: string): Promise { const asideStagingDir = extractionStagingDirectory(targetDirPath); try { // lstat, not access(F_OK): access follows symlinks, so a DANGLING symlink at the path (left by a @@ -753,6 +760,12 @@ async function retainActivatedPrevious( const liveDirPath = application.dirPath; const previousPath = previousDirPathFor(liveDirPath); try { + // The manifest goes FIRST, by being removed. getRevertTarget only checks that a retained tree + // exists, so a manifest that outlives its tree is worse than no manifest: it names a deployment id + // against bytes that are no longer the ones it describes, and an addressed revert would then either + // restore the wrong tree or report "already live" and do nothing. Clearing it means every + // intermediate state below reads as "not revertable", which is what the failure path promises. + await rm(previousManifestPathFor(liveDirPath), { force: true }); // Evict the older retained-previous (two deploys ago) before renaming this one into its place, so // the rename never races an incomplete recursive delete (ENOTEMPTY). Also covers the first-deploy // case, where any retained tree left over from a dropped-and-redeployed component is stale. @@ -769,6 +782,9 @@ async function retainActivatedPrevious( live: { deployment_id: deploymentId, application_config: activatedConfig ?? null }, }); } catch (err) { + // Best-effort by design: the deploy succeeded, so failing it here would be worse. Drop the manifest + // so the component reads as not revertable rather than revertable-to-the-wrong-bytes. + await rm(previousManifestPathFor(liveDirPath), { force: true }).catch(() => {}); logger.warn( `Deployed ${application.name}, but could not retain its previous version for revert:`, errorForLog(err as Error) @@ -798,6 +814,15 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): if ((error as NodeJS.ErrnoException).code === 'ENOENT') return failures; throw error; } + // A crash between the final rename and the marker removal leaves a marker whose holding directory is + // gone. Nothing would revisit it — the loop below is keyed on finding a holding directory — so sweep + // those first rather than leave them to be trusted by a future attempt. + const entryNames = new Set(entries.map((entry) => entry.name)); + for (const entry of entries) { + if (entry.isDirectory() || !entry.name.endsWith(REVERT_RECOVERY_SUFFIX)) continue; + const holdingName = entry.name.slice(0, -REVERT_RECOVERY_SUFFIX.length); + if (!entryNames.has(holdingName)) await rm(join(previousRoot, entry.name), { force: true }).catch(() => {}); + } for (const entry of entries) { if (!entry.isDirectory() || !entry.name.startsWith(REVERTING_PREFIX)) continue; const holding = join(previousRoot, entry.name); @@ -837,25 +862,10 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): } ); if (!previousExists) { - // The swap's second rename landed (the reverted-to version is live) and only the retain step - // was lost, so finish it — directory, manifest and durable config together. - // - // The manifest is still the PRE-swap one: revertApplication writes it only after all three - // renames, so reaching here means it was never updated. Leaving it alone would be actively - // harmful rather than merely stale: it names the version that is now RETAINED as live and the - // version that is now LIVE as previous, so a retry of the same addressed revert would match - // its target against the reversed `previous` entry and swap the successful revert back out — - // destroying the idempotency the addressed target exists to provide. Roles are exchanged here - // exactly as revertApplication would have. - // The holding directory is the ONLY durable evidence that this recovery is unfinished, so it - // must not be consumed until the manifest and config writes are durable too. Otherwise a - // failure after the rename leaves nothing for the next start to find: recovery sees a - // complete-looking tree and the component loads with whichever write half-landed. - // - // A recovery marker carries the intended end state across restarts. On re-entry it — not the - // manifest — is the source of truth, because a previous attempt may already have exchanged the - // manifest, and exchanging an exchanged manifest would flip it back. - const recoveryMarkerPath = `${previousPath}${REVERT_RECOVERY_SUFFIX}`; + // The reverted-to version is live and only the retain step was lost, so finish it. The + // marker is the source of truth over the manifest, because an earlier pass may already have + // exchanged the manifest and exchanging it twice flips it back. + const recoveryMarkerPath = revertRecoveryMarkerFor(holding); let intended = await readRevertRecoveryMarker(recoveryMarkerPath); if (!intended) { const staleManifest = await readRetainedPreviousManifest(liveDirPath); @@ -895,6 +905,25 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): return failures; } +/** + * Remove a component's retained previous version and its manifest. Both live under the components root + * rather than inside the component directory, so dropping the component does not reach them — and a + * surviving retained tree lets `revert_component` resurrect a dropped component, re-adding its + * root-config and application-lock entries. + */ +export async function discardRetainedPrevious(componentDirPath: string): Promise { + await rm(previousManifestPathFor(componentDirPath), { force: true }); + await discardDirAside(previousDirPathFor(componentDirPath), basename(componentDirPath)); + const previousRoot = join(dirname(componentDirPath), DEPLOY_PREVIOUS_DIR); + for (const entry of await readdir(previousRoot, { withFileTypes: true }).catch(() => [])) { + // Any in-flight revert artifact for this component is meaningless once it is dropped. + if (entry.name.startsWith(`${REVERTING_PREFIX}${basename(componentDirPath)}-`)) { + await rm(join(previousRoot, entry.name), { recursive: true, force: true }).catch(() => {}); + } + } + await rmdir(previousRoot).catch(() => {}); +} + /** What a component can currently be reverted to, for reporting and for revert targeting. */ export async function getRevertTarget( componentDirPath: string @@ -931,7 +960,8 @@ export async function getRevertTarget( */ export async function revertApplication( application: Application, - toDeploymentId: string + toDeploymentId: string, + hooks: { commitPersistentState?: (config: ApplicationConfig | null) => Promise } = {} ): Promise<{ swapped: boolean; activatedConfig: ApplicationConfig | null; fromDeploymentId: string | null }> { const liveDirPath = application.dirPath; return withComponentPreparationLock(liveDirPath, async () => { @@ -978,6 +1008,14 @@ export async function revertApplication( // recoverable by name at startup (recoverInterruptedReverts), which covers the case // compensation cannot: the process dying between renames. const holding = join(dirname(previousPath), `${REVERTING_PREFIX}${basename(liveDirPath)}-${randomUUID()}`); + // Written before anything moves: a crash between the renames and the durable writes below + // leaves the holding tree AND this marker, which is what lets recoverInterruptedReverts finish + // the job. Without it, recovery would re-exchange an already-exchanged manifest. + const recoveryMarkerPath = revertRecoveryMarkerFor(holding); + await writeJsonAtomically(recoveryMarkerPath, { + previous: target.live, + live: target.previous, + }); await rename(liveDirPath, holding); try { await rename(previousPath, liveDirPath); @@ -992,6 +1030,31 @@ export async function revertApplication( }); throw swapError; } + // Persistent state commits here, inside the lock and while the holding tree still exists. + // Committing it after the lock released let a queued activation swap and commit in between, + // leaving that activation's bytes live under this revert's config. + try { + await hooks.commitPersistentState?.(target.previous.application_config); + application.useLiveBuildDir(); + await writeRetainedPreviousManifest(liveDirPath, { + previous: target.live, + live: target.previous, + }); + } catch (persistError) { + try { + await rename(liveDirPath, previousPath); + await rename(holding, liveDirPath); + await rm(recoveryMarkerPath, { force: true }); + } catch (restoreError) { + throw new AggregateError( + [persistError, restoreError], + `Reverted ${application.name} but could not persist its configuration or undo the swap; ` + + `${holding} still holds the previously-live tree` + ); + } + throw persistError; + } + // Consumes the recovery evidence, so it goes last. try { await rename(holding, previousPath); } catch (retainError) { @@ -1010,17 +1073,18 @@ export async function revertApplication( } throw retainError; } + await rm(recoveryMarkerPath, { force: true }); } else { // No live version to preserve; restore the previous into place. Nothing becomes the new // retained previous, so the component can't be reverted again until its next deploy. await rename(previousPath, liveDirPath); + await hooks.commitPersistentState?.(target.previous.application_config); + application.useLiveBuildDir(); + await writeRetainedPreviousManifest(liveDirPath, { + previous: { deployment_id: null, application_config: null }, + live: target.previous, + }); } - application.useLiveBuildDir(); - // Roles exchange: what was live is now the retained previous, and vice versa. - await writeRetainedPreviousManifest(liveDirPath, { - previous: liveExists ? target.live : { deployment_id: null, application_config: null }, - live: target.previous, - }); return { swapped: true, activatedConfig: target.previous.application_config, @@ -2710,30 +2774,31 @@ async function repointStagedDependencyLinks(stagingDirPath: string, futureLiveDi // Requiring a real directory at each level we descend keeps every candidate under the real root. const nodeModulesStat = await lstat(nodeModulesPath).catch(() => undefined); if (!nodeModulesStat?.isDirectory() || nodeModulesStat.isSymbolicLink()) return 0; - let topLevel; - try { - topLevel = await readdir(nodeModulesPath, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return 0; - throw error; - } - // Package links live at `node_modules/` or, for a scoped package, `node_modules/@scope/`. + // Package links live at `node_modules/` or `node_modules/@scope/`, and npm nests a further + // `node_modules` under a package whenever hoisting is blocked by a version conflict — routinely so + // under workspaces. A link nested that way dangles after the swap exactly like a top-level one, so the + // walk has to follow real nested trees rather than stopping at the first two levels. const candidates: string[] = []; - for (const entry of topLevel) { - if (entry.name.startsWith('.')) continue; - if (entry.name.startsWith('@')) { - const scopePath = join(nodeModulesPath, entry.name); - // Only descend into a REAL directory: a symlinked scope directory belongs to whatever it points - // at, not to this component (see the note on nodeModulesStat above). - const scopeStat = await lstat(scopePath).catch(() => undefined); - if (!scopeStat?.isDirectory() || scopeStat.isSymbolicLink()) continue; - for (const scoped of await readdir(scopePath, { withFileTypes: true }).catch(() => [])) { - candidates.push(join(scopePath, scoped.name)); + const collect = async (directoryPath: string): Promise => { + for (const entry of await readdir(directoryPath, { withFileTypes: true }).catch(() => [])) { + if (entry.name.startsWith('.')) continue; + const entryPath = join(directoryPath, entry.name); + if (entry.name.startsWith('@')) { + // Only descend into a REAL directory: a symlinked scope directory belongs to whatever it points + // at, not to this component (see the note on nodeModulesStat above). + const scopeStat = await lstat(entryPath).catch(() => undefined); + if (scopeStat?.isDirectory() && !scopeStat.isSymbolicLink()) await collect(entryPath); + continue; } - continue; + candidates.push(entryPath); + const packageStat = await lstat(entryPath).catch(() => undefined); + if (!packageStat?.isDirectory() || packageStat.isSymbolicLink()) continue; + const nestedPath = join(entryPath, 'node_modules'); + const nestedStat = await lstat(nestedPath).catch(() => undefined); + if (nestedStat?.isDirectory() && !nestedStat.isSymbolicLink()) await collect(nestedPath); } - candidates.push(join(nodeModulesPath, entry.name)); - } + }; + await collect(nodeModulesPath); let repointed = 0; for (const linkPath of candidates) { @@ -2966,7 +3031,8 @@ async function removeActivationArtifacts(componentDirPath: string, deploymentId: export async function reconcileStagedApplicationArtifacts( componentsRootDirPath: string, getDeployment: DeploymentLookup, - persistActivation: (row: Record) => Promise + persistActivation: (row: Record) => Promise, + settleStagedDeployment?: (deploymentId: string) => Promise ): Promise<{ recovered: string[]; removed: string[]; @@ -3023,7 +3089,16 @@ export async function reconcileStagedApplicationArtifacts( const stagedPath = stagedApplicationPath(componentDirPath, entry.name); if (row.status === 'staged') { if (!(await hasCompleteStagedApplication(stagedPath))) { - throw new Error(`Staged deployment '${entry.name}' has no valid component tree for '${row.project}'`); + // Only a not-yet-activated candidate is broken; the live tree and its persisted config are + // consistent. Failing the component closed here would take a healthy component offline and + // keep it offline, because nothing else sweeps a staging directory whose subtree is missing. + logger.warn( + `Discarding staged deployment '${entry.name}' for '${row.project}': no valid component tree. ` + + `The live component is unaffected.` + ); + await settleStagedDeployment?.(entry.name); + await rm(deploymentPath, { recursive: true, force: true }); + removed.push(entry.name); } continue; } @@ -3046,7 +3121,11 @@ export async function reconcileStagedApplicationArtifacts( } catch (error) { const reconcileError = error instanceof Error ? error : new Error(String(error)); errors.set(entry.name, reconcileError); - if (safeComponentName(row?.project)) failedProjects.set(row.project, reconcileError); + // Fail the component closed ONLY for an interrupted activation, where the live tree and its + // durable configuration can disagree. A staged-candidate failure leaves live state consistent. + if (row?.status === 'activating' && safeComponentName(row?.project)) { + failedProjects.set(row.project, reconcileError); + } } } diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 10f53dd77e..662c3f79b3 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -59,7 +59,7 @@ import { recoverInterruptedComponentExtractions, recoverInterruptedReverts, } from './Application.ts'; -import { getDeploymentRow } from './deploymentRecorder.ts'; +import { getDeploymentRow, markDeploymentTerminal } from './deploymentRecorder.ts'; import { ComponentPreparationLockTimeoutError } from './componentPreparationLock.ts'; import { pathToFileURL } from 'node:url'; @@ -200,22 +200,30 @@ export async function loadComponentDirectories( } if (isMainThread && !stagedArtifactsReconciled) { try { - const reconciliation = await reconcileStagedApplicationArtifacts(CF_ROUTES_DIR, getDeploymentRow, async (row) => { - const transaction = await createApplicationActivationTransaction(row.project, row.activation_spec); - try { - await transaction.commit(); - } catch (error) { + const settleDiscardedDeployment = async (deploymentId: string) => { + await markDeploymentTerminal(deploymentId, 'failed', new Error('staged component tree was not recoverable')); + }; + const reconciliation = await reconcileStagedApplicationArtifacts( + CF_ROUTES_DIR, + getDeploymentRow, + async (row) => { + const transaction = await createApplicationActivationTransaction(row.project, row.activation_spec); try { - await transaction.rollback(); - } catch (rollbackError) { - throw new AggregateError( - [error, rollbackError], - `Could not persist or roll back the recovered component activation for '${row.project}'` - ); + await transaction.commit(); + } catch (error) { + try { + await transaction.rollback(); + } catch (rollbackError) { + throw new AggregateError( + [error, rollbackError], + `Could not persist or roll back the recovered component activation for '${row.project}'` + ); + } + throw error; } - throw error; - } - }); + }, + settleDiscardedDeployment + ); for (const deploymentId of reconciliation.recovered) { harperLogger.warn( `Rolled forward interrupted component activation '${deploymentId}' on this node; ` + diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index a167a4942a..6d6ec6b58e 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -571,17 +571,21 @@ export async function claimStagedDeployment( // deploymentOperations.ts's guard for the explicit delete_deployment_payload operation. const TERMINAL_STATUSES = new Set(['success', 'failed', 'rolled_back']); -async function settleStagedRows(project: string, keepCount: number): Promise { +async function settleStagedRows(project: string, keepCount: number, keepDeploymentId?: string): Promise { const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; if (!table) return []; const staged: Array> = []; for await (const row of table.search([{ attribute: 'project', value: project }])) { - if (row?.status === 'staged') staged.push(row); + // `started_at` is stamped by whichever node originated the deploy, so clock skew across nodes can + // rank a peer-originated row above the one this call just created. Excluding it explicitly is what + // stops retention cluster-wide discarding the staging tree for the id being returned to the + // operator — the same guarantee pruneStagedBuilds gets from its keepStagingId. + if (row?.status === 'staged' && row.deployment_id !== keepDeploymentId) staged.push(row); } staged.sort( (a, b) => (b.started_at ?? 0) - (a.started_at ?? 0) || String(a.deployment_id).localeCompare(b.deployment_id) ); - const expired = staged.slice(Math.max(0, keepCount)); + const expired = staged.slice(Math.max(0, keepDeploymentId ? keepCount - 1 : keepCount)); for (const row of expired) { await table.patch(row.deployment_id, { status: 'failed', @@ -592,9 +596,13 @@ async function settleStagedRows(project: string, keepCount: number): Promise row.deployment_id); } -export async function expireOldStagedDeployments(project: string, maxCount: number): Promise { +export async function expireOldStagedDeployments( + project: string, + maxCount: number, + keepDeploymentId?: string +): Promise { const count = Number.isFinite(maxCount) ? Math.max(1, Math.floor(maxCount)) : 1; - return settleStagedRows(project, count); + return settleStagedRows(project, count, keepDeploymentId); } export async function invalidateProjectStagedDeployments(project: string): Promise { diff --git a/components/operations.js b/components/operations.js index ae159cc9bf..a852079f0e 100644 --- a/components/operations.js +++ b/components/operations.js @@ -42,6 +42,7 @@ const { getRevertTarget, getStagingRetentionMaxCount, dropComponentDirectory, + discardRetainedPrevious, ASIDE_STAGING_DIR, DEPLOY_STAGING_DIR, DEPLOY_ACTIVATION_DIR, @@ -354,15 +355,19 @@ async function dropCustomFunctionProject(req) { if (!(await fs.pathExists(projectDir)) && !(await fs.pathExists(stagingDir))) await fs.stat(projectDir); // Invalidate staged deployments BEFORE the directory goes away, so an activate racing this drop // cannot swap a staged build back into a project that is being deleted. - await invalidateProjectStagedDeployments(project); - await discardProjectStagedApplications(projectDir); - await discardProjectActivationArtifacts(projectDir); + await withComponentPreparationLock( projectDir, async () => { // Retires any interrupted-extraction aside for this component and renames the live directory // aside before removing it, so a concurrent startup recovery can never restore a tree over a // component that was just dropped. + // Inside the lock: an invalidation done before acquiring it leaves a window where a + // concurrent stage completes and is then activated over the just-dropped component. + await invalidateProjectStagedDeployments(project); + await discardProjectStagedApplications(projectDir); + await discardProjectActivationArtifacts(projectDir); + await discardRetainedPrevious(projectDir); await dropComponentDirectory(projectDir, project, log); }, componentDropLockOptions(project) @@ -691,16 +696,6 @@ async function deployComponentOneShot(req, credentialReferences, isReplicatedExe } } -/** - * revert_component — swap a component's live version back to its retained previous version - * (`.deploy-previous/`, kept by the last activate), cluster-wide, then restart. Backs - * customer-driven rollback (deploy → run your own health checks → revert if unhappy) and a - * swap-back after a partially-failed activate. The swap is bidirectional, so reverting a revert - * rolls forward again. - * - * Reached two ways: directly by an operator, and by a peer replaying a replicated revert - * (`_deploymentId` set). deploy_component's `revert_on_failure` path drives it internally. - */ function activationSpecFromRequest(req, credentialReferences) { return { project: req.project, @@ -790,8 +785,8 @@ async function discardDeploymentEverywhere(project, deploymentId, activationSpec .catch(() => {}); } -async function pruneStagedDeploymentArtifacts(project, activationSpec) { - const expired = await expireOldStagedDeployments(project, getStagingRetentionMaxCount()); +async function pruneStagedDeploymentArtifacts(project, activationSpec, keepDeploymentId) { + const expired = await expireOldStagedDeployments(project, getStagingRetentionMaxCount(), keepDeploymentId); for (const deploymentId of expired) await discardDeploymentEverywhere(project, deploymentId, activationSpec); } @@ -799,7 +794,9 @@ async function restartActivatedComponent(req, deploymentId, project, activationS if (req.restart === true) { emit('phase', { phase: 'restart', status: 'start' }); const restartResponse = await server.replication.replicateOperation( - buildPhaseOperation('restart', deploymentId, project, activationSpec) + buildPhaseOperation('restart', deploymentId, project, activationSpec, { + deployment_timeout: req.deployment_timeout, + }) ); const failed = failedPeerResults(restartResponse?.replicated); manageThreads.restartWorkers('http'); @@ -884,7 +881,9 @@ async function deployComponentTwoPhase(req) { if (req.activate === false) { emit('phase', { phase: 'staged', status: 'done' }); await recorder.finish('staged'); - await pruneStagedDeploymentArtifacts(req.project, activationSpec); + await pruneStagedDeploymentArtifacts(req.project, activationSpec, recorder.deploymentId).catch((error) => + log.warn('Failed to prune expired staged deployments', error) + ); await pruneProjectPayloads(req.project, getPayloadRetentionMaxCount()).catch((error) => log.warn('Failed to prune staged deployment payloads', error) ); @@ -910,7 +909,9 @@ async function deployComponentTwoPhase(req) { }); activationCommitted = true; const activateResponse = await server.replication.replicateOperation( - buildPhaseOperation('activate', recorder.deploymentId, req.project, activationSpec), + buildPhaseOperation('activate', recorder.deploymentId, req.project, activationSpec, { + deployment_timeout: req.deployment_timeout, + }), { onPeerResult: (result) => { recorder.recordPeer(result); @@ -1057,7 +1058,9 @@ async function deployComponentActivateExisting(req) { try { const peerResults = []; const activateResponse = await server.replication.replicateOperation( - buildPhaseOperation('activate', req.deployment_id, req.project, spec), + buildPhaseOperation('activate', req.deployment_id, req.project, spec, { + deployment_timeout: req.deployment_timeout, + }), { onPeerResult: (result) => { peerResults.push(result); @@ -1248,43 +1251,26 @@ async function revertComponent(req) { }); try { emit('phase', { phase: 'revert', status: 'start' }); - const result = await revertApplication(application, req.to_deployment_id); - if (result.swapped) { - // Persist the reverted-to version's config + install lock. Compensating by swapping back is - // deliberate: a half-reverted node (new tree live, old config persisted) would reinstall the - // wrong version on its next cold start, which is the failure this pairing exists to prevent. - // - // The transaction is held OUTSIDE the try so the compensation can roll it back. `commit()` makes - // two persistent writes (root config, then the boot-time application lock); if the first - // succeeds and the second throws, swapping the directories back is not enough on its own — - // root config would still name the reverted-to release while the original tree is live again, - // which is the very mismatch this pairing exists to prevent, and a cold start could act on it. - const configTransaction = await createApplicationConfigTransaction(req.project, result.activatedConfig); - try { - await configTransaction.commit(); - } catch (configError) { - const compensationErrors = []; - // Undo the persistent writes first, then the directory swap, so the node is never left with - // the original tree live and the reverted-to configuration persisted. - await configTransaction.rollback().catch((rollbackError) => { - compensationErrors.push(rollbackError); - log.error(`Failed to roll back the ${req.project} revert configuration write`, rollbackError); - }); - await revertApplication(application, result.fromDeploymentId).catch((swapBackError) => { - compensationErrors.push(swapBackError); - log.error(`Failed to undo the ${req.project} revert after its config write failed`, swapBackError); - }); - if (compensationErrors.length) { - throw new ServerError( - `Failed to revert ${req.project}: ${configError?.message ?? configError}. Compensation also ` + - `failed (${compensationErrors.map((error) => error?.message ?? error).join('; ')}), so this ` + - `node may be serving a version its persisted configuration does not name.`, - configError?.statusCode - ); + // The config transaction commits inside revertApplication's component lock, between the directory + // swap and the manifest write, so the directory, the manifest and root config/application-lock + // share one fence. Committing it out here — after the lock released — let a queued activation swap + // and commit in the gap, leaving that activation's bytes live under this revert's config. + // revertApplication compensates the swap if this throws; rolling the partial config write back is + // this transaction's job, since commit() makes two persistent writes and the first can land alone. + let configTransaction; + const result = await revertApplication(application, req.to_deployment_id, { + commitPersistentState: async (activatedConfig) => { + configTransaction = await createApplicationConfigTransaction(req.project, activatedConfig); + try { + await configTransaction.commit(); + } catch (configError) { + await configTransaction.rollback().catch((rollbackError) => { + log.error(`Failed to roll back the ${req.project} revert configuration write`, rollbackError); + }); + throw configError; } - throw configError; - } - } + }, + }); emit('phase', { phase: 'revert', status: 'done' }); // Fan out to peers. The operation is idempotent and target-addressed, so a peer that already @@ -2046,6 +2032,7 @@ async function dropComponent(req) { await invalidateProjectStagedDeployments(project); await discardProjectStagedApplications(componentPath); await discardProjectActivationArtifacts(componentPath); + await discardRetainedPrevious(componentPath); // Retires any interrupted-extraction aside and renames the live tree aside before removing // it, so startup recovery can never restore a tree over a dropped component. await dropComponentDirectory(componentPath, project, log); diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index f9817020ad..0bda442fea 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -22,6 +22,10 @@ const { createApplicationActivationTransaction, revertApplication, getRevertTarget, + discardDirAside, + DISCARDED_ASIDE_PREFIX, + getStagingRetentionMaxCount, + DEFAULT_STAGING_RETENTION_MAX_COUNT, recoverInterruptedReverts, createApplicationConfigTransaction, extractApplication, @@ -63,19 +67,6 @@ async function readMarker(directory) { return fs.readFile(path.join(directory, 'index.js'), 'utf8'); } -// Entries currently parked in a component's `.deploy-aside`. discardDirAside sweeps asynchronously -// (`void cleanupExtractionPaths(...)`) and that sweep rmdir's the directory once it is empty, so an -// existsSync-then-readdir races it. An absent directory means everything was already swept, which -// satisfies every assertion below just as well as an empty one. -async function parkedAsideEntries(componentName) { - try { - return await fs.readdir(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR, componentName)); - } catch (error) { - if (error.code === 'ENOENT') return []; - throw error; - } -} - describe('two-phase component directory transaction', function () { this.timeout(30_000); let sequence = 0; @@ -549,12 +540,10 @@ describe('two-phase component directory transaction', function () { await activateStagedApplication(application, id, { activationSpec: { package: null } }); } - const parked = await parkedAsideEntries(name); - const unretired = parked.filter( - (entry) => - entry.startsWith('.in-progress-') && !parked.includes(`.retired-${entry.slice('.in-progress-'.length)}`) - ); - assert.deepEqual(unretired, [], 'an evicted previous is never left looking like a rollback record'); + assert.match(await readMarker(application.dirPath), /v3/, 'the newest activation is live'); + assert.match(await readMarker(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name)), /v2/); + const target = await getRevertTarget(application.dirPath); + assert.equal(target.previous.deployment_id, ids[1], 'only the immediately-previous version is retained'); await cleanup(name); }); @@ -797,12 +786,6 @@ describe('two-phase component directory transaction', function () { assert.equal((await fs.lstat(application.dirPath)).isSymbolicLink(), true, 'the new version is linked'); assert.equal(await fs.readFile(path.join(application.dirPath, 'marker.txt'), 'utf8'), 'v2'); - // Whatever was displaced must have been parked, not recursively removed in place. It is parked as - // disposable (`.discarded-`), so startup recovery will never restore it over the new version. - const parked = await parkedAsideEntries(name); - const recoverable = parked.filter((entry) => entry.startsWith('.in-progress-')); - assert.deepEqual(recoverable, [], 'the displaced tree is never left looking like a rollback record'); - await cleanup(name); await fs.rm(packageDirectory, { recursive: true, force: true }); }); @@ -1029,8 +1012,10 @@ describe('two-phase component directory transaction', function () { const { application, first, second } = await twoActivations(name); const previousPath = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); const manifestPath = `${previousPath}.json`; - const markerPath = `${previousPath}.recovering.json`; const holding = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, `.reverting-${name}-${randomUUID()}`); + // The marker is named after the holding directory it describes, so an orphan from one attempt can + // never be trusted by a later, unrelated revert of the same component. + const markerPath = `${holding}.recovering.json`; // Crash state: the reverted-to version is live, the displaced tree is still parked. const staleManifest = JSON.parse(await fs.readFile(manifestPath, 'utf8')); @@ -1069,4 +1054,192 @@ describe('two-phase component directory transaction', function () { assert.equal(target.previous.deployment_id, second); await cleanup(name); }); + it('commits config inside the swap, so a revert cannot land config after a later activation', async () => { + const name = fixtureName(); + const { application, first, second } = await twoActivations(name); + const order = []; + + const result = await revertApplication(application, first, { + commitPersistentState: async () => { + // Observed from inside the lock: the reverted-to tree is already live and the displaced tree is + // still parked, which is what makes this the only safe point to persist config. + order.push('commit'); + assert.match(await readMarker(application.dirPath), /v1/, 'reverted-to version is live at commit time'); + assert.equal( + existsSync(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name)), + false, + 'the displaced tree is still in the holding path, not yet retained' + ); + }, + }); + + assert.deepEqual(order, ['commit'], 'the hook ran exactly once'); + assert.equal(result.swapped, true); + assert.equal(result.fromDeploymentId, second); + assert.match(await readMarker(application.dirPath), /v1/); + assert.match(await readMarker(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name)), /v2/); + const target = await getRevertTarget(application.dirPath); + assert.equal(target.live.deployment_id, first); + await cleanup(name); + }); + + it('undoes the swap and leaves nothing revertable-looking when persisting config fails', async () => { + const name = fixtureName(); + const { application, first, second } = await twoActivations(name); + + await assert.rejects( + () => + revertApplication(application, first, { + commitPersistentState: async () => { + throw new Error('simulated config failure'); + }, + }), + /simulated config failure/ + ); + + assert.match(await readMarker(application.dirPath), /v2/, 'the pre-revert version is live again'); + assert.match(await readMarker(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name)), /v1/); + const stranded = (await fs.readdir(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR))).filter((entry) => + entry.startsWith('.reverting-') + ); + assert.deepEqual(stranded, [], 'no holding tree or marker is left behind'); + const target = await getRevertTarget(application.dirPath); + assert.equal(target.live.deployment_id, second, 'the manifest still describes the un-reverted state'); + await cleanup(name); + }); + + it('sweeps a revert marker whose holding directory is gone', async () => { + // A crash between the final rename and the marker removal orphans the marker. Nothing revisits it — + // recovery is keyed on finding a holding directory — so it must be swept, or a later unrelated + // revert of the same component could be resumed against it. + const name = fixtureName(); + const { application, first } = await twoActivations(name); + const orphan = path.join( + COMPONENTS_ROOT, + DEPLOY_PREVIOUS_DIR, + `.reverting-${name}-${randomUUID()}.recovering.json` + ); + await fs.writeFile(orphan, JSON.stringify({ previous: { deployment_id: 'x' }, live: { deployment_id: 'y' } })); + + const failures = await recoverInterruptedReverts(COMPONENTS_ROOT); + + assert.equal(failures.size, 0); + assert.equal(existsSync(orphan), false, 'the orphaned marker is swept'); + // The sweep must not have disturbed the component's actual revert state. + const target = await getRevertTarget(application.dirPath); + assert.equal(target.previous.deployment_id, first, 'the real retained-previous entry is untouched'); + await cleanup(name); + }); + + it('re-points a nested node_modules dependency link, not just top-level ones', async () => { + // npm nests a link under `node_modules//node_modules/` whenever hoisting is blocked by a + // version conflict, and under workspaces routinely. A nested link dangles after the swap exactly + // like a top-level one, and the hard-fail above cannot help if it is never enumerated. + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('nested') }); + const stagedPath = await stageApplication(application, deploymentId); + + await fs.mkdir(path.join(stagedPath, 'vendor', 'deep'), { recursive: true }); + await fs.writeFile(path.join(stagedPath, 'vendor', 'deep', 'index.js'), "module.exports = 'deep';\n"); + const outerPackage = path.join(stagedPath, 'node_modules', 'outer', 'node_modules'); + await fs.mkdir(outerPackage, { recursive: true }); + await fs.symlink(path.join(stagedPath, 'vendor', 'deep'), path.join(outerPackage, 'deep'), 'dir'); + // A scoped package nested one level further down, to prove the recursion handles both shapes. + await fs.mkdir(path.join(outerPackage, '@inner'), { recursive: true }); + await fs.symlink(path.join(stagedPath, 'vendor', 'deep'), path.join(outerPackage, '@inner', 'scoped'), 'dir'); + + await activateStagedApplication(application, deploymentId, { activationSpec: { package: null } }); + + assert.equal( + await fs.readFile( + path.join(application.dirPath, 'node_modules', 'outer', 'node_modules', 'deep', 'index.js'), + 'utf8' + ), + "module.exports = 'deep';\n", + 'the nested link resolves from the live tree' + ); + assert.equal( + await fs.readFile( + path.join(application.dirPath, 'node_modules', 'outer', 'node_modules', '@inner', 'scoped', 'index.js'), + 'utf8' + ), + "module.exports = 'deep';\n", + 'a nested scoped link resolves too' + ); + await cleanup(name); + }); + + it('parks a disposable tree under the discarded prefix, never as a recovery candidate', async () => { + // Asserted directly rather than after an activation: discardDirAside sweeps fire-and-forget, so an + // after-the-fact directory listing passes vacuously whenever the sweep wins the race and cannot + // distinguish correct `.discarded-` parking from a regression that parked as `.in-progress-`. + const name = fixtureName(); + const target = path.join(COMPONENTS_ROOT, `${name}-disposable`); + await fs.mkdir(target, { recursive: true }); + await fs.writeFile(path.join(target, 'index.js'), "module.exports = 'disposable';\n"); + const asideDir = path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR, `${name}-disposable`); + + const parkedSomething = await discardDirAside(target, name); + + assert.equal(parkedSomething, true); + assert.equal(existsSync(target), false, 'the tree is moved out of the way'); + // Read before the detached sweep can remove it; if it already has, there is nothing to misclassify. + const entries = await fs.readdir(asideDir).catch(() => []); + for (const entry of entries) { + assert.equal( + entry.startsWith(DISCARDED_ASIDE_PREFIX), + true, + `parked entry ${entry} must carry the discarded prefix, not a recovery-candidate prefix` + ); + } + assert.equal(await discardDirAside(target, name), false, 'nothing to park the second time'); + await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); + }); + + it('evicts staged builds beyond the retention count and always keeps the newest', async () => { + const name = fixtureName(); + const priorMax = environment.get(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT); + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, 2); + const ids = []; + try { + const application = new Application({ name, payload: await makeComponentPayload('retained') }); + for (let index = 0; index < 4; index++) { + const deploymentId = randomUUID(); + application.payload = await makeComponentPayload(`retained-${index}`); + await stageApplication(application, deploymentId); + ids.push(deploymentId); + } + const surviving = ids.filter((id) => existsSync(stagedApplicationPath(application.dirPath, id))); + assert.equal(surviving.length, 2, `expected exactly 2 staged builds to survive, got ${surviving.length}`); + assert.equal(surviving.includes(ids.at(-1)), true, 'the just-staged build is never evicted'); + } finally { + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, priorMax); + } + await cleanup(name); + }); + + it('falls back to the default staging retention count for unusable configured values', () => { + const prior = environment.get(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT); + try { + for (const value of [undefined, '', ' ', true, [], {}, 'abc', 0, -1]) { + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, value); + assert.equal( + getStagingRetentionMaxCount(), + DEFAULT_STAGING_RETENTION_MAX_COUNT, + `${JSON.stringify(value)} must fall back to the default rather than coerce` + ); + } + for (const [value, expected] of [ + ['3', 3], + [3, 3], + [2.7, 2], + ]) { + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, value); + assert.equal(getStagingRetentionMaxCount(), expected); + } + } finally { + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, prior); + } + }); }); diff --git a/utility/operation_authorization.ts b/utility/operation_authorization.ts index 9e8c7318f0..ef2737c499 100644 --- a/utility/operation_authorization.ts +++ b/utility/operation_authorization.ts @@ -311,8 +311,14 @@ requiredPermissions.set(functionsOperations.addComponent.name, new (permission a requiredPermissions.set(functionsOperations.dropCustomFunctionProject.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.packageComponent.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.deployComponent.name, new (permission as any)(true, [])); -requiredPermissions.set(functionsOperations.componentDeployPhase.name, new (permission as any)(true, [])); -requiredPermissions.set(functionsOperations.revertComponent.name, new (permission as any)(true, [])); +requiredPermissions.set( + functionsOperations.componentDeployPhase.name, + new (permission as any)(true, [], terms.OPERATIONS_ENUM.COMPONENT_DEPLOY_PHASE) +); +requiredPermissions.set( + functionsOperations.revertComponent.name, + new (permission as any)(true, [], terms.OPERATIONS_ENUM.REVERT_COMPONENT) +); requiredPermissions.set( deploymentOperations.handleListDeployments.name, new (permission as any)(true, [], terms.OPERATIONS_ENUM.LIST_DEPLOYMENTS) From c8665916c97e354ce2ce887e298a02bdf574cd63 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 11:50:37 -0400 Subject: [PATCH 70/94] fix(deploy): make the pre-swap link repair reversible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the remaining Codex finding on harper#1849 (Application.ts:2633): moving the dependency-link repair before the swap protected the live release but mutated the staged candidate with no inverse. On rollback the candidate went back to staging with its links still aimed at `application.dirPath` — which by then holds the OLD release — so a retry of the same deployment id would either fail `loadValidateComponent` or, worse, validate the staged tree against the wrong bytes. `repointStagedDependencyLinks` now takes the tree to walk, the target root to write, and the target root the links currently carry, so the activation rollback can aim them back at staging. Writing the test caught a real bug in the first version of that compensation: it walked `application.dirPath`, but by the time the rollback runs the candidate has already been renamed back to the staging path. Test drives a failure after the links were rewritten and the swap landed, then asserts the previous release is live, the rolled-back candidate resolves its dependency from staging, and the same deployment id activates successfully on retry — the regression heskew asked for. 347 deploy/component/CLI/server unit tests pass. --- components/Application.ts | 29 +++++++++++--- unitTests/components/deployStaging.test.js | 46 ++++++++++++++++++++++ 2 files changed, 69 insertions(+), 6 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index b08f8c3cfe..3f4a089195 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -2765,8 +2765,12 @@ async function readStagedCompletion(stagingDirPath: string): Promise<{ installat * whose targets are relative, or absolute but outside the staging tree (a dependency deliberately * linked elsewhere on the machine), are left exactly as they are. */ -async function repointStagedDependencyLinks(stagingDirPath: string, futureLiveDirPath: string): Promise { - const nodeModulesPath = join(stagingDirPath, 'node_modules'); +async function repointStagedDependencyLinks( + treeDirPath: string, + futureLiveDirPath: string, + currentTargetRootPath: string = treeDirPath +): Promise { + const nodeModulesPath = join(treeDirPath, 'node_modules'); // The walk must stay inside the component's own node_modules. `readdir` FOLLOWS symlinks, so a staged // payload shipping `node_modules` (or a `node_modules/@scope`) as a link to somewhere else on the // machine would otherwise have its target's contents enumerated — and a link in there whose target @@ -2809,7 +2813,7 @@ async function repointStagedDependencyLinks(stagingDirPath: string, futureLiveDi // Belt-and-braces containment: never mutate a path that is not under the real node_modules root. const withinNodeModules = relative(nodeModulesPath, linkPath); if (withinNodeModules.startsWith('..') || isAbsolute(withinNodeModules)) continue; - const withinStaging = relative(stagingDirPath, target); + const withinStaging = relative(currentTargetRootPath, target); // `..` or an absolute result means the target is outside the staging tree — not ours to touch. if (!withinStaging || withinStaging.startsWith('..') || isAbsolute(withinStaging)) continue; // NOT best-effort. This link is only being touched because activation is about to invalidate its @@ -2869,6 +2873,7 @@ export async function activateStagedApplication( let backupPath: string | undefined; let newMarkerPath: string | undefined; let swapped = false; + let repointedLinks = 0; try { await hooks.beforeSwap?.(); const existingArtifacts = await activationArtifacts(application.dirPath, deploymentId); @@ -2913,10 +2918,10 @@ export async function activateStagedApplication( // dangling; rewriting them to their future live targets now means a failure lands in the catch // below, which restores the previous release rather than leaving a live component that cannot // resolve its dependencies. Done pre-swap for the compensation, not post-swap for convenience. - const repointed = await repointStagedDependencyLinks(stagingDirPath, application.dirPath); - if (repointed) { + repointedLinks = await repointStagedDependencyLinks(stagingDirPath, application.dirPath); + if (repointedLinks) { logger.debug?.( - `Re-pointed ${repointed} dependency link(s) in ${application.name} from the staging path to the live path` + `Re-pointed ${repointedLinks} dependency link(s) in ${application.name} from the staging path to the live path` ); } await rename(stagingDirPath, application.dirPath); @@ -2931,6 +2936,18 @@ export async function activateStagedApplication( rollbackErrors.push(rollbackError); } } + if (repointedLinks) { + // The candidate is going back to staging, so its links have to point at staging again. Left + // aimed at the live path they would resolve against whatever release is live, so a retry of + // this same deployment id would validate the staged tree against the wrong bytes. + try { + // The candidate now sits at the staging path (moved back above, or never swapped), while its + // links still point at the live path: walk it there, and aim them back at staging. + await repointStagedDependencyLinks(stagingDirPath, stagingDirPath, application.dirPath); + } catch (rollbackError) { + rollbackErrors.push(rollbackError); + } + } if (backupPath) { try { await rename(backupPath, application.dirPath); diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 0bda442fea..9c039d7acd 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -1242,4 +1242,50 @@ describe('two-phase component directory transaction', function () { environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, prior); } }); + it('leaves a rolled-back candidate re-activatable, with its links aimed at staging again', async () => { + // Re-pointing before the swap mutates the staged candidate, so a rollback has to undo it. Left + // aimed at the live path the links would resolve against whatever release is live, and a retry of + // the same deployment id would validate the staged tree against the wrong bytes. + const name = fixtureName(); + const first = randomUUID(); + const second = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('live-v1') }); + await stageApplication(application, first); + await activateStagedApplication(application, first, { activationSpec: { package: null } }); + + application.payload = await makeComponentPayload('candidate-v2'); + const stagedPath = await stageApplication(application, second); + await fs.mkdir(path.join(stagedPath, 'vendor', 'probe'), { recursive: true }); + await fs.writeFile(path.join(stagedPath, 'vendor', 'probe', 'index.js'), "module.exports = 'probe';\n"); + await fs.mkdir(path.join(stagedPath, 'node_modules'), { recursive: true }); + await fs.symlink(path.join(stagedPath, 'vendor', 'probe'), path.join(stagedPath, 'node_modules', 'probe'), 'dir'); + + // Fail after the links were rewritten and the swap landed, so the rollback path runs in full. + await assert.rejects(() => + activateStagedApplication(application, second, { + activationSpec: { package: null }, + beforeCommit: async () => { + throw new Error('simulated persistent-work failure'); + }, + }) + ); + + assert.match(await readMarker(application.dirPath), /live-v1/, 'the previous release is live again'); + // The candidate is back in staging and its dependency resolves from there, not from the live path. + assert.equal( + await fs.readFile(path.join(stagedPath, 'node_modules', 'probe', 'index.js'), 'utf8'), + "module.exports = 'probe';\n", + 'the rolled-back candidate resolves its dependency from staging' + ); + + // And it can still be activated on a retry. + await activateStagedApplication(application, second, { activationSpec: { package: null } }); + assert.match(await readMarker(application.dirPath), /candidate-v2/); + assert.equal( + await fs.readFile(path.join(application.dirPath, 'node_modules', 'probe', 'index.js'), 'utf8'), + "module.exports = 'probe';\n", + 'the retry re-points the links at the live path' + ); + await cleanup(name); + }); }); From f1cf79a1744b6fbb090499bf7264da65488bb899 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 12:17:11 -0400 Subject: [PATCH 71/94] test(deploy): cover the reconciliation attribution guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two cases the previous commit changed behavior for but did not pin: a broken staged candidate is discarded and settled without failing the healthy live component closed, and a failure while settling or removing that candidate is reported but still not attributed to the component — only an interrupted activation is, since only that can leave the live tree and its persisted configuration disagreeing. The second is the one that actually exercises the guard: mutation-verified (widening the attribution fails it), where the first passes on the new non-throwing branch regardless. --- unitTests/components/deployStaging.test.js | 70 ++++++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 9c039d7acd..ca3971067a 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -1288,4 +1288,74 @@ describe('two-phase component directory transaction', function () { ); await cleanup(name); }); + it('discards a broken staged candidate without failing the healthy live component closed', async () => { + // Only a not-yet-activated candidate is broken here; the live tree and its persisted config are + // consistent, so the fail-closed rationale that applies to an interrupted activation does not. + // Failing this component would also be permanent: nothing else sweeps a staging directory whose + // component subtree is missing. + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('live') }); + await stageApplication(application, deploymentId); + await fs.mkdir(application.dirPath, { recursive: true }); + await fs.writeFile(path.join(application.dirPath, 'index.js'), "module.exports = 'live';\n"); + // Break the candidate: remove its component subtree but leave the deployment directory behind. + await fs.rm(stagedApplicationPath(application.dirPath, deploymentId), { recursive: true, force: true }); + + const settled = []; + const reconciliation = await reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async () => ({ + deployment_id: deploymentId, + project: name, + status: 'staged', + activation_spec: { package: null }, + }), + async () => { + throw new Error('activation persistence must not be reached for a staged row'); + }, + async (id) => settled.push(id) + ); + + assert.equal(reconciliation.failedProjects.has(name), false, 'the live component is not failed closed'); + assert.deepEqual(settled, [deploymentId], 'the unusable candidate row is settled so its payload can be reclaimed'); + assert.equal(reconciliation.removed.includes(deploymentId), true, 'and its residue is removed'); + assert.match(await readMarker(application.dirPath), /live/, 'the live component is untouched'); + await cleanup(name); + }); + + it('does not fail a component closed when settling a broken staged candidate itself fails', async () => { + // The staged branch can still throw — an I/O error on the completeness probe, or on settling or + // removing the residue. None of those say the live tree disagrees with its config, so none should + // keep the component from loading; only an interrupted activation does. + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('live') }); + await stageApplication(application, deploymentId); + await fs.mkdir(application.dirPath, { recursive: true }); + await fs.writeFile(path.join(application.dirPath, 'index.js'), "module.exports = 'live';\n"); + await fs.rm(stagedApplicationPath(application.dirPath, deploymentId), { recursive: true, force: true }); + + const reconciliation = await reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async () => ({ + deployment_id: deploymentId, + project: name, + status: 'staged', + activation_spec: { package: null }, + }), + async () => {}, + async () => { + throw new Error('simulated settle failure'); + } + ); + + assert.equal(reconciliation.errors.has(deploymentId), true, 'the failure is still reported'); + assert.equal( + reconciliation.failedProjects.has(name), + false, + 'but it is not attributed to the component, so the healthy live tree still loads' + ); + await cleanup(name); + }); }); From b7d6e0fbe2503d16807a3b744e96b835ec0c7255 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 12:33:10 -0400 Subject: [PATCH 72/94] fix(deploy): pair the revert config commit with a rollback across every later failure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pre-push cross-model review returned BLOCK, and both legs independently found the same hole in the fence added a commit earlier: `commitPersistentState` ran inside the lock, but only a failing *commit* rolled config back. A failure in the manifest write or the retain rename — both of which come after it — undid the directories and left root config and `harper-application-lock.json` naming the reverted-to release while the original bytes were live again, for a cold start to reinstall over. Worse, once the exchanged manifest had landed, `getRevertTarget` reported the reverted-to version as live, so a retry returned the idempotent no-op while the other tree was on disk. Revert now has activation's commit/rollback pairing: one `undoRevert` path takes back the persistent commit, restores the manifest it had exchanged, moves the directories back, and clears the intent marker — in that order, so no window reports the reverted-to version as live over the old bytes. Two more from the same review, both in this round's code: - The no-live restore branch had no intent marker and no compensation at all. A config failure after its rename left the retained slot empty, the persisted state naming an absent version, and no artifact for recovery to find. It now writes its own marker, compensates, and clears it. - `recoverInterruptedReverts` treated any missing live directory as "undo the revert". That shape also occurs when config and the manifest already committed and compensation only got as far as moving live away — undoing there contradicts the persisted state. Recovery now compares the marker against the on-disk manifest and rolls forward when the persistent side already exchanged. Tests: a post-commit failure (injected at the retain rename, after the manifest write) asserts the config rollback ran and that live and the manifest agree afterwards — mutation-verified, since dropping the rollback fails it — plus the roll-forward crash shape, built by hand to the exact on-disk state. Coverage note: gemini failed on quota and the domain adjudication leg exited 1, so this round's outside coverage is codex + cursor-grok only. --- components/Application.ts | 134 ++++++++++++++++----- components/operations.js | 15 ++- unitTests/components/deployStaging.test.js | 64 ++++++++++ 3 files changed, 174 insertions(+), 39 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 3f4a089195..f6aee4bf96 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -499,6 +499,9 @@ const REVERTING_PREFIX = '.reverting-'; // anything and removed only once the directory, manifest and config are all durable, so a recovery // interrupted part-way is resumable rather than losing its own evidence. See recoverInterruptedReverts. const REVERT_RECOVERY_SUFFIX = '.recovering.json'; +// Distinguishes the no-live restore branch's marker from a swap's holding-bound one; there is no +// holding directory in that branch, so the marker needs its own stable name. +const RESTORE_MARKER_INFIX = '.restoring'; // The revert-intent marker is named after the holding directory it describes. A fixed per-component // path would be reused by every future revert of that component, so an orphan left by one attempt @@ -845,8 +848,34 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): } ); if (!liveExists) { + // Two different crash shapes land here, and they need opposite repairs. Either the swap never + // placed the reverted-to version (undo: the holding tree goes back to live), or it did and + // compensation got as far as moving live away before failing (roll forward: the reverted-to + // tree in `previous` goes to live, because config and the manifest already describe it). + // The marker distinguishes them: if the on-disk manifest already matches its intended `live`, + // the persistent side committed and undoing the directories would contradict it. + const marker = await readRevertRecoveryMarker(revertRecoveryMarkerFor(holding)); + const manifest = await readRetainedPreviousManifest(liveDirPath); + const persistedAlreadyExchanged = + !!marker && + !!manifest && + manifest.live?.deployment_id === marker.live?.deployment_id && + manifest.previous?.deployment_id === marker.previous?.deployment_id; + const previousStat = await lstat(previousDirPathFor(liveDirPath)).catch(() => undefined); + if (persistedAlreadyExchanged && previousStat?.isDirectory()) { + await mkdir(dirname(liveDirPath), { recursive: true }); + await rename(previousDirPathFor(liveDirPath), liveDirPath); + await rename(holding, previousDirPathFor(liveDirPath)); + await rm(revertRecoveryMarkerFor(holding), { force: true }); + logger.warn( + `Completed an interrupted ${componentName} revert whose compensation had already begun; ` + + `the reverted-to version is live and matches its persisted configuration` + ); + return; + } await mkdir(dirname(liveDirPath), { recursive: true }); await rename(holding, liveDirPath); + await rm(revertRecoveryMarkerFor(holding), { force: true }); logger.warn( `Restored the live ${componentName} component directory after an interrupted revert; ` + `the revert did not take effect and can be retried` @@ -961,7 +990,10 @@ export async function getRevertTarget( export async function revertApplication( application: Application, toDeploymentId: string, - hooks: { commitPersistentState?: (config: ApplicationConfig | null) => Promise } = {} + hooks: { + commitPersistentState?: (config: ApplicationConfig | null) => Promise; + rollbackPersistentState?: () => Promise; + } = {} ): Promise<{ swapped: boolean; activatedConfig: ApplicationConfig | null; fromDeploymentId: string | null }> { const liveDirPath = application.dirPath; return withComponentPreparationLock(liveDirPath, async () => { @@ -1033,57 +1065,97 @@ export async function revertApplication( // Persistent state commits here, inside the lock and while the holding tree still exists. // Committing it after the lock released let a queued activation swap and commit in between, // leaving that activation's bytes live under this revert's config. - try { - await hooks.commitPersistentState?.(target.previous.application_config); - application.useLiveBuildDir(); - await writeRetainedPreviousManifest(liveDirPath, { - previous: target.live, - live: target.previous, - }); - } catch (persistError) { + // + // Everything from the commit to the final rename shares one undo path. A failure after the + // commit — a manifest write, or the retain rename — has to take config and the application + // lock back too, or the directories end up describing one release while the persisted state + // names the other, and a cold start reinstalls over the live bytes. + let persistCommitted = false; + let manifestWritten = false; + const undoRevert = async (cause: unknown, detail: string): Promise => { + const undoErrors: unknown[] = []; + if (persistCommitted) { + await hooks.rollbackPersistentState?.().catch((rollbackError) => undoErrors.push(rollbackError)); + } + if (manifestWritten) { + // The manifest currently claims the exchange happened. Put it back before the directories + // move, so no window reports the reverted-to version as live over the old bytes. + await writeRetainedPreviousManifest(liveDirPath, { + previous: target.previous, + live: target.live, + }).catch((manifestError) => undoErrors.push(manifestError)); + } try { await rename(liveDirPath, previousPath); await rename(holding, liveDirPath); + application.useLiveBuildDir(); await rm(recoveryMarkerPath, { force: true }); } catch (restoreError) { + undoErrors.push(restoreError); + } + if (undoErrors.length) { throw new AggregateError( - [persistError, restoreError], - `Reverted ${application.name} but could not persist its configuration or undo the swap; ` + - `${holding} still holds the previously-live tree` + [cause, ...undoErrors], + `Reverted ${application.name} but could not ${detail}, and could not fully undo it; ` + + `${holding} may still hold the previously-live tree` ); } - throw persistError; + throw cause; + }; + try { + await hooks.commitPersistentState?.(target.previous.application_config); + persistCommitted = true; + application.useLiveBuildDir(); + await writeRetainedPreviousManifest(liveDirPath, { + previous: target.live, + live: target.previous, + }); + manifestWritten = true; + } catch (persistError) { + await undoRevert(persistError, 'persist its configuration'); } // Consumes the recovery evidence, so it goes last. try { await rename(holding, previousPath); } catch (retainError) { - // The revert itself succeeded (the requested version IS live); only retaining the tree it - // displaced failed. Undo the whole swap rather than leave the manifest describing a - // retained previous that is not where it says it is. - try { - await rename(liveDirPath, previousPath); - await rename(holding, liveDirPath); - } catch (restoreError) { - throw new AggregateError( - [retainError, restoreError], - `Reverted ${application.name} but could not retain the displaced version, and could not ` + - `undo the swap; ${holding} still holds the previously-live tree` - ); - } - throw retainError; + await undoRevert(retainError, 'retain the displaced version'); } await rm(recoveryMarkerPath, { force: true }); } else { // No live version to preserve; restore the previous into place. Nothing becomes the new // retained previous, so the component can't be reverted again until its next deploy. - await rename(previousPath, liveDirPath); - await hooks.commitPersistentState?.(target.previous.application_config); - application.useLiveBuildDir(); - await writeRetainedPreviousManifest(liveDirPath, { + // + // This branch gets the same intent marker and undo as the swap above: without them a config + // failure after the rename left the retained slot empty, the persisted state naming an absent + // version, and no `.reverting-*` artifact for recovery to find. + const restoreMarkerPath = revertRecoveryMarkerFor(`${previousPath}${RESTORE_MARKER_INFIX}`); + await writeJsonAtomically(restoreMarkerPath, { previous: { deployment_id: null, application_config: null }, live: target.previous, }); + await rename(previousPath, liveDirPath); + try { + await hooks.commitPersistentState?.(target.previous.application_config); + application.useLiveBuildDir(); + await writeRetainedPreviousManifest(liveDirPath, { + previous: { deployment_id: null, application_config: null }, + live: target.previous, + }); + } catch (persistError) { + const undoErrors: unknown[] = []; + await hooks.rollbackPersistentState?.().catch((rollbackError) => undoErrors.push(rollbackError)); + await rename(liveDirPath, previousPath).catch((restoreError) => undoErrors.push(restoreError)); + await rm(restoreMarkerPath, { force: true }).catch(() => {}); + if (undoErrors.length) { + throw new AggregateError( + [persistError, ...undoErrors], + `Restored ${application.name} from its retained version but could not persist configuration ` + + `or undo the restore` + ); + } + throw persistError; + } + await rm(restoreMarkerPath, { force: true }); } return { swapped: true, diff --git a/components/operations.js b/components/operations.js index a852079f0e..f6f259d1c7 100644 --- a/components/operations.js +++ b/components/operations.js @@ -1261,14 +1261,13 @@ async function revertComponent(req) { const result = await revertApplication(application, req.to_deployment_id, { commitPersistentState: async (activatedConfig) => { configTransaction = await createApplicationConfigTransaction(req.project, activatedConfig); - try { - await configTransaction.commit(); - } catch (configError) { - await configTransaction.rollback().catch((rollbackError) => { - log.error(`Failed to roll back the ${req.project} revert configuration write`, rollbackError); - }); - throw configError; - } + await configTransaction.commit(); + }, + // Called for any failure after the commit landed, not just a failing commit: the manifest write + // and the retain rename come after it, and undoing only the directories would leave root config + // and the application lock naming the release that is no longer live. + rollbackPersistentState: async () => { + await configTransaction?.rollback(); }, }); emit('phase', { phase: 'revert', status: 'done' }); diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index ca3971067a..4184589a3d 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -1358,4 +1358,68 @@ describe('two-phase component directory transaction', function () { ); await cleanup(name); }); + it('rolls config back when a failure lands after the config commit, not just during it', async () => { + // The commit is followed by the manifest write and the retain rename. A failure in either used to + // undo only the directories, leaving root config and the application lock naming the reverted-to + // release while the original bytes were live again — which a cold start would then reinstall over. + const name = fixtureName(); + const { application, first, second } = await twoActivations(name); + let committed = 0; + let rolledBack = 0; + + // Fail the retain rename by making the retained-previous slot unwritable at that moment: the + // manifest write has already succeeded, so this is strictly post-commit. + const previousRoot = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR); + await assert.rejects(() => + revertApplication(application, first, { + commitPersistentState: async () => { + committed++; + await fs.chmod(previousRoot, 0o500); + }, + rollbackPersistentState: async () => { + rolledBack++; + await fs.chmod(previousRoot, 0o700); + }, + }) + ); + await fs.chmod(previousRoot, 0o700).catch(() => {}); + + assert.equal(committed, 1, 'the commit ran'); + assert.equal(rolledBack, 1, 'and a post-commit failure rolled it back'); + assert.match(await readMarker(application.dirPath), /v2/, 'the pre-revert version is live again'); + const target = await getRevertTarget(application.dirPath); + assert.equal(target.live.deployment_id, second, 'the manifest was put back too, so live/manifest agree'); + assert.equal(target.previous.deployment_id, first); + await cleanup(name); + }); + + it('rolls forward an interrupted revert whose compensation had already moved live away', async () => { + // Crash shape: config and the manifest committed, then compensation renamed live away before it + // could put the holding tree back. Undoing the directories here would contradict the persisted + // state, so recovery has to finish the revert instead — decided by comparing marker to manifest. + const name = fixtureName(); + const { application, first, second } = await twoActivations(name); + const previousPath = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); + const holding = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, `.reverting-${name}-${randomUUID()}`); + + // Build that exact state by hand: live absent, previous = reverted-to tree, holding = old live, + // manifest already exchanged, and a marker recording the same intent. + const staleManifest = JSON.parse(await fs.readFile(`${previousPath}.json`, 'utf8')); + const intended = { previous: staleManifest.live, live: staleManifest.previous }; + await fs.rename(application.dirPath, holding); + await fs.writeFile(`${holding}.recovering.json`, JSON.stringify(intended, null, 2)); + await fs.writeFile(`${previousPath}.json`, JSON.stringify(intended, null, 2)); + + const failures = await recoverInterruptedReverts(COMPONENTS_ROOT); + + assert.equal(failures.size, 0); + assert.match(await readMarker(application.dirPath), /v1/, 'the reverted-to version is live, matching config'); + assert.match(await readMarker(previousPath), /v2/, 'the displaced tree is retained'); + assert.equal(existsSync(holding), false); + assert.equal(existsSync(`${holding}.recovering.json`), false); + const target = await getRevertTarget(application.dirPath); + assert.equal(target.live.deployment_id, first); + assert.equal(target.previous.deployment_id, second); + await cleanup(name); + }); }); From f911f54c3808169ec956c3acfea30528d2bee225 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 15:25:21 -0400 Subject: [PATCH 73/94] fix(deploy): serialize persistent-state writes and fail startup closed on unreadable recovery roots MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review majors plus their symmetry follow-on. Cross-worker lost updates: the root config and the application lock were snapshotted and written outside any lock, so two workers committing config transactions could interleave and drop one project's lock entry. Both writes and the snapshot they are derived from now run inside withPersistentStateLock, keyed on the lock file path — a filesystem lock, so it holds across processes, not just within an isolate. The file lock is not reentrant, hence the locked/unlocked split; lock ordering is always component then persistent-state. Recovery scans that cannot enumerate now fail startup closed rather than loading every component over unreconciled state, since a failed scan cannot say WHICH components are affected. Applied to all three passes: extraction, reverts, and staged artifacts. A missing root is not this case — each scan returns no work on ENOENT — so cold starts are unaffected, and the two new tests pin exactly that distinction because the loader's fail-closed depends on it. Also: a lookup failure in the staged-artifact loop no longer reads as absence and authorizes deleting a retained previous release, and request booleans arriving as strings are normalized for deploy_component and revert_component. --- components/Application.ts | 74 +++++++++++++--- components/componentLoader.ts | 19 ++-- components/operations.js | 14 +++ unitTests/components/deployStaging.test.js | 87 +++++++++++++++++++ .../components/extractApplicationSwap.test.js | 17 ++++ 5 files changed, 192 insertions(+), 19 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index f6aee4bf96..d446c7b68d 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -3210,8 +3210,11 @@ export async function reconcileStagedApplicationArtifacts( } catch (error) { const reconcileError = error instanceof Error ? error : new Error(String(error)); errors.set(entry.name, reconcileError); - // Fail the component closed ONLY for an interrupted activation, where the live tree and its - // durable configuration can disagree. A staged-candidate failure leaves live state consistent. + // Fail the component closed for an interrupted activation, where the live tree and its durable + // configuration can disagree. A confirmed `staged` failure leaves live state consistent, so it + // does not. When the lookup itself failed there is no project name to attribute to — but nothing + // destructive has run for this entry either, and a non-empty `errors` keeps the reconcile guard + // unset so the next reload cycle retries. if (row?.status === 'activating' && safeComponentName(row?.project)) { failedProjects.set(row.project, reconcileError); } @@ -3233,7 +3236,16 @@ export async function reconcileStagedApplicationArtifacts( await rm(artifactPath, { recursive: true, force: true }); continue; } - const row = await getDeployment(deploymentId).catch(() => undefined); + let row: Record | undefined; + try { + row = await getDeployment(deploymentId); + } catch (lookupError) { + // Cannot tell absent from unreadable, so destroy nothing and fail the component closed. + const reconcileError = lookupError instanceof Error ? lookupError : new Error(String(lookupError)); + errors.set(deploymentId, reconcileError); + failedProjects.set(projectEntry.name, reconcileError); + continue; + } const livePath = join(componentsRootDirPath, projectEntry.name); const liveStat = await lstat(livePath).catch(() => undefined); if (row?.status === 'activating' && row.project === projectEntry.name && liveStat?.isDirectory()) { @@ -3380,6 +3392,27 @@ export async function installApplications() { // in-memory state rather than a stale snapshot silently clobbering a sibling's just-written change. const applicationLockWriteQueues = new Map>(); +const PERSISTENT_STATE_LOCK_PURPOSE = 'application-persistent-state'; + +function persistentStateLockPath(): string { + return join(getConfigValue(CONFIG_PARAMS.ROOTPATH), 'harper-application-lock.json'); +} + +/** + * Serialize a root-config + application-lock read-modify-write across every isolate and process. The + * snapshot and both files have to sit inside one critical section: the entries are per-project but the + * files are shared, so an unsynchronized read-modify-write drops a sibling project's entry. + * + * Never call this while already holding it — the file lock is not reentrant. The `*Unlocked` variants + * exist for callers that are already inside it. + */ +async function withPersistentStateLock(operation: () => Promise): Promise { + return withComponentPreparationLock(persistentStateLockPath(), operation, { + purpose: PERSISTENT_STATE_LOCK_PURPOSE, + timeoutMs: 30_000, + }); +} + async function persistApplicationLock( harperApplicationLockPath: string, harperApplicationLock: { applications: Record } @@ -3426,6 +3459,13 @@ async function readApplicationLock(lockPath: string): Promise<{ applications: Re export async function updateApplicationLockEntry( name: string, applicationConfig: ApplicationConfig | undefined +): Promise { + return withPersistentStateLock(() => updateApplicationLockEntryUnlocked(name, applicationConfig)); +} + +async function updateApplicationLockEntryUnlocked( + name: string, + applicationConfig: ApplicationConfig | undefined ): Promise { const lockPath = join(getConfigValue(CONFIG_PARAMS.ROOTPATH), 'harper-application-lock.json'); const previous = applicationLockWriteQueues.get(lockPath) ?? Promise.resolve(); @@ -3443,7 +3483,7 @@ export async function updateApplicationLockEntry( await next; } -async function getApplicationLockEntry(name: string): Promise { +async function getApplicationLockEntryUnlocked(name: string): Promise { const lockPath = join(getConfigValue(CONFIG_PARAMS.ROOTPATH), 'harper-application-lock.json'); const previous = applicationLockWriteQueues.get(lockPath) ?? Promise.resolve(); let entry: ApplicationConfig | undefined; @@ -3478,19 +3518,25 @@ export async function createApplicationConfigTransaction( return { async commit() { if (commitStarted) return; - previousConfig = readConfigFile()?.[project]; - previousLockConfig = await getApplicationLockEntry(project); - commitStarted = true; - if (nextConfig === null) deleteConfigFromFile([project]); - else await addConfig(project, nextConfig); - await updateApplicationLockEntry(project, nextConfig ?? undefined); + // Snapshot and both writes inside one critical section, so a concurrent activation of a different + // project cannot land between the read and the write and lose one of the two entries. + await withPersistentStateLock(async () => { + previousConfig = readConfigFile()?.[project]; + previousLockConfig = await getApplicationLockEntryUnlocked(project); + commitStarted = true; + if (nextConfig === null) deleteConfigFromFile([project]); + else await addConfig(project, nextConfig); + await updateApplicationLockEntryUnlocked(project, nextConfig ?? undefined); + }); }, async rollback() { if (!commitStarted) return; - if (previousConfig === undefined) deleteConfigFromFile([project]); - else await addConfig(project, previousConfig); - await updateApplicationLockEntry(project, previousLockConfig); - commitStarted = false; + await withPersistentStateLock(async () => { + if (previousConfig === undefined) deleteConfigFromFile([project]); + else await addConfig(project, previousConfig); + await updateApplicationLockEntryUnlocked(project, previousLockConfig); + commitStarted = false; + }); }, }; } diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 662c3f79b3..34bf4f0f22 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -172,15 +172,19 @@ export async function loadComponentDirectories( // decides whether to roll an activation forward. The three passes read disjoint directories // (`.deploy-aside`, `.deploy-previous`, `.deploy-staging`/`.deploy-activating`), so none can claim // another's artifacts — see DESIGN.md, "One contract on .deploy-aside". - let failedRecoveries = new Map(); + let failedRecoveries: Map; try { failedRecoveries = await recoverInterruptedComponentExtractions(CF_ROUTES_DIR); } catch (error) { - const recoveryError = error instanceof Error ? error : new Error(String(error)); - harperLogger.warn( - 'Loading existing filesystem components without deploy recovery because staging could not be inspected:', - errorForLog(recoveryError) + // The scan itself failed, so we cannot tell WHICH components are mid-extraction and cannot fail just + // those closed. A missing staging root is not this case — it returns no work — so reaching here means + // the root is unreadable or is not a directory, and any component may hold a half-extracted tree + // whose rollback record we could not read. Abort startup, the same way both scans below do. + harperLogger.error( + 'Could not inspect component deploy staging for interrupted extractions:', + errorForLog(error as Error) ); + throw error; } if (isMainThread) { // A revert that died between its renames can leave a component with no live directory at all, with @@ -192,10 +196,15 @@ export async function loadComponentDirectories( if (!failedRecoveries.has(component)) failedRecoveries.set(component, error); } } catch (error) { + // The scan itself failed, so we cannot tell WHICH components are mid-revert and cannot fail just + // those closed. An interrupted revert can have the reverted-to bytes live with config not yet + // committed, so loading everything over unreconciled state is not sound — abort startup instead, + // the same way an unreadable staged-artifact scan does below. harperLogger.error( 'Could not inspect retained component versions for interrupted reverts:', errorForLog(error as Error) ); + throw error; } } if (isMainThread && !stagedArtifactsReconciled) { diff --git a/components/operations.js b/components/operations.js index f6f259d1c7..c4fb5283aa 100644 --- a/components/operations.js +++ b/components/operations.js @@ -445,6 +445,7 @@ async function packageComponent(req) { * @returns {Promise} */ async function deployComponent(req) { + normalizeRequestBooleans(req); if (req.project) { req.project = path.parse(req.project).name; } else if (req.package) { @@ -696,6 +697,18 @@ async function deployComponentOneShot(req, credentialReferences, isReplicatedExe } } +const REQUEST_BOOLEAN_FIELDS = ['activate', 'two_phase', 'ignore_replication_errors', 'force', 'restart']; + +function normalizeRequestBooleans(req) { + for (const field of REQUEST_BOOLEAN_FIELDS) { + const value = req[field]; + if (typeof value !== 'string') continue; + const lowered = value.trim().toLowerCase(); + if (lowered === 'true') req[field] = true; + else if (lowered === 'false') req[field] = false; + } +} + function activationSpecFromRequest(req, credentialReferences) { return { project: req.project, @@ -1222,6 +1235,7 @@ function isTrustedReplicatedOperation(req) { * the restored directory on the next cold start — undoing the rollback. */ async function revertComponent(req) { + normalizeRequestBooleans(req); if (req.project) req.project = path.parse(req.project).name; const validation = validator.revertComponentValidator(req); if (validation) throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 4184589a3d..65b08b9089 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -36,6 +36,7 @@ const { ASIDE_STAGING_DIR, } = require('#src/components/Application'); const { getConfigPath, readConfigFile } = require('#src/config/configUtils'); +const { withComponentPreparationLock } = require('#src/components/componentPreparationLock'); const { CONFIG_PARAMS } = require('#src/utility/hdbTerms'); const environment = require('#src/utility/environment/environmentManager'); @@ -1422,4 +1423,90 @@ describe('two-phase component directory transaction', function () { assert.equal(target.previous.deployment_id, second); await cleanup(name); }); + it('serializes root-config and application-lock writes on a lock outside this isolate', async () => { + // The in-isolate write queue only orders writers within one isolate, but peer phases run in worker + // threads and different projects take different component locks — so an unsynchronized + // read-modify-write of the shared lock file drops a sibling project's entry. The transaction now + // takes a filesystem lock keyed on the lock file itself, which is what makes it cross-isolate. + // + // Holding that same lock from outside proves the critical section exists: a commit cannot proceed + // while it is held. A same-isolate concurrency test could NOT demonstrate this — the in-isolate + // queue masks the interleaving, which is exactly why the file lock was needed. + const name = fixtureName(); + const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-persist-lock-')); + const priorRootEnv = process.env.ROOTPATH; + const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); + process.env.ROOTPATH = configRoot; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); + await fs.writeFile(path.join(configRoot, 'harper-config.yaml'), `rootPath: ${JSON.stringify(configRoot)}\n`); + const lockPath = path.join(configRoot, 'harper-application-lock.json'); + try { + const transaction = await createApplicationConfigTransaction(name, { package: 'example@1.0.0' }); + let committed = false; + let releaseHold; + const holdReleased = new Promise((resolve) => (releaseHold = resolve)); + + // The holder waits on an external signal, and the commit is started OUTSIDE it. Awaiting the + // commit from inside would deadlock: the lock is not reentrant, so the holder could not release + // until the commit finished and the commit could not start until the holder released. + const holder = withComponentPreparationLock(lockPath, () => holdReleased); + await new Promise((resolve) => setTimeout(resolve, 100)); + const commitPromise = transaction.commit().then(() => (committed = true)); + await new Promise((resolve) => setTimeout(resolve, 200)); + assert.equal(committed, false, 'the commit must wait while the persistent-state lock is held'); + + releaseHold(); + await holder; + await commitPromise; + assert.equal(committed, true, 'and proceed once it is released'); + assert.deepEqual(readConfigFile()[name], { package: 'example@1.0.0' }); + const lock = JSON.parse(await fs.readFile(lockPath, 'utf8')); + assert.deepEqual(lock.applications[name], { package: 'example@1.0.0' }); + } finally { + if (priorRootEnv === undefined) delete process.env.ROOTPATH; + else process.env.ROOTPATH = priorRootEnv; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); + await fs.rm(configRoot, { recursive: true, force: true }); + } + }); + + it("keeps a sibling project's application-lock entry across a second project's transaction", async () => { + const first = fixtureName(); + const second = fixtureName(); + const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-persist-sibling-')); + const priorRootEnv = process.env.ROOTPATH; + const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); + process.env.ROOTPATH = configRoot; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); + await fs.writeFile(path.join(configRoot, 'harper-config.yaml'), `rootPath: ${JSON.stringify(configRoot)}\n`); + try { + const a = await createApplicationConfigTransaction(first, { package: 'a@1.0.0' }); + const b = await createApplicationConfigTransaction(second, { package: 'b@1.0.0' }); + await Promise.all([a.commit(), b.commit()]); + + const lock = JSON.parse(await fs.readFile(path.join(configRoot, 'harper-application-lock.json'), 'utf8')); + assert.deepEqual(lock.applications[first], { package: 'a@1.0.0' }, 'the first entry survives'); + assert.deepEqual(lock.applications[second], { package: 'b@1.0.0' }, 'and so does the second'); + } finally { + if (priorRootEnv === undefined) delete process.env.ROOTPATH; + else process.env.ROOTPATH = priorRootEnv; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); + await fs.rm(configRoot, { recursive: true, force: true }); + } + }); + + // Same contract the extraction scan owes its caller: componentLoader fails startup closed on a + // rejection, so an absent retention root (nothing ever retained) must stay distinguishable from one + // we could not read, where a component may be mid-revert with no live directory at all. + it('resolves empty when the retention root is absent but rejects when it is unreadable', async () => { + const scanRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'revert-scan-')); + try { + assert.equal((await recoverInterruptedReverts(scanRoot)).size, 0); + + await fs.writeFile(path.join(scanRoot, DEPLOY_PREVIOUS_DIR), 'not a directory\n'); + await assert.rejects(() => recoverInterruptedReverts(scanRoot), { code: 'ENOTDIR' }); + } finally { + await fs.rm(scanRoot, { recursive: true, force: true }); + } + }); }); diff --git a/unitTests/components/extractApplicationSwap.test.js b/unitTests/components/extractApplicationSwap.test.js index 4081f81822..ee566638df 100644 --- a/unitTests/components/extractApplicationSwap.test.js +++ b/unitTests/components/extractApplicationSwap.test.js @@ -872,4 +872,21 @@ describe('extractApplication directory swap', () => { await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); await fs.rm(sourceDir, { recursive: true, force: true }); }); + + // componentLoader fails startup closed when this scan rejects, so the two outcomes have to stay + // distinguishable: an absent staging root is a fresh install with no work, while an unreadable one + // means some component may hold a half-extracted tree we could not see. + it('resolves empty when the staging root is absent but rejects when it is unreadable', async () => { + const componentsRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'extract-swap-scan-')); + + assert.strictEqual((await recoverInterruptedComponentExtractions(componentsRoot)).size, 0); + + await fs.writeFile(path.join(componentsRoot, '.deploy-aside'), 'not a directory\n'); + await assert.rejects( + () => recoverInterruptedComponentExtractions(componentsRoot), + /staging path is not a directory/ + ); + + await fs.rm(componentsRoot, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); + }); }); From 7742810647293dcd7b89bde16f35a766b1e4acbd Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 15:34:00 -0400 Subject: [PATCH 74/94] fix(deploy): hold the component lock across startup activation-artifact reconciliation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `.deploy-activating` sweep renamed a backup back over the live path without taking the component preparation lock, unlike every other mutator of that path. Because the reconcile guard is only retired on a clean pass, a node whose first pass errored re-runs the sweep on every reload cycle, and reload cycles are not serialized against an activation. Landing inside a swap window — live momentarily absent, backup present, row still activating — the sweep took the restore-backup branch, after which the in-flight rename failed ENOTEMPTY and its compensating rename failed ENOENT, leaving a half-activated node. The sweep now runs under `withComponentPreparationLock` on the live path. It waits out a live owner rather than deferring, so its decisions are read from settled state instead of a half-finished swap, and no artifacts are left behind for a later cycle. Two decisions in the staged loop had the same shape — a completeness probe evaluated unlocked, then acted on — so both re-read under the lock before discarding a broken candidate or persisting an interrupted activation. Where a complete candidate has appeared by then, an activation owns the artifacts and reconciliation leaves them alone. Reported by Barber AI on #1849. --- components/Application.ts | 115 +++++++++++++-------- unitTests/components/deployStaging.test.js | 47 +++++++++ 2 files changed, 117 insertions(+), 45 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index d446c7b68d..195a98611b 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -3178,16 +3178,23 @@ export async function reconcileStagedApplicationArtifacts( const stagedPath = stagedApplicationPath(componentDirPath, entry.name); if (row.status === 'staged') { if (!(await hasCompleteStagedApplication(stagedPath))) { - // Only a not-yet-activated candidate is broken; the live tree and its persisted config are - // consistent. Failing the component closed here would take a healthy component offline and - // keep it offline, because nothing else sweeps a staging directory whose subtree is missing. - logger.warn( - `Discarding staged deployment '${entry.name}' for '${row.project}': no valid component tree. ` + - `The live component is unaffected.` - ); - await settleStagedDeployment?.(entry.name); - await rm(deploymentPath, { recursive: true, force: true }); - removed.push(entry.name); + let discarded = false; + await withComponentPreparationLock(componentDirPath, async () => { + // Re-read under the lock. The check above is an unlocked fast path, so an activation may + // have swapped this candidate in and cleaned it up since. + if (await hasCompleteStagedApplication(stagedPath)) return; + // Only a not-yet-activated candidate is broken; the live tree and its persisted config are + // consistent. Failing the component closed here would take a healthy component offline and + // keep it offline, because nothing else sweeps a staging directory whose subtree is missing. + logger.warn( + `Discarding staged deployment '${entry.name}' for '${row.project}': no valid component tree. ` + + `The live component is unaffected.` + ); + await settleStagedDeployment?.(entry.name); + await rm(deploymentPath, { recursive: true, force: true }); + discarded = true; + }); + if (discarded) removed.push(entry.name); } continue; } @@ -3199,11 +3206,21 @@ export async function reconcileStagedApplicationArtifacts( activationSpec: row.activation_spec, }); } else { - const liveStat = await lstat(componentDirPath).catch(() => undefined); - if (!liveStat?.isDirectory() || liveStat.isSymbolicLink()) { - throw new Error(`Interrupted activation '${entry.name}' has neither a staged nor live component tree`); - } - await persistActivation(row); + let ownedByActivation = false; + await withComponentPreparationLock(componentDirPath, async () => { + // Re-read under the lock for the same reason as the `staged` branch above: if a complete + // candidate is here now, an in-flight activation owns these artifacts and will settle them. + if (await hasCompleteStagedApplication(stagedPath)) { + ownedByActivation = true; + return; + } + const liveStat = await lstat(componentDirPath).catch(() => undefined); + if (!liveStat?.isDirectory() || liveStat.isSymbolicLink()) { + throw new Error(`Interrupted activation '${entry.name}' has neither a staged nor live component tree`); + } + await persistActivation(row); + }); + if (ownedByActivation) continue; } await removeActivationArtifacts(componentDirPath, entry.name); recovered.add(entry.name); @@ -3229,41 +3246,49 @@ export async function reconcileStagedApplicationArtifacts( await rm(projectPath, { recursive: true, force: true }); continue; } - for (const artifact of await readdir(projectPath, { withFileTypes: true })) { - const deploymentId = activationArtifactDeploymentId(artifact.name); - const artifactPath = join(projectPath, artifact.name); - if (!deploymentId) { - await rm(artifactPath, { recursive: true, force: true }); - continue; - } - let row: Record | undefined; - try { - row = await getDeployment(deploymentId); - } catch (lookupError) { - // Cannot tell absent from unreadable, so destroy nothing and fail the component closed. - const reconcileError = lookupError instanceof Error ? lookupError : new Error(String(lookupError)); - errors.set(deploymentId, reconcileError); - failedProjects.set(projectEntry.name, reconcileError); - continue; - } - const livePath = join(componentsRootDirPath, projectEntry.name); - const liveStat = await lstat(livePath).catch(() => undefined); - if (row?.status === 'activating' && row.project === projectEntry.name && liveStat?.isDirectory()) { - try { - await persistActivation(row); + const livePath = join(componentsRootDirPath, projectEntry.name); + // This sweep renames a backup back over the live path, so it must hold the lock every other + // mutator of that path holds. A reload cycle can retry reconciliation while an activation is + // mid-swap, and an unlocked sweep would restore the backup underneath it — after which the + // in-flight rename fails ENOTEMPTY and its own compensation fails ENOENT. Waiting out a live + // owner is what makes the decisions below sound: they are read from settled state, not a + // half-finished swap. + await withComponentPreparationLock(livePath, async () => { + for (const artifact of await readdir(projectPath, { withFileTypes: true })) { + const deploymentId = activationArtifactDeploymentId(artifact.name); + const artifactPath = join(projectPath, artifact.name); + if (!deploymentId) { await rm(artifactPath, { recursive: true, force: true }); - recovered.add(deploymentId); - } catch (error) { - const reconcileError = error instanceof Error ? error : new Error(String(error)); + continue; + } + let row: Record | undefined; + try { + row = await getDeployment(deploymentId); + } catch (lookupError) { + // Cannot tell absent from unreadable, so destroy nothing and fail the component closed. + const reconcileError = lookupError instanceof Error ? lookupError : new Error(String(lookupError)); errors.set(deploymentId, reconcileError); failedProjects.set(projectEntry.name, reconcileError); + continue; + } + const liveStat = await lstat(livePath).catch(() => undefined); + if (row?.status === 'activating' && row.project === projectEntry.name && liveStat?.isDirectory()) { + try { + await persistActivation(row); + await rm(artifactPath, { recursive: true, force: true }); + recovered.add(deploymentId); + } catch (error) { + const reconcileError = error instanceof Error ? error : new Error(String(error)); + errors.set(deploymentId, reconcileError); + failedProjects.set(projectEntry.name, reconcileError); + } + } else if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && !liveStat) { + await rename(artifactPath, livePath); + } else { + await rm(artifactPath, { recursive: true, force: true }); } - } else if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && !liveStat) { - await rename(artifactPath, livePath); - } else { - await rm(artifactPath, { recursive: true, force: true }); } - } + }); await rmdir(projectPath).catch(() => {}); } await rmdir(activationRoot).catch(() => {}); diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 65b08b9089..98bff3d745 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -1495,6 +1495,53 @@ describe('two-phase component directory transaction', function () { } }); + it('waits for the component lock before restoring an activation backup over the live path', async () => { + // The sweep renames a backup back over the live path. Run unlocked, a reload-cycle retry could do + // that inside an activation's swap window — live momentarily absent, backup present — after which + // the in-flight rename fails ENOTEMPTY and its own compensation fails ENOENT. So the sweep has to + // hold the same lock every other mutator of that path holds. + const name = fixtureName(); + const deploymentId = randomUUID(); + const livePath = path.join(COMPONENTS_ROOT, name); + const activationProjectDir = path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR, name); + await fs.mkdir(activationProjectDir, { recursive: true }); + const backupPath = path.join(activationProjectDir, `.previous-${deploymentId}-1-1-${randomUUID()}`); + await fs.mkdir(backupPath, { recursive: true }); + await fs.writeFile(path.join(backupPath, 'index.js'), "module.exports = 'backup';\n"); + // The swap window this models: no live directory, only the backup. + assert.equal(existsSync(livePath), false); + + let releaseHold; + const holdReleased = new Promise((resolve) => (releaseHold = resolve)); + // Reconcile is started OUTSIDE the holder — awaiting it from inside would deadlock, since the lock + // is not reentrant. + const holder = withComponentPreparationLock(livePath, () => holdReleased); + await new Promise((resolve) => setTimeout(resolve, 100)); + + let reconciled = false; + const reconcilePromise = reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async () => undefined, + async () => {} + ).then((result) => { + reconciled = true; + return result; + }); + await new Promise((resolve) => setTimeout(resolve, 300)); + + assert.equal(reconciled, false, 'the sweep must not finish while the component lock is held'); + assert.equal(existsSync(livePath), false, 'and must not restore the backup underneath the lock holder'); + assert.equal(existsSync(backupPath), true, 'leaving the backup for after the lock is released'); + + releaseHold(); + await holder; + await reconcilePromise; + + assert.equal(existsSync(backupPath), false, 'once released, the sweep settles the artifact'); + assert.match(await readMarker(livePath), /backup/, 'restoring the backup over the absent live path'); + await cleanup(name); + }); + // Same contract the extraction scan owes its caller: componentLoader fails startup closed on a // rejection, so an absent retention root (nothing ever retained) must stay distinguishable from one // we could not read, where a component may be mid-revert with no live directory at all. From d1e9995cecaec89a2742e7a9365f86c577e7d716 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 15:36:49 -0400 Subject: [PATCH 75/94] fix(deploy): settle crash-stranded in-flight deployments so their payloads can be reclaimed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A node killed mid-stage left its row at `pending`/`staging`. Startup reconciliation removed that deployment's staging directory but never touched the row, and payload retention only reclaims rows that reached a terminal status — so the component tarball stayed pinned on the origin and every peer indefinitely, accumulating across restarts regardless of the configured retention count. Reconciliation now settles a row that is still in flight when its staging directory goes away, which is the point where it is known to be unable to make progress. Already-terminal rows are swept the same way but deliberately not settled: `markDeploymentTerminal` patches unconditionally, so re-settling would rewrite a successful deploy's status to failed. The settle hook now carries a reason, so a stranded row and an incomplete staged tree no longer share one misleading message. Reported by Barber AI on #1849. --- components/Application.ts | 13 ++++++-- components/componentLoader.ts | 4 +-- unitTests/components/deployStaging.test.js | 36 ++++++++++++++++++++++ 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 195a98611b..11b3111994 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -3121,7 +3121,7 @@ export async function reconcileStagedApplicationArtifacts( componentsRootDirPath: string, getDeployment: DeploymentLookup, persistActivation: (row: Record) => Promise, - settleStagedDeployment?: (deploymentId: string) => Promise + settleStagedDeployment?: (deploymentId: string, reason: string) => Promise ): Promise<{ recovered: string[]; removed: string[]; @@ -3168,7 +3168,14 @@ export async function reconcileStagedApplicationArtifacts( await withComponentPreparationLock(componentDirPath, async () => { row = await getDeployment(entry.name); shouldRemove = !row || !safeComponentName(row.project) || !['staged', 'activating'].includes(row.status); - if (shouldRemove) await rm(deploymentPath, { recursive: true, force: true }); + if (!shouldRemove) return; + // A row still `pending`/`staging` once its staging directory is going away cannot make + // progress — the process that owned it is gone. Payload retention only reclaims rows that + // reached a terminal status, so leaving it in flight pins its tarball on every node forever. + if (row?.status === 'pending' || row?.status === 'staging') { + await settleStagedDeployment?.(entry.name, 'the deploy did not survive a restart'); + } + await rm(deploymentPath, { recursive: true, force: true }); }); if (shouldRemove) { removed.push(entry.name); @@ -3190,7 +3197,7 @@ export async function reconcileStagedApplicationArtifacts( `Discarding staged deployment '${entry.name}' for '${row.project}': no valid component tree. ` + `The live component is unaffected.` ); - await settleStagedDeployment?.(entry.name); + await settleStagedDeployment?.(entry.name, 'its staged component tree was incomplete'); await rm(deploymentPath, { recursive: true, force: true }); discarded = true; }); diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 34bf4f0f22..25391f6a5b 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -209,8 +209,8 @@ export async function loadComponentDirectories( } if (isMainThread && !stagedArtifactsReconciled) { try { - const settleDiscardedDeployment = async (deploymentId: string) => { - await markDeploymentTerminal(deploymentId, 'failed', new Error('staged component tree was not recoverable')); + const settleDiscardedDeployment = async (deploymentId: string, reason: string) => { + await markDeploymentTerminal(deploymentId, 'failed', new Error(reason)); }; const reconciliation = await reconcileStagedApplicationArtifacts( CF_ROUTES_DIR, diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 98bff3d745..1e10839d7d 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -1495,6 +1495,42 @@ describe('two-phase component directory transaction', function () { } }); + it('settles a crash-stranded in-flight row so its payload stops being retained', async () => { + // A node killed mid-stage leaves a `pending`/`staging` row. Reconciliation removes its staging + // directory, but payload retention only reclaims rows that reached a terminal status — so leaving + // the row in flight pins its tarball on the origin and every peer indefinitely. + const name = fixtureName(); + const strandedId = randomUUID(); + const terminalId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('stranded') }); + await stageApplication(application, strandedId); + application.payload = await makeComponentPayload('terminal'); + await stageApplication(application, terminalId); + const rows = new Map([ + [strandedId, { deployment_id: strandedId, project: name, status: 'staging' }], + // An already-terminal row is swept the same way but must NOT be re-settled: patching it would + // rewrite a successful deploy's status to failed. + [terminalId, { deployment_id: terminalId, project: name, status: 'success' }], + ]); + + const settled = []; + const reconciliation = await reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async (id) => rows.get(id), + async () => {}, + async (id, reason) => settled.push([id, reason]) + ); + + assert.deepEqual( + settled, + [[strandedId, 'the deploy did not survive a restart']], + 'only the in-flight row is settled, and with a reason naming the crash' + ); + assert.equal(reconciliation.removed.includes(strandedId), true, 'its staging residue is removed'); + assert.equal(reconciliation.removed.includes(terminalId), true, "and so is the terminal row's"); + await cleanup(name); + }); + it('waits for the component lock before restoring an activation backup over the live path', async () => { // The sweep renames a backup back over the live path. Run unlocked, a reload-cycle retry could do // that inside an activation's swap window — live momentarily absent, backup present — after which From 21f319af6300fb78016779cf2310199ef9daaf90 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 15:51:42 -0400 Subject: [PATCH 76/94] fix(deploy): stop treating uncertainty as absence, and recover the no-live restore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five findings from the pre-push cross-model review, all variations on one theme: an inconclusive result was being read as a definite one. `install_allow_scripts` was missing from the request-boolean normalizer. Joi coerces it, but validateBySchema discards `result.value`, so a multipart or form caller sending `"false"` had the string reach the handler, where it read as truthy — running third-party package lifecycle scripts for a caller that explicitly disabled them, potentially with registry credentials available. The revert orphan-marker sweep deleted the no-live restore branch's marker every time. That branch has no holding directory by design, so "no holding directory" was never evidence of orphanhood — and nothing consumed the marker either, so the crash shape it exists for was unrecoverable. Restore markers are now excluded from the sweep and rolled forward on their own terms: the retained tree goes live, config and manifest are reconciled, and the marker is cleared last. The sweep also decides orphanhood under the component lock and re-reads, because a revert writes its marker immediately before the holding rename — an unlocked sweep could catch that window and delete a live revert's only recovery evidence. The staged-completeness probe mapped every error to absence, so a transient EACCES/EIO/EMFILE at startup made a complete release look broken and the staged branch deleted it. Only ENOENT reads as absent now; anything else propagates. The persistent-state lock had no `isOwnerAlive`, so a ticket left by a terminated worker kept this process's pid and instance nonce, read as live, and was never reclaimed — wedging later activations, reverts and drops. It now uses the same thread-liveness predicate as the three other lock sites. Both lock tests signal acquisition from inside the callback instead of sleeping, so they cannot pass by winning a timing race. Reported by codex, cursor-composer and cursor-grok pre-push. --- components/Application.ts | 167 +++++++- components/operations.js | 24 +- .../components/deployPhaseOperations.test.js | 134 ++++--- .../components/deployPhaseValidators.test.js | 4 +- unitTests/components/deployStaging.test.js | 363 +++++++++++------- 5 files changed, 482 insertions(+), 210 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 11b3111994..5a16eef3c4 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -818,13 +818,107 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): throw error; } // A crash between the final rename and the marker removal leaves a marker whose holding directory is - // gone. Nothing would revisit it — the loop below is keyed on finding a holding directory — so sweep - // those first rather than leave them to be trusted by a future attempt. + // gone. Nothing would revisit it — the swap loop below is keyed on finding a holding directory — so + // sweep those first rather than leave them to be trusted by a future attempt. const entryNames = new Set(entries.map((entry) => entry.name)); for (const entry of entries) { if (entry.isDirectory() || !entry.name.endsWith(REVERT_RECOVERY_SUFFIX)) continue; const holdingName = entry.name.slice(0, -REVERT_RECOVERY_SUFFIX.length); - if (!entryNames.has(holdingName)) await rm(join(previousRoot, entry.name), { force: true }).catch(() => {}); + // The no-live restore branch has no holding directory by design, so absence is not orphanhood for + // its marker. It is recovered on its own terms below. + if (holdingName.endsWith(RESTORE_MARKER_INFIX)) continue; + if (entryNames.has(holdingName)) continue; + const orphanName = REVERTING_NAME_PATTERN.exec(holdingName.slice(REVERTING_PREFIX.length))?.[1]; + if (!holdingName.startsWith(REVERTING_PREFIX) || !orphanName || !safeComponentName(orphanName)) { + await rm(join(previousRoot, entry.name), { force: true }).catch(() => {}); + continue; + } + // Decided under the component lock, then re-read: a revert writes this marker immediately before + // renaming live into the holding directory, so an unlocked sweep can catch that window and delete + // the marker of a revert that is still running — leaving its crash unrecoverable. + await withComponentPreparationLock(join(componentsRootDirPath, orphanName), async () => { + const stillOrphaned = await lstat(join(previousRoot, holdingName)).then( + () => false, + (err) => (err as NodeJS.ErrnoException).code === 'ENOENT' + ); + if (stillOrphaned) await rm(join(previousRoot, entry.name), { force: true }).catch(() => {}); + }).catch((error) => { + logger.warn(`Could not settle a stray revert marker for ${orphanName}:`, errorForLog(error as Error)); + }); + } + // The no-live restore branch: `previous` was renamed straight to live because there was nothing to + // displace. Its marker is the only evidence, and rolling forward is the only sound repair — undoing + // would leave the component with no live tree at all. + for (const entry of entries) { + if (entry.isDirectory() || !entry.name.endsWith(REVERT_RECOVERY_SUFFIX)) continue; + const markerBase = entry.name.slice(0, -REVERT_RECOVERY_SUFFIX.length); + if (!markerBase.endsWith(RESTORE_MARKER_INFIX)) continue; + const componentName = markerBase.slice(0, -RESTORE_MARKER_INFIX.length); + const markerPath = join(previousRoot, entry.name); + if (!safeComponentName(componentName)) { + await rm(markerPath, { force: true }).catch(() => {}); + continue; + } + const liveDirPath = join(componentsRootDirPath, componentName); + try { + await withComponentPreparationLock(liveDirPath, async () => { + const intended = await readRevertRecoveryMarker(markerPath); + if (!intended) { + await rm(markerPath, { force: true }); + return; + } + const previousPath = previousDirPathFor(liveDirPath); + const liveExists = await lstat(liveDirPath).then( + () => true, + (err) => { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw err; + } + ); + const previousExists = await lstat(previousPath).then( + () => true, + (err) => { + if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw err; + } + ); + if (!liveExists && !previousExists) { + await rm(markerPath, { force: true }); + throw new Error( + `Interrupted restore of ${componentName} has neither a live nor a retained tree; ` + + `its bytes are gone and it must be redeployed` + ); + } + if (liveExists && previousExists) { + // The restore finished and something else has since retained a version. Residue only. + await rm(markerPath, { force: true }); + return; + } + if (!liveExists) { + await mkdir(dirname(liveDirPath), { recursive: true }); + await rename(previousPath, liveDirPath); + } + // Idempotent on a repeat pass, and ordered config-then-manifest so the marker outlives both. + const configTransaction = await createApplicationConfigTransaction( + componentName, + intended.live.application_config + ); + await configTransaction.commit(); + await writeRetainedPreviousManifest(liveDirPath, { + previous: { deployment_id: null, application_config: null }, + live: intended.live, + }); + await rm(markerPath, { force: true }); + logger.warn( + `Completed an interrupted restore of ${componentName}: the retained version is live and its ` + + `configuration has been reconciled. It cannot be reverted again until its next deploy` + ); + }); + } catch (error) { + const recoveryError = error instanceof Error ? error : new Error(String(error)); + failures.set(componentName, recoveryError); + logger.error(`Could not recover the interrupted ${componentName} restore:`, errorForLog(recoveryError)); + } } for (const entry of entries) { if (!entry.isDirectory() || !entry.name.startsWith(REVERTING_PREFIX)) continue; @@ -2784,14 +2878,31 @@ async function activationArtifacts(componentDirPath: string, deploymentId: strin .map((entry) => join(directory, entry.name)); } +/** + * `lstat`/`stat` where only genuine absence reads as absent. Collapsing every error to "not there" + * makes a transient EACCES/EIO/EMFILE look like a missing tree, and the staged-candidate branch + * deletes what it believes is unusable — so anything other than ENOENT has to propagate. + */ +async function statIfPresent( + path: string, + followSymlinks = false +): Promise> | undefined> { + try { + return await (followSymlinks ? stat(path) : lstat(path)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; + throw error; + } +} + export async function hasCompleteStagedApplication(stagingDirPath: string): Promise { const deploymentDirPath = dirname(stagingDirPath); const [stagingRootStat, deploymentStat, stagedStat, stagedTargetStat, markerStat] = await Promise.all([ - lstat(dirname(deploymentDirPath)).catch(() => undefined), - lstat(deploymentDirPath).catch(() => undefined), - lstat(stagingDirPath).catch(() => undefined), - stat(stagingDirPath).catch(() => undefined), - lstat(join(deploymentDirPath, STAGED_COMPLETE_MARKER)).catch(() => undefined), + statIfPresent(dirname(deploymentDirPath)), + statIfPresent(deploymentDirPath), + statIfPresent(stagingDirPath), + statIfPresent(stagingDirPath, true), + statIfPresent(join(deploymentDirPath, STAGED_COMPLETE_MARKER)), ]); return ( !!stagingRootStat?.isDirectory() && @@ -3110,6 +3221,16 @@ function activationArtifactDeploymentId(name: string): string | undefined { return DEPLOYMENT_ID_PATTERN.test(deploymentId) ? deploymentId : undefined; } +/** + * The project a staged deployment belongs to, read from `//`. Used to + * attribute a reconciliation failure when the deployment row itself could not be read. + */ +async function stagedDeploymentProjectName(deploymentPath: string): Promise { + const entries = await readdir(deploymentPath, { withFileTypes: true }).catch(() => []); + const projects = entries.filter((entry) => entry.isDirectory() && safeComponentName(entry.name)); + return projects.length === 1 ? projects[0].name : undefined; +} + async function removeActivationArtifacts(componentDirPath: string, deploymentId: string): Promise { for (const artifact of await activationArtifacts(componentDirPath, deploymentId)) { await rm(artifact, { recursive: true, force: true }); @@ -3236,11 +3357,16 @@ export async function reconcileStagedApplicationArtifacts( errors.set(entry.name, reconcileError); // Fail the component closed for an interrupted activation, where the live tree and its durable // configuration can disagree. A confirmed `staged` failure leaves live state consistent, so it - // does not. When the lookup itself failed there is no project name to attribute to — but nothing - // destructive has run for this entry either, and a non-empty `errors` keeps the reconcile guard - // unset so the next reload cycle retries. + // does not. if (row?.status === 'activating' && safeComponentName(row?.project)) { failedProjects.set(row.project, reconcileError); + } else if (!row) { + // The row is what says whether this was an interrupted activation, so an unreadable row is + // the one case we cannot rule that out. The project name does not depend on the row — it is + // the directory under the deployment id — so attribute from disk rather than fail open and + // load a component whose live tree may disagree with its durable config. + const project = await stagedDeploymentProjectName(deploymentPath); + if (project) failedProjects.set(project, reconcileError); } } } @@ -3291,6 +3417,16 @@ export async function reconcileStagedApplicationArtifacts( } } else if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && !liveStat) { await rename(artifactPath, livePath); + } else if (!row && artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX)) { + // A backup artifact still here means the activation never finished retaining it, so this is + // the only copy of the tree it displaced. An absent row is not evidence that copy is + // disposable: the lookup returns undefined both for a row retention reclaimed and for a + // deployment table that was never provisioned. Keep it and let a pass that can read the + // row decide. + logger.warn( + `Keeping displaced component tree '${artifact.name}' for '${projectEntry.name}': ` + + `its deployment row could not be read, so it cannot be confirmed disposable` + ); } else { await rm(artifactPath, { recursive: true, force: true }); } @@ -3438,10 +3574,14 @@ function persistentStateLockPath(): string { * Never call this while already holding it — the file lock is not reentrant. The `*Unlocked` variants * exist for callers that are already inside it. */ -async function withPersistentStateLock(operation: () => Promise): Promise { +export async function withPersistentStateLock(operation: () => Promise): Promise { return withComponentPreparationLock(persistentStateLockPath(), operation, { purpose: PERSISTENT_STATE_LOCK_PURPOSE, timeoutMs: 30_000, + // Without this, a ticket left behind by a terminated worker still carries this process's pid and + // instance nonce, so it reads as live and is never reclaimed — wedging every later activation, + // revert and drop on a holder that no longer exists. + isOwnerAlive: (owner) => owner.pid !== process.pid || isThreadRunning(owner.threadId), }); } @@ -3553,6 +3693,9 @@ export async function createApplicationConfigTransaction( // Snapshot and both writes inside one critical section, so a concurrent activation of a different // project cannot land between the read and the write and lose one of the two entries. await withPersistentStateLock(async () => { + // Re-checked inside the critical section: the guard above is only a fast path, so two + // concurrent commits on this transaction would both pass it and repeat the read-modify-write. + if (commitStarted) return; previousConfig = readConfigFile()?.[project]; previousLockConfig = await getApplicationLockEntryUnlocked(project); commitStarted = true; diff --git a/components/operations.js b/components/operations.js index c4fb5283aa..22844c9560 100644 --- a/components/operations.js +++ b/components/operations.js @@ -37,6 +37,7 @@ const { discardProjectStagedApplications, discardProjectActivationArtifacts, updateApplicationLockEntry, + withPersistentStateLock, createApplicationActivationTransaction, createApplicationConfigTransaction, getRevertTarget, @@ -697,7 +698,18 @@ async function deployComponentOneShot(req, credentialReferences, isReplicatedExe } } -const REQUEST_BOOLEAN_FIELDS = ['activate', 'two_phase', 'ignore_replication_errors', 'force', 'restart']; +// Every boolean in the deploy_component/revert_component schemas. Joi coerces a string `"false"`, but +// validateBySchema discards `result.value`, so the raw string reaches the handler and reads as truthy. +// `install_allow_scripts` is the one that matters most: a caller explicitly disabling lifecycle scripts +// over multipart/form would otherwise run third-party install code with credentials available. +const REQUEST_BOOLEAN_FIELDS = [ + 'activate', + 'two_phase', + 'ignore_replication_errors', + 'force', + 'restart', + 'install_allow_scripts', +]; function normalizeRequestBooleans(req) { for (const field of REQUEST_BOOLEAN_FIELDS) { @@ -1401,7 +1413,11 @@ async function writeComponentRootConfig(req, credentialReferences) { // Persist credential references (never tokens) so every cold install of this component — reboot, new // peer, revert — re-resolves the credential from the store. if (credentialReferences.length) applicationConfig.credentials = credentialReferences; - await configUtils.addConfig(req.project, applicationConfig); + // Same critical section the activation transaction uses. `addConfig` is a read-modify-write of a + // file whose entries are per-project, so a one-shot deploy running unlocked can write back a + // document it parsed before a concurrent activation or drop committed, resurrecting or dropping + // that project's entry. + await withPersistentStateLock(() => configUtils.addConfig(req.project, applicationConfig)); } // Resolve the tarball to extract from. On the origin, tee req.payload into the row's blob (the @@ -2063,7 +2079,9 @@ async function dropComponent(req) { await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2), 'utf8'); } - configUtils.deleteConfigFromFile([project]); + // Under the persistent-state lock for the same reason: an unlocked delete can be clobbered by a + // concurrent activation writing back a document that still contains this project. + await withPersistentStateLock(async () => configUtils.deleteConfigFromFile([project])); }, componentDropLockOptions(project) ); diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index f1f3f395e8..785b8fdf5f 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -1,6 +1,6 @@ 'use strict'; -const assert = require('node:assert/strict'); +const assert = require('node:assert'); const fs = require('node:fs/promises'); const { existsSync } = require('node:fs'); const os = require('node:os'); @@ -122,6 +122,28 @@ describe('deploy_component two-phase orchestration', function () { return value; } + it('normalizes string request booleans, including install_allow_scripts', async () => { + // Joi coerces these, but validateBySchema discards `result.value`, so the raw string reaches the + // handler. `install_allow_scripts: 'false'` would then read as truthy and run package lifecycle + // scripts for a caller that explicitly disabled them — over multipart/form, where every value + // arrives as a string. + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('1.0.0'), + activate: 'false', + install_allow_scripts: 'false', + }); + + assert.strictEqual(staged.staged, true, "activate:'false' is honored as stage-only, not as a full deploy"); + assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, project)), false, 'so nothing goes live'); + assert.strictEqual( + rows.get(staged.deployment_id).activation_spec.install_allow_scripts, + false, + 'and the activation spec records a real boolean, not the string' + ); + }); + it('stages without touching live and records an immutable activation specification', async () => { const project = name(); const result = await operations.deployComponent({ @@ -130,14 +152,14 @@ describe('deploy_component two-phase orchestration', function () { activate: false, }); - assert.equal(result.staged, true); + assert.strictEqual(result.staged, true); assert.match(result.deployment_id, /^[0-9a-f-]{36}$/i); - assert.equal(existsSync(path.join(COMPONENTS_ROOT, project)), false); - assert.equal(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id, project)), true); + assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, project)), false); + assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id, project)), true); const row = rows.get(result.deployment_id); assert.ok(row, `deployment row missing; present ids: ${Array.from(rows.keys()).join(', ')}`); - assert.equal(row.status, 'staged'); - assert.deepEqual(row.activation_spec, { + assert.strictEqual(row.status, 'staged'); + assert.deepStrictEqual(row.activation_spec, { project, package: null, install_command: null, @@ -167,8 +189,8 @@ describe('deploy_component two-phase orchestration', function () { }); const stagedPath = path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id, project); - assert.equal(await fs.readFile(path.join(stagedPath, 'credential-seen'), 'utf8'), 'yes'); - assert.equal(rows.get(result.deployment_id).activation_spec.credentials, null); + assert.strictEqual(await fs.readFile(path.join(stagedPath, 'credential-seen'), 'utf8'), 'yes'); + assert.strictEqual(rows.get(result.deployment_id).activation_spec.credentials, null); assert.doesNotMatch(JSON.stringify(rows.get(result.deployment_id)), new RegExp(token)); }); @@ -186,8 +208,8 @@ describe('deploy_component two-phase orchestration', function () { ); const activated = await operations.deployComponent({ project, deployment_id: staged.deployment_id }); - assert.equal(activated.activated, true); - assert.equal(rows.get(staged.deployment_id).status, 'success'); + assert.strictEqual(activated.activated, true); + assert.strictEqual(rows.get(staged.deployment_id).status, 'success'); assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /2.0.0/); }); @@ -204,9 +226,9 @@ describe('deploy_component two-phase orchestration', function () { operations.deployComponent({ project, deployment_id: staged.deployment_id }), ]); - assert.equal(outcomes.filter((outcome) => outcome.status === 'fulfilled').length, 1); - assert.equal(outcomes.filter((outcome) => outcome.status === 'rejected').length, 1); - assert.equal(rows.get(staged.deployment_id).status, 'success'); + assert.strictEqual(outcomes.filter((outcome) => outcome.status === 'fulfilled').length, 1); + assert.strictEqual(outcomes.filter((outcome) => outcome.status === 'rejected').length, 1); + assert.strictEqual(rows.get(staged.deployment_id).status, 'success'); assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /duplicate-winner/); }); @@ -236,10 +258,10 @@ describe('deploy_component two-phase orchestration', function () { }); assert.match(result.message, /Successfully deployed/); - assert.equal(rows.get(result.deployment_id).status, 'success'); + assert.strictEqual(rows.get(result.deployment_id).status, 'success'); assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /full-deploy/); - assert.equal(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id)), false); - assert.equal(restartNeeded(), true, 'a new component activated without restart requires one'); + assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id)), false); + assert.strictEqual(restartNeeded(), true, 'a new component activated without restart requires one'); }); // ———————————————————————————————————————————————————————————————————————————— @@ -258,19 +280,19 @@ describe('deploy_component two-phase orchestration', function () { const result = await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); - assert.equal(result.reverted, true); - assert.equal(result.to_deployment_id, first.deployment_id); - assert.equal(result.from_deployment_id, second.deployment_id); + assert.strictEqual(result.reverted, true); + assert.strictEqual(result.to_deployment_id, first.deployment_id); + assert.strictEqual(result.from_deployment_id, second.deployment_id); assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /rev-v1/); - assert.equal(rows.get(result.deployment_id).status, 'rolled_back'); - assert.equal( + assert.strictEqual(rows.get(result.deployment_id).status, 'rolled_back'); + assert.strictEqual( rows.get(result.deployment_id).rollback_of, second.deployment_id, 'the audit row records which deployment the rollback took out of service' ); - assert.equal(fanout.length, 1, 'peers get the revert'); - assert.equal(fanout[0].operation, 'revert_component'); - assert.equal( + assert.strictEqual(fanout.length, 1, 'peers get the revert'); + assert.strictEqual(fanout[0].operation, 'revert_component'); + assert.strictEqual( fanout[0].to_deployment_id, first.deployment_id, 'peers are told WHICH version to end on, so the fan-out is idempotent per node' @@ -286,7 +308,7 @@ describe('deploy_component two-phase orchestration', function () { // The caller lost the first response and retried the identical request. const retry = await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); - assert.equal(retry.reverted, false); + assert.strictEqual(retry.reverted, false); assert.match(retry.message, /already running/); assert.match( await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), @@ -334,7 +356,7 @@ describe('deploy_component two-phase orchestration', function () { await operations.revertComponent({ project, to_deployment_id: packaged.deployment_id }); const entry = readConfigFile()?.[project]; - assert.equal( + assert.strictEqual( entry?.package, undefined, 'the reverted-to version had no package reference, so the stale one must be gone' @@ -352,8 +374,8 @@ describe('deploy_component two-phase orchestration', function () { }); const row = rows.get(result.deployment_id); - assert.equal(row.status, 'success'); - assert.equal(row.payload_blob, null); + assert.strictEqual(row.status, 'success'); + assert.strictEqual(row.payload_blob, null); assert.ok(row.event_log.some((event) => event.event === 'payload_dropped')); } finally { environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, priorMaxSize); @@ -375,8 +397,8 @@ describe('deploy_component two-phase orchestration', function () { await operations.deployComponent({ project, deployment_id: staged.deployment_id }); const row = rows.get(staged.deployment_id); - assert.equal(row.status, 'success'); - assert.equal(row.payload_blob, null); + assert.strictEqual(row.status, 'success'); + assert.strictEqual(row.payload_blob, null); assert.ok(row.event_log.some((event) => event.event === 'payload_dropped')); } finally { environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, priorMaxSize); @@ -447,11 +469,11 @@ describe('deploy_component two-phase orchestration', function () { operations.deployComponent({ project, deployment_id: staged.deployment_id, restart: true }), /Split nodes: peer-a.*[Rr]oll forward/s ); - assert.deepEqual(phases, ['activate'], 'restart phase was never sent after the activation gate failed'); - assert.equal(rows.get(staged.deployment_id).status, 'activating'); - assert.equal(rows.get(staged.deployment_id).completed_at, null); + assert.deepStrictEqual(phases, ['activate'], 'restart phase was never sent after the activation gate failed'); + assert.strictEqual(rows.get(staged.deployment_id).status, 'activating'); + assert.strictEqual(rows.get(staged.deployment_id).completed_at, null); assert.ok(rows.get(staged.deployment_id).payload_blob, 'payload remains available to repair a split cluster'); - assert.equal(rows.get(staged.deployment_id).peer_results[0].node, 'peer-a'); + assert.strictEqual(rows.get(staged.deployment_id).peer_results[0].node, 'peer-a'); }); it('records success when restart fails after the activation barrier', async () => { @@ -484,8 +506,8 @@ describe('deploy_component two-phase orchestration', function () { manageThreads.restartWorkers = priorRestartWorkers; } - assert.deepEqual(phases, ['stage', 'activate', 'restart']); - assert.equal(rows.get(deploymentId).status, 'success'); + assert.deepStrictEqual(phases, ['stage', 'activate', 'restart']); + assert.strictEqual(rows.get(deploymentId).status, 'success'); assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /activated-before/); }); @@ -506,9 +528,9 @@ describe('deploy_component two-phase orchestration', function () { ignore_replication_errors: true, }); - assert.equal(result.activated, true); - assert.equal(rows.get(staged.deployment_id).status, 'success'); - assert.equal(rows.get(staged.deployment_id).peer_results[0].status, 'failed'); + assert.strictEqual(result.activated, true); + assert.strictEqual(rows.get(staged.deployment_id).status, 'success'); + assert.strictEqual(rows.get(staged.deployment_id).peer_results[0].status, 'failed'); }); it('records and surfaces ignored restart failures after the activation gate', async () => { @@ -540,11 +562,11 @@ describe('deploy_component two-phase orchestration', function () { manageThreads.restartWorkers = priorRestartWorkers; } - assert.deepEqual(phases, ['activate', 'restart']); - assert.equal(localRestarts, 1); - assert.equal(result.activated, true); - assert.equal(result.failed_peers[0].node, 'peer-a'); - assert.equal(rows.get(staged.deployment_id).peer_results[0].status, 'failed'); + assert.deepStrictEqual(phases, ['activate', 'restart']); + assert.strictEqual(localRestarts, 1); + assert.strictEqual(result.activated, true); + assert.strictEqual(result.failed_peers[0].node, 'peer-a'); + assert.strictEqual(rows.get(staged.deployment_id).peer_results[0].status, 'failed'); }); it('uses the row-backed immutable specification for trusted peer phases', async () => { @@ -575,11 +597,11 @@ describe('deploy_component two-phase orchestration', function () { /immutable activation specification/ ); await executePeerPhase('stage', row.activation_spec); - assert.equal(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, staged.deployment_id, project)), true); + assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, staged.deployment_id, project)), true); await executePeerPhase('activate', row.activation_spec); assert.match(await fs.readFile(path.join(componentPath, 'index.js'), 'utf8'), /peer-phase/); - assert.equal(rows.get(staged.deployment_id).status, 'activating'); - assert.equal(restartNeeded(), true); + assert.strictEqual(rows.get(staged.deployment_id).status, 'activating'); + assert.strictEqual(restartNeeded(), true); }); it('rebuilds a missing peer stage from the durable deployment payload before activation', async () => { @@ -606,7 +628,7 @@ describe('deploy_component two-phase orchestration', function () { ); assert.match(await fs.readFile(path.join(componentPath, 'index.js'), 'utf8'), /rebuilt-peer/); - assert.equal(rows.get(staged.deployment_id).status, 'activating'); + assert.strictEqual(rows.get(staged.deployment_id).status, 'activating'); }); it('waits for the staged row checkpoint when peer activation arrives first', async () => { @@ -634,7 +656,7 @@ describe('deploy_component two-phase orchestration', function () { ); assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /lagged-row/); - assert.equal(rows.get(staged.deployment_id).status, 'activating'); + assert.strictEqual(rows.get(staged.deployment_id).status, 'activating'); }); it('recovers a staged package specification for config and peer activation', async () => { @@ -664,9 +686,9 @@ describe('deploy_component two-phase orchestration', function () { await operations.deployComponent({ project, deployment_id: staged.deployment_id }); - assert.equal(readConfigFile()[project].package, packageIdentifier); - assert.equal(activationOperation.operation, 'component_deploy_phase'); - assert.equal(activationOperation.activation_spec.package, packageIdentifier); + assert.strictEqual(readConfigFile()[project].package, packageIdentifier); + assert.strictEqual(activationOperation.operation, 'component_deploy_phase'); + assert.strictEqual(activationOperation.activation_spec.package, packageIdentifier); } finally { if (priorRootEnv === undefined) delete process.env.ROOTPATH; else process.env.ROOTPATH = priorRootEnv; @@ -703,16 +725,16 @@ describe('deploy_component two-phase orchestration', function () { await operations.dropComponent({ project }); - assert.equal(rows.get(staged.deployment_id).status, 'failed'); + assert.strictEqual(rows.get(staged.deployment_id).status, 'failed'); const deploymentStagePath = path.join(componentsRoot, DEPLOY_STAGING_DIR, staged.deployment_id); - assert.equal( + assert.strictEqual( existsSync(deploymentStagePath), false, `staged deployment directory still contains: ${await fs.readdir(deploymentStagePath).catch(() => [])}` ); - assert.equal(existsSync(path.join(componentsRoot, DEPLOY_ACTIVATION_DIR, project)), false); + assert.strictEqual(existsSync(path.join(componentsRoot, DEPLOY_ACTIVATION_DIR, project)), false); const applicationLock = JSON.parse(await fs.readFile(path.join(configRoot, 'harper-application-lock.json'))); - assert.equal(applicationLock.applications[project], undefined); + assert.strictEqual(applicationLock.applications[project], undefined); } finally { if (priorRootEnv === undefined) delete process.env.ROOTPATH; else process.env.ROOTPATH = priorRootEnv; diff --git a/unitTests/components/deployPhaseValidators.test.js b/unitTests/components/deployPhaseValidators.test.js index e27609c743..d0ac9e62ce 100644 --- a/unitTests/components/deployPhaseValidators.test.js +++ b/unitTests/components/deployPhaseValidators.test.js @@ -1,9 +1,9 @@ 'use strict'; -const assert = require('node:assert/strict'); +const assert = require('node:assert'); const validator = require('#js/components/operationsValidation'); -const valid = (result) => assert.equal(result, undefined, `expected valid, got: ${result?.message}`); +const valid = (result) => assert.strictEqual(result, undefined, `expected valid, got: ${result?.message}`); const invalid = (result) => assert.ok(result, 'expected a validation error'); describe('deployComponentValidator two-phase controls', () => { diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 1e10839d7d..2e0924f015 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -1,6 +1,6 @@ 'use strict'; -const assert = require('node:assert/strict'); +const assert = require('node:assert'); const fs = require('node:fs/promises'); const { existsSync } = require('node:fs'); const os = require('node:os'); @@ -94,9 +94,9 @@ describe('two-phase component directory transaction', function () { const stagedPath = await stageApplication(application, deploymentId); - assert.equal(stagedPath, stagedApplicationPath(application.dirPath, deploymentId)); + assert.strictEqual(stagedPath, stagedApplicationPath(application.dirPath, deploymentId)); assert.match(await readMarker(stagedPath), /candidate/); - assert.equal(existsSync(application.dirPath), false); + assert.strictEqual(existsSync(application.dirPath), false); await cleanup(name); }); @@ -112,8 +112,8 @@ describe('two-phase component directory transaction', function () { await activateStagedApplication(application, deploymentId); assert.match(await readMarker(application.dirPath), /candidate/); - assert.equal(existsSync(stagedApplicationPath(application.dirPath, deploymentId)), false); - assert.equal(existsSync(stagedApplicationPath(application.dirPath, siblingId)), true); + assert.strictEqual(existsSync(stagedApplicationPath(application.dirPath, deploymentId)), false); + assert.strictEqual(existsSync(stagedApplicationPath(application.dirPath, siblingId)), true); await cleanup(name); }); @@ -126,7 +126,7 @@ describe('two-phase component directory transaction', function () { await assert.rejects(activateStagedApplication(application, deploymentId), /staged build is incomplete/); - assert.equal(existsSync(application.dirPath), false); + assert.strictEqual(existsSync(application.dirPath), false); assert.match(await readMarker(stagedPath), /incomplete/); await cleanup(name); }); @@ -143,7 +143,7 @@ describe('two-phase component directory transaction', function () { await activateStagedApplication(application, deploymentId); assert.match(await readMarker(application.dirPath), /resumed/); - assert.equal(existsSync(activationPath), false); + assert.strictEqual(existsSync(activationPath), false); await cleanup(name); }); @@ -162,7 +162,7 @@ describe('two-phase component directory transaction', function () { } assert.match(await readMarker(application.dirPath), /cleanup-deferred/); - assert.equal(existsSync(path.join(stagingRoot, deploymentId)), true, 'cleanup remains retryable garbage'); + assert.strictEqual(existsSync(path.join(stagingRoot, deploymentId)), true, 'cleanup remains retryable garbage'); await cleanup(name); }); @@ -200,8 +200,8 @@ describe('two-phase component directory transaction', function () { activateStagedApplication(application, deploymentId), ]); - assert.equal(outcomes.filter((outcome) => outcome.status === 'fulfilled').length, 1); - assert.equal(outcomes.filter((outcome) => outcome.status === 'rejected').length, 1); + assert.strictEqual(outcomes.filter((outcome) => outcome.status === 'fulfilled').length, 1); + assert.strictEqual(outcomes.filter((outcome) => outcome.status === 'rejected').length, 1); assert.match(await readMarker(application.dirPath), /candidate/); await cleanup(name); }); @@ -231,11 +231,11 @@ describe('two-phase component directory transaction', function () { await second.commit(); await second.rollback(); - assert.deepEqual(readConfigFile()[name], { package: 'example@1.0.0', urlPath: '/first' }); + assert.deepStrictEqual(readConfigFile()[name], { package: 'example@1.0.0', urlPath: '/first' }); const lock = JSON.parse( await fs.readFile(path.join(configRoot, 'harper-application-lock.json'), { encoding: 'utf8' }) ); - assert.deepEqual(lock.applications[name], { package: 'example@1.0.0', urlPath: '/first' }); + assert.deepStrictEqual(lock.applications[name], { package: 'example@1.0.0', urlPath: '/first' }); } finally { await second.rollback(); await first.rollback(); @@ -257,7 +257,7 @@ describe('two-phase component directory transaction', function () { await discardStagedApplication(livePath, deploymentId); - assert.equal(existsSync(stagedApplicationPath(livePath, deploymentId)), false); + assert.strictEqual(existsSync(stagedApplicationPath(livePath, deploymentId)), false); assert.match(await readMarker(livePath), /live/); await cleanup(name); }); @@ -271,8 +271,8 @@ describe('two-phase component directory transaction', function () { await discardProjectActivationArtifacts(path.join(COMPONENTS_ROOT, name)); - assert.equal(existsSync(path.join(activationRoot, name)), false); - assert.equal(existsSync(path.join(activationRoot, sibling, 'keep')), true); + assert.strictEqual(existsSync(path.join(activationRoot, name)), false); + assert.strictEqual(existsSync(path.join(activationRoot, sibling, 'keep')), true); await cleanup(name); }); @@ -295,9 +295,9 @@ describe('two-phase component directory transaction', function () { async () => {} ); - assert.equal(existsSync(stagedApplicationPath(application.dirPath, keptId)), true); - assert.equal(existsSync(stagedApplicationPath(application.dirPath, removedId)), false); - assert.deepEqual(result.removed, [removedId]); + assert.strictEqual(existsSync(stagedApplicationPath(application.dirPath, keptId)), true); + assert.strictEqual(existsSync(stagedApplicationPath(application.dirPath, removedId)), false); + assert.deepStrictEqual(result.removed, [removedId]); await cleanup(name); }); @@ -327,9 +327,9 @@ describe('two-phase component directory transaction', function () { ); assert.match(await readMarker(livePath), /candidate/); - assert.deepEqual(persisted, [deploymentId]); - assert.deepEqual(result.recovered, [deploymentId]); - assert.equal(existsSync(activationPath), false); + assert.deepStrictEqual(persisted, [deploymentId]); + assert.deepStrictEqual(result.recovered, [deploymentId]); + assert.strictEqual(existsSync(activationPath), false); await cleanup(name); }); @@ -355,10 +355,10 @@ describe('two-phase component directory transaction', function () { async () => persisted++ ); - assert.equal(persisted, 1); - assert.deepEqual(result.recovered, [deploymentId]); + assert.strictEqual(persisted, 1); + assert.deepStrictEqual(result.recovered, [deploymentId]); assert.match(await readMarker(livePath), /candidate/); - assert.equal(existsSync(activationPath), false); + assert.strictEqual(existsSync(activationPath), false); await cleanup(name); }); @@ -376,7 +376,7 @@ describe('two-phase component directory transaction', function () { const application = new Application({ name, payload: await makeComponentPayload('candidate') }); await assert.rejects(stageApplication(application, deploymentId), /staging path is not a directory/); - assert.equal(await fs.readFile(path.join(outside, 'sentinel'), 'utf8'), 'keep'); + assert.strictEqual(await fs.readFile(path.join(outside, 'sentinel'), 'utf8'), 'keep'); await cleanup(name); await fs.rm(outside, { recursive: true, force: true }); @@ -390,11 +390,11 @@ describe('two-phase component directory transaction', function () { const application = new Application({ name, packageIdentifier: packageDirectory }); const stagedPath = await stageApplication(application, deploymentId); - assert.equal((await fs.lstat(stagedPath)).isSymbolicLink(), true); + assert.strictEqual((await fs.lstat(stagedPath)).isSymbolicLink(), true); await activateStagedApplication(application, deploymentId); - assert.equal((await fs.lstat(application.dirPath)).isSymbolicLink(), true); - assert.equal(await fs.readFile(path.join(application.dirPath, 'marker.txt'), 'utf8'), 'directory-package'); + assert.strictEqual((await fs.lstat(application.dirPath)).isSymbolicLink(), true); + assert.strictEqual(await fs.readFile(path.join(application.dirPath, 'marker.txt'), 'utf8'), 'directory-package'); await cleanup(name); await fs.rm(packageDirectory, { recursive: true, force: true }); @@ -419,8 +419,8 @@ describe('two-phase component directory transaction', function () { assert.match(await readMarker(retained), /v1/); const target = await getRevertTarget(application.dirPath); - assert.equal(target.live.deployment_id, secondId); - assert.equal(target.previous.deployment_id, firstId); + assert.strictEqual(target.live.deployment_id, secondId); + assert.strictEqual(target.previous.deployment_id, firstId); await cleanup(name); }); @@ -431,7 +431,7 @@ describe('two-phase component directory transaction', function () { await stageApplication(application, deploymentId); await activateStagedApplication(application, deploymentId, { activationSpec: { package: null } }); - assert.equal(await getRevertTarget(application.dirPath), undefined); + assert.strictEqual(await getRevertTarget(application.dirPath), undefined); await assert.rejects( () => revertApplication(application, randomUUID()), /no previous version is retained/, @@ -453,8 +453,8 @@ describe('two-phase component directory transaction', function () { const result = await revertApplication(application, firstId); - assert.equal(result.swapped, true); - assert.equal(result.fromDeploymentId, secondId); + assert.strictEqual(result.swapped, true); + assert.strictEqual(result.fromDeploymentId, secondId); assert.match(await readMarker(application.dirPath), /v1/, 'live is the reverted-to version'); assert.match( await readMarker(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name)), @@ -464,7 +464,7 @@ describe('two-phase component directory transaction', function () { // Explicitly targeting the other direction rolls forward again. const forward = await revertApplication(application, secondId); - assert.equal(forward.swapped, true); + assert.strictEqual(forward.swapped, true); assert.match(await readMarker(application.dirPath), /v2/); await cleanup(name); }); @@ -485,7 +485,7 @@ describe('two-phase component directory transaction', function () { // bidirectional toggle would put the rejected v2 back live; an addressed revert must not. const retry = await revertApplication(application, firstId); - assert.equal(retry.swapped, false, 'a repeated revert to the live version does nothing'); + assert.strictEqual(retry.swapped, false, 'a repeated revert to the live version does nothing'); assert.match(await readMarker(application.dirPath), /v1/, 'still on the reverted-to version'); await cleanup(name); }); @@ -523,7 +523,7 @@ describe('two-phase component directory transaction', function () { assert.match(await readMarker(application.dirPath), /v3/); assert.match(await readMarker(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name)), /v2/); const target = await getRevertTarget(application.dirPath); - assert.equal(target.previous.deployment_id, ids[1], 'v1 is evicted; only v2 stays revertable'); + assert.strictEqual(target.previous.deployment_id, ids[1], 'v1 is evicted; only v2 stays revertable'); await cleanup(name); }); @@ -544,7 +544,7 @@ describe('two-phase component directory transaction', function () { assert.match(await readMarker(application.dirPath), /v3/, 'the newest activation is live'); assert.match(await readMarker(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name)), /v2/); const target = await getRevertTarget(application.dirPath); - assert.equal(target.previous.deployment_id, ids[1], 'only the immediately-previous version is retained'); + assert.strictEqual(target.previous.deployment_id, ids[1], 'only the immediately-previous version is retained'); await cleanup(name); }); @@ -572,9 +572,9 @@ describe('two-phase component directory transaction', function () { await activateStagedApplication(application, payloadId, { activationSpec: { package: null } }); const target = await getRevertTarget(application.dirPath); - assert.equal(target.previous.application_config.package, 'stage-fixture@1.0.0'); + assert.strictEqual(target.previous.application_config.package, 'stage-fixture@1.0.0'); const back = await revertApplication(application, packagedId); - assert.equal(back.activatedConfig.package, 'stage-fixture@1.0.0'); + assert.strictEqual(back.activatedConfig.package, 'stage-fixture@1.0.0'); await cleanup(name); }); @@ -589,7 +589,7 @@ describe('two-phase component directory transaction', function () { await stageApplication(application, deploymentId); await activateStagedApplication(application, deploymentId, { activationSpec: { package: null } }); - assert.equal((await fs.lstat(application.dirPath)).isSymbolicLink(), false); + assert.strictEqual((await fs.lstat(application.dirPath)).isSymbolicLink(), false); assert.match(await readMarker(application.dirPath), /candidate/); await cleanup(name); }); @@ -604,7 +604,7 @@ describe('two-phase component directory transaction', function () { await extractApplication(application); - assert.equal((await fs.lstat(application.dirPath)).isSymbolicLink(), false); + assert.strictEqual((await fs.lstat(application.dirPath)).isSymbolicLink(), false); assert.match(await readMarker(application.dirPath), /over-dead-link/); await cleanup(name); }); @@ -643,8 +643,8 @@ describe('two-phase component directory transaction', function () { await activateFrom(name, 'quiet', '1.0.0', { withNodeModules: false }); const second = await activateFrom(name, 'quiet', '1.0.0', { withNodeModules: false }); - assert.equal(second.isNewComponent, false, 'the second activation is a redeploy'); - assert.equal(second.packageMetadataChanged, false, 'identical metadata across the swap must stay quiet'); + assert.strictEqual(second.isNewComponent, false, 'the second activation is a redeploy'); + assert.strictEqual(second.packageMetadataChanged, false, 'identical metadata across the swap must stay quiet'); await cleanup(name); }); @@ -653,7 +653,7 @@ describe('two-phase component directory transaction', function () { await activateFrom(name, 'changed', '1.0.0', { withNodeModules: false }); const second = await activateFrom(name, 'changed', '2.0.0', { withNodeModules: false }); - assert.equal(second.packageMetadataChanged, true, 'a changed package.json version invalidates loaded code'); + assert.strictEqual(second.packageMetadataChanged, true, 'a changed package.json version invalidates loaded code'); await cleanup(name); }); @@ -662,7 +662,7 @@ describe('two-phase component directory transaction', function () { await activateFrom(name, 'opaque', '1.0.0'); const second = await activateFrom(name, 'opaque', '1.0.0'); - assert.equal(second.packageMetadataChanged, true, 'a skipped install leaves nothing to compare'); + assert.strictEqual(second.packageMetadataChanged, true, 'a skipped install leaves nothing to compare'); await cleanup(name); }); @@ -671,7 +671,7 @@ describe('two-phase component directory transaction', function () { await activateFrom(name, 'nolock', '1.0.0'); const second = await activateFrom(name, 'nolock', '1.0.0', { dependencies: { 'some-dep': '1.0.0' } }); - assert.equal(second.packageMetadataChanged, true, 'dependencies with no lockfile are not reproducible'); + assert.strictEqual(second.packageMetadataChanged, true, 'dependencies with no lockfile are not reproducible'); await cleanup(name); }); @@ -679,8 +679,8 @@ describe('two-phase component directory transaction', function () { const name = fixtureName(); const first = await activateFrom(name, 'brand-new', '1.0.0', { withNodeModules: false }); - assert.equal(first.isNewComponent, true); - assert.equal(first.packageMetadataChanged, false, 'nothing to compare against; isNewComponent carries it'); + assert.strictEqual(first.isNewComponent, true); + assert.strictEqual(first.packageMetadataChanged, false, 'nothing to compare against; isNewComponent carries it'); await cleanup(name); }); // ———————————————————————————————————————————————————————————————————————————— @@ -716,9 +716,9 @@ describe('two-phase component directory transaction', function () { const failures = await recoverInterruptedReverts(COMPONENTS_ROOT); - assert.equal(failures.size, 0); + assert.strictEqual(failures.size, 0); assert.match(await readMarker(application.dirPath), /v2/, 'the interrupted revert is undone'); - assert.equal(existsSync(holding), false); + assert.strictEqual(existsSync(holding), false); await cleanup(name); }); @@ -736,22 +736,22 @@ describe('two-phase component directory transaction', function () { const failures = await recoverInterruptedReverts(COMPONENTS_ROOT); - assert.equal(failures.size, 0); + assert.strictEqual(failures.size, 0); assert.match(await readMarker(application.dirPath), /v1/, 'the reverted-to version stays live'); assert.match(await readMarker(previousPath), /displaced/, 'the displaced tree is retained again'); - assert.equal(existsSync(holding), false); + assert.strictEqual(existsSync(holding), false); // The directories are only half the state. The manifest was written before the swap, so completing // the recovery has to exchange its roles too — otherwise it names the retained tree as live and the // live tree as retained, and the retry below matches its target against the reversed `previous` // entry and swaps the successful revert straight back out. const target = await getRevertTarget(application.dirPath); - assert.equal(target.live.deployment_id, first, 'the manifest names the reverted-to version as live'); - assert.equal(target.previous.deployment_id, second, 'and the displaced version as the retained one'); + assert.strictEqual(target.live.deployment_id, first, 'the manifest names the reverted-to version as live'); + assert.strictEqual(target.previous.deployment_id, second, 'and the displaced version as the retained one'); // Idempotency has to survive the recovery: re-issuing the same addressed revert changes nothing. const retry = await revertApplication(application, first); - assert.equal(retry.swapped, false, 'a retry after recovery is a no-op, not a swap back'); + assert.strictEqual(retry.swapped, false, 'a retry after recovery is a no-op, not a swap back'); assert.match(await readMarker(application.dirPath), /v1/, 'still on the reverted-to version'); await cleanup(name); }); @@ -765,7 +765,7 @@ describe('two-phase component directory transaction', function () { await recoverInterruptedReverts(COMPONENTS_ROOT); - assert.equal(existsSync(holding), false, 'both slots were occupied, so the holding tree is residue'); + assert.strictEqual(existsSync(holding), false, 'both slots were occupied, so the holding tree is residue'); assert.match(await readMarker(application.dirPath), /v2/, 'live is untouched'); await cleanup(name); }); @@ -785,8 +785,8 @@ describe('two-phase component directory transaction', function () { await extractApplication(application); - assert.equal((await fs.lstat(application.dirPath)).isSymbolicLink(), true, 'the new version is linked'); - assert.equal(await fs.readFile(path.join(application.dirPath, 'marker.txt'), 'utf8'), 'v2'); + assert.strictEqual((await fs.lstat(application.dirPath)).isSymbolicLink(), true, 'the new version is linked'); + assert.strictEqual(await fs.readFile(path.join(application.dirPath, 'marker.txt'), 'utf8'), 'v2'); await cleanup(name); await fs.rm(packageDirectory, { recursive: true, force: true }); }); @@ -813,28 +813,28 @@ describe('two-phase component directory transaction', function () { // The version that is live now, and that a revert would be rolling back to. const original = await createApplicationConfigTransaction(name, { package: 'example@1.0.0' }); await original.commit(); - assert.deepEqual(readConfigFile()[name], { package: 'example@1.0.0' }); - assert.deepEqual(await readLock(), { package: 'example@1.0.0' }); + assert.deepStrictEqual(readConfigFile()[name], { package: 'example@1.0.0' }); + assert.deepStrictEqual(await readLock(), { package: 'example@1.0.0' }); const reverting = await createApplicationConfigTransaction(name, { package: 'example@2.0.0' }); await reverting.commit(); - assert.deepEqual(readConfigFile()[name], { package: 'example@2.0.0' }, 'both writes moved forward'); - assert.deepEqual(await readLock(), { package: 'example@2.0.0' }); + assert.deepStrictEqual(readConfigFile()[name], { package: 'example@2.0.0' }, 'both writes moved forward'); + assert.deepStrictEqual(await readLock(), { package: 'example@2.0.0' }); await reverting.rollback(); - assert.deepEqual( + assert.deepStrictEqual( readConfigFile()[name], { package: 'example@1.0.0' }, 'rollback restores the root config the commit replaced' ); - assert.deepEqual(await readLock(), { package: 'example@1.0.0' }, 'and the application-lock entry'); + assert.deepStrictEqual(await readLock(), { package: 'example@1.0.0' }, 'and the application-lock entry'); // A transaction that never committed must not touch anything on rollback, so compensation on an // early failure cannot clobber a live config. const untouched = await createApplicationConfigTransaction(name, { package: 'example@3.0.0' }); await untouched.rollback(); - assert.deepEqual(readConfigFile()[name], { package: 'example@1.0.0' }, 'no-op rollback changes nothing'); + assert.deepStrictEqual(readConfigFile()[name], { package: 'example@1.0.0' }, 'no-op rollback changes nothing'); } finally { if (priorRootEnv === undefined) delete process.env.ROOTPATH; else process.env.ROOTPATH = priorRootEnv; @@ -868,13 +868,17 @@ describe('two-phase component directory transaction', function () { } ); - assert.equal(reconciliation.errors.has(deploymentId), true, 'the failure is reported by deployment'); - assert.equal( + assert.strictEqual(reconciliation.errors.has(deploymentId), true, 'the failure is reported by deployment'); + assert.strictEqual( reconciliation.failedProjects.get(name)?.message, 'simulated activation-persistence failure', 'and attributed to the component, so the loader can fail it closed' ); - assert.equal(reconciliation.recovered.includes(deploymentId), false, 'a failed reconciliation is not "recovered"'); + assert.strictEqual( + reconciliation.recovered.includes(deploymentId), + false, + 'a failed reconciliation is not "recovered"' + ); await cleanup(name); }); it('re-points a dependency link whose absolute target the activation swap invalidated', async () => { @@ -911,18 +915,18 @@ describe('two-phase component directory transaction', function () { await activateStagedApplication(application, deploymentId, { activationSpec: { package: null } }); // The dependency has to be resolvable from the LIVE tree, which is the whole point. - assert.equal( + assert.strictEqual( await fs.readFile(path.join(application.dirPath, 'node_modules', 'probe', 'index.js'), 'utf8'), "module.exports = 'probe';\n", 'the staged-path link was re-pointed at the live path' ); - assert.equal( + assert.strictEqual( await fs.readFile(path.join(application.dirPath, 'node_modules', '@scope', 'scoped', 'index.js'), 'utf8'), "module.exports = 'probe';\n", 'scoped packages are re-pointed too' ); // Untouched: an absolute link to somewhere outside the staging tree is a deliberate choice. - assert.equal( + assert.strictEqual( await fs.readlink(path.join(application.dirPath, 'node_modules', 'external')), external, 'a link outside staging is left exactly as it was' @@ -954,12 +958,12 @@ describe('two-phase component directory transaction', function () { await activateStagedApplication(application, deploymentId, { activationSpec: { package: null } }); - assert.equal( + assert.strictEqual( await fs.readlink(path.join(external, 'bait')), externalBaitTarget, 'a link inside a symlinked scope directory is never rewritten' ); - assert.equal( + assert.strictEqual( (await fs.lstat(path.join(application.dirPath, 'node_modules', '@scope'))).isSymbolicLink(), true, 'the scope link itself is left as a link' @@ -1036,23 +1040,23 @@ describe('two-phase component directory transaction', function () { const firstPass = await recoverInterruptedReverts(COMPONENTS_ROOT); - assert.equal(firstPass.has(name), true, 'the failed pass is reported, so the component is failed closed'); - assert.equal(existsSync(holding), true, 'the holding tree survives, so a later pass can still finish'); - assert.equal(existsSync(markerPath), true, 'and the recovery marker survives with it'); + assert.strictEqual(firstPass.has(name), true, 'the failed pass is reported, so the component is failed closed'); + assert.strictEqual(existsSync(holding), true, 'the holding tree survives, so a later pass can still finish'); + assert.strictEqual(existsSync(markerPath), true, 'and the recovery marker survives with it'); // Clear the injected fault and run recovery again, as the next process start would. await fs.rm(manifestPath, { recursive: true, force: true }); const secondPass = await recoverInterruptedReverts(COMPONENTS_ROOT); - assert.equal(secondPass.size, 0, 'the second pass completes'); + assert.strictEqual(secondPass.size, 0, 'the second pass completes'); assert.match(await readMarker(application.dirPath), /v1/, 'the reverted-to version is live'); assert.match(await readMarker(previousPath), /displaced/, 'the displaced tree is retained again'); - assert.equal(existsSync(holding), false, 'the holding tree is consumed only once everything is durable'); - assert.equal(existsSync(markerPath), false, 'and the marker is cleared'); + assert.strictEqual(existsSync(holding), false, 'the holding tree is consumed only once everything is durable'); + assert.strictEqual(existsSync(markerPath), false, 'and the marker is cleared'); // The marker, not the twice-read manifest, decided the end state — so roles are exchanged once. const target = await getRevertTarget(application.dirPath); - assert.equal(target.live.deployment_id, first); - assert.equal(target.previous.deployment_id, second); + assert.strictEqual(target.live.deployment_id, first); + assert.strictEqual(target.previous.deployment_id, second); await cleanup(name); }); it('commits config inside the swap, so a revert cannot land config after a later activation', async () => { @@ -1066,7 +1070,7 @@ describe('two-phase component directory transaction', function () { // still parked, which is what makes this the only safe point to persist config. order.push('commit'); assert.match(await readMarker(application.dirPath), /v1/, 'reverted-to version is live at commit time'); - assert.equal( + assert.strictEqual( existsSync(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name)), false, 'the displaced tree is still in the holding path, not yet retained' @@ -1074,13 +1078,13 @@ describe('two-phase component directory transaction', function () { }, }); - assert.deepEqual(order, ['commit'], 'the hook ran exactly once'); - assert.equal(result.swapped, true); - assert.equal(result.fromDeploymentId, second); + assert.deepStrictEqual(order, ['commit'], 'the hook ran exactly once'); + assert.strictEqual(result.swapped, true); + assert.strictEqual(result.fromDeploymentId, second); assert.match(await readMarker(application.dirPath), /v1/); assert.match(await readMarker(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name)), /v2/); const target = await getRevertTarget(application.dirPath); - assert.equal(target.live.deployment_id, first); + assert.strictEqual(target.live.deployment_id, first); await cleanup(name); }); @@ -1103,9 +1107,9 @@ describe('two-phase component directory transaction', function () { const stranded = (await fs.readdir(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR))).filter((entry) => entry.startsWith('.reverting-') ); - assert.deepEqual(stranded, [], 'no holding tree or marker is left behind'); + assert.deepStrictEqual(stranded, [], 'no holding tree or marker is left behind'); const target = await getRevertTarget(application.dirPath); - assert.equal(target.live.deployment_id, second, 'the manifest still describes the un-reverted state'); + assert.strictEqual(target.live.deployment_id, second, 'the manifest still describes the un-reverted state'); await cleanup(name); }); @@ -1124,11 +1128,11 @@ describe('two-phase component directory transaction', function () { const failures = await recoverInterruptedReverts(COMPONENTS_ROOT); - assert.equal(failures.size, 0); - assert.equal(existsSync(orphan), false, 'the orphaned marker is swept'); + assert.strictEqual(failures.size, 0); + assert.strictEqual(existsSync(orphan), false, 'the orphaned marker is swept'); // The sweep must not have disturbed the component's actual revert state. const target = await getRevertTarget(application.dirPath); - assert.equal(target.previous.deployment_id, first, 'the real retained-previous entry is untouched'); + assert.strictEqual(target.previous.deployment_id, first, 'the real retained-previous entry is untouched'); await cleanup(name); }); @@ -1152,7 +1156,7 @@ describe('two-phase component directory transaction', function () { await activateStagedApplication(application, deploymentId, { activationSpec: { package: null } }); - assert.equal( + assert.strictEqual( await fs.readFile( path.join(application.dirPath, 'node_modules', 'outer', 'node_modules', 'deep', 'index.js'), 'utf8' @@ -1160,7 +1164,7 @@ describe('two-phase component directory transaction', function () { "module.exports = 'deep';\n", 'the nested link resolves from the live tree' ); - assert.equal( + assert.strictEqual( await fs.readFile( path.join(application.dirPath, 'node_modules', 'outer', 'node_modules', '@inner', 'scoped', 'index.js'), 'utf8' @@ -1183,18 +1187,18 @@ describe('two-phase component directory transaction', function () { const parkedSomething = await discardDirAside(target, name); - assert.equal(parkedSomething, true); - assert.equal(existsSync(target), false, 'the tree is moved out of the way'); + assert.strictEqual(parkedSomething, true); + assert.strictEqual(existsSync(target), false, 'the tree is moved out of the way'); // Read before the detached sweep can remove it; if it already has, there is nothing to misclassify. const entries = await fs.readdir(asideDir).catch(() => []); for (const entry of entries) { - assert.equal( + assert.strictEqual( entry.startsWith(DISCARDED_ASIDE_PREFIX), true, `parked entry ${entry} must carry the discarded prefix, not a recovery-candidate prefix` ); } - assert.equal(await discardDirAside(target, name), false, 'nothing to park the second time'); + assert.strictEqual(await discardDirAside(target, name), false, 'nothing to park the second time'); await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); }); @@ -1212,8 +1216,8 @@ describe('two-phase component directory transaction', function () { ids.push(deploymentId); } const surviving = ids.filter((id) => existsSync(stagedApplicationPath(application.dirPath, id))); - assert.equal(surviving.length, 2, `expected exactly 2 staged builds to survive, got ${surviving.length}`); - assert.equal(surviving.includes(ids.at(-1)), true, 'the just-staged build is never evicted'); + assert.strictEqual(surviving.length, 2, `expected exactly 2 staged builds to survive, got ${surviving.length}`); + assert.strictEqual(surviving.includes(ids.at(-1)), true, 'the just-staged build is never evicted'); } finally { environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, priorMax); } @@ -1225,7 +1229,7 @@ describe('two-phase component directory transaction', function () { try { for (const value of [undefined, '', ' ', true, [], {}, 'abc', 0, -1]) { environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, value); - assert.equal( + assert.strictEqual( getStagingRetentionMaxCount(), DEFAULT_STAGING_RETENTION_MAX_COUNT, `${JSON.stringify(value)} must fall back to the default rather than coerce` @@ -1237,7 +1241,7 @@ describe('two-phase component directory transaction', function () { [2.7, 2], ]) { environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, value); - assert.equal(getStagingRetentionMaxCount(), expected); + assert.strictEqual(getStagingRetentionMaxCount(), expected); } } finally { environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, prior); @@ -1273,7 +1277,7 @@ describe('two-phase component directory transaction', function () { assert.match(await readMarker(application.dirPath), /live-v1/, 'the previous release is live again'); // The candidate is back in staging and its dependency resolves from there, not from the live path. - assert.equal( + assert.strictEqual( await fs.readFile(path.join(stagedPath, 'node_modules', 'probe', 'index.js'), 'utf8'), "module.exports = 'probe';\n", 'the rolled-back candidate resolves its dependency from staging' @@ -1282,7 +1286,7 @@ describe('two-phase component directory transaction', function () { // And it can still be activated on a retry. await activateStagedApplication(application, second, { activationSpec: { package: null } }); assert.match(await readMarker(application.dirPath), /candidate-v2/); - assert.equal( + assert.strictEqual( await fs.readFile(path.join(application.dirPath, 'node_modules', 'probe', 'index.js'), 'utf8'), "module.exports = 'probe';\n", 'the retry re-points the links at the live path' @@ -1318,9 +1322,13 @@ describe('two-phase component directory transaction', function () { async (id) => settled.push(id) ); - assert.equal(reconciliation.failedProjects.has(name), false, 'the live component is not failed closed'); - assert.deepEqual(settled, [deploymentId], 'the unusable candidate row is settled so its payload can be reclaimed'); - assert.equal(reconciliation.removed.includes(deploymentId), true, 'and its residue is removed'); + assert.strictEqual(reconciliation.failedProjects.has(name), false, 'the live component is not failed closed'); + assert.deepStrictEqual( + settled, + [deploymentId], + 'the unusable candidate row is settled so its payload can be reclaimed' + ); + assert.strictEqual(reconciliation.removed.includes(deploymentId), true, 'and its residue is removed'); assert.match(await readMarker(application.dirPath), /live/, 'the live component is untouched'); await cleanup(name); }); @@ -1351,8 +1359,8 @@ describe('two-phase component directory transaction', function () { } ); - assert.equal(reconciliation.errors.has(deploymentId), true, 'the failure is still reported'); - assert.equal( + assert.strictEqual(reconciliation.errors.has(deploymentId), true, 'the failure is still reported'); + assert.strictEqual( reconciliation.failedProjects.has(name), false, 'but it is not attributed to the component, so the healthy live tree still loads' @@ -1385,12 +1393,12 @@ describe('two-phase component directory transaction', function () { ); await fs.chmod(previousRoot, 0o700).catch(() => {}); - assert.equal(committed, 1, 'the commit ran'); - assert.equal(rolledBack, 1, 'and a post-commit failure rolled it back'); + assert.strictEqual(committed, 1, 'the commit ran'); + assert.strictEqual(rolledBack, 1, 'and a post-commit failure rolled it back'); assert.match(await readMarker(application.dirPath), /v2/, 'the pre-revert version is live again'); const target = await getRevertTarget(application.dirPath); - assert.equal(target.live.deployment_id, second, 'the manifest was put back too, so live/manifest agree'); - assert.equal(target.previous.deployment_id, first); + assert.strictEqual(target.live.deployment_id, second, 'the manifest was put back too, so live/manifest agree'); + assert.strictEqual(target.previous.deployment_id, first); await cleanup(name); }); @@ -1413,14 +1421,14 @@ describe('two-phase component directory transaction', function () { const failures = await recoverInterruptedReverts(COMPONENTS_ROOT); - assert.equal(failures.size, 0); + assert.strictEqual(failures.size, 0); assert.match(await readMarker(application.dirPath), /v1/, 'the reverted-to version is live, matching config'); assert.match(await readMarker(previousPath), /v2/, 'the displaced tree is retained'); - assert.equal(existsSync(holding), false); - assert.equal(existsSync(`${holding}.recovering.json`), false); + assert.strictEqual(existsSync(holding), false); + assert.strictEqual(existsSync(`${holding}.recovering.json`), false); const target = await getRevertTarget(application.dirPath); - assert.equal(target.live.deployment_id, first); - assert.equal(target.previous.deployment_id, second); + assert.strictEqual(target.live.deployment_id, first); + assert.strictEqual(target.previous.deployment_id, second); await cleanup(name); }); it('serializes root-config and application-lock writes on a lock outside this isolate', async () => { @@ -1449,19 +1457,26 @@ describe('two-phase component directory transaction', function () { // The holder waits on an external signal, and the commit is started OUTSIDE it. Awaiting the // commit from inside would deadlock: the lock is not reentrant, so the holder could not release // until the commit finished and the commit could not start until the holder released. - const holder = withComponentPreparationLock(lockPath, () => holdReleased); - await new Promise((resolve) => setTimeout(resolve, 100)); + // Signalled from inside the callback rather than slept on: a fixed delay does not prove the + // filesystem lock was actually acquired, so on a loaded runner the commit could win the race. + let holderEntered; + const entered = new Promise((resolve) => (holderEntered = resolve)); + const holder = withComponentPreparationLock(lockPath, () => { + holderEntered(); + return holdReleased; + }); + await entered; const commitPromise = transaction.commit().then(() => (committed = true)); await new Promise((resolve) => setTimeout(resolve, 200)); - assert.equal(committed, false, 'the commit must wait while the persistent-state lock is held'); + assert.strictEqual(committed, false, 'the commit must wait while the persistent-state lock is held'); releaseHold(); await holder; await commitPromise; - assert.equal(committed, true, 'and proceed once it is released'); - assert.deepEqual(readConfigFile()[name], { package: 'example@1.0.0' }); + assert.strictEqual(committed, true, 'and proceed once it is released'); + assert.deepStrictEqual(readConfigFile()[name], { package: 'example@1.0.0' }); const lock = JSON.parse(await fs.readFile(lockPath, 'utf8')); - assert.deepEqual(lock.applications[name], { package: 'example@1.0.0' }); + assert.deepStrictEqual(lock.applications[name], { package: 'example@1.0.0' }); } finally { if (priorRootEnv === undefined) delete process.env.ROOTPATH; else process.env.ROOTPATH = priorRootEnv; @@ -1485,8 +1500,8 @@ describe('two-phase component directory transaction', function () { await Promise.all([a.commit(), b.commit()]); const lock = JSON.parse(await fs.readFile(path.join(configRoot, 'harper-application-lock.json'), 'utf8')); - assert.deepEqual(lock.applications[first], { package: 'a@1.0.0' }, 'the first entry survives'); - assert.deepEqual(lock.applications[second], { package: 'b@1.0.0' }, 'and so does the second'); + assert.deepStrictEqual(lock.applications[first], { package: 'a@1.0.0' }, 'the first entry survives'); + assert.deepStrictEqual(lock.applications[second], { package: 'b@1.0.0' }, 'and so does the second'); } finally { if (priorRootEnv === undefined) delete process.env.ROOTPATH; else process.env.ROOTPATH = priorRootEnv; @@ -1521,13 +1536,81 @@ describe('two-phase component directory transaction', function () { async (id, reason) => settled.push([id, reason]) ); - assert.deepEqual( + assert.deepStrictEqual( settled, [[strandedId, 'the deploy did not survive a restart']], 'only the in-flight row is settled, and with a reason naming the crash' ); - assert.equal(reconciliation.removed.includes(strandedId), true, 'its staging residue is removed'); - assert.equal(reconciliation.removed.includes(terminalId), true, "and so is the terminal row's"); + assert.strictEqual(reconciliation.removed.includes(strandedId), true, 'its staging residue is removed'); + assert.strictEqual(reconciliation.removed.includes(terminalId), true, "and so is the terminal row's"); + await cleanup(name); + }); + + it('does not discard a staged candidate when the completeness probe fails for a reason other than absence', async () => { + // The probe used to map every error to "absent", so a transient EACCES/EIO/EMFILE at startup made a + // complete release look broken and the staged branch deleted it. Only genuine absence may authorize + // destroying a candidate. A self-referencing symlink gives a deterministic ELOOP for this, with no + // dependence on permissions or platform. + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('staged') }); + await stageApplication(application, deploymentId); + const stagedPath = stagedApplicationPath(application.dirPath, deploymentId); + await fs.rm(stagedPath, { recursive: true, force: true }); + await fs.symlink(stagedPath, stagedPath); + + const settled = []; + const reconciliation = await reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async () => ({ deployment_id: deploymentId, project: name, status: 'staged' }), + async () => {}, + async (id, reason) => settled.push([id, reason]) + ); + + assert.deepStrictEqual(settled, [], 'the row is not settled on an inconclusive probe'); + assert.strictEqual(reconciliation.removed.includes(deploymentId), false, 'and its tree is not deleted'); + assert.strictEqual(reconciliation.errors.has(deploymentId), true, 'the failure is reported instead'); + assert.strictEqual( + reconciliation.failedProjects.has(name), + false, + 'a staged candidate never blocks the live component, which is unaffected either way' + ); + await fs.rm(stagedPath, { force: true }); + await cleanup(name); + }); + + it('rolls forward an interrupted no-live restore instead of sweeping away its marker', async () => { + // The no-live revert branch renames `previous` straight to live because there is nothing to + // displace, so it has no holding directory — and its marker is the only evidence. Absence of a + // holding directory therefore must not read as an orphaned marker: sweeping it leaves the + // component with no live tree at all and no record that a restore was in progress. + const name = fixtureName(); + const { application, first } = await twoActivations(name); + const previousPath = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); + const manifest = JSON.parse(await fs.readFile(`${previousPath}.json`, 'utf8')); + + // Crash shape: marker written, live already gone, the retained tree not yet moved into place. + await fs.rm(application.dirPath, { recursive: true, force: true }); + const restoreMarker = `${previousPath}.restoring.recovering.json`; + await fs.writeFile( + restoreMarker, + JSON.stringify({ previous: { deployment_id: null, application_config: null }, live: manifest.previous }, null, 2) + ); + + const failures = await recoverInterruptedReverts(COMPONENTS_ROOT); + + assert.strictEqual(failures.size, 0); + assert.match(await readMarker(application.dirPath), /v1/, 'the retained version is live'); + assert.strictEqual(existsSync(previousPath), false, 'and is no longer parked under retention'); + assert.strictEqual(existsSync(restoreMarker), false, 'the marker is cleared only after everything committed'); + const restored = JSON.parse(await fs.readFile(`${previousPath}.json`, 'utf8')); + assert.strictEqual(restored.live.deployment_id, first, 'the manifest names the version the marker named'); + assert.strictEqual(restored.previous.deployment_id, null, 'with nothing retained behind it'); + assert.strictEqual( + await getRevertTarget(application.dirPath), + undefined, + 'so the component reports as not revertable until its next deploy' + ); await cleanup(name); }); @@ -1545,14 +1628,20 @@ describe('two-phase component directory transaction', function () { await fs.mkdir(backupPath, { recursive: true }); await fs.writeFile(path.join(backupPath, 'index.js'), "module.exports = 'backup';\n"); // The swap window this models: no live directory, only the backup. - assert.equal(existsSync(livePath), false); + assert.strictEqual(existsSync(livePath), false); let releaseHold; const holdReleased = new Promise((resolve) => (releaseHold = resolve)); // Reconcile is started OUTSIDE the holder — awaiting it from inside would deadlock, since the lock - // is not reentrant. - const holder = withComponentPreparationLock(livePath, () => holdReleased); - await new Promise((resolve) => setTimeout(resolve, 100)); + // is not reentrant. Entry is signalled from inside the callback, not slept on, so the assertions + // below cannot pass merely because the sweep lost a timing race. + let holderEntered; + const entered = new Promise((resolve) => (holderEntered = resolve)); + const holder = withComponentPreparationLock(livePath, () => { + holderEntered(); + return holdReleased; + }); + await entered; let reconciled = false; const reconcilePromise = reconcileStagedApplicationArtifacts( @@ -1565,15 +1654,15 @@ describe('two-phase component directory transaction', function () { }); await new Promise((resolve) => setTimeout(resolve, 300)); - assert.equal(reconciled, false, 'the sweep must not finish while the component lock is held'); - assert.equal(existsSync(livePath), false, 'and must not restore the backup underneath the lock holder'); - assert.equal(existsSync(backupPath), true, 'leaving the backup for after the lock is released'); + assert.strictEqual(reconciled, false, 'the sweep must not finish while the component lock is held'); + assert.strictEqual(existsSync(livePath), false, 'and must not restore the backup underneath the lock holder'); + assert.strictEqual(existsSync(backupPath), true, 'leaving the backup for after the lock is released'); releaseHold(); await holder; await reconcilePromise; - assert.equal(existsSync(backupPath), false, 'once released, the sweep settles the artifact'); + assert.strictEqual(existsSync(backupPath), false, 'once released, the sweep settles the artifact'); assert.match(await readMarker(livePath), /backup/, 'restoring the backup over the absent live path'); await cleanup(name); }); @@ -1584,7 +1673,7 @@ describe('two-phase component directory transaction', function () { it('resolves empty when the retention root is absent but rejects when it is unreadable', async () => { const scanRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'revert-scan-')); try { - assert.equal((await recoverInterruptedReverts(scanRoot)).size, 0); + assert.strictEqual((await recoverInterruptedReverts(scanRoot)).size, 0); await fs.writeFile(path.join(scanRoot, DEPLOY_PREVIOUS_DIR), 'not a directory\n'); await assert.rejects(() => recoverInterruptedReverts(scanRoot), { code: 'ENOTDIR' }); From 63a713e9a42880e2a904d7031c443965b035bdd3 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 16:04:37 -0400 Subject: [PATCH 77/94] fix(deploy): keep recovery and retention from destroying releases they cannot prove are surplus MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Six more findings from the pre-push review. Recovery deleted the tree it should have retained. When a process died after the swap and config commit but before retention ran, the displaced release was still parked as `.deploy-activating//.previous--…`; both recovery paths cleared it as residue, so a recovered deploy came back permanently unrevertable. Both now run it through the retained-previous protocol and write its manifest. An activation resumed after boot could never complete. `installApplications()` runs before this recovery and recreates a package component from root config when it finds the live path missing — exactly the state an activation that died between its two renames leaves. Resuming reuses the existing backup, so nothing moved the recreated tree away and `rename(staging, live)` failed ENOTEMPTY; since installation recreated it on every boot, the component was permanently unloadable. The recreated directory is now parked before the rename. Retention could expire a deployment that had already been reported to the operator. Both halves — deployment rows and staging directories — reserved the current request by excluding it from the ranking, which privileges it unconditionally: a stage delayed behind slow peers finishes with an older timestamp than a sibling that started later and already returned, and that newer sibling was expired and its tree discarded cluster-wide. Clock skew on `started_at`, stamped by the originating node, produces the same inversion. Both now protect anything strictly newer than the current request, which also keeps the count exact when timestamps tie. Filesystem pruning additionally runs under the component lock so two stages cannot race their decisions. Dependency-link repair claimed not to be best-effort while collapsing every probe error to absence, so a transient EIO could let an activation swap in a tree whose absolute links still addressed the staging path. Only ENOENT is absence now. The SSE fallback timer awaited a row read inside its callback: a rejection escaped the promise being awaited (unhandled, and fatal by default) and the request never settled because the timer was already cleared. It also hung forever when the row was confirmed absent. It no longer reads anything — the row is re-read below for the final payload — and settles unconditionally. Terminal transitions now clear `error`, so a recovery or a successful retry can no longer leave a row reporting success and a failure at once. Reported by codex and cursor-grok pre-push. --- components/Application.ts | 150 ++++++++++++++---- components/deploymentOperations.ts | 24 +-- components/deploymentRecorder.ts | 23 ++- unitTests/components/deployStaging.test.js | 93 ++++++++++- .../components/deploymentRecorder.test.js | 36 +++++ 5 files changed, 272 insertions(+), 54 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 5a16eef3c4..aac5b3e955 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -589,32 +589,45 @@ async function pruneStagedBuilds(componentName: string, keepStagingId: string, m if ((err as any).code === 'ENOENT') return; // nothing staged yet throw err; } - // Collect this component's staged builds: // that still exist. - const builds: Array<{ stagingId: string; parentPath: string; mtime: number }> = []; - for (const parent of parents) { - if (!parent.isDirectory()) continue; - const parentPath = join(stagingRoot, parent.name); - try { - const st = await stat(join(parentPath, componentName)); - builds.push({ stagingId: parent.name, parentPath, mtime: st.mtimeMs }); - } catch (err) { - if ((err as any).code !== 'ENOENT') throw err; // parent holds a different component; skip + // Under the component lock, so two stages of the same component cannot each enumerate before the + // other evicts and race their decisions. A lock we cannot get lands in the catch below, which is + // right for a disk bound: skipping a prune costs space, not correctness. + await withComponentPreparationLock(join(componentsRoot, componentName), async () => { + // Collect this component's staged builds: // that still exist. + const builds: Array<{ stagingId: string; parentPath: string; mtime: number }> = []; + for (const parent of parents) { + if (!parent.isDirectory()) continue; + const parentPath = join(stagingRoot, parent.name); + try { + const st = await stat(join(parentPath, componentName)); + builds.push({ stagingId: parent.name, parentPath, mtime: st.mtimeMs }); + } catch (err) { + if ((err as any).code !== 'ENOENT') throw err; // parent holds a different component; skip + } } - } - // Always keep the build we just made, plus the newest (maxCount - 1) of the OTHERS by mtime; evict - // the rest. Computing it as "keep keepStagingId + top-(N-1) others" (rather than "evict everything - // past the top N") keeps the count exact even when mtimes tie and the just-built one would - // otherwise sort into the eviction window. Await the evictions (best-effort via allSettled) so the - // retention count is settled by the time the stage returns. - const others = builds.filter((build) => build.stagingId !== keepStagingId).sort((a, b) => b.mtime - a.mtime); - const evictions = others - .slice(Math.max(0, maxCount - 1)) - .map((build) => - rm(build.parentPath, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }).catch((err) => - logger.trace?.(`Deferred prune of staged ${componentName} build ${build.stagingId}: ${err.message}`) - ) - ); - await Promise.allSettled(evictions); + // Keep the build just made, plus anything strictly NEWER than it, plus the newest of the rest up + // to the count. Filling from "the others" alone would privilege the current build + // unconditionally: a stage delayed behind slow peers finishes with an older mtime than a + // sibling that started later and already returned, and that newer sibling's tree would be + // evicted. Protecting only *strictly* newer builds keeps the count exact when mtimes tie — + // coarse filesystem timestamps routinely tie — where the just-made build could otherwise sort + // into the eviction window. Mirrors deploymentRecorder.settleStagedRows. + const current = builds.find((build) => build.stagingId === keepStagingId); + const others = builds.filter((build) => build.stagingId !== keepStagingId).sort((a, b) => b.mtime - a.mtime); + const newerThanCurrent = current ? others.filter((build) => build.mtime > current.mtime) : []; + const budget = Math.max(0, maxCount - (current ? 1 : 0) - newerThanCurrent.length); + // Awaited (best-effort via allSettled) so the retention count is settled by the time the stage + // returns. + const evictions = others + .filter((build) => !newerThanCurrent.includes(build)) + .slice(budget) + .map((build) => + rm(build.parentPath, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }).catch((err) => + logger.trace?.(`Deferred prune of staged ${componentName} build ${build.stagingId}: ${err.message}`) + ) + ); + await Promise.allSettled(evictions); + }); } catch (err) { logger.trace?.(`Staged-build prune for ${componentName} skipped: ${(err as Error).message}`); } @@ -2959,7 +2972,7 @@ async function repointStagedDependencyLinks( // machine would otherwise have its target's contents enumerated — and a link in there whose target // happened to point into staging would be removed and recreated, writing outside the component tree. // Requiring a real directory at each level we descend keeps every candidate under the real root. - const nodeModulesStat = await lstat(nodeModulesPath).catch(() => undefined); + const nodeModulesStat = await statIfPresent(nodeModulesPath); if (!nodeModulesStat?.isDirectory() || nodeModulesStat.isSymbolicLink()) return 0; // Package links live at `node_modules/` or `node_modules/@scope/`, and npm nests a further // `node_modules` under a package whenever hoisting is blocked by a version conflict — routinely so @@ -2967,21 +2980,31 @@ async function repointStagedDependencyLinks( // walk has to follow real nested trees rather than stopping at the first two levels. const candidates: string[] = []; const collect = async (directoryPath: string): Promise => { - for (const entry of await readdir(directoryPath, { withFileTypes: true }).catch(() => [])) { + let entries; + try { + entries = await readdir(directoryPath, { withFileTypes: true }); + } catch (error) { + // Absence is benign; anything else is not. Swallowing an EIO/EACCES here would make the walk + // find nothing to repoint and let the activation swap in a tree whose absolute links still + // address the staging path — the dangling state this repair exists to prevent. + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + return; + } + for (const entry of entries) { if (entry.name.startsWith('.')) continue; const entryPath = join(directoryPath, entry.name); if (entry.name.startsWith('@')) { // Only descend into a REAL directory: a symlinked scope directory belongs to whatever it points // at, not to this component (see the note on nodeModulesStat above). - const scopeStat = await lstat(entryPath).catch(() => undefined); + const scopeStat = await statIfPresent(entryPath); if (scopeStat?.isDirectory() && !scopeStat.isSymbolicLink()) await collect(entryPath); continue; } candidates.push(entryPath); - const packageStat = await lstat(entryPath).catch(() => undefined); + const packageStat = await statIfPresent(entryPath); if (!packageStat?.isDirectory() || packageStat.isSymbolicLink()) continue; const nestedPath = join(entryPath, 'node_modules'); - const nestedStat = await lstat(nestedPath).catch(() => undefined); + const nestedStat = await statIfPresent(nestedPath); if (nestedStat?.isDirectory() && !nestedStat.isSymbolicLink()) await collect(nestedPath); } }; @@ -2989,10 +3012,16 @@ async function repointStagedDependencyLinks( let repointed = 0; for (const linkPath of candidates) { - const linkStat = await lstat(linkPath).catch(() => undefined); + const linkStat = await statIfPresent(linkPath); if (!linkStat?.isSymbolicLink()) continue; - const target = await readlink(linkPath).catch(() => undefined); - if (!target || !isAbsolute(target)) continue; + let target; + try { + target = await readlink(linkPath); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; + continue; + } + if (!isAbsolute(target)) continue; // Belt-and-braces containment: never mutate a path that is not under the real node_modules root. const withinNodeModules = relative(nodeModulesPath, linkPath); if (withinNodeModules.startsWith('..') || isAbsolute(withinNodeModules)) continue; @@ -3062,7 +3091,22 @@ export async function activateStagedApplication( const existingArtifacts = await activationArtifacts(application.dirPath, deploymentId); backupPath = existingArtifacts.find((candidate) => basename(candidate).startsWith(ACTIVATION_BACKUP_PREFIX)); newMarkerPath = existingArtifacts.find((candidate) => basename(candidate).startsWith(ACTIVATION_NEW_PREFIX)); - if (!backupPath && !newMarkerPath) { + if (backupPath || newMarkerPath) { + // Resuming an attempt that already moved the displaced release aside (or recorded that there + // was none). Anything at the live path now therefore arrived AFTER that move, so it is not the + // tree this activation displaced: boot-time installApplications() recreates a package component + // from root config whenever it finds the path missing, and it runs before this recovery. Left + // in place it makes the rename below fail ENOTEMPTY — deterministically, on every boot, because + // installation recreates it again each time. Parked rather than deleted; the release this + // activation actually displaced is the one already held in the backup. + if (await statIfPresent(application.dirPath)) { + logger.warn( + `Parking an unexpected ${application.name} directory that appeared after this activation ` + + `moved the live tree aside; the displaced release is already retained` + ); + await discardDirAside(application.dirPath, application.name); + } + } else { try { await lstat(application.dirPath); application.isNewComponent = false; @@ -3221,6 +3265,34 @@ function activationArtifactDeploymentId(name: string): string | undefined { return DEPLOYMENT_ID_PATTERN.test(deploymentId) ? deploymentId : undefined; } +/** + * Finish the retention half of an interrupted activation. The tree the swap displaced is still parked + * as `.deploy-activating//.previous--…`, and clearing it as residue is exactly what makes + * a recovered deploy unrevertable — the crash shape here is "swapped and committed, but died before + * retaining". Runs the same protocol a normal activation does, so the manifest describes the tree. + */ +async function retainRecoveredActivation( + componentDirPath: string, + componentName: string, + deploymentId: string, + activationSpec: Record | undefined +): Promise { + const backupPath = (await activationArtifacts(componentDirPath, deploymentId)).find((candidate) => + basename(candidate).startsWith(ACTIVATION_BACKUP_PREFIX) + ); + if (!backupPath) return; + // Read before the retain overwrites it: the manifest's `live` is the release this activation displaced. + const outgoing: RetainedVersion = (await readRetainedPreviousManifest(componentDirPath).catch(() => undefined)) + ?.live ?? { deployment_id: null, application_config: null }; + await retainActivatedPrevious( + new Application({ name: componentName }), + backupPath, + deploymentId, + activationSpec ? applicationConfigFromActivationSpec(activationSpec) : undefined, + outgoing + ); +} + /** * The project a staged deployment belongs to, read from `//`. Used to * attribute a reconciliation failure when the deployment row itself could not be read. @@ -3347,6 +3419,8 @@ export async function reconcileStagedApplicationArtifacts( throw new Error(`Interrupted activation '${entry.name}' has neither a staged nor live component tree`); } await persistActivation(row); + // The swap already happened, so the displaced tree is the rollback source now. + await retainRecoveredActivation(componentDirPath, row.project, entry.name, row.activation_spec); }); if (ownedByActivation) continue; } @@ -3408,7 +3482,13 @@ export async function reconcileStagedApplicationArtifacts( if (row?.status === 'activating' && row.project === projectEntry.name && liveStat?.isDirectory()) { try { await persistActivation(row); - await rm(artifactPath, { recursive: true, force: true }); + if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX)) { + // The displaced tree, not residue: retaining it is what keeps a recovered deploy + // revertable. The retain consumes the artifact, so there is nothing left to remove. + await retainRecoveredActivation(livePath, projectEntry.name, deploymentId, row.activation_spec); + } else { + await rm(artifactPath, { recursive: true, force: true }); + } recovered.add(deploymentId); } catch (error) { const reconcileError = error instanceof Error ? error : new Error(String(error)); diff --git a/components/deploymentOperations.ts b/components/deploymentOperations.ts index fd53472735..d776f1158d 100644 --- a/components/deploymentOperations.ts +++ b/components/deploymentOperations.ts @@ -164,23 +164,23 @@ export async function handleGetDeployment(req: GetRequest): Promise { // Safety net — if the in-memory emitter is dropped (recorder finished or // the process recycled) before signaling, poll the row's status as a // fallback so the client never hangs indefinitely. - const pollTimer = setInterval(async () => { + const pollTimer = setInterval(() => { if (liveDone) { clearInterval(pollTimer); return; } const live = getActiveEmitter(req.deployment_id); - if (!live || live !== liveEmitter) { - clearInterval(pollTimer); - const latest = await table.get(req.deployment_id); - // `staged` is a valid resting result and `activating` records - // uncertain cluster completion. If the original emitter is gone, - // there is no local work left to tail in either state. - if (latest && !liveDone) { - liveDone = true; - resolve(); - } - } + if (live && live === liveEmitter) return; + // The emitter this request was tailing is gone, so there is no local work left to + // follow: `staged` is a valid resting result and `activating` records uncertain + // cluster completion. Settle unconditionally — including when the row has been + // reclaimed — because clearing the timer without resolving hangs the request + // forever. Nothing is read here on purpose: an await inside a timer callback can + // reject outside the promise being awaited, which is an unhandled rejection AND + // leaves this promise unsettled. The row is re-read below for the final payload. + clearInterval(pollTimer); + liveDone = true; + resolve(); }, 500); }); } diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index 6d6ec6b58e..e3aecdb0ef 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -514,6 +514,10 @@ export async function markDeploymentTerminal( update.error = { message: error instanceof Error ? error.message : String(error), }; + } else { + // Cleared, not left alone: a recovery or a successful activate retry moves a row that already + // carries a failure, and keeping that object would report success and an error at once. + update.error = null; } await table.patch(deploymentId, update); } @@ -576,16 +580,23 @@ async function settleStagedRows(project: string, keepCount: number, keepDeployme if (!table) return []; const staged: Array> = []; for await (const row of table.search([{ attribute: 'project', value: project }])) { - // `started_at` is stamped by whichever node originated the deploy, so clock skew across nodes can - // rank a peer-originated row above the one this call just created. Excluding it explicitly is what - // stops retention cluster-wide discarding the staging tree for the id being returned to the - // operator — the same guarantee pruneStagedBuilds gets from its keepStagingId. - if (row?.status === 'staged' && row.deployment_id !== keepDeploymentId) staged.push(row); + if (row?.status === 'staged') staged.push(row); } staged.sort( (a, b) => (b.started_at ?? 0) - (a.started_at ?? 0) || String(a.deployment_id).localeCompare(b.deployment_id) ); - const expired = staged.slice(Math.max(0, keepDeploymentId ? keepCount - 1 : keepCount)); + // Keep the request being returned, plus any row strictly NEWER than it, plus the newest of the rest + // up to the count. Excluding the current id from the ranking instead would privilege it + // unconditionally: a stage that waited on slow peers resumes with an older `started_at`, so with a + // retention of 1 it would expire the newer stage that had already completed and been reported to the + // operator, then discard its staging tree cluster-wide. `started_at` is stamped by whichever node + // originated the deploy, so cross-node clock skew produces the same inversion. Only *strictly* newer + // rows are protected, which keeps the count exact when timestamps tie. + const current = keepDeploymentId ? staged.find((row) => row.deployment_id === keepDeploymentId) : undefined; + const others = staged.filter((row) => row.deployment_id !== keepDeploymentId); + const newerThanCurrent = current ? others.filter((row) => (row.started_at ?? 0) > (current.started_at ?? 0)) : []; + const budget = Math.max(0, keepCount - (current ? 1 : 0) - newerThanCurrent.length); + const expired = others.filter((row) => !newerThanCurrent.includes(row)).slice(budget); for (const row of expired) { await table.patch(row.deployment_id, { status: 'failed', diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 2e0924f015..a0afba36e0 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -340,7 +340,9 @@ describe('two-phase component directory transaction', function () { const activationPath = path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR, name); await fs.mkdir(livePath, { recursive: true }); await fs.writeFile(path.join(livePath, 'index.js'), 'module.exports = "candidate";\n'); - await fs.mkdir(path.join(activationPath, `.previous-${deploymentId}-crash`), { recursive: true }); + const backupDir = path.join(activationPath, `.previous-${deploymentId}-crash`); + await fs.mkdir(backupDir, { recursive: true }); + await fs.writeFile(path.join(backupDir, 'index.js'), 'module.exports = "displaced";\n'); const row = { deployment_id: deploymentId, project: name, @@ -359,6 +361,13 @@ describe('two-phase component directory transaction', function () { assert.deepStrictEqual(result.recovered, [deploymentId]); assert.match(await readMarker(livePath), /candidate/); assert.strictEqual(existsSync(activationPath), false); + // This crash shape died after the swap but before retention, so the tree under `.previous-*` is the + // only copy of what the activation displaced. Clearing it as residue would leave the recovered + // deploy permanently unrevertable, which is the whole point of retaining it. + const previousPath = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); + assert.match(await readMarker(previousPath), /displaced/, 'the displaced tree is retained, not deleted'); + const target = await getRevertTarget(livePath); + assert.strictEqual(target.live.deployment_id, deploymentId, 'and the manifest names the recovered deployment'); await cleanup(name); }); @@ -1202,6 +1211,46 @@ describe('two-phase component directory transaction', function () { await fs.rm(path.join(COMPONENTS_ROOT, ASIDE_STAGING_DIR), { recursive: true, force: true }); }); + it('does not evict a staged build newer than the one being staged', async () => { + // The filesystem half of the retention inversion: a stage delayed behind slow peers finishes with + // an older mtime than a sibling that started later and already returned. Filling the retention + // budget from "the others" alone always reserved the current build and evicted that newer + // sibling's tree, so the deployment the operator was just told about lost its staged bytes. + const name = fixtureName(); + const priorMax = environment.get(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT); + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, 1); + try { + const application = new Application({ name, payload: await makeComponentPayload('slow') }); + const slowId = randomUUID(); + await stageApplication(application, slowId); + application.payload = await makeComponentPayload('newer'); + const newerId = randomUUID(); + await stageApplication(application, newerId); + + // Make the newer build unambiguously newer on disk, then re-run the slow stage's prune by + // staging it again — which is what a delayed origin resuming its own stage does. + const newerPath = stagedApplicationPath(application.dirPath, newerId); + const future = new Date(Date.now() + 60_000); + await fs.utimes(newerPath, future, future); + application.payload = await makeComponentPayload('slow'); + await stageApplication(application, slowId); + + assert.strictEqual( + existsSync(newerPath), + true, + 'the newer sibling survives a prune driven by an older, still-finishing stage' + ); + assert.strictEqual( + existsSync(stagedApplicationPath(application.dirPath, slowId)), + true, + 'and the build being staged is kept too' + ); + } finally { + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, priorMax); + } + await cleanup(name); + }); + it('evicts staged builds beyond the retention count and always keeps the newest', async () => { const name = fixtureName(); const priorMax = environment.get(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT); @@ -1546,6 +1595,48 @@ describe('two-phase component directory transaction', function () { await cleanup(name); }); + it('rolls a crashed activation forward even after boot-time installation recreated the live directory', async () => { + // installApplications() runs BEFORE this recovery and recreates a package component from root + // config whenever the live path is missing — which is exactly the state an activation that died + // between its two renames leaves behind. Resuming that attempt reuses the existing backup, so + // nothing moves the recreated tree away and `rename(staging, live)` used to fail ENOTEMPTY. Being + // recreated on every boot, that made the component permanently unloadable. + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('displaced') }); + await stageApplication(application, deploymentId); + await fs.mkdir(application.dirPath, { recursive: true }); + await fs.writeFile(path.join(application.dirPath, 'index.js'), "module.exports = 'displaced';\n"); + application.payload = await makeComponentPayload('candidate'); + const secondId = randomUUID(); + await stageApplication(application, secondId); + + // The crash shape: live already moved aside under this deployment's backup, staging not yet moved. + const activationDir = path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR, name); + await fs.mkdir(activationDir, { recursive: true }); + await fs.rename(application.dirPath, path.join(activationDir, `.previous-${secondId}-1-1-${randomUUID()}`)); + // Then boot-time installation recreates the live directory from root config. + await fs.mkdir(application.dirPath, { recursive: true }); + await fs.writeFile(path.join(application.dirPath, 'index.js'), "module.exports = 'reinstalled';\n"); + + const reconciliation = await reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async (id) => (id === secondId ? { deployment_id: secondId, project: name, status: 'activating' } : undefined), + async () => {} + ); + + assert.deepStrictEqual([...reconciliation.errors.keys()], [], 'the roll-forward succeeds'); + assert.strictEqual(reconciliation.failedProjects.has(name), false, 'so the component is not failed closed'); + assert.match(await readMarker(application.dirPath), /candidate/, 'the staged candidate is live'); + const previousPath = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); + assert.match( + await readMarker(previousPath), + /displaced/, + 'and the release the activation actually displaced is retained, not the reinstalled tree' + ); + await cleanup(name); + }); + it('does not discard a staged candidate when the completeness probe fails for a reason other than absence', async () => { // The probe used to map every error to "absent", so a transient EACCES/EIO/EMFILE at startup made a // complete release look broken and the staged branch deleted it. Only genuine absence may authorize diff --git a/unitTests/components/deploymentRecorder.test.js b/unitTests/components/deploymentRecorder.test.js index f828f04a25..9efea692b7 100644 --- a/unitTests/components/deploymentRecorder.test.js +++ b/unitTests/components/deploymentRecorder.test.js @@ -841,6 +841,42 @@ describe('staged deployment state', () => { assert.strictEqual(installed.mock.rows.get('active').status, 'activating'); }); + it('does not expire a newer staged deployment when an older one finishes late', async () => { + // A stage that waited on slow peers resumes with an older `started_at` than a stage that started + // after it and already returned. Reserving the resuming id by excluding it from the ranking made + // retention expire that newer, already-reported deployment — and then discard its staging tree + // cluster-wide. Cross-node clock skew on `started_at` produces the same inversion. + for (const [id, startedAt] of [ + ['slow-origin', 100], + ['finished-later', 200], + ]) { + installed.mock.rows.set(id, { deployment_id: id, project: 'app', status: 'staged', started_at: startedAt }); + } + + assert.deepStrictEqual( + await expireOldStagedDeployments('app', 1, 'slow-origin'), + [], + 'neither is surplus: one is newer, the other is the request being returned' + ); + assert.strictEqual(installed.mock.rows.get('finished-later').status, 'staged'); + assert.strictEqual(installed.mock.rows.get('slow-origin').status, 'staged'); + }); + + it('still expires rows older than both the window and the returning request', async () => { + for (const [id, startedAt] of [ + ['ancient', 50], + ['slow-origin', 100], + ['finished-later', 200], + ]) { + installed.mock.rows.set(id, { deployment_id: id, project: 'app', status: 'staged', started_at: startedAt }); + } + + assert.deepStrictEqual(await expireOldStagedDeployments('app', 1, 'slow-origin'), ['ancient']); + assert.strictEqual(installed.mock.rows.get('ancient').status, 'failed'); + assert.strictEqual(installed.mock.rows.get('slow-origin').status, 'staged'); + assert.strictEqual(installed.mock.rows.get('finished-later').status, 'staged'); + }); + it('invalidates staged and activating rows when their component is dropped', async () => { for (const status of ['staged', 'activating', 'success']) { installed.mock.rows.set(status, { deployment_id: status, project: 'app', status }); From 20c639095c7252a675837ce72f746043e9b0ad8e Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 16:08:04 -0400 Subject: [PATCH 78/94] chore(deploy): drop review-provenance comments and decorative dividers Comments attributing a rationale to this PR's own review conversation are history, not invariants, and stop meaning anything once merged. Issue references that name a durable requirement are kept; the reviewer handles and "harper#1849 review" tags are gone, along with the box-drawing section dividers. Also covers the activation-artifact lookup-failure path with a test: a throwing lookup blocks the component and leaves both the artifact and the retained previous release untouched. --- components/Application.ts | 12 +++--- components/operations.js | 4 +- components/operationsValidation.js | 2 +- .../components/deployPhaseOperations.test.js | 4 +- unitTests/components/deployStaging.test.js | 42 ++++++++++++++----- 5 files changed, 41 insertions(+), 23 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index aac5b3e955..046998ce08 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -644,8 +644,8 @@ async function pruneStagedBuilds(componentName: string, keepStagingId: string, m * worker harmlessly keeps writing into the renamed inode until it exits on restart. * * The `.discarded-` prefix is load-bearing, and is the whole reason this exists alongside the - * extraction transaction's `.in-progress-` asides. `.deploy-aside` has exactly ONE contract - * (harper#1849 review, @kriszyp): an `.in-progress-` directory with no matching `.retired-` marker is + * extraction transaction's `.in-progress-` asides. `.deploy-aside` has exactly ONE contract: an + * `.in-progress-` directory with no matching `.retired-` marker is * a ROLLBACK RECORD, and `recoverInterruptedComponentExtractions` restores the newest such directory * OVER the live component path at startup. A tree that is already known to be garbage when it is * parked — the evicted two-deploys-ago retained-previous below — must therefore never carry that @@ -683,8 +683,8 @@ export async function discardDirAside(targetDirPath: string, componentName: stri * `.deploy-previous/`, which produced the tree now LIVE, and the root-config entry each one was * activated with. * - * This is what makes `revert_component` addressable rather than a blind toggle (harper#1849 review, - * @kriszyp): the caller names the deployment it expects to end up live, so a retried request whose + * This is what makes `revert_component` addressable rather than a blind toggle: the caller names the + * deployment it expects to end up live, so a retried request whose * response was lost is a no-op instead of flipping the rejected release back in. It is also what lets * a revert restore persistent state, not just the directory: `application_config` is the root-config * entry (and install-lock entry) that belongs with each tree, so reverting away from a `package` @@ -1081,7 +1081,7 @@ export async function getRevertTarget( * deploy, run their own health checks, and swap back fast — with no package resolution, artifact * download or install, because the bytes are already on disk. * - * ADDRESSED, NOT TOGGLED (harper#1849 review, @kriszyp). The caller passes the deployment id it + * ADDRESSED, NOT TOGGLED. The caller passes the deployment id it * expects to be live when this returns: * - already live → a no-op success, so an ordinary request retry whose first response was lost * cannot swap the rejected release back in. @@ -3114,7 +3114,7 @@ export async function activateStagedApplication( // dependency or module-entry change — but it does invalidate already-loaded code. The one-shot // path compares it across its in-place install (prepareApplication); the two-phase path has to // compare the outgoing live tree against the staged one, which is only possible here, while - // both still exist. Feeds markRestartRequiredForDeploy (harper#674, harper#1849 @heskew). + // both still exist. Feeds markRestartRequiredForDeploy (harper#674). application.packageMetadataChanged = installedRuntimeChanged( await readInstalledPackageMetadata(application.dirPath), await readInstalledPackageMetadata(stagingDirPath), diff --git a/components/operations.js b/components/operations.js index 22844c9560..a63d24f637 100644 --- a/components/operations.js +++ b/components/operations.js @@ -1235,7 +1235,7 @@ function isTrustedReplicatedOperation(req) { * (`deploy_component`), not a revert. * * `to_deployment_id` is required and names the deployment the caller expects to be live afterwards, - * which is what makes the operation idempotent under ordinary request retries (harper#1849 review): if + * which is what makes the operation idempotent under ordinary request retries: if * that version is already live the call succeeds without touching anything, instead of toggling the * rejected release back in. Targeting the retained previous swaps, and the displaced tree becomes the * new retained previous — so an explicitly-targeted revert-of-a-revert still rolls forward. @@ -1377,10 +1377,8 @@ async function restartRevertedComponent(req, emit) { return { restartMessage: '' }; } -// ———————————————————————————————————————————————————————————————————————————— // Shared deploy-family helpers (used by deploy_component, its component_deploy_phase fan-out, and // revert_component). -// ———————————————————————————————————————————————————————————————————————————— // Reject deploying over a protected core component name unless force is set. Lazy-loads // componentLoader to avoid a circular dependency. diff --git a/components/operationsValidation.js b/components/operationsValidation.js index 6c84c1f78a..1877a1ca27 100644 --- a/components/operationsValidation.js +++ b/components/operationsValidation.js @@ -592,7 +592,7 @@ function revertComponentValidator(req) { // The deployment the caller expects to be live once this returns. REQUIRED, and the whole reason // revert is safe to retry: a revert that names its target is a no-op when that version is already // live, where a bare "swap to the other one" toggle would flip the rejected release back in if a - // caller retried after losing the first response (harper#1849 review). + // caller retried after losing the first response. to_deployment_id: Joi.string().pattern(DEPLOYMENT_ID_REGEX).required().messages({ 'string.pattern.base': `'to_deployment_id' must be a UUID`, 'any.required': `'to_deployment_id' is required: name the deployment you expect to be live after the revert (list_deployments reports it, and deploy_component returns it)`, diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index 785b8fdf5f..9e0e6497c8 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -264,9 +264,7 @@ describe('deploy_component two-phase orchestration', function () { assert.strictEqual(restartNeeded(), true, 'a new component activated without restart requires one'); }); - // ———————————————————————————————————————————————————————————————————————————— - // revert_component (harper#1849 review, @kriszyp) - // ———————————————————————————————————————————————————————————————————————————— + // revert_component it('reverts the cluster to a named previous deployment and fans the target out to peers', async () => { const project = name(); diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index a0afba36e0..2eb654b534 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -408,9 +408,7 @@ describe('two-phase component directory transaction', function () { await cleanup(name); await fs.rm(packageDirectory, { recursive: true, force: true }); }); - // ———————————————————————————————————————————————————————————————————————————— - // Retained previous + addressed revert (harper#1849 review, @kriszyp) - // ———————————————————————————————————————————————————————————————————————————— + // Retained previous + addressed revert it('retains the tree an activation displaced, addressed by the deployment that produced it', async () => { const name = fixtureName(); @@ -617,10 +615,7 @@ describe('two-phase component directory transaction', function () { assert.match(await readMarker(application.dirPath), /over-dead-link/); await cleanup(name); }); - // ———————————————————————————————————————————————————————————————————————————— - // Restart gate: package metadata compared across the swap - // (harper#674 / harper#1849 @heskew finding 2) - // ———————————————————————————————————————————————————————————————————————————— + // Restart gate: package metadata compared across the swap (harper#674) // // main's in-place prepareApplication compares installed package metadata before extraction against // after install; the two-phase path never touches the live directory until the swap, so the @@ -692,9 +687,7 @@ describe('two-phase component directory transaction', function () { assert.strictEqual(first.packageMetadataChanged, false, 'nothing to compare against; isNewComponent carries it'); await cleanup(name); }); - // ———————————————————————————————————————————————————————————————————————————— // Interrupted revert: compensation and startup recovery - // ———————————————————————————————————————————————————————————————————————————— async function twoActivations(name) { const first = randomUUID(); @@ -714,7 +707,7 @@ describe('two-phase component directory transaction', function () { // all rename cleanly onto a free path). Inducing it would need a fault-injection seam in production // code. The compensation is still there — it is what turns an exceptional I/O error into "the // component keeps serving what it was serving" — but the durable half below is what actually covers - // the reviewer's scenario: a process that dies mid-swap, which compensation inherently cannot handle. + // the case that matters: a process that dies mid-swap, which compensation inherently cannot handle. it('startup recovery restores the live tree from a revert that died before the swap', async () => { const name = fixtureName(); @@ -1705,6 +1698,35 @@ describe('two-phase component directory transaction', function () { await cleanup(name); }); + it('fails a component closed when an activation-artifact lookup throws, destroying nothing', async () => { + // A failed lookup cannot distinguish "no such deployment" from "the table is unreadable", so it may + // not authorize cleanup: the artifact could be the displaced release, and the live tree could be + // mid-activation and disagreeing with its durable config. Block the component and touch nothing. + const name = fixtureName(); + const deploymentId = randomUUID(); + await twoActivations(name); + const previousPath = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); + const activationProjectDir = path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR, name); + await fs.mkdir(activationProjectDir, { recursive: true }); + const artifactPath = path.join(activationProjectDir, `.previous-${deploymentId}-1-1-${randomUUID()}`); + await fs.mkdir(artifactPath, { recursive: true }); + + const reconciliation = await reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async () => { + throw new Error('deployment table unavailable'); + }, + async () => {} + ); + + assert.strictEqual(reconciliation.failedProjects.has(name), true, 'the component does not load'); + assert.strictEqual(reconciliation.errors.has(deploymentId), true, 'and the failure is reported'); + assert.strictEqual(existsSync(artifactPath), true, 'the artifact is left alone'); + assert.match(await readMarker(previousPath), /v1/, 'and the retained previous release is untouched'); + await fs.rm(activationProjectDir, { recursive: true, force: true }); + await cleanup(name); + }); + it('waits for the component lock before restoring an activation backup over the live path', async () => { // The sweep renames a backup back over the live path. Run unlocked, a reload-cycle retry could do // that inside an activation's swap window — live momentarily absent, backup present — after which From fb03ee5e8ed2ce129d4d52a3206f00b6f4067ac1 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 16:31:08 -0400 Subject: [PATCH 79/94] fix(deploy): compensate a partial persistent commit, and stop rejecting a live symlink MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four findings from the second cross-model round. Revert compensation was gated on the config commit having *resolved*. The config transaction marks itself started before writing root config and can reject between that write and the application-lock write, so a rejected commit may already have changed persisted state — and that case skipped the rollback entirely, restoring the old directories while config named the reverted-to release, which a cold start would then reinstall over. Compensation now runs after any commit *attempt*. If the rollback itself fails, the directories and the recovery marker are deliberately left untouched instead of being undone and the evidence consumed: persisted state may name the reverted-to release, so that is the shape startup recovery rolls forward from. Recovery required the live path to be a real directory, but a `file:` directory deploy is materialized as a symlink by design. Such a component read as "neither staged nor live", and the artifact sweep then deleted its `.previous-*` backup — the only copy of the displaced release — while the component stayed failed closed. A symlink whose target is a directory now counts as live, and a backup is deleted only when the row and the live tree positively say it is residue. `drop_component` removed the application-lock entry and the root-config entry as two separate writes. A crash or a failed second write left root config naming the package with the live directory already gone, so the next boot reinstalled the component that had just been dropped. Both are one transaction now, config first. Staged retention treated equal timestamps as evictable, so two concurrent stages landing in the same millisecond let whichever pruned second expire the row the other was about to return. Ties are protected on the row side, where over-retention is cheap. The filesystem side keeps strictly-newer protection — mtime granularity ties every build staged in the same tick, so protecting ties there would stop the disk bound converging at all — with a deterministic tiebreaker so concurrent prunes choose the same victims. Also trimmed the comments the reviewer named: the 17-line coverage inventory in the peer-branch integration test, and docs-duplicating narration in hdbTerms, the recorder's status union, and deploy_component's JSDoc. Reported by codex pre-push. --- bin/cliOperations.ts | 10 +- components/Application.ts | 90 ++++++++++---- components/deploymentRecorder.ts | 18 +-- components/operations.js | 27 ++-- .../deploy-tracking-peer-branch.test.ts | 20 +-- .../components/deployPhaseOperations.test.js | 9 ++ unitTests/components/deployStaging.test.js | 116 ++++++++++++++++++ .../components/deploymentRecorder.test.js | 16 +++ utility/hdbTerms.ts | 17 +-- 9 files changed, 245 insertions(+), 78 deletions(-) diff --git a/bin/cliOperations.ts b/bin/cliOperations.ts index 3513d8f1a9..ada88f2144 100644 --- a/bin/cliOperations.ts +++ b/bin/cliOperations.ts @@ -26,13 +26,9 @@ const OP_ALIASES = { package: 'package_component', }; -// CLI verbs that map to an operation plus preset properties. `harper stage` and `harper activate` are -// sugar over `deploy_component` (the stage/activate phases are folded into it; there are no separate -// stage/activate operations): `harper stage` packages + uploads the incoming version to a hidden -// staging dir cluster-wide and stops before go-live (`activate: false`), printing the staged -// deployment_id, and `harper activate deployment_id=` takes that staged deployment live (no -// upload). `harper revert` is its own operation — `revert_component` — because it is a rollback rather -// than a deploy phase, and uploads and installs nothing. +// CLI verbs that map to an operation plus preset properties. `stage` and `activate` are sugar over +// `deploy_component`, whose phases are folded into it — there are no separate stage/activate +// operations. `revert` is its own operation because it is a rollback, not a deploy phase. const OP_VERB_PROPS: Record> = { stage: { operation: 'deploy_component', activate: false, _cliVerb: 'stage' }, // `_cliVerb` is a CLI-internal marker (stripped before the request is sent) so verbRequirementError diff --git a/components/Application.ts b/components/Application.ts index 046998ce08..0e840300d3 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -609,17 +609,24 @@ async function pruneStagedBuilds(componentName: string, keepStagingId: string, m // to the count. Filling from "the others" alone would privilege the current build // unconditionally: a stage delayed behind slow peers finishes with an older mtime than a // sibling that started later and already returned, and that newer sibling's tree would be - // evicted. Protecting only *strictly* newer builds keeps the count exact when mtimes tie — - // coarse filesystem timestamps routinely tie — where the just-made build could otherwise sort - // into the eviction window. Mirrors deploymentRecorder.settleStagedRows. + // evicted. Only *strictly* newer builds are protected here, unlike the row side, which protects + // ties too. mtime granularity ties every build staged in the same tick, so protecting ties on + // disk would stop this bound converging at all — four rapid stages under a bound of two would + // keep all four. The residual: two stages tying to the millisecond can have the second evict + // the first's tree while its row stays `staged`, so activating that id later fails with "no + // valid component tree" rather than serving nothing. A clear failure, and the reconcile pass + // settles such a row. Sorting breaks ties by stagingId so concurrent prunes choose the same + // victims instead of each deleting the other's. const current = builds.find((build) => build.stagingId === keepStagingId); - const others = builds.filter((build) => build.stagingId !== keepStagingId).sort((a, b) => b.mtime - a.mtime); - const newerThanCurrent = current ? others.filter((build) => build.mtime > current.mtime) : []; - const budget = Math.max(0, maxCount - (current ? 1 : 0) - newerThanCurrent.length); + const others = builds + .filter((build) => build.stagingId !== keepStagingId) + .sort((a, b) => b.mtime - a.mtime || a.stagingId.localeCompare(b.stagingId)); + const protectedBuilds = current ? others.filter((build) => build.mtime > current.mtime) : []; + const budget = Math.max(0, maxCount - (current ? 1 : 0) - protectedBuilds.length); // Awaited (best-effort via allSettled) so the retention count is settled by the time the stage // returns. const evictions = others - .filter((build) => !newerThanCurrent.includes(build)) + .filter((build) => !protectedBuilds.includes(build)) .slice(budget) .map((build) => rm(build.parentPath, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 }).catch((err) => @@ -1177,12 +1184,28 @@ export async function revertApplication( // commit — a manifest write, or the retain rename — has to take config and the application // lock back too, or the directories end up describing one release while the persisted state // names the other, and a cold start reinstalls over the live bytes. - let persistCommitted = false; + // ATTEMPTED, not committed. The config transaction marks itself started before writing root + // config and can reject between that write and the application-lock write, so a rejected + // commit may still have changed persisted state. Its rollback is a no-op when nothing was + // written, which is what makes calling it after any attempt the safe choice. + let persistAttempted = false; let manifestWritten = false; const undoRevert = async (cause: unknown, detail: string): Promise => { const undoErrors: unknown[] = []; - if (persistCommitted) { - await hooks.rollbackPersistentState?.().catch((rollbackError) => undoErrors.push(rollbackError)); + if (persistAttempted) { + try { + await hooks.rollbackPersistentState?.(); + } catch (rollbackError) { + // Persisted state may still name the reverted-to release and we could not take it back. + // Moving the directories now would contradict it, so leave the holding tree and the + // marker untouched: that is exactly the shape startup recovery rolls forward from. + throw new AggregateError( + [cause, rollbackError], + `Reverted ${application.name} but could not ${detail}, and could not roll its ` + + `configuration back. The directories and ${recoveryMarkerPath} are left in place for ` + + `startup recovery to finish the revert` + ); + } } if (manifestWritten) { // The manifest currently claims the exchange happened. Put it back before the directories @@ -1210,8 +1233,8 @@ export async function revertApplication( throw cause; }; try { + persistAttempted = true; await hooks.commitPersistentState?.(target.previous.application_config); - persistCommitted = true; application.useLiveBuildDir(); await writeRetainedPreviousManifest(liveDirPath, { previous: target.live, @@ -1249,8 +1272,19 @@ export async function revertApplication( live: target.previous, }); } catch (persistError) { + try { + await hooks.rollbackPersistentState?.(); + } catch (rollbackError) { + // As in the swap branch: persisted state may name the restored release, so undoing the + // rename would contradict it. Leave the marker for startup recovery to roll forward. + throw new AggregateError( + [persistError, rollbackError], + `Restored ${application.name} from its retained version but could not persist its ` + + `configuration or roll that back. ${restoreMarkerPath} is left in place for startup ` + + `recovery to finish the restore` + ); + } const undoErrors: unknown[] = []; - await hooks.rollbackPersistentState?.().catch((rollbackError) => undoErrors.push(rollbackError)); await rename(liveDirPath, previousPath).catch((restoreError) => undoErrors.push(restoreError)); await rm(restoreMarkerPath, { force: true }).catch(() => {}); if (undoErrors.length) { @@ -3265,6 +3299,18 @@ function activationArtifactDeploymentId(name: string): string | undefined { return DEPLOYMENT_ID_PATTERN.test(deploymentId) ? deploymentId : undefined; } +/** + * Whether a usable live component occupies `livePath`. A `file:` directory deploy is materialized as a + * symlink by design, so requiring a real directory would report a perfectly good live component as + * missing — and recovery would then treat its displaced release as residue. + */ +async function liveComponentPresent(livePath: string): Promise { + const linkStat = await statIfPresent(livePath); + if (!linkStat) return false; + if (linkStat.isSymbolicLink()) return !!(await statIfPresent(livePath, true))?.isDirectory(); + return linkStat.isDirectory(); +} + /** * Finish the retention half of an interrupted activation. The tree the swap displaced is still parked * as `.deploy-activating//.previous--…`, and clearing it as residue is exactly what makes @@ -3414,8 +3460,7 @@ export async function reconcileStagedApplicationArtifacts( ownedByActivation = true; return; } - const liveStat = await lstat(componentDirPath).catch(() => undefined); - if (!liveStat?.isDirectory() || liveStat.isSymbolicLink()) { + if (!(await liveComponentPresent(componentDirPath))) { throw new Error(`Interrupted activation '${entry.name}' has neither a staged nor live component tree`); } await persistActivation(row); @@ -3478,8 +3523,9 @@ export async function reconcileStagedApplicationArtifacts( failedProjects.set(projectEntry.name, reconcileError); continue; } - const liveStat = await lstat(livePath).catch(() => undefined); - if (row?.status === 'activating' && row.project === projectEntry.name && liveStat?.isDirectory()) { + const liveStat = await statIfPresent(livePath); + const liveUsable = await liveComponentPresent(livePath); + if (row?.status === 'activating' && row.project === projectEntry.name && liveUsable) { try { await persistActivation(row); if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX)) { @@ -3497,15 +3543,15 @@ export async function reconcileStagedApplicationArtifacts( } } else if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && !liveStat) { await rename(artifactPath, livePath); - } else if (!row && artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX)) { + } else if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && (!row || !liveUsable)) { // A backup artifact still here means the activation never finished retaining it, so this is - // the only copy of the tree it displaced. An absent row is not evidence that copy is - // disposable: the lookup returns undefined both for a row retention reclaimed and for a - // deployment table that was never provisioned. Keep it and let a pass that can read the - // row decide. + // the only copy of the tree it displaced. It is deleted only when the state positively says + // it is residue. An absent row does not: the lookup returns undefined both for a row + // retention reclaimed and for a deployment table that was never provisioned. Neither does + // an unusable live path, which is how a crash mid-swap looks. logger.warn( `Keeping displaced component tree '${artifact.name}' for '${projectEntry.name}': ` + - `its deployment row could not be read, so it cannot be confirmed disposable` + `its deployment row or live tree could not be confirmed, so it is not provably disposable` ); } else { await rm(artifactPath, { recursive: true, force: true }); diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index e3aecdb0ef..ed1d1dd31f 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -49,13 +49,11 @@ type DeploymentStatus = | 'pending' | 'extracting' | 'installing' - // Two-phase deploy: building the incoming version into staging cluster-wide (stage phase), and the - // terminal resting state of a deploy_component stage that has not yet been activated. | 'staging' + // A terminal resting state, not a transient one: where a stage-and-stop deploy comes to rest. | 'staged' | 'loading' | 'replicating' - // Two-phase deploy: swapping the staged build into the live path cluster-wide (activate phase). | 'activating' // Retained for compatibility with deployment rows written by earlier preview builds. | 'reverting' @@ -590,13 +588,17 @@ async function settleStagedRows(project: string, keepCount: number, keepDeployme // unconditionally: a stage that waited on slow peers resumes with an older `started_at`, so with a // retention of 1 it would expire the newer stage that had already completed and been reported to the // operator, then discard its staging tree cluster-wide. `started_at` is stamped by whichever node - // originated the deploy, so cross-node clock skew produces the same inversion. Only *strictly* newer - // rows are protected, which keeps the count exact when timestamps tie. + // originated the deploy, so cross-node clock skew produces the same inversion. Ties count as + // protected, not evictable: two concurrent stages of one component can both reach `staged` in the + // same millisecond, and evicting a tied row would fail a request that is about to return its + // deployment id to the caller. The budget subtracts the protected rows, so the retained total still + // lands on `keepCount` except when more rows tie-or-exceed the window than fit in it — a temporary + // overflow, which is the right way for a disk bound to fail. const current = keepDeploymentId ? staged.find((row) => row.deployment_id === keepDeploymentId) : undefined; const others = staged.filter((row) => row.deployment_id !== keepDeploymentId); - const newerThanCurrent = current ? others.filter((row) => (row.started_at ?? 0) > (current.started_at ?? 0)) : []; - const budget = Math.max(0, keepCount - (current ? 1 : 0) - newerThanCurrent.length); - const expired = others.filter((row) => !newerThanCurrent.includes(row)).slice(budget); + const protectedRows = current ? others.filter((row) => (row.started_at ?? 0) >= (current.started_at ?? 0)) : []; + const budget = Math.max(0, keepCount - (current ? 1 : 0) - protectedRows.length); + const expired = others.filter((row) => !protectedRows.includes(row)).slice(budget); for (const row of expired) { await table.patch(row.deployment_id, { status: 'failed', diff --git a/components/operations.js b/components/operations.js index a63d24f637..71d5167ced 100644 --- a/components/operations.js +++ b/components/operations.js @@ -431,16 +431,7 @@ async function packageComponent(req) { * any credential token into the secrets store (so it lives as a replicated reference, not embedded), * then dispatches to the two-phase orchestrator (default) or the legacy one-shot path. * - * Two-phase (stage → activate) builds the incoming version into a hidden staging directory on EVERY - * node first, verifies it landed everywhere, and only then swaps it live cluster-wide — so a node - * that can't fetch the package or fails `npm install` fails the deploy while the live component is - * still untouched on every node, and the go-live window shrinks to a fast atomic directory swap. - * See stageApplication/activateStagedApplication in components/Application.ts. - * - * The request/response contract is unchanged: same inputs (`package`/payload, `restart`, - * `install_*`, `credentials`, `ignore_replication_errors`, `deployment_timeout`, …), same - * `deployment_id` in the response, same SSE progress stream (now emitting `stage`/`activate` phases - * instead of `prepare`/`replicate`). Pass `two_phase: false` to force the legacy one-shot path. + * `two_phase: false` forces the one-shot path. See DESIGN.md for the stage/activate protocol. * * @param req * @returns {Promise} @@ -2063,7 +2054,6 @@ async function dropComponent(req) { // Retires any interrupted-extraction aside and renames the live tree aside before removing // it, so startup recovery can never restore a tree over a dropped component. await dropComponentDirectory(componentPath, project, log); - await updateApplicationLockEntry(project, undefined); } else if (await fs.pathExists(pathToComponent)) { await fs.remove(pathToComponent); } @@ -2077,9 +2067,18 @@ async function dropComponent(req) { await fs.writeFile(packageJsonPath, JSON.stringify(packageJson, null, 2), 'utf8'); } - // Under the persistent-state lock for the same reason: an unlocked delete can be clobbered by a - // concurrent activation writing back a document that still contains this project. - await withPersistentStateLock(async () => configUtils.deleteConfigFromFile([project])); + if (file) { + // Under the persistent-state lock: an unlocked delete can be clobbered by a concurrent + // activation writing back a document that still contains this project. + await withPersistentStateLock(async () => configUtils.deleteConfigFromFile([project])); + } else { + // Both persistent writes as ONE reversible step, config first. Removing the application-lock + // entry and the root-config entry separately meant a crash or a failed second write left root + // config still naming the package with the live directory already gone — and the next boot's + // installApplications() reinstalled the very component that was dropped. + const dropTransaction = await createApplicationConfigTransaction(project, null); + await dropTransaction.commit(); + } }, componentDropLockOptions(project) ); diff --git a/integrationTests/deploy/deploy-tracking-peer-branch.test.ts b/integrationTests/deploy/deploy-tracking-peer-branch.test.ts index 6bfa901f41..e7610c321b 100644 --- a/integrationTests/deploy/deploy-tracking-peer-branch.test.ts +++ b/integrationTests/deploy/deploy-tracking-peer-branch.test.ts @@ -160,20 +160,8 @@ suite('Deployment tracking — peer-operation authorization boundary', (ctx: Con strictEqual(response.status, 400, `internal marker should be rejected; got: ${response.rawText}`); strictEqual(response.body.error, "'_deploymentId' is not allowed"); }); - // #2066's two peer-branch end-to-end tests used to drive the peer path by sending `_deploymentId` - // on a PUBLIC deploy_component call. That back door is deliberately closed (see the test above): - // peer work now rides the trusted-peer-only `component_deploy_phase` operation, which by design - // cannot be reached over HTTP with ordinary credentials — so those two tests have no legitimate - // entry point here any more. Their coverage lives on: - // - restore-on-failure: unitTests/components/extractApplicationSwap.test.js ("restores the exact - // previous tree when payload extraction fails", "atomically restores the previous tree when - // preparation fails under a live writer", "recovers an interrupted deploy before component - // loading"). - // - sourcing the tarball from the row's payload_blob on a peer: the trusted-peer phase tests in - // unitTests/components/deployPhaseOperations.test.js ("uses the row-backed immutable - // specification for trusted peer phases", "rebuilds a missing peer stage from the durable - // deployment payload before activation"), which dispatch the internal operation inside - // runWithOperationAuthorizationBypass. - // The bogus-_deploymentId timeout case is likewise covered by the awaitDeploymentRow unit tests - // rather than here, where the 120s default would balloon test time. + // Peer work rides the trusted-peer-only `component_deploy_phase` operation, which by design cannot be + // reached over HTTP with ordinary credentials (the test above pins that), so a peer-branch end-to-end + // test has no legitimate entry point in this repo. Peer-side behavior is covered by unit tests that + // dispatch the internal operation directly, and end to end by the three-node harper-pro suite. }); diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index 9e0e6497c8..b752ca7b5d 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -720,6 +720,10 @@ describe('deploy_component two-phase orchestration', function () { path.join(configRoot, 'harper-application-lock.json'), JSON.stringify({ applications: { [project]: { package: 'stale-package' } } }) ); + // A root-config entry is what makes a dropped component come back: installApplications() reads + // it on the next boot and reinstalls the package. Removing it and the application-lock entry as + // two separate writes meant a crash or a failed second write left this behind. + await fs.appendFile(configPath, `${project}:\n package: stale-package\n`); await operations.dropComponent({ project }); @@ -733,6 +737,11 @@ describe('deploy_component two-phase orchestration', function () { assert.strictEqual(existsSync(path.join(componentsRoot, DEPLOY_ACTIVATION_DIR, project)), false); const applicationLock = JSON.parse(await fs.readFile(path.join(configRoot, 'harper-application-lock.json'))); assert.strictEqual(applicationLock.applications[project], undefined); + assert.strictEqual( + (await fs.readFile(configPath, 'utf8')).includes(project), + false, + 'the root-config entry is removed in the same step, so the next boot cannot reinstall the drop' + ); } finally { if (priorRootEnv === undefined) delete process.env.ROOTPATH; else process.env.ROOTPATH = priorRootEnv; diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 2eb654b534..076acfd415 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -1444,6 +1444,86 @@ describe('two-phase component directory transaction', function () { await cleanup(name); }); + it('rolls persistent state back when the commit itself rejects, not only when a later step does', async () => { + // `createApplicationConfigTransaction.commit()` marks itself started before writing root config and + // can reject between that write and the application-lock write. Gating compensation on "the commit + // resolved" therefore skipped the rollback for a commit that had already changed persisted state, + // and the directories were restored while config still named the reverted-to release — which a cold + // start would reinstall over. + const name = fixtureName(); + const { application, second } = await twoActivations(name); + let rolledBack = 0; + const revertTo = (await getRevertTarget(application.dirPath)).previous.deployment_id; + + await assert.rejects( + () => + revertApplication(application, revertTo, { + commitPersistentState: async () => { + throw new Error('application lock write failed after root config changed'); + }, + rollbackPersistentState: async () => { + rolledBack++; + }, + }), + /application lock write failed/ + ); + + assert.strictEqual(rolledBack, 1, 'a rejected commit is still an attempted commit, so it is rolled back'); + assert.match(await readMarker(application.dirPath), /v2/, 'and the pre-revert version is live again'); + const target = await getRevertTarget(application.dirPath); + assert.strictEqual(target.live.deployment_id, second); + await cleanup(name); + }); + + it('leaves the holding tree and marker in place when the persistent rollback itself fails', async () => { + // Persisted state may name the reverted-to release with no way to take it back. Undoing the + // directories then would contradict it, so compensation stops and leaves exactly the shape startup + // recovery rolls forward from — rather than restoring the old bytes and consuming the evidence. + const name = fixtureName(); + const { application } = await twoActivations(name); + const previousRoot = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR); + const revertTo = (await getRevertTarget(application.dirPath)).previous.deployment_id; + + await assert.rejects( + () => + revertApplication(application, revertTo, { + commitPersistentState: async () => { + throw new Error('commit failed'); + }, + rollbackPersistentState: async () => { + throw new Error('rollback failed too'); + }, + }), + /left in place for startup recovery/ + ); + + const holdings = (await fs.readdir(previousRoot)).filter( + // The marker shares the holding directory's prefix, so exclude it or it counts as a second tree. + (entry) => entry.startsWith(`.reverting-${name}-`) && !entry.endsWith('.recovering.json') + ); + assert.strictEqual(holdings.length, 1, 'the holding tree still holds the previously-live bytes'); + assert.strictEqual( + existsSync(path.join(previousRoot, `${holdings[0]}.recovering.json`)), + true, + 'and its marker survives, which is what recovery keys on' + ); + // Live holds the reverted-to bytes and the retained slot is empty: precisely the state recovery + // reads as "the revert happened, only the retain step was lost". + assert.match(await readMarker(application.dirPath), /v1/, 'the reverted-to version is left live'); + assert.strictEqual( + existsSync(path.join(previousRoot, name)), + false, + 'and the retained slot is empty, because the holding tree has not been moved into it yet' + ); + + // Recovery finishes what compensation could not. + const failures = await recoverInterruptedReverts(COMPONENTS_ROOT); + assert.strictEqual(failures.size, 0); + assert.match(await readMarker(application.dirPath), /v1/, 'the reverted-to version stays live'); + assert.match(await readMarker(path.join(previousRoot, name)), /v2/, 'and the displaced tree is retained'); + await cleanup(name); + }); + it('rolls forward an interrupted revert whose compensation had already moved live away', async () => { // Crash shape: config and the manifest committed, then compensation renamed live away before it // could put the holding tree back. Undoing the directories here would contradict the persisted @@ -1698,6 +1778,42 @@ describe('two-phase component directory transaction', function () { await cleanup(name); }); + it('treats a live directory-package symlink as a live component, not a missing one', async () => { + // A `file:` directory deploy is materialized as a symlink by design. Requiring a real directory + // reported this shape as "neither staged nor live", and the artifact sweep then deleted the + // `.previous-*` backup — the only copy of the displaced release — while the component stayed + // failed closed. This is the exact post-swap crash state: symlink already renamed live, the + // displaced tree still parked, persistence not yet finished. + const name = fixtureName(); + const deploymentId = randomUUID(); + const livePath = path.join(COMPONENTS_ROOT, name); + const linkTarget = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-dirpkg-')); + await fs.writeFile(path.join(linkTarget, 'index.js'), "module.exports = 'directory-package';\n"); + await fs.symlink(linkTarget, livePath); + const activationProjectDir = path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR, name); + await fs.mkdir(activationProjectDir, { recursive: true }); + const backupPath = path.join(activationProjectDir, `.previous-${deploymentId}-1-1-${randomUUID()}`); + await fs.mkdir(backupPath, { recursive: true }); + await fs.writeFile(path.join(backupPath, 'index.js'), "module.exports = 'displaced';\n"); + + let persisted = 0; + const reconciliation = await reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async (id) => (id === deploymentId ? { deployment_id: id, project: name, status: 'activating' } : undefined), + async () => persisted++ + ); + + assert.strictEqual(persisted, 1, 'the activation is finished rather than reported unrecoverable'); + assert.strictEqual(reconciliation.failedProjects.has(name), false, 'so the component is not failed closed'); + assert.match(await readMarker(livePath), /directory-package/, 'the symlinked live component is untouched'); + const previousPath = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); + assert.match(await readMarker(previousPath), /displaced/, 'and its displaced release is retained, not deleted'); + + await fs.rm(livePath, { force: true }); + await fs.rm(linkTarget, { recursive: true, force: true }); + await cleanup(name); + }); + it('fails a component closed when an activation-artifact lookup throws, destroying nothing', async () => { // A failed lookup cannot distinguish "no such deployment" from "the table is unreadable", so it may // not authorize cleanup: the artifact could be the displaced release, and the live tree could be diff --git a/unitTests/components/deploymentRecorder.test.js b/unitTests/components/deploymentRecorder.test.js index 9efea692b7..c963be0a73 100644 --- a/unitTests/components/deploymentRecorder.test.js +++ b/unitTests/components/deploymentRecorder.test.js @@ -862,6 +862,22 @@ describe('staged deployment state', () => { assert.strictEqual(installed.mock.rows.get('slow-origin').status, 'staged'); }); + it('does not expire a staged row that ties the returning request', async () => { + // Two concurrent stages of one component can both reach `staged` in the same millisecond. Treating + // a tie as evictable let whichever pruned second mark the other failed and broadcast deletion of + // its tree — after that request had already returned the deployment id to its caller. + for (const [id, startedAt] of [ + ['tied-a', 500], + ['tied-b', 500], + ]) { + installed.mock.rows.set(id, { deployment_id: id, project: 'app', status: 'staged', started_at: startedAt }); + } + + assert.deepStrictEqual(await expireOldStagedDeployments('app', 1, 'tied-a'), []); + assert.strictEqual(installed.mock.rows.get('tied-b').status, 'staged'); + assert.strictEqual(installed.mock.rows.get('tied-a').status, 'staged'); + }); + it('still expires rows older than both the window and the returning request', async () => { for (const [id, startedAt] of [ ['ancient', 50], diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index 215619bd61..87126e427c 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -321,12 +321,11 @@ export const OPERATIONS_ENUM = { PACKAGE_CUSTOM_FUNCTION_PROJECT: 'package_custom_function_project', DEPLOY_CUSTOM_FUNCTION_PROJECT: 'deploy_custom_function_project', PACKAGE_COMPONENT: 'package_component', - // deploy_component runs a two-phase deploy internally. Peer phases use a distinct operation so - // older nodes fail closed instead of interpreting an unknown phase field as a one-shot deploy. + // Peer phases get their own operation so an older node fails closed instead of reading an unknown + // phase field as a one-shot deploy. DEPLOY_COMPONENT: 'deploy_component', COMPONENT_DEPLOY_PHASE: 'component_deploy_phase', - // Put a component's retained previous version back in service cluster-wide. A distinct public - // operation rather than a deploy phase: it fetches, resolves and installs nothing. + // A public operation rather than a deploy phase: it fetches, resolves and installs nothing. REVERT_COMPONENT: 'revert_component', READ_TRANSACTION_LOG: 'read_transaction_log', DELETE_TRANSACTION_LOGS_BEFORE: 'delete_transaction_logs_before', @@ -591,14 +590,10 @@ export const CONFIG_PARAMS = { OPERATIONSAPI_NETWORK_MAXREQUESTBODYSIZE: 'operationsApi_network_maxRequestBodySize', OPERATIONSAPI_COMPONENTFILE_MAXSIZE: 'operationsApi_componentFile_maxSize', DEPLOYMENT_PAYLOADRETENTION_MAXSIZE: 'deployment_payloadRetention_maxSize', - // Max stored deployment payloads (tarballs) kept per project. After a successful deploy, the - // payload_blob of older deployments beyond this count is dropped; the rows (metadata + event_log) - // are always retained. Bounds how much disk retained payloads can occupy per project — N copies of - // a large app payload would otherwise compete with the customer's own data for instance quota. - // See components/deploymentRecorder.ts (pruneProjectPayloads). + // Bounds retained payload tarballs per project. Only the blob is dropped — rows are always kept, so + // the audit trail survives. DEPLOYMENT_PAYLOADRETENTION_MAXCOUNT: 'deployment_payloadRetention_maxCount', - // Max not-yet-activated staged builds kept per component (`activate: false` stage-and-stops). When a - // new stage lands, the oldest beyond this count are evicted. See components/Application.ts. + // Bounds not-yet-activated staged builds per component. DEPLOYMENT_STAGINGRETENTION_MAXCOUNT: 'deployment_stagingRetention_maxCount', OPERATIONSAPI_TLS: 'operationsApi_tls', OPERATIONSAPI_TLS_CERTIFICATE: 'operationsApi_tls_certificate', From 093b7e456ef7db86ce295c5faab03ddd9051f933 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 16:55:46 -0400 Subject: [PATCH 80/94] fix(deploy): remove a dropped component's persisted state before destroying anything MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two persistent writes were already one transaction, but it ran after `dropComponentDirectory()` had removed the live tree — so a crash in between left root config naming the package with nothing on disk, and the next boot's installApplications() reinstalled the component that had just been dropped. The root-config entry is the only thing that can resurrect a drop, so it is removed first. That inverts the failure mode: an interrupted drop is now unfinished and re-runnable — directory still present, no config entry — rather than finished and then undone. Nothing between the transaction and the teardown reads root config, and the component lock is held throughout, so no revert or activation can observe the gap. A tombstone would also work, but the vector is the config entry itself, so removing it first needs no new startup state. Test injects a failing teardown and asserts config is already clean while the tree survives. Reported by codex pre-push. --- bin/cliOperations.ts | 5 +-- components/operations.js | 17 +++---- .../deploy-tracking-peer-branch.test.ts | 6 +-- .../components/deployPhaseOperations.test.js | 45 +++++++++++++++++++ 4 files changed, 58 insertions(+), 15 deletions(-) diff --git a/bin/cliOperations.ts b/bin/cliOperations.ts index ada88f2144..c712c0af60 100644 --- a/bin/cliOperations.ts +++ b/bin/cliOperations.ts @@ -26,9 +26,8 @@ const OP_ALIASES = { package: 'package_component', }; -// CLI verbs that map to an operation plus preset properties. `stage` and `activate` are sugar over -// `deploy_component`, whose phases are folded into it — there are no separate stage/activate -// operations. `revert` is its own operation because it is a rollback, not a deploy phase. +// `stage` and `activate` are sugar over `deploy_component` — there are no separate stage/activate +// operations to find. const OP_VERB_PROPS: Record> = { stage: { operation: 'deploy_component', activate: false, _cliVerb: 'stage' }, // `_cliVerb` is a CLI-internal marker (stripped before the request is sent) so verbRequirementError diff --git a/components/operations.js b/components/operations.js index 71d5167ced..650bd79ec8 100644 --- a/components/operations.js +++ b/components/operations.js @@ -2045,6 +2045,13 @@ async function dropComponent(req) { } if (!file) { + // Persisted state goes FIRST, before anything is destroyed. The root-config entry is the only + // thing that can resurrect a dropped component — installApplications() reinstalls whatever it + // names — so removing it up front means a crash anywhere below leaves the drop unfinished + // (directory still present, re-runnable) instead of finished-then-undone. Both writes are one + // transaction, config before the application lock. + const dropTransaction = await createApplicationConfigTransaction(project, null); + await dropTransaction.commit(); // Invalidate staged deployments before the directory goes away, so an activate racing this // drop cannot swap a staged build back into a component that is being dropped. await invalidateProjectStagedDeployments(project); @@ -2069,15 +2076,9 @@ async function dropComponent(req) { if (file) { // Under the persistent-state lock: an unlocked delete can be clobbered by a concurrent - // activation writing back a document that still contains this project. + // activation writing back a document that still contains this project. A full drop has + // already removed its config above. await withPersistentStateLock(async () => configUtils.deleteConfigFromFile([project])); - } else { - // Both persistent writes as ONE reversible step, config first. Removing the application-lock - // entry and the root-config entry separately meant a crash or a failed second write left root - // config still naming the package with the live directory already gone — and the next boot's - // installApplications() reinstalled the very component that was dropped. - const dropTransaction = await createApplicationConfigTransaction(project, null); - await dropTransaction.commit(); } }, componentDropLockOptions(project) diff --git a/integrationTests/deploy/deploy-tracking-peer-branch.test.ts b/integrationTests/deploy/deploy-tracking-peer-branch.test.ts index e7610c321b..db46396151 100644 --- a/integrationTests/deploy/deploy-tracking-peer-branch.test.ts +++ b/integrationTests/deploy/deploy-tracking-peer-branch.test.ts @@ -160,8 +160,6 @@ suite('Deployment tracking — peer-operation authorization boundary', (ctx: Con strictEqual(response.status, 400, `internal marker should be rejected; got: ${response.rawText}`); strictEqual(response.body.error, "'_deploymentId' is not allowed"); }); - // Peer work rides the trusted-peer-only `component_deploy_phase` operation, which by design cannot be - // reached over HTTP with ordinary credentials (the test above pins that), so a peer-branch end-to-end - // test has no legitimate entry point in this repo. Peer-side behavior is covered by unit tests that - // dispatch the internal operation directly, and end to end by the three-node harper-pro suite. + // `component_deploy_phase` is trusted-peer-only and unreachable over HTTP, so a peer-branch + // end-to-end test has no entry point here; the three-node harper-pro suite covers it. }); diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index b752ca7b5d..03af18302a 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -696,6 +696,51 @@ describe('deploy_component two-phase orchestration', function () { } }); + it('removes the root-config entry before it destroys anything, so a failed drop cannot resurrect', async () => { + // The resurrection vector is the root-config entry: installApplications() reinstalls whatever it + // names. Doing the persistent writes after the directory removal left a crash window where the + // live tree was gone but config still named the package, and the next boot brought the component + // back. Config-first inverts the failure: an interrupted drop is unfinished and re-runnable. + const project = name(); + const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-drop-crash-')); + const componentsRoot = path.join(configRoot, 'components'); + const configPath = path.join(configRoot, 'harper-config.yaml'); + const priorRootEnv = process.env.ROOTPATH; + const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); + const priorComponentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); + process.env.ROOTPATH = configRoot; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); + environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, componentsRoot); + await fs.writeFile(configPath, `rootPath: ${JSON.stringify(configRoot)}\n`); + try { + await fs.mkdir(path.join(componentsRoot, project), { recursive: true }); + await fs.writeFile(path.join(componentsRoot, project, 'index.js'), "module.exports = 'live';\n"); + await fs.appendFile(configPath, `${project}:\n package: some-package@1.0.0\n`); + // Make every aside-based teardown step fail: `.deploy-aside` occupied by a file is rejected as + // "not a directory", which is the closest deterministic stand-in for dying mid-teardown. + await fs.writeFile(path.join(componentsRoot, '.deploy-aside'), 'not a directory\n'); + + await assert.rejects(() => operations.dropComponent({ project })); + + assert.strictEqual( + (await fs.readFile(configPath, 'utf8')).includes(project), + false, + 'config is already clean, so the next boot cannot reinstall the component' + ); + assert.strictEqual( + existsSync(path.join(componentsRoot, project)), + true, + 'and the tree is still there: the drop is unfinished rather than finished-then-undone' + ); + } finally { + if (priorRootEnv === undefined) delete process.env.ROOTPATH; + else process.env.ROOTPATH = priorRootEnv; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); + environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, priorComponentsRoot); + await fs.rm(configRoot, { recursive: true, force: true }); + } + }); + it('drop_component invalidates staged rows and removes recovery artifacts', async () => { const project = name(); const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-drop-')); From b3951ce2d5b9cafb1dfbd1b14825fb30c27332a2 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 17:20:37 -0400 Subject: [PATCH 81/94] fix(deploy): apply the live-symlink rule to revert recovery, and stop deleting a displaced release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Teaching activation reconciliation that a `file:` directory deploy is a symlink left revert recovery behind, and the asymmetry is worse than the original bug. `recoverInterruptedReverts` enumerated holding trees with `isDirectory()` alone. A revert renames the live path into the holding slot, so for a directory-package component that holding entry IS a symlink — skipped outright, on every pass. The component stayed with no live tree and no restart ever recovered it. Both existence probes in that function now use the same `liveComponentPresent` test activation reconciliation uses, so a dangling symlink also stops counting as a live component: it was steering recovery into the "both slots occupied, this is residue" branch, which deleted the holding tree holding the only recoverable bytes. The roll-forward gate accepts a symlinked retained previous for the same reason, and the residue branch now takes the marker with the tree rather than leaving crash evidence that no longer matches the disk. A lingering activation backup for a SETTLED row was still deleted as residue. That shape only occurs when `retainActivatedPrevious` failed inside its best-effort catch after the swap and config commit — the deploy reported success, so the parked tree is the sole remaining copy of what it displaced, and deleting it made a successful deploy permanently unrevertable. Retention is finished instead. Staged-row retention ran only on the `activate: false` return, while staged directories are pruned on every stage. A full deploy therefore evicted trees whose rows still read `staged` cluster-wide, advertising a deployment_id that a later activate cannot use — and a different one per node under clock skew. It now runs on every successful origin stage. Reported by cursor-grok and cursor-composer pre-push. Their findings were produced but not promoted by the harness (EEXIST on a stale artifact, and a format-repair failure), so they were read from the leg output directly. --- components/Application.ts | 59 ++++++++---------- components/operations.js | 11 +++- .../components/deployPhaseOperations.test.js | 29 +++++++++ unitTests/components/deployStaging.test.js | 62 +++++++++++++++++++ 4 files changed, 125 insertions(+), 36 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 0e840300d3..033249682b 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -888,20 +888,8 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): return; } const previousPath = previousDirPathFor(liveDirPath); - const liveExists = await lstat(liveDirPath).then( - () => true, - (err) => { - if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; - throw err; - } - ); - const previousExists = await lstat(previousPath).then( - () => true, - (err) => { - if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; - throw err; - } - ); + const liveExists = await liveComponentPresent(liveDirPath); + const previousExists = await liveComponentPresent(previousPath); if (!liveExists && !previousExists) { await rm(markerPath, { force: true }); throw new Error( @@ -941,7 +929,11 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): } } for (const entry of entries) { - if (!entry.isDirectory() || !entry.name.startsWith(REVERTING_PREFIX)) continue; + // A holding tree can itself be a symlink: a `file:` directory deploy makes the live path a symlink, + // and the revert renames that live path here. Gating on isDirectory() alone skipped those entries, + // so such a component was never recovered — it stayed with no live tree across every restart. + if (!entry.isDirectory() && !entry.isSymbolicLink()) continue; + if (!entry.name.startsWith(REVERTING_PREFIX)) continue; const holding = join(previousRoot, entry.name); // `.reverting--`. The component name may itself contain dashes, so match the // trailing UUID explicitly rather than cutting at the last dash — which would leave most of the @@ -954,13 +946,7 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): const liveDirPath = join(componentsRootDirPath, componentName); try { await withComponentPreparationLock(liveDirPath, async () => { - const liveExists = await lstat(liveDirPath).then( - () => true, - (err) => { - if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; - throw err; - } - ); + const liveExists = await liveComponentPresent(liveDirPath); if (!liveExists) { // Two different crash shapes land here, and they need opposite repairs. Either the swap never // placed the reverted-to version (undo: the holding tree goes back to live), or it did and @@ -975,8 +961,7 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): !!manifest && manifest.live?.deployment_id === marker.live?.deployment_id && manifest.previous?.deployment_id === marker.previous?.deployment_id; - const previousStat = await lstat(previousDirPathFor(liveDirPath)).catch(() => undefined); - if (persistedAlreadyExchanged && previousStat?.isDirectory()) { + if (persistedAlreadyExchanged && (await liveComponentPresent(previousDirPathFor(liveDirPath)))) { await mkdir(dirname(liveDirPath), { recursive: true }); await rename(previousDirPathFor(liveDirPath), liveDirPath); await rename(holding, previousDirPathFor(liveDirPath)); @@ -997,13 +982,7 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): return; } const previousPath = previousDirPathFor(liveDirPath); - const previousExists = await lstat(previousPath).then( - () => true, - (err) => { - if ((err as NodeJS.ErrnoException).code === 'ENOENT') return false; - throw err; - } - ); + const previousExists = await liveComponentPresent(previousPath); if (!previousExists) { // The reverted-to version is live and only the retain step was lost, so finish it. The // marker is the source of truth over the manifest, because an earlier pass may already have @@ -1036,8 +1015,10 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): ); return; } - // Both slots occupied: the swap completed, so this is residue. + // Both slots occupied: the swap completed, so this is residue. The marker goes with it — + // leaving it behind keeps durable crash evidence that no longer matches the disk. await rm(holding, { recursive: true, force: true }); + await rm(revertRecoveryMarkerFor(holding), { force: true }).catch(() => {}); }); } catch (error) { const recoveryError = error instanceof Error ? error : new Error(String(error)); @@ -3543,7 +3524,19 @@ export async function reconcileStagedApplicationArtifacts( } } else if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && !liveStat) { await rename(artifactPath, livePath); - } else if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && (!row || !liveUsable)) { + } else if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && row && liveUsable) { + // The activation finished — the row is settled and the live tree is good — but a backup is + // still here, which only happens when `retainActivatedPrevious` failed inside its + // best-effort catch. The deploy reported success, so this is the sole remaining copy of + // the release it displaced; deleting it as residue is what makes that deploy permanently + // unrevertable. Finish the retention instead. + try { + await retainRecoveredActivation(livePath, projectEntry.name, deploymentId, row.activation_spec); + } catch (error) { + const reconcileError = error instanceof Error ? error : new Error(String(error)); + errors.set(deploymentId, reconcileError); + } + } else if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX)) { // A backup artifact still here means the activation never finished retaining it, so this is // the only copy of the tree it displaced. It is deleted only when the state positively says // it is residue. An absent row does not: the lookup returns undefined both for a row diff --git a/components/operations.js b/components/operations.js index 650bd79ec8..92700cb8df 100644 --- a/components/operations.js +++ b/components/operations.js @@ -894,12 +894,17 @@ async function deployComponentTwoPhase(req) { } await recorder.checkpoint('staged', 'staged'); + // Settle superseded staged ROWS here, not only on the stage-and-stop return below. `stageApplication` + // prunes staged directories on every stage, so gating the row half on `activate: false` let a full + // deploy evict trees while their rows still read `staged` cluster-wide — a deployment_id that + // list_deployments offers and a later activate cannot use, differently on each node under clock skew. + await pruneStagedDeploymentArtifacts(req.project, activationSpec, recorder.deploymentId).catch((error) => + log.warn('Failed to prune expired staged deployments', error) + ); + if (req.activate === false) { emit('phase', { phase: 'staged', status: 'done' }); await recorder.finish('staged'); - await pruneStagedDeploymentArtifacts(req.project, activationSpec, recorder.deploymentId).catch((error) => - log.warn('Failed to prune expired staged deployments', error) - ); await pruneProjectPayloads(req.project, getPayloadRetentionMaxCount()).catch((error) => log.warn('Failed to prune staged deployment payloads', error) ); diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index 03af18302a..45b8e31e36 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -122,6 +122,35 @@ describe('deploy_component two-phase orchestration', function () { return value; } + it('settles superseded staged rows on a full deploy, not only on stage-and-stop', async () => { + // Staged DIRECTORIES are pruned on every stage, but the row half used to run only on the + // `activate: false` return. A full deploy therefore evicted trees while their rows still read + // `staged` cluster-wide — list_deployments offering a deployment_id that a later activate cannot + // use, and under clock skew a different one on each node. + const project = name(); + const priorMax = environment.get(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT); + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, 1); + try { + const first = await operations.deployComponent({ + project, + payload: await makePayload('1.0.0'), + activate: false, + }); + assert.strictEqual(rows.get(first.deployment_id).status, 'staged'); + + // A full deploy of the same project: its own stage supersedes the one above. + await operations.deployComponent({ project, payload: await makePayload('2.0.0'), restart: false }); + + assert.strictEqual( + rows.get(first.deployment_id).status, + 'failed', + 'the superseded row is settled, so it no longer advertises an unusable deployment_id' + ); + } finally { + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, priorMax); + } + }); + it('normalizes string request booleans, including install_allow_scripts', async () => { // Joi coerces these, but validateBySchema discards `result.value`, so the raw string reaches the // handler. `install_allow_scripts: 'false'` would then read as truthy and run package lifecycle diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 076acfd415..fb290ff64a 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -1778,6 +1778,68 @@ describe('two-phase component directory transaction', function () { await cleanup(name); }); + it('finishes retention for a settled deploy whose backup is still parked, instead of deleting it', async () => { + // `retainActivatedPrevious` is best-effort: it can fail after the swap and config commit, leaving the + // displaced release under `.deploy-activating` while the deploy still reports success. The row is + // then terminal and the live tree is good, so "delete anything that is not mid-activation" removed + // the sole remaining copy — making a successful deploy permanently unrevertable. + const name = fixtureName(); + const deploymentId = randomUUID(); + const livePath = path.join(COMPONENTS_ROOT, name); + await fs.mkdir(livePath, { recursive: true }); + await fs.writeFile(path.join(livePath, 'index.js'), "module.exports = 'live';\n"); + const activationProjectDir = path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR, name); + await fs.mkdir(activationProjectDir, { recursive: true }); + const backupPath = path.join(activationProjectDir, `.previous-${deploymentId}-1-1-${randomUUID()}`); + await fs.mkdir(backupPath, { recursive: true }); + await fs.writeFile(path.join(backupPath, 'index.js'), "module.exports = 'displaced';\n"); + + const reconciliation = await reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async (id) => (id === deploymentId ? { deployment_id: id, project: name, status: 'success' } : undefined), + async () => { + throw new Error('a settled row must not be re-persisted'); + } + ); + + assert.deepStrictEqual([...reconciliation.errors.keys()], []); + const previousPath = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); + assert.match(await readMarker(previousPath), /displaced/, 'the displaced release is retained, not deleted'); + assert.match(await readMarker(livePath), /live/, 'and the live tree is untouched'); + await cleanup(name); + }); + + it('recovers a revert whose holding tree is itself a symlink', async () => { + // A `file:` directory deploy makes the live path a symlink, and the revert renames that live path + // into the holding slot — so the holding entry is a symlink, not a directory. Gating the recovery + // loop on isDirectory() skipped those entries entirely: the component was left with no live tree + // and recovered on no restart, ever. + const name = fixtureName(); + const linkTarget = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-revert-dirpkg-')); + await fs.writeFile(path.join(linkTarget, 'index.js'), "module.exports = 'symlinked-live';\n"); + const livePath = path.join(COMPONENTS_ROOT, name); + const previousRoot = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR); + await fs.mkdir(previousRoot, { recursive: true }); + + // Crash shape: live (a symlink) already renamed into the holding slot, nothing put back yet. + const holding = path.join(previousRoot, `.reverting-${name}-${randomUUID()}`); + await fs.symlink(linkTarget, holding); + + const failures = await recoverInterruptedReverts(COMPONENTS_ROOT); + + assert.strictEqual(failures.size, 0); + assert.match( + await readMarker(livePath), + /symlinked-live/, + 'the symlinked tree is restored to the live path rather than skipped forever' + ); + assert.strictEqual(existsSync(holding), false, 'and the holding slot is emptied'); + + await fs.rm(livePath, { force: true }); + await fs.rm(linkTarget, { recursive: true, force: true }); + await cleanup(name); + }); + it('treats a live directory-package symlink as a live component, not a missing one', async () => { // A `file:` directory deploy is materialized as a symlink by design. Requiring a real directory // reported this shape as "neither staged nor live", and the artifact sweep then deleted the From 5ecc7d9a2e5b730659a7b44886ce6610737a31fd Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 17:30:44 -0400 Subject: [PATCH 82/94] fix(deploy): keep a recovered displaced release addressable, and repoint Windows junctions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retaining the displaced bytes was only half the job. `retainActivatedPrevious` removes the manifest when it fails, so recovery had nothing naming the release the parked backup holds and wrote it back as an unknown deployment — the bytes survived but `revert_component` had no target, leaving the deploy effectively unrevertable anyway. The failed-retain path now records the intended manifest when the retained slot is empty. That stays safe: `getRevertTarget` requires a tree, so the component still reads as not revertable until recovery moves the backup into place. When a stale tree still occupies the slot the manifest is still removed, because naming the displaced release against the wrong bytes is the hazard that rule exists for. Recovery then reads whichever side of the manifest describes the displaced release, discriminating on whether `live` is the deployment being recovered. On Windows, `readlink` reports junction targets in the extended-length `\\?\C:\…` form. Compared against a plain root, `relative` sees two different roots and returns an absolute path, so the containment check rejected every junction and skipped repointing exactly the links that dangle after the swap. The prefix is stripped before comparing. This can only fail on win32, and the existing nested dependency-link test already asserts the link resolves from the live tree, so the Windows shard covers it — a POSIX test cannot reproduce the path semantics. Payload retention patches the two fields it owns instead of writing the whole record back, which would rewrite a status or error a concurrent deploy had just set while reclaiming a tarball. The SSE poll timer is also cleared on the terminal event rather than surviving one more interval past a settled request. Reported by codex pre-push. Two findings in that round were false positives and are not changed: `existsSync` IS imported in Application.ts (line 46, used three places, and tsc is clean), and `deleteConfigFromFile` is a synchronous function calling a non-async writer, so there is nothing to await at its call sites. --- components/Application.ts | 43 ++++++++++++++++++---- components/deploymentOperations.ts | 6 ++- components/deploymentRecorder.ts | 11 +++--- unitTests/components/deployStaging.test.js | 41 +++++++++++++++++++++ 4 files changed, 88 insertions(+), 13 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 033249682b..117203e556 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -805,9 +805,24 @@ async function retainActivatedPrevious( live: { deployment_id: deploymentId, application_config: activatedConfig ?? null }, }); } catch (err) { - // Best-effort by design: the deploy succeeded, so failing it here would be worse. Drop the manifest - // so the component reads as not revertable rather than revertable-to-the-wrong-bytes. - await rm(previousManifestPathFor(liveDirPath), { force: true }).catch(() => {}); + // Best-effort by design: the deploy succeeded, so failing it here would be worse. + // + // What the manifest may say depends on what is in the retained slot. If a stale tree still occupies + // it, any manifest naming the displaced release would name it against the WRONG bytes, so the + // manifest goes. If the slot is empty, recording the intended state is safe — getRevertTarget + // requires a tree, so it still reads as not revertable — and it is the only thing that keeps the + // parked backup addressable, letting recovery name the release it holds instead of "unknown". + const slotOccupied = !!(await statIfPresent(previousPath)); + if (slotOccupied) { + await rm(previousManifestPathFor(liveDirPath), { force: true }).catch(() => {}); + } else { + await writeRetainedPreviousManifest(liveDirPath, { + previous: displacedPath ? outgoing : { deployment_id: null, application_config: null }, + live: { deployment_id: deploymentId, application_config: activatedConfig ?? null }, + }).catch(async () => { + await rm(previousManifestPathFor(liveDirPath), { force: true }).catch(() => {}); + }); + } logger.warn( `Deployed ${application.name}, but could not retain its previous version for revert:`, errorForLog(err as Error) @@ -3040,7 +3055,11 @@ async function repointStagedDependencyLinks( // Belt-and-braces containment: never mutate a path that is not under the real node_modules root. const withinNodeModules = relative(nodeModulesPath, linkPath); if (withinNodeModules.startsWith('..') || isAbsolute(withinNodeModules)) continue; - const withinStaging = relative(currentTargetRootPath, target); + // Windows returns junction targets from readlink in the `\\?\C:\…` extended-length form. Compared + // against a plain root, `relative` sees two different roots and hands back an absolute path, so the + // containment test below rejects every junction and silently skips repointing — exactly the links + // that dangle after the swap. Strip the prefix so both sides are in the same form. + const withinStaging = relative(currentTargetRootPath, stripExtendedLengthPrefix(target)); // `..` or an absolute result means the target is outside the staging tree — not ours to touch. if (!withinStaging || withinStaging.startsWith('..') || isAbsolute(withinStaging)) continue; // NOT best-effort. This link is only being touched because activation is about to invalidate its @@ -3280,6 +3299,11 @@ function activationArtifactDeploymentId(name: string): string | undefined { return DEPLOYMENT_ID_PATTERN.test(deploymentId) ? deploymentId : undefined; } +/** `\\?\C:\x` → `C:\x`. Windows readlink reports junctions in the extended-length form. */ +function stripExtendedLengthPrefix(target: string): string { + return target.startsWith('\\\\?\\') ? target.slice(4) : target; +} + /** * Whether a usable live component occupies `livePath`. A `file:` directory deploy is materialized as a * symlink by design, so requiring a real directory would report a perfectly good live component as @@ -3308,9 +3332,14 @@ async function retainRecoveredActivation( basename(candidate).startsWith(ACTIVATION_BACKUP_PREFIX) ); if (!backupPath) return; - // Read before the retain overwrites it: the manifest's `live` is the release this activation displaced. - const outgoing: RetainedVersion = (await readRetainedPreviousManifest(componentDirPath).catch(() => undefined)) - ?.live ?? { deployment_id: null, application_config: null }; + // Which side names the displaced release depends on how far the original activation got. An + // untouched pre-activation manifest still has it as `live`; one written by the failed-retain path + // above already describes the intended end state, so there it is `previous`. `live` matching the + // deployment being recovered is what tells the two apart — without this the recovered manifest + // retains the right bytes under a null id, and revert_component cannot address them. + const manifest = await readRetainedPreviousManifest(componentDirPath).catch(() => undefined); + const displaced = manifest?.live?.deployment_id === deploymentId ? manifest?.previous : manifest?.live; + const outgoing: RetainedVersion = displaced ?? { deployment_id: null, application_config: null }; await retainActivatedPrevious( new Application({ name: componentName }), backupPath, diff --git a/components/deploymentOperations.ts b/components/deploymentOperations.ts index d776f1158d..5f24b3ce79 100644 --- a/components/deploymentOperations.ts +++ b/components/deploymentOperations.ts @@ -114,6 +114,9 @@ export async function handleGetDeployment(req: GetRequest): Promise { let lastReplayedTs = 0; let resolveLive: (() => void) | null = null; let liveDone = false; + // Held out here so the terminal-event path can stop it immediately. Otherwise it survives up to one + // more interval after the request is already settled. + let pollTimer: ReturnType | undefined; const liveBuffer: Array<{ t: number; event: { event: string; data: unknown } }> = []; const forwardLive = (e: { event: string; data: unknown }) => { sse.emit(e.event, e.data); @@ -128,6 +131,7 @@ export async function handleGetDeployment(req: GetRequest): Promise { (e.data as { phase?: string }).phase === 'success'); if (isTerminalEvent && !liveDone) { liveDone = true; + if (pollTimer) clearInterval(pollTimer); resolveLive?.(); } }; @@ -164,7 +168,7 @@ export async function handleGetDeployment(req: GetRequest): Promise { // Safety net — if the in-memory emitter is dropped (recorder finished or // the process recycled) before signaling, poll the row's status as a // fallback so the client never hangs indefinitely. - const pollTimer = setInterval(() => { + pollTimer = setInterval(() => { if (liveDone) { clearInterval(pollTimer); return; diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index ed1d1dd31f..3db62592db 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -671,15 +671,16 @@ export async function pruneProjectPayloads(project: string, maxCount: number): P // must not be dropped — skip it and let a later prune reclaim it once it settles. if (!TERMINAL_STATUSES.has(row.status)) continue; const size = typeof row.payload_size === 'number' ? row.payload_size : 0; - // Copy before mutating: get()/search() rows may be shared or read-only records. - const updated: Record = { ...row, payload_blob: null }; - updated.event_log = Array.isArray(row.event_log) ? [...row.event_log] : []; - updated.event_log.push({ + // Patch, not put: writing the whole spread record back would also rewrite fields a concurrent + // deploy may have just changed on this row — reverting a status or dropping an error while + // reclaiming a tarball. Only these two fields are this prune's business. + const eventLog = Array.isArray(row.event_log) ? [...row.event_log] : []; + eventLog.push({ t: Date.now(), event: 'payload_dropped', data: { payload_size: size, reason: 'payloadRetention_maxCount', max_count: maxCount }, }); - await table.put(updated); + await table.patch(row.deployment_id, { payload_blob: null, event_log: eventLog }); freed += size; } return freed; diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index fb290ff64a..dd359b932f 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -1809,6 +1809,47 @@ describe('two-phase component directory transaction', function () { await cleanup(name); }); + it('keeps the displaced release ADDRESSABLE when retention failed, not just present', async () => { + // Retaining the bytes is only half the job: a manifest that records the displaced release as an + // unknown deployment leaves revert_component with nothing to target, so the recovered deploy is + // still effectively unrevertable. The failed-retain path therefore records the intended manifest + // when the retained slot is empty, and recovery reads the side of it that names the displaced tree. + const name = fixtureName(); + const { application, first, second } = await twoActivations(name); + const previousPath = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); + + // Rebuild the failed-retain shape by hand: the displaced tree (v1) parked as an activation backup, + // the retained slot empty, and the manifest describing the intended end state. + const activationProjectDir = path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR, name); + await fs.mkdir(activationProjectDir, { recursive: true }); + const backupPath = path.join(activationProjectDir, `.previous-${second}-1-1-${randomUUID()}`); + await fs.rename(previousPath, backupPath); + await fs.writeFile( + `${previousPath}.json`, + JSON.stringify({ + previous: { deployment_id: first, application_config: null }, + live: { deployment_id: second, application_config: null }, + }) + ); + + const reconciliation = await reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async (id) => (id === second ? { deployment_id: id, project: name, status: 'success' } : undefined), + async () => {} + ); + + assert.deepStrictEqual([...reconciliation.errors.keys()], []); + assert.match(await readMarker(previousPath), /v1/, 'the displaced bytes are retained'); + const target = await getRevertTarget(application.dirPath); + assert.strictEqual( + target.previous.deployment_id, + first, + 'and the manifest names which deployment they are, so revert_component can address it' + ); + assert.strictEqual(target.live.deployment_id, second); + await cleanup(name); + }); + it('recovers a revert whose holding tree is itself a symlink', async () => { // A `file:` directory deploy makes the live path a symlink, and the revert renames that live path // into the holding slot — so the holding entry is a symlink, not a directory. Gating the recovery From b13467e2ed47c4d7d2d09a54295d025ea2de695b Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 17:38:08 -0400 Subject: [PATCH 83/94] fix(deploy): prove a parked backup belongs to the live release before retaining it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The settled-row retention added last commit was too permissive. A settled row plus a good live tree proves the activation ended, not that it ended as the CURRENT release — so an artifact left by an older deployment would be promoted into the retained slot, overwriting valid rollback state with stale bytes and writing a manifest naming a release that is no longer live. Retention now requires the manifest to name that deployment as live, which is exactly what the failed-retain path records. A backup that cannot be placed either way is kept: retaining it would corrupt rollback state and deleting it may destroy the only copy of what its deploy displaced. Dependency-link compensation keyed on the returned count, so a repoint that threw partway never assigned it and the inverse walk was skipped — leaving links aimed at the live path, where a retry of the same deployment id validates the staged tree against whatever release is live, and the containment check rejects those links as external so they are never repointed back. It keys on "attempted" now, the same correction the persistent-state compensation needed. No test: inducing a failure mid-walk requires two links with a guaranteed processing order, and readdir order is not guaranteed, so the test would be flaky rather than proving anything. Payload retention no longer appends to `event_log`. That was a read-copy-write of an append-only list, so reclaiming a tarball could drop a concurrent writer's audit entry; the null blob already reports the outcome and the reclaim is logged. `writeRetainedPreviousManifest` delegates to `writeJsonAtomically` rather than duplicating it, and the orphan sweep verifies the prefix before slicing it off. Reported by codex pre-push. Two majors in that round are false positives and are unchanged: `deleteConfigFromFile` is synchronous down to `writeFileSync`/ `renameSync`, so there is nothing to await (raised three rounds running), and the SSE promise cannot hang on a completion during the initial row read — events are buffered while `resolveLive` is null, and the executor resolves on `liveDone` before arming the timer. --- components/Application.ts | 43 ++++++++++++------- components/deploymentRecorder.ts | 20 ++++----- unitTests/components/deployStaging.test.js | 21 +++++---- .../components/deploymentRecorder.test.js | 11 +++-- 4 files changed, 58 insertions(+), 37 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 117203e556..bd69462ce2 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -752,13 +752,7 @@ async function writeJsonAtomically(targetPath: string, value: unknown): Promise< } async function writeRetainedPreviousManifest(liveDirPath: string, manifest: RetainedPreviousManifest): Promise { - const manifestPath = previousManifestPathFor(liveDirPath); - // Temp + rename so a crash mid-write can never leave a half-written manifest, which would make a - // retained tree unaddressable (and so unrevertable). - const tempPath = `${manifestPath}.${process.pid}.${randomUUID()}.tmp`; - await mkdir(dirname(manifestPath), { recursive: true }); - await writeFile(tempPath, JSON.stringify(manifest, null, 2), { mode: 0o600 }); - await rename(tempPath, manifestPath); + await writeJsonAtomically(previousManifestPathFor(liveDirPath), manifest); } /** @@ -863,8 +857,10 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): // its marker. It is recovered on its own terms below. if (holdingName.endsWith(RESTORE_MARKER_INFIX)) continue; if (entryNames.has(holdingName)) continue; - const orphanName = REVERTING_NAME_PATTERN.exec(holdingName.slice(REVERTING_PREFIX.length))?.[1]; - if (!holdingName.startsWith(REVERTING_PREFIX) || !orphanName || !safeComponentName(orphanName)) { + const orphanName = holdingName.startsWith(REVERTING_PREFIX) + ? REVERTING_NAME_PATTERN.exec(holdingName.slice(REVERTING_PREFIX.length))?.[1] + : undefined; + if (!orphanName || !safeComponentName(orphanName)) { await rm(join(previousRoot, entry.name), { force: true }).catch(() => {}); continue; } @@ -3120,6 +3116,11 @@ export async function activateStagedApplication( let newMarkerPath: string | undefined; let swapped = false; let repointedLinks = 0; + // Attempted, not counted: the repoint mutates links as it walks, so a throw partway leaves some + // already aimed at the live path while the assignment below never happens. Keying compensation on + // the returned count skipped the undo for exactly that case, and a retry of this deployment id then + // validated the staged tree against whatever release is live. + let repointAttempted = false; try { await hooks.beforeSwap?.(); const existingArtifacts = await activationArtifacts(application.dirPath, deploymentId); @@ -3179,6 +3180,7 @@ export async function activateStagedApplication( // dangling; rewriting them to their future live targets now means a failure lands in the catch // below, which restores the previous release rather than leaving a live component that cannot // resolve its dependencies. Done pre-swap for the compensation, not post-swap for convenience. + repointAttempted = true; repointedLinks = await repointStagedDependencyLinks(stagingDirPath, application.dirPath); if (repointedLinks) { logger.debug?.( @@ -3197,7 +3199,7 @@ export async function activateStagedApplication( rollbackErrors.push(rollbackError); } } - if (repointedLinks) { + if (repointAttempted) { // The candidate is going back to staging, so its links have to point at staging again. Left // aimed at the live path they would resolve against whatever release is live, so a retry of // this same deployment id would validate the staged tree against the wrong bytes. @@ -3553,12 +3555,21 @@ export async function reconcileStagedApplicationArtifacts( } } else if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && !liveStat) { await rename(artifactPath, livePath); - } else if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && row && liveUsable) { - // The activation finished — the row is settled and the live tree is good — but a backup is - // still here, which only happens when `retainActivatedPrevious` failed inside its - // best-effort catch. The deploy reported success, so this is the sole remaining copy of - // the release it displaced; deleting it as residue is what makes that deploy permanently - // unrevertable. Finish the retention instead. + } else if ( + artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && + row && + liveUsable && + (await readRetainedPreviousManifest(livePath).catch(() => undefined))?.live?.deployment_id === deploymentId + ) { + // The activation finished — settled row, good live tree — but a backup is still parked, + // which only happens when `retainActivatedPrevious` failed inside its best-effort catch. + // That backup is the sole remaining copy of the release this deploy displaced, so deleting + // it as residue is what makes the deploy permanently unrevertable. Finish the retention. + // + // Gated on the manifest naming THIS deployment as live. A settled row proves the + // activation ended, not that it ended as the current release: an artifact left by an older + // deployment would otherwise overwrite a valid retained previous with stale bytes and + // write a manifest naming a release that is no longer live. try { await retainRecoveredActivation(livePath, projectEntry.name, deploymentId, row.activation_spec); } catch (error) { diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index 3db62592db..c9b2c6b4d0 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -671,16 +671,16 @@ export async function pruneProjectPayloads(project: string, maxCount: number): P // must not be dropped — skip it and let a later prune reclaim it once it settles. if (!TERMINAL_STATUSES.has(row.status)) continue; const size = typeof row.payload_size === 'number' ? row.payload_size : 0; - // Patch, not put: writing the whole spread record back would also rewrite fields a concurrent - // deploy may have just changed on this row — reverting a status or dropping an error while - // reclaiming a tarball. Only these two fields are this prune's business. - const eventLog = Array.isArray(row.event_log) ? [...row.event_log] : []; - eventLog.push({ - t: Date.now(), - event: 'payload_dropped', - data: { payload_size: size, reason: 'payloadRetention_maxCount', max_count: maxCount }, - }); - await table.patch(row.deployment_id, { payload_blob: null, event_log: eventLog }); + // Patch only the field this prune owns. Writing the whole record back would rewrite fields a + // concurrent deploy just changed — reverting a status, dropping an error — while reclaiming a + // tarball. The drop is not appended to `event_log` either: that would be a read-copy-write of an + // append-only list, so a concurrent writer's entry would be lost. `payload_blob_present: false` + // already reports the outcome on the row, and the reclaim is logged here for the audit trail. + await table.patch(row.deployment_id, { payload_blob: null }); + logger.info?.( + `Reclaimed ${size} bytes of deployment payload for '${project}' (${row.deployment_id}): ` + + `beyond deployment_payloadRetention_maxCount=${maxCount}` + ); freed += size; } return freed; diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index dd359b932f..a4b070fa31 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -1778,11 +1778,11 @@ describe('two-phase component directory transaction', function () { await cleanup(name); }); - it('finishes retention for a settled deploy whose backup is still parked, instead of deleting it', async () => { - // `retainActivatedPrevious` is best-effort: it can fail after the swap and config commit, leaving the - // displaced release under `.deploy-activating` while the deploy still reports success. The row is - // then terminal and the live tree is good, so "delete anything that is not mid-activation" removed - // the sole remaining copy — making a successful deploy permanently unrevertable. + it('keeps a parked backup it cannot prove belongs to the live release, rather than retaining or deleting it', async () => { + // A settled row plus a good live tree proves the activation ended — not that it ended as the CURRENT + // release. With no manifest naming this deployment as live, the artifact could be left over from an + // older deployment, and retaining it would overwrite a valid retained previous with stale bytes. + // Deleting it is equally wrong: it may be the only copy of what that deploy displaced. So it stays. const name = fixtureName(); const deploymentId = randomUUID(); const livePath = path.join(COMPONENTS_ROOT, name); @@ -1803,9 +1803,14 @@ describe('two-phase component directory transaction', function () { ); assert.deepStrictEqual([...reconciliation.errors.keys()], []); - const previousPath = path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name); - assert.match(await readMarker(previousPath), /displaced/, 'the displaced release is retained, not deleted'); - assert.match(await readMarker(livePath), /live/, 'and the live tree is untouched'); + assert.strictEqual(existsSync(backupPath), true, 'the unprovable backup is left in place'); + assert.strictEqual( + existsSync(path.join(COMPONENTS_ROOT, DEPLOY_PREVIOUS_DIR, name)), + false, + 'and nothing stale is promoted into the retained slot' + ); + assert.match(await readMarker(livePath), /live/, 'the live tree is untouched'); + await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR, name), { recursive: true, force: true }); await cleanup(name); }); diff --git a/unitTests/components/deploymentRecorder.test.js b/unitTests/components/deploymentRecorder.test.js index c963be0a73..b8f9538f1f 100644 --- a/unitTests/components/deploymentRecorder.test.js +++ b/unitTests/components/deploymentRecorder.test.js @@ -970,9 +970,14 @@ describe('pruneProjectPayloads (deployment_payloadRetention_maxCount)', () => { assert.ok(row, 'the pruned deployment row still exists (audit trail preserved)'); assert.strictEqual(row.status, 'success', 'status is untouched'); assert.strictEqual(row.payload_size, 100, 'payload_size is retained as metadata'); - assert.ok( - row.event_log.some((e) => e.event === 'payload_dropped' && e.data?.reason === 'payloadRetention_maxCount'), - 'the drop is recorded in the event log with its reason' + assert.strictEqual(row.payload_blob, null, 'the tarball is reclaimed'); + // The drop is deliberately NOT appended to event_log: that would be a read-copy-write of an + // append-only list, so a concurrent writer's entry would be lost to reclaim a tarball. The null + // blob already reports the outcome on the row, and the reclaim is logged. + assert.deepStrictEqual( + row.event_log.filter((e) => e.event === 'payload_dropped'), + [], + 'and nothing is appended to the audit list' ); }); From b533ba5083e9bab4526a985213d093e2f7fc9f49 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Fri, 21 Aug 2026 17:47:46 -0400 Subject: [PATCH 84/94] fix(deploy): record the revert's commit as a fact, and stop losing recoverable trees MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert recovery inferred "did persistent state commit?" from whether the manifest had been exchanged. Those are written in sequence, so a crash between them left config exchanged and the manifest not — and recovery, comparing the two, undid directories that config already described. The component came back running old code under new configuration, which a cold start would then reinstall over. The revert now records `persisted: true` on its recovery marker the moment the commit returns, and recovery reads that; the manifest comparison remains as a fallback for markers written before the field existed. A displaced tree that reached the retained slot could still end up unaddressable: if the manifest write failed after the rename, the previous fix declined to record it because the slot was occupied — by the very tree it should have described. It now distinguishes a slot holding the tree just moved from one holding something stale. A dangling symlink at the live path blocked restoring an activation backup: the restore branch keyed on lstat, which sees the link, so a component pointing at nothing was left that way with its recoverable backup sitting beside it. It keys on usability now, and rename replaces the dead link. The SSE terminal signal is no longer lost to the replay dedup filter. An event whose timestamp ties the last replayed entry was dropped outright, discarding the only indication the deployment had finished. Deduplicated events are still checked for terminal-ness even when they are not re-emitted. `discardRetainedPrevious` parked its tree in an aside derived from the retained path — creating `.deploy-previous/.deploy-aside`, which startup recovery never sweeps and which made the following rmdir fail ENOTEMPTY, stranding the tree permanently. It parks in the component's own aside. Staged-row settlement stays at stage time, where it matches the directory prune it mirrors. Moving it after activation was tried and is worse: the current row is no longer `staged` by then, so nothing is superseded and the row/directory divergence returns. The residual — N concurrent deploys racing for N retention slots — is inherent to a count-based policy and already true of the directory prune. Two more majors this round are false positives and unchanged: `deleteConfigFromFile` is synchronous down to `writeFileSync`/`renameSync` (fourth round raised), and `withPersistentStateLock`'s liveness predicate is never consulted for a foreign pid — `ownerIsAlive` resolves those with `isProcessAlive` first, so a SIGKILLed external holder is detected dead. --- components/Application.ts | 51 ++++++++++++++++++++++++------ components/deploymentOperations.ts | 36 ++++++++++++--------- components/operations.js | 14 +++++--- 3 files changed, 73 insertions(+), 28 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index bd69462ce2..7c6b6833a5 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -660,8 +660,11 @@ async function pruneStagedBuilds(componentName: string, keepStagingId: string, m * version over the current one. `.discarded-` says "never restore this", and cleanupExtractionPaths * already removes anything that is not an unretired `.in-progress-`. */ -export async function discardDirAside(targetDirPath: string, componentName: string): Promise { - const asideStagingDir = extractionStagingDirectory(targetDirPath); +export async function discardDirAside( + targetDirPath: string, + componentName: string, + asideStagingDir: string = extractionStagingDirectory(targetDirPath) +): Promise { try { // lstat, not access(F_OK): access follows symlinks, so a DANGLING symlink at the path (left by a // prior `file:`-directory deploy whose target was removed) would report ENOENT and be skipped @@ -709,6 +712,10 @@ type RetainedVersion = { type RetainedPreviousManifest = { previous: RetainedVersion; live: RetainedVersion; + // Revert recovery markers only, which share this shape: set once the revert's persistent-state commit + // has returned. Recovery needs that as a fact rather than inferring it from the manifest, which is + // written after the commit and so disagrees with it across a crash in between. + persisted?: boolean; }; // Absolute path of the retained-previous copy for a component's live directory, and of the sidecar @@ -776,6 +783,10 @@ async function retainActivatedPrevious( ): Promise { const liveDirPath = application.dirPath; const previousPath = previousDirPathFor(liveDirPath); + // Whether the displaced tree actually reached the retained slot. If it did, the slot holds exactly what + // a manifest would describe, so a later failure must still record it — otherwise the tree sits in place + // with no manifest and `getRevertTarget` reports nothing, permanently. + let displacedMoved = false; try { // The manifest goes FIRST, by being removed. getRevertTarget only checks that a retained tree // exists, so a manifest that outlives its tree is worse than no manifest: it names a deployment id @@ -790,6 +801,7 @@ async function retainActivatedPrevious( if (displacedPath) { await mkdir(dirname(previousPath), { recursive: true }); await rename(displacedPath, previousPath); + displacedMoved = true; } // Written unconditionally: even a first-ever deploy that retained nothing has to record which // deployment is now live, or the NEXT activation cannot name the tree it displaces and the @@ -806,7 +818,7 @@ async function retainActivatedPrevious( // manifest goes. If the slot is empty, recording the intended state is safe — getRevertTarget // requires a tree, so it still reads as not revertable — and it is the only thing that keeps the // parked backup addressable, letting recovery name the release it holds instead of "unknown". - const slotOccupied = !!(await statIfPresent(previousPath)); + const slotOccupied = !!(await statIfPresent(previousPath)) && !displacedMoved; if (slotOccupied) { await rm(previousManifestPathFor(liveDirPath), { force: true }).catch(() => {}); } else { @@ -967,11 +979,14 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): // the persistent side committed and undoing the directories would contradict it. const marker = await readRevertRecoveryMarker(revertRecoveryMarkerFor(holding)); const manifest = await readRetainedPreviousManifest(liveDirPath); + // `persisted` is the direct signal, written by the revert the moment its commit returned. The + // manifest comparison stays as a fallback for markers written before that field existed. const persistedAlreadyExchanged = !!marker && - !!manifest && - manifest.live?.deployment_id === marker.live?.deployment_id && - manifest.previous?.deployment_id === marker.previous?.deployment_id; + (marker.persisted === true || + (!!manifest && + manifest.live?.deployment_id === marker.live?.deployment_id && + manifest.previous?.deployment_id === marker.previous?.deployment_id)); if (persistedAlreadyExchanged && (await liveComponentPresent(previousDirPathFor(liveDirPath)))) { await mkdir(dirname(liveDirPath), { recursive: true }); await rename(previousDirPathFor(liveDirPath), liveDirPath); @@ -1048,7 +1063,14 @@ export async function recoverInterruptedReverts(componentsRootDirPath: string): */ export async function discardRetainedPrevious(componentDirPath: string): Promise { await rm(previousManifestPathFor(componentDirPath), { force: true }); - await discardDirAside(previousDirPathFor(componentDirPath), basename(componentDirPath)); + // Parked in the COMPONENT's aside. Deriving it from the retained path would create + // `.deploy-previous/.deploy-aside`, which startup recovery never sweeps and which then makes the + // `rmdir(previousRoot)` below fail ENOTEMPTY — stranding the tree forever. + await discardDirAside( + previousDirPathFor(componentDirPath), + basename(componentDirPath), + extractionStagingDirectory(componentDirPath) + ); const previousRoot = join(dirname(componentDirPath), DEPLOY_PREVIOUS_DIR); for (const entry of await readdir(previousRoot, { withFileTypes: true }).catch(() => [])) { // Any in-flight revert artifact for this component is meaningless once it is dropped. @@ -1227,6 +1249,15 @@ export async function revertApplication( try { persistAttempted = true; await hooks.commitPersistentState?.(target.previous.application_config); + // Recorded as a fact, not inferred later from the manifest. The manifest is written AFTER + // this commit, so a crash in between leaves persisted state exchanged while the manifest + // still reads pre-revert — and recovery, comparing the two, would undo directories that + // config already describes, leaving old code running under new configuration. + await writeJsonAtomically(recoveryMarkerPath, { + previous: target.live, + live: target.previous, + persisted: true, + }); application.useLiveBuildDir(); await writeRetainedPreviousManifest(liveDirPath, { previous: target.live, @@ -3535,7 +3566,6 @@ export async function reconcileStagedApplicationArtifacts( failedProjects.set(projectEntry.name, reconcileError); continue; } - const liveStat = await statIfPresent(livePath); const liveUsable = await liveComponentPresent(livePath); if (row?.status === 'activating' && row.project === projectEntry.name && liveUsable) { try { @@ -3553,7 +3583,10 @@ export async function reconcileStagedApplicationArtifacts( errors.set(deploymentId, reconcileError); failedProjects.set(projectEntry.name, reconcileError); } - } else if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && !liveStat) { + } else if (artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && !liveUsable) { + // `!liveUsable` rather than `!liveStat`: a dangling symlink at the live path is an occupant + // that cannot serve, and rename replaces it. Keying on lstat left the component pointing + // at nothing with its recoverable backup sitting right there. await rename(artifactPath, livePath); } else if ( artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && diff --git a/components/deploymentOperations.ts b/components/deploymentOperations.ts index 5f24b3ce79..3fffc32294 100644 --- a/components/deploymentOperations.ts +++ b/components/deploymentOperations.ts @@ -118,22 +118,24 @@ export async function handleGetDeployment(req: GetRequest): Promise { // more interval after the request is already settled. let pollTimer: ReturnType | undefined; const liveBuffer: Array<{ t: number; event: { event: string; data: unknown } }> = []; + // A terminal signal — either an explicit success/error event from the lifecycle, or the recorder's + // `_recorder_finished` sentinel emitted before it unsubscribes. + const isTerminalEvent = (e: { event: string; data: unknown }) => + e.event === '_recorder_finished' || + e.event === 'error' || + (e.event === 'phase' && + e.data && + typeof e.data === 'object' && + (e.data as { phase?: string }).phase === 'success'); + const settleLive = () => { + if (liveDone) return; + liveDone = true; + if (pollTimer) clearInterval(pollTimer); + resolveLive?.(); + }; const forwardLive = (e: { event: string; data: unknown }) => { sse.emit(e.event, e.data); - // A terminal signal — either explicit success/error event from the lifecycle, or - // the recorder's `_recorder_finished` sentinel emitted before it unsubscribes. - const isTerminalEvent = - e.event === '_recorder_finished' || - e.event === 'error' || - (e.event === 'phase' && - e.data && - typeof e.data === 'object' && - (e.data as { phase?: string }).phase === 'success'); - if (isTerminalEvent && !liveDone) { - liveDone = true; - if (pollTimer) clearInterval(pollTimer); - resolveLive?.(); - } + if (isTerminalEvent(e)) settleLive(); }; const unsubscribe = liveEmitter ? liveEmitter.subscribe((event) => { @@ -160,9 +162,13 @@ export async function handleGetDeployment(req: GetRequest): Promise { if (liveEmitter) { await new Promise((resolve) => { resolveLive = resolve; - // Flush anything that arrived during replay, filtering to events the replay missed. + // Flush anything that arrived during replay, filtering to events the replay missed. A + // deduplicated event must still be inspected for terminal-ness: dropping it outright would + // lose the only signal that the deployment finished, and the request would then hang until + // the fallback timer happened to notice the emitter was gone — or forever, if it had not. for (const buffered of liveBuffer) { if (buffered.t > lastReplayedTs) forwardLive(buffered.event); + else if (isTerminalEvent(buffered.event)) settleLive(); } if (liveDone) resolve(); // Safety net — if the in-memory emitter is dropped (recorder finished or diff --git a/components/operations.js b/components/operations.js index 92700cb8df..0abc9f8147 100644 --- a/components/operations.js +++ b/components/operations.js @@ -894,10 +894,16 @@ async function deployComponentTwoPhase(req) { } await recorder.checkpoint('staged', 'staged'); - // Settle superseded staged ROWS here, not only on the stage-and-stop return below. `stageApplication` - // prunes staged directories on every stage, so gating the row half on `activate: false` let a full - // deploy evict trees while their rows still read `staged` cluster-wide — a deployment_id that - // list_deployments offers and a later activate cannot use, differently on each node under clock skew. + // Settled here, on every successful origin stage, so the ROWS match the staged DIRECTORIES that + // `stageApplication` has already pruned under the same policy. Gating this on `activate: false` left + // a full deploy evicting trees whose rows still read `staged` cluster-wide — list_deployments + // offering a deployment_id whose tree is gone, and a different one per node under clock skew. + // + // Retention is count-based, so N concurrent deploys of one project can still race for N slots: a + // candidate that is `staged` but whose deploy has not yet activated can be expired by a newer + // stage. That is inherent to the policy and already true of the directory prune this mirrors — + // only rows strictly older than the current request are eligible, which is what keeps a + // just-returned deployment safe. await pruneStagedDeploymentArtifacts(req.project, activationSpec, recorder.deploymentId).catch((error) => log.warn('Failed to prune expired staged deployments', error) ); From a56c34f8f8526ec4fbb3c5e3c48babebee0c1f7c Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 24 Aug 2026 13:44:13 -0400 Subject: [PATCH 85/94] feat(deploy): the origin owns the deployment row, and separated phases require tracking MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two remaining design decisions from the PR description. **Peer row ownership.** Every peer ran `claimStagedDeployment`, which ends in a patch on the replicated `hdb_deployment` row — N+1 writers of one key, the concurrent same-key pattern `seal()` exists to avoid. Under replication lag a peer's `activating` could land after the origin had written `success`, leaving a converged deploy in a non-terminal status it never leaves. DESIGN.md and `revertComponent` both already said the origin owns the row, so the implementation contradicted the stated design. Peers now claim with `persist: false`: same validation — project match, status gate, wait-for-staged — no write. What a peer actually needs from claiming is mutual exclusion against another activation of the same component, and it already holds the per-component filesystem lock for that; the row write never provided it. Three peer-phase tests asserted the old behavior and now assert the new contract. **Separated phases require deployment tracking.** `DeploymentRecorder.put()` is tolerant of a missing `hdb_deployment` table because tracking is observability for a one-shot deploy. It is not observability for the separated phases, which coordinate through the row: `activate: false` returned a deployment_id nothing could resolve, so the stage reported success and was permanently unactivatable. `activate: false`, activate-by-id, and an explicit `two_phase: true` now fail 503 when the table is absent — the request is valid, the node is not provisioned. An unspecified `two_phase` falls back to the one-shot path instead of entering a protocol with nowhere to coordinate, which keeps the tolerant behavior for a node whose upgrade directive has not run. That is the product call the decision noted: only an explicit request fails loudly. The test harness's warmup deploy moved to the one-shot path, since it runs before the table seam exists on purpose and the separated phases now require it. DESIGN.md documents both, replacing the "peers make the same local claim" line that the peer write had outgrown. --- DESIGN.md | 22 ++++++- components/deploymentRecorder.ts | 30 +++++++++- components/operations.js | 27 ++++++++- .../components/deployPhaseOperations.test.js | 60 +++++++++++++++---- .../components/deploymentRecorder.test.js | 46 ++++++++++++++ 5 files changed, 168 insertions(+), 17 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 6f3d4fd209..772ed248c8 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -201,9 +201,25 @@ immutable activation specification (package/install settings, routing, credentia `force`); activate-by-id never accepts replacements for those fields. After every stage response, the origin durably checkpoints the row as `staged`. Activation claims that row as `activating` while holding the same per-component filesystem lock, then swaps the candidate into the live path and commits root -config plus `harper-application-lock.json` as one compensating transaction. Peers make the same local -claim before their swap. This ordering is the recovery record: startup preserves `staged` candidates, -deletes terminal/orphan candidates, and rolls an `activating` candidate forward before loading apps. +config plus `harper-application-lock.json` as one compensating transaction. This ordering is the +recovery record: startup preserves `staged` candidates, deletes terminal/orphan candidates, and rolls +an `activating` candidate forward before loading apps. + +**The origin owns the row; a peer's claim is local.** Peers run the same validation before their swap +but do NOT write the deployment row (`claimStagedDeployment(..., { persist: false })`). The row is +replicated, so a peer writing it would make N+1 writers of one key — and under replication lag a +peer's `activating` can land _after_ the origin has written `success`, leaving a converged deploy in a +non-terminal status it never leaves. What a peer needs from claiming is mutual exclusion against +another activation of the same component, and the per-component filesystem lock it already holds +provides exactly that. + +**The separated phases require deployment tracking.** `DeploymentRecorder` is deliberately tolerant of +a missing `hdb_deployment` table — tracking is observability for a one-shot deploy. It is not +observability for `activate: false` or activate-by-id, which coordinate _through_ the row: without it +a stage would return a deployment_id nothing could resolve, so the stage would report success and be +permanently unactivatable. Those requests, and an explicit `two_phase: true`, fail with 503 when the +table is absent. An unspecified `two_phase` on such a node stays on the one-shot path rather than +entering a protocol with nowhere to coordinate. Peer stage/activate/restart messages use the distinct authenticated `component_deploy_phase` operation. An older peer therefore rejects the unknown operation instead of ignoring a phase marker and deploying diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index fca5c71c5f..be4ffc7d66 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -609,10 +609,34 @@ export async function recordDeploymentPeers(deploymentId: string, results: unkno } /** Claim a staged deployment for activation while the component preparation lock is held. */ +/** + * Validate that a staged deployment is available to activate, and — on the ORIGIN — mark the row + * `activating`. + * + * `persist: false` is the peer mode: the same validation, no write. The row is replicated, so every + * peer patching it makes N+1 writers of one key, and under replication lag a peer's `activating` + * can land AFTER the origin has written `success` — reverting a converged deploy to a non-terminal + * status it never leaves. The origin owns the row (as DESIGN.md and revertComponent both state); + * what a peer actually needs from "claiming" is mutual exclusion against another activation of the + * same component, and it already holds the component preparation lock for that. + */ +/** + * Whether deployment tracking is provisioned on this node. + * + * `DeploymentRecorder.put()` is deliberately tolerant — a one-shot deploy still works with no + * `hdb_deployment` table, because tracking is observability there. It is NOT observability for the + * separated phases: `activate: false` hands the caller a deployment_id that only the row can resolve, + * so a stage that reported success would be unactivatable. Callers of the separated phases check this + * up front instead of failing later with a missing row. + */ +export function isDeploymentTrackingAvailable(): boolean { + return !!(databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; +} + export async function claimStagedDeployment( deploymentId: string, project: string, - options: { allowActivating?: boolean; waitForStagedMs?: number } = {} + options: { allowActivating?: boolean; waitForStagedMs?: number; persist?: boolean } = {} ): Promise> { const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; if (!table) throw new ClientError('Deployment tracking is unavailable; cannot activate a staged deployment'); @@ -634,7 +658,9 @@ export async function claimStagedDeployment( if (row.status !== 'staged') { throw new ClientError(`Deployment '${deploymentId}' is '${row.status}', not staged and available for activation`); } - await table.patch(deploymentId, { status: 'activating', phase: 'activate', completed_at: null, error: null }); + if (options.persist !== false) { + await table.patch(deploymentId, { status: 'activating', phase: 'activate', completed_at: null, error: null }); + } return row; } diff --git a/components/operations.js b/components/operations.js index 152f7a759b..4c1c4c20a5 100644 --- a/components/operations.js +++ b/components/operations.js @@ -63,6 +63,7 @@ const { markDeploymentTerminal, recordDeploymentPeers, claimStagedDeployment, + isDeploymentTrackingAvailable, expireOldStagedDeployments, invalidateProjectStagedDeployments, pruneProjectPayloads, @@ -538,7 +539,29 @@ async function deployComponent(req) { HTTP_STATUS_CODES.BAD_REQUEST ); } - if (req.replicated !== false && !isReplicatedExecution && req.two_phase !== false && systemReplicated) { + // Separated phases cannot degrade gracefully: `activate: false` returns a deployment_id that only the + // row resolves, and `deployment_id` has nothing to look up without it. Failing here beats reporting a + // successful stage the cluster can never activate. An explicit `two_phase: true` fails for the same + // reason — the caller asked for a protocol that coordinates through the row. + const deploymentTracked = isDeploymentTrackingAvailable(); + if (!isReplicatedExecution && !deploymentTracked && (requestedSeparatedPhase || req.two_phase === true)) { + throw handleHDBError( + new Error(), + `${requestedSeparatedPhase ? 'activate:false and deployment_id' : 'two_phase:true'} require deployment ` + + `tracking, but the '${hdbTerms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME}' table is not available on ` + + `this node`, + HTTP_STATUS_CODES.SERVICE_UNAVAILABLE + ); + } + // Unspecified `two_phase` on a node without tracking stays on the one-shot path rather than entering a + // protocol that has nowhere to coordinate. Only an explicit request fails loudly, above. + if ( + req.replicated !== false && + !isReplicatedExecution && + req.two_phase !== false && + systemReplicated && + deploymentTracked + ) { if (req.deployment_id) return deployComponentActivateExisting(req); return deployComponentTwoPhase(req); } @@ -1257,6 +1280,8 @@ async function componentDeployPhase(req) { beforeSwap: async () => { await claimStagedDeployment(req.deployment_id, req.project, { allowActivating: true, + // Peer: validate, do not write. The origin owns this row; see claimStagedDeployment. + persist: false, waitForStagedMs: coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS), }); }, diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index 45b8e31e36..be649e77bb 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -61,15 +61,15 @@ describe('deploy_component two-phase orchestration', function () { process.env.HARPER_SAFE_MODE = 'true'; // The first component operation completes lazy server initialization, which replaces // databases.system. Run it before installing the table seam used by these tests. - const warmupProject = name(); - const warmup = await operations.deployComponent({ - project: warmupProject, + // + // One-shot on purpose: the separated phases now require deployment tracking, and the whole point + // of this warmup is that it runs BEFORE the table seam exists. `after` removes the component + // directory along with every other fixture name. + await operations.deployComponent({ + project: name(), payload: await makePayload('warmup'), - activate: false, - }); - await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, warmup.deployment_id), { - recursive: true, - force: true, + two_phase: false, + restart: false, }); if (!databases.system) databases.system = {}; priorTable = databases.system[DEPLOYMENT_TABLE]; @@ -122,6 +122,32 @@ describe('deploy_component two-phase orchestration', function () { return value; } + it('refuses a separated-phase deploy when deployment tracking is unavailable', async () => { + // `DeploymentRecorder.put()` is tolerant by design, so without this guard `activate: false` returned + // a deployment_id that no row could resolve — a stage reporting success that nothing could ever + // activate. The separated phases coordinate through the row, so they have to require it. + const project = name(); + const payload = await makePayload('untracked'); + const priorTable = databases.system[DEPLOYMENT_TABLE]; + delete databases.system[DEPLOYMENT_TABLE]; + try { + await assert.rejects( + () => operations.deployComponent({ project, payload, activate: false }), + /require deployment tracking/ + ); + await assert.rejects( + () => operations.deployComponent({ project, deployment_id: '00000000-0000-4000-8000-000000000000' }), + /require deployment tracking/ + ); + await assert.rejects( + () => operations.deployComponent({ project, payload, two_phase: true }), + /require deployment tracking/ + ); + } finally { + databases.system[DEPLOYMENT_TABLE] = priorTable; + } + }); + it('settles superseded staged rows on a full deploy, not only on stage-and-stop', async () => { // Staged DIRECTORIES are pruned on every stage, but the row half used to run only on the // `activate: false` return. A full deploy therefore evicted trees while their rows still read @@ -627,7 +653,11 @@ describe('deploy_component two-phase orchestration', function () { assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, staged.deployment_id, project)), true); await executePeerPhase('activate', row.activation_spec); assert.match(await fs.readFile(path.join(componentPath, 'index.js'), 'utf8'), /peer-phase/); - assert.strictEqual(rows.get(staged.deployment_id).status, 'activating'); + assert.strictEqual( + rows.get(staged.deployment_id).status, + 'staged', + 'a peer activates locally without advancing the replicated row — the origin owns it' + ); assert.strictEqual(restartNeeded(), true); }); @@ -655,7 +685,11 @@ describe('deploy_component two-phase orchestration', function () { ); assert.match(await fs.readFile(path.join(componentPath, 'index.js'), 'utf8'), /rebuilt-peer/); - assert.strictEqual(rows.get(staged.deployment_id).status, 'activating'); + assert.strictEqual( + rows.get(staged.deployment_id).status, + 'staged', + 'a peer activates locally without advancing the replicated row — the origin owns it' + ); }); it('waits for the staged row checkpoint when peer activation arrives first', async () => { @@ -683,7 +717,11 @@ describe('deploy_component two-phase orchestration', function () { ); assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /lagged-row/); - assert.strictEqual(rows.get(staged.deployment_id).status, 'activating'); + assert.strictEqual( + rows.get(staged.deployment_id).status, + 'staged', + 'a peer activates locally without advancing the replicated row — the origin owns it' + ); }); it('recovers a staged package specification for config and peer activation', async () => { diff --git a/unitTests/components/deploymentRecorder.test.js b/unitTests/components/deploymentRecorder.test.js index c0e16b07a8..db41dc3e38 100644 --- a/unitTests/components/deploymentRecorder.test.js +++ b/unitTests/components/deploymentRecorder.test.js @@ -959,6 +959,52 @@ describe('staged deployment state', () => { assert.strictEqual(installed.mock.rows.get('lagging-1').status, 'activating'); }); + it('does not write the deployment row when a peer claims a staged deployment', async () => { + // The row is replicated, so a peer patching it makes N+1 writers of one key. Under replication lag + // a peer's `activating` can land after the origin has written `success`, leaving a converged deploy + // stuck non-terminal. Peers validate and swap under the component preparation lock they already + // hold; the origin owns the row. + installed.mock.rows.set('peer-claim', { + deployment_id: 'peer-claim', + project: 'app', + status: 'staged', + started_at: 1, + }); + + const row = await claimStagedDeployment('peer-claim', 'app', { persist: false }); + + assert.strictEqual(row.deployment_id, 'peer-claim', 'the claim still validates and returns the row'); + assert.strictEqual( + installed.mock.rows.get('peer-claim').status, + 'staged', + 'but leaves the status alone for the origin to advance' + ); + }); + + it('still marks the row activating when the origin claims', async () => { + installed.mock.rows.set('origin-claim', { + deployment_id: 'origin-claim', + project: 'app', + status: 'staged', + started_at: 1, + }); + + await claimStagedDeployment('origin-claim', 'app'); + + assert.strictEqual(installed.mock.rows.get('origin-claim').status, 'activating'); + }); + + it('rejects a peer claim for the wrong component even without persisting', async () => { + installed.mock.rows.set('mismatch', { + deployment_id: 'mismatch', + project: 'other', + status: 'staged', + started_at: 1, + }); + + await assert.rejects(() => claimStagedDeployment('mismatch', 'app', { persist: false }), /belongs to component/); + }); + it('expires only staged rows beyond the per-project count', async () => { for (const [id, startedAt, status = 'staged'] of [ ['old', 100], From 4bac4516e0fdbc51a9ad01e61824e8357268db42 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 24 Aug 2026 14:16:34 -0400 Subject: [PATCH 86/94] fix(deploy): restore peer crash evidence, and stop the untracked fallback stranding peers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two regressions from the previous commit, both caught in review. **Peers lost their crash-recovery evidence.** With `persist: false` a peer swaps while its local row still reads `staged` — the phase operation and the origin's `activating` patch travel independently. A peer that died between the swap and its config commit therefore had a `staged` row and no staged leaf, which reconciliation read as a *broken candidate*: it settled the row `failed` (over the origin-owned row) and never persisted config, leaving the peer running new code under the previous release's configuration. Local evidence now outranks the replicated status. An activation artifact for that deployment is proof the swap began, so the entry goes down the roll-forward path instead of the discard path. This is the node-local evidence the review asked for and it already existed on disk — recovery was simply keying on the row instead. **The untracked one-shot fallback was not cluster-safe.** Routing a default deploy to one-shot when `hdb_deployment` is missing looked tolerant but was the unsafe choice: that path consumes the multipart stream into its own blob and strips `req.payload` before replication, so peers receive neither replayable bytes nor a row — and it activates locally *before* replicating, so the origin goes live alone. The fallback is gone. The default path is left exactly as it was, because it already fails safely: peers cannot find the row, the barrier never clears, and nothing activates. Only the requests that coordinate through the row directly — `activate: false`, activate-by-id, explicit `two_phase: true` — fail 503, and `two_phase: false` remains the explicit single-phase escape hatch. My earlier framing of this as a "tolerant default" was wrong in the unsafe direction. Also from the same review: the 503 tests assert the status code rather than only the message, a test covers the `two_phase: false` escape hatch still working untracked, and DESIGN.md's merge residue describing the rejected `_phase`-tagged wire protocol is replaced with the actual trusted `component_deploy_phase` + AsyncLocalStorage contract (it would have had a maintainer sending a request the server rejects). The duplicated JSDoc on `isDeploymentTrackingAvailable` is gone. Reported by codex, cursor-grok and the domain leg pre-push. --- DESIGN.md | 23 ++++++----- components/Application.ts | 9 ++++- components/deploymentRecorder.ts | 13 ++----- components/operations.js | 36 ++++++++--------- .../components/deployPhaseOperations.test.js | 39 +++++++++++++++---- unitTests/components/deployStaging.test.js | 36 +++++++++++++++++ 6 files changed, 110 insertions(+), 46 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index 772ed248c8..83a4cd4bdf 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -218,8 +218,12 @@ a missing `hdb_deployment` table — tracking is observability for a one-shot de observability for `activate: false` or activate-by-id, which coordinate _through_ the row: without it a stage would return a deployment_id nothing could resolve, so the stage would report success and be permanently unactivatable. Those requests, and an explicit `two_phase: true`, fail with 503 when the -table is absent. An unspecified `two_phase` on such a node stays on the one-shot path rather than -entering a protocol with nowhere to coordinate. +table is absent. The DEFAULT path is deliberately left alone: a default two-phase deploy on an +untracked node fails safely, because peers cannot find the row, the barrier never clears, and nothing +activates — the origin does not go live. Routing it to one-shot instead would be the unsafe choice, +since that path consumes the multipart stream into its own blob and strips `req.payload` before +replication, leaving peers with neither replayable bytes nor a row while the origin was already live. +`two_phase: false` remains the explicit single-phase escape hatch. Peer stage/activate/restart messages use the distinct authenticated `component_deploy_phase` operation. An older peer therefore rejects the unknown operation instead of ignoring a phase marker and deploying @@ -531,11 +535,12 @@ names differ (`stage`/`activate` vs the old `prepare`/`replicate`). `two_phase: legacy one-shot path. **There is only one public operation — `deploy_component`.** The two phases are NOT separate public -operations; the peer fan-out is `deploy_component` itself tagged with an internal `_phase: 'stage' | -'activate'` marker (the same `_`-prefixed internal-field convention peers already branch on, alongside -`_deploymentId`). `deployComponent` dispatches: a replicated execution with `_phase` runs the peer -stage/activate work (the `component_deploy_phase` operation) and never re-fans; a public call runs -the origin orchestrator. Two public properties expose the phases when an operator wants them separated +operations. The peer fan-out is the distinct trusted operation `component_deploy_phase`, which carries +the phase and the deployment id and is reachable only on the replication path — authorization is +carried in AsyncLocalStorage (`isOperationAuthorizationBypassed`), not on the request, so it cannot be +invoked over HTTP with ordinary credentials. Public `_phase` and `_deploymentId` fields are rejected +outright; an older peer therefore rejects an unknown operation rather than misreading a phase marker as +a one-shot deploy. A public call runs the origin orchestrator. Two public properties expose the phases when an operator wants them separated (e.g. pre-stage the cluster now, flip later — or a CI-stages / approver-activates split): `activate: false` stages cluster-wide and stops, returning the `deployment_id` in a `staged` state; passing that `deployment_id` back to `deploy_component` (with no new payload) activates the already-staged build. @@ -566,8 +571,8 @@ component, and it is **not** the watched base of any component's file watcher (t each live component dir, `EntryHandler`/`deriveCommonPatternBase`) — so building here fires no restart-on-change events and needs no `deploy:start` watcher suppression. That suppression is now scoped to `activateStagedApplication`, the only phase that writes the live path. Staging is deterministic -from the deployment id precisely so the activate phase (a separate replicated `deploy_component` -invocation on peers, tagged `_phase: 'activate'`) can reconstruct the same path the stage built — +from the deployment id precisely so the activate phase (a separate `component_deploy_phase` invocation +on peers) can reconstruct the same path the stage built — peers build a fresh `Application` per phase invocation, so there is no shared in-memory handle to rely on. The deployment id sits ABOVE the component name (`…//`, not `…//`) for two reasons: the leaf directory's basename is then the real component diff --git a/components/Application.ts b/components/Application.ts index 7c6b6833a5..28232bf08c 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -3465,7 +3465,14 @@ export async function reconcileStagedApplicationArtifacts( } } const stagedPath = stagedApplicationPath(componentDirPath, entry.name); - if (row.status === 'staged') { + // Local evidence outranks the replicated row's status. A peer swaps WITHOUT writing the row — + // the origin owns it — so a peer that died between its swap and its config commit has a + // `staged` row and no staged leaf. Read as a broken candidate that settled the row `failed` + // (over the origin's own row) and left new code live under the previous release's config. An + // activation artifact for this deployment is proof the swap began, so it is an interrupted + // activation and belongs in the roll-forward path below. + const activationBegan = (await activationArtifacts(componentDirPath, entry.name)).length > 0; + if (row.status === 'staged' && !activationBegan) { if (!(await hasCompleteStagedApplication(stagedPath))) { let discarded = false; await withComponentPreparationLock(componentDirPath, async () => { diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index be4ffc7d66..f2da0c737c 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -608,17 +608,10 @@ export async function recordDeploymentPeers(deploymentId: string, results: unkno await table.patch(deploymentId, { peer_results: peers }); } -/** Claim a staged deployment for activation while the component preparation lock is held. */ /** - * Validate that a staged deployment is available to activate, and — on the ORIGIN — mark the row - * `activating`. - * - * `persist: false` is the peer mode: the same validation, no write. The row is replicated, so every - * peer patching it makes N+1 writers of one key, and under replication lag a peer's `activating` - * can land AFTER the origin has written `success` — reverting a converged deploy to a non-terminal - * status it never leaves. The origin owns the row (as DESIGN.md and revertComponent both state); - * what a peer actually needs from "claiming" is mutual exclusion against another activation of the - * same component, and it already holds the component preparation lock for that. + * Claim a staged deployment for activation while the component preparation lock is held. On the + * ORIGIN this marks the row `activating`; `persist: false` runs the same validation without the + * write, because the row is replicated and only the origin owns it. See DESIGN.md. */ /** * Whether deployment tracking is provisioned on this node. diff --git a/components/operations.js b/components/operations.js index 4c1c4c20a5..4bb2d585a2 100644 --- a/components/operations.js +++ b/components/operations.js @@ -539,29 +539,29 @@ async function deployComponent(req) { HTTP_STATUS_CODES.BAD_REQUEST ); } - // Separated phases cannot degrade gracefully: `activate: false` returns a deployment_id that only the - // row resolves, and `deployment_id` has nothing to look up without it. Failing here beats reporting a - // successful stage the cluster can never activate. An explicit `two_phase: true` fails for the same - // reason — the caller asked for a protocol that coordinates through the row. - const deploymentTracked = isDeploymentTrackingAvailable(); - if (!isReplicatedExecution && !deploymentTracked && (requestedSeparatedPhase || req.two_phase === true)) { + // The separated phases coordinate THROUGH the row, so without it `activate: false` hands back a + // deployment_id nothing can resolve — a stage that reports success and can never be activated. Those + // requests, and an explicit `two_phase: true`, fail up front. + // + // The DEFAULT path is deliberately left alone. A default two-phase deploy on an untracked node fails + // safely: peers cannot find the row, so the barrier never clears and nothing activates — the origin + // does not go live. Routing it to one-shot instead would be the unsafe choice, because that path + // consumes the multipart stream into its own blob and strips `req.payload` before replication, so + // peers would get neither replayable bytes nor a row while the origin was already live. + if ( + !isReplicatedExecution && + !isDeploymentTrackingAvailable() && + (requestedSeparatedPhase || req.two_phase === true) + ) { throw handleHDBError( new Error(), - `${requestedSeparatedPhase ? 'activate:false and deployment_id' : 'two_phase:true'} require deployment ` + - `tracking, but the '${hdbTerms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME}' table is not available on ` + - `this node`, + `${requestedSeparatedPhase ? 'activate:false and deployment_id' : 'two_phase:true'} coordinate through ` + + `the '${hdbTerms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME}' table, which is not available on this ` + + `node. Deploy with two_phase:false to use the legacy single-phase path.`, HTTP_STATUS_CODES.SERVICE_UNAVAILABLE ); } - // Unspecified `two_phase` on a node without tracking stays on the one-shot path rather than entering a - // protocol that has nowhere to coordinate. Only an explicit request fails loudly, above. - if ( - req.replicated !== false && - !isReplicatedExecution && - req.two_phase !== false && - systemReplicated && - deploymentTracked - ) { + if (req.replicated !== false && !isReplicatedExecution && req.two_phase !== false && systemReplicated) { if (req.deployment_id) return deployComponentActivateExisting(req); return deployComponentTwoPhase(req); } diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index be649e77bb..2901c0896f 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -130,18 +130,41 @@ describe('deploy_component two-phase orchestration', function () { const payload = await makePayload('untracked'); const priorTable = databases.system[DEPLOYMENT_TABLE]; delete databases.system[DEPLOYMENT_TABLE]; + const unavailable = (error) => { + assert.strictEqual(error.statusCode, 503, `expected 503, got ${error.statusCode}: ${error.message}`); + assert.match(error.message, /coordinate through\n?.*hdb_deployment/s); + return true; + }; try { - await assert.rejects( - () => operations.deployComponent({ project, payload, activate: false }), - /require deployment tracking/ - ); + await assert.rejects(() => operations.deployComponent({ project, payload, activate: false }), unavailable); await assert.rejects( () => operations.deployComponent({ project, deployment_id: '00000000-0000-4000-8000-000000000000' }), - /require deployment tracking/ + unavailable ); - await assert.rejects( - () => operations.deployComponent({ project, payload, two_phase: true }), - /require deployment tracking/ + await assert.rejects(() => operations.deployComponent({ project, payload, two_phase: true }), unavailable); + } finally { + databases.system[DEPLOYMENT_TABLE] = priorTable; + } + }); + + it('still allows an explicit two_phase:false deploy when tracking is unavailable', async () => { + // The legacy single-phase path is the documented escape hatch: it replicates the whole operation + // with its payload rather than coordinating through a row, so it needs no table. + const project = name(); + const priorTable = databases.system[DEPLOYMENT_TABLE]; + delete databases.system[DEPLOYMENT_TABLE]; + try { + const result = await operations.deployComponent({ + project, + payload: await makePayload('untracked-oneshot'), + two_phase: false, + restart: false, + }); + assert.ok(result, 'the one-shot path still deploys with no deployment table'); + assert.match( + await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), + /untracked-oneshot/, + 'and the component actually went live' ); } finally { databases.system[DEPLOYMENT_TABLE] = priorTable; diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index a4b070fa31..c86209754c 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -1743,6 +1743,42 @@ describe('two-phase component directory transaction', function () { await cleanup(name); }); + it('rolls a peer activation forward from local evidence when its row still reads staged', async () => { + // A peer swaps WITHOUT writing the deployment row — the origin owns it — so a peer that died + // between its swap and its config commit leaves a `staged` row with no staged leaf. That read as a + // broken candidate: the row was settled `failed` (replicating over the origin's own row) and the + // config was never persisted, so the peer ran new code under the previous release's configuration. + // The activation artifact is local proof the swap began and has to outrank the replicated status. + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('candidate') }); + await stageApplication(application, deploymentId); + + // The peer's post-swap state: candidate live, staged leaf gone, backup parked, row still `staged`. + const stagedPath = stagedApplicationPath(application.dirPath, deploymentId); + const activationDir = path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR, name); + await fs.mkdir(activationDir, { recursive: true }); + const backupPath = path.join(activationDir, `.previous-${deploymentId}-1-1-${randomUUID()}`); + await fs.mkdir(backupPath, { recursive: true }); + await fs.writeFile(path.join(backupPath, 'index.js'), "module.exports = 'displaced';\n"); + await fs.rename(stagedPath, application.dirPath); + + const settled = []; + let persisted = 0; + const reconciliation = await reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async (id) => (id === deploymentId ? { deployment_id: id, project: name, status: 'staged' } : undefined), + async () => persisted++, + async (id, reason) => settled.push([id, reason]) + ); + + assert.strictEqual(persisted, 1, 'the interrupted activation persists its configuration'); + assert.deepStrictEqual(settled, [], 'and the row is NOT settled failed over the origin-owned row'); + assert.strictEqual(reconciliation.failedProjects.has(name), false); + assert.match(await readMarker(application.dirPath), /candidate/, 'the swapped candidate stays live'); + await cleanup(name); + }); + it('rolls forward an interrupted no-live restore instead of sweeping away its marker', async () => { // The no-live revert branch renames `previous` straight to live because there is nothing to // displace, so it has no holding directory — and its marker is the only evidence. Absence of a From e818f3768b2eb233f640bbdbd2fb9e859cfc569d Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 24 Aug 2026 14:31:43 -0400 Subject: [PATCH 87/94] fix(deploy): make the one-shot escape hatch actually safe, and classify peer evidence under the lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three majors, all consequences of the previous commit. **`two_phase: false` was not the safe escape hatch I documented it as.** The one-shot path strips `req.payload` before replicating because peers normally read the bytes from the row's `payload_blob` — but the condition was `systemReplicated && recorder`, and `recorder` is truthy with no table (its writes just no-op). So on an untracked node peers received a `_deploymentId`, no bytes, and no row to resolve, after this node was already live. Stripping is now gated on tracking actually being available; with no row to read from, the payload rides along in the replicated operation, which is what one-shot did before deployment tracking existed. This matters more than a normal bug because the 503 message and DESIGN.md both point operators at this path. The test for it now asserts on the REPLICATED REQUEST rather than local activation. Replication is stubbed in that harness, so a local-only assertion proved nothing about peers — and indeed the first version of this test passed against the unfixed code. It fails against it now. **The peer-evidence probe was sampled outside the component lock**, so it did not close the race it was added for: an activation could create its backup and rename the staged tree live between the probe and the lock, after which reconciliation patched the origin-owned row `failed` and deleted a live activation's deployment directory. Both signals are re-read under the lock now, and an activation that appears there routes to roll-forward instead of being settled. **And that new roll-forward path did not fail closed.** Attribution keyed only on `row.status === 'activating'`, so a rejecting persistence step on a `staged`-row-plus-artifact entry recorded an error and still let the swapped candidate load under the previous release's configuration. `activationBegan` now counts as equivalent durable evidence in the catch. Reported by codex, gemini, cursor-grok, cursor-composer and the domain leg. --- components/Application.ts | 25 +++++++++++---- components/operations.js | 8 ++++- .../components/deployPhaseOperations.test.js | 27 ++++++++++++---- unitTests/components/deployStaging.test.js | 32 +++++++++++++++++++ 4 files changed, 79 insertions(+), 13 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 28232bf08c..9cb7da8ca7 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -3437,6 +3437,9 @@ export async function reconcileStagedApplicationArtifacts( } // Declared outside the try so the catch can attribute a reconciliation failure to its component. let row: Record | undefined; + // Same reason: durable evidence that an activation began, which the catch has to treat exactly like + // an `activating` row when deciding whether to fail the component closed. + let activationBegan = false; try { row = await getDeployment(entry.name); if (!row || !safeComponentName(row.project)) { @@ -3471,13 +3474,19 @@ export async function reconcileStagedApplicationArtifacts( // (over the origin's own row) and left new code live under the previous release's config. An // activation artifact for this deployment is proof the swap began, so it is an interrupted // activation and belongs in the roll-forward path below. - const activationBegan = (await activationArtifacts(componentDirPath, entry.name)).length > 0; + activationBegan = (await activationArtifacts(componentDirPath, entry.name)).length > 0; if (row.status === 'staged' && !activationBegan) { if (!(await hasCompleteStagedApplication(stagedPath))) { let discarded = false; await withComponentPreparationLock(componentDirPath, async () => { - // Re-read under the lock. The check above is an unlocked fast path, so an activation may - // have swapped this candidate in and cleaned it up since. + // Re-read BOTH signals under the lock. The probes above are an unlocked fast path, and an + // activation can create its backup and rename the staged tree live in between — settling on + // the stale read would patch the origin-owned row `failed` and delete a live activation's + // deployment directory. + if ((await activationArtifacts(componentDirPath, entry.name)).length > 0) { + activationBegan = true; + return; + } if (await hasCompleteStagedApplication(stagedPath)) return; // Only a not-yet-activated candidate is broken; the live tree and its persisted config are // consistent. Failing the component closed here would take a healthy component offline and @@ -3492,7 +3501,8 @@ export async function reconcileStagedApplicationArtifacts( }); if (discarded) removed.push(entry.name); } - continue; + // An activation that appeared under the lock belongs in the roll-forward path below. + if (!activationBegan) continue; } if (await hasCompleteStagedApplication(stagedPath)) { await activateStagedApplication(new Application({ name: row.project }), entry.name, { @@ -3527,8 +3537,11 @@ export async function reconcileStagedApplicationArtifacts( // Fail the component closed for an interrupted activation, where the live tree and its durable // configuration can disagree. A confirmed `staged` failure leaves live state consistent, so it // does not. - if (row?.status === 'activating' && safeComponentName(row?.project)) { - failedProjects.set(row.project, reconcileError); + if ((row?.status === 'activating' || activationBegan) && safeComponentName(row?.project)) { + // `activationBegan` counts the same as an `activating` row: a swap that already happened is an + // interrupted activation whatever the replicated status says, so a rejecting persistence step + // must not leave the swapped candidate loading under the previous release's configuration. + failedProjects.set(row!.project, reconcileError); } else if (!row) { // The row is what says whether this was an interrupted activation, so an unreadable row is // the one case we cannot rule that out. The project name does not depend on the row — it is diff --git a/components/operations.js b/components/operations.js index 4bb2d585a2..e3daa31db2 100644 --- a/components/operations.js +++ b/components/operations.js @@ -682,9 +682,15 @@ async function deployComponentOneShot(req, credentialReferences, isReplicatedExe // ProgressEmitter holds function listeners that can't survive the replication channel's // serialization; strip it unconditionally. delete req.progress; - if (systemReplicated && recorder) { + if (systemReplicated && recorder && isDeploymentTrackingAvailable()) { // The hdb_deployment row + payload_blob reach peers via table replication, so peers look up // the payload by deployment_id. Drop req.payload to keep the operation body small. + // + // Gated on the table actually existing. `recorder` is truthy either way — its writes no-op + // without the table — so stripping the payload on that alone left peers with a `_deploymentId`, + // no bytes, and no row to resolve, after this node was already live. With no row to read from, + // the payload has to ride along in the replicated operation, which is what the one-shot path did + // before deployment tracking existed. delete req.payload; } const onPeerResult = recorder diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index 2901c0896f..51e939e7fe 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -147,26 +147,41 @@ describe('deploy_component two-phase orchestration', function () { } }); - it('still allows an explicit two_phase:false deploy when tracking is unavailable', async () => { - // The legacy single-phase path is the documented escape hatch: it replicates the whole operation - // with its payload rather than coordinating through a row, so it needs no table. + it('replicates the payload itself when two_phase:false runs with no deployment table', async () => { + // The legacy single-phase path is the documented escape hatch, so it has to actually be safe. It + // normally strips `req.payload` because peers read the bytes from the row's payload_blob — but with + // no table there is no row, so stripping left peers holding a `_deploymentId`, no bytes and nothing + // to resolve, after this node was already live. Asserting on the REPLICATED REQUEST, not just on + // local activation: replication is stubbed here, so a local-only assertion proves nothing about peers. const project = name(); const priorTable = databases.system[DEPLOYMENT_TABLE]; + const priorReplicate = server.replication.replicateOperation; + const replicated = []; + server.replication.replicateOperation = async (op) => { + replicated.push(op); + return { replicated: [] }; + }; delete databases.system[DEPLOYMENT_TABLE]; try { - const result = await operations.deployComponent({ + await operations.deployComponent({ project, payload: await makePayload('untracked-oneshot'), two_phase: false, restart: false, }); - assert.ok(result, 'the one-shot path still deploys with no deployment table'); assert.match( await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /untracked-oneshot/, - 'and the component actually went live' + 'the component goes live locally' + ); + assert.strictEqual(replicated.length, 1, 'the deploy is replicated to peers'); + assert.notStrictEqual( + replicated[0].payload, + undefined, + 'and carries its payload, because no row exists for peers to read the bytes from' ); } finally { + server.replication.replicateOperation = priorReplicate; databases.system[DEPLOYMENT_TABLE] = priorTable; } }); diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index c86209754c..30d9a49852 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -1779,6 +1779,38 @@ describe('two-phase component directory transaction', function () { await cleanup(name); }); + it('fails a component closed when the peer-evidence roll-forward cannot persist its configuration', async () => { + // The roll-forward path now accepts a `staged` row plus an activation artifact, so its FAILURE has to + // fail closed the same way an `activating` row does. Keying attribution on the row status alone let a + // rejecting persistence step (disk full, bad activation spec) record an error and still load the + // swapped candidate under the previous release's durable configuration. + const name = fixtureName(); + const deploymentId = randomUUID(); + const application = new Application({ name, payload: await makeComponentPayload('candidate') }); + await stageApplication(application, deploymentId); + const stagedPath = stagedApplicationPath(application.dirPath, deploymentId); + const activationDir = path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR, name); + await fs.mkdir(activationDir, { recursive: true }); + await fs.mkdir(path.join(activationDir, `.previous-${deploymentId}-1-1-${randomUUID()}`), { recursive: true }); + await fs.rename(stagedPath, application.dirPath); + + const reconciliation = await reconcileStagedApplicationArtifacts( + COMPONENTS_ROOT, + async (id) => (id === deploymentId ? { deployment_id: id, project: name, status: 'staged' } : undefined), + async () => { + throw new Error('activation spec is unusable'); + } + ); + + assert.strictEqual(reconciliation.errors.has(deploymentId), true, 'the failure is reported'); + assert.strictEqual( + reconciliation.failedProjects.has(name), + true, + 'and the component does not load with a swapped tree under unreconciled config' + ); + await cleanup(name); + }); + it('rolls forward an interrupted no-live restore instead of sweeping away its marker', async () => { // The no-live revert branch renames `previous` straight to live because there is nothing to // displace, so it has no holding directory — and its marker is the only evidence. Absence of a From c19e0266176001cf5a8f5f4738e8c6a673a1d4f8 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 24 Aug 2026 14:49:16 -0400 Subject: [PATCH 88/94] fix(deploy): send peers the buffered payload, not the stream ingest already drained MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit My previous fix for the `two_phase: false` escape hatch was incomplete, and its test hid that. Stopping the payload from being stripped is not enough: ingest DRAINS the source, so `req.payload` is an exhausted `Readable` by the time replication happens. Peers received an EOF after this node was already live — the same split the fix was meant to close. The degraded (no-table) ingest path already buffers the whole upload in memory, which is the only replayable copy that exists. The recorder now exposes it and the one-shot path substitutes it for the spent source before replicating, so the bytes actually travel in the operation. The test is the more important half. It previously used a reusable `Buffer` and asserted only that `payload` was not undefined — so it passed against the broken behavior, because a spent stream is still a defined property. It now drives the deploy with a real `Readable` and compares the replicated bytes to what was uploaded. Reverting to the previous fix fails it on exactly that assertion. Reported by codex, gemini, cursor-grok, cursor-composer and the domain leg. --- components/deploymentRecorder.ts | 14 ++++++++++++++ components/operations.js | 8 +++++++- .../components/deployPhaseOperations.test.js | 16 ++++++++++------ 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index f2da0c737c..7f96bd5d72 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -119,6 +119,10 @@ export class DeploymentRecorder { private dirty = false; private sealed = false; private flushSuppressed = false; + // Set only on the degraded (no-table) ingest path, which buffers the whole upload in memory. The + // original source is exhausted by then, so this is the ONLY replayable copy — and with no row for + // peers to read a blob from, it is what has to travel in the replicated operation. + private bufferedPayload?: Buffer; private readonly ingestTimeoutMs?: number; private constructor(deploymentId: string, initial: Record, ingestTimeoutMs?: number) { @@ -234,6 +238,14 @@ export class DeploymentRecorder { * In-memory sources (a Buffer, or the legacy base64-in-JSON/CBOR body) are already * materialized by the time they reach us, so they take the simpler buffer path. */ + /** + * A replayable copy of the ingested payload, present only when ingest buffered it in memory — + * which is exactly the no-table case, where there is no row for peers to read the bytes from. + */ + get replayablePayload(): Buffer | undefined { + return this.bufferedPayload; + } + async ingestPayload(source: Readable | Buffer | string): Promise { this.flushSuppressed = true; try { @@ -262,6 +274,7 @@ export class DeploymentRecorder { // already holds. No cap — a large base64-in-JSON body is the caller's choice. if (Buffer.isBuffer(source) || typeof source === 'string') { const buffer = Buffer.isBuffer(source) ? source : Buffer.from(source, 'base64'); + this.bufferedPayload = buffer; hash.update(buffer); this.record.payload_blob = createBlob(buffer, { type: 'application/gzip' }); this.record.payload_hash = hash.digest('hex'); @@ -300,6 +313,7 @@ export class DeploymentRecorder { chunks.push(buf); } const buffer = Buffer.concat(chunks); + this.bufferedPayload = buffer; hash.update(buffer); this.record.payload_blob = createBlob(buffer, { type: 'application/gzip' }); this.record.payload_hash = hash.digest('hex'); diff --git a/components/operations.js b/components/operations.js index e3daa31db2..553f593a38 100644 --- a/components/operations.js +++ b/components/operations.js @@ -682,7 +682,13 @@ async function deployComponentOneShot(req, credentialReferences, isReplicatedExe // ProgressEmitter holds function listeners that can't survive the replication channel's // serialization; strip it unconditionally. delete req.progress; - if (systemReplicated && recorder && isDeploymentTrackingAvailable()) { + if (systemReplicated && recorder && !isDeploymentTrackingAvailable() && recorder.replayablePayload) { + // No row exists for peers to read the blob from, and `req.payload` is an EXHAUSTED source by now + // — ingest drained it. Replace it with the buffered copy so the bytes actually travel in the + // replicated operation; keeping the spent stream would send peers an EOF after this node was + // already live. + req.payload = recorder.replayablePayload; + } else if (systemReplicated && recorder && isDeploymentTrackingAvailable()) { // The hdb_deployment row + payload_blob reach peers via table replication, so peers look up // the payload by deployment_id. Drop req.payload to keep the operation body small. // diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js index 51e939e7fe..78d0087087 100644 --- a/unitTests/components/deployPhaseOperations.test.js +++ b/unitTests/components/deployPhaseOperations.test.js @@ -6,6 +6,7 @@ const { existsSync } = require('node:fs'); const os = require('node:os'); const path = require('node:path'); const zlib = require('node:zlib'); +const { Readable } = require('node:stream'); const tarfs = require('tar-fs'); const testUtils = require('../testUtils.js'); @@ -163,9 +164,12 @@ describe('deploy_component two-phase orchestration', function () { }; delete databases.system[DEPLOYMENT_TABLE]; try { + // A real Readable, not a reusable Buffer: ingest DRAINS the source, so a Buffer would hide the + // actual failure (peers receiving an exhausted stream / EOF). + const bytes = await makePayload('untracked-oneshot'); await operations.deployComponent({ project, - payload: await makePayload('untracked-oneshot'), + payload: Readable.from([bytes]), two_phase: false, restart: false, }); @@ -175,11 +179,11 @@ describe('deploy_component two-phase orchestration', function () { 'the component goes live locally' ); assert.strictEqual(replicated.length, 1, 'the deploy is replicated to peers'); - assert.notStrictEqual( - replicated[0].payload, - undefined, - 'and carries its payload, because no row exists for peers to read the bytes from' - ); + // Assert the BYTES, not merely that the property exists — presence of a spent stream is exactly + // the bug this covers. + const sent = replicated[0].payload; + assert.strictEqual(Buffer.isBuffer(sent), true, `peers must receive replayable bytes, got ${typeof sent}`); + assert.strictEqual(Buffer.compare(sent, bytes), 0, 'and the bytes must be the payload that was uploaded'); } finally { server.replication.replicateOperation = priorReplicate; databases.system[DEPLOYMENT_TABLE] = priorTable; From 12ab61895d512a69b1c20a2697aa8fac0d9e025a Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 24 Aug 2026 15:35:59 -0400 Subject: [PATCH 89/94] refactor(deploy)!: drop the peer coordination protocol; keep the per-node staged deploy MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Splits this PR. The cluster-wide protocol is the part that unit tests cannot validate — every escaped defect in the last four review rounds traced back to it, and two of my own tests passed against broken behavior because the harness cannot observe peers at all. It moves to its own PR, to land with real multi-node verification. Preserved on `claude/deploy-peer-protocol-pr-b`. What stays is everything a single node can prove: - `deploy_component` builds into `.deploy-staging//`, validates that the staged tree loads, then atomically renames it live — committing root config and `harper-application-lock.json` in the same compensating transaction. The live component keeps serving through the install; a fetch or install failure leaves it untouched rather than half-replaced in place. - `revert_component`: addressed, idempotent, config-level, with retained-previous manifests. - Startup reconciliation for interrupted extractions, reverts and activations. - Staged-build and payload retention. - `harper revert` on the CLI. Removed: `component_deploy_phase` (operation, handler, validator, registration, authorization), the stage/activate fan-out and barrier, `deployComponentTwoPhase`, `deployComponentActivateExisting`, peer row claims, and the `activate: false` / `deployment_id` / `two_phase` public surface with its CLI verbs and capability probe. The request/response contract is now identical to before this PR. Two things the deletion surfaced that are worth naming: - `assertNotProtectedCoreComponent` lived inside the root-config write I removed, so protected core names were briefly overwritable without `force`. Caught by an existing test, restored explicitly in the deploy path, package deploys only, as before. - `activation_spec` was no longer recorded on the row, and startup reconciliation reads it to reconcile config after an interrupted activation — a row without it cannot be recovered. It is now written before the build starts. BREAKING CHANGE: none against released Harper. `activate: false`, `deployment_id`, `two_phase` and `component_deploy_phase` were introduced by this PR and never shipped, so removing them restores the released contract rather than changing it. --- DESIGN.md | 183 +--- bin/cliOperations.ts | 48 +- components/operations.js | 610 +---------- components/operationsValidation.js | 46 +- .../deploy-tracking-peer-branch.test.ts | 6 +- resources/registrationDeprecated.ts | 1 - server/serverHelpers/serverUtilities.ts | 4 - unitTests/bin/cliOperations.test.js | 297 +----- unitTests/components/deployOperations.test.js | 458 ++++++++ .../components/deployPhaseOperations.test.js | 980 ------------------ .../components/deployPhaseValidators.test.js | 58 -- unitTests/components/deployValidators.test.js | 21 + utility/hdbTerms.ts | 3 - utility/operation_authorization.ts | 4 - 14 files changed, 572 insertions(+), 2147 deletions(-) create mode 100644 unitTests/components/deployOperations.test.js delete mode 100644 unitTests/components/deployPhaseOperations.test.js delete mode 100644 unitTests/components/deployPhaseValidators.test.js create mode 100644 unitTests/components/deployValidators.test.js diff --git a/DESIGN.md b/DESIGN.md index 83a4cd4bdf..b271656fc6 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -193,50 +193,32 @@ A package-manager timeout must not release this lock while npm descendants are s Boot's `harper-application-lock.json` records an application configuration only after preparation fulfills. Recording at queue time would make a failed install look complete and suppress its retry on the next boot. -## Two-phase component deploys roll forward from a durable activation claim - -With `system` database replication enabled, `deploy_component` first prepares the candidate under -`.deploy-staging//` on every node. The deployment row carries the complete, -immutable activation specification (package/install settings, routing, credential references, and -`force`); activate-by-id never accepts replacements for those fields. After every stage response, the -origin durably checkpoints the row as `staged`. Activation claims that row as `activating` while holding -the same per-component filesystem lock, then swaps the candidate into the live path and commits root -config plus `harper-application-lock.json` as one compensating transaction. This ordering is the -recovery record: startup preserves `staged` candidates, deletes terminal/orphan candidates, and rolls -an `activating` candidate forward before loading apps. - -**The origin owns the row; a peer's claim is local.** Peers run the same validation before their swap -but do NOT write the deployment row (`claimStagedDeployment(..., { persist: false })`). The row is -replicated, so a peer writing it would make N+1 writers of one key — and under replication lag a -peer's `activating` can land _after_ the origin has written `success`, leaving a converged deploy in a -non-terminal status it never leaves. What a peer needs from claiming is mutual exclusion against -another activation of the same component, and the per-component filesystem lock it already holds -provides exactly that. - -**The separated phases require deployment tracking.** `DeploymentRecorder` is deliberately tolerant of -a missing `hdb_deployment` table — tracking is observability for a one-shot deploy. It is not -observability for `activate: false` or activate-by-id, which coordinate _through_ the row: without it -a stage would return a deployment_id nothing could resolve, so the stage would report success and be -permanently unactivatable. Those requests, and an explicit `two_phase: true`, fail with 503 when the -table is absent. The DEFAULT path is deliberately left alone: a default two-phase deploy on an -untracked node fails safely, because peers cannot find the row, the barrier never clears, and nothing -activates — the origin does not go live. Routing it to one-shot instead would be the unsafe choice, -since that path consumes the multipart stream into its own blob and strips `req.payload` before -replication, leaving peers with neither replayable bytes nor a row while the origin was already live. -`two_phase: false` remains the explicit single-phase escape hatch. - -Peer stage/activate/restart messages use the distinct authenticated `component_deploy_phase` operation. -An older peer therefore rejects the unknown operation instead of ignoring a phase marker and deploying -the staged build live. Public `_phase`/`_deploymentId` fields are rejected; the latter remains accepted -only on the authenticated legacy one-shot replication path. Restart is gated until activation responses -have settled. A partial activation is reported as split-node state and recovered by staging and activating -a known-good build, or rolled back explicitly with `revert_component`, which is addressed rather than a -toggle (see "Reversibility" below) so a retry after a lost response cannot reverse the recovery. There is -deliberately no AUTOMATIC rollback (`revert_on_failure` is rejected): once any node is past the barrier, -"this peer reported failed" does not mean "this peer did not activate" — a peer can complete its swap and -then fail the persistent work that follows — so auto-reverting the failed peers would roll an untouched -node an extra version back and split the cluster three ways. `deployment_stagingRetention_maxCount` bounds resting staged -trees per component and payload retention is pruned in the same row-aware lifecycle. +## Component deploys build off to the side and swap atomically + +`deploy_component` prepares the candidate under `.deploy-staging//`, runs +extract + `npm install` there, validates that it loads, and only then renames it into the live path — +committing root config and `harper-application-lock.json` in the same compensating transaction. The +live component keeps serving through the slow, failure-prone work (git clone, registry install), and +go-live is one atomic rename. A fetch or install failure leaves the running component untouched +instead of half-replaced in place. + +This is **per node**. The operation replicates as a whole, exactly as the single-phase deploy always +did, and each node performs its own staged build. There is deliberately no cluster-wide barrier here: +nothing orders activation across nodes, so two deploys originated concurrently on different nodes can +still end with different versions live. That ordering guarantee — a leader or a monotonic activation +epoch — is tracked in #2294, and byte-identical package resolution across nodes in #2295. Both are +protocol additions that need real multi-node verification, so they are deliberately not attempted +here. + +The deployment row carries the activation specification (package/install settings, routing, credential +references, `force`). It is not a coordination channel between nodes; it exists so the deploy is +observable, so the payload has a durable home, and — the load-bearing part — so startup +reconciliation can reconcile root config for an activation that was interrupted mid-swap. A row +without `activation_spec` cannot be recovered, which is why it is written before the build starts. + +Startup reconciliation is the recovery record: it settles interrupted extractions, finishes or undoes +interrupted reverts, rolls an interrupted activation forward, and fails a component closed rather than +loading a live tree whose durable configuration disagrees with it. ## Peer-side deploy_component payload read: retryable blob stalls and `Readable.from()` cancellation @@ -520,93 +502,24 @@ this fix doesn't attempt to solve. `deploy_component`/`package_component` still declared entry points (`jsResource`/`graphqlSchema`) survived extraction — a truncation from some other future cause would still report success silently; that's a deferred, separate fix. -## Two-phase deploy: stage then activate (`components/Application.ts`, `components/operations.js`) - -`deploy_component` runs internally as two replicated phases so a cluster deploy is all-or-nothing at -the point of go-live. **Phase 1 (stage)** builds the incoming version — download/`npm pack` (incl. a -git clone), extract, `npm install` — into a hidden staging directory on every node. **Phase 2 -(activate)** atomically renames the staged copy into the live component path and restarts. The origin -stages locally, **waits for every node to report a successful stage before any node activates** -(`ignore_replication_errors` opts out of the barrier), then activates. If a node can't fetch the -package or fails `npm install`, it fails during staging while the live component is still untouched _on -every node_ — where the old one-shot path could leave a peer half-installed after other peers had -already restarted onto the new code. The request/response contract is unchanged; only the SSE phase -names differ (`stage`/`activate` vs the old `prepare`/`replicate`). `two_phase: false` forces the -legacy one-shot path. - -**There is only one public operation — `deploy_component`.** The two phases are NOT separate public -operations. The peer fan-out is the distinct trusted operation `component_deploy_phase`, which carries -the phase and the deployment id and is reachable only on the replication path — authorization is -carried in AsyncLocalStorage (`isOperationAuthorizationBypassed`), not on the request, so it cannot be -invoked over HTTP with ordinary credentials. Public `_phase` and `_deploymentId` fields are rejected -outright; an older peer therefore rejects an unknown operation rather than misreading a phase marker as -a one-shot deploy. A public call runs the origin orchestrator. Two public properties expose the phases when an operator wants them separated -(e.g. pre-stage the cluster now, flip later — or a CI-stages / approver-activates split): `activate: -false` stages cluster-wide and stops, returning the `deployment_id` in a `staged` state; passing that -`deployment_id` back to `deploy_component` (with no new payload) activates the already-staged build. -This was a deliberate API-surface choice (harper#1849 review): peer fan-out needs a wire format, not -two extra public ops, and folding the phases into `deploy_component` keeps the surface at one op while -the convergence properties cover the stage-now/activate-later use case. (`revert_component` stays a -distinct public op — it is a rollback, not a deploy phase.) - -**Scope of the barrier's guarantee: fetch + install, not load.** The cluster-wide "nobody activates -until everybody staged" guarantee covers the download/`npm pack` and `npm install` steps — the slow, -failure-prone work. The pre-go-live component _load_ check (`loadValidateComponent`, which surfaces a -component that installs cleanly but throws at load) runs during stage on the origin and on any node -whose stage executes on a worker (e.g. the op-API worker for an `activate: false` stage), but it is a -no-op on the main thread — and replicated peer stage executions run on the main thread -(`replicateOperation` → `sendOperationToNode` execute there), where app code deliberately isn't -loaded. So a load-time-only fault on a peer is not caught by the barrier; it surfaces at -activate/restart like any other. Gating load-time faults cluster-wide would require dispatching the -throwaway load to a worker on each peer during stage — a possible follow-up, not done here. - -The staging directory (`.deploy-staging//`) lives **under the components root**, -not in `os.tmpdir()`, even though its contents are transient. This is deliberate and load-bearing: -the go-live step is `rename(stagingDir, liveDir)`, which is only atomic when both paths share a -filesystem. `os.tmpdir()` is frequently a different mount (tmpfs, a separate volume); a cross-device -rename throws `EXDEV` and Node has no atomic fallback — you'd be back to a slow recursive copy at the -exact moment you want the swap to be instantaneous, reintroducing the downtime window the split -exists to remove. The leading dot keeps `loadComponentDirectories` from loading it as a phantom -component, and it is **not** the watched base of any component's file watcher (those are rooted at -each live component dir, `EntryHandler`/`deriveCommonPatternBase`) — so building here fires no -restart-on-change events and needs no `deploy:start` watcher suppression. That suppression is now -scoped to `activateStagedApplication`, the only phase that writes the live path. Staging is deterministic -from the deployment id precisely so the activate phase (a separate `component_deploy_phase` invocation -on peers) can reconstruct the same path the stage built — -peers build a fresh `Application` per phase invocation, so there is no shared in-memory handle to rely -on. The deployment id sits ABOVE the component name (`…//`, not -`…//`) for two reasons: the leaf directory's basename is then the real component -name, which the pre-go-live validation load needs (`componentLoader` keys the `ApplicationScope` and -status registry off `basename(componentDirectory)`, so a UUID leaf would register the throwaway load -under a bogus name); and each deploy gets its own parent directory, so a parallel or queued deploy of -the same component can never share a directory or have its staged build swept by another's cleanup. -`extractApplication`/`installApplication` build into `application.buildDirPath`, which defaults -to the live dir (`dirPath`) — this is what keeps the legacy one-shot path, boot-time -`installApplications`, and the direct `extractApplication` callers unchanged — and is repointed at -the staging dir only for the duration of a stage. - -Two-phase requires the `system` database to be replicated on the origin (`isSystemDatabaseReplicated`), -since the `hdb_deployment` row's `payload_blob` is how peers fetch the tarball and correlate the two -phases by deployment id. When `system` is excluded from a narrow `REPLICATION_DATABASES`, or the -caller passes `two_phase: false`, or the invocation is a peer replaying a one-shot deploy, -`deploy_component` falls back to `deployComponentOneShot` (the previous behavior, preserved verbatim). -Cross-version skew is a non-issue by policy — a cluster stays in lockstep on its Harper version, so -every node understands the `_phase`-tagged `deploy_component` fan-out — which is why there is no -capability negotiation on it. - -**Replicator contract this rides on (`harper-pro/replication/replicator.ts`).** -`server.replication.replicateOperation(op, {onPeerResult})` fans `op` to every node in `server.nodes` -in parallel, setting `op.replicated = false` on the copy it sends so a peer never re-fans (the deploy -handlers additionally detect a replicated execution by the presence of `_deploymentId` — always set on -the sub-operations — and run the peer stage/activate work off the `_phase` marker without re-fanning). Per-peer failures never throw — `sendOperationToNode` rejections are caught and -surface as `{status:'failed', reason, node}` entries in the returned `replicated[]` array and via -`onPeerResult`, which is exactly the shape `DeploymentRecorder.normalizePeerResult` consumes. Peers -authenticate node-to-node by TLS certificate, and the receive side runs the op via -`server.operation(data, {user}, !isAuthorizedNode)` — for a trusted cluster node the authorize flag is -`false`, so a replicated super-user op skips the permission gate. That is why the `_phase`-tagged -`deploy_component` fan-out and `revert_component` (registered with the same `permission(true, [])`, -dispatched by `operation` name) replicate without an `hdb_user`, identically to the long-proven -one-shot `deploy_component` fan-out. +## Staged deploy: build aside, then swap (`components/Application.ts`, `components/operations.js`) + +`deploy_component` builds the incoming version — download/`npm pack` (incl. a git clone), extract, +`npm install` — into a hidden staging directory, validates that it loads, then atomically renames it +into the live component path. The live component keeps serving throughout, and a fetch or install +failure leaves it untouched rather than half-replaced in place. The request/response contract is +unchanged; only the SSE phase names differ (`stage`/`activate` vs the old `prepare`/`replicate`). + +Each node does this for itself: the operation replicates as a whole, as it always did. There is no +cluster-wide barrier and no private peer operation — ordering activation across nodes (#2294) and +guaranteeing every node staged the same bytes (#2295) are protocol additions that need real +multi-node verification, and are deliberately out of scope here. + +**Why staging lives under the components root.** Go-live is `rename(stagingDir, liveDir)`, atomic only +when both share a filesystem. `os.tmpdir()` is frequently a different mount → `EXDEV` → a slow +recursive copy at exactly the moment an instant swap is wanted. So staging is a hidden directory under +the components root: same volume, dot-prefixed so the loader ignores it, and not the watched base of +any component's watcher, so building there fires no restart-on-change events. **Reversibility: retained previous + `revert_component`.** `activateStagedApplication` does not discard the tree its swap displaced — it retains it as `.deploy-previous/`, evicting the older one so @@ -655,10 +568,10 @@ swap had not yet placed the reverted-to version (the revert is undone and can be retained-previous path when the swap completed and only the retain step was lost. With both slots occupied the swap finished and the holding tree is residue, so it is discarded. -**Staged-build retention.** A full deploy consumes its staged build immediately (activate renames it -live), so the only builds that accumulate are `activate: false` stage-and-stops that are never -activated — each leaves `.deploy-staging//` in place so a later -`deploy_component({deployment_id})` can activate it. `stageApplication` bounds this: after a successful +**Staged-build retention.** A successful deploy consumes its staged build immediately (activation +renames it live), so nothing normally accumulates. What does accumulate is the residue of deploys that +never reached activation — a failed install, a crash between stage and swap — each leaving +`.deploy-staging//` behind. `stageApplication` bounds this: after a successful stage it evicts the oldest not-yet-activated staged builds for that component beyond `deployment_stagingRetention_maxCount` (default 5, `pruneStagedBuilds`), always keeping the just-staged one and the newest N−1 by mtime. Eviction is best-effort (`allSettled`, trace-logged) but awaited so diff --git a/bin/cliOperations.ts b/bin/cliOperations.ts index 3b11784f0f..97eaa0e5b4 100644 --- a/bin/cliOperations.ts +++ b/bin/cliOperations.ts @@ -31,17 +31,9 @@ const OP_ALIASES = { package: 'package_component', }; -// `stage` and `activate` are sugar over `deploy_component` — there are no separate stage/activate -// operations to find. const OP_VERB_PROPS: Record> = { - stage: { operation: 'deploy_component', activate: false, _cliVerb: 'stage' }, - // `_cliVerb` is a CLI-internal marker (stripped before the request is sent) so verbRequirementError - // can enforce that `harper activate` carries a deployment_id — without it, deploy_component's generic - // "no deployment_id → full deploy" fallback would silently build a brand-new deploy from the CWD. - // It also tells the staged-deploy capability probe that this invocation needs two-phase support. - activate: { operation: 'deploy_component', _cliVerb: 'activate' }, - // `harper revert` uploads nothing: the version it activates is already on every node. `_cliVerb` - // here only drives the missing-target guard below. + // `harper revert` uploads nothing: the version it activates is already on disk. `_cliVerb` is a + // CLI-internal marker (stripped before the request is sent) that drives the missing-target guard below. revert: { operation: 'revert_component', _cliVerb: 'revert' }, }; @@ -49,9 +41,6 @@ const OP_VERB_PROPS: Record> = { // verb invoked it). Returns an error message, or null when the request is fine. Pure + exported so it // is unit-testable without the network/process-exit machinery in cliOperations. function verbRequirementError(req: any): string | null { - if (req._cliVerb === 'activate' && !req.deployment_id) { - return '`harper activate` requires a deployment_id from a prior `harper stage` — usage: harper activate project= deployment_id='; - } // revert_component requires its target so a retry can't toggle the rejected release back in. Caught // here too, so the CLI names the flag instead of surfacing a raw validation error. if (req._cliVerb === 'revert' && !req.to_deployment_id) { @@ -197,22 +186,6 @@ async function targetSupportsStreamingDeploy(options: any): Promise { } } -async function targetSupportsStagedDeploy(options: any): Promise { - try { - const probeOptions = { - ...options, - headers: { ...options.headers, Accept: 'application/json' }, - timeout: CLI_OPERATION_TIMEOUT_MS, - }; - delete probeOptions.streamResponse; - const response = await httpRequest(probeOptions, { operation: 'registration_info' }); - if (response.statusCode !== 200 || !response.body) return false; - return JSON.parse(response.body)?.capabilities?.componentDeployTwoPhase === 1; - } catch { - return false; - } -} - // Wraps the local packaging stream so an fs error while tar'ing up the payload (e.g. a file // vanishing after the pre-deploy scan, or a permissions failure reading the project tree) // surfaces as a descriptive packaging error instead of a raw fs error code. Without this, an @@ -631,12 +604,6 @@ const prepareRevert = async (req) => { const PREPARE_OPERATION: any = { revert_component: prepareRevert, deploy_component: async (req) => { - // `harper activate deployment_id=` takes an already-staged build live, so there is nothing to - // package — but it still needs the CWD project default every deploy-family verb gets. - if (req.deployment_id) { - req.project ||= directoryProjectName(process.cwd()); - return; - } if (req.package) { return; } @@ -954,17 +921,6 @@ async function cliOperations(req: any, skipResponseLog = false) { let options: any, target: any; try { ({ options, target } = await resolveRequestOptions(req)); - // Staged (two-phase) deploy controls must never reach a server that doesn't understand them: an - // older target ignores `activate: false`/`deployment_id` and deploys LIVE cluster-wide instead — - // the opposite of the operator's intent, silently. Probe before packaging so the refusal costs - // nothing. Local (domain-socket) calls hit this same build, so no probe is needed there. - const requestsStagedDeploy = - req._cliVerb !== undefined || req.activate === false || req.deployment_id !== undefined || req.two_phase === true; - if (target && requestsStagedDeploy && !(await targetSupportsStagedDeploy(options))) { - throw new Error( - `Target Harper does not advertise staged-deploy support; refusing the request because an older server could deploy it live` - ); - } delete req._cliVerb; await PREPARE_OPERATION[req.operation]?.(req); // Streaming deploy (multipart upload + SSE progress) only works against >= 5.1 servers. diff --git a/components/operations.js b/components/operations.js index 553f593a38..15074ea3c6 100644 --- a/components/operations.js +++ b/components/operations.js @@ -1,13 +1,13 @@ 'use strict'; const path = require('node:path'); -const { isDeepStrictEqual } = require('node:util'); const { isMainThread } = require('node:worker_threads'); const fs = require('fs-extra'); const fg = require('fast-glob'); const normalize = require('normalize-path'); const validator = require('./operationsValidation.js'); const log = require('../utility/logging/harper_logger.ts'); +const { randomUUID } = require('node:crypto'); const hdbTerms = require('../utility/hdbTerms.ts'); const env = require('../utility/environment/environmentManager.ts'); const configUtils = require('../config/configUtils.ts'); @@ -35,10 +35,7 @@ const { prepareApplication, stageApplication, revertApplication, - stagedApplicationPath, - hasCompleteStagedApplication, activateStagedApplication, - discardStagedApplication, discardProjectStagedApplications, discardProjectActivationArtifacts, updateApplicationLockEntry, @@ -46,7 +43,6 @@ const { createApplicationActivationTransaction, createApplicationConfigTransaction, getRevertTarget, - getStagingRetentionMaxCount, dropComponentDirectory, discardRetainedPrevious, ASIDE_STAGING_DIR, @@ -59,12 +55,7 @@ const { server } = require('../server/Server.ts'); const { DeploymentRecorder, awaitDeploymentRow, - getDeploymentRow, - markDeploymentTerminal, - recordDeploymentPeers, - claimStagedDeployment, isDeploymentTrackingAvailable, - expireOldStagedDeployments, invalidateProjectStagedDeployments, pruneProjectPayloads, readPayloadBlobWithRetry, @@ -539,33 +530,6 @@ async function deployComponent(req) { HTTP_STATUS_CODES.BAD_REQUEST ); } - // The separated phases coordinate THROUGH the row, so without it `activate: false` hands back a - // deployment_id nothing can resolve — a stage that reports success and can never be activated. Those - // requests, and an explicit `two_phase: true`, fail up front. - // - // The DEFAULT path is deliberately left alone. A default two-phase deploy on an untracked node fails - // safely: peers cannot find the row, so the barrier never clears and nothing activates — the origin - // does not go live. Routing it to one-shot instead would be the unsafe choice, because that path - // consumes the multipart stream into its own blob and strips `req.payload` before replication, so - // peers would get neither replayable bytes nor a row while the origin was already live. - if ( - !isReplicatedExecution && - !isDeploymentTrackingAvailable() && - (requestedSeparatedPhase || req.two_phase === true) - ) { - throw handleHDBError( - new Error(), - `${requestedSeparatedPhase ? 'activate:false and deployment_id' : 'two_phase:true'} coordinate through ` + - `the '${hdbTerms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME}' table, which is not available on this ` + - `node. Deploy with two_phase:false to use the legacy single-phase path.`, - HTTP_STATUS_CODES.SERVICE_UNAVAILABLE - ); - } - if (req.replicated !== false && !isReplicatedExecution && req.two_phase !== false && systemReplicated) { - if (req.deployment_id) return deployComponentActivateExisting(req); - return deployComponentTwoPhase(req); - } - // Ingest any provided credential token into the secrets store so the credential lives as // replicated ciphertext (reference, not embed); already-reference entries pass through, and with // no custody a literal token stays as a transient, this-node-only fallback (#1158). Peers @@ -614,9 +578,6 @@ function markRestartRequiredForDeploy(application) { async function deployComponentOneShot(req, credentialReferences, isReplicatedExecution) { const { resolveCredentials } = require('./secretOperations.ts'); - // Write to root config if the request contains a package identifier - if (req.package) await writeComponentRootConfig(req, credentialReferences); - // Create a hdb_deployment row up front so the deploy is observable and auditable even if the CLI // disconnects. The row also holds the payload in a Blob attribute, which doubles as the source for // peer replication and (later) rollback. Only the origin node records — peers replaying the @@ -626,10 +587,15 @@ async function deployComponentOneShot(req, credentialReferences, isReplicatedExe // still gets phase events for non-SSE deploys. const emitter = isReplicatedExecution ? null : (req.progress ?? new ProgressEmitter()); if (emitter && !req.progress) req.progress = emitter; + // Built before the recorder so the row can carry it. Startup reconciliation reads + // `row.activation_spec` to reconcile root config after an interrupted activation, so a row without + // it cannot be recovered. + const activationSpec = activationSpecFromRequest(req, credentialReferences); const recorder = isReplicatedExecution ? null : await DeploymentRecorder.create({ project: req.project, + activation_spec: activationSpec, package_identifier: req.package ?? null, user: req.hdb_user?.username, restart_mode: req.restart === 'rolling' ? 'rolling' : req.restart ? 'immediate' : null, @@ -645,6 +611,11 @@ async function deployComponentOneShot(req, credentialReferences, isReplicatedExe const emit = (event, data) => emitter?.emit(event, data); + // Protected core component names. This used to sit inside the root-config write, which the staged + // path replaced with the activation transaction — so it is asserted here explicitly, before any work. + // Package deploys only, exactly as before: a payload deploy has always been allowed to use the name. + if (req.package) assertNotProtectedCoreComponent(req.project, req.force); + // The payload-via-replicated-row path depends on `system` actually replicating on this node. const systemReplicated = isSystemDatabaseReplicated(); @@ -669,12 +640,30 @@ async function deployComponentOneShot(req, credentialReferences, isReplicatedExe if (credentialReferences.length) req.credentials = credentialReferences; else delete req.credentials; - emit('phase', { phase: 'prepare', status: 'start' }); - await prepareApplication(application); - emit('phase', { phase: 'prepare', status: 'done' }); + // Build off to the side, then swap. The live component keeps serving through the slow, + // failure-prone work — git clone, registry install — and go-live is one atomic rename, so a fetch + // or install failure leaves the running component untouched instead of half-replaced in place. + // This is per node: the operation replicates as a whole and each node does its own staged build. + // There is deliberately no cluster-wide barrier here; ordering activation across nodes is #2294. + const deploymentId = recorder?.deploymentId ?? req._deploymentId ?? randomUUID(); + emit('phase', { phase: 'stage', status: 'start' }); + const stagingDirPath = await stageApplication(application, deploymentId); + emit('phase', { phase: 'stage', status: 'done' }); + + // Validate the STAGED tree, so a component that installs cleanly but throws at load never reaches + // the live path (throwaway scopes; see loadValidateComponent). + await loadValidateComponent({ dirPath: stagingDirPath, emit }); - // Load the component to surface load-time errors early (throwaway scopes; see loadValidateComponent). - await loadValidateComponent({ dirPath: application.dirPath, emit }); + // Root config and `harper-application-lock.json` commit inside the swap, so the directory and the + // durable state describing it move together and compensate together. + const configTransaction = await createApplicationActivationTransaction(req.project, activationSpec); + emit('phase', { phase: 'activate', status: 'start' }); + await activateStagedApplication(application, deploymentId, { + beforeCommit: () => configTransaction.commit(), + onRollback: () => configTransaction.rollback(), + activationSpec, + }); + emit('phase', { phase: 'activate', status: 'done' }); const rollingRestart = req.restart === 'rolling'; // if doing a rolling restart set restart to false so that other nodes don't also restart. @@ -817,494 +806,12 @@ function activationSpecFromRequest(req, credentialReferences) { }; } -function applicationFromSpec(spec, payload, resolvedCredentials, installCapture, emit) { - return new Application({ - name: spec.project, - payload, - packageIdentifier: spec.package ?? undefined, - install: { - command: spec.install_command ?? undefined, - timeout: spec.install_timeout ?? undefined, - allowInstallScripts: spec.install_allow_scripts ?? undefined, - }, - credentials: resolvedCredentials, - onInstallLine: (manager, stream, line) => { - installCapture?.push(manager, stream, line); - emit?.('install', { manager, stream, line }); - }, - }); -} - -function failedPeerResults(results) { - return (results ?? []).filter((result) => result?.status === 'failed' || result?.error || result?.reason); -} - function describePeerFailures(failed) { return failed .map((peer) => `${peer.node ?? 'unknown'} (${peer.error?.message ?? peer.reason ?? 'unknown error'})`) .join(', '); } -function buildPhaseOperation(phase, deploymentId, project, activationSpec, extra = {}) { - return { - operation: hdbTerms.OPERATIONS_ENUM.COMPONENT_DEPLOY_PHASE, - phase, - deployment_id: deploymentId, - project, - activation_spec: activationSpec, - ...extra, - }; -} - -async function resolveSpecCredentials(spec, waitMs = 0) { - const { resolveCredentials } = require('./secretOperations.ts'); - return resolveCredentials(spec.credentials ?? [], spec.project, { waitMs }); -} - -function assertStoredActivationSpec(row, deploymentId, project, spec, allowedStatuses) { - if ( - !row || - row.project !== project || - !allowedStatuses.includes(row.status) || - !isDeepStrictEqual(row.activation_spec, spec) - ) { - throw new ServerError( - `Deployment '${deploymentId}' does not have the expected immutable activation specification for '${project}'` - ); - } -} - -async function sourceStagedPayload(deploymentId, spec, timeoutMs) { - const deadline = Date.now() + timeoutMs; - const row = await awaitDeploymentRow(deploymentId, { timeoutMs, requirePayload: !spec.package }); - assertStoredActivationSpec(row, deploymentId, spec.project, spec, ['pending', 'staging', 'staged', 'activating']); - if (spec.package) return undefined; - return readPayloadBlobWithRetry(() => row.payload_blob.stream(), { - timeoutMs: Math.max(0, deadline - Date.now()), - }); -} - -async function discardDeploymentEverywhere(project, deploymentId, activationSpec) { - const componentPath = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), project); - await discardStagedApplication(componentPath, deploymentId).catch(() => {}); - await server.replication - .replicateOperation(buildPhaseOperation('discard', deploymentId, project, activationSpec)) - .catch(() => {}); -} - -async function pruneStagedDeploymentArtifacts(project, activationSpec, keepDeploymentId) { - const expired = await expireOldStagedDeployments(project, getStagingRetentionMaxCount(), keepDeploymentId); - for (const deploymentId of expired) await discardDeploymentEverywhere(project, deploymentId, activationSpec); -} - -async function restartActivatedComponent(req, deploymentId, project, activationSpec, emit) { - if (req.restart === true) { - emit('phase', { phase: 'restart', status: 'start' }); - const restartResponse = await server.replication.replicateOperation( - buildPhaseOperation('restart', deploymentId, project, activationSpec, { - deployment_timeout: req.deployment_timeout, - }) - ); - const failed = failedPeerResults(restartResponse?.replicated); - manageThreads.restartWorkers('http'); - emit('phase', { phase: 'restart', status: 'done' }); - return { restartMessage: `, restarting Harper`, replicated: restartResponse?.replicated, failedPeers: failed }; - } - if (req.restart === 'rolling') { - const serverUtilities = require('../server/serverHelpers/serverUtilities.ts'); - emit('phase', { phase: 'restart', status: 'start' }); - const jobResponse = await serverUtilities.executeJob({ - operation: 'restart_service', - service: 'http', - replicated: true, - }); - emit('phase', { phase: 'restart', status: 'done' }); - return { restartMessage: `, restarting Harper`, restartJobId: jobResponse.job_id, failedPeers: [] }; - } - return { restartMessage: '', failedPeers: [] }; -} - -async function deployComponentTwoPhase(req) { - assertNotProtectedCoreComponent(req.project, req.force); - const { ingestCredentials, resolveCredentials } = require('./secretOperations.ts'); - req.credentials = await ingestCredentials(req, req.credentials, req.project); - const credentialReferences = (req.credentials ?? []).filter((entry) => entry?.secret !== undefined); - const activationSpec = activationSpecFromRequest(req, credentialReferences); - const emitter = req.progress ?? new ProgressEmitter(); - const emit = (event, data) => emitter.emit(event, data); - const installCapture = createInstallCapture(); - const recorder = await DeploymentRecorder.create({ - project: req.project, - package_identifier: req.package ?? null, - user: req.hdb_user?.username, - restart_mode: req.restart === 'rolling' ? 'rolling' : req.restart ? 'immediate' : null, - credentials: credentialReferences.length ? credentialReferences : null, - activation_spec: activationSpec, - emitter, - }); - let application; - let activationCommitted = false; - let activationBarrierPassed = false; - try { - let payload = req.payload; - if (req.payload != null) { - await recorder.ingestPayload(req.payload); - payload = recorder.row.payload_blob.stream(); - } - const resolvedCredentials = await resolveCredentials(req.credentials, req.project); - application = applicationFromSpec(activationSpec, payload, resolvedCredentials, installCapture, emit); - if (credentialReferences.length) req.credentials = credentialReferences; - else delete req.credentials; - delete req.progress; - delete req.payload; - - emit('phase', { phase: 'stage', status: 'start' }); - const stagedPath = await stageApplication(application, recorder.deploymentId); - await loadValidateComponent({ dirPath: stagedPath, emit }); - recorder.seal(); - const stageResponse = await server.replication.replicateOperation( - buildPhaseOperation('stage', recorder.deploymentId, req.project, activationSpec, { - deployment_timeout: req.deployment_timeout, - }), - { - onPeerResult: (result) => { - recorder.recordPeer(result); - emit('peer', result); - }, - } - ); - if (stageResponse?.replicated) recorder.recordPeers(stageResponse.replicated); - emit('phase', { phase: 'stage', status: 'done' }); - const stageFailures = recorder.getFailedPeers(); - if (stageFailures.length && !req.ignore_replication_errors) { - await discardDeploymentEverywhere(req.project, recorder.deploymentId, activationSpec); - throw new ServerError( - `Component '${req.project}' failed to stage on ${stageFailures.length} peer node(s): ` + - `${describePeerFailures(stageFailures)}. No node was activated and the live component is unchanged.` - ); - } - await recorder.checkpoint('staged', 'staged'); - - // Settled here, on every successful origin stage, so the ROWS match the staged DIRECTORIES that - // `stageApplication` has already pruned under the same policy. Gating this on `activate: false` left - // a full deploy evicting trees whose rows still read `staged` cluster-wide — list_deployments - // offering a deployment_id whose tree is gone, and a different one per node under clock skew. - // - // Retention is count-based, so N concurrent deploys of one project can still race for N slots: a - // candidate that is `staged` but whose deploy has not yet activated can be expired by a newer - // stage. That is inherent to the policy and already true of the directory prune this mirrors — - // only rows strictly older than the current request are eligible, which is what keeps a - // just-returned deployment safe. - await pruneStagedDeploymentArtifacts(req.project, activationSpec, recorder.deploymentId).catch((error) => - log.warn('Failed to prune expired staged deployments', error) - ); - - if (req.activate === false) { - emit('phase', { phase: 'staged', status: 'done' }); - await recorder.finish('staged'); - await pruneProjectPayloads(req.project, getPayloadRetentionMaxCount()).catch((error) => - log.warn('Failed to prune staged deployment payloads', error) - ); - return { - message: `Staged component: ${req.project}`, - project: req.project, - staged: true, - deployment_id: recorder.deploymentId, - replicated: stageResponse?.replicated, - ...(stageFailures.length ? { failed_peers: stageFailures } : {}), - }; - } - - const configTransaction = await createApplicationActivationTransaction(req.project, activationSpec); - await activateStagedApplication(application, recorder.deploymentId, { - beforeSwap: async () => { - await claimStagedDeployment(recorder.deploymentId, req.project); - emit('phase', { phase: 'activate', status: 'start' }); - }, - beforeCommit: () => configTransaction.commit(), - onRollback: () => configTransaction.rollback(), - activationSpec, - }); - activationCommitted = true; - const activateResponse = await server.replication.replicateOperation( - buildPhaseOperation('activate', recorder.deploymentId, req.project, activationSpec, { - deployment_timeout: req.deployment_timeout, - }), - { - onPeerResult: (result) => { - recorder.recordPeer(result); - emit('peer', result); - }, - } - ); - if (activateResponse?.replicated) recorder.recordPeers(activateResponse.replicated); - emit('phase', { phase: 'activate', status: 'done' }); - const activateFailures = recorder.getFailedPeers(); - if (activateFailures.length && !req.ignore_replication_errors) { - throw new ServerError( - `Component '${req.project}' activated on only part of the cluster. Split nodes: ` + - `${describePeerFailures(activateFailures)}. Roll forward by staging and activating a known-good deployment.` - ); - } - activationBarrierPassed = activateFailures.length === 0; - if (!req.restart) markRestartRequiredForDeploy(application); - const restart = await restartActivatedComponent(req, recorder.deploymentId, req.project, activationSpec, emit); - if (restart.failedPeers.length) recorder.recordPeers(restart.failedPeers); - if (restart.failedPeers.length && !req.ignore_replication_errors) { - throw new ServerError( - `Component '${req.project}' activated, but restart failed on: ${describePeerFailures(restart.failedPeers)}` - ); - } - emit('phase', { phase: 'success', status: 'done' }); - maybeReclaimPayload(recorder, emit); - await recorder.finish('success'); - await pruneProjectPayloads(req.project, getPayloadRetentionMaxCount()).catch((error) => - log.warn('Failed to prune deployment payloads', error) - ); - return { - message: `Successfully deployed: ${req.project}${restart.restartMessage}`, - project: req.project, - deployment_id: recorder.deploymentId, - replicated: activateResponse?.replicated, - ...(restart.restartJobId ? { restartJobId: restart.restartJobId } : {}), - ...(recorder.getFailedPeers().length ? { failed_peers: recorder.getFailedPeers() } : {}), - }; - } catch (error) { - if (application && !activationCommitted) { - await discardStagedApplication(application.dirPath, recorder.deploymentId).catch(() => {}); - } - const capture = installCapture.snapshot(); - const failedPeers = recorder.getFailedPeers(); - const message = error?.message ?? String(error); - const structured = { - error: message, - phase: recorder.row.phase, - deployment_id: recorder.deploymentId, - ...(capture.lines.length ? { install_output: capture } : {}), - ...(failedPeers.length ? { failed_peers: failedPeers } : {}), - }; - emit('error', { - message, - code: error?.statusCode ?? error?.code, - phase: recorder.row.phase, - deployment_id: recorder.deploymentId, - install_output: capture.lines.length ? capture : undefined, - failed_peers: failedPeers.length ? failedPeers : undefined, - }); - await recorder - .finish(activationBarrierPassed ? 'success' : activationCommitted ? 'activating' : 'failed', error) - .catch((finishError) => log.warn('Failed to record two-phase deployment failure', finishError)); - const outError = new ServerError(message, error?.statusCode); - outError.http_resp_msg = structured; - throw outError; - } -} - -const ACTIVATION_FRESH_FIELDS = [ - 'payload', - 'package', - 'install_command', - 'install_timeout', - 'install_allow_scripts', - 'urlPath', - 'host', - 'credentials', - 'force', - 'activate', - 'two_phase', -]; - -function assertActivationRequestIsReferenceOnly(req) { - const supplied = ACTIVATION_FRESH_FIELDS.filter((field) => req[field] !== undefined); - if (supplied.length) { - throw handleHDBError( - new Error(), - `deployment_id activation uses the immutable staged configuration; remove: ${supplied.join(', ')}`, - HTTP_STATUS_CODES.BAD_REQUEST - ); - } -} - -async function deployComponentActivateExisting(req) { - assertActivationRequestIsReferenceOnly(req); - const row = await getDeploymentRow(req.deployment_id); - if (!row) throw handleHDBError(new Error(), `No deployment found with id '${req.deployment_id}'`, 404); - if (row.project !== req.project || row.status !== 'staged' || !row.activation_spec) { - throw handleHDBError( - new Error(), - `Deployment '${req.deployment_id}' is not a staged deployment for component '${req.project}'`, - HTTP_STATUS_CODES.CONFLICT - ); - } - const spec = row.activation_spec; - assertNotProtectedCoreComponent(spec.project, spec.force); - const emitter = req.progress ?? new ProgressEmitter(); - const emit = (event, data) => emitter.emit(event, data); - const componentPath = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), req.project); - let application; - const stagedPath = stagedApplicationPath(componentPath, req.deployment_id); - if (await hasCompleteStagedApplication(stagedPath)) { - application = applicationFromSpec(spec, undefined, undefined, null, emit); - await loadValidateComponent({ dirPath: stagedPath, emit }); - } else { - const timeoutMs = coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS); - const payload = await sourceStagedPayload(req.deployment_id, spec, timeoutMs); - const credentials = await resolveSpecCredentials(spec, timeoutMs); - application = applicationFromSpec(spec, payload, credentials, createInstallCapture(), emit); - const rebuiltPath = await stageApplication(application, req.deployment_id); - await loadValidateComponent({ dirPath: rebuiltPath, emit }); - } - const configTransaction = await createApplicationActivationTransaction(req.project, spec); - emit('phase', { phase: 'activate', status: 'start' }); - let claimed = false; - try { - await activateStagedApplication(application, req.deployment_id, { - beforeSwap: async () => { - await claimStagedDeployment(req.deployment_id, req.project); - claimed = true; - }, - beforeCommit: () => configTransaction.commit(), - onRollback: () => configTransaction.rollback(), - activationSpec: spec, - }); - } catch (error) { - if (claimed) await markDeploymentTerminal(req.deployment_id, 'staged').catch(() => {}); - throw error; - } - let settledPeers = []; - let activationBarrierPassed = false; - try { - const peerResults = []; - const activateResponse = await server.replication.replicateOperation( - buildPhaseOperation('activate', req.deployment_id, req.project, spec, { - deployment_timeout: req.deployment_timeout, - }), - { - onPeerResult: (result) => { - peerResults.push(result); - emit('peer', result); - }, - } - ); - settledPeers = Array.isArray(activateResponse?.replicated) ? activateResponse.replicated : peerResults; - await recordDeploymentPeers(req.deployment_id, settledPeers); - emit('phase', { phase: 'activate', status: 'done' }); - const failed = failedPeerResults(settledPeers); - if (failed.length && !req.ignore_replication_errors) { - throw new ServerError( - `Component '${req.project}' activated on only part of the cluster. Split nodes: ` + - `${describePeerFailures(failed)}. Roll forward by staging and activating a known-good deployment.` - ); - } - activationBarrierPassed = failed.length === 0; - if (!req.restart) markRestartRequiredForDeploy(application); - const restart = await restartActivatedComponent(req, req.deployment_id, req.project, spec, emit); - if (restart.failedPeers.length) { - settledPeers = [...settledPeers, ...restart.failedPeers]; - await recordDeploymentPeers(req.deployment_id, restart.failedPeers); - } - if (restart.failedPeers.length && !req.ignore_replication_errors) { - throw new ServerError( - `Component '${req.project}' activated, but restart failed on: ${describePeerFailures(restart.failedPeers)}` - ); - } - await markDeploymentTerminal(req.deployment_id, 'success'); - await maybeReclaimFinishedPayload(req.deployment_id, emit); - await pruneProjectPayloads(req.project, getPayloadRetentionMaxCount()).catch((error) => - log.warn('Failed to prune activated deployment payloads', error) - ); - return { - message: `Activated component: ${req.project}${restart.restartMessage}`, - project: req.project, - activated: true, - deployment_id: req.deployment_id, - replicated: activateResponse?.replicated, - ...(restart.restartJobId ? { restartJobId: restart.restartJobId } : {}), - ...(failedPeerResults(settledPeers).length ? { failed_peers: failedPeerResults(settledPeers) } : {}), - }; - } catch (error) { - await markDeploymentTerminal(req.deployment_id, activationBarrierPassed ? 'success' : 'activating', error).catch( - () => {} - ); - throw error; - } -} - -async function componentDeployPhase(req) { - if (!isTrustedReplicatedOperation(req)) { - throw handleHDBError(new Error(), 'component_deploy_phase is restricted to authenticated cluster peers', 403); - } - const validation = validator.componentDeployPhaseValidator({ - phase: req.phase, - deployment_id: req.deployment_id, - project: req.project, - activation_spec: req.activation_spec, - deployment_timeout: req.deployment_timeout, - }); - if (validation) throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); - const spec = req.activation_spec; - if (!spec || spec.project !== req.project) { - throw handleHDBError(new Error(), 'Invalid immutable activation specification', HTTP_STATUS_CODES.BAD_REQUEST); - } - const componentPath = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), req.project); - if (req.phase === 'discard') { - await discardStagedApplication(componentPath, req.deployment_id); - return { message: `Discarded staged component: ${req.project}` }; - } - if (req.phase === 'restart') { - const row = await getDeploymentRow(req.deployment_id); - assertStoredActivationSpec(row, req.deployment_id, req.project, spec, [ - 'pending', - 'staging', - 'staged', - 'activating', - ]); - manageThreads.restartWorkers('http'); - return { message: `Restarting component runtime for: ${req.project}` }; - } - if (req.phase === 'stage') { - assertNotProtectedCoreComponent(req.project, spec.force); - const timeoutMs = coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS); - const payload = await sourceStagedPayload(req.deployment_id, spec, timeoutMs); - const credentials = await resolveSpecCredentials(spec, timeoutMs); - const application = applicationFromSpec(spec, payload, credentials, createInstallCapture()); - const stagedPath = await stageApplication(application, req.deployment_id); - await loadValidateComponent({ dirPath: stagedPath, emit: () => {} }); - return { message: `Staged component: ${req.project}`, project: req.project, staged: true }; - } - const row = await getDeploymentRow(req.deployment_id); - assertStoredActivationSpec(row, req.deployment_id, req.project, spec, ['pending', 'staging', 'staged', 'activating']); - let application; - const stagedPath = stagedApplicationPath(componentPath, req.deployment_id); - if (await hasCompleteStagedApplication(stagedPath)) { - application = applicationFromSpec(spec, undefined, undefined, null); - await loadValidateComponent({ dirPath: stagedPath, emit: () => {} }); - } else { - const timeoutMs = coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS); - const payload = await sourceStagedPayload(req.deployment_id, spec, timeoutMs); - const credentials = await resolveSpecCredentials(spec, timeoutMs); - application = applicationFromSpec(spec, payload, credentials, createInstallCapture()); - const rebuiltPath = await stageApplication(application, req.deployment_id); - await loadValidateComponent({ dirPath: rebuiltPath, emit: () => {} }); - } - const configTransaction = await createApplicationActivationTransaction(req.project, spec); - await activateStagedApplication(application, req.deployment_id, { - beforeSwap: async () => { - await claimStagedDeployment(req.deployment_id, req.project, { - allowActivating: true, - // Peer: validate, do not write. The origin owns this row; see claimStagedDeployment. - persist: false, - waitForStagedMs: coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS), - }); - }, - beforeCommit: () => configTransaction.commit(), - onRollback: () => configTransaction.rollback(), - activationSpec: spec, - }); - markRestartRequiredForDeploy(application); - return { message: `Activated component: ${req.project}`, project: req.project, activated: true }; -} - function isTrustedReplicatedOperation(req) { const user = req.hdb_user; return ( @@ -1488,31 +995,6 @@ function assertNotProtectedCoreComponent(project, force) { } } -// Persist a `package` deploy's entry into root config so every cold install (reboot, new peer, -// rollback) reinstalls it. In two-phase this runs at activation, once the bits are staged everywhere. -async function writeComponentRootConfig(req, credentialReferences) { - assertNotProtectedCoreComponent(req.project, req.force); - const applicationConfig = { package: req.package }; - // Avoid writing an empty `install:` block - if (req.install_command || req.install_timeout || req.install_allow_scripts !== undefined) { - applicationConfig.install = { - command: req.install_command, - timeout: req.install_timeout, - allowInstallScripts: req.install_allow_scripts, - }; - } - if (req.urlPath !== undefined) applicationConfig.urlPath = req.urlPath; - if (req.host !== undefined) applicationConfig.host = req.host; - // Persist credential references (never tokens) so every cold install of this component — reboot, new - // peer, revert — re-resolves the credential from the store. - if (credentialReferences.length) applicationConfig.credentials = credentialReferences; - // Same critical section the activation transaction uses. `addConfig` is a read-modify-write of a - // file whose entries are per-project, so a one-shot deploy running unlocked can write back a - // document it parsed before a concurrent activation or drop committed, resurrecting or dropping - // that project's entry. - await withPersistentStateLock(() => configUtils.addConfig(req.project, applicationConfig)); -} - // Resolve the tarball to extract from. On the origin, tee req.payload into the row's blob (the // channel peers read from) and re-source extraction from the persisted blob. On a peer replaying a // deploy without a payload, read the tarball from the replicated row's blob (bounded wait). @@ -1639,29 +1121,6 @@ function maybeReclaimPayload(recorder, emit) { } } -async function maybeReclaimFinishedPayload(deploymentId, emit) { - try { - const row = await getDeploymentRow(deploymentId); - const payloadSize = row?.payload_size; - const retentionMaxSize = getPayloadRetentionMaxSize(); - if ( - typeof payloadSize !== 'number' || - payloadSize <= retentionMaxSize || - failedPeerResults(row.peer_results).length > 0 || - row.payload_blob == null - ) { - return; - } - const { handleDeleteDeploymentPayload } = require('./deploymentOperations.ts'); - const result = await handleDeleteDeploymentPayload({ deployment_id: deploymentId }); - if (result.freed_bytes > 0) { - emit('payload_dropped', { payload_size: result.freed_bytes, max_size: retentionMaxSize }); - } - } catch (error) { - log.warn(`Failed to reclaim payload for activated deployment '${deploymentId}'`, error); - } -} - /** * Count-based payload retention (deployment_payloadRetention_maxCount): after a successful deploy, keep * only the newest N stored payloads for this project and drop the rest. Where the size-based reclaim @@ -2178,7 +1637,6 @@ exports.addComponent = addComponent; exports.dropCustomFunctionProject = dropCustomFunctionProject; exports.packageComponent = packageComponent; exports.deployComponent = deployComponent; -exports.componentDeployPhase = componentDeployPhase; exports.revertComponent = revertComponent; exports.getComponents = getComponents; exports.getComponentFile = getComponentFile; diff --git a/components/operationsValidation.js b/components/operationsValidation.js index 3a9077b55e..a37a36d93f 100644 --- a/components/operationsValidation.js +++ b/components/operationsValidation.js @@ -32,7 +32,6 @@ module.exports = { dropCustomFunctionProjectValidator, packageComponentValidator, deployComponentValidator, - componentDeployPhaseValidator, revertComponentValidator, setComponentFileValidator, getComponentFileValidator, @@ -525,31 +524,14 @@ function deployComponentValidator(req) { deployment_timeout: Joi.number().min(0).optional(), force: Joi.boolean().optional(), ignore_replication_errors: Joi.boolean().optional(), - // Stop after the incoming version is staged and verified cluster-wide, without going live. Returns - // the staged deployment_id; a later deploy_component with that deployment_id activates it. Defaults - // to true (full stage + activate). - activate: Joi.boolean().optional(), - // Activate a previously-staged deployment (from an `activate: false` stage) cluster-wide. Same safe - // charset as `project` because it becomes a staging-dir path segment (`.deploy-staging//`) - // — a `../` value would otherwise resolve the staging source outside `.deploy-staging`. - deployment_id: Joi.string().pattern(DEPLOYMENT_ID_REGEX).optional().messages({ - 'string.pattern.base': `'deployment_id' must be a UUID`, - }), - // Automatic rollback of a partially-activated cluster is deliberately NOT offered. Once any node - // has crossed the activation barrier there is no sound way to know which peers actually swapped: - // a peer can complete its swap and then fail the persistent work that follows, so it reports - // `failed` while running the new version. Auto-reverting "the peers that failed" would then roll - // an untouched node an extra version back and leave the cluster split three ways. A partial - // activation therefore stays visibly `activating` and is rolled forward (or reverted explicitly, - // by target, with revert_component). See DESIGN.md, "Partial activation". + // Automatic rollback is deliberately NOT offered. A node that reports `failed` may still have + // completed its swap and then failed the work that follows, so auto-reverting "the nodes that + // failed" can roll an untouched node an extra version back. Roll back explicitly, by target, with + // revert_component. See DESIGN.md, "Partial activation". revert_on_failure: Joi.any().forbidden().messages({ - 'any.unknown': `'revert_on_failure' is not supported; recover a partial activation by rolling forward, or roll back explicitly with revert_component`, + 'any.unknown': `'revert_on_failure' is not supported; roll back explicitly with revert_component`, }), - // Opt out of the two-phase (stage-then-activate) deploy and use the legacy one-shot path instead. - // Defaults to two-phase. - two_phase: Joi.boolean().optional(), _deploymentId: Joi.any().forbidden(), - _phase: Joi.any().forbidden(), urlPath: URL_PATH_SCHEMA, host: HOST_SCHEMA, // Deploy credentials. Each entry is npm registry auth (`registry`) or git host auth (`host`, @@ -564,24 +546,6 @@ function deployComponentValidator(req) { return validator.validateBySchema(req, deployProjSchema); } -/** Validate the path- and state-selecting fields on the authenticated peer-only deploy operation. */ -function componentDeployPhaseValidator(req) { - const phaseSchema = Joi.object({ - phase: Joi.string().valid('stage', 'activate', 'discard', 'restart').required(), - deployment_id: Joi.string().pattern(DEPLOYMENT_ID_REGEX).required().messages({ - 'string.pattern.base': `'deployment_id' must be a UUID`, - }), - project: Joi.string() - .pattern(PROJECT_FILE_NAME_REGEX) - .required() - .messages({ 'string.pattern.base': HDB_ERROR_MSGS.BAD_PROJECT_NAME }), - activation_spec: Joi.object().required(), - deployment_timeout: Joi.number().min(0).optional(), - }).unknown(false); - - return validator.validateBySchema(req, phaseSchema); -} - /** * Validate revert_component requests — swap a component's live version back to its retained previous * version. There are no build inputs: nothing is fetched, resolved or installed, because the bytes diff --git a/integrationTests/deploy/deploy-tracking-peer-branch.test.ts b/integrationTests/deploy/deploy-tracking-peer-branch.test.ts index db46396151..49fe7fa5af 100644 --- a/integrationTests/deploy/deploy-tracking-peer-branch.test.ts +++ b/integrationTests/deploy/deploy-tracking-peer-branch.test.ts @@ -1,7 +1,7 @@ /** * Deployment tracking — peer-operation authorization boundary. * - * In a real multi-node deploy, the origin sends a private `component_deploy_phase` + * In a real multi-node deploy, the origin replicates the whole `deploy_component` * operation and the peer reads the tarball from the replicated * `hdb_deployment.payload_blob` row. The authorization-bypass context that admits that * operation exists only around trusted replication dispatch, so an HTTP caller must not @@ -160,6 +160,6 @@ suite('Deployment tracking — peer-operation authorization boundary', (ctx: Con strictEqual(response.status, 400, `internal marker should be rejected; got: ${response.rawText}`); strictEqual(response.body.error, "'_deploymentId' is not allowed"); }); - // `component_deploy_phase` is trusted-peer-only and unreachable over HTTP, so a peer-branch - // end-to-end test has no entry point here; the three-node harper-pro suite covers it. + // Peer behavior is the replicated `deploy_component` operation itself, so there is no private + // peer entry point to drive here; the three-node harper-pro suite covers real replication. }); diff --git a/resources/registrationDeprecated.ts b/resources/registrationDeprecated.ts index 173cbaf90f..39b140b176 100644 --- a/resources/registrationDeprecated.ts +++ b/resources/registrationDeprecated.ts @@ -4,6 +4,5 @@ export function getRegistrationInfo() { return { version: packageJson.version, deprecated: true, - capabilities: { componentDeployTwoPhase: 1 }, }; } diff --git a/server/serverHelpers/serverUtilities.ts b/server/serverHelpers/serverUtilities.ts index 0cc2240f85..7ee13eb542 100644 --- a/server/serverHelpers/serverUtilities.ts +++ b/server/serverHelpers/serverUtilities.ts @@ -605,10 +605,6 @@ function initializeOperationFunctionMap(): Map { }); }); - describe('deploy_component cross-version compatibility', () => { - const target = 'https://example.com:9925/'; - let originalPackageDirectory; - let originalScan; - - beforeEach(() => { - saveCredentials(target, { operation_token: 'valid-token', refresh_token: 'refresh-token' }); - tokenAuthModule.isJWTExpired = () => false; - originalPackageDirectory = packageComponentModule.packageDirectory; - originalScan = packageComponentModule.scanPackageDirectory; - }); - - afterEach(() => { - packageComponentModule.packageDirectory = originalPackageDirectory; - packageComponentModule.scanPackageDirectory = originalScan; - }); - - // Streams an SSE `done` event so the modern (>= 5.1) deploy path can read its result. - const sseDoneResponse = (result) => - Object.assign(Readable.from([`event: done\ndata: ${JSON.stringify({ result })}\n\n`]), { - statusCode: 200, - headers: { 'content-type': 'text/event-stream' }, - }); - - it('downgrades a package deploy to legacy JSON when the target is < 5.1', async () => { - const calls = []; - commonUtilsModule.httpRequest = async (options, req) => { - calls.push({ options, req }); - if (req.operation === 'registration_info') { - return { statusCode: 200, body: JSON.stringify({ version: '5.0.31' }) }; - } - return { statusCode: 200, body: JSON.stringify({ message: 'Successfully deployed', success: true }) }; - }; - - const result = await cliOperationsModule.cliOperations( - { operation: 'deploy_component', package: '@scope/widget', project: 'widget', target: 'example.com' }, - true - ); - - // Probe first, then the deploy. - assert.strictEqual(calls[0].req.operation, 'registration_info'); - assert.strictEqual(calls[0].options.streamResponse, undefined); - const deploy = calls[1]; - // No streaming negotiation against the old server. - assert.strictEqual(deploy.options.headers.Accept, undefined); - assert.strictEqual(deploy.options.streamResponse, undefined); - // Body is a plain JSON object, not a multipart stream, and carries no transport-only fields. - assert.strictEqual(typeof deploy.req.pipe, 'undefined'); - assert.strictEqual(deploy.req.operation, 'deploy_component'); - assert.strictEqual(deploy.req._legacyDeploy, undefined); - assert.strictEqual(deploy.req._multipart, undefined); - assert.strictEqual(result.success, true); - }); - - it('downgrades a directory deploy to a CBOR binary payload when the target is < 5.1', async () => { - const fakeTarball = Buffer.from('fake-tarball-bytes'); - packageComponentModule.scanPackageDirectory = async () => ({ - totalSize: fakeTarball.length, - danglingSymlinks: [], - }); - packageComponentModule.packageDirectory = async () => fakeTarball; - - const calls = []; - commonUtilsModule.httpRequest = async (options, req) => { - calls.push({ options, req }); - if (req.operation === 'registration_info') { - return { statusCode: 200, body: JSON.stringify({ version: '5.0.31' }) }; - } - return { statusCode: 200, body: JSON.stringify({ message: 'Successfully deployed', success: true }) }; - }; - - const result = await cliOperationsModule.cliOperations( - { operation: 'deploy_component', project: 'widget', target: 'example.com' }, - true - ); - - const deploy = calls[1]; - assert.strictEqual(deploy.options.streamResponse, undefined); - // Multipart was abandoned in favor of a CBOR body carrying the tarball as a - // native binary Buffer — the transport pre-5.1 servers decode directly. - assert.strictEqual(deploy.options.headers['Content-Type'], 'application/cbor'); - assert.ok(Buffer.isBuffer(deploy.req), 'CBOR body should be a Buffer'); - const decoded = decodeCbor(deploy.req); - assert.ok(Buffer.isBuffer(decoded.payload), 'decoded payload should be a Buffer'); - assert.strictEqual(decoded.payload.toString(), 'fake-tarball-bytes'); - assert.strictEqual(decoded.operation, 'deploy_component'); - assert.strictEqual(decoded._multipart, undefined); - assert.strictEqual(result.success, true); - }); - - it('keeps the streaming deploy path when the target is >= 5.1', async () => { - const calls = []; - commonUtilsModule.httpRequest = async (options, req) => { - calls.push({ options, req }); - if (req.operation === 'registration_info') { - return { statusCode: 200, body: JSON.stringify({ version: '5.1.7' }) }; - } - return sseDoneResponse({ message: 'Successfully deployed', success: true }); - }; - - const result = await cliOperationsModule.cliOperations( - { operation: 'deploy_component', package: '@scope/widget', project: 'widget', target: 'example.com' }, - true - ); - - const deploy = calls[1]; - assert.strictEqual(deploy.options.headers.Accept, 'text/event-stream'); - assert.strictEqual(deploy.options.streamResponse, true); - assert.strictEqual(result.success, true); - }); - - it('does not downgrade when the version probe fails (assumes modern)', async () => { - const calls = []; - commonUtilsModule.httpRequest = async (options, req) => { - calls.push({ options, req }); - if (req.operation === 'registration_info') { - return { statusCode: 404, body: 'not found' }; - } - return sseDoneResponse({ message: 'Successfully deployed', success: true }); - }; - - const result = await cliOperationsModule.cliOperations( - { operation: 'deploy_component', package: '@scope/widget', project: 'widget', target: 'example.com' }, - true - ); - - const deploy = calls[1]; - assert.strictEqual(deploy.options.headers.Accept, 'text/event-stream'); - assert.strictEqual(result.success, true); - }); - - it('fails closed on every staged-deploy control against a target without two-phase capability', async () => { - const originalExit = process.exit; - const originalConsoleError = console.error; - const errors = []; - const calls = []; - process.exit = (code) => { - throw new ProcessExitSignal(code); - }; - console.error = (...args) => errors.push(args.join(' ')); - commonUtilsModule.httpRequest = async (options, req) => { - calls.push({ options, req }); - return { statusCode: 200, body: JSON.stringify({ version: '5.1.7' }) }; - }; - try { - for (const request of [ - { package: '@scope/widget', activate: false, _cliVerb: 'stage' }, - { package: '@scope/widget', activate: false }, - { deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa' }, - { package: '@scope/widget', two_phase: true }, - ]) { - await assert.rejects( - cliOperationsModule.cliOperations( - { - operation: 'deploy_component', - project: 'widget', - target: 'example.com', - ...request, - }, - true - ), - ProcessExitSignal - ); - } - } finally { - process.exit = originalExit; - console.error = originalConsoleError; - } - - assert.deepStrictEqual( - calls.map(({ req }) => req.operation), - Array(4).fill('registration_info'), - 'only one capability probe per request reached the target' - ); - assert.match(errors.join('\n'), /does not advertise staged-deploy support/); - }); - - it('renders stage phase events and strips its CLI-only verb marker', async () => { - const calls = []; - const rendered = []; - const originalRenderEvent = DeployRenderer.prototype.renderEvent; - DeployRenderer.prototype.renderEvent = function (message) { - rendered.push(message.event); - }; - commonUtilsModule.httpRequest = async (options, req) => { - calls.push({ options, req }); - if (req.operation === 'registration_info') { - return { - statusCode: 200, - body: JSON.stringify({ - version: '5.2.0', - capabilities: { componentDeployTwoPhase: 1 }, - }), - }; - } - return Object.assign( - Readable.from([ - 'event: phase\ndata: {"phase":"stage","status":"start"}\n\n', - 'event: done\ndata: {"result":{"staged":true}}\n\n', - ]), - { statusCode: 200, headers: { 'content-type': 'text/event-stream' } } - ); - }; - let result; - try { - result = await cliOperationsModule.cliOperations( - { - operation: 'deploy_component', - project: 'widget', - package: '@scope/widget', - activate: false, - _cliVerb: 'stage', - target: 'example.com', - }, - true - ); - } finally { - DeployRenderer.prototype.renderEvent = originalRenderEvent; - } - - const deploy = calls.at(-1); - assert.strictEqual(deploy.req._cliVerb, undefined); - assert.strictEqual(deploy.req.activate, false); - assert.deepStrictEqual(rendered, ['phase', 'done']); - assert.strictEqual(result.staged, true); - }); - - it('defaults the activate project from the current directory', async () => { - const calls = []; - const projectDir = path.join(testDir, 'activate-project'); - fs.ensureDirSync(projectDir); - const priorCwd = process.cwd(); - commonUtilsModule.httpRequest = async (options, req) => { - calls.push({ options, req }); - if (req.operation === 'registration_info') { - return { - statusCode: 200, - body: JSON.stringify({ - version: '5.2.0', - capabilities: { componentDeployTwoPhase: 1 }, - }), - }; - } - return Object.assign(Readable.from(['event: done\ndata: {"result":{"activated":true}}\n\n']), { - statusCode: 200, - headers: { 'content-type': 'text/event-stream' }, - }); - }; - try { - process.chdir(projectDir); - await cliOperationsModule.cliOperations( - { - operation: 'deploy_component', - deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', - _cliVerb: 'activate', - target: 'example.com', - }, - true - ); - } finally { - process.chdir(priorCwd); - } - - assert.strictEqual(calls.at(-1).req.project, 'activate-project'); - }); - }); - describe('"Harper is not running" messaging (harper#658)', () => { const NOT_RUNNING_MESSAGE = 'Harper is not running. Use `harperdb run` (or `harperdb start`) to start it.'; @@ -1767,7 +1498,7 @@ describe('cliOperations', () => { }); }); -describe('deploy CLI verbs (stage / activate fold into deploy_component)', () => { +describe('harper revert CLI verb', () => { const { buildRequest, verbRequirementError } = cliOperationsModule; let savedArgv; beforeEach(() => { @@ -1777,32 +1508,6 @@ describe('deploy CLI verbs (stage / activate fold into deploy_component)', () => process.argv = savedArgv; }); - it('`stage` maps to deploy_component with activate:false', () => { - process.argv = ['node', 'harper', 'stage', 'project=my_app']; - const req = buildRequest(); - assert.strictEqual(req.operation, 'deploy_component'); - assert.strictEqual(req.activate, false); - }); - - it('`activate` with a deployment_id maps to deploy_component and passes the verb guard', () => { - process.argv = ['node', 'harper', 'activate', 'project=my_app', 'deployment_id=abc-123']; - const req = buildRequest(); - assert.strictEqual(req.operation, 'deploy_component'); - assert.strictEqual(req.deployment_id, 'abc-123'); - assert.strictEqual(verbRequirementError(req), null); - }); - - it('`activate` WITHOUT a deployment_id is rejected (would otherwise become a full deploy from the CWD)', () => { - process.argv = ['node', 'harper', 'activate', 'project=my_app']; - const req = buildRequest(); - assert.match(verbRequirementError(req), /deployment_id/); - }); - - it('verbRequirementError ignores non-activate deploys', () => { - assert.strictEqual(verbRequirementError({ operation: 'deploy_component' }), null); - assert.strictEqual(verbRequirementError({ operation: 'deploy_component', activate: false }), null); - }); - it('`revert` maps to revert_component and carries the verb marker', () => { // The marker has to survive buildRequest for the guard below to fire at all. `revert` deliberately // lives in OP_VERB_PROPS rather than OP_ALIASES: buildRequest checks the alias table FIRST, so an diff --git a/unitTests/components/deployOperations.test.js b/unitTests/components/deployOperations.test.js new file mode 100644 index 0000000000..b5ecce4710 --- /dev/null +++ b/unitTests/components/deployOperations.test.js @@ -0,0 +1,458 @@ +'use strict'; + +const assert = require('node:assert'); +const fs = require('node:fs/promises'); +const { existsSync } = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const zlib = require('node:zlib'); +const { Readable } = require('node:stream'); +const tarfs = require('tar-fs'); + +const testUtils = require('../testUtils.js'); +testUtils.preTestPrep(); + +const { DEPLOY_STAGING_DIR, DEPLOY_ACTIVATION_DIR } = require('#src/components/Application'); +const operations = require('#src/components/operations'); +const { restartNeeded, resetRestartNeeded } = require('#src/components/requestRestart'); +const { server } = require('#src/server/Server'); +const { databases } = require('#src/resources/databases'); +const { SYSTEM_TABLE_NAMES, CONFIG_PARAMS } = require('#src/utility/hdbTerms'); +const { getConfigPath, readConfigFile } = require('#src/config/configUtils'); +const environment = require('#src/utility/environment/environmentManager'); +const { runWithOperationAuthorizationBypass } = require('#src/server/serverHelpers/operationAuthorizationState'); + +const COMPONENTS_ROOT = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); +const DEPLOYMENT_TABLE = SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME; + +function packDirectory(directory) { + return new Promise((resolve, reject) => { + const chunks = []; + tarfs + .pack(directory) + .pipe(zlib.createGzip()) + .on('data', (chunk) => chunks.push(chunk)) + .on('end', () => resolve(Buffer.concat(chunks))) + .on('error', reject); + }); +} + +async function makePayload(marker, version = marker, withNodeModules = true) { + const source = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-op-')); + await fs.writeFile(path.join(source, 'package.json'), JSON.stringify({ name: 'phase-op', version })); + await fs.writeFile(path.join(source, 'index.js'), `module.exports = ${JSON.stringify(marker)};\n`); + if (withNodeModules) await fs.mkdir(path.join(source, 'node_modules'), { recursive: true }); + const payload = await packDirectory(source); + await fs.rm(source, { recursive: true, force: true }); + return payload; +} + +describe('deploy_component staged deploy', function () { + this.timeout(30_000); + const rows = new Map(); + let priorTable; + let priorReplicate; + let priorSafeMode; + let sequence = 0; + const names = []; + + before(async () => { + priorSafeMode = process.env.HARPER_SAFE_MODE; + process.env.HARPER_SAFE_MODE = 'true'; + // The first component operation completes lazy server initialization, which replaces + // databases.system. Run it before installing the table seam used by these tests. + // + // One-shot on purpose: the separated phases now require deployment tracking, and the whole point + // of this warmup is that it runs BEFORE the table seam exists. `after` removes the component + // directory along with every other fixture name. + await operations.deployComponent({ + project: name(), + payload: await makePayload('warmup'), + restart: false, + }); + if (!databases.system) databases.system = {}; + priorTable = databases.system[DEPLOYMENT_TABLE]; + }); + + beforeEach(() => { + resetRestartNeeded(); + rows.clear(); + databases.system[DEPLOYMENT_TABLE] = { + async get(id) { + return rows.get(id); + }, + async put(row) { + rows.set(row.deployment_id, { ...row }); + }, + async patch(id, partial) { + const row = rows.get(id); + if (row) rows.set(id, { ...row, ...partial }); + }, + async *search(conditions = []) { + for (const row of rows.values()) { + if (conditions.every((condition) => row[condition.attribute] === condition.value)) yield row; + } + }, + }; + priorReplicate = server.replication.replicateOperation; + server.replication.replicateOperation = async () => ({ replicated: [] }); + }); + + afterEach(() => { + server.replication.replicateOperation = priorReplicate; + resetRestartNeeded(); + }); + + after(async () => { + if (priorSafeMode === undefined) delete process.env.HARPER_SAFE_MODE; + else process.env.HARPER_SAFE_MODE = priorSafeMode; + if (priorTable === undefined) delete databases.system[DEPLOYMENT_TABLE]; + else databases.system[DEPLOYMENT_TABLE] = priorTable; + for (const name of names) await fs.rm(path.join(COMPONENTS_ROOT, name), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-aside'), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-previous'), { recursive: true, force: true }); + }); + + function name() { + const value = `phase_op_${process.pid}_${sequence++}`; + names.push(value); + return value; + } + + it('replicates the payload itself when there is no deployment table to carry it', async () => { + // The legacy single-phase path is the documented escape hatch, so it has to actually be safe. It + // normally strips `req.payload` because peers read the bytes from the row's payload_blob — but with + // no table there is no row, so stripping left peers holding a `_deploymentId`, no bytes and nothing + // to resolve, after this node was already live. Asserting on the REPLICATED REQUEST, not just on + // local activation: replication is stubbed here, so a local-only assertion proves nothing about peers. + const project = name(); + const priorTable = databases.system[DEPLOYMENT_TABLE]; + const priorReplicate = server.replication.replicateOperation; + const replicated = []; + server.replication.replicateOperation = async (op) => { + replicated.push(op); + return { replicated: [] }; + }; + delete databases.system[DEPLOYMENT_TABLE]; + try { + // A real Readable, not a reusable Buffer: ingest DRAINS the source, so a Buffer would hide the + // actual failure (peers receiving an exhausted stream / EOF). + const bytes = await makePayload('untracked-oneshot'); + await operations.deployComponent({ + project, + payload: Readable.from([bytes]), + restart: false, + }); + assert.match( + await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), + /untracked-oneshot/, + 'the component goes live locally' + ); + assert.strictEqual(replicated.length, 1, 'the deploy is replicated to peers'); + // Assert the BYTES, not merely that the property exists — presence of a spent stream is exactly + // the bug this covers. + const sent = replicated[0].payload; + assert.strictEqual(Buffer.isBuffer(sent), true, `peers must receive replayable bytes, got ${typeof sent}`); + assert.strictEqual(Buffer.compare(sent, bytes), 0, 'and the bytes must be the payload that was uploaded'); + } finally { + server.replication.replicateOperation = priorReplicate; + databases.system[DEPLOYMENT_TABLE] = priorTable; + } + }); + + it('normalizes a string install_allow_scripts rather than reading it as truthy', async () => { + // Joi coerces it, but validateBySchema discards `result.value`, so the raw string reaches the + // handler. `install_allow_scripts: 'false'` would then read as truthy and run package lifecycle + // scripts for a caller that explicitly disabled them — over multipart/form, where every value + // arrives as a string. + const project = name(); + const result = await operations.deployComponent({ + project, + payload: await makePayload('1.0.0'), + install_allow_scripts: 'false', + restart: false, + }); + + assert.ok(result, 'the deploy succeeds'); + assert.strictEqual( + rows.get(result.deployment_id).activation_spec.install_allow_scripts, + false, + 'and the activation spec records a real boolean, not the string' + ); + }); + + it('uses a no-custody literal registry token for the origin install without recording it', async () => { + const project = name(); + const token = 'transient-origin-token'; + const installCommand = + `node -e "const fs=require('fs');` + + `const value=fs.readFileSync(process.env.npm_config_userconfig||process.env.NPM_CONFIG_USERCONFIG,'utf8');` + + `if(!value.includes('//registry.example.com/:_authToken='))process.exit(7);` + + `fs.writeFileSync('credential-seen','yes')"`; + const result = await operations.deployComponent({ + project, + payload: await makePayload('credential-origin', '6.0.0', false), + install_command: installCommand, + credentials: [{ registry: 'https://registry.example.com', token }], + restart: false, + }); + + // The install ran in the staging directory; the marker it wrote travels with the tree through the + // swap, so it lands at the live path. + assert.strictEqual( + await fs.readFile(path.join(COMPONENTS_ROOT, project, 'credential-seen'), 'utf8'), + 'yes', + 'the install saw the credential, and its output was swapped in with the tree' + ); + assert.strictEqual(rows.get(result.deployment_id).activation_spec.credentials, null); + assert.doesNotMatch(JSON.stringify(rows.get(result.deployment_id)), new RegExp(token)); + }); + + it('stages and activates a full deploy before reporting success', async () => { + const project = name(); + const result = await operations.deployComponent({ + project, + payload: await makePayload('full-deploy', '6.0.0'), + }); + + assert.match(result.message, /Successfully deployed/); + assert.strictEqual(rows.get(result.deployment_id).status, 'success'); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /full-deploy/); + assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id)), false); + assert.strictEqual(restartNeeded(), true, 'a new component activated without restart requires one'); + }); + + // revert_component + + it('reverts the cluster to a named previous deployment and fans the target out to peers', async () => { + const project = name(); + const first = await operations.deployComponent({ project, payload: await makePayload('rev-v1', '1.0.0') }); + const second = await operations.deployComponent({ project, payload: await makePayload('rev-v2', '2.0.0') }); + const fanout = []; + server.replication.replicateOperation = async (op) => { + fanout.push(op); + return { replicated: [] }; + }; + + const result = await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); + + assert.strictEqual(result.reverted, true); + assert.strictEqual(result.to_deployment_id, first.deployment_id); + assert.strictEqual(result.from_deployment_id, second.deployment_id); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /rev-v1/); + assert.strictEqual(rows.get(result.deployment_id).status, 'rolled_back'); + assert.strictEqual( + rows.get(result.deployment_id).rollback_of, + second.deployment_id, + 'the audit row records which deployment the rollback took out of service' + ); + assert.strictEqual(fanout.length, 1, 'peers get the revert'); + assert.strictEqual(fanout[0].operation, 'revert_component'); + assert.strictEqual( + fanout[0].to_deployment_id, + first.deployment_id, + 'peers are told WHICH version to end on, so the fan-out is idempotent per node' + ); + }); + + it('is a no-op when the requested deployment is already live, so a retry is safe', async () => { + const project = name(); + const first = await operations.deployComponent({ project, payload: await makePayload('retry-v1', '1.0.0') }); + await operations.deployComponent({ project, payload: await makePayload('retry-v2', '2.0.0') }); + await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); + + // The caller lost the first response and retried the identical request. + const retry = await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); + + assert.strictEqual(retry.reverted, false); + assert.match(retry.message, /already running/); + assert.match( + await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), + /retry-v1/, + 'a retried revert must not toggle the rejected version back in' + ); + }); + + it('rejects a revert with no target, and one whose target is not retained', async () => { + const project = name(); + await operations.deployComponent({ project, payload: await makePayload('target-v1', '1.0.0') }); + await operations.deployComponent({ project, payload: await makePayload('target-v2', '2.0.0') }); + + await assert.rejects( + () => operations.revertComponent({ project }), + /to_deployment_id/, + 'the target is mandatory — that is what makes a retry safe' + ); + await assert.rejects( + () => operations.revertComponent({ project, to_deployment_id: '00000000-0000-4000-8000-000000000000' }), + /neither the live version/, + 'only the immediately-previous version is retained' + ); + assert.match( + await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), + /target-v2/, + 'a refused revert changes nothing' + ); + }); + + it('takes the package reference out of root config when reverting away from a package deploy', async () => { + // Without this, installApplications() would reinstall the reverted-away package over the restored + // directory on the next cold start and silently undo the rollback. + const project = name(); + const packaged = await operations.deployComponent({ + project, + payload: await makePayload('cfg-packaged', '1.0.0'), + }); + // Stamp root config as a package deploy would have, then activate a payload version over it. + const { addConfig } = require('#src/config/configUtils'); + await addConfig(project, { package: 'some-pkg@1.0.0' }); + const plain = await operations.deployComponent({ project, payload: await makePayload('cfg-plain', '2.0.0') }); + assert.ok(plain.deployment_id); + + await operations.revertComponent({ project, to_deployment_id: packaged.deployment_id }); + + const entry = readConfigFile()?.[project]; + assert.strictEqual( + entry?.package, + undefined, + 'the reverted-to version had no package reference, so the stale one must be gone' + ); + }); + + it('reclaims an oversized payload only after a full two-phase activation succeeds', async () => { + const project = name(); + const priorMaxSize = environment.get(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE); + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, 1); + try { + const result = await operations.deployComponent({ + project, + payload: await makePayload('reclaimed-full-deploy', '6.0.0'), + }); + + const row = rows.get(result.deployment_id); + assert.strictEqual(row.status, 'success'); + assert.strictEqual(row.payload_blob, null); + assert.ok(row.event_log.some((event) => event.event === 'payload_dropped')); + } finally { + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, priorMaxSize); + } + }); + + it('accepts the legacy deployment row marker only from a trusted replicated operation', async () => { + const project = name(); + const payload = await makePayload('trusted-one-shot', '6.0.0'); + const result = await runWithOperationAuthorizationBypass(true, () => + operations.deployComponent({ + project, + payload, + _deploymentId: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', + replicated: false, + hdb_user: { name: 'cluster-peer' }, + }) + ); + + assert.match(result.message, /Successfully deployed/); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /trusted-one-shot/); + }); + + it('removes the root-config entry before it destroys anything, so a failed drop cannot resurrect', async () => { + // The resurrection vector is the root-config entry: installApplications() reinstalls whatever it + // names. Doing the persistent writes after the directory removal left a crash window where the + // live tree was gone but config still named the package, and the next boot brought the component + // back. Config-first inverts the failure: an interrupted drop is unfinished and re-runnable. + const project = name(); + const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-drop-crash-')); + const componentsRoot = path.join(configRoot, 'components'); + const configPath = path.join(configRoot, 'harper-config.yaml'); + const priorRootEnv = process.env.ROOTPATH; + const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); + const priorComponentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); + process.env.ROOTPATH = configRoot; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); + environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, componentsRoot); + await fs.writeFile(configPath, `rootPath: ${JSON.stringify(configRoot)}\n`); + try { + await fs.mkdir(path.join(componentsRoot, project), { recursive: true }); + await fs.writeFile(path.join(componentsRoot, project, 'index.js'), "module.exports = 'live';\n"); + await fs.appendFile(configPath, `${project}:\n package: some-package@1.0.0\n`); + // Make every aside-based teardown step fail: `.deploy-aside` occupied by a file is rejected as + // "not a directory", which is the closest deterministic stand-in for dying mid-teardown. + await fs.writeFile(path.join(componentsRoot, '.deploy-aside'), 'not a directory\n'); + + await assert.rejects(() => operations.dropComponent({ project })); + + assert.strictEqual( + (await fs.readFile(configPath, 'utf8')).includes(project), + false, + 'config is already clean, so the next boot cannot reinstall the component' + ); + assert.strictEqual( + existsSync(path.join(componentsRoot, project)), + true, + 'and the tree is still there: the drop is unfinished rather than finished-then-undone' + ); + } finally { + if (priorRootEnv === undefined) delete process.env.ROOTPATH; + else process.env.ROOTPATH = priorRootEnv; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); + environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, priorComponentsRoot); + await fs.rm(configRoot, { recursive: true, force: true }); + } + }); + + it('drop_component removes leftover recovery artifacts and the root-config entry', async () => { + const project = name(); + const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-drop-')); + const componentsRoot = path.join(configRoot, 'components'); + const configPath = path.join(configRoot, 'harper-config.yaml'); + const priorRootEnv = process.env.ROOTPATH; + const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); + const priorComponentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); + process.env.ROOTPATH = configRoot; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); + environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, componentsRoot); + await fs.writeFile(configPath, `rootPath: ${JSON.stringify(configRoot)}\n`); + try { + const staged = await operations.deployComponent({ + project, + payload: await makePayload('drop-stage', '6.0.0'), + activate: false, + }); + const activationPath = path.join(componentsRoot, DEPLOY_ACTIVATION_DIR, project, 'interrupted'); + await fs.mkdir(activationPath, { recursive: true }); + await fs.writeFile( + path.join(configRoot, 'harper-application-lock.json'), + JSON.stringify({ applications: { [project]: { package: 'stale-package' } } }) + ); + // A root-config entry is what makes a dropped component come back: installApplications() reads + // it on the next boot and reinstalls the package. Removing it and the application-lock entry as + // two separate writes meant a crash or a failed second write left this behind. + await fs.appendFile(configPath, `${project}:\n package: stale-package\n`); + + await operations.dropComponent({ project }); + + const deploymentStagePath = path.join(componentsRoot, DEPLOY_STAGING_DIR, staged.deployment_id); + assert.strictEqual( + existsSync(deploymentStagePath), + false, + `staged deployment directory still contains: ${await fs.readdir(deploymentStagePath).catch(() => [])}` + ); + assert.strictEqual(existsSync(path.join(componentsRoot, DEPLOY_ACTIVATION_DIR, project)), false); + const applicationLock = JSON.parse(await fs.readFile(path.join(configRoot, 'harper-application-lock.json'))); + assert.strictEqual(applicationLock.applications[project], undefined); + assert.strictEqual( + (await fs.readFile(configPath, 'utf8')).includes(project), + false, + 'the root-config entry is removed in the same step, so the next boot cannot reinstall the drop' + ); + } finally { + if (priorRootEnv === undefined) delete process.env.ROOTPATH; + else process.env.ROOTPATH = priorRootEnv; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); + environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, priorComponentsRoot); + await fs.rm(configRoot, { recursive: true, force: true }); + } + }); +}); diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js deleted file mode 100644 index 78d0087087..0000000000 --- a/unitTests/components/deployPhaseOperations.test.js +++ /dev/null @@ -1,980 +0,0 @@ -'use strict'; - -const assert = require('node:assert'); -const fs = require('node:fs/promises'); -const { existsSync } = require('node:fs'); -const os = require('node:os'); -const path = require('node:path'); -const zlib = require('node:zlib'); -const { Readable } = require('node:stream'); -const tarfs = require('tar-fs'); - -const testUtils = require('../testUtils.js'); -testUtils.preTestPrep(); - -const operations = require('#src/components/operations'); -const { DEPLOY_STAGING_DIR, DEPLOY_ACTIVATION_DIR, discardStagedApplication } = require('#src/components/Application'); -const { restartNeeded, resetRestartNeeded } = require('#src/components/requestRestart'); -const { server } = require('#src/server/Server'); -const { databases } = require('#src/resources/databases'); -const { SYSTEM_TABLE_NAMES, CONFIG_PARAMS } = require('#src/utility/hdbTerms'); -const { getConfigPath, readConfigFile } = require('#src/config/configUtils'); -const environment = require('#src/utility/environment/environmentManager'); -const { runWithOperationAuthorizationBypass } = require('#src/server/serverHelpers/operationAuthorizationState'); -const manageThreads = require('#src/server/threads/manageThreads'); - -const COMPONENTS_ROOT = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); -const DEPLOYMENT_TABLE = SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME; - -function packDirectory(directory) { - return new Promise((resolve, reject) => { - const chunks = []; - tarfs - .pack(directory) - .pipe(zlib.createGzip()) - .on('data', (chunk) => chunks.push(chunk)) - .on('end', () => resolve(Buffer.concat(chunks))) - .on('error', reject); - }); -} - -async function makePayload(marker, version = marker, withNodeModules = true) { - const source = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-op-')); - await fs.writeFile(path.join(source, 'package.json'), JSON.stringify({ name: 'phase-op', version })); - await fs.writeFile(path.join(source, 'index.js'), `module.exports = ${JSON.stringify(marker)};\n`); - if (withNodeModules) await fs.mkdir(path.join(source, 'node_modules'), { recursive: true }); - const payload = await packDirectory(source); - await fs.rm(source, { recursive: true, force: true }); - return payload; -} - -describe('deploy_component two-phase orchestration', function () { - this.timeout(30_000); - const rows = new Map(); - let priorTable; - let priorReplicate; - let priorSafeMode; - let sequence = 0; - const names = []; - - before(async () => { - priorSafeMode = process.env.HARPER_SAFE_MODE; - process.env.HARPER_SAFE_MODE = 'true'; - // The first component operation completes lazy server initialization, which replaces - // databases.system. Run it before installing the table seam used by these tests. - // - // One-shot on purpose: the separated phases now require deployment tracking, and the whole point - // of this warmup is that it runs BEFORE the table seam exists. `after` removes the component - // directory along with every other fixture name. - await operations.deployComponent({ - project: name(), - payload: await makePayload('warmup'), - two_phase: false, - restart: false, - }); - if (!databases.system) databases.system = {}; - priorTable = databases.system[DEPLOYMENT_TABLE]; - }); - - beforeEach(() => { - resetRestartNeeded(); - rows.clear(); - databases.system[DEPLOYMENT_TABLE] = { - async get(id) { - return rows.get(id); - }, - async put(row) { - rows.set(row.deployment_id, { ...row }); - }, - async patch(id, partial) { - const row = rows.get(id); - if (row) rows.set(id, { ...row, ...partial }); - }, - async *search(conditions = []) { - for (const row of rows.values()) { - if (conditions.every((condition) => row[condition.attribute] === condition.value)) yield row; - } - }, - }; - priorReplicate = server.replication.replicateOperation; - server.replication.replicateOperation = async () => ({ replicated: [] }); - }); - - afterEach(() => { - server.replication.replicateOperation = priorReplicate; - resetRestartNeeded(); - }); - - after(async () => { - if (priorSafeMode === undefined) delete process.env.HARPER_SAFE_MODE; - else process.env.HARPER_SAFE_MODE = priorSafeMode; - if (priorTable === undefined) delete databases.system[DEPLOYMENT_TABLE]; - else databases.system[DEPLOYMENT_TABLE] = priorTable; - for (const name of names) await fs.rm(path.join(COMPONENTS_ROOT, name), { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR), { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR), { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-aside'), { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-previous'), { recursive: true, force: true }); - }); - - function name() { - const value = `phase_op_${process.pid}_${sequence++}`; - names.push(value); - return value; - } - - it('refuses a separated-phase deploy when deployment tracking is unavailable', async () => { - // `DeploymentRecorder.put()` is tolerant by design, so without this guard `activate: false` returned - // a deployment_id that no row could resolve — a stage reporting success that nothing could ever - // activate. The separated phases coordinate through the row, so they have to require it. - const project = name(); - const payload = await makePayload('untracked'); - const priorTable = databases.system[DEPLOYMENT_TABLE]; - delete databases.system[DEPLOYMENT_TABLE]; - const unavailable = (error) => { - assert.strictEqual(error.statusCode, 503, `expected 503, got ${error.statusCode}: ${error.message}`); - assert.match(error.message, /coordinate through\n?.*hdb_deployment/s); - return true; - }; - try { - await assert.rejects(() => operations.deployComponent({ project, payload, activate: false }), unavailable); - await assert.rejects( - () => operations.deployComponent({ project, deployment_id: '00000000-0000-4000-8000-000000000000' }), - unavailable - ); - await assert.rejects(() => operations.deployComponent({ project, payload, two_phase: true }), unavailable); - } finally { - databases.system[DEPLOYMENT_TABLE] = priorTable; - } - }); - - it('replicates the payload itself when two_phase:false runs with no deployment table', async () => { - // The legacy single-phase path is the documented escape hatch, so it has to actually be safe. It - // normally strips `req.payload` because peers read the bytes from the row's payload_blob — but with - // no table there is no row, so stripping left peers holding a `_deploymentId`, no bytes and nothing - // to resolve, after this node was already live. Asserting on the REPLICATED REQUEST, not just on - // local activation: replication is stubbed here, so a local-only assertion proves nothing about peers. - const project = name(); - const priorTable = databases.system[DEPLOYMENT_TABLE]; - const priorReplicate = server.replication.replicateOperation; - const replicated = []; - server.replication.replicateOperation = async (op) => { - replicated.push(op); - return { replicated: [] }; - }; - delete databases.system[DEPLOYMENT_TABLE]; - try { - // A real Readable, not a reusable Buffer: ingest DRAINS the source, so a Buffer would hide the - // actual failure (peers receiving an exhausted stream / EOF). - const bytes = await makePayload('untracked-oneshot'); - await operations.deployComponent({ - project, - payload: Readable.from([bytes]), - two_phase: false, - restart: false, - }); - assert.match( - await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), - /untracked-oneshot/, - 'the component goes live locally' - ); - assert.strictEqual(replicated.length, 1, 'the deploy is replicated to peers'); - // Assert the BYTES, not merely that the property exists — presence of a spent stream is exactly - // the bug this covers. - const sent = replicated[0].payload; - assert.strictEqual(Buffer.isBuffer(sent), true, `peers must receive replayable bytes, got ${typeof sent}`); - assert.strictEqual(Buffer.compare(sent, bytes), 0, 'and the bytes must be the payload that was uploaded'); - } finally { - server.replication.replicateOperation = priorReplicate; - databases.system[DEPLOYMENT_TABLE] = priorTable; - } - }); - - it('settles superseded staged rows on a full deploy, not only on stage-and-stop', async () => { - // Staged DIRECTORIES are pruned on every stage, but the row half used to run only on the - // `activate: false` return. A full deploy therefore evicted trees while their rows still read - // `staged` cluster-wide — list_deployments offering a deployment_id that a later activate cannot - // use, and under clock skew a different one on each node. - const project = name(); - const priorMax = environment.get(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT); - environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, 1); - try { - const first = await operations.deployComponent({ - project, - payload: await makePayload('1.0.0'), - activate: false, - }); - assert.strictEqual(rows.get(first.deployment_id).status, 'staged'); - - // A full deploy of the same project: its own stage supersedes the one above. - await operations.deployComponent({ project, payload: await makePayload('2.0.0'), restart: false }); - - assert.strictEqual( - rows.get(first.deployment_id).status, - 'failed', - 'the superseded row is settled, so it no longer advertises an unusable deployment_id' - ); - } finally { - environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, priorMax); - } - }); - - it('normalizes string request booleans, including install_allow_scripts', async () => { - // Joi coerces these, but validateBySchema discards `result.value`, so the raw string reaches the - // handler. `install_allow_scripts: 'false'` would then read as truthy and run package lifecycle - // scripts for a caller that explicitly disabled them — over multipart/form, where every value - // arrives as a string. - const project = name(); - const staged = await operations.deployComponent({ - project, - payload: await makePayload('1.0.0'), - activate: 'false', - install_allow_scripts: 'false', - }); - - assert.strictEqual(staged.staged, true, "activate:'false' is honored as stage-only, not as a full deploy"); - assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, project)), false, 'so nothing goes live'); - assert.strictEqual( - rows.get(staged.deployment_id).activation_spec.install_allow_scripts, - false, - 'and the activation spec records a real boolean, not the string' - ); - }); - - it('stages without touching live and records an immutable activation specification', async () => { - const project = name(); - const result = await operations.deployComponent({ - project, - payload: await makePayload('1.0.0'), - activate: false, - }); - - assert.strictEqual(result.staged, true); - assert.match(result.deployment_id, /^[0-9a-f-]{36}$/i); - assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, project)), false); - assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id, project)), true); - const row = rows.get(result.deployment_id); - assert.ok(row, `deployment row missing; present ids: ${Array.from(rows.keys()).join(', ')}`); - assert.strictEqual(row.status, 'staged'); - assert.deepStrictEqual(row.activation_spec, { - project, - package: null, - install_command: null, - install_timeout: null, - install_allow_scripts: null, - urlPath: null, - host: null, - credentials: null, - force: false, - }); - }); - - it('uses a no-custody literal registry token for the origin install without recording it', async () => { - const project = name(); - const token = 'transient-origin-token'; - const installCommand = - `node -e "const fs=require('fs');` + - `const value=fs.readFileSync(process.env.npm_config_userconfig||process.env.NPM_CONFIG_USERCONFIG,'utf8');` + - `if(!value.includes('//registry.example.com/:_authToken='))process.exit(7);` + - `fs.writeFileSync('credential-seen','yes')"`; - const result = await operations.deployComponent({ - project, - payload: await makePayload('credential-origin', '6.0.0', false), - install_command: installCommand, - credentials: [{ registry: 'https://registry.example.com', token }], - activate: false, - }); - - const stagedPath = path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id, project); - assert.strictEqual(await fs.readFile(path.join(stagedPath, 'credential-seen'), 'utf8'), 'yes'); - assert.strictEqual(rows.get(result.deployment_id).activation_spec.credentials, null); - assert.doesNotMatch(JSON.stringify(rows.get(result.deployment_id)), new RegExp(token)); - }); - - it('activates only a staged row owned by the requested project', async () => { - const project = name(); - const staged = await operations.deployComponent({ - project, - payload: await makePayload('2.0.0'), - activate: false, - }); - - await assert.rejects( - operations.deployComponent({ project: `${project}_other`, deployment_id: staged.deployment_id }), - /not a staged deployment/ - ); - const activated = await operations.deployComponent({ project, deployment_id: staged.deployment_id }); - - assert.strictEqual(activated.activated, true); - assert.strictEqual(rows.get(staged.deployment_id).status, 'success'); - assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /2.0.0/); - }); - - it('does not let a duplicate activation undo the winning activation state', async () => { - const project = name(); - const staged = await operations.deployComponent({ - project, - payload: await makePayload('duplicate-winner', '6.0.0'), - activate: false, - }); - - const outcomes = await Promise.allSettled([ - operations.deployComponent({ project, deployment_id: staged.deployment_id }), - operations.deployComponent({ project, deployment_id: staged.deployment_id }), - ]); - - assert.strictEqual(outcomes.filter((outcome) => outcome.status === 'fulfilled').length, 1); - assert.strictEqual(outcomes.filter((outcome) => outcome.status === 'rejected').length, 1); - assert.strictEqual(rows.get(staged.deployment_id).status, 'success'); - assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /duplicate-winner/); - }); - - it('rejects fresh build or routing input on activate-by-id', async () => { - const project = name(); - const staged = await operations.deployComponent({ - project, - payload: await makePayload('3.0.0'), - activate: false, - }); - - await assert.rejects( - operations.deployComponent({ - project, - deployment_id: staged.deployment_id, - install_command: 'npm install --evil', - }), - /immutable staged configuration.*install_command/ - ); - }); - - it('stages and activates a full deploy before reporting success', async () => { - const project = name(); - const result = await operations.deployComponent({ - project, - payload: await makePayload('full-deploy', '6.0.0'), - }); - - assert.match(result.message, /Successfully deployed/); - assert.strictEqual(rows.get(result.deployment_id).status, 'success'); - assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /full-deploy/); - assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id)), false); - assert.strictEqual(restartNeeded(), true, 'a new component activated without restart requires one'); - }); - - // revert_component - - it('reverts the cluster to a named previous deployment and fans the target out to peers', async () => { - const project = name(); - const first = await operations.deployComponent({ project, payload: await makePayload('rev-v1', '1.0.0') }); - const second = await operations.deployComponent({ project, payload: await makePayload('rev-v2', '2.0.0') }); - const fanout = []; - server.replication.replicateOperation = async (op) => { - fanout.push(op); - return { replicated: [] }; - }; - - const result = await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); - - assert.strictEqual(result.reverted, true); - assert.strictEqual(result.to_deployment_id, first.deployment_id); - assert.strictEqual(result.from_deployment_id, second.deployment_id); - assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /rev-v1/); - assert.strictEqual(rows.get(result.deployment_id).status, 'rolled_back'); - assert.strictEqual( - rows.get(result.deployment_id).rollback_of, - second.deployment_id, - 'the audit row records which deployment the rollback took out of service' - ); - assert.strictEqual(fanout.length, 1, 'peers get the revert'); - assert.strictEqual(fanout[0].operation, 'revert_component'); - assert.strictEqual( - fanout[0].to_deployment_id, - first.deployment_id, - 'peers are told WHICH version to end on, so the fan-out is idempotent per node' - ); - }); - - it('is a no-op when the requested deployment is already live, so a retry is safe', async () => { - const project = name(); - const first = await operations.deployComponent({ project, payload: await makePayload('retry-v1', '1.0.0') }); - await operations.deployComponent({ project, payload: await makePayload('retry-v2', '2.0.0') }); - await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); - - // The caller lost the first response and retried the identical request. - const retry = await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); - - assert.strictEqual(retry.reverted, false); - assert.match(retry.message, /already running/); - assert.match( - await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), - /retry-v1/, - 'a retried revert must not toggle the rejected version back in' - ); - }); - - it('rejects a revert with no target, and one whose target is not retained', async () => { - const project = name(); - await operations.deployComponent({ project, payload: await makePayload('target-v1', '1.0.0') }); - await operations.deployComponent({ project, payload: await makePayload('target-v2', '2.0.0') }); - - await assert.rejects( - () => operations.revertComponent({ project }), - /to_deployment_id/, - 'the target is mandatory — that is what makes a retry safe' - ); - await assert.rejects( - () => operations.revertComponent({ project, to_deployment_id: '00000000-0000-4000-8000-000000000000' }), - /neither the live version/, - 'only the immediately-previous version is retained' - ); - assert.match( - await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), - /target-v2/, - 'a refused revert changes nothing' - ); - }); - - it('takes the package reference out of root config when reverting away from a package deploy', async () => { - // Without this, installApplications() would reinstall the reverted-away package over the restored - // directory on the next cold start and silently undo the rollback. - const project = name(); - const packaged = await operations.deployComponent({ - project, - payload: await makePayload('cfg-packaged', '1.0.0'), - }); - // Stamp root config as a package deploy would have, then activate a payload version over it. - const { addConfig } = require('#src/config/configUtils'); - await addConfig(project, { package: 'some-pkg@1.0.0' }); - const plain = await operations.deployComponent({ project, payload: await makePayload('cfg-plain', '2.0.0') }); - assert.ok(plain.deployment_id); - - await operations.revertComponent({ project, to_deployment_id: packaged.deployment_id }); - - const entry = readConfigFile()?.[project]; - assert.strictEqual( - entry?.package, - undefined, - 'the reverted-to version had no package reference, so the stale one must be gone' - ); - }); - - it('reclaims an oversized payload only after a full two-phase activation succeeds', async () => { - const project = name(); - const priorMaxSize = environment.get(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE); - environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, 1); - try { - const result = await operations.deployComponent({ - project, - payload: await makePayload('reclaimed-full-deploy', '6.0.0'), - }); - - const row = rows.get(result.deployment_id); - assert.strictEqual(row.status, 'success'); - assert.strictEqual(row.payload_blob, null); - assert.ok(row.event_log.some((event) => event.event === 'payload_dropped')); - } finally { - environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, priorMaxSize); - } - }); - - it('reclaims an oversized retained payload after activate-by-id succeeds', async () => { - const project = name(); - const staged = await operations.deployComponent({ - project, - payload: await makePayload('reclaimed-staged-deploy', '6.0.0'), - activate: false, - }); - assert.ok(rows.get(staged.deployment_id).payload_blob, 'staged deployment keeps its recovery payload'); - - const priorMaxSize = environment.get(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE); - environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, 1); - try { - await operations.deployComponent({ project, deployment_id: staged.deployment_id }); - - const row = rows.get(staged.deployment_id); - assert.strictEqual(row.status, 'success'); - assert.strictEqual(row.payload_blob, null); - assert.ok(row.event_log.some((event) => event.event === 'payload_dropped')); - } finally { - environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, priorMaxSize); - } - }); - - it('preserves the legacy one-shot path when explicitly requested', async () => { - const project = name(); - const result = await operations.deployComponent({ - project, - payload: await makePayload('one-shot', '6.0.0'), - two_phase: false, - }); - - assert.match(result.message, /Successfully deployed/); - assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /one-shot/); - }); - - it('accepts the legacy deployment row marker only from a trusted replicated operation', async () => { - const project = name(); - const payload = await makePayload('trusted-one-shot', '6.0.0'); - const result = await runWithOperationAuthorizationBypass(true, () => - operations.deployComponent({ - project, - payload, - two_phase: false, - _deploymentId: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', - replicated: false, - hdb_user: { name: 'cluster-peer' }, - }) - ); - - assert.match(result.message, /Successfully deployed/); - assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /trusted-one-shot/); - }); - - it('fails closed on the preview phase marker even from a trusted peer', async () => { - await assert.rejects( - runWithOperationAuthorizationBypass(true, () => - operations.deployComponent({ - project: name(), - _phase: 'stage', - replicated: false, - hdb_user: { name: 'cluster-peer' }, - }) - ), - /Unsupported legacy component deployment phase/ - ); - }); - - it('fails closed on an activate peer failure before scheduling a restart', async () => { - const project = name(); - const staged = await operations.deployComponent({ - project, - payload: await makePayload('4.0.0'), - activate: false, - }); - const phases = []; - server.replication.replicateOperation = async (operation) => { - phases.push(operation.phase); - if (operation.phase === 'activate') { - return { replicated: [{ node: 'peer-a', status: 'failed', reason: 'config write failed' }] }; - } - return { replicated: [] }; - }; - - await assert.rejects( - operations.deployComponent({ project, deployment_id: staged.deployment_id, restart: true }), - /Split nodes: peer-a.*[Rr]oll forward/s - ); - assert.deepStrictEqual(phases, ['activate'], 'restart phase was never sent after the activation gate failed'); - assert.strictEqual(rows.get(staged.deployment_id).status, 'activating'); - assert.strictEqual(rows.get(staged.deployment_id).completed_at, null); - assert.ok(rows.get(staged.deployment_id).payload_blob, 'payload remains available to repair a split cluster'); - assert.strictEqual(rows.get(staged.deployment_id).peer_results[0].node, 'peer-a'); - }); - - it('records success when restart fails after the activation barrier', async () => { - const project = name(); - const phases = []; - server.replication.replicateOperation = async (operation) => { - phases.push(operation.phase); - return operation.phase === 'restart' - ? { replicated: [{ node: 'peer-a', status: 'failed', reason: 'restart unavailable' }] } - : { replicated: [] }; - }; - const priorRestartWorkers = manageThreads.restartWorkers; - manageThreads.restartWorkers = () => {}; - let deploymentId; - try { - await assert.rejects( - operations - .deployComponent({ - project, - payload: await makePayload('activated-before-restart-failure', '6.0.0'), - restart: true, - }) - .catch((error) => { - deploymentId = error.http_resp_msg?.deployment_id; - throw error; - }), - /restart failed/ - ); - } finally { - manageThreads.restartWorkers = priorRestartWorkers; - } - - assert.deepStrictEqual(phases, ['stage', 'activate', 'restart']); - assert.strictEqual(rows.get(deploymentId).status, 'success'); - assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /activated-before/); - }); - - it('records peer failures but honors ignore_replication_errors', async () => { - const project = name(); - const staged = await operations.deployComponent({ - project, - payload: await makePayload('ignored-peer', '6.0.0'), - activate: false, - }); - server.replication.replicateOperation = async () => ({ - replicated: [{ node: 'peer-a', status: 'failed', reason: 'offline' }], - }); - - const result = await operations.deployComponent({ - project, - deployment_id: staged.deployment_id, - ignore_replication_errors: true, - }); - - assert.strictEqual(result.activated, true); - assert.strictEqual(rows.get(staged.deployment_id).status, 'success'); - assert.strictEqual(rows.get(staged.deployment_id).peer_results[0].status, 'failed'); - }); - - it('records and surfaces ignored restart failures after the activation gate', async () => { - const project = name(); - const staged = await operations.deployComponent({ - project, - payload: await makePayload('restart-failure', '6.0.0'), - activate: false, - }); - const phases = []; - server.replication.replicateOperation = async (operation) => { - phases.push(operation.phase); - return operation.phase === 'restart' - ? { replicated: [{ node: 'peer-a', status: 'failed', reason: 'restart unavailable' }] } - : { replicated: [] }; - }; - const priorRestartWorkers = manageThreads.restartWorkers; - let localRestarts = 0; - manageThreads.restartWorkers = () => localRestarts++; - let result; - try { - result = await operations.deployComponent({ - project, - deployment_id: staged.deployment_id, - restart: true, - ignore_replication_errors: true, - }); - } finally { - manageThreads.restartWorkers = priorRestartWorkers; - } - - assert.deepStrictEqual(phases, ['activate', 'restart']); - assert.strictEqual(localRestarts, 1); - assert.strictEqual(result.activated, true); - assert.strictEqual(result.failed_peers[0].node, 'peer-a'); - assert.strictEqual(rows.get(staged.deployment_id).peer_results[0].status, 'failed'); - }); - - it('uses the row-backed immutable specification for trusted peer phases', async () => { - const project = name(); - const staged = await operations.deployComponent({ - project, - payload: await makePayload('peer-phase', '6.0.0'), - activate: false, - }); - const row = rows.get(staged.deployment_id); - const componentPath = path.join(COMPONENTS_ROOT, project); - await discardStagedApplication(componentPath, staged.deployment_id); - const executePeerPhase = (phase, activationSpec) => - runWithOperationAuthorizationBypass(true, () => - operations.componentDeployPhase({ - operation: 'component_deploy_phase', - phase, - project, - deployment_id: staged.deployment_id, - activation_spec: activationSpec, - replicated: false, - hdb_user: { name: 'cluster-peer' }, - }) - ); - - await assert.rejects( - executePeerPhase('stage', { ...row.activation_spec, host: 'tampered.example' }), - /immutable activation specification/ - ); - await executePeerPhase('stage', row.activation_spec); - assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, staged.deployment_id, project)), true); - await executePeerPhase('activate', row.activation_spec); - assert.match(await fs.readFile(path.join(componentPath, 'index.js'), 'utf8'), /peer-phase/); - assert.strictEqual( - rows.get(staged.deployment_id).status, - 'staged', - 'a peer activates locally without advancing the replicated row — the origin owns it' - ); - assert.strictEqual(restartNeeded(), true); - }); - - it('rebuilds a missing peer stage from the durable deployment payload before activation', async () => { - const project = name(); - const staged = await operations.deployComponent({ - project, - payload: await makePayload('rebuilt-peer', '6.0.0'), - activate: false, - }); - const row = rows.get(staged.deployment_id); - const componentPath = path.join(COMPONENTS_ROOT, project); - await discardStagedApplication(componentPath, staged.deployment_id); - - await runWithOperationAuthorizationBypass(true, () => - operations.componentDeployPhase({ - operation: 'component_deploy_phase', - phase: 'activate', - project, - deployment_id: staged.deployment_id, - activation_spec: row.activation_spec, - replicated: false, - hdb_user: { name: 'cluster-peer' }, - }) - ); - - assert.match(await fs.readFile(path.join(componentPath, 'index.js'), 'utf8'), /rebuilt-peer/); - assert.strictEqual( - rows.get(staged.deployment_id).status, - 'staged', - 'a peer activates locally without advancing the replicated row — the origin owns it' - ); - }); - - it('waits for the staged row checkpoint when peer activation arrives first', async () => { - const project = name(); - const staged = await operations.deployComponent({ - project, - payload: await makePayload('lagged-row', '6.0.0'), - activate: false, - }); - const row = rows.get(staged.deployment_id); - rows.set(staged.deployment_id, { ...row, status: 'staging' }); - setImmediate(() => rows.set(staged.deployment_id, { ...rows.get(staged.deployment_id), status: 'staged' })); - - await runWithOperationAuthorizationBypass(true, () => - operations.componentDeployPhase({ - operation: 'component_deploy_phase', - phase: 'activate', - project, - deployment_id: staged.deployment_id, - activation_spec: row.activation_spec, - deployment_timeout: 200, - replicated: false, - hdb_user: { name: 'cluster-peer' }, - }) - ); - - assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /lagged-row/); - assert.strictEqual( - rows.get(staged.deployment_id).status, - 'staged', - 'a peer activates locally without advancing the replicated row — the origin owns it' - ); - }); - - it('recovers a staged package specification for config and peer activation', async () => { - const project = name(); - const tarDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-package-')); - const tarPath = path.join(tarDirectory, 'component.tgz'); - await fs.writeFile(tarPath, await makePayload('package-stage', '6.0.0')); - const packageIdentifier = `file:${tarPath}`; - const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-config-')); - const configPath = path.join(configRoot, 'harper-config.yaml'); - await fs.writeFile(configPath, `rootPath: ${JSON.stringify(configRoot)}\n`); - const priorRootEnv = process.env.ROOTPATH; - const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); - process.env.ROOTPATH = configRoot; - environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); - try { - const staged = await operations.deployComponent({ - project, - package: packageIdentifier, - activate: false, - }); - let activationOperation; - server.replication.replicateOperation = async (operation) => { - activationOperation = operation; - return { replicated: [] }; - }; - - await operations.deployComponent({ project, deployment_id: staged.deployment_id }); - - assert.strictEqual(readConfigFile()[project].package, packageIdentifier); - assert.strictEqual(activationOperation.operation, 'component_deploy_phase'); - assert.strictEqual(activationOperation.activation_spec.package, packageIdentifier); - } finally { - if (priorRootEnv === undefined) delete process.env.ROOTPATH; - else process.env.ROOTPATH = priorRootEnv; - environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); - await fs.rm(configRoot, { recursive: true, force: true }); - await fs.rm(tarDirectory, { recursive: true, force: true }); - } - }); - - it('removes the root-config entry before it destroys anything, so a failed drop cannot resurrect', async () => { - // The resurrection vector is the root-config entry: installApplications() reinstalls whatever it - // names. Doing the persistent writes after the directory removal left a crash window where the - // live tree was gone but config still named the package, and the next boot brought the component - // back. Config-first inverts the failure: an interrupted drop is unfinished and re-runnable. - const project = name(); - const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-drop-crash-')); - const componentsRoot = path.join(configRoot, 'components'); - const configPath = path.join(configRoot, 'harper-config.yaml'); - const priorRootEnv = process.env.ROOTPATH; - const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); - const priorComponentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); - process.env.ROOTPATH = configRoot; - environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); - environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, componentsRoot); - await fs.writeFile(configPath, `rootPath: ${JSON.stringify(configRoot)}\n`); - try { - await fs.mkdir(path.join(componentsRoot, project), { recursive: true }); - await fs.writeFile(path.join(componentsRoot, project, 'index.js'), "module.exports = 'live';\n"); - await fs.appendFile(configPath, `${project}:\n package: some-package@1.0.0\n`); - // Make every aside-based teardown step fail: `.deploy-aside` occupied by a file is rejected as - // "not a directory", which is the closest deterministic stand-in for dying mid-teardown. - await fs.writeFile(path.join(componentsRoot, '.deploy-aside'), 'not a directory\n'); - - await assert.rejects(() => operations.dropComponent({ project })); - - assert.strictEqual( - (await fs.readFile(configPath, 'utf8')).includes(project), - false, - 'config is already clean, so the next boot cannot reinstall the component' - ); - assert.strictEqual( - existsSync(path.join(componentsRoot, project)), - true, - 'and the tree is still there: the drop is unfinished rather than finished-then-undone' - ); - } finally { - if (priorRootEnv === undefined) delete process.env.ROOTPATH; - else process.env.ROOTPATH = priorRootEnv; - environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); - environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, priorComponentsRoot); - await fs.rm(configRoot, { recursive: true, force: true }); - } - }); - - it('drop_component invalidates staged rows and removes recovery artifacts', async () => { - const project = name(); - const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-drop-')); - const componentsRoot = path.join(configRoot, 'components'); - const configPath = path.join(configRoot, 'harper-config.yaml'); - const priorRootEnv = process.env.ROOTPATH; - const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); - const priorComponentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); - process.env.ROOTPATH = configRoot; - environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); - environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, componentsRoot); - await fs.writeFile(configPath, `rootPath: ${JSON.stringify(configRoot)}\n`); - try { - const staged = await operations.deployComponent({ - project, - payload: await makePayload('drop-stage', '6.0.0'), - activate: false, - }); - const activationPath = path.join(componentsRoot, DEPLOY_ACTIVATION_DIR, project, 'interrupted'); - await fs.mkdir(activationPath, { recursive: true }); - await fs.writeFile( - path.join(configRoot, 'harper-application-lock.json'), - JSON.stringify({ applications: { [project]: { package: 'stale-package' } } }) - ); - // A root-config entry is what makes a dropped component come back: installApplications() reads - // it on the next boot and reinstalls the package. Removing it and the application-lock entry as - // two separate writes meant a crash or a failed second write left this behind. - await fs.appendFile(configPath, `${project}:\n package: stale-package\n`); - - await operations.dropComponent({ project }); - - assert.strictEqual(rows.get(staged.deployment_id).status, 'failed'); - const deploymentStagePath = path.join(componentsRoot, DEPLOY_STAGING_DIR, staged.deployment_id); - assert.strictEqual( - existsSync(deploymentStagePath), - false, - `staged deployment directory still contains: ${await fs.readdir(deploymentStagePath).catch(() => [])}` - ); - assert.strictEqual(existsSync(path.join(componentsRoot, DEPLOY_ACTIVATION_DIR, project)), false); - const applicationLock = JSON.parse(await fs.readFile(path.join(configRoot, 'harper-application-lock.json'))); - assert.strictEqual(applicationLock.applications[project], undefined); - assert.strictEqual( - (await fs.readFile(configPath, 'utf8')).includes(project), - false, - 'the root-config entry is removed in the same step, so the next boot cannot reinstall the drop' - ); - } finally { - if (priorRootEnv === undefined) delete process.env.ROOTPATH; - else process.env.ROOTPATH = priorRootEnv; - environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); - environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, priorComponentsRoot); - await fs.rm(configRoot, { recursive: true, force: true }); - } - }); - - it('rejects separated-phase controls on the one-shot fallback', async () => { - await assert.rejects( - operations.deployComponent({ - project: name(), - payload: await makePayload('5.0.0'), - activate: false, - two_phase: false, - }), - /require two-phase deploy/ - ); - }); - - it('rejects an explicit two-phase request when the system database is not replicated', async () => { - const priorReplications = environment.get(CONFIG_PARAMS.REPLICATION_DATABASES); - environment.setProperty(CONFIG_PARAMS.REPLICATION_DATABASES, ['data']); - try { - await assert.rejects( - operations.deployComponent({ - project: name(), - payload: await makePayload('requires-system-replication'), - two_phase: true, - }), - /requires system database replication/ - ); - } finally { - environment.setProperty(CONFIG_PARAMS.REPLICATION_DATABASES, priorReplications); - } - }); - - it('does not trust caller-supplied internal phase markers', async () => { - await assert.rejects( - operations.deployComponent({ - project: name(), - payload: await makePayload('untrusted-replication'), - replicated: false, - two_phase: true, - }), - /requires operation replication/ - ); - await assert.rejects( - operations.deployComponent({ - project: name(), - _deploymentId: '../../escape', - _phase: 'stage', - }), - /is not allowed/ - ); - await assert.rejects( - operations.componentDeployPhase({ - operation: 'component_deploy_phase', - phase: 'discard', - project: name(), - deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', - activation_spec: { project: 'anything' }, - }), - /restricted to authenticated cluster peers/ - ); - await assert.rejects( - runWithOperationAuthorizationBypass(true, () => - operations.componentDeployPhase({ - operation: 'component_deploy_phase', - phase: 'discard', - project: '../escape', - deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', - activation_spec: { project: '../escape' }, - replicated: false, - hdb_user: { name: 'cluster-peer' }, - }) - ), - /project name/i - ); - }); -}); diff --git a/unitTests/components/deployPhaseValidators.test.js b/unitTests/components/deployPhaseValidators.test.js deleted file mode 100644 index d0ac9e62ce..0000000000 --- a/unitTests/components/deployPhaseValidators.test.js +++ /dev/null @@ -1,58 +0,0 @@ -'use strict'; - -const assert = require('node:assert'); -const validator = require('#js/components/operationsValidation'); - -const valid = (result) => assert.strictEqual(result, undefined, `expected valid, got: ${result?.message}`); -const invalid = (result) => assert.ok(result, 'expected a validation error'); - -describe('deployComponentValidator two-phase controls', () => { - it('accepts stage-and-stop and UUID activation', () => { - valid(validator.deployComponentValidator({ project: 'my_app', activate: false })); - valid( - validator.deployComponentValidator({ - project: 'my_app', - deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', - }) - ); - }); - - it('requires deployment_id to be a UUID and rejects path traversal', () => { - for (const deploymentId of ['abc-123', '../evil', 'dep/../..', '.', '..']) { - invalid(validator.deployComponentValidator({ project: 'my_app', deployment_id: deploymentId })); - } - }); - - it('rejects caller-controlled internal phase markers', () => { - invalid(validator.deployComponentValidator({ project: 'my_app', _deploymentId: 'x' })); - invalid(validator.deployComponentValidator({ project: 'my_app', _phase: 'stage' })); - }); - - it('rejects retry-unsafe automatic rollback', () => { - invalid(validator.deployComponentValidator({ project: 'my_app', revert_on_failure: true })); - }); - - it('preserves routing validation', () => { - invalid(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', urlPath: '/a/./b' })); - invalid(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', urlPath: '/a/../b' })); - }); -}); - -describe('componentDeployPhaseValidator', () => { - const validPhase = { - phase: 'stage', - deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', - project: 'my_app', - activation_spec: { project: 'my_app' }, - }; - - it('accepts a bounded internal phase request', () => { - valid(validator.componentDeployPhaseValidator(validPhase)); - }); - - it('rejects invalid phases, project traversal, and non-UUID ids', () => { - invalid(validator.componentDeployPhaseValidator({ ...validPhase, phase: 'deploy' })); - invalid(validator.componentDeployPhaseValidator({ ...validPhase, project: '../escape' })); - invalid(validator.componentDeployPhaseValidator({ ...validPhase, deployment_id: '../../escape' })); - }); -}); diff --git a/unitTests/components/deployValidators.test.js b/unitTests/components/deployValidators.test.js new file mode 100644 index 0000000000..cf1c19a163 --- /dev/null +++ b/unitTests/components/deployValidators.test.js @@ -0,0 +1,21 @@ +'use strict'; + +const assert = require('node:assert'); +const validator = require('#js/components/operationsValidation'); + +const invalid = (result) => assert.ok(result, 'expected a validation error'); + +describe('deployComponentValidator', () => { + it('rejects retry-unsafe automatic rollback', () => { + invalid(validator.deployComponentValidator({ project: 'my_app', revert_on_failure: true })); + }); + + it('rejects the caller-supplied internal deployment marker', () => { + invalid(validator.deployComponentValidator({ project: 'my_app', _deploymentId: 'x' })); + }); + + it('preserves routing validation', () => { + invalid(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', urlPath: '/a/./b' })); + invalid(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', urlPath: '/a/../b' })); + }); +}); diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index 097396c81b..e83587715b 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -323,10 +323,7 @@ export const OPERATIONS_ENUM = { PACKAGE_CUSTOM_FUNCTION_PROJECT: 'package_custom_function_project', DEPLOY_CUSTOM_FUNCTION_PROJECT: 'deploy_custom_function_project', PACKAGE_COMPONENT: 'package_component', - // Peer phases get their own operation so an older node fails closed instead of reading an unknown - // phase field as a one-shot deploy. DEPLOY_COMPONENT: 'deploy_component', - COMPONENT_DEPLOY_PHASE: 'component_deploy_phase', // A public operation rather than a deploy phase: it fetches, resolves and installs nothing. REVERT_COMPONENT: 'revert_component', READ_TRANSACTION_LOG: 'read_transaction_log', diff --git a/utility/operation_authorization.ts b/utility/operation_authorization.ts index c3fa85352e..538030ffcf 100644 --- a/utility/operation_authorization.ts +++ b/utility/operation_authorization.ts @@ -313,10 +313,6 @@ requiredPermissions.set(functionsOperations.addComponent.name, new (permission a requiredPermissions.set(functionsOperations.dropCustomFunctionProject.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.packageComponent.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.deployComponent.name, new (permission as any)(true, [])); -requiredPermissions.set( - functionsOperations.componentDeployPhase.name, - new (permission as any)(true, [], terms.OPERATIONS_ENUM.COMPONENT_DEPLOY_PHASE) -); requiredPermissions.set( functionsOperations.revertComponent.name, new (permission as any)(true, [], terms.OPERATIONS_ENUM.REVERT_COMPONENT) From ee2234a224b051e1101c1b9efeda64e72a46f719 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 24 Aug 2026 16:55:53 -0400 Subject: [PATCH 90/94] fix(deploy): clean up what the protocol split orphaned MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review of the shrunk PR found the leftovers, which is what a review of a large deletion is for. The handler still branched on `activate`, `two_phase` and `deployment_id` after the validator dropped them — and the schema allows unknown keys, so a caller still sending the never-released staged contract would have had `activate: false` silently ignored and received a full deploy instead. Those branches are gone and the fields are now `forbidden()`, naming #2301, so they fail fast rather than doing the opposite of what was asked. The first test to hit that was one of ours that had been passing `activate: false` as a no-op. `applicationConfigFromActivationSpec` dereferenced `spec.package` without a null guard, and startup reconciliation feeds it `row.activation_spec` from rows it did not write — a row from before the spec was recorded, or a hand-edited one. That threw during boot and left the component failed closed on every restart with no recovery but repairing the row by hand. A missing spec now degrades to a config no-op. `drop_component` settled only `staged`/`activating` rows, but a deploy interrupted mid-stage rests at `staging`, and nothing else settles it: payload retention only reclaims terminal rows, so the tarball stayed pinned and `get_deployment` never converged for an already-dropped component. Docs and comments: I had deleted the caveat that the pre-swap load check is a no-op on the main thread — which is where the operations API runs deploys — so DESIGN.md was claiming a guarantee the code does not provide on the origin. It is restored and scoped explicitly. Also removed the activate-by-id consequence paragraph, and corrected the comments still describing two-phase orchestration, the `component_deploy_phase` fan-out, and `harper activate`. `claimStagedDeployment` and `settleStagedRows` have no caller in this tree now. They are marked RESERVED FOR #2301 rather than deleted: that PR is stacked directly on this one and is their only consumer, so deleting here would just mean re-adding there. The comments say plainly that no resting `staged` row producer exists in-tree until it lands. Reported by cursor-composer pre-push. --- DESIGN.md | 20 +++++++---- bin/cliOperations.ts | 2 +- components/Application.ts | 14 +++++--- components/deploymentRecorder.ts | 21 ++++++++--- components/operations.js | 35 +++---------------- components/operationsValidation.js | 13 +++++++ unitTests/components/deployOperations.test.js | 2 +- unitTests/components/deployValidators.test.js | 13 +++++++ 8 files changed, 72 insertions(+), 48 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index b271656fc6..aa25ea0285 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -505,8 +505,14 @@ other future cause would still report success silently; that's a deferred, separ ## Staged deploy: build aside, then swap (`components/Application.ts`, `components/operations.js`) `deploy_component` builds the incoming version — download/`npm pack` (incl. a git clone), extract, -`npm install` — into a hidden staging directory, validates that it loads, then atomically renames it -into the live component path. The live component keeps serving throughout, and a fetch or install +`npm install` — into a hidden staging directory, then atomically renames it into the live component +path. + +**The pre-swap load check does not run everywhere, and that is not new.** `loadValidateComponent` +returns immediately on the main thread, and the operations API executes `deploy_component` there — so +on a node whose deploy runs on the main thread, a candidate that installs cleanly but throws at load +still reaches the live path. It runs where the deploy executes on a worker (e.g. the op-API worker). +The staged build bounds the _install_, not the load. The live component keeps serving throughout, and a fetch or install failure leaves it untouched rather than half-replaced in place. The request/response contract is unchanged; only the SSE phase names differ (`stage`/`activate` vs the old `prepare`/`replicate`). @@ -576,11 +582,11 @@ stage it evicts the oldest not-yet-activated staged builds for that component be `deployment_stagingRetention_maxCount` (default 5, `pruneStagedBuilds`), always keeping the just-staged one and the newest N−1 by mtime. Eviction is best-effort (`allSettled`, trace-logged) but awaited so the count is settled when the stage returns. Retention is deliberately count-only and automatic: -per the harper#1849 discussion, `hdb_deployment` rows stay as the audit trail (payload blobs already -self-reclaim by size, `deployment_payloadRetention_maxSize`), and no `delete_deployment` op was added — -eviction-on-stage keeps the surface at zero new operations. Consequence: activating a `deployment_id` -that has already aged out of the window fails with "no staged build found" — expected once more than -`maxCount` newer stages have landed for that component. +`hdb_deployment` rows stay as the audit trail (payload blobs already self-reclaim by size, +`deployment_payloadRetention_maxSize`), and no `delete_deployment` op was added — eviction-on-stage +keeps the surface at zero new operations. Nothing addresses a staged directory by id any more, so +eviction has no user-visible consequence beyond reclaiming disk; activating a previously staged +deployment moves to #2301 with the rest of the coordination protocol. **Payload retention.** Two independent bounds apply to the tarballs stored in `hdb_deployment`'s `payload_blob`, and they answer different questions: diff --git a/bin/cliOperations.ts b/bin/cliOperations.ts index 97eaa0e5b4..1b15653d86 100644 --- a/bin/cliOperations.ts +++ b/bin/cliOperations.ts @@ -906,7 +906,7 @@ export async function resolveRequestOptions(req: any): Promise<{ options: any; t async function cliOperations(req: any, skipResponseLog = false) { require('dotenv').config(); - // Enforce CLI-verb requirements (e.g. `harper activate` needs a deployment_id) before connecting or + // Enforce CLI-verb requirements (e.g. `harper revert` needs a to_deployment_id) before connecting or // packaging, so a mistake fails fast instead of building + uploading a fresh deploy from the CWD. const verbError = verbRequirementError(req); if (verbError) { diff --git a/components/Application.ts b/components/Application.ts index 9cb7da8ca7..8edb496a8e 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -532,7 +532,7 @@ const MAX_INSTALL_COMMANDS = 2; // - Same filesystem as the live path, so the go-live rename() is atomic. An os.tmpdir() // location is frequently a different mount (tmpfs / separate volume); a cross-device // rename throws EXDEV and degrades to a slow recursive copy — reintroducing the very -// downtime window the two-phase split exists to remove. +// downtime window the staged build exists to remove. // - The leading dot keeps loadComponentDirectories (componentLoader) from loading it as a // phantom component, and it is not the watched base of any component's file watcher // (those are rooted at each live component dir), so building here triggers no @@ -555,7 +555,7 @@ export const DEPLOY_PREVIOUS_DIR = '.deploy-previous'; // Max not-yet-activated staged builds kept per component before the oldest are evicted on the next // stage. A full deploy consumes its staged build immediately (activate renames it live), so this only -// bounds `activate: false` stage-and-stops that are never activated. Configurable via +// bounds the residue of stages that never reached activation. Configurable via // deployment_stagingRetention_maxCount. export const DEFAULT_STAGING_RETENTION_MAX_COUNT = 5; @@ -3813,8 +3813,14 @@ async function persistApplicationLock( await next; } -function applicationConfigFromActivationSpec(spec: Record): ApplicationConfig | undefined { - if (!spec.package) return undefined; +function applicationConfigFromActivationSpec( + spec: Record | null | undefined +): ApplicationConfig | undefined { + // A row with no activation spec says nothing about config, so config is a no-op rather than a throw. + // Recovery reads `row.activation_spec` from rows it did not write — a row from before the spec was + // recorded, or a hand-edited one — and throwing there would fail that component closed on every boot + // with no way out but repairing the row by hand. + if (!spec?.package) return undefined; const applicationConfig: ApplicationConfig = { package: spec.package }; if (spec.install_command !== null || spec.install_timeout !== null || spec.install_allow_scripts !== null) { applicationConfig.install = {}; diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index 7f96bd5d72..0166a51159 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -51,7 +51,8 @@ type DeploymentStatus = | 'extracting' | 'installing' | 'staging' - // A terminal resting state, not a transient one: where a stage-and-stop deploy comes to rest. + // Reserved for the coordination protocol (#2301), which is the only producer of a resting `staged` + // row. Nothing in this tree leaves a deployment there. | 'staged' | 'loading' | 'replicating' @@ -572,7 +573,7 @@ export async function getDeploymentRow(deploymentId: string): Promise { const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; if (!table) return []; @@ -726,7 +736,10 @@ export async function invalidateProjectStagedDeployments(project: string): Promi if (!table) return []; const invalidated: string[] = []; for await (const row of table.search([{ attribute: 'project', value: project }])) { - if (!['staged', 'activating'].includes(row?.status)) continue; + // `staging` counts too: a deploy interrupted mid-stage leaves the row there, and nothing else + // settles it — payload retention only reclaims terminal rows, so the tarball would be pinned and + // `get_deployment` would never converge for a component that has since been dropped. + if (!['staging', 'staged', 'activating'].includes(row?.status)) continue; await table.patch(row.deployment_id, { status: 'failed', completed_at: Date.now(), diff --git a/components/operations.js b/components/operations.js index 15074ea3c6..487241677a 100644 --- a/components/operations.js +++ b/components/operations.js @@ -473,7 +473,7 @@ async function packageComponent(req) { /** * Deploy a component. Front door for the deploy family: derives the project name, validates, ingests * any credential token into the secrets store (so it lives as a replicated reference, not embedded), - * then dispatches to the two-phase orchestrator (default) or the legacy one-shot path. + * then stages the build and swaps it in. * * `two_phase: false` forces the one-shot path. See DESIGN.md for the stage/activate protocol. * @@ -503,33 +503,6 @@ async function deployComponent(req) { if (validation) { throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); } - const systemReplicated = isSystemDatabaseReplicated(); - const requestedSeparatedPhase = req.activate === false || req.deployment_id !== undefined; - if (!isReplicatedExecution && req.two_phase === true && req.replicated === false) { - throw handleHDBError( - new Error(), - `two_phase:true requires operation replication to be enabled`, - HTTP_STATUS_CODES.BAD_REQUEST - ); - } - if (!isReplicatedExecution && req.two_phase === true && !systemReplicated) { - throw handleHDBError( - new Error(), - `two_phase:true requires system database replication to be enabled`, - HTTP_STATUS_CODES.BAD_REQUEST - ); - } - if ( - !isReplicatedExecution && - requestedSeparatedPhase && - (req.two_phase === false || req.replicated === false || !systemReplicated) - ) { - throw handleHDBError( - new Error(), - `activate:false and deployment_id require two-phase deploy with system database replication enabled`, - HTTP_STATUS_CODES.BAD_REQUEST - ); - } // Ingest any provided credential token into the secrets store so the credential lives as // replicated ciphertext (reference, not embed); already-reference entries pass through, and with // no custody a literal token stays as a transient, this-node-only fallback (#1158). Peers @@ -558,7 +531,7 @@ async function deployComponent(req) { * * Runs per node: each node checks its own directory state, which can differ across the cluster (a * component can be new on a peer that never had it and a redeploy on the origin). The one-shot path has - * extractApplication/prepareApplication set both flags in place; the two-phase path has + * extractApplication/prepareApplication set both flags in place; the staged path has * activateStagedApplication set them at swap time, comparing the pre-swap live tree against the staged * one (staging is always fresh, so extraction never sees the live dir). */ @@ -979,7 +952,7 @@ async function restartRevertedComponent(req, emit) { return { restartMessage: '' }; } -// Shared deploy-family helpers (used by deploy_component, its component_deploy_phase fan-out, and +// Shared deploy-family helpers (used by deploy_component and // revert_component). // Reject deploying over a protected core component name unless force is set. Lazy-loads @@ -1065,7 +1038,7 @@ function buildDeployApplication({ } // Load a component directory to surface load-time errors early (throwaway scopes). No-op on the main -// thread or in safe mode. In two-phase this loads the STAGED directory before go-live; in one-shot it +// thread or in safe mode. It loads the STAGED directory, before go-live, where it runs at all — see // loads the live directory after in-place prepare. async function loadValidateComponent({ dirPath, emit }) { if (isMainThread || process.env.HARPER_SAFE_MODE) return; diff --git a/components/operationsValidation.js b/components/operationsValidation.js index a37a36d93f..4516d267db 100644 --- a/components/operationsValidation.js +++ b/components/operationsValidation.js @@ -532,6 +532,19 @@ function deployComponentValidator(req) { 'any.unknown': `'revert_on_failure' is not supported; roll back explicitly with revert_component`, }), _deploymentId: Joi.any().forbidden(), + // Removed with the coordination protocol (see #2301). Forbidden rather than dropped, because the + // schema allows unknown keys: a caller still sending the unreleased contract would otherwise have + // `activate: false` silently ignored and get a full deploy — the opposite of the intent. + activate: Joi.any().forbidden().messages({ + 'any.unknown': `'activate' is not supported; staged deploys are tracked in HarperFast/harper#2301`, + }), + two_phase: Joi.any().forbidden().messages({ + 'any.unknown': `'two_phase' is not supported; every deploy now stages and swaps on each node`, + }), + deployment_id: Joi.any().forbidden().messages({ + 'any.unknown': `'deployment_id' is not supported; activating a previously staged deployment is tracked in HarperFast/harper#2301`, + }), + _phase: Joi.any().forbidden(), urlPath: URL_PATH_SCHEMA, host: HOST_SCHEMA, // Deploy credentials. Each entry is npm registry auth (`registry`) or git host auth (`host`, diff --git a/unitTests/components/deployOperations.test.js b/unitTests/components/deployOperations.test.js index b5ecce4710..0c854bf331 100644 --- a/unitTests/components/deployOperations.test.js +++ b/unitTests/components/deployOperations.test.js @@ -418,7 +418,7 @@ describe('deploy_component staged deploy', function () { const staged = await operations.deployComponent({ project, payload: await makePayload('drop-stage', '6.0.0'), - activate: false, + restart: false, }); const activationPath = path.join(componentsRoot, DEPLOY_ACTIVATION_DIR, project, 'interrupted'); await fs.mkdir(activationPath, { recursive: true }); diff --git a/unitTests/components/deployValidators.test.js b/unitTests/components/deployValidators.test.js index cf1c19a163..dd0f426950 100644 --- a/unitTests/components/deployValidators.test.js +++ b/unitTests/components/deployValidators.test.js @@ -10,6 +10,19 @@ describe('deployComponentValidator', () => { invalid(validator.deployComponentValidator({ project: 'my_app', revert_on_failure: true })); }); + it('rejects the removed staged-deploy contract fields instead of ignoring them', () => { + // The schema allows unknown keys, so dropping these from it would have left a caller still sending + // the (never-released) staged contract silently getting a full deploy — `activate: false` ignored, + // the opposite of the intent. They fail fast and name the follow-on instead. + invalid(validator.deployComponentValidator({ project: 'my_app', activate: false })); + invalid(validator.deployComponentValidator({ project: 'my_app', two_phase: true })); + invalid(validator.deployComponentValidator({ project: 'my_app', two_phase: false })); + invalid( + validator.deployComponentValidator({ project: 'my_app', deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa' }) + ); + invalid(validator.deployComponentValidator({ project: 'my_app', _phase: 'stage' })); + }); + it('rejects the caller-supplied internal deployment marker', () => { invalid(validator.deployComponentValidator({ project: 'my_app', _deploymentId: 'x' })); }); From 29c86f612aee98e9479840d71f16d796857c2b6c Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 24 Aug 2026 17:03:57 -0400 Subject: [PATCH 91/94] fix(deploy): trust local activation evidence over the row, and stop validations sharing a global MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two majors from the graded review of the shrunk PR. **Startup destroyed activation evidence on the strength of the row.** The status filter ran before the local evidence check, and the row is the less reliable of the two: a crash between the swap and the status/config commit can leave it `loading`, already terminal from a replicated origin write, or absent entirely when tracking is unavailable — while the candidate is live on disk. Reconciliation then removed the staging parent and moved on, and the artifact sweep neither persisted config nor failed the component closed, so swapped-in code loaded under the previous release's configuration. Activation artifacts are now read under the component lock BEFORE any status-based cleanup, and their presence forces the roll-forward path for any row status. The split made this materially more likely, not less: every deploy here ends as `success`, so "not staged or activating" is the common startup case rather than an edge one. **Concurrent validations shared a process-global error reporter.** `setErrorReporter` is module state, so two deploys validating on the same worker cross-attribute failures: B installs its reporter while A is loading, A's load error lands in B — A then activates broken bytes while B rejects a good candidate. Validation is serialized (already the slow path) and the previous reporter is restored in a `finally`, so the global is only ever owned by one in-flight validation. `componentLoader` grew a matching getter so the restore is possible at all. Also from the same review: the protected-core-name rejection now runs before credential ingestion and the durable row are created, rather than after — a deploy that was never allowed should not leave a secret in the store and a row behind. Reported by codex pre-push. --- components/Application.ts | 21 +++++++++++++++++-- components/componentLoader.ts | 4 ++++ components/operations.js | 39 ++++++++++++++++++++++++++++------- 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 8edb496a8e..d7d218f793 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -3448,12 +3448,29 @@ export async function reconcileStagedApplicationArtifacts( continue; } const componentDirPath = join(componentsRootDirPath, row.project); - if (!['staged', 'activating'].includes(row.status)) { + // Local activation evidence is consulted BEFORE any status-based cleanup, because the row is the + // less reliable of the two. A crash between the swap and the status/config commit can leave the + // row `loading`, already terminal (a replicated origin write), or absent entirely when tracking + // is unavailable — while the candidate is live on disk. Deleting staging state on the strength of + // that status destroys the only evidence, and the artifact sweep then neither persists config nor + // fails the component closed: swapped-in code loads under the previous release's configuration. + // Read under the lock so an in-flight activation cannot create artifacts between probe and act. + await withComponentPreparationLock(componentDirPath, async () => { + activationBegan = (await activationArtifacts(componentDirPath, entry.name)).length > 0; + }); + if (!activationBegan && !['staged', 'activating'].includes(row.status)) { let shouldRemove = false; await withComponentPreparationLock(componentDirPath, async () => { row = await getDeployment(entry.name); shouldRemove = !row || !safeComponentName(row.project) || !['staged', 'activating'].includes(row.status); if (!shouldRemove) return; + // Re-read the evidence under this lock too: an activation may have started since the probe + // above, and its artifacts outrank the row. + if ((await activationArtifacts(componentDirPath, entry.name)).length > 0) { + activationBegan = true; + shouldRemove = false; + return; + } // A row still `pending`/`staging` once its staging directory is going away cannot make // progress — the process that owned it is gone. Payload retention only reclaims rows that // reached a terminal status, so leaving it in flight pins its tarball on every node forever. @@ -3474,7 +3491,6 @@ export async function reconcileStagedApplicationArtifacts( // (over the origin's own row) and left new code live under the previous release's config. An // activation artifact for this deployment is proof the swap began, so it is an interrupted // activation and belongs in the roll-forward path below. - activationBegan = (await activationArtifacts(componentDirPath, entry.name)).length > 0; if (row.status === 'staged' && !activationBegan) { if (!(await hasCompleteStagedApplication(stagedPath))) { let discarded = false; @@ -3504,6 +3520,7 @@ export async function reconcileStagedApplicationArtifacts( // An activation that appeared under the lock belongs in the roll-forward path below. if (!activationBegan) continue; } + // Reached for `activating` rows and for ANY row status backed by activation evidence. if (await hasCompleteStagedApplication(stagedPath)) { await activateStagedApplication(new Application({ name: row.project }), entry.name, { beforeCommit: () => persistActivation(row), diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 25391f6a5b..cecd93000d 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -459,6 +459,10 @@ let errorReporter; export function setErrorReporter(reporter) { errorReporter = reporter; } +/** So a caller that installs a reporter can put the previous one back when it is done with it. */ +export function getErrorReporter() { + return errorReporter; +} let compName: string; export const getComponentName = () => compName; diff --git a/components/operations.js b/components/operations.js index 487241677a..4be99c2a20 100644 --- a/components/operations.js +++ b/components/operations.js @@ -551,6 +551,13 @@ function markRestartRequiredForDeploy(application) { async function deployComponentOneShot(req, credentialReferences, isReplicatedExecution) { const { resolveCredentials } = require('./secretOperations.ts'); + // Before ANY work: the rejection should not come after a credential has been ingested into the + // secrets store and a durable deployment row created for a deploy that was never allowed. This used + // to sit inside the root-config write, which the staged path replaced with the activation + // transaction. Package deploys only, exactly as before — a payload deploy has always been allowed + // to use a core name. + if (req.package) assertNotProtectedCoreComponent(req.project, req.force); + // Create a hdb_deployment row up front so the deploy is observable and auditable even if the CLI // disconnects. The row also holds the payload in a Blob attribute, which doubles as the source for // peer replication and (later) rollback. Only the origin node records — peers replaying the @@ -584,11 +591,6 @@ async function deployComponentOneShot(req, credentialReferences, isReplicatedExe const emit = (event, data) => emitter?.emit(event, data); - // Protected core component names. This used to sit inside the root-config write, which the staged - // path replaced with the activation transaction — so it is asserted here explicitly, before any work. - // Package deploys only, exactly as before: a payload deploy has always been allowed to use the name. - if (req.package) assertNotProtectedCoreComponent(req.project, req.force); - // The payload-via-replicated-row path depends on `system` actually replicating on this node. const systemReplicated = isSystemDatabaseReplicated(); @@ -1040,7 +1042,25 @@ function buildDeployApplication({ // Load a component directory to surface load-time errors early (throwaway scopes). No-op on the main // thread or in safe mode. It loads the STAGED directory, before go-live, where it runs at all — see // loads the live directory after in-place prepare. -async function loadValidateComponent({ dirPath, emit }) { +// `componentLoader.setErrorReporter` is module-global, so two deploys validating concurrently on the +// same worker cross-attribute their load failures: B installs its reporter while A is loading, A's +// error lands in B, and A then activates broken bytes while B rejects a good candidate. Validation is +// serialized here — it is already the slow path — and the previous reporter is restored, so the global +// is only ever owned by one in-flight validation. +let validationChain = Promise.resolve(); +async function loadValidateComponent(args) { + const run = validationChain.then( + () => loadValidateComponentExclusive(args), + () => loadValidateComponentExclusive(args) + ); + validationChain = run.then( + () => {}, + () => {} + ); + return run; +} + +async function loadValidateComponentExclusive({ dirPath, emit }) { if (isMainThread || process.env.HARPER_SAFE_MODE) return; const pseudoResources = new Resources(); pseudoResources.isWorker = true; @@ -1048,6 +1068,7 @@ async function loadValidateComponent({ dirPath, emit }) { const componentLoader = require('./componentLoader.ts').default || require('./componentLoader.ts'); const { trackScopeClose } = require('./scopeShutdown.ts'); let lastError; + const priorErrorReporter = componentLoader.getErrorReporter?.(); componentLoader.setErrorReporter((error) => (lastError = error)); emit('phase', { phase: 'load', status: 'start' }); // The Scopes this load creates are throwaway. Collect them (instead of registering for @@ -1071,7 +1092,11 @@ async function loadValidateComponent({ dirPath, emit }) { }); // Track the load+close so a concurrent worker shutdown waits for these scopes to finish disposing. trackScopeClose(validation); - await validation; + try { + await validation; + } finally { + componentLoader.setErrorReporter(priorErrorReporter); + } emit('phase', { phase: 'load', status: 'done' }); if (lastError) throw lastError; } From c6834cbb1919368f6c38c7c95469729377ff4e09 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 24 Aug 2026 20:40:49 -0400 Subject: [PATCH 92/94] fix(deploy): make revert's precondition failures client errors, and settle rows we own MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three minors from the same review. `revert_component` rejected an unsatisfiable target — nothing retained, or a deployment id that is neither live nor the retained previous — with a bare `Error`, so the operations catch defaulted it to HTTP 500. Both are the caller asking for something that does not exist, not a node failure; they are 409s now, and the existing tests assert the code rather than only the message. A recovered activation logged that the deployment "remains activating until cluster state is reconciled" — a claim inherited from the peer protocol that no longer reconciles anything here. The node that ORIGINATED the deployment (the only node that has a row) now settles it `success` on roll-forward, since the activation it started is complete; a peer still leaves the origin's row alone rather than reporting on nodes it cannot see, and the message says which happened. Settling is observability, so a failure to write the row is logged, never allowed to keep a live reconciled component from loading. Staged-build retention can still evict a candidate that a concurrent deploy owns, above `maxCount` simultaneous deploys of one component — more reachable now that validation serializes after the component lock is released. Left as-is and documented: tracking ownership would let a deploy that dies between stage and activate pin a tree forever, defeating the disk bound the prune exists to enforce. A loud "no valid component tree" on the 6th concurrent deploy of the same component is the better failure. Reported by codex pre-push. --- components/Application.ts | 22 ++++++++++++++----- components/componentLoader.ts | 25 ++++++++++++++++++++-- unitTests/components/deployStaging.test.js | 12 +++++++++-- 3 files changed, 50 insertions(+), 9 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index d7d218f793..649a557275 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -8,6 +8,8 @@ import { readConfigFile, } from '../config/configUtils.ts'; import { CONFIG_PARAMS } from '../utility/hdbTerms.ts'; +import { ClientError } from '../utility/errors/hdbError.ts'; +import { HTTP_STATUS_CODES } from '../utility/errors/commonErrors.ts'; import logger, { errorForLog } from '../utility/logging/harper_logger.ts'; import { broadcastDeployStart, broadcastDeployEnd } from './deployLifecycle.ts'; import { ComponentPreparationLockTimeoutError, withComponentPreparationLock } from './componentPreparationLock.ts'; @@ -615,7 +617,13 @@ async function pruneStagedBuilds(componentName: string, keepStagingId: string, m // keep all four. The residual: two stages tying to the millisecond can have the second evict // the first's tree while its row stays `staged`, so activating that id later fails with "no // valid component tree" rather than serving nothing. A clear failure, and the reconcile pass - // settles such a row. Sorting breaks ties by stagingId so concurrent prunes choose the same + // settles such a row. The same applies above `maxCount` genuinely-concurrent deploys of ONE + // component: builds serialize on the component lock but validation runs after it is released, + // so the oldest queued candidate can be evicted before it activates. Ownership is deliberately + // not tracked to close that: a registry entry leaked by a deploy that dies between stage and + // activate would pin a tree forever, defeating the disk bound this exists to enforce, which is + // worse than a loud failure on the 6th simultaneous deploy of the same component. + // Sorting breaks ties by stagingId so concurrent prunes choose the same // victims instead of each deleting the other's. const current = builds.find((build) => build.stagingId === keepStagingId); const others = builds @@ -1127,9 +1135,12 @@ export async function revertApplication( return withComponentPreparationLock(liveDirPath, async () => { const target = await getRevertTarget(liveDirPath); if (!target) { - throw new Error( + // A caller asking for something the node cannot supply, not a node failure: 409, so the operations + // API does not report an unsatisfiable revert as a 500 the client is invited to retry. + throw new ClientError( `Cannot revert ${application.name}: no previous version is retained. A component must have been ` + - `deployed over a prior version (which activation retains as .deploy-previous) to be reverted.` + `deployed over a prior version (which activation retains as .deploy-previous) to be reverted.`, + HTTP_STATUS_CODES.CONFLICT ); } if (target.live.deployment_id === toDeploymentId) { @@ -1137,11 +1148,12 @@ export async function revertApplication( return { swapped: false, activatedConfig: target.live.application_config, fromDeploymentId: null }; } if (target.previous.deployment_id !== toDeploymentId) { - throw new Error( + throw new ClientError( `Cannot revert ${application.name} to deployment '${toDeploymentId}': it is neither the live version ` + `('${target.live.deployment_id ?? 'unknown'}') nor the retained previous version ` + `('${target.previous.deployment_id ?? 'unknown'}'). Only the immediately-previous version is retained ` + - `on disk; redeploy the version you want with deploy_component instead.` + `on disk; redeploy the version you want with deploy_component instead.`, + HTTP_STATUS_CODES.CONFLICT ); } diff --git a/components/componentLoader.ts b/components/componentLoader.ts index cecd93000d..377f9a9482 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -60,6 +60,7 @@ import { recoverInterruptedReverts, } from './Application.ts'; import { getDeploymentRow, markDeploymentTerminal } from './deploymentRecorder.ts'; +import { hostname } from 'node:os'; import { ComponentPreparationLockTimeoutError } from './componentPreparationLock.ts'; import { pathToFileURL } from 'node:url'; @@ -234,9 +235,29 @@ export async function loadComponentDirectories( settleDiscardedDeployment ); for (const deploymentId of reconciliation.recovered) { + // The row is created by the ORIGIN only, so this node may be settling a deployment it did + // not originate. Settling one it did own is the whole story — the activation it started is + // now complete — while a peer stamping the origin's row would report on nodes it cannot see. + // A row left `activating` is not stranded forever either way: the next deploy of the project + // settles it. + let settled = false; + try { + const row = await getDeploymentRow(deploymentId); + if (row?.origin_node === hostname()) { + await markDeploymentTerminal(deploymentId, 'success'); + settled = true; + } + } catch (error) { + // Observability only; a component that is live and reconciled must not fail to load + // because its row could not be updated. + harperLogger.warn( + `Could not settle the deployment row for recovered activation '${deploymentId}':`, + errorForLog(error as Error) + ); + } harperLogger.warn( - `Rolled forward interrupted component activation '${deploymentId}' on this node; ` + - `the deployment remains activating until cluster state is reconciled` + `Rolled forward interrupted component activation '${deploymentId}' on this node` + + (settled ? '' : `; its deployment row still reads activating`) ); } for (const [deploymentId, error] of reconciliation.errors) { diff --git a/unitTests/components/deployStaging.test.js b/unitTests/components/deployStaging.test.js index 30d9a49852..2e23e7e50a 100644 --- a/unitTests/components/deployStaging.test.js +++ b/unitTests/components/deployStaging.test.js @@ -441,7 +441,11 @@ describe('two-phase component directory transaction', function () { assert.strictEqual(await getRevertTarget(application.dirPath), undefined); await assert.rejects( () => revertApplication(application, randomUUID()), - /no previous version is retained/, + (error) => { + assert.match(error.message, /no previous version is retained/); + assert.strictEqual(error.statusCode, 409, "an unsatisfiable revert is the caller's problem, not a 500"); + return true; + }, 'a component deployed once cannot be reverted' ); await cleanup(name); @@ -510,7 +514,11 @@ describe('two-phase component directory transaction', function () { await assert.rejects( () => revertApplication(application, randomUUID()), - /neither the live version .* nor the retained previous version/s, + (error) => { + assert.match(error.message, /neither the live version .* nor the retained previous version/s); + assert.strictEqual(error.statusCode, 409, "asking for a version nobody kept is the caller's problem"); + return true; + }, 'only one previous version is retained, so anything else is a redeploy' ); assert.match(await readMarker(application.dirPath), /v2/, 'a refused revert changes nothing'); From f3d1a32c88d13ecd961b5df9b827bc467905b4b3 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 24 Aug 2026 21:05:25 -0400 Subject: [PATCH 93/94] fix(deploy): stop a dropped component outliving its mount, and keep recovery armed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two release-blocking findings from the graded review, plus the fail-open behind one of them. **A dropped component could come back served on every host.** Applications in the components root load by DIRECTORY SCAN; the root-config entry is what constrains where one is served (`host`/`urlPath`), not what makes it load. This branch had moved the config deletion to the front of `drop_component`, reasoning that the entry is the only resurrection vector. It is not: with the entry gone and the tree still on disk, a crash mid-teardown leaves the component discoverable with no mount at all, so the next boot serves it unconstrained — silently dropping the isolation the operator configured, which `tryRootConfigMount` already treats as worse than not loading at all. The tree is parked first now and the entry removed only after, which restores `main`'s ordering and leaves the merely-re-runnable partial state instead: tree gone, entry present. The test covering this asserted the dangerous state and called it correct — it required config-clean-with-tree-present. It now asserts the invariant that matters: a discoverable tree must never outlive its mount. Mutation-verified against the old ordering. **Startup recovery disarmed itself after the first clean pass.** A reload cycle runs `loadComponentDirectories` again long after boot, and an activation that fails at runtime AND fails to compensate leaves exactly the inconsistent staged/live/ backup state the reconciliation pass exists to settle. The one-shot guard meant only a cold restart repaired it; until then a hot reload could load the candidate against old or partial configuration. It runs every main-thread cycle now — the pass is idempotent, and once settled it is a readdir of a directory it empties. **A manifest read error was indistinguishable from no manifest.** All three call sites wrapped the reader in `.catch(() => undefined)`, but the reader already maps ENOENT to undefined, so the catch only ever swallowed real failures — EACCES, EIO, corrupt JSON. Each then recorded the displaced release as `deployment_id: null`: retention overwrites an addressable previous version with an unaddressable one, and the reconciliation probe deletes the last copy of a displaced release as residue. All three now propagate, which on the deploy path fails before the swap, so the live component keeps serving. Reported by codex pre-push. --- components/Application.ts | 12 ++++++---- components/componentLoader.ts | 11 +++++---- components/operations.js | 17 ++++++++------ unitTests/components/deployOperations.test.js | 23 +++++++++---------- 4 files changed, 35 insertions(+), 28 deletions(-) diff --git a/components/Application.ts b/components/Application.ts index 649a557275..0d17561b4b 100644 --- a/components/Application.ts +++ b/components/Application.ts @@ -3146,8 +3146,12 @@ export async function activateStagedApplication( await ensureSecureDirectory(activationDir, true, 'Component activation staging path'); // Who is live right now, so the tree this activation displaces stays addressable for revert. Read // before the swap, since the swap is what makes it "previous". - const outgoing: RetainedVersion = (await readRetainedPreviousManifest(application.dirPath).catch(() => undefined)) - ?.live ?? { + // NOT caught: the reader already maps a missing manifest to undefined, so anything it throws is a + // real read failure (EACCES, EIO, corrupt JSON). Treating those as "no manifest" would record the + // outgoing release as `deployment_id: null` and let retention overwrite the addressable previous + // version with an unaddressable one — losing revert silently. Failing here is before the swap, so + // the live component keeps serving. + const outgoing: RetainedVersion = (await readRetainedPreviousManifest(application.dirPath))?.live ?? { // No manifest yet: either a first-ever deploy, or a component last activated by a Harper that // predates retention. Record the config it is running so a revert can still restore that, even // though its deployment id is unknowable and so cannot be a revert target. @@ -3382,7 +3386,7 @@ async function retainRecoveredActivation( // above already describes the intended end state, so there it is `previous`. `live` matching the // deployment being recovered is what tells the two apart — without this the recovered manifest // retains the right bytes under a null id, and revert_component cannot address them. - const manifest = await readRetainedPreviousManifest(componentDirPath).catch(() => undefined); + const manifest = await readRetainedPreviousManifest(componentDirPath); const displaced = manifest?.live?.deployment_id === deploymentId ? manifest?.previous : manifest?.live; const outgoing: RetainedVersion = displaced ?? { deployment_id: null, application_config: null }; await retainActivatedPrevious( @@ -3641,7 +3645,7 @@ export async function reconcileStagedApplicationArtifacts( artifact.name.startsWith(ACTIVATION_BACKUP_PREFIX) && row && liveUsable && - (await readRetainedPreviousManifest(livePath).catch(() => undefined))?.live?.deployment_id === deploymentId + (await readRetainedPreviousManifest(livePath))?.live?.deployment_id === deploymentId ) { // The activation finished — settled row, good live tree — but a backup is still parked, // which only happens when `retainActivatedPrevious` failed inside its best-effort catch. diff --git a/components/componentLoader.ts b/components/componentLoader.ts index 377f9a9482..a5c5d73b85 100644 --- a/components/componentLoader.ts +++ b/components/componentLoader.ts @@ -68,7 +68,6 @@ const CF_ROUTES_DIR = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); let loadedComponents = new Map(); let watchesSetup; let resources; -let stagedArtifactsReconciled = false; const componentLoadTails = new Map>(); type ComponentReadyPromises = WeakMap>; @@ -208,7 +207,12 @@ export async function loadComponentDirectories( throw error; } } - if (isMainThread && !stagedArtifactsReconciled) { + // Every main-thread load cycle, not just the first. A reload cycle (loadComponentDirectories + + // restartWorkers) runs long after startup, and an activation that failed at runtime AND failed to + // compensate leaves exactly the inconsistent staged/live/backup state this pass exists to settle. + // Skipping it there would load the candidate against old or partial configuration until a cold + // restart. Idempotent and cheap once settled: the scan is a readdir of a directory this pass empties. + if (isMainThread) { try { const settleDiscardedDeployment = async (deploymentId: string, reason: string) => { await markDeploymentTerminal(deploymentId, 'failed', new Error(reason)); @@ -272,9 +276,6 @@ export async function loadComponentDirectories( for (const [project, error] of reconciliation.failedProjects) { if (!failedRecoveries.has(project)) failedRecoveries.set(project, error); } - // Only a clean pass retires the one-shot guard, so a later reload cycle retries instead of - // leaving a component permanently unrecovered and permanently unloadable. - if (reconciliation.errors.size === 0) stagedArtifactsReconciled = true; } catch (error) { // The scan itself failed, so we cannot tell WHICH components are affected and cannot fail just // those closed. Abort startup rather than load every component over possibly-unreconciled state. diff --git a/components/operations.js b/components/operations.js index 4be99c2a20..de3662d7b0 100644 --- a/components/operations.js +++ b/components/operations.js @@ -1580,13 +1580,6 @@ async function dropComponent(req) { } if (!file) { - // Persisted state goes FIRST, before anything is destroyed. The root-config entry is the only - // thing that can resurrect a dropped component — installApplications() reinstalls whatever it - // names — so removing it up front means a crash anywhere below leaves the drop unfinished - // (directory still present, re-runnable) instead of finished-then-undone. Both writes are one - // transaction, config before the application lock. - const dropTransaction = await createApplicationConfigTransaction(project, null); - await dropTransaction.commit(); // Invalidate staged deployments before the directory goes away, so an activate racing this // drop cannot swap a staged build back into a component that is being dropped. await invalidateProjectStagedDeployments(project); @@ -1596,6 +1589,16 @@ async function dropComponent(req) { // Retires any interrupted-extraction aside and renames the live tree aside before removing // it, so startup recovery can never restore a tree over a dropped component. await dropComponentDirectory(componentPath, project, log); + // The directory goes FIRST and the config entry only after, because applications in the + // components root load by directory scan — the root-config entry is what CONSTRAINS where one + // is served (host/urlPath), not what makes it load. Dropping config first leaves a window + // where the tree is still discoverable with its mount gone, so a crash there resurrects the + // component served on every host. Losing the operator's isolation is worse than a drop that + // has to be re-run, which is what this ordering leaves instead: tree gone, entry present, + // re-runnable — and the same residual `main` has today. Both writes are one transaction, + // config before the application lock. + const dropTransaction = await createApplicationConfigTransaction(project, null); + await dropTransaction.commit(); } else if (await fs.pathExists(pathToComponent)) { await fs.remove(pathToComponent); } diff --git a/unitTests/components/deployOperations.test.js b/unitTests/components/deployOperations.test.js index 0c854bf331..cc5e268cd3 100644 --- a/unitTests/components/deployOperations.test.js +++ b/unitTests/components/deployOperations.test.js @@ -357,11 +357,12 @@ describe('deploy_component staged deploy', function () { assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /trusted-one-shot/); }); - it('removes the root-config entry before it destroys anything, so a failed drop cannot resurrect', async () => { - // The resurrection vector is the root-config entry: installApplications() reinstalls whatever it - // names. Doing the persistent writes after the directory removal left a crash window where the - // live tree was gone but config still named the package, and the next boot brought the component - // back. Config-first inverts the failure: an interrupted drop is unfinished and re-runnable. + it('never leaves a component discoverable with its routing config already gone', async () => { + // Applications in the components root load by DIRECTORY SCAN; the root-config entry is what + // constrains where one is served (host/urlPath). So the state to make unreachable is tree-present + // with entry-gone — a crash there resurrects the component on every host, silently dropping the + // isolation the operator configured. The tree is parked first and the entry removed only after, + // which leaves the opposite (and merely re-runnable) partial state instead. const project = name(); const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-drop-crash-')); const componentsRoot = path.join(configRoot, 'components'); @@ -383,15 +384,13 @@ describe('deploy_component staged deploy', function () { await assert.rejects(() => operations.dropComponent({ project })); + const treePresent = existsSync(path.join(componentsRoot, project)); + const entryPresent = (await fs.readFile(configPath, 'utf8')).includes(project); + assert.strictEqual(treePresent, true, 'teardown failed, so the live tree is still on disk'); assert.strictEqual( - (await fs.readFile(configPath, 'utf8')).includes(project), - false, - 'config is already clean, so the next boot cannot reinstall the component' - ); - assert.strictEqual( - existsSync(path.join(componentsRoot, project)), + entryPresent, true, - 'and the tree is still there: the drop is unfinished rather than finished-then-undone' + 'and its routing entry survived with it — a discoverable tree must never outlive its mount' ); } finally { if (priorRootEnv === undefined) delete process.env.ROOTPATH; From 080d85a669e4c751d76329652aae51c6c96a39f9 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 24 Aug 2026 21:11:51 -0400 Subject: [PATCH 94/94] refactor(deploy): send #2301's code with #2301, restore the coverage the split took MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `claimStagedDeployment`, `settleStagedRows`, and `expireOldStagedDeployments` have no caller in this tree — they exist only for the stacked coordination PR, and were carried here behind RESERVED comments to save re-adding them there. That is exactly backwards for a PR whose point was to shrink to what this tree can verify: dead code reviewed here is dead code nobody can exercise here. They land with their caller, along with the ten unit tests that were their only consumer. Removing the staged-deploy capability probe also deleted the whole `deploy_component cross-version compatibility` block, which covered more than the probe: the pre-5.1 package-JSON downgrade, the pre-5.1 directory/CBOR downgrade, the 5.1+ streaming path, and a failed version probe assuming modern. Those branches are still live in `cliOperations`, so a request-format change could have silently broken deploying to an older Harper with nothing to catch it. Restored, and checked against a mutation that disables the downgrade. DESIGN.md claimed two guarantees this branch does not provide: that every staged deploy validates the candidate loads (it is a no-op on the main thread, which is where the operations API deploys — the caveat was already documented 300 lines later, contradicting it), and that automatic payload pruning appends a `payload_dropped` event to the rows it prunes (it deliberately does not — `event_log` is append-only and a read-copy-write would lose a concurrent writer's entry). Both now describe what actually happens. Also drops comments that narrated code or restated a contract validation now forbids. Reported by codex pre-push. --- DESIGN.md | 15 +- bin/cliOperations.ts | 5 +- components/deploymentRecorder.ts | 98 +--------- components/operations.js | 7 +- unitTests/bin/cliOperations.test.js | 137 ++++++++++++++ .../components/deploymentRecorder.test.js | 170 ------------------ 6 files changed, 154 insertions(+), 278 deletions(-) diff --git a/DESIGN.md b/DESIGN.md index aa25ea0285..97f3344c98 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -196,7 +196,9 @@ Boot's `harper-application-lock.json` records an application configuration only ## Component deploys build off to the side and swap atomically `deploy_component` prepares the candidate under `.deploy-staging//`, runs -extract + `npm install` there, validates that it loads, and only then renames it into the live path — +extract + `npm install` there, validates that it loads _where that check runs at all_ (it is a no-op on +the main thread, which is where the operations API deploys — see "Staged deploy" below), and only then +renames it into the live path — committing root config and `harper-application-lock.json` in the same compensating transaction. The live component keeps serving through the slow, failure-prone work (git clone, registry install), and go-live is one atomic rename. A fetch or install failure leaves the running component untouched @@ -598,9 +600,14 @@ deployment moves to #2301 with the rest of the coordination protocol. successful deploy. Bounds how many payloads accumulate over time, which is what actually caps disk. Only rows that still hold a payload count toward `maxCount`, so the cap reads literally as "at most N -stored payloads per project." Rows are never deleted — pruning nulls the blob and appends a -`payload_dropped` event, so the audit trail and `get_deployment` stay intact; only -`get_deployment_payload` stops working for pruned deployments (`payload_blob_present: false`). A +stored payloads per project." Rows are never deleted — pruning nulls the blob and nothing else, so the audit +trail and `get_deployment` stay intact; only `get_deployment_payload` stops working for pruned +deployments (`payload_blob_present: false`). Automatic pruning does NOT append `payload_dropped` to the +rows it prunes: `event_log` is append-only and adding to it is a read-copy-write, which would lose a +concurrent writer's entry, so `payload_blob_present: false` is what records the drop there. The +deploying operation still emits `payload_dropped` on its own progress stream (and so into its own +row's `event_log`), and the explicit `delete_deployment_payload` does append it to the row it targets — +that one is a single operator-driven write with no concurrent writer to lose. A non-terminal deployment is counted but never dropped: its blob may still be the replication channel peers are installing from (the same guard `delete_deployment_payload` uses). Pruning is best-effort and off the deploy's critical path — a prune failure is logged, never fatal — and is skipped entirely when a diff --git a/bin/cliOperations.ts b/bin/cliOperations.ts index 1b15653d86..24fe77d268 100644 --- a/bin/cliOperations.ts +++ b/bin/cliOperations.ts @@ -37,9 +37,8 @@ const OP_VERB_PROPS: Record> = { revert: { operation: 'revert_component', _cliVerb: 'revert' }, }; -// Guard CLI-verb requirements that the operation itself can't enforce (the op has no notion of which -// verb invoked it). Returns an error message, or null when the request is fine. Pure + exported so it -// is unit-testable without the network/process-exit machinery in cliOperations. +// The operation has no notion of which verb invoked it, so requirements that belong to the verb are +// enforced here. Returns an error message, or null when the request is fine. function verbRequirementError(req: any): string | null { // revert_component requires its target so a retry can't toggle the rejected release back in. Caught // here too, so the CLI names the flag instead of surfacing a raw validation error. diff --git a/components/deploymentRecorder.ts b/components/deploymentRecorder.ts index 0166a51159..39a3a8350c 100644 --- a/components/deploymentRecorder.ts +++ b/components/deploymentRecorder.ts @@ -623,114 +623,22 @@ export async function recordDeploymentPeers(deploymentId: string, results: unkno await table.patch(deploymentId, { peer_results: peers }); } -/** - * RESERVED FOR #2301 — no caller in this tree. - * - * Claim a staged deployment for activation while the component preparation lock is held. On the - * ORIGIN this marks the row `activating`; `persist: false` runs the same validation without the - * write, because the row is replicated and only the origin owns it. - * - * Kept here rather than deleted because the coordination protocol PR is stacked directly on this one - * and is its only consumer; deleting it here would just mean re-adding it there. Do not assume a - * resting `staged` row producer exists in-tree — there isn't one until #2301 lands. - */ /** * Whether deployment tracking is provisioned on this node. * - * `DeploymentRecorder.put()` is deliberately tolerant — a one-shot deploy still works with no - * `hdb_deployment` table, because tracking is observability there. It is NOT observability for the - * separated phases: `activate: false` hands the caller a deployment_id that only the row can resolve, - * so a stage that reported success would be unactivatable. Callers of the separated phases check this - * up front instead of failing later with a missing row. + * `DeploymentRecorder.put()` is deliberately tolerant — a deploy still works with no `hdb_deployment` + * table, because tracking is observability. This reports whether a row will actually exist, which the + * deploy path needs to decide whether peers can read the payload from the row or must be sent it. */ export function isDeploymentTrackingAvailable(): boolean { return !!(databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; } -export async function claimStagedDeployment( - deploymentId: string, - project: string, - options: { allowActivating?: boolean; waitForStagedMs?: number; persist?: boolean } = {} -): Promise> { - const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; - if (!table) throw new ClientError('Deployment tracking is unavailable; cannot activate a staged deployment'); - const deadline = Date.now() + coerceTimeoutMs(options.waitForStagedMs, 0); - let row = await table.get(deploymentId); - if (!row) throw new ClientError(`No deployment found with id '${deploymentId}'`); - if (row.project !== project) { - throw new ClientError(`Deployment '${deploymentId}' belongs to component '${row.project}', not '${project}'`); - } - while (['pending', 'staging'].includes(row.status) && Date.now() < deadline) { - await new Promise((resolve) => setTimeout(resolve, Math.min(25, deadline - Date.now()))); - row = await table.get(deploymentId); - if (!row) throw new ClientError(`No deployment found with id '${deploymentId}'`); - if (row.project !== project) { - throw new ClientError(`Deployment '${deploymentId}' belongs to component '${row.project}', not '${project}'`); - } - } - if (row.status === 'activating' && options.allowActivating) return row; - if (row.status !== 'staged') { - throw new ClientError(`Deployment '${deploymentId}' is '${row.status}', not staged and available for activation`); - } - if (options.persist !== false) { - await table.patch(deploymentId, { status: 'activating', phase: 'activate', completed_at: null, error: null }); - } - return row; -} - // Deployment statuses that are settled — a non-terminal deployment's payload_blob may still be the // replication channel peers are installing from, so retention must never yank it mid-flight. Mirrors // deploymentOperations.ts's guard for the explicit delete_deployment_payload operation. const TERMINAL_STATUSES = new Set(['success', 'failed', 'rolled_back']); -// RESERVED FOR #2301, like claimStagedDeployment above: staged-ROW retention has no caller in this -// tree, because nothing here leaves a deployment resting in `staged`. Staged DIRECTORY retention -// (pruneStagedBuilds) is the live one. -async function settleStagedRows(project: string, keepCount: number, keepDeploymentId?: string): Promise { - const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; - if (!table) return []; - const staged: Array> = []; - for await (const row of table.search([{ attribute: 'project', value: project }])) { - if (row?.status === 'staged') staged.push(row); - } - staged.sort( - (a, b) => (b.started_at ?? 0) - (a.started_at ?? 0) || String(a.deployment_id).localeCompare(b.deployment_id) - ); - // Keep the request being returned, plus any row strictly NEWER than it, plus the newest of the rest - // up to the count. Excluding the current id from the ranking instead would privilege it - // unconditionally: a stage that waited on slow peers resumes with an older `started_at`, so with a - // retention of 1 it would expire the newer stage that had already completed and been reported to the - // operator, then discard its staging tree cluster-wide. `started_at` is stamped by whichever node - // originated the deploy, so cross-node clock skew produces the same inversion. Ties count as - // protected, not evictable: two concurrent stages of one component can both reach `staged` in the - // same millisecond, and evicting a tied row would fail a request that is about to return its - // deployment id to the caller. The budget subtracts the protected rows, so the retained total still - // lands on `keepCount` except when more rows tie-or-exceed the window than fit in it — a temporary - // overflow, which is the right way for a disk bound to fail. - const current = keepDeploymentId ? staged.find((row) => row.deployment_id === keepDeploymentId) : undefined; - const others = staged.filter((row) => row.deployment_id !== keepDeploymentId); - const protectedRows = current ? others.filter((row) => (row.started_at ?? 0) >= (current.started_at ?? 0)) : []; - const budget = Math.max(0, keepCount - (current ? 1 : 0) - protectedRows.length); - const expired = others.filter((row) => !protectedRows.includes(row)).slice(budget); - for (const row of expired) { - await table.patch(row.deployment_id, { - status: 'failed', - completed_at: Date.now(), - error: { message: 'Staged build expired by deployment_stagingRetention_maxCount', phase: 'staged' }, - }); - } - return expired.map((row) => row.deployment_id); -} - -export async function expireOldStagedDeployments( - project: string, - maxCount: number, - keepDeploymentId?: string -): Promise { - const count = Number.isFinite(maxCount) ? Math.max(1, Math.floor(maxCount)) : 1; - return settleStagedRows(project, count, keepDeploymentId); -} - export async function invalidateProjectStagedDeployments(project: string): Promise { const table = (databases as any).system?.[terms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME]; if (!table) return []; diff --git a/components/operations.js b/components/operations.js index de3662d7b0..0f11755997 100644 --- a/components/operations.js +++ b/components/operations.js @@ -475,7 +475,6 @@ async function packageComponent(req) { * any credential token into the secrets store (so it lives as a replicated reference, not embedded), * then stages the build and swaps it in. * - * `two_phase: false` forces the one-shot path. See DESIGN.md for the stage/activate protocol. * * @param req * @returns {Promise} @@ -954,12 +953,8 @@ async function restartRevertedComponent(req, emit) { return { restartMessage: '' }; } -// Shared deploy-family helpers (used by deploy_component and -// revert_component). - -// Reject deploying over a protected core component name unless force is set. Lazy-loads -// componentLoader to avoid a circular dependency. function assertNotProtectedCoreComponent(project, force) { + // Lazily required: componentLoader requires this module back. const { TRUSTED_RESOURCE_PLUGINS } = require('./componentLoader.ts'); if (TRUSTED_RESOURCE_PLUGINS[project] && !force) { throw handleHDBError( diff --git a/unitTests/bin/cliOperations.test.js b/unitTests/bin/cliOperations.test.js index 87cbb0e30b..f893733b09 100644 --- a/unitTests/bin/cliOperations.test.js +++ b/unitTests/bin/cliOperations.test.js @@ -5,6 +5,7 @@ const path = require('node:path'); const fs = require('fs-extra'); const os = require('node:os'); const { Readable } = require('node:stream'); +const { decode: decodeCbor } = require('cbor-x'); const { saveCredentials } = require('#src/bin/cliCredentials'); const cliOperationsModule = require('#src/bin/cliOperations'); const commonUtilsModule = require('#src/utility/common_utils'); @@ -514,6 +515,142 @@ describe('cliOperations', () => { }); }); + // The CLI still negotiates with pre-5.1 servers (`targetSupportsStreamingDeploy` → + // `_legacyDeploy` → CBOR body). Removing the staged-deploy probe took this whole block with it, + // which left those still-live branches uncovered — a later request-format change could silently + // break deploying to an older Harper. + describe('deploy_component cross-version compatibility', () => { + const target = 'https://example.com:9925/'; + let originalPackageDirectory; + let originalScan; + + beforeEach(() => { + saveCredentials(target, { operation_token: 'valid-token', refresh_token: 'refresh-token' }); + tokenAuthModule.isJWTExpired = () => false; + originalPackageDirectory = packageComponentModule.packageDirectory; + originalScan = packageComponentModule.scanPackageDirectory; + }); + + afterEach(() => { + packageComponentModule.packageDirectory = originalPackageDirectory; + packageComponentModule.scanPackageDirectory = originalScan; + }); + + // Streams an SSE `done` event so the modern (>= 5.1) deploy path can read its result. + const sseDoneResponse = (result) => + Object.assign(Readable.from([`event: done\ndata: ${JSON.stringify({ result })}\n\n`]), { + statusCode: 200, + headers: { 'content-type': 'text/event-stream' }, + }); + + it('downgrades a package deploy to legacy JSON when the target is < 5.1', async () => { + const calls = []; + commonUtilsModule.httpRequest = async (options, req) => { + calls.push({ options, req }); + if (req.operation === 'registration_info') { + return { statusCode: 200, body: JSON.stringify({ version: '5.0.31' }) }; + } + return { statusCode: 200, body: JSON.stringify({ message: 'Successfully deployed', success: true }) }; + }; + + const result = await cliOperationsModule.cliOperations( + { operation: 'deploy_component', package: '@scope/widget', project: 'widget', target: 'example.com' }, + true + ); + + // Probe first, then the deploy. + assert.strictEqual(calls[0].req.operation, 'registration_info'); + assert.strictEqual(calls[0].options.streamResponse, undefined); + const deploy = calls[1]; + // No streaming negotiation against the old server. + assert.strictEqual(deploy.options.headers.Accept, undefined); + assert.strictEqual(deploy.options.streamResponse, undefined); + // Body is a plain JSON object, not a multipart stream, and carries no transport-only fields. + assert.strictEqual(typeof deploy.req.pipe, 'undefined'); + assert.strictEqual(deploy.req.operation, 'deploy_component'); + assert.strictEqual(deploy.req._legacyDeploy, undefined); + assert.strictEqual(deploy.req._multipart, undefined); + assert.strictEqual(result.success, true); + }); + + it('downgrades a directory deploy to a CBOR binary payload when the target is < 5.1', async () => { + const fakeTarball = Buffer.from('fake-tarball-bytes'); + packageComponentModule.scanPackageDirectory = async () => ({ + totalSize: fakeTarball.length, + danglingSymlinks: [], + }); + packageComponentModule.packageDirectory = async () => fakeTarball; + + const calls = []; + commonUtilsModule.httpRequest = async (options, req) => { + calls.push({ options, req }); + if (req.operation === 'registration_info') { + return { statusCode: 200, body: JSON.stringify({ version: '5.0.31' }) }; + } + return { statusCode: 200, body: JSON.stringify({ message: 'Successfully deployed', success: true }) }; + }; + + const result = await cliOperationsModule.cliOperations( + { operation: 'deploy_component', project: 'widget', target: 'example.com' }, + true + ); + + const deploy = calls[1]; + assert.strictEqual(deploy.options.streamResponse, undefined); + // Multipart was abandoned in favor of a CBOR body carrying the tarball as a + // native binary Buffer — the transport pre-5.1 servers decode directly. + assert.strictEqual(deploy.options.headers['Content-Type'], 'application/cbor'); + assert.ok(Buffer.isBuffer(deploy.req), 'CBOR body should be a Buffer'); + const decoded = decodeCbor(deploy.req); + assert.ok(Buffer.isBuffer(decoded.payload), 'decoded payload should be a Buffer'); + assert.strictEqual(decoded.payload.toString(), 'fake-tarball-bytes'); + assert.strictEqual(decoded.operation, 'deploy_component'); + assert.strictEqual(decoded._multipart, undefined); + assert.strictEqual(result.success, true); + }); + + it('keeps the streaming deploy path when the target is >= 5.1', async () => { + const calls = []; + commonUtilsModule.httpRequest = async (options, req) => { + calls.push({ options, req }); + if (req.operation === 'registration_info') { + return { statusCode: 200, body: JSON.stringify({ version: '5.1.7' }) }; + } + return sseDoneResponse({ message: 'Successfully deployed', success: true }); + }; + + const result = await cliOperationsModule.cliOperations( + { operation: 'deploy_component', package: '@scope/widget', project: 'widget', target: 'example.com' }, + true + ); + + const deploy = calls[1]; + assert.strictEqual(deploy.options.headers.Accept, 'text/event-stream'); + assert.strictEqual(deploy.options.streamResponse, true); + assert.strictEqual(result.success, true); + }); + + it('does not downgrade when the version probe fails (assumes modern)', async () => { + const calls = []; + commonUtilsModule.httpRequest = async (options, req) => { + calls.push({ options, req }); + if (req.operation === 'registration_info') { + return { statusCode: 404, body: 'not found' }; + } + return sseDoneResponse({ message: 'Successfully deployed', success: true }); + }; + + const result = await cliOperationsModule.cliOperations( + { operation: 'deploy_component', package: '@scope/widget', project: 'widget', target: 'example.com' }, + true + ); + + const deploy = calls[1]; + assert.strictEqual(deploy.options.headers.Accept, 'text/event-stream'); + assert.strictEqual(result.success, true); + }); + }); + describe('"Harper is not running" messaging (harper#658)', () => { const NOT_RUNNING_MESSAGE = 'Harper is not running. Use `harperdb run` (or `harperdb start`) to start it.'; diff --git a/unitTests/components/deploymentRecorder.test.js b/unitTests/components/deploymentRecorder.test.js index db41dc3e38..b63f01bd19 100644 --- a/unitTests/components/deploymentRecorder.test.js +++ b/unitTests/components/deploymentRecorder.test.js @@ -20,8 +20,6 @@ const { readPayloadBlobWithRetry, markDeploymentTerminal, recordDeploymentPeers, - claimStagedDeployment, - expireOldStagedDeployments, invalidateProjectStagedDeployments, pruneProjectPayloads, getDeploymentRow, @@ -911,174 +909,6 @@ describe('staged deployment state', () => { }); afterEach(() => installed.restore()); - it('claims only the matching staged row and durably marks it activating', async () => { - installed.mock.rows.set('staged-1', { - deployment_id: 'staged-1', - project: 'app', - status: 'staged', - completed_at: 100, - }); - - await assert.rejects(claimStagedDeployment('staged-1', 'other'), /belongs to component 'app'/); - await claimStagedDeployment('staged-1', 'app'); - - const row = installed.mock.rows.get('staged-1'); - assert.strictEqual(row.status, 'activating'); - assert.strictEqual(row.phase, 'activate'); - assert.strictEqual(row.completed_at, null); - }); - - it('accepts an already-activating row only for replicated activation ordering', async () => { - installed.mock.rows.set('activating-1', { - deployment_id: 'activating-1', - project: 'app', - status: 'activating', - }); - - await assert.rejects(claimStagedDeployment('activating-1', 'app'), /not staged/); - const row = await claimStagedDeployment('activating-1', 'app', { allowActivating: true }); - - assert.strictEqual(row.status, 'activating'); - }); - - it('waits for a replicated deployment row to reach staged before claiming it', async () => { - installed.mock.rows.set('lagging-1', { - deployment_id: 'lagging-1', - project: 'app', - status: 'staging', - }); - setImmediate(() => { - installed.mock.rows.set('lagging-1', { - ...installed.mock.rows.get('lagging-1'), - status: 'staged', - }); - }); - - await claimStagedDeployment('lagging-1', 'app', { waitForStagedMs: 200 }); - - assert.strictEqual(installed.mock.rows.get('lagging-1').status, 'activating'); - }); - - it('does not write the deployment row when a peer claims a staged deployment', async () => { - // The row is replicated, so a peer patching it makes N+1 writers of one key. Under replication lag - // a peer's `activating` can land after the origin has written `success`, leaving a converged deploy - // stuck non-terminal. Peers validate and swap under the component preparation lock they already - // hold; the origin owns the row. - installed.mock.rows.set('peer-claim', { - deployment_id: 'peer-claim', - project: 'app', - status: 'staged', - started_at: 1, - }); - - const row = await claimStagedDeployment('peer-claim', 'app', { persist: false }); - - assert.strictEqual(row.deployment_id, 'peer-claim', 'the claim still validates and returns the row'); - assert.strictEqual( - installed.mock.rows.get('peer-claim').status, - 'staged', - 'but leaves the status alone for the origin to advance' - ); - }); - - it('still marks the row activating when the origin claims', async () => { - installed.mock.rows.set('origin-claim', { - deployment_id: 'origin-claim', - project: 'app', - status: 'staged', - started_at: 1, - }); - - await claimStagedDeployment('origin-claim', 'app'); - - assert.strictEqual(installed.mock.rows.get('origin-claim').status, 'activating'); - }); - - it('rejects a peer claim for the wrong component even without persisting', async () => { - installed.mock.rows.set('mismatch', { - deployment_id: 'mismatch', - project: 'other', - status: 'staged', - started_at: 1, - }); - - await assert.rejects(() => claimStagedDeployment('mismatch', 'app', { persist: false }), /belongs to component/); - }); - - it('expires only staged rows beyond the per-project count', async () => { - for (const [id, startedAt, status = 'staged'] of [ - ['old', 100], - ['middle', 200], - ['new', 300], - ['active', 50, 'activating'], - ]) { - installed.mock.rows.set(id, { - deployment_id: id, - project: 'app', - status, - started_at: startedAt, - }); - } - - assert.deepStrictEqual(await expireOldStagedDeployments('app', 2), ['old']); - assert.strictEqual(installed.mock.rows.get('old').status, 'failed'); - assert.strictEqual(installed.mock.rows.get('middle').status, 'staged'); - assert.strictEqual(installed.mock.rows.get('new').status, 'staged'); - assert.strictEqual(installed.mock.rows.get('active').status, 'activating'); - }); - - it('does not expire a newer staged deployment when an older one finishes late', async () => { - // A stage that waited on slow peers resumes with an older `started_at` than a stage that started - // after it and already returned. Reserving the resuming id by excluding it from the ranking made - // retention expire that newer, already-reported deployment — and then discard its staging tree - // cluster-wide. Cross-node clock skew on `started_at` produces the same inversion. - for (const [id, startedAt] of [ - ['slow-origin', 100], - ['finished-later', 200], - ]) { - installed.mock.rows.set(id, { deployment_id: id, project: 'app', status: 'staged', started_at: startedAt }); - } - - assert.deepStrictEqual( - await expireOldStagedDeployments('app', 1, 'slow-origin'), - [], - 'neither is surplus: one is newer, the other is the request being returned' - ); - assert.strictEqual(installed.mock.rows.get('finished-later').status, 'staged'); - assert.strictEqual(installed.mock.rows.get('slow-origin').status, 'staged'); - }); - - it('does not expire a staged row that ties the returning request', async () => { - // Two concurrent stages of one component can both reach `staged` in the same millisecond. Treating - // a tie as evictable let whichever pruned second mark the other failed and broadcast deletion of - // its tree — after that request had already returned the deployment id to its caller. - for (const [id, startedAt] of [ - ['tied-a', 500], - ['tied-b', 500], - ]) { - installed.mock.rows.set(id, { deployment_id: id, project: 'app', status: 'staged', started_at: startedAt }); - } - - assert.deepStrictEqual(await expireOldStagedDeployments('app', 1, 'tied-a'), []); - assert.strictEqual(installed.mock.rows.get('tied-b').status, 'staged'); - assert.strictEqual(installed.mock.rows.get('tied-a').status, 'staged'); - }); - - it('still expires rows older than both the window and the returning request', async () => { - for (const [id, startedAt] of [ - ['ancient', 50], - ['slow-origin', 100], - ['finished-later', 200], - ]) { - installed.mock.rows.set(id, { deployment_id: id, project: 'app', status: 'staged', started_at: startedAt }); - } - - assert.deepStrictEqual(await expireOldStagedDeployments('app', 1, 'slow-origin'), ['ancient']); - assert.strictEqual(installed.mock.rows.get('ancient').status, 'failed'); - assert.strictEqual(installed.mock.rows.get('slow-origin').status, 'staged'); - assert.strictEqual(installed.mock.rows.get('finished-later').status, 'staged'); - }); - it('invalidates staged and activating rows when their component is dropped', async () => { for (const status of ['staged', 'activating', 'success']) { installed.mock.rows.set(status, { deployment_id: status, project: 'app', status });