From 22ae86559683250e9d41e038729519dc67976142 Mon Sep 17 00:00:00 2001 From: Dawson Toth Date: Mon, 24 Aug 2026 16:38:02 -0400 Subject: [PATCH] feat(deploy): cluster-wide stage/activate barrier and the trusted peer phase operation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Restores the peer coordination protocol that #1849 carried before it was split, so it can be reviewed on its own and land with real multi-node verification. This is the half unit tests cannot validate: the harness mocks `replicateOperation` and enters the authorization bypass directly, so it cannot observe a peer at all. On top of #1849's per-node staged deploy, this adds: - `component_deploy_phase`, a trusted peer-only operation carrying the phase and deployment id. Authorization travels in AsyncLocalStorage rather than on the request, so it is unreachable over HTTP with ordinary credentials, and an older peer rejects an unknown operation instead of misreading a phase marker as a one-shot deploy. - The cluster-wide barrier: the origin stages everywhere and waits for every node to report a good stage before any node activates. A node that cannot fetch or install fails during staging while the live component is untouched everywhere. - The separated public phases — `activate: false` returns a staged deployment_id, `deployment_id` activates it later, `two_phase: false` forces single-phase — plus the `harper stage` / `harper activate` CLI verbs and the capability probe that stops a staged request reaching a server that would silently deploy it live. - The deployment row as the peer channel: the immutable activation specification, the payload blob peers read their bytes from, and staged-row retention. - Origin-owned row semantics: peers claim with `persist: false` so the replicated row has exactly one writer, with local activation artifacts as their crash evidence. Known open work, carried from the review rounds on #1849 rather than hidden: - No in-repo end-to-end coverage of the trusted dispatch path. This is the gap that motivated the split; the three-node harper-pro suite has to run against this revision before it merges. - #2294 — nothing orders two concurrently-originated deploys, so both can report success with different versions live. The barrier orders stage-before-activate, not deploy-against-deploy. - #2295 — a `package:` deploy resolves per node, so the barrier guarantees "everyone staged something", not "everyone staged the same bytes". Stacked on #1849; the diff here is exactly the coordination protocol. --- DESIGN.md | 183 +++- bin/cliOperations.ts | 48 +- components/operations.js | 610 ++++++++++- components/operationsValidation.js | 46 +- .../deploy-tracking-peer-branch.test.ts | 6 +- resources/registrationDeprecated.ts | 1 + server/serverHelpers/serverUtilities.ts | 4 + unitTests/bin/cliOperations.test.js | 297 +++++- unitTests/components/deployOperations.test.js | 458 -------- .../components/deployPhaseOperations.test.js | 980 ++++++++++++++++++ .../components/deployPhaseValidators.test.js | 58 ++ unitTests/components/deployValidators.test.js | 21 - utility/hdbTerms.ts | 3 + utility/operation_authorization.ts | 4 + 14 files changed, 2147 insertions(+), 572 deletions(-) delete mode 100644 unitTests/components/deployOperations.test.js create mode 100644 unitTests/components/deployPhaseOperations.test.js create mode 100644 unitTests/components/deployPhaseValidators.test.js delete mode 100644 unitTests/components/deployValidators.test.js diff --git a/DESIGN.md b/DESIGN.md index b271656fc6..83a4cd4bdf 100644 --- a/DESIGN.md +++ b/DESIGN.md @@ -193,32 +193,50 @@ A package-manager timeout must not release this lock while npm descendants are s Boot's `harper-application-lock.json` records an application configuration only after preparation fulfills. Recording at queue time would make a failed install look complete and suppress its retry on the next boot. -## Component deploys build off to the side and swap atomically - -`deploy_component` prepares the candidate under `.deploy-staging//`, runs -extract + `npm install` there, validates that it loads, and only then renames it into the live path — -committing root config and `harper-application-lock.json` in the same compensating transaction. The -live component keeps serving through the slow, failure-prone work (git clone, registry install), and -go-live is one atomic rename. A fetch or install failure leaves the running component untouched -instead of half-replaced in place. - -This is **per node**. The operation replicates as a whole, exactly as the single-phase deploy always -did, and each node performs its own staged build. There is deliberately no cluster-wide barrier here: -nothing orders activation across nodes, so two deploys originated concurrently on different nodes can -still end with different versions live. That ordering guarantee — a leader or a monotonic activation -epoch — is tracked in #2294, and byte-identical package resolution across nodes in #2295. Both are -protocol additions that need real multi-node verification, so they are deliberately not attempted -here. - -The deployment row carries the activation specification (package/install settings, routing, credential -references, `force`). It is not a coordination channel between nodes; it exists so the deploy is -observable, so the payload has a durable home, and — the load-bearing part — so startup -reconciliation can reconcile root config for an activation that was interrupted mid-swap. A row -without `activation_spec` cannot be recovered, which is why it is written before the build starts. - -Startup reconciliation is the recovery record: it settles interrupted extractions, finishes or undoes -interrupted reverts, rolls an interrupted activation forward, and fails a component closed rather than -loading a live tree whose durable configuration disagrees with it. +## Two-phase component deploys roll forward from a durable activation claim + +With `system` database replication enabled, `deploy_component` first prepares the candidate under +`.deploy-staging//` on every node. The deployment row carries the complete, +immutable activation specification (package/install settings, routing, credential references, and +`force`); activate-by-id never accepts replacements for those fields. After every stage response, the +origin durably checkpoints the row as `staged`. Activation claims that row as `activating` while holding +the same per-component filesystem lock, then swaps the candidate into the live path and commits root +config plus `harper-application-lock.json` as one compensating transaction. This ordering is the +recovery record: startup preserves `staged` candidates, deletes terminal/orphan candidates, and rolls +an `activating` candidate forward before loading apps. + +**The origin owns the row; a peer's claim is local.** Peers run the same validation before their swap +but do NOT write the deployment row (`claimStagedDeployment(..., { persist: false })`). The row is +replicated, so a peer writing it would make N+1 writers of one key — and under replication lag a +peer's `activating` can land _after_ the origin has written `success`, leaving a converged deploy in a +non-terminal status it never leaves. What a peer needs from claiming is mutual exclusion against +another activation of the same component, and the per-component filesystem lock it already holds +provides exactly that. + +**The separated phases require deployment tracking.** `DeploymentRecorder` is deliberately tolerant of +a missing `hdb_deployment` table — tracking is observability for a one-shot deploy. It is not +observability for `activate: false` or activate-by-id, which coordinate _through_ the row: without it +a stage would return a deployment_id nothing could resolve, so the stage would report success and be +permanently unactivatable. Those requests, and an explicit `two_phase: true`, fail with 503 when the +table is absent. The DEFAULT path is deliberately left alone: a default two-phase deploy on an +untracked node fails safely, because peers cannot find the row, the barrier never clears, and nothing +activates — the origin does not go live. Routing it to one-shot instead would be the unsafe choice, +since that path consumes the multipart stream into its own blob and strips `req.payload` before +replication, leaving peers with neither replayable bytes nor a row while the origin was already live. +`two_phase: false` remains the explicit single-phase escape hatch. + +Peer stage/activate/restart messages use the distinct authenticated `component_deploy_phase` operation. +An older peer therefore rejects the unknown operation instead of ignoring a phase marker and deploying +the staged build live. Public `_phase`/`_deploymentId` fields are rejected; the latter remains accepted +only on the authenticated legacy one-shot replication path. Restart is gated until activation responses +have settled. A partial activation is reported as split-node state and recovered by staging and activating +a known-good build, or rolled back explicitly with `revert_component`, which is addressed rather than a +toggle (see "Reversibility" below) so a retry after a lost response cannot reverse the recovery. There is +deliberately no AUTOMATIC rollback (`revert_on_failure` is rejected): once any node is past the barrier, +"this peer reported failed" does not mean "this peer did not activate" — a peer can complete its swap and +then fail the persistent work that follows — so auto-reverting the failed peers would roll an untouched +node an extra version back and split the cluster three ways. `deployment_stagingRetention_maxCount` bounds resting staged +trees per component and payload retention is pruned in the same row-aware lifecycle. ## Peer-side deploy_component payload read: retryable blob stalls and `Readable.from()` cancellation @@ -502,24 +520,93 @@ this fix doesn't attempt to solve. `deploy_component`/`package_component` still declared entry points (`jsResource`/`graphqlSchema`) survived extraction — a truncation from some other future cause would still report success silently; that's a deferred, separate fix. -## Staged deploy: build aside, then swap (`components/Application.ts`, `components/operations.js`) - -`deploy_component` builds the incoming version — download/`npm pack` (incl. a git clone), extract, -`npm install` — into a hidden staging directory, validates that it loads, then atomically renames it -into the live component path. The live component keeps serving throughout, and a fetch or install -failure leaves it untouched rather than half-replaced in place. The request/response contract is -unchanged; only the SSE phase names differ (`stage`/`activate` vs the old `prepare`/`replicate`). - -Each node does this for itself: the operation replicates as a whole, as it always did. There is no -cluster-wide barrier and no private peer operation — ordering activation across nodes (#2294) and -guaranteeing every node staged the same bytes (#2295) are protocol additions that need real -multi-node verification, and are deliberately out of scope here. - -**Why staging lives under the components root.** Go-live is `rename(stagingDir, liveDir)`, atomic only -when both share a filesystem. `os.tmpdir()` is frequently a different mount → `EXDEV` → a slow -recursive copy at exactly the moment an instant swap is wanted. So staging is a hidden directory under -the components root: same volume, dot-prefixed so the loader ignores it, and not the watched base of -any component's watcher, so building there fires no restart-on-change events. +## Two-phase deploy: stage then activate (`components/Application.ts`, `components/operations.js`) + +`deploy_component` runs internally as two replicated phases so a cluster deploy is all-or-nothing at +the point of go-live. **Phase 1 (stage)** builds the incoming version — download/`npm pack` (incl. a +git clone), extract, `npm install` — into a hidden staging directory on every node. **Phase 2 +(activate)** atomically renames the staged copy into the live component path and restarts. The origin +stages locally, **waits for every node to report a successful stage before any node activates** +(`ignore_replication_errors` opts out of the barrier), then activates. If a node can't fetch the +package or fails `npm install`, it fails during staging while the live component is still untouched _on +every node_ — where the old one-shot path could leave a peer half-installed after other peers had +already restarted onto the new code. The request/response contract is unchanged; only the SSE phase +names differ (`stage`/`activate` vs the old `prepare`/`replicate`). `two_phase: false` forces the +legacy one-shot path. + +**There is only one public operation — `deploy_component`.** The two phases are NOT separate public +operations. The peer fan-out is the distinct trusted operation `component_deploy_phase`, which carries +the phase and the deployment id and is reachable only on the replication path — authorization is +carried in AsyncLocalStorage (`isOperationAuthorizationBypassed`), not on the request, so it cannot be +invoked over HTTP with ordinary credentials. Public `_phase` and `_deploymentId` fields are rejected +outright; an older peer therefore rejects an unknown operation rather than misreading a phase marker as +a one-shot deploy. A public call runs the origin orchestrator. Two public properties expose the phases when an operator wants them separated +(e.g. pre-stage the cluster now, flip later — or a CI-stages / approver-activates split): `activate: +false` stages cluster-wide and stops, returning the `deployment_id` in a `staged` state; passing that +`deployment_id` back to `deploy_component` (with no new payload) activates the already-staged build. +This was a deliberate API-surface choice (harper#1849 review): peer fan-out needs a wire format, not +two extra public ops, and folding the phases into `deploy_component` keeps the surface at one op while +the convergence properties cover the stage-now/activate-later use case. (`revert_component` stays a +distinct public op — it is a rollback, not a deploy phase.) + +**Scope of the barrier's guarantee: fetch + install, not load.** The cluster-wide "nobody activates +until everybody staged" guarantee covers the download/`npm pack` and `npm install` steps — the slow, +failure-prone work. The pre-go-live component _load_ check (`loadValidateComponent`, which surfaces a +component that installs cleanly but throws at load) runs during stage on the origin and on any node +whose stage executes on a worker (e.g. the op-API worker for an `activate: false` stage), but it is a +no-op on the main thread — and replicated peer stage executions run on the main thread +(`replicateOperation` → `sendOperationToNode` execute there), where app code deliberately isn't +loaded. So a load-time-only fault on a peer is not caught by the barrier; it surfaces at +activate/restart like any other. Gating load-time faults cluster-wide would require dispatching the +throwaway load to a worker on each peer during stage — a possible follow-up, not done here. + +The staging directory (`.deploy-staging//`) lives **under the components root**, +not in `os.tmpdir()`, even though its contents are transient. This is deliberate and load-bearing: +the go-live step is `rename(stagingDir, liveDir)`, which is only atomic when both paths share a +filesystem. `os.tmpdir()` is frequently a different mount (tmpfs, a separate volume); a cross-device +rename throws `EXDEV` and Node has no atomic fallback — you'd be back to a slow recursive copy at the +exact moment you want the swap to be instantaneous, reintroducing the downtime window the split +exists to remove. The leading dot keeps `loadComponentDirectories` from loading it as a phantom +component, and it is **not** the watched base of any component's file watcher (those are rooted at +each live component dir, `EntryHandler`/`deriveCommonPatternBase`) — so building here fires no +restart-on-change events and needs no `deploy:start` watcher suppression. That suppression is now +scoped to `activateStagedApplication`, the only phase that writes the live path. Staging is deterministic +from the deployment id precisely so the activate phase (a separate `component_deploy_phase` invocation +on peers) can reconstruct the same path the stage built — +peers build a fresh `Application` per phase invocation, so there is no shared in-memory handle to rely +on. The deployment id sits ABOVE the component name (`…//`, not +`…//`) for two reasons: the leaf directory's basename is then the real component +name, which the pre-go-live validation load needs (`componentLoader` keys the `ApplicationScope` and +status registry off `basename(componentDirectory)`, so a UUID leaf would register the throwaway load +under a bogus name); and each deploy gets its own parent directory, so a parallel or queued deploy of +the same component can never share a directory or have its staged build swept by another's cleanup. +`extractApplication`/`installApplication` build into `application.buildDirPath`, which defaults +to the live dir (`dirPath`) — this is what keeps the legacy one-shot path, boot-time +`installApplications`, and the direct `extractApplication` callers unchanged — and is repointed at +the staging dir only for the duration of a stage. + +Two-phase requires the `system` database to be replicated on the origin (`isSystemDatabaseReplicated`), +since the `hdb_deployment` row's `payload_blob` is how peers fetch the tarball and correlate the two +phases by deployment id. When `system` is excluded from a narrow `REPLICATION_DATABASES`, or the +caller passes `two_phase: false`, or the invocation is a peer replaying a one-shot deploy, +`deploy_component` falls back to `deployComponentOneShot` (the previous behavior, preserved verbatim). +Cross-version skew is a non-issue by policy — a cluster stays in lockstep on its Harper version, so +every node understands the `_phase`-tagged `deploy_component` fan-out — which is why there is no +capability negotiation on it. + +**Replicator contract this rides on (`harper-pro/replication/replicator.ts`).** +`server.replication.replicateOperation(op, {onPeerResult})` fans `op` to every node in `server.nodes` +in parallel, setting `op.replicated = false` on the copy it sends so a peer never re-fans (the deploy +handlers additionally detect a replicated execution by the presence of `_deploymentId` — always set on +the sub-operations — and run the peer stage/activate work off the `_phase` marker without re-fanning). Per-peer failures never throw — `sendOperationToNode` rejections are caught and +surface as `{status:'failed', reason, node}` entries in the returned `replicated[]` array and via +`onPeerResult`, which is exactly the shape `DeploymentRecorder.normalizePeerResult` consumes. Peers +authenticate node-to-node by TLS certificate, and the receive side runs the op via +`server.operation(data, {user}, !isAuthorizedNode)` — for a trusted cluster node the authorize flag is +`false`, so a replicated super-user op skips the permission gate. That is why the `_phase`-tagged +`deploy_component` fan-out and `revert_component` (registered with the same `permission(true, [])`, +dispatched by `operation` name) replicate without an `hdb_user`, identically to the long-proven +one-shot `deploy_component` fan-out. **Reversibility: retained previous + `revert_component`.** `activateStagedApplication` does not discard the tree its swap displaced — it retains it as `.deploy-previous/`, evicting the older one so @@ -568,10 +655,10 @@ swap had not yet placed the reverted-to version (the revert is undone and can be retained-previous path when the swap completed and only the retain step was lost. With both slots occupied the swap finished and the holding tree is residue, so it is discarded. -**Staged-build retention.** A successful deploy consumes its staged build immediately (activation -renames it live), so nothing normally accumulates. What does accumulate is the residue of deploys that -never reached activation — a failed install, a crash between stage and swap — each leaving -`.deploy-staging//` behind. `stageApplication` bounds this: after a successful +**Staged-build retention.** A full deploy consumes its staged build immediately (activate renames it +live), so the only builds that accumulate are `activate: false` stage-and-stops that are never +activated — each leaves `.deploy-staging//` in place so a later +`deploy_component({deployment_id})` can activate it. `stageApplication` bounds this: after a successful stage it evicts the oldest not-yet-activated staged builds for that component beyond `deployment_stagingRetention_maxCount` (default 5, `pruneStagedBuilds`), always keeping the just-staged one and the newest N−1 by mtime. Eviction is best-effort (`allSettled`, trace-logged) but awaited so diff --git a/bin/cliOperations.ts b/bin/cliOperations.ts index 97eaa0e5b4..3b11784f0f 100644 --- a/bin/cliOperations.ts +++ b/bin/cliOperations.ts @@ -31,9 +31,17 @@ const OP_ALIASES = { package: 'package_component', }; +// `stage` and `activate` are sugar over `deploy_component` — there are no separate stage/activate +// operations to find. const OP_VERB_PROPS: Record> = { - // `harper revert` uploads nothing: the version it activates is already on disk. `_cliVerb` is a - // CLI-internal marker (stripped before the request is sent) that drives the missing-target guard below. + stage: { operation: 'deploy_component', activate: false, _cliVerb: 'stage' }, + // `_cliVerb` is a CLI-internal marker (stripped before the request is sent) so verbRequirementError + // can enforce that `harper activate` carries a deployment_id — without it, deploy_component's generic + // "no deployment_id → full deploy" fallback would silently build a brand-new deploy from the CWD. + // It also tells the staged-deploy capability probe that this invocation needs two-phase support. + activate: { operation: 'deploy_component', _cliVerb: 'activate' }, + // `harper revert` uploads nothing: the version it activates is already on every node. `_cliVerb` + // here only drives the missing-target guard below. revert: { operation: 'revert_component', _cliVerb: 'revert' }, }; @@ -41,6 +49,9 @@ const OP_VERB_PROPS: Record> = { // verb invoked it). Returns an error message, or null when the request is fine. Pure + exported so it // is unit-testable without the network/process-exit machinery in cliOperations. function verbRequirementError(req: any): string | null { + if (req._cliVerb === 'activate' && !req.deployment_id) { + return '`harper activate` requires a deployment_id from a prior `harper stage` — usage: harper activate project= deployment_id='; + } // revert_component requires its target so a retry can't toggle the rejected release back in. Caught // here too, so the CLI names the flag instead of surfacing a raw validation error. if (req._cliVerb === 'revert' && !req.to_deployment_id) { @@ -186,6 +197,22 @@ async function targetSupportsStreamingDeploy(options: any): Promise { } } +async function targetSupportsStagedDeploy(options: any): Promise { + try { + const probeOptions = { + ...options, + headers: { ...options.headers, Accept: 'application/json' }, + timeout: CLI_OPERATION_TIMEOUT_MS, + }; + delete probeOptions.streamResponse; + const response = await httpRequest(probeOptions, { operation: 'registration_info' }); + if (response.statusCode !== 200 || !response.body) return false; + return JSON.parse(response.body)?.capabilities?.componentDeployTwoPhase === 1; + } catch { + return false; + } +} + // Wraps the local packaging stream so an fs error while tar'ing up the payload (e.g. a file // vanishing after the pre-deploy scan, or a permissions failure reading the project tree) // surfaces as a descriptive packaging error instead of a raw fs error code. Without this, an @@ -604,6 +631,12 @@ const prepareRevert = async (req) => { const PREPARE_OPERATION: any = { revert_component: prepareRevert, deploy_component: async (req) => { + // `harper activate deployment_id=` takes an already-staged build live, so there is nothing to + // package — but it still needs the CWD project default every deploy-family verb gets. + if (req.deployment_id) { + req.project ||= directoryProjectName(process.cwd()); + return; + } if (req.package) { return; } @@ -921,6 +954,17 @@ async function cliOperations(req: any, skipResponseLog = false) { let options: any, target: any; try { ({ options, target } = await resolveRequestOptions(req)); + // Staged (two-phase) deploy controls must never reach a server that doesn't understand them: an + // older target ignores `activate: false`/`deployment_id` and deploys LIVE cluster-wide instead — + // the opposite of the operator's intent, silently. Probe before packaging so the refusal costs + // nothing. Local (domain-socket) calls hit this same build, so no probe is needed there. + const requestsStagedDeploy = + req._cliVerb !== undefined || req.activate === false || req.deployment_id !== undefined || req.two_phase === true; + if (target && requestsStagedDeploy && !(await targetSupportsStagedDeploy(options))) { + throw new Error( + `Target Harper does not advertise staged-deploy support; refusing the request because an older server could deploy it live` + ); + } delete req._cliVerb; await PREPARE_OPERATION[req.operation]?.(req); // Streaming deploy (multipart upload + SSE progress) only works against >= 5.1 servers. diff --git a/components/operations.js b/components/operations.js index 15074ea3c6..553f593a38 100644 --- a/components/operations.js +++ b/components/operations.js @@ -1,13 +1,13 @@ 'use strict'; const path = require('node:path'); +const { isDeepStrictEqual } = require('node:util'); const { isMainThread } = require('node:worker_threads'); const fs = require('fs-extra'); const fg = require('fast-glob'); const normalize = require('normalize-path'); const validator = require('./operationsValidation.js'); const log = require('../utility/logging/harper_logger.ts'); -const { randomUUID } = require('node:crypto'); const hdbTerms = require('../utility/hdbTerms.ts'); const env = require('../utility/environment/environmentManager.ts'); const configUtils = require('../config/configUtils.ts'); @@ -35,7 +35,10 @@ const { prepareApplication, stageApplication, revertApplication, + stagedApplicationPath, + hasCompleteStagedApplication, activateStagedApplication, + discardStagedApplication, discardProjectStagedApplications, discardProjectActivationArtifacts, updateApplicationLockEntry, @@ -43,6 +46,7 @@ const { createApplicationActivationTransaction, createApplicationConfigTransaction, getRevertTarget, + getStagingRetentionMaxCount, dropComponentDirectory, discardRetainedPrevious, ASIDE_STAGING_DIR, @@ -55,7 +59,12 @@ const { server } = require('../server/Server.ts'); const { DeploymentRecorder, awaitDeploymentRow, + getDeploymentRow, + markDeploymentTerminal, + recordDeploymentPeers, + claimStagedDeployment, isDeploymentTrackingAvailable, + expireOldStagedDeployments, invalidateProjectStagedDeployments, pruneProjectPayloads, readPayloadBlobWithRetry, @@ -530,6 +539,33 @@ async function deployComponent(req) { HTTP_STATUS_CODES.BAD_REQUEST ); } + // The separated phases coordinate THROUGH the row, so without it `activate: false` hands back a + // deployment_id nothing can resolve — a stage that reports success and can never be activated. Those + // requests, and an explicit `two_phase: true`, fail up front. + // + // The DEFAULT path is deliberately left alone. A default two-phase deploy on an untracked node fails + // safely: peers cannot find the row, so the barrier never clears and nothing activates — the origin + // does not go live. Routing it to one-shot instead would be the unsafe choice, because that path + // consumes the multipart stream into its own blob and strips `req.payload` before replication, so + // peers would get neither replayable bytes nor a row while the origin was already live. + if ( + !isReplicatedExecution && + !isDeploymentTrackingAvailable() && + (requestedSeparatedPhase || req.two_phase === true) + ) { + throw handleHDBError( + new Error(), + `${requestedSeparatedPhase ? 'activate:false and deployment_id' : 'two_phase:true'} coordinate through ` + + `the '${hdbTerms.SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME}' table, which is not available on this ` + + `node. Deploy with two_phase:false to use the legacy single-phase path.`, + HTTP_STATUS_CODES.SERVICE_UNAVAILABLE + ); + } + if (req.replicated !== false && !isReplicatedExecution && req.two_phase !== false && systemReplicated) { + if (req.deployment_id) return deployComponentActivateExisting(req); + return deployComponentTwoPhase(req); + } + // Ingest any provided credential token into the secrets store so the credential lives as // replicated ciphertext (reference, not embed); already-reference entries pass through, and with // no custody a literal token stays as a transient, this-node-only fallback (#1158). Peers @@ -578,6 +614,9 @@ function markRestartRequiredForDeploy(application) { async function deployComponentOneShot(req, credentialReferences, isReplicatedExecution) { const { resolveCredentials } = require('./secretOperations.ts'); + // Write to root config if the request contains a package identifier + if (req.package) await writeComponentRootConfig(req, credentialReferences); + // Create a hdb_deployment row up front so the deploy is observable and auditable even if the CLI // disconnects. The row also holds the payload in a Blob attribute, which doubles as the source for // peer replication and (later) rollback. Only the origin node records — peers replaying the @@ -587,15 +626,10 @@ async function deployComponentOneShot(req, credentialReferences, isReplicatedExe // still gets phase events for non-SSE deploys. const emitter = isReplicatedExecution ? null : (req.progress ?? new ProgressEmitter()); if (emitter && !req.progress) req.progress = emitter; - // Built before the recorder so the row can carry it. Startup reconciliation reads - // `row.activation_spec` to reconcile root config after an interrupted activation, so a row without - // it cannot be recovered. - const activationSpec = activationSpecFromRequest(req, credentialReferences); const recorder = isReplicatedExecution ? null : await DeploymentRecorder.create({ project: req.project, - activation_spec: activationSpec, package_identifier: req.package ?? null, user: req.hdb_user?.username, restart_mode: req.restart === 'rolling' ? 'rolling' : req.restart ? 'immediate' : null, @@ -611,11 +645,6 @@ async function deployComponentOneShot(req, credentialReferences, isReplicatedExe const emit = (event, data) => emitter?.emit(event, data); - // Protected core component names. This used to sit inside the root-config write, which the staged - // path replaced with the activation transaction — so it is asserted here explicitly, before any work. - // Package deploys only, exactly as before: a payload deploy has always been allowed to use the name. - if (req.package) assertNotProtectedCoreComponent(req.project, req.force); - // The payload-via-replicated-row path depends on `system` actually replicating on this node. const systemReplicated = isSystemDatabaseReplicated(); @@ -640,30 +669,12 @@ async function deployComponentOneShot(req, credentialReferences, isReplicatedExe if (credentialReferences.length) req.credentials = credentialReferences; else delete req.credentials; - // Build off to the side, then swap. The live component keeps serving through the slow, - // failure-prone work — git clone, registry install — and go-live is one atomic rename, so a fetch - // or install failure leaves the running component untouched instead of half-replaced in place. - // This is per node: the operation replicates as a whole and each node does its own staged build. - // There is deliberately no cluster-wide barrier here; ordering activation across nodes is #2294. - const deploymentId = recorder?.deploymentId ?? req._deploymentId ?? randomUUID(); - emit('phase', { phase: 'stage', status: 'start' }); - const stagingDirPath = await stageApplication(application, deploymentId); - emit('phase', { phase: 'stage', status: 'done' }); - - // Validate the STAGED tree, so a component that installs cleanly but throws at load never reaches - // the live path (throwaway scopes; see loadValidateComponent). - await loadValidateComponent({ dirPath: stagingDirPath, emit }); + emit('phase', { phase: 'prepare', status: 'start' }); + await prepareApplication(application); + emit('phase', { phase: 'prepare', status: 'done' }); - // Root config and `harper-application-lock.json` commit inside the swap, so the directory and the - // durable state describing it move together and compensate together. - const configTransaction = await createApplicationActivationTransaction(req.project, activationSpec); - emit('phase', { phase: 'activate', status: 'start' }); - await activateStagedApplication(application, deploymentId, { - beforeCommit: () => configTransaction.commit(), - onRollback: () => configTransaction.rollback(), - activationSpec, - }); - emit('phase', { phase: 'activate', status: 'done' }); + // Load the component to surface load-time errors early (throwaway scopes; see loadValidateComponent). + await loadValidateComponent({ dirPath: application.dirPath, emit }); const rollingRestart = req.restart === 'rolling'; // if doing a rolling restart set restart to false so that other nodes don't also restart. @@ -806,12 +817,494 @@ function activationSpecFromRequest(req, credentialReferences) { }; } +function applicationFromSpec(spec, payload, resolvedCredentials, installCapture, emit) { + return new Application({ + name: spec.project, + payload, + packageIdentifier: spec.package ?? undefined, + install: { + command: spec.install_command ?? undefined, + timeout: spec.install_timeout ?? undefined, + allowInstallScripts: spec.install_allow_scripts ?? undefined, + }, + credentials: resolvedCredentials, + onInstallLine: (manager, stream, line) => { + installCapture?.push(manager, stream, line); + emit?.('install', { manager, stream, line }); + }, + }); +} + +function failedPeerResults(results) { + return (results ?? []).filter((result) => result?.status === 'failed' || result?.error || result?.reason); +} + function describePeerFailures(failed) { return failed .map((peer) => `${peer.node ?? 'unknown'} (${peer.error?.message ?? peer.reason ?? 'unknown error'})`) .join(', '); } +function buildPhaseOperation(phase, deploymentId, project, activationSpec, extra = {}) { + return { + operation: hdbTerms.OPERATIONS_ENUM.COMPONENT_DEPLOY_PHASE, + phase, + deployment_id: deploymentId, + project, + activation_spec: activationSpec, + ...extra, + }; +} + +async function resolveSpecCredentials(spec, waitMs = 0) { + const { resolveCredentials } = require('./secretOperations.ts'); + return resolveCredentials(spec.credentials ?? [], spec.project, { waitMs }); +} + +function assertStoredActivationSpec(row, deploymentId, project, spec, allowedStatuses) { + if ( + !row || + row.project !== project || + !allowedStatuses.includes(row.status) || + !isDeepStrictEqual(row.activation_spec, spec) + ) { + throw new ServerError( + `Deployment '${deploymentId}' does not have the expected immutable activation specification for '${project}'` + ); + } +} + +async function sourceStagedPayload(deploymentId, spec, timeoutMs) { + const deadline = Date.now() + timeoutMs; + const row = await awaitDeploymentRow(deploymentId, { timeoutMs, requirePayload: !spec.package }); + assertStoredActivationSpec(row, deploymentId, spec.project, spec, ['pending', 'staging', 'staged', 'activating']); + if (spec.package) return undefined; + return readPayloadBlobWithRetry(() => row.payload_blob.stream(), { + timeoutMs: Math.max(0, deadline - Date.now()), + }); +} + +async function discardDeploymentEverywhere(project, deploymentId, activationSpec) { + const componentPath = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), project); + await discardStagedApplication(componentPath, deploymentId).catch(() => {}); + await server.replication + .replicateOperation(buildPhaseOperation('discard', deploymentId, project, activationSpec)) + .catch(() => {}); +} + +async function pruneStagedDeploymentArtifacts(project, activationSpec, keepDeploymentId) { + const expired = await expireOldStagedDeployments(project, getStagingRetentionMaxCount(), keepDeploymentId); + for (const deploymentId of expired) await discardDeploymentEverywhere(project, deploymentId, activationSpec); +} + +async function restartActivatedComponent(req, deploymentId, project, activationSpec, emit) { + if (req.restart === true) { + emit('phase', { phase: 'restart', status: 'start' }); + const restartResponse = await server.replication.replicateOperation( + buildPhaseOperation('restart', deploymentId, project, activationSpec, { + deployment_timeout: req.deployment_timeout, + }) + ); + const failed = failedPeerResults(restartResponse?.replicated); + manageThreads.restartWorkers('http'); + emit('phase', { phase: 'restart', status: 'done' }); + return { restartMessage: `, restarting Harper`, replicated: restartResponse?.replicated, failedPeers: failed }; + } + if (req.restart === 'rolling') { + const serverUtilities = require('../server/serverHelpers/serverUtilities.ts'); + emit('phase', { phase: 'restart', status: 'start' }); + const jobResponse = await serverUtilities.executeJob({ + operation: 'restart_service', + service: 'http', + replicated: true, + }); + emit('phase', { phase: 'restart', status: 'done' }); + return { restartMessage: `, restarting Harper`, restartJobId: jobResponse.job_id, failedPeers: [] }; + } + return { restartMessage: '', failedPeers: [] }; +} + +async function deployComponentTwoPhase(req) { + assertNotProtectedCoreComponent(req.project, req.force); + const { ingestCredentials, resolveCredentials } = require('./secretOperations.ts'); + req.credentials = await ingestCredentials(req, req.credentials, req.project); + const credentialReferences = (req.credentials ?? []).filter((entry) => entry?.secret !== undefined); + const activationSpec = activationSpecFromRequest(req, credentialReferences); + const emitter = req.progress ?? new ProgressEmitter(); + const emit = (event, data) => emitter.emit(event, data); + const installCapture = createInstallCapture(); + const recorder = await DeploymentRecorder.create({ + project: req.project, + package_identifier: req.package ?? null, + user: req.hdb_user?.username, + restart_mode: req.restart === 'rolling' ? 'rolling' : req.restart ? 'immediate' : null, + credentials: credentialReferences.length ? credentialReferences : null, + activation_spec: activationSpec, + emitter, + }); + let application; + let activationCommitted = false; + let activationBarrierPassed = false; + try { + let payload = req.payload; + if (req.payload != null) { + await recorder.ingestPayload(req.payload); + payload = recorder.row.payload_blob.stream(); + } + const resolvedCredentials = await resolveCredentials(req.credentials, req.project); + application = applicationFromSpec(activationSpec, payload, resolvedCredentials, installCapture, emit); + if (credentialReferences.length) req.credentials = credentialReferences; + else delete req.credentials; + delete req.progress; + delete req.payload; + + emit('phase', { phase: 'stage', status: 'start' }); + const stagedPath = await stageApplication(application, recorder.deploymentId); + await loadValidateComponent({ dirPath: stagedPath, emit }); + recorder.seal(); + const stageResponse = await server.replication.replicateOperation( + buildPhaseOperation('stage', recorder.deploymentId, req.project, activationSpec, { + deployment_timeout: req.deployment_timeout, + }), + { + onPeerResult: (result) => { + recorder.recordPeer(result); + emit('peer', result); + }, + } + ); + if (stageResponse?.replicated) recorder.recordPeers(stageResponse.replicated); + emit('phase', { phase: 'stage', status: 'done' }); + const stageFailures = recorder.getFailedPeers(); + if (stageFailures.length && !req.ignore_replication_errors) { + await discardDeploymentEverywhere(req.project, recorder.deploymentId, activationSpec); + throw new ServerError( + `Component '${req.project}' failed to stage on ${stageFailures.length} peer node(s): ` + + `${describePeerFailures(stageFailures)}. No node was activated and the live component is unchanged.` + ); + } + await recorder.checkpoint('staged', 'staged'); + + // Settled here, on every successful origin stage, so the ROWS match the staged DIRECTORIES that + // `stageApplication` has already pruned under the same policy. Gating this on `activate: false` left + // a full deploy evicting trees whose rows still read `staged` cluster-wide — list_deployments + // offering a deployment_id whose tree is gone, and a different one per node under clock skew. + // + // Retention is count-based, so N concurrent deploys of one project can still race for N slots: a + // candidate that is `staged` but whose deploy has not yet activated can be expired by a newer + // stage. That is inherent to the policy and already true of the directory prune this mirrors — + // only rows strictly older than the current request are eligible, which is what keeps a + // just-returned deployment safe. + await pruneStagedDeploymentArtifacts(req.project, activationSpec, recorder.deploymentId).catch((error) => + log.warn('Failed to prune expired staged deployments', error) + ); + + if (req.activate === false) { + emit('phase', { phase: 'staged', status: 'done' }); + await recorder.finish('staged'); + await pruneProjectPayloads(req.project, getPayloadRetentionMaxCount()).catch((error) => + log.warn('Failed to prune staged deployment payloads', error) + ); + return { + message: `Staged component: ${req.project}`, + project: req.project, + staged: true, + deployment_id: recorder.deploymentId, + replicated: stageResponse?.replicated, + ...(stageFailures.length ? { failed_peers: stageFailures } : {}), + }; + } + + const configTransaction = await createApplicationActivationTransaction(req.project, activationSpec); + await activateStagedApplication(application, recorder.deploymentId, { + beforeSwap: async () => { + await claimStagedDeployment(recorder.deploymentId, req.project); + emit('phase', { phase: 'activate', status: 'start' }); + }, + beforeCommit: () => configTransaction.commit(), + onRollback: () => configTransaction.rollback(), + activationSpec, + }); + activationCommitted = true; + const activateResponse = await server.replication.replicateOperation( + buildPhaseOperation('activate', recorder.deploymentId, req.project, activationSpec, { + deployment_timeout: req.deployment_timeout, + }), + { + onPeerResult: (result) => { + recorder.recordPeer(result); + emit('peer', result); + }, + } + ); + if (activateResponse?.replicated) recorder.recordPeers(activateResponse.replicated); + emit('phase', { phase: 'activate', status: 'done' }); + const activateFailures = recorder.getFailedPeers(); + if (activateFailures.length && !req.ignore_replication_errors) { + throw new ServerError( + `Component '${req.project}' activated on only part of the cluster. Split nodes: ` + + `${describePeerFailures(activateFailures)}. Roll forward by staging and activating a known-good deployment.` + ); + } + activationBarrierPassed = activateFailures.length === 0; + if (!req.restart) markRestartRequiredForDeploy(application); + const restart = await restartActivatedComponent(req, recorder.deploymentId, req.project, activationSpec, emit); + if (restart.failedPeers.length) recorder.recordPeers(restart.failedPeers); + if (restart.failedPeers.length && !req.ignore_replication_errors) { + throw new ServerError( + `Component '${req.project}' activated, but restart failed on: ${describePeerFailures(restart.failedPeers)}` + ); + } + emit('phase', { phase: 'success', status: 'done' }); + maybeReclaimPayload(recorder, emit); + await recorder.finish('success'); + await pruneProjectPayloads(req.project, getPayloadRetentionMaxCount()).catch((error) => + log.warn('Failed to prune deployment payloads', error) + ); + return { + message: `Successfully deployed: ${req.project}${restart.restartMessage}`, + project: req.project, + deployment_id: recorder.deploymentId, + replicated: activateResponse?.replicated, + ...(restart.restartJobId ? { restartJobId: restart.restartJobId } : {}), + ...(recorder.getFailedPeers().length ? { failed_peers: recorder.getFailedPeers() } : {}), + }; + } catch (error) { + if (application && !activationCommitted) { + await discardStagedApplication(application.dirPath, recorder.deploymentId).catch(() => {}); + } + const capture = installCapture.snapshot(); + const failedPeers = recorder.getFailedPeers(); + const message = error?.message ?? String(error); + const structured = { + error: message, + phase: recorder.row.phase, + deployment_id: recorder.deploymentId, + ...(capture.lines.length ? { install_output: capture } : {}), + ...(failedPeers.length ? { failed_peers: failedPeers } : {}), + }; + emit('error', { + message, + code: error?.statusCode ?? error?.code, + phase: recorder.row.phase, + deployment_id: recorder.deploymentId, + install_output: capture.lines.length ? capture : undefined, + failed_peers: failedPeers.length ? failedPeers : undefined, + }); + await recorder + .finish(activationBarrierPassed ? 'success' : activationCommitted ? 'activating' : 'failed', error) + .catch((finishError) => log.warn('Failed to record two-phase deployment failure', finishError)); + const outError = new ServerError(message, error?.statusCode); + outError.http_resp_msg = structured; + throw outError; + } +} + +const ACTIVATION_FRESH_FIELDS = [ + 'payload', + 'package', + 'install_command', + 'install_timeout', + 'install_allow_scripts', + 'urlPath', + 'host', + 'credentials', + 'force', + 'activate', + 'two_phase', +]; + +function assertActivationRequestIsReferenceOnly(req) { + const supplied = ACTIVATION_FRESH_FIELDS.filter((field) => req[field] !== undefined); + if (supplied.length) { + throw handleHDBError( + new Error(), + `deployment_id activation uses the immutable staged configuration; remove: ${supplied.join(', ')}`, + HTTP_STATUS_CODES.BAD_REQUEST + ); + } +} + +async function deployComponentActivateExisting(req) { + assertActivationRequestIsReferenceOnly(req); + const row = await getDeploymentRow(req.deployment_id); + if (!row) throw handleHDBError(new Error(), `No deployment found with id '${req.deployment_id}'`, 404); + if (row.project !== req.project || row.status !== 'staged' || !row.activation_spec) { + throw handleHDBError( + new Error(), + `Deployment '${req.deployment_id}' is not a staged deployment for component '${req.project}'`, + HTTP_STATUS_CODES.CONFLICT + ); + } + const spec = row.activation_spec; + assertNotProtectedCoreComponent(spec.project, spec.force); + const emitter = req.progress ?? new ProgressEmitter(); + const emit = (event, data) => emitter.emit(event, data); + const componentPath = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), req.project); + let application; + const stagedPath = stagedApplicationPath(componentPath, req.deployment_id); + if (await hasCompleteStagedApplication(stagedPath)) { + application = applicationFromSpec(spec, undefined, undefined, null, emit); + await loadValidateComponent({ dirPath: stagedPath, emit }); + } else { + const timeoutMs = coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS); + const payload = await sourceStagedPayload(req.deployment_id, spec, timeoutMs); + const credentials = await resolveSpecCredentials(spec, timeoutMs); + application = applicationFromSpec(spec, payload, credentials, createInstallCapture(), emit); + const rebuiltPath = await stageApplication(application, req.deployment_id); + await loadValidateComponent({ dirPath: rebuiltPath, emit }); + } + const configTransaction = await createApplicationActivationTransaction(req.project, spec); + emit('phase', { phase: 'activate', status: 'start' }); + let claimed = false; + try { + await activateStagedApplication(application, req.deployment_id, { + beforeSwap: async () => { + await claimStagedDeployment(req.deployment_id, req.project); + claimed = true; + }, + beforeCommit: () => configTransaction.commit(), + onRollback: () => configTransaction.rollback(), + activationSpec: spec, + }); + } catch (error) { + if (claimed) await markDeploymentTerminal(req.deployment_id, 'staged').catch(() => {}); + throw error; + } + let settledPeers = []; + let activationBarrierPassed = false; + try { + const peerResults = []; + const activateResponse = await server.replication.replicateOperation( + buildPhaseOperation('activate', req.deployment_id, req.project, spec, { + deployment_timeout: req.deployment_timeout, + }), + { + onPeerResult: (result) => { + peerResults.push(result); + emit('peer', result); + }, + } + ); + settledPeers = Array.isArray(activateResponse?.replicated) ? activateResponse.replicated : peerResults; + await recordDeploymentPeers(req.deployment_id, settledPeers); + emit('phase', { phase: 'activate', status: 'done' }); + const failed = failedPeerResults(settledPeers); + if (failed.length && !req.ignore_replication_errors) { + throw new ServerError( + `Component '${req.project}' activated on only part of the cluster. Split nodes: ` + + `${describePeerFailures(failed)}. Roll forward by staging and activating a known-good deployment.` + ); + } + activationBarrierPassed = failed.length === 0; + if (!req.restart) markRestartRequiredForDeploy(application); + const restart = await restartActivatedComponent(req, req.deployment_id, req.project, spec, emit); + if (restart.failedPeers.length) { + settledPeers = [...settledPeers, ...restart.failedPeers]; + await recordDeploymentPeers(req.deployment_id, restart.failedPeers); + } + if (restart.failedPeers.length && !req.ignore_replication_errors) { + throw new ServerError( + `Component '${req.project}' activated, but restart failed on: ${describePeerFailures(restart.failedPeers)}` + ); + } + await markDeploymentTerminal(req.deployment_id, 'success'); + await maybeReclaimFinishedPayload(req.deployment_id, emit); + await pruneProjectPayloads(req.project, getPayloadRetentionMaxCount()).catch((error) => + log.warn('Failed to prune activated deployment payloads', error) + ); + return { + message: `Activated component: ${req.project}${restart.restartMessage}`, + project: req.project, + activated: true, + deployment_id: req.deployment_id, + replicated: activateResponse?.replicated, + ...(restart.restartJobId ? { restartJobId: restart.restartJobId } : {}), + ...(failedPeerResults(settledPeers).length ? { failed_peers: failedPeerResults(settledPeers) } : {}), + }; + } catch (error) { + await markDeploymentTerminal(req.deployment_id, activationBarrierPassed ? 'success' : 'activating', error).catch( + () => {} + ); + throw error; + } +} + +async function componentDeployPhase(req) { + if (!isTrustedReplicatedOperation(req)) { + throw handleHDBError(new Error(), 'component_deploy_phase is restricted to authenticated cluster peers', 403); + } + const validation = validator.componentDeployPhaseValidator({ + phase: req.phase, + deployment_id: req.deployment_id, + project: req.project, + activation_spec: req.activation_spec, + deployment_timeout: req.deployment_timeout, + }); + if (validation) throw handleHDBError(validation, validation.message, HTTP_STATUS_CODES.BAD_REQUEST); + const spec = req.activation_spec; + if (!spec || spec.project !== req.project) { + throw handleHDBError(new Error(), 'Invalid immutable activation specification', HTTP_STATUS_CODES.BAD_REQUEST); + } + const componentPath = path.join(configUtils.getConfigPath(hdbTerms.CONFIG_PARAMS.COMPONENTSROOT), req.project); + if (req.phase === 'discard') { + await discardStagedApplication(componentPath, req.deployment_id); + return { message: `Discarded staged component: ${req.project}` }; + } + if (req.phase === 'restart') { + const row = await getDeploymentRow(req.deployment_id); + assertStoredActivationSpec(row, req.deployment_id, req.project, spec, [ + 'pending', + 'staging', + 'staged', + 'activating', + ]); + manageThreads.restartWorkers('http'); + return { message: `Restarting component runtime for: ${req.project}` }; + } + if (req.phase === 'stage') { + assertNotProtectedCoreComponent(req.project, spec.force); + const timeoutMs = coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS); + const payload = await sourceStagedPayload(req.deployment_id, spec, timeoutMs); + const credentials = await resolveSpecCredentials(spec, timeoutMs); + const application = applicationFromSpec(spec, payload, credentials, createInstallCapture()); + const stagedPath = await stageApplication(application, req.deployment_id); + await loadValidateComponent({ dirPath: stagedPath, emit: () => {} }); + return { message: `Staged component: ${req.project}`, project: req.project, staged: true }; + } + const row = await getDeploymentRow(req.deployment_id); + assertStoredActivationSpec(row, req.deployment_id, req.project, spec, ['pending', 'staging', 'staged', 'activating']); + let application; + const stagedPath = stagedApplicationPath(componentPath, req.deployment_id); + if (await hasCompleteStagedApplication(stagedPath)) { + application = applicationFromSpec(spec, undefined, undefined, null); + await loadValidateComponent({ dirPath: stagedPath, emit: () => {} }); + } else { + const timeoutMs = coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS); + const payload = await sourceStagedPayload(req.deployment_id, spec, timeoutMs); + const credentials = await resolveSpecCredentials(spec, timeoutMs); + application = applicationFromSpec(spec, payload, credentials, createInstallCapture()); + const rebuiltPath = await stageApplication(application, req.deployment_id); + await loadValidateComponent({ dirPath: rebuiltPath, emit: () => {} }); + } + const configTransaction = await createApplicationActivationTransaction(req.project, spec); + await activateStagedApplication(application, req.deployment_id, { + beforeSwap: async () => { + await claimStagedDeployment(req.deployment_id, req.project, { + allowActivating: true, + // Peer: validate, do not write. The origin owns this row; see claimStagedDeployment. + persist: false, + waitForStagedMs: coerceTimeoutMs(req.deployment_timeout, DEFAULT_AWAIT_ROW_TIMEOUT_MS), + }); + }, + beforeCommit: () => configTransaction.commit(), + onRollback: () => configTransaction.rollback(), + activationSpec: spec, + }); + markRestartRequiredForDeploy(application); + return { message: `Activated component: ${req.project}`, project: req.project, activated: true }; +} + function isTrustedReplicatedOperation(req) { const user = req.hdb_user; return ( @@ -995,6 +1488,31 @@ function assertNotProtectedCoreComponent(project, force) { } } +// Persist a `package` deploy's entry into root config so every cold install (reboot, new peer, +// rollback) reinstalls it. In two-phase this runs at activation, once the bits are staged everywhere. +async function writeComponentRootConfig(req, credentialReferences) { + assertNotProtectedCoreComponent(req.project, req.force); + const applicationConfig = { package: req.package }; + // Avoid writing an empty `install:` block + if (req.install_command || req.install_timeout || req.install_allow_scripts !== undefined) { + applicationConfig.install = { + command: req.install_command, + timeout: req.install_timeout, + allowInstallScripts: req.install_allow_scripts, + }; + } + if (req.urlPath !== undefined) applicationConfig.urlPath = req.urlPath; + if (req.host !== undefined) applicationConfig.host = req.host; + // Persist credential references (never tokens) so every cold install of this component — reboot, new + // peer, revert — re-resolves the credential from the store. + if (credentialReferences.length) applicationConfig.credentials = credentialReferences; + // Same critical section the activation transaction uses. `addConfig` is a read-modify-write of a + // file whose entries are per-project, so a one-shot deploy running unlocked can write back a + // document it parsed before a concurrent activation or drop committed, resurrecting or dropping + // that project's entry. + await withPersistentStateLock(() => configUtils.addConfig(req.project, applicationConfig)); +} + // Resolve the tarball to extract from. On the origin, tee req.payload into the row's blob (the // channel peers read from) and re-source extraction from the persisted blob. On a peer replaying a // deploy without a payload, read the tarball from the replicated row's blob (bounded wait). @@ -1121,6 +1639,29 @@ function maybeReclaimPayload(recorder, emit) { } } +async function maybeReclaimFinishedPayload(deploymentId, emit) { + try { + const row = await getDeploymentRow(deploymentId); + const payloadSize = row?.payload_size; + const retentionMaxSize = getPayloadRetentionMaxSize(); + if ( + typeof payloadSize !== 'number' || + payloadSize <= retentionMaxSize || + failedPeerResults(row.peer_results).length > 0 || + row.payload_blob == null + ) { + return; + } + const { handleDeleteDeploymentPayload } = require('./deploymentOperations.ts'); + const result = await handleDeleteDeploymentPayload({ deployment_id: deploymentId }); + if (result.freed_bytes > 0) { + emit('payload_dropped', { payload_size: result.freed_bytes, max_size: retentionMaxSize }); + } + } catch (error) { + log.warn(`Failed to reclaim payload for activated deployment '${deploymentId}'`, error); + } +} + /** * Count-based payload retention (deployment_payloadRetention_maxCount): after a successful deploy, keep * only the newest N stored payloads for this project and drop the rest. Where the size-based reclaim @@ -1637,6 +2178,7 @@ exports.addComponent = addComponent; exports.dropCustomFunctionProject = dropCustomFunctionProject; exports.packageComponent = packageComponent; exports.deployComponent = deployComponent; +exports.componentDeployPhase = componentDeployPhase; exports.revertComponent = revertComponent; exports.getComponents = getComponents; exports.getComponentFile = getComponentFile; diff --git a/components/operationsValidation.js b/components/operationsValidation.js index a37a36d93f..3a9077b55e 100644 --- a/components/operationsValidation.js +++ b/components/operationsValidation.js @@ -32,6 +32,7 @@ module.exports = { dropCustomFunctionProjectValidator, packageComponentValidator, deployComponentValidator, + componentDeployPhaseValidator, revertComponentValidator, setComponentFileValidator, getComponentFileValidator, @@ -524,14 +525,31 @@ function deployComponentValidator(req) { deployment_timeout: Joi.number().min(0).optional(), force: Joi.boolean().optional(), ignore_replication_errors: Joi.boolean().optional(), - // Automatic rollback is deliberately NOT offered. A node that reports `failed` may still have - // completed its swap and then failed the work that follows, so auto-reverting "the nodes that - // failed" can roll an untouched node an extra version back. Roll back explicitly, by target, with - // revert_component. See DESIGN.md, "Partial activation". + // Stop after the incoming version is staged and verified cluster-wide, without going live. Returns + // the staged deployment_id; a later deploy_component with that deployment_id activates it. Defaults + // to true (full stage + activate). + activate: Joi.boolean().optional(), + // Activate a previously-staged deployment (from an `activate: false` stage) cluster-wide. Same safe + // charset as `project` because it becomes a staging-dir path segment (`.deploy-staging//`) + // — a `../` value would otherwise resolve the staging source outside `.deploy-staging`. + deployment_id: Joi.string().pattern(DEPLOYMENT_ID_REGEX).optional().messages({ + 'string.pattern.base': `'deployment_id' must be a UUID`, + }), + // Automatic rollback of a partially-activated cluster is deliberately NOT offered. Once any node + // has crossed the activation barrier there is no sound way to know which peers actually swapped: + // a peer can complete its swap and then fail the persistent work that follows, so it reports + // `failed` while running the new version. Auto-reverting "the peers that failed" would then roll + // an untouched node an extra version back and leave the cluster split three ways. A partial + // activation therefore stays visibly `activating` and is rolled forward (or reverted explicitly, + // by target, with revert_component). See DESIGN.md, "Partial activation". revert_on_failure: Joi.any().forbidden().messages({ - 'any.unknown': `'revert_on_failure' is not supported; roll back explicitly with revert_component`, + 'any.unknown': `'revert_on_failure' is not supported; recover a partial activation by rolling forward, or roll back explicitly with revert_component`, }), + // Opt out of the two-phase (stage-then-activate) deploy and use the legacy one-shot path instead. + // Defaults to two-phase. + two_phase: Joi.boolean().optional(), _deploymentId: Joi.any().forbidden(), + _phase: Joi.any().forbidden(), urlPath: URL_PATH_SCHEMA, host: HOST_SCHEMA, // Deploy credentials. Each entry is npm registry auth (`registry`) or git host auth (`host`, @@ -546,6 +564,24 @@ function deployComponentValidator(req) { return validator.validateBySchema(req, deployProjSchema); } +/** Validate the path- and state-selecting fields on the authenticated peer-only deploy operation. */ +function componentDeployPhaseValidator(req) { + const phaseSchema = Joi.object({ + phase: Joi.string().valid('stage', 'activate', 'discard', 'restart').required(), + deployment_id: Joi.string().pattern(DEPLOYMENT_ID_REGEX).required().messages({ + 'string.pattern.base': `'deployment_id' must be a UUID`, + }), + project: Joi.string() + .pattern(PROJECT_FILE_NAME_REGEX) + .required() + .messages({ 'string.pattern.base': HDB_ERROR_MSGS.BAD_PROJECT_NAME }), + activation_spec: Joi.object().required(), + deployment_timeout: Joi.number().min(0).optional(), + }).unknown(false); + + return validator.validateBySchema(req, phaseSchema); +} + /** * Validate revert_component requests — swap a component's live version back to its retained previous * version. There are no build inputs: nothing is fetched, resolved or installed, because the bytes diff --git a/integrationTests/deploy/deploy-tracking-peer-branch.test.ts b/integrationTests/deploy/deploy-tracking-peer-branch.test.ts index 49fe7fa5af..db46396151 100644 --- a/integrationTests/deploy/deploy-tracking-peer-branch.test.ts +++ b/integrationTests/deploy/deploy-tracking-peer-branch.test.ts @@ -1,7 +1,7 @@ /** * Deployment tracking — peer-operation authorization boundary. * - * In a real multi-node deploy, the origin replicates the whole `deploy_component` + * In a real multi-node deploy, the origin sends a private `component_deploy_phase` * operation and the peer reads the tarball from the replicated * `hdb_deployment.payload_blob` row. The authorization-bypass context that admits that * operation exists only around trusted replication dispatch, so an HTTP caller must not @@ -160,6 +160,6 @@ suite('Deployment tracking — peer-operation authorization boundary', (ctx: Con strictEqual(response.status, 400, `internal marker should be rejected; got: ${response.rawText}`); strictEqual(response.body.error, "'_deploymentId' is not allowed"); }); - // Peer behavior is the replicated `deploy_component` operation itself, so there is no private - // peer entry point to drive here; the three-node harper-pro suite covers real replication. + // `component_deploy_phase` is trusted-peer-only and unreachable over HTTP, so a peer-branch + // end-to-end test has no entry point here; the three-node harper-pro suite covers it. }); diff --git a/resources/registrationDeprecated.ts b/resources/registrationDeprecated.ts index 39b140b176..173cbaf90f 100644 --- a/resources/registrationDeprecated.ts +++ b/resources/registrationDeprecated.ts @@ -4,5 +4,6 @@ export function getRegistrationInfo() { return { version: packageJson.version, deprecated: true, + capabilities: { componentDeployTwoPhase: 1 }, }; } diff --git a/server/serverHelpers/serverUtilities.ts b/server/serverHelpers/serverUtilities.ts index 7ee13eb542..0cc2240f85 100644 --- a/server/serverHelpers/serverUtilities.ts +++ b/server/serverHelpers/serverUtilities.ts @@ -605,6 +605,10 @@ function initializeOperationFunctionMap(): Map { }); }); + describe('deploy_component cross-version compatibility', () => { + const target = 'https://example.com:9925/'; + let originalPackageDirectory; + let originalScan; + + beforeEach(() => { + saveCredentials(target, { operation_token: 'valid-token', refresh_token: 'refresh-token' }); + tokenAuthModule.isJWTExpired = () => false; + originalPackageDirectory = packageComponentModule.packageDirectory; + originalScan = packageComponentModule.scanPackageDirectory; + }); + + afterEach(() => { + packageComponentModule.packageDirectory = originalPackageDirectory; + packageComponentModule.scanPackageDirectory = originalScan; + }); + + // Streams an SSE `done` event so the modern (>= 5.1) deploy path can read its result. + const sseDoneResponse = (result) => + Object.assign(Readable.from([`event: done\ndata: ${JSON.stringify({ result })}\n\n`]), { + statusCode: 200, + headers: { 'content-type': 'text/event-stream' }, + }); + + it('downgrades a package deploy to legacy JSON when the target is < 5.1', async () => { + const calls = []; + commonUtilsModule.httpRequest = async (options, req) => { + calls.push({ options, req }); + if (req.operation === 'registration_info') { + return { statusCode: 200, body: JSON.stringify({ version: '5.0.31' }) }; + } + return { statusCode: 200, body: JSON.stringify({ message: 'Successfully deployed', success: true }) }; + }; + + const result = await cliOperationsModule.cliOperations( + { operation: 'deploy_component', package: '@scope/widget', project: 'widget', target: 'example.com' }, + true + ); + + // Probe first, then the deploy. + assert.strictEqual(calls[0].req.operation, 'registration_info'); + assert.strictEqual(calls[0].options.streamResponse, undefined); + const deploy = calls[1]; + // No streaming negotiation against the old server. + assert.strictEqual(deploy.options.headers.Accept, undefined); + assert.strictEqual(deploy.options.streamResponse, undefined); + // Body is a plain JSON object, not a multipart stream, and carries no transport-only fields. + assert.strictEqual(typeof deploy.req.pipe, 'undefined'); + assert.strictEqual(deploy.req.operation, 'deploy_component'); + assert.strictEqual(deploy.req._legacyDeploy, undefined); + assert.strictEqual(deploy.req._multipart, undefined); + assert.strictEqual(result.success, true); + }); + + it('downgrades a directory deploy to a CBOR binary payload when the target is < 5.1', async () => { + const fakeTarball = Buffer.from('fake-tarball-bytes'); + packageComponentModule.scanPackageDirectory = async () => ({ + totalSize: fakeTarball.length, + danglingSymlinks: [], + }); + packageComponentModule.packageDirectory = async () => fakeTarball; + + const calls = []; + commonUtilsModule.httpRequest = async (options, req) => { + calls.push({ options, req }); + if (req.operation === 'registration_info') { + return { statusCode: 200, body: JSON.stringify({ version: '5.0.31' }) }; + } + return { statusCode: 200, body: JSON.stringify({ message: 'Successfully deployed', success: true }) }; + }; + + const result = await cliOperationsModule.cliOperations( + { operation: 'deploy_component', project: 'widget', target: 'example.com' }, + true + ); + + const deploy = calls[1]; + assert.strictEqual(deploy.options.streamResponse, undefined); + // Multipart was abandoned in favor of a CBOR body carrying the tarball as a + // native binary Buffer — the transport pre-5.1 servers decode directly. + assert.strictEqual(deploy.options.headers['Content-Type'], 'application/cbor'); + assert.ok(Buffer.isBuffer(deploy.req), 'CBOR body should be a Buffer'); + const decoded = decodeCbor(deploy.req); + assert.ok(Buffer.isBuffer(decoded.payload), 'decoded payload should be a Buffer'); + assert.strictEqual(decoded.payload.toString(), 'fake-tarball-bytes'); + assert.strictEqual(decoded.operation, 'deploy_component'); + assert.strictEqual(decoded._multipart, undefined); + assert.strictEqual(result.success, true); + }); + + it('keeps the streaming deploy path when the target is >= 5.1', async () => { + const calls = []; + commonUtilsModule.httpRequest = async (options, req) => { + calls.push({ options, req }); + if (req.operation === 'registration_info') { + return { statusCode: 200, body: JSON.stringify({ version: '5.1.7' }) }; + } + return sseDoneResponse({ message: 'Successfully deployed', success: true }); + }; + + const result = await cliOperationsModule.cliOperations( + { operation: 'deploy_component', package: '@scope/widget', project: 'widget', target: 'example.com' }, + true + ); + + const deploy = calls[1]; + assert.strictEqual(deploy.options.headers.Accept, 'text/event-stream'); + assert.strictEqual(deploy.options.streamResponse, true); + assert.strictEqual(result.success, true); + }); + + it('does not downgrade when the version probe fails (assumes modern)', async () => { + const calls = []; + commonUtilsModule.httpRequest = async (options, req) => { + calls.push({ options, req }); + if (req.operation === 'registration_info') { + return { statusCode: 404, body: 'not found' }; + } + return sseDoneResponse({ message: 'Successfully deployed', success: true }); + }; + + const result = await cliOperationsModule.cliOperations( + { operation: 'deploy_component', package: '@scope/widget', project: 'widget', target: 'example.com' }, + true + ); + + const deploy = calls[1]; + assert.strictEqual(deploy.options.headers.Accept, 'text/event-stream'); + assert.strictEqual(result.success, true); + }); + + it('fails closed on every staged-deploy control against a target without two-phase capability', async () => { + const originalExit = process.exit; + const originalConsoleError = console.error; + const errors = []; + const calls = []; + process.exit = (code) => { + throw new ProcessExitSignal(code); + }; + console.error = (...args) => errors.push(args.join(' ')); + commonUtilsModule.httpRequest = async (options, req) => { + calls.push({ options, req }); + return { statusCode: 200, body: JSON.stringify({ version: '5.1.7' }) }; + }; + try { + for (const request of [ + { package: '@scope/widget', activate: false, _cliVerb: 'stage' }, + { package: '@scope/widget', activate: false }, + { deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa' }, + { package: '@scope/widget', two_phase: true }, + ]) { + await assert.rejects( + cliOperationsModule.cliOperations( + { + operation: 'deploy_component', + project: 'widget', + target: 'example.com', + ...request, + }, + true + ), + ProcessExitSignal + ); + } + } finally { + process.exit = originalExit; + console.error = originalConsoleError; + } + + assert.deepStrictEqual( + calls.map(({ req }) => req.operation), + Array(4).fill('registration_info'), + 'only one capability probe per request reached the target' + ); + assert.match(errors.join('\n'), /does not advertise staged-deploy support/); + }); + + it('renders stage phase events and strips its CLI-only verb marker', async () => { + const calls = []; + const rendered = []; + const originalRenderEvent = DeployRenderer.prototype.renderEvent; + DeployRenderer.prototype.renderEvent = function (message) { + rendered.push(message.event); + }; + commonUtilsModule.httpRequest = async (options, req) => { + calls.push({ options, req }); + if (req.operation === 'registration_info') { + return { + statusCode: 200, + body: JSON.stringify({ + version: '5.2.0', + capabilities: { componentDeployTwoPhase: 1 }, + }), + }; + } + return Object.assign( + Readable.from([ + 'event: phase\ndata: {"phase":"stage","status":"start"}\n\n', + 'event: done\ndata: {"result":{"staged":true}}\n\n', + ]), + { statusCode: 200, headers: { 'content-type': 'text/event-stream' } } + ); + }; + let result; + try { + result = await cliOperationsModule.cliOperations( + { + operation: 'deploy_component', + project: 'widget', + package: '@scope/widget', + activate: false, + _cliVerb: 'stage', + target: 'example.com', + }, + true + ); + } finally { + DeployRenderer.prototype.renderEvent = originalRenderEvent; + } + + const deploy = calls.at(-1); + assert.strictEqual(deploy.req._cliVerb, undefined); + assert.strictEqual(deploy.req.activate, false); + assert.deepStrictEqual(rendered, ['phase', 'done']); + assert.strictEqual(result.staged, true); + }); + + it('defaults the activate project from the current directory', async () => { + const calls = []; + const projectDir = path.join(testDir, 'activate-project'); + fs.ensureDirSync(projectDir); + const priorCwd = process.cwd(); + commonUtilsModule.httpRequest = async (options, req) => { + calls.push({ options, req }); + if (req.operation === 'registration_info') { + return { + statusCode: 200, + body: JSON.stringify({ + version: '5.2.0', + capabilities: { componentDeployTwoPhase: 1 }, + }), + }; + } + return Object.assign(Readable.from(['event: done\ndata: {"result":{"activated":true}}\n\n']), { + statusCode: 200, + headers: { 'content-type': 'text/event-stream' }, + }); + }; + try { + process.chdir(projectDir); + await cliOperationsModule.cliOperations( + { + operation: 'deploy_component', + deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', + _cliVerb: 'activate', + target: 'example.com', + }, + true + ); + } finally { + process.chdir(priorCwd); + } + + assert.strictEqual(calls.at(-1).req.project, 'activate-project'); + }); + }); + describe('"Harper is not running" messaging (harper#658)', () => { const NOT_RUNNING_MESSAGE = 'Harper is not running. Use `harperdb run` (or `harperdb start`) to start it.'; @@ -1498,7 +1767,7 @@ describe('cliOperations', () => { }); }); -describe('harper revert CLI verb', () => { +describe('deploy CLI verbs (stage / activate fold into deploy_component)', () => { const { buildRequest, verbRequirementError } = cliOperationsModule; let savedArgv; beforeEach(() => { @@ -1508,6 +1777,32 @@ describe('harper revert CLI verb', () => { process.argv = savedArgv; }); + it('`stage` maps to deploy_component with activate:false', () => { + process.argv = ['node', 'harper', 'stage', 'project=my_app']; + const req = buildRequest(); + assert.strictEqual(req.operation, 'deploy_component'); + assert.strictEqual(req.activate, false); + }); + + it('`activate` with a deployment_id maps to deploy_component and passes the verb guard', () => { + process.argv = ['node', 'harper', 'activate', 'project=my_app', 'deployment_id=abc-123']; + const req = buildRequest(); + assert.strictEqual(req.operation, 'deploy_component'); + assert.strictEqual(req.deployment_id, 'abc-123'); + assert.strictEqual(verbRequirementError(req), null); + }); + + it('`activate` WITHOUT a deployment_id is rejected (would otherwise become a full deploy from the CWD)', () => { + process.argv = ['node', 'harper', 'activate', 'project=my_app']; + const req = buildRequest(); + assert.match(verbRequirementError(req), /deployment_id/); + }); + + it('verbRequirementError ignores non-activate deploys', () => { + assert.strictEqual(verbRequirementError({ operation: 'deploy_component' }), null); + assert.strictEqual(verbRequirementError({ operation: 'deploy_component', activate: false }), null); + }); + it('`revert` maps to revert_component and carries the verb marker', () => { // The marker has to survive buildRequest for the guard below to fire at all. `revert` deliberately // lives in OP_VERB_PROPS rather than OP_ALIASES: buildRequest checks the alias table FIRST, so an diff --git a/unitTests/components/deployOperations.test.js b/unitTests/components/deployOperations.test.js deleted file mode 100644 index b5ecce4710..0000000000 --- a/unitTests/components/deployOperations.test.js +++ /dev/null @@ -1,458 +0,0 @@ -'use strict'; - -const assert = require('node:assert'); -const fs = require('node:fs/promises'); -const { existsSync } = require('node:fs'); -const os = require('node:os'); -const path = require('node:path'); -const zlib = require('node:zlib'); -const { Readable } = require('node:stream'); -const tarfs = require('tar-fs'); - -const testUtils = require('../testUtils.js'); -testUtils.preTestPrep(); - -const { DEPLOY_STAGING_DIR, DEPLOY_ACTIVATION_DIR } = require('#src/components/Application'); -const operations = require('#src/components/operations'); -const { restartNeeded, resetRestartNeeded } = require('#src/components/requestRestart'); -const { server } = require('#src/server/Server'); -const { databases } = require('#src/resources/databases'); -const { SYSTEM_TABLE_NAMES, CONFIG_PARAMS } = require('#src/utility/hdbTerms'); -const { getConfigPath, readConfigFile } = require('#src/config/configUtils'); -const environment = require('#src/utility/environment/environmentManager'); -const { runWithOperationAuthorizationBypass } = require('#src/server/serverHelpers/operationAuthorizationState'); - -const COMPONENTS_ROOT = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); -const DEPLOYMENT_TABLE = SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME; - -function packDirectory(directory) { - return new Promise((resolve, reject) => { - const chunks = []; - tarfs - .pack(directory) - .pipe(zlib.createGzip()) - .on('data', (chunk) => chunks.push(chunk)) - .on('end', () => resolve(Buffer.concat(chunks))) - .on('error', reject); - }); -} - -async function makePayload(marker, version = marker, withNodeModules = true) { - const source = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-op-')); - await fs.writeFile(path.join(source, 'package.json'), JSON.stringify({ name: 'phase-op', version })); - await fs.writeFile(path.join(source, 'index.js'), `module.exports = ${JSON.stringify(marker)};\n`); - if (withNodeModules) await fs.mkdir(path.join(source, 'node_modules'), { recursive: true }); - const payload = await packDirectory(source); - await fs.rm(source, { recursive: true, force: true }); - return payload; -} - -describe('deploy_component staged deploy', function () { - this.timeout(30_000); - const rows = new Map(); - let priorTable; - let priorReplicate; - let priorSafeMode; - let sequence = 0; - const names = []; - - before(async () => { - priorSafeMode = process.env.HARPER_SAFE_MODE; - process.env.HARPER_SAFE_MODE = 'true'; - // The first component operation completes lazy server initialization, which replaces - // databases.system. Run it before installing the table seam used by these tests. - // - // One-shot on purpose: the separated phases now require deployment tracking, and the whole point - // of this warmup is that it runs BEFORE the table seam exists. `after` removes the component - // directory along with every other fixture name. - await operations.deployComponent({ - project: name(), - payload: await makePayload('warmup'), - restart: false, - }); - if (!databases.system) databases.system = {}; - priorTable = databases.system[DEPLOYMENT_TABLE]; - }); - - beforeEach(() => { - resetRestartNeeded(); - rows.clear(); - databases.system[DEPLOYMENT_TABLE] = { - async get(id) { - return rows.get(id); - }, - async put(row) { - rows.set(row.deployment_id, { ...row }); - }, - async patch(id, partial) { - const row = rows.get(id); - if (row) rows.set(id, { ...row, ...partial }); - }, - async *search(conditions = []) { - for (const row of rows.values()) { - if (conditions.every((condition) => row[condition.attribute] === condition.value)) yield row; - } - }, - }; - priorReplicate = server.replication.replicateOperation; - server.replication.replicateOperation = async () => ({ replicated: [] }); - }); - - afterEach(() => { - server.replication.replicateOperation = priorReplicate; - resetRestartNeeded(); - }); - - after(async () => { - if (priorSafeMode === undefined) delete process.env.HARPER_SAFE_MODE; - else process.env.HARPER_SAFE_MODE = priorSafeMode; - if (priorTable === undefined) delete databases.system[DEPLOYMENT_TABLE]; - else databases.system[DEPLOYMENT_TABLE] = priorTable; - for (const name of names) await fs.rm(path.join(COMPONENTS_ROOT, name), { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR), { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR), { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-aside'), { recursive: true, force: true }); - await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-previous'), { recursive: true, force: true }); - }); - - function name() { - const value = `phase_op_${process.pid}_${sequence++}`; - names.push(value); - return value; - } - - it('replicates the payload itself when there is no deployment table to carry it', async () => { - // The legacy single-phase path is the documented escape hatch, so it has to actually be safe. It - // normally strips `req.payload` because peers read the bytes from the row's payload_blob — but with - // no table there is no row, so stripping left peers holding a `_deploymentId`, no bytes and nothing - // to resolve, after this node was already live. Asserting on the REPLICATED REQUEST, not just on - // local activation: replication is stubbed here, so a local-only assertion proves nothing about peers. - const project = name(); - const priorTable = databases.system[DEPLOYMENT_TABLE]; - const priorReplicate = server.replication.replicateOperation; - const replicated = []; - server.replication.replicateOperation = async (op) => { - replicated.push(op); - return { replicated: [] }; - }; - delete databases.system[DEPLOYMENT_TABLE]; - try { - // A real Readable, not a reusable Buffer: ingest DRAINS the source, so a Buffer would hide the - // actual failure (peers receiving an exhausted stream / EOF). - const bytes = await makePayload('untracked-oneshot'); - await operations.deployComponent({ - project, - payload: Readable.from([bytes]), - restart: false, - }); - assert.match( - await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), - /untracked-oneshot/, - 'the component goes live locally' - ); - assert.strictEqual(replicated.length, 1, 'the deploy is replicated to peers'); - // Assert the BYTES, not merely that the property exists — presence of a spent stream is exactly - // the bug this covers. - const sent = replicated[0].payload; - assert.strictEqual(Buffer.isBuffer(sent), true, `peers must receive replayable bytes, got ${typeof sent}`); - assert.strictEqual(Buffer.compare(sent, bytes), 0, 'and the bytes must be the payload that was uploaded'); - } finally { - server.replication.replicateOperation = priorReplicate; - databases.system[DEPLOYMENT_TABLE] = priorTable; - } - }); - - it('normalizes a string install_allow_scripts rather than reading it as truthy', async () => { - // Joi coerces it, but validateBySchema discards `result.value`, so the raw string reaches the - // handler. `install_allow_scripts: 'false'` would then read as truthy and run package lifecycle - // scripts for a caller that explicitly disabled them — over multipart/form, where every value - // arrives as a string. - const project = name(); - const result = await operations.deployComponent({ - project, - payload: await makePayload('1.0.0'), - install_allow_scripts: 'false', - restart: false, - }); - - assert.ok(result, 'the deploy succeeds'); - assert.strictEqual( - rows.get(result.deployment_id).activation_spec.install_allow_scripts, - false, - 'and the activation spec records a real boolean, not the string' - ); - }); - - it('uses a no-custody literal registry token for the origin install without recording it', async () => { - const project = name(); - const token = 'transient-origin-token'; - const installCommand = - `node -e "const fs=require('fs');` + - `const value=fs.readFileSync(process.env.npm_config_userconfig||process.env.NPM_CONFIG_USERCONFIG,'utf8');` + - `if(!value.includes('//registry.example.com/:_authToken='))process.exit(7);` + - `fs.writeFileSync('credential-seen','yes')"`; - const result = await operations.deployComponent({ - project, - payload: await makePayload('credential-origin', '6.0.0', false), - install_command: installCommand, - credentials: [{ registry: 'https://registry.example.com', token }], - restart: false, - }); - - // The install ran in the staging directory; the marker it wrote travels with the tree through the - // swap, so it lands at the live path. - assert.strictEqual( - await fs.readFile(path.join(COMPONENTS_ROOT, project, 'credential-seen'), 'utf8'), - 'yes', - 'the install saw the credential, and its output was swapped in with the tree' - ); - assert.strictEqual(rows.get(result.deployment_id).activation_spec.credentials, null); - assert.doesNotMatch(JSON.stringify(rows.get(result.deployment_id)), new RegExp(token)); - }); - - it('stages and activates a full deploy before reporting success', async () => { - const project = name(); - const result = await operations.deployComponent({ - project, - payload: await makePayload('full-deploy', '6.0.0'), - }); - - assert.match(result.message, /Successfully deployed/); - assert.strictEqual(rows.get(result.deployment_id).status, 'success'); - assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /full-deploy/); - assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id)), false); - assert.strictEqual(restartNeeded(), true, 'a new component activated without restart requires one'); - }); - - // revert_component - - it('reverts the cluster to a named previous deployment and fans the target out to peers', async () => { - const project = name(); - const first = await operations.deployComponent({ project, payload: await makePayload('rev-v1', '1.0.0') }); - const second = await operations.deployComponent({ project, payload: await makePayload('rev-v2', '2.0.0') }); - const fanout = []; - server.replication.replicateOperation = async (op) => { - fanout.push(op); - return { replicated: [] }; - }; - - const result = await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); - - assert.strictEqual(result.reverted, true); - assert.strictEqual(result.to_deployment_id, first.deployment_id); - assert.strictEqual(result.from_deployment_id, second.deployment_id); - assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /rev-v1/); - assert.strictEqual(rows.get(result.deployment_id).status, 'rolled_back'); - assert.strictEqual( - rows.get(result.deployment_id).rollback_of, - second.deployment_id, - 'the audit row records which deployment the rollback took out of service' - ); - assert.strictEqual(fanout.length, 1, 'peers get the revert'); - assert.strictEqual(fanout[0].operation, 'revert_component'); - assert.strictEqual( - fanout[0].to_deployment_id, - first.deployment_id, - 'peers are told WHICH version to end on, so the fan-out is idempotent per node' - ); - }); - - it('is a no-op when the requested deployment is already live, so a retry is safe', async () => { - const project = name(); - const first = await operations.deployComponent({ project, payload: await makePayload('retry-v1', '1.0.0') }); - await operations.deployComponent({ project, payload: await makePayload('retry-v2', '2.0.0') }); - await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); - - // The caller lost the first response and retried the identical request. - const retry = await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); - - assert.strictEqual(retry.reverted, false); - assert.match(retry.message, /already running/); - assert.match( - await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), - /retry-v1/, - 'a retried revert must not toggle the rejected version back in' - ); - }); - - it('rejects a revert with no target, and one whose target is not retained', async () => { - const project = name(); - await operations.deployComponent({ project, payload: await makePayload('target-v1', '1.0.0') }); - await operations.deployComponent({ project, payload: await makePayload('target-v2', '2.0.0') }); - - await assert.rejects( - () => operations.revertComponent({ project }), - /to_deployment_id/, - 'the target is mandatory — that is what makes a retry safe' - ); - await assert.rejects( - () => operations.revertComponent({ project, to_deployment_id: '00000000-0000-4000-8000-000000000000' }), - /neither the live version/, - 'only the immediately-previous version is retained' - ); - assert.match( - await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), - /target-v2/, - 'a refused revert changes nothing' - ); - }); - - it('takes the package reference out of root config when reverting away from a package deploy', async () => { - // Without this, installApplications() would reinstall the reverted-away package over the restored - // directory on the next cold start and silently undo the rollback. - const project = name(); - const packaged = await operations.deployComponent({ - project, - payload: await makePayload('cfg-packaged', '1.0.0'), - }); - // Stamp root config as a package deploy would have, then activate a payload version over it. - const { addConfig } = require('#src/config/configUtils'); - await addConfig(project, { package: 'some-pkg@1.0.0' }); - const plain = await operations.deployComponent({ project, payload: await makePayload('cfg-plain', '2.0.0') }); - assert.ok(plain.deployment_id); - - await operations.revertComponent({ project, to_deployment_id: packaged.deployment_id }); - - const entry = readConfigFile()?.[project]; - assert.strictEqual( - entry?.package, - undefined, - 'the reverted-to version had no package reference, so the stale one must be gone' - ); - }); - - it('reclaims an oversized payload only after a full two-phase activation succeeds', async () => { - const project = name(); - const priorMaxSize = environment.get(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE); - environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, 1); - try { - const result = await operations.deployComponent({ - project, - payload: await makePayload('reclaimed-full-deploy', '6.0.0'), - }); - - const row = rows.get(result.deployment_id); - assert.strictEqual(row.status, 'success'); - assert.strictEqual(row.payload_blob, null); - assert.ok(row.event_log.some((event) => event.event === 'payload_dropped')); - } finally { - environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, priorMaxSize); - } - }); - - it('accepts the legacy deployment row marker only from a trusted replicated operation', async () => { - const project = name(); - const payload = await makePayload('trusted-one-shot', '6.0.0'); - const result = await runWithOperationAuthorizationBypass(true, () => - operations.deployComponent({ - project, - payload, - _deploymentId: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', - replicated: false, - hdb_user: { name: 'cluster-peer' }, - }) - ); - - assert.match(result.message, /Successfully deployed/); - assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /trusted-one-shot/); - }); - - it('removes the root-config entry before it destroys anything, so a failed drop cannot resurrect', async () => { - // The resurrection vector is the root-config entry: installApplications() reinstalls whatever it - // names. Doing the persistent writes after the directory removal left a crash window where the - // live tree was gone but config still named the package, and the next boot brought the component - // back. Config-first inverts the failure: an interrupted drop is unfinished and re-runnable. - const project = name(); - const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-drop-crash-')); - const componentsRoot = path.join(configRoot, 'components'); - const configPath = path.join(configRoot, 'harper-config.yaml'); - const priorRootEnv = process.env.ROOTPATH; - const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); - const priorComponentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); - process.env.ROOTPATH = configRoot; - environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); - environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, componentsRoot); - await fs.writeFile(configPath, `rootPath: ${JSON.stringify(configRoot)}\n`); - try { - await fs.mkdir(path.join(componentsRoot, project), { recursive: true }); - await fs.writeFile(path.join(componentsRoot, project, 'index.js'), "module.exports = 'live';\n"); - await fs.appendFile(configPath, `${project}:\n package: some-package@1.0.0\n`); - // Make every aside-based teardown step fail: `.deploy-aside` occupied by a file is rejected as - // "not a directory", which is the closest deterministic stand-in for dying mid-teardown. - await fs.writeFile(path.join(componentsRoot, '.deploy-aside'), 'not a directory\n'); - - await assert.rejects(() => operations.dropComponent({ project })); - - assert.strictEqual( - (await fs.readFile(configPath, 'utf8')).includes(project), - false, - 'config is already clean, so the next boot cannot reinstall the component' - ); - assert.strictEqual( - existsSync(path.join(componentsRoot, project)), - true, - 'and the tree is still there: the drop is unfinished rather than finished-then-undone' - ); - } finally { - if (priorRootEnv === undefined) delete process.env.ROOTPATH; - else process.env.ROOTPATH = priorRootEnv; - environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); - environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, priorComponentsRoot); - await fs.rm(configRoot, { recursive: true, force: true }); - } - }); - - it('drop_component removes leftover recovery artifacts and the root-config entry', async () => { - const project = name(); - const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-drop-')); - const componentsRoot = path.join(configRoot, 'components'); - const configPath = path.join(configRoot, 'harper-config.yaml'); - const priorRootEnv = process.env.ROOTPATH; - const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); - const priorComponentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); - process.env.ROOTPATH = configRoot; - environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); - environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, componentsRoot); - await fs.writeFile(configPath, `rootPath: ${JSON.stringify(configRoot)}\n`); - try { - const staged = await operations.deployComponent({ - project, - payload: await makePayload('drop-stage', '6.0.0'), - activate: false, - }); - const activationPath = path.join(componentsRoot, DEPLOY_ACTIVATION_DIR, project, 'interrupted'); - await fs.mkdir(activationPath, { recursive: true }); - await fs.writeFile( - path.join(configRoot, 'harper-application-lock.json'), - JSON.stringify({ applications: { [project]: { package: 'stale-package' } } }) - ); - // A root-config entry is what makes a dropped component come back: installApplications() reads - // it on the next boot and reinstalls the package. Removing it and the application-lock entry as - // two separate writes meant a crash or a failed second write left this behind. - await fs.appendFile(configPath, `${project}:\n package: stale-package\n`); - - await operations.dropComponent({ project }); - - const deploymentStagePath = path.join(componentsRoot, DEPLOY_STAGING_DIR, staged.deployment_id); - assert.strictEqual( - existsSync(deploymentStagePath), - false, - `staged deployment directory still contains: ${await fs.readdir(deploymentStagePath).catch(() => [])}` - ); - assert.strictEqual(existsSync(path.join(componentsRoot, DEPLOY_ACTIVATION_DIR, project)), false); - const applicationLock = JSON.parse(await fs.readFile(path.join(configRoot, 'harper-application-lock.json'))); - assert.strictEqual(applicationLock.applications[project], undefined); - assert.strictEqual( - (await fs.readFile(configPath, 'utf8')).includes(project), - false, - 'the root-config entry is removed in the same step, so the next boot cannot reinstall the drop' - ); - } finally { - if (priorRootEnv === undefined) delete process.env.ROOTPATH; - else process.env.ROOTPATH = priorRootEnv; - environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); - environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, priorComponentsRoot); - await fs.rm(configRoot, { recursive: true, force: true }); - } - }); -}); diff --git a/unitTests/components/deployPhaseOperations.test.js b/unitTests/components/deployPhaseOperations.test.js new file mode 100644 index 0000000000..78d0087087 --- /dev/null +++ b/unitTests/components/deployPhaseOperations.test.js @@ -0,0 +1,980 @@ +'use strict'; + +const assert = require('node:assert'); +const fs = require('node:fs/promises'); +const { existsSync } = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const zlib = require('node:zlib'); +const { Readable } = require('node:stream'); +const tarfs = require('tar-fs'); + +const testUtils = require('../testUtils.js'); +testUtils.preTestPrep(); + +const operations = require('#src/components/operations'); +const { DEPLOY_STAGING_DIR, DEPLOY_ACTIVATION_DIR, discardStagedApplication } = require('#src/components/Application'); +const { restartNeeded, resetRestartNeeded } = require('#src/components/requestRestart'); +const { server } = require('#src/server/Server'); +const { databases } = require('#src/resources/databases'); +const { SYSTEM_TABLE_NAMES, CONFIG_PARAMS } = require('#src/utility/hdbTerms'); +const { getConfigPath, readConfigFile } = require('#src/config/configUtils'); +const environment = require('#src/utility/environment/environmentManager'); +const { runWithOperationAuthorizationBypass } = require('#src/server/serverHelpers/operationAuthorizationState'); +const manageThreads = require('#src/server/threads/manageThreads'); + +const COMPONENTS_ROOT = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); +const DEPLOYMENT_TABLE = SYSTEM_TABLE_NAMES.DEPLOYMENT_TABLE_NAME; + +function packDirectory(directory) { + return new Promise((resolve, reject) => { + const chunks = []; + tarfs + .pack(directory) + .pipe(zlib.createGzip()) + .on('data', (chunk) => chunks.push(chunk)) + .on('end', () => resolve(Buffer.concat(chunks))) + .on('error', reject); + }); +} + +async function makePayload(marker, version = marker, withNodeModules = true) { + const source = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-op-')); + await fs.writeFile(path.join(source, 'package.json'), JSON.stringify({ name: 'phase-op', version })); + await fs.writeFile(path.join(source, 'index.js'), `module.exports = ${JSON.stringify(marker)};\n`); + if (withNodeModules) await fs.mkdir(path.join(source, 'node_modules'), { recursive: true }); + const payload = await packDirectory(source); + await fs.rm(source, { recursive: true, force: true }); + return payload; +} + +describe('deploy_component two-phase orchestration', function () { + this.timeout(30_000); + const rows = new Map(); + let priorTable; + let priorReplicate; + let priorSafeMode; + let sequence = 0; + const names = []; + + before(async () => { + priorSafeMode = process.env.HARPER_SAFE_MODE; + process.env.HARPER_SAFE_MODE = 'true'; + // The first component operation completes lazy server initialization, which replaces + // databases.system. Run it before installing the table seam used by these tests. + // + // One-shot on purpose: the separated phases now require deployment tracking, and the whole point + // of this warmup is that it runs BEFORE the table seam exists. `after` removes the component + // directory along with every other fixture name. + await operations.deployComponent({ + project: name(), + payload: await makePayload('warmup'), + two_phase: false, + restart: false, + }); + if (!databases.system) databases.system = {}; + priorTable = databases.system[DEPLOYMENT_TABLE]; + }); + + beforeEach(() => { + resetRestartNeeded(); + rows.clear(); + databases.system[DEPLOYMENT_TABLE] = { + async get(id) { + return rows.get(id); + }, + async put(row) { + rows.set(row.deployment_id, { ...row }); + }, + async patch(id, partial) { + const row = rows.get(id); + if (row) rows.set(id, { ...row, ...partial }); + }, + async *search(conditions = []) { + for (const row of rows.values()) { + if (conditions.every((condition) => row[condition.attribute] === condition.value)) yield row; + } + }, + }; + priorReplicate = server.replication.replicateOperation; + server.replication.replicateOperation = async () => ({ replicated: [] }); + }); + + afterEach(() => { + server.replication.replicateOperation = priorReplicate; + resetRestartNeeded(); + }); + + after(async () => { + if (priorSafeMode === undefined) delete process.env.HARPER_SAFE_MODE; + else process.env.HARPER_SAFE_MODE = priorSafeMode; + if (priorTable === undefined) delete databases.system[DEPLOYMENT_TABLE]; + else databases.system[DEPLOYMENT_TABLE] = priorTable; + for (const name of names) await fs.rm(path.join(COMPONENTS_ROOT, name), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, DEPLOY_ACTIVATION_DIR), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-aside'), { recursive: true, force: true }); + await fs.rm(path.join(COMPONENTS_ROOT, '.deploy-previous'), { recursive: true, force: true }); + }); + + function name() { + const value = `phase_op_${process.pid}_${sequence++}`; + names.push(value); + return value; + } + + it('refuses a separated-phase deploy when deployment tracking is unavailable', async () => { + // `DeploymentRecorder.put()` is tolerant by design, so without this guard `activate: false` returned + // a deployment_id that no row could resolve — a stage reporting success that nothing could ever + // activate. The separated phases coordinate through the row, so they have to require it. + const project = name(); + const payload = await makePayload('untracked'); + const priorTable = databases.system[DEPLOYMENT_TABLE]; + delete databases.system[DEPLOYMENT_TABLE]; + const unavailable = (error) => { + assert.strictEqual(error.statusCode, 503, `expected 503, got ${error.statusCode}: ${error.message}`); + assert.match(error.message, /coordinate through\n?.*hdb_deployment/s); + return true; + }; + try { + await assert.rejects(() => operations.deployComponent({ project, payload, activate: false }), unavailable); + await assert.rejects( + () => operations.deployComponent({ project, deployment_id: '00000000-0000-4000-8000-000000000000' }), + unavailable + ); + await assert.rejects(() => operations.deployComponent({ project, payload, two_phase: true }), unavailable); + } finally { + databases.system[DEPLOYMENT_TABLE] = priorTable; + } + }); + + it('replicates the payload itself when two_phase:false runs with no deployment table', async () => { + // The legacy single-phase path is the documented escape hatch, so it has to actually be safe. It + // normally strips `req.payload` because peers read the bytes from the row's payload_blob — but with + // no table there is no row, so stripping left peers holding a `_deploymentId`, no bytes and nothing + // to resolve, after this node was already live. Asserting on the REPLICATED REQUEST, not just on + // local activation: replication is stubbed here, so a local-only assertion proves nothing about peers. + const project = name(); + const priorTable = databases.system[DEPLOYMENT_TABLE]; + const priorReplicate = server.replication.replicateOperation; + const replicated = []; + server.replication.replicateOperation = async (op) => { + replicated.push(op); + return { replicated: [] }; + }; + delete databases.system[DEPLOYMENT_TABLE]; + try { + // A real Readable, not a reusable Buffer: ingest DRAINS the source, so a Buffer would hide the + // actual failure (peers receiving an exhausted stream / EOF). + const bytes = await makePayload('untracked-oneshot'); + await operations.deployComponent({ + project, + payload: Readable.from([bytes]), + two_phase: false, + restart: false, + }); + assert.match( + await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), + /untracked-oneshot/, + 'the component goes live locally' + ); + assert.strictEqual(replicated.length, 1, 'the deploy is replicated to peers'); + // Assert the BYTES, not merely that the property exists — presence of a spent stream is exactly + // the bug this covers. + const sent = replicated[0].payload; + assert.strictEqual(Buffer.isBuffer(sent), true, `peers must receive replayable bytes, got ${typeof sent}`); + assert.strictEqual(Buffer.compare(sent, bytes), 0, 'and the bytes must be the payload that was uploaded'); + } finally { + server.replication.replicateOperation = priorReplicate; + databases.system[DEPLOYMENT_TABLE] = priorTable; + } + }); + + it('settles superseded staged rows on a full deploy, not only on stage-and-stop', async () => { + // Staged DIRECTORIES are pruned on every stage, but the row half used to run only on the + // `activate: false` return. A full deploy therefore evicted trees while their rows still read + // `staged` cluster-wide — list_deployments offering a deployment_id that a later activate cannot + // use, and under clock skew a different one on each node. + const project = name(); + const priorMax = environment.get(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT); + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, 1); + try { + const first = await operations.deployComponent({ + project, + payload: await makePayload('1.0.0'), + activate: false, + }); + assert.strictEqual(rows.get(first.deployment_id).status, 'staged'); + + // A full deploy of the same project: its own stage supersedes the one above. + await operations.deployComponent({ project, payload: await makePayload('2.0.0'), restart: false }); + + assert.strictEqual( + rows.get(first.deployment_id).status, + 'failed', + 'the superseded row is settled, so it no longer advertises an unusable deployment_id' + ); + } finally { + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_STAGINGRETENTION_MAXCOUNT, priorMax); + } + }); + + it('normalizes string request booleans, including install_allow_scripts', async () => { + // Joi coerces these, but validateBySchema discards `result.value`, so the raw string reaches the + // handler. `install_allow_scripts: 'false'` would then read as truthy and run package lifecycle + // scripts for a caller that explicitly disabled them — over multipart/form, where every value + // arrives as a string. + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('1.0.0'), + activate: 'false', + install_allow_scripts: 'false', + }); + + assert.strictEqual(staged.staged, true, "activate:'false' is honored as stage-only, not as a full deploy"); + assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, project)), false, 'so nothing goes live'); + assert.strictEqual( + rows.get(staged.deployment_id).activation_spec.install_allow_scripts, + false, + 'and the activation spec records a real boolean, not the string' + ); + }); + + it('stages without touching live and records an immutable activation specification', async () => { + const project = name(); + const result = await operations.deployComponent({ + project, + payload: await makePayload('1.0.0'), + activate: false, + }); + + assert.strictEqual(result.staged, true); + assert.match(result.deployment_id, /^[0-9a-f-]{36}$/i); + assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, project)), false); + assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id, project)), true); + const row = rows.get(result.deployment_id); + assert.ok(row, `deployment row missing; present ids: ${Array.from(rows.keys()).join(', ')}`); + assert.strictEqual(row.status, 'staged'); + assert.deepStrictEqual(row.activation_spec, { + project, + package: null, + install_command: null, + install_timeout: null, + install_allow_scripts: null, + urlPath: null, + host: null, + credentials: null, + force: false, + }); + }); + + it('uses a no-custody literal registry token for the origin install without recording it', async () => { + const project = name(); + const token = 'transient-origin-token'; + const installCommand = + `node -e "const fs=require('fs');` + + `const value=fs.readFileSync(process.env.npm_config_userconfig||process.env.NPM_CONFIG_USERCONFIG,'utf8');` + + `if(!value.includes('//registry.example.com/:_authToken='))process.exit(7);` + + `fs.writeFileSync('credential-seen','yes')"`; + const result = await operations.deployComponent({ + project, + payload: await makePayload('credential-origin', '6.0.0', false), + install_command: installCommand, + credentials: [{ registry: 'https://registry.example.com', token }], + activate: false, + }); + + const stagedPath = path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id, project); + assert.strictEqual(await fs.readFile(path.join(stagedPath, 'credential-seen'), 'utf8'), 'yes'); + assert.strictEqual(rows.get(result.deployment_id).activation_spec.credentials, null); + assert.doesNotMatch(JSON.stringify(rows.get(result.deployment_id)), new RegExp(token)); + }); + + it('activates only a staged row owned by the requested project', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('2.0.0'), + activate: false, + }); + + await assert.rejects( + operations.deployComponent({ project: `${project}_other`, deployment_id: staged.deployment_id }), + /not a staged deployment/ + ); + const activated = await operations.deployComponent({ project, deployment_id: staged.deployment_id }); + + assert.strictEqual(activated.activated, true); + assert.strictEqual(rows.get(staged.deployment_id).status, 'success'); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /2.0.0/); + }); + + it('does not let a duplicate activation undo the winning activation state', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('duplicate-winner', '6.0.0'), + activate: false, + }); + + const outcomes = await Promise.allSettled([ + operations.deployComponent({ project, deployment_id: staged.deployment_id }), + operations.deployComponent({ project, deployment_id: staged.deployment_id }), + ]); + + assert.strictEqual(outcomes.filter((outcome) => outcome.status === 'fulfilled').length, 1); + assert.strictEqual(outcomes.filter((outcome) => outcome.status === 'rejected').length, 1); + assert.strictEqual(rows.get(staged.deployment_id).status, 'success'); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /duplicate-winner/); + }); + + it('rejects fresh build or routing input on activate-by-id', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('3.0.0'), + activate: false, + }); + + await assert.rejects( + operations.deployComponent({ + project, + deployment_id: staged.deployment_id, + install_command: 'npm install --evil', + }), + /immutable staged configuration.*install_command/ + ); + }); + + it('stages and activates a full deploy before reporting success', async () => { + const project = name(); + const result = await operations.deployComponent({ + project, + payload: await makePayload('full-deploy', '6.0.0'), + }); + + assert.match(result.message, /Successfully deployed/); + assert.strictEqual(rows.get(result.deployment_id).status, 'success'); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /full-deploy/); + assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, result.deployment_id)), false); + assert.strictEqual(restartNeeded(), true, 'a new component activated without restart requires one'); + }); + + // revert_component + + it('reverts the cluster to a named previous deployment and fans the target out to peers', async () => { + const project = name(); + const first = await operations.deployComponent({ project, payload: await makePayload('rev-v1', '1.0.0') }); + const second = await operations.deployComponent({ project, payload: await makePayload('rev-v2', '2.0.0') }); + const fanout = []; + server.replication.replicateOperation = async (op) => { + fanout.push(op); + return { replicated: [] }; + }; + + const result = await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); + + assert.strictEqual(result.reverted, true); + assert.strictEqual(result.to_deployment_id, first.deployment_id); + assert.strictEqual(result.from_deployment_id, second.deployment_id); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /rev-v1/); + assert.strictEqual(rows.get(result.deployment_id).status, 'rolled_back'); + assert.strictEqual( + rows.get(result.deployment_id).rollback_of, + second.deployment_id, + 'the audit row records which deployment the rollback took out of service' + ); + assert.strictEqual(fanout.length, 1, 'peers get the revert'); + assert.strictEqual(fanout[0].operation, 'revert_component'); + assert.strictEqual( + fanout[0].to_deployment_id, + first.deployment_id, + 'peers are told WHICH version to end on, so the fan-out is idempotent per node' + ); + }); + + it('is a no-op when the requested deployment is already live, so a retry is safe', async () => { + const project = name(); + const first = await operations.deployComponent({ project, payload: await makePayload('retry-v1', '1.0.0') }); + await operations.deployComponent({ project, payload: await makePayload('retry-v2', '2.0.0') }); + await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); + + // The caller lost the first response and retried the identical request. + const retry = await operations.revertComponent({ project, to_deployment_id: first.deployment_id }); + + assert.strictEqual(retry.reverted, false); + assert.match(retry.message, /already running/); + assert.match( + await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), + /retry-v1/, + 'a retried revert must not toggle the rejected version back in' + ); + }); + + it('rejects a revert with no target, and one whose target is not retained', async () => { + const project = name(); + await operations.deployComponent({ project, payload: await makePayload('target-v1', '1.0.0') }); + await operations.deployComponent({ project, payload: await makePayload('target-v2', '2.0.0') }); + + await assert.rejects( + () => operations.revertComponent({ project }), + /to_deployment_id/, + 'the target is mandatory — that is what makes a retry safe' + ); + await assert.rejects( + () => operations.revertComponent({ project, to_deployment_id: '00000000-0000-4000-8000-000000000000' }), + /neither the live version/, + 'only the immediately-previous version is retained' + ); + assert.match( + await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), + /target-v2/, + 'a refused revert changes nothing' + ); + }); + + it('takes the package reference out of root config when reverting away from a package deploy', async () => { + // Without this, installApplications() would reinstall the reverted-away package over the restored + // directory on the next cold start and silently undo the rollback. + const project = name(); + const packaged = await operations.deployComponent({ + project, + payload: await makePayload('cfg-packaged', '1.0.0'), + }); + // Stamp root config as a package deploy would have, then activate a payload version over it. + const { addConfig } = require('#src/config/configUtils'); + await addConfig(project, { package: 'some-pkg@1.0.0' }); + const plain = await operations.deployComponent({ project, payload: await makePayload('cfg-plain', '2.0.0') }); + assert.ok(plain.deployment_id); + + await operations.revertComponent({ project, to_deployment_id: packaged.deployment_id }); + + const entry = readConfigFile()?.[project]; + assert.strictEqual( + entry?.package, + undefined, + 'the reverted-to version had no package reference, so the stale one must be gone' + ); + }); + + it('reclaims an oversized payload only after a full two-phase activation succeeds', async () => { + const project = name(); + const priorMaxSize = environment.get(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE); + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, 1); + try { + const result = await operations.deployComponent({ + project, + payload: await makePayload('reclaimed-full-deploy', '6.0.0'), + }); + + const row = rows.get(result.deployment_id); + assert.strictEqual(row.status, 'success'); + assert.strictEqual(row.payload_blob, null); + assert.ok(row.event_log.some((event) => event.event === 'payload_dropped')); + } finally { + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, priorMaxSize); + } + }); + + it('reclaims an oversized retained payload after activate-by-id succeeds', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('reclaimed-staged-deploy', '6.0.0'), + activate: false, + }); + assert.ok(rows.get(staged.deployment_id).payload_blob, 'staged deployment keeps its recovery payload'); + + const priorMaxSize = environment.get(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE); + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, 1); + try { + await operations.deployComponent({ project, deployment_id: staged.deployment_id }); + + const row = rows.get(staged.deployment_id); + assert.strictEqual(row.status, 'success'); + assert.strictEqual(row.payload_blob, null); + assert.ok(row.event_log.some((event) => event.event === 'payload_dropped')); + } finally { + environment.setProperty(CONFIG_PARAMS.DEPLOYMENT_PAYLOADRETENTION_MAXSIZE, priorMaxSize); + } + }); + + it('preserves the legacy one-shot path when explicitly requested', async () => { + const project = name(); + const result = await operations.deployComponent({ + project, + payload: await makePayload('one-shot', '6.0.0'), + two_phase: false, + }); + + assert.match(result.message, /Successfully deployed/); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /one-shot/); + }); + + it('accepts the legacy deployment row marker only from a trusted replicated operation', async () => { + const project = name(); + const payload = await makePayload('trusted-one-shot', '6.0.0'); + const result = await runWithOperationAuthorizationBypass(true, () => + operations.deployComponent({ + project, + payload, + two_phase: false, + _deploymentId: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', + replicated: false, + hdb_user: { name: 'cluster-peer' }, + }) + ); + + assert.match(result.message, /Successfully deployed/); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /trusted-one-shot/); + }); + + it('fails closed on the preview phase marker even from a trusted peer', async () => { + await assert.rejects( + runWithOperationAuthorizationBypass(true, () => + operations.deployComponent({ + project: name(), + _phase: 'stage', + replicated: false, + hdb_user: { name: 'cluster-peer' }, + }) + ), + /Unsupported legacy component deployment phase/ + ); + }); + + it('fails closed on an activate peer failure before scheduling a restart', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('4.0.0'), + activate: false, + }); + const phases = []; + server.replication.replicateOperation = async (operation) => { + phases.push(operation.phase); + if (operation.phase === 'activate') { + return { replicated: [{ node: 'peer-a', status: 'failed', reason: 'config write failed' }] }; + } + return { replicated: [] }; + }; + + await assert.rejects( + operations.deployComponent({ project, deployment_id: staged.deployment_id, restart: true }), + /Split nodes: peer-a.*[Rr]oll forward/s + ); + assert.deepStrictEqual(phases, ['activate'], 'restart phase was never sent after the activation gate failed'); + assert.strictEqual(rows.get(staged.deployment_id).status, 'activating'); + assert.strictEqual(rows.get(staged.deployment_id).completed_at, null); + assert.ok(rows.get(staged.deployment_id).payload_blob, 'payload remains available to repair a split cluster'); + assert.strictEqual(rows.get(staged.deployment_id).peer_results[0].node, 'peer-a'); + }); + + it('records success when restart fails after the activation barrier', async () => { + const project = name(); + const phases = []; + server.replication.replicateOperation = async (operation) => { + phases.push(operation.phase); + return operation.phase === 'restart' + ? { replicated: [{ node: 'peer-a', status: 'failed', reason: 'restart unavailable' }] } + : { replicated: [] }; + }; + const priorRestartWorkers = manageThreads.restartWorkers; + manageThreads.restartWorkers = () => {}; + let deploymentId; + try { + await assert.rejects( + operations + .deployComponent({ + project, + payload: await makePayload('activated-before-restart-failure', '6.0.0'), + restart: true, + }) + .catch((error) => { + deploymentId = error.http_resp_msg?.deployment_id; + throw error; + }), + /restart failed/ + ); + } finally { + manageThreads.restartWorkers = priorRestartWorkers; + } + + assert.deepStrictEqual(phases, ['stage', 'activate', 'restart']); + assert.strictEqual(rows.get(deploymentId).status, 'success'); + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /activated-before/); + }); + + it('records peer failures but honors ignore_replication_errors', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('ignored-peer', '6.0.0'), + activate: false, + }); + server.replication.replicateOperation = async () => ({ + replicated: [{ node: 'peer-a', status: 'failed', reason: 'offline' }], + }); + + const result = await operations.deployComponent({ + project, + deployment_id: staged.deployment_id, + ignore_replication_errors: true, + }); + + assert.strictEqual(result.activated, true); + assert.strictEqual(rows.get(staged.deployment_id).status, 'success'); + assert.strictEqual(rows.get(staged.deployment_id).peer_results[0].status, 'failed'); + }); + + it('records and surfaces ignored restart failures after the activation gate', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('restart-failure', '6.0.0'), + activate: false, + }); + const phases = []; + server.replication.replicateOperation = async (operation) => { + phases.push(operation.phase); + return operation.phase === 'restart' + ? { replicated: [{ node: 'peer-a', status: 'failed', reason: 'restart unavailable' }] } + : { replicated: [] }; + }; + const priorRestartWorkers = manageThreads.restartWorkers; + let localRestarts = 0; + manageThreads.restartWorkers = () => localRestarts++; + let result; + try { + result = await operations.deployComponent({ + project, + deployment_id: staged.deployment_id, + restart: true, + ignore_replication_errors: true, + }); + } finally { + manageThreads.restartWorkers = priorRestartWorkers; + } + + assert.deepStrictEqual(phases, ['activate', 'restart']); + assert.strictEqual(localRestarts, 1); + assert.strictEqual(result.activated, true); + assert.strictEqual(result.failed_peers[0].node, 'peer-a'); + assert.strictEqual(rows.get(staged.deployment_id).peer_results[0].status, 'failed'); + }); + + it('uses the row-backed immutable specification for trusted peer phases', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('peer-phase', '6.0.0'), + activate: false, + }); + const row = rows.get(staged.deployment_id); + const componentPath = path.join(COMPONENTS_ROOT, project); + await discardStagedApplication(componentPath, staged.deployment_id); + const executePeerPhase = (phase, activationSpec) => + runWithOperationAuthorizationBypass(true, () => + operations.componentDeployPhase({ + operation: 'component_deploy_phase', + phase, + project, + deployment_id: staged.deployment_id, + activation_spec: activationSpec, + replicated: false, + hdb_user: { name: 'cluster-peer' }, + }) + ); + + await assert.rejects( + executePeerPhase('stage', { ...row.activation_spec, host: 'tampered.example' }), + /immutable activation specification/ + ); + await executePeerPhase('stage', row.activation_spec); + assert.strictEqual(existsSync(path.join(COMPONENTS_ROOT, DEPLOY_STAGING_DIR, staged.deployment_id, project)), true); + await executePeerPhase('activate', row.activation_spec); + assert.match(await fs.readFile(path.join(componentPath, 'index.js'), 'utf8'), /peer-phase/); + assert.strictEqual( + rows.get(staged.deployment_id).status, + 'staged', + 'a peer activates locally without advancing the replicated row — the origin owns it' + ); + assert.strictEqual(restartNeeded(), true); + }); + + it('rebuilds a missing peer stage from the durable deployment payload before activation', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('rebuilt-peer', '6.0.0'), + activate: false, + }); + const row = rows.get(staged.deployment_id); + const componentPath = path.join(COMPONENTS_ROOT, project); + await discardStagedApplication(componentPath, staged.deployment_id); + + await runWithOperationAuthorizationBypass(true, () => + operations.componentDeployPhase({ + operation: 'component_deploy_phase', + phase: 'activate', + project, + deployment_id: staged.deployment_id, + activation_spec: row.activation_spec, + replicated: false, + hdb_user: { name: 'cluster-peer' }, + }) + ); + + assert.match(await fs.readFile(path.join(componentPath, 'index.js'), 'utf8'), /rebuilt-peer/); + assert.strictEqual( + rows.get(staged.deployment_id).status, + 'staged', + 'a peer activates locally without advancing the replicated row — the origin owns it' + ); + }); + + it('waits for the staged row checkpoint when peer activation arrives first', async () => { + const project = name(); + const staged = await operations.deployComponent({ + project, + payload: await makePayload('lagged-row', '6.0.0'), + activate: false, + }); + const row = rows.get(staged.deployment_id); + rows.set(staged.deployment_id, { ...row, status: 'staging' }); + setImmediate(() => rows.set(staged.deployment_id, { ...rows.get(staged.deployment_id), status: 'staged' })); + + await runWithOperationAuthorizationBypass(true, () => + operations.componentDeployPhase({ + operation: 'component_deploy_phase', + phase: 'activate', + project, + deployment_id: staged.deployment_id, + activation_spec: row.activation_spec, + deployment_timeout: 200, + replicated: false, + hdb_user: { name: 'cluster-peer' }, + }) + ); + + assert.match(await fs.readFile(path.join(COMPONENTS_ROOT, project, 'index.js'), 'utf8'), /lagged-row/); + assert.strictEqual( + rows.get(staged.deployment_id).status, + 'staged', + 'a peer activates locally without advancing the replicated row — the origin owns it' + ); + }); + + it('recovers a staged package specification for config and peer activation', async () => { + const project = name(); + const tarDirectory = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-package-')); + const tarPath = path.join(tarDirectory, 'component.tgz'); + await fs.writeFile(tarPath, await makePayload('package-stage', '6.0.0')); + const packageIdentifier = `file:${tarPath}`; + const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-config-')); + const configPath = path.join(configRoot, 'harper-config.yaml'); + await fs.writeFile(configPath, `rootPath: ${JSON.stringify(configRoot)}\n`); + const priorRootEnv = process.env.ROOTPATH; + const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); + process.env.ROOTPATH = configRoot; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); + try { + const staged = await operations.deployComponent({ + project, + package: packageIdentifier, + activate: false, + }); + let activationOperation; + server.replication.replicateOperation = async (operation) => { + activationOperation = operation; + return { replicated: [] }; + }; + + await operations.deployComponent({ project, deployment_id: staged.deployment_id }); + + assert.strictEqual(readConfigFile()[project].package, packageIdentifier); + assert.strictEqual(activationOperation.operation, 'component_deploy_phase'); + assert.strictEqual(activationOperation.activation_spec.package, packageIdentifier); + } finally { + if (priorRootEnv === undefined) delete process.env.ROOTPATH; + else process.env.ROOTPATH = priorRootEnv; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); + await fs.rm(configRoot, { recursive: true, force: true }); + await fs.rm(tarDirectory, { recursive: true, force: true }); + } + }); + + it('removes the root-config entry before it destroys anything, so a failed drop cannot resurrect', async () => { + // The resurrection vector is the root-config entry: installApplications() reinstalls whatever it + // names. Doing the persistent writes after the directory removal left a crash window where the + // live tree was gone but config still named the package, and the next boot brought the component + // back. Config-first inverts the failure: an interrupted drop is unfinished and re-runnable. + const project = name(); + const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-drop-crash-')); + const componentsRoot = path.join(configRoot, 'components'); + const configPath = path.join(configRoot, 'harper-config.yaml'); + const priorRootEnv = process.env.ROOTPATH; + const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); + const priorComponentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); + process.env.ROOTPATH = configRoot; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); + environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, componentsRoot); + await fs.writeFile(configPath, `rootPath: ${JSON.stringify(configRoot)}\n`); + try { + await fs.mkdir(path.join(componentsRoot, project), { recursive: true }); + await fs.writeFile(path.join(componentsRoot, project, 'index.js'), "module.exports = 'live';\n"); + await fs.appendFile(configPath, `${project}:\n package: some-package@1.0.0\n`); + // Make every aside-based teardown step fail: `.deploy-aside` occupied by a file is rejected as + // "not a directory", which is the closest deterministic stand-in for dying mid-teardown. + await fs.writeFile(path.join(componentsRoot, '.deploy-aside'), 'not a directory\n'); + + await assert.rejects(() => operations.dropComponent({ project })); + + assert.strictEqual( + (await fs.readFile(configPath, 'utf8')).includes(project), + false, + 'config is already clean, so the next boot cannot reinstall the component' + ); + assert.strictEqual( + existsSync(path.join(componentsRoot, project)), + true, + 'and the tree is still there: the drop is unfinished rather than finished-then-undone' + ); + } finally { + if (priorRootEnv === undefined) delete process.env.ROOTPATH; + else process.env.ROOTPATH = priorRootEnv; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); + environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, priorComponentsRoot); + await fs.rm(configRoot, { recursive: true, force: true }); + } + }); + + it('drop_component invalidates staged rows and removes recovery artifacts', async () => { + const project = name(); + const configRoot = await fs.mkdtemp(path.join(os.tmpdir(), 'harper-phase-drop-')); + const componentsRoot = path.join(configRoot, 'components'); + const configPath = path.join(configRoot, 'harper-config.yaml'); + const priorRootEnv = process.env.ROOTPATH; + const priorRootConfig = getConfigPath(CONFIG_PARAMS.ROOTPATH); + const priorComponentsRoot = getConfigPath(CONFIG_PARAMS.COMPONENTSROOT); + process.env.ROOTPATH = configRoot; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, configRoot); + environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, componentsRoot); + await fs.writeFile(configPath, `rootPath: ${JSON.stringify(configRoot)}\n`); + try { + const staged = await operations.deployComponent({ + project, + payload: await makePayload('drop-stage', '6.0.0'), + activate: false, + }); + const activationPath = path.join(componentsRoot, DEPLOY_ACTIVATION_DIR, project, 'interrupted'); + await fs.mkdir(activationPath, { recursive: true }); + await fs.writeFile( + path.join(configRoot, 'harper-application-lock.json'), + JSON.stringify({ applications: { [project]: { package: 'stale-package' } } }) + ); + // A root-config entry is what makes a dropped component come back: installApplications() reads + // it on the next boot and reinstalls the package. Removing it and the application-lock entry as + // two separate writes meant a crash or a failed second write left this behind. + await fs.appendFile(configPath, `${project}:\n package: stale-package\n`); + + await operations.dropComponent({ project }); + + assert.strictEqual(rows.get(staged.deployment_id).status, 'failed'); + const deploymentStagePath = path.join(componentsRoot, DEPLOY_STAGING_DIR, staged.deployment_id); + assert.strictEqual( + existsSync(deploymentStagePath), + false, + `staged deployment directory still contains: ${await fs.readdir(deploymentStagePath).catch(() => [])}` + ); + assert.strictEqual(existsSync(path.join(componentsRoot, DEPLOY_ACTIVATION_DIR, project)), false); + const applicationLock = JSON.parse(await fs.readFile(path.join(configRoot, 'harper-application-lock.json'))); + assert.strictEqual(applicationLock.applications[project], undefined); + assert.strictEqual( + (await fs.readFile(configPath, 'utf8')).includes(project), + false, + 'the root-config entry is removed in the same step, so the next boot cannot reinstall the drop' + ); + } finally { + if (priorRootEnv === undefined) delete process.env.ROOTPATH; + else process.env.ROOTPATH = priorRootEnv; + environment.setProperty(CONFIG_PARAMS.ROOTPATH, priorRootConfig); + environment.setProperty(CONFIG_PARAMS.COMPONENTSROOT, priorComponentsRoot); + await fs.rm(configRoot, { recursive: true, force: true }); + } + }); + + it('rejects separated-phase controls on the one-shot fallback', async () => { + await assert.rejects( + operations.deployComponent({ + project: name(), + payload: await makePayload('5.0.0'), + activate: false, + two_phase: false, + }), + /require two-phase deploy/ + ); + }); + + it('rejects an explicit two-phase request when the system database is not replicated', async () => { + const priorReplications = environment.get(CONFIG_PARAMS.REPLICATION_DATABASES); + environment.setProperty(CONFIG_PARAMS.REPLICATION_DATABASES, ['data']); + try { + await assert.rejects( + operations.deployComponent({ + project: name(), + payload: await makePayload('requires-system-replication'), + two_phase: true, + }), + /requires system database replication/ + ); + } finally { + environment.setProperty(CONFIG_PARAMS.REPLICATION_DATABASES, priorReplications); + } + }); + + it('does not trust caller-supplied internal phase markers', async () => { + await assert.rejects( + operations.deployComponent({ + project: name(), + payload: await makePayload('untrusted-replication'), + replicated: false, + two_phase: true, + }), + /requires operation replication/ + ); + await assert.rejects( + operations.deployComponent({ + project: name(), + _deploymentId: '../../escape', + _phase: 'stage', + }), + /is not allowed/ + ); + await assert.rejects( + operations.componentDeployPhase({ + operation: 'component_deploy_phase', + phase: 'discard', + project: name(), + deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', + activation_spec: { project: 'anything' }, + }), + /restricted to authenticated cluster peers/ + ); + await assert.rejects( + runWithOperationAuthorizationBypass(true, () => + operations.componentDeployPhase({ + operation: 'component_deploy_phase', + phase: 'discard', + project: '../escape', + deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', + activation_spec: { project: '../escape' }, + replicated: false, + hdb_user: { name: 'cluster-peer' }, + }) + ), + /project name/i + ); + }); +}); diff --git a/unitTests/components/deployPhaseValidators.test.js b/unitTests/components/deployPhaseValidators.test.js new file mode 100644 index 0000000000..d0ac9e62ce --- /dev/null +++ b/unitTests/components/deployPhaseValidators.test.js @@ -0,0 +1,58 @@ +'use strict'; + +const assert = require('node:assert'); +const validator = require('#js/components/operationsValidation'); + +const valid = (result) => assert.strictEqual(result, undefined, `expected valid, got: ${result?.message}`); +const invalid = (result) => assert.ok(result, 'expected a validation error'); + +describe('deployComponentValidator two-phase controls', () => { + it('accepts stage-and-stop and UUID activation', () => { + valid(validator.deployComponentValidator({ project: 'my_app', activate: false })); + valid( + validator.deployComponentValidator({ + project: 'my_app', + deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', + }) + ); + }); + + it('requires deployment_id to be a UUID and rejects path traversal', () => { + for (const deploymentId of ['abc-123', '../evil', 'dep/../..', '.', '..']) { + invalid(validator.deployComponentValidator({ project: 'my_app', deployment_id: deploymentId })); + } + }); + + it('rejects caller-controlled internal phase markers', () => { + invalid(validator.deployComponentValidator({ project: 'my_app', _deploymentId: 'x' })); + invalid(validator.deployComponentValidator({ project: 'my_app', _phase: 'stage' })); + }); + + it('rejects retry-unsafe automatic rollback', () => { + invalid(validator.deployComponentValidator({ project: 'my_app', revert_on_failure: true })); + }); + + it('preserves routing validation', () => { + invalid(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', urlPath: '/a/./b' })); + invalid(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', urlPath: '/a/../b' })); + }); +}); + +describe('componentDeployPhaseValidator', () => { + const validPhase = { + phase: 'stage', + deployment_id: '41faded8-6cf5-4a2a-95f8-863e7ea498fa', + project: 'my_app', + activation_spec: { project: 'my_app' }, + }; + + it('accepts a bounded internal phase request', () => { + valid(validator.componentDeployPhaseValidator(validPhase)); + }); + + it('rejects invalid phases, project traversal, and non-UUID ids', () => { + invalid(validator.componentDeployPhaseValidator({ ...validPhase, phase: 'deploy' })); + invalid(validator.componentDeployPhaseValidator({ ...validPhase, project: '../escape' })); + invalid(validator.componentDeployPhaseValidator({ ...validPhase, deployment_id: '../../escape' })); + }); +}); diff --git a/unitTests/components/deployValidators.test.js b/unitTests/components/deployValidators.test.js deleted file mode 100644 index cf1c19a163..0000000000 --- a/unitTests/components/deployValidators.test.js +++ /dev/null @@ -1,21 +0,0 @@ -'use strict'; - -const assert = require('node:assert'); -const validator = require('#js/components/operationsValidation'); - -const invalid = (result) => assert.ok(result, 'expected a validation error'); - -describe('deployComponentValidator', () => { - it('rejects retry-unsafe automatic rollback', () => { - invalid(validator.deployComponentValidator({ project: 'my_app', revert_on_failure: true })); - }); - - it('rejects the caller-supplied internal deployment marker', () => { - invalid(validator.deployComponentValidator({ project: 'my_app', _deploymentId: 'x' })); - }); - - it('preserves routing validation', () => { - invalid(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', urlPath: '/a/./b' })); - invalid(validator.deployComponentValidator({ project: 'my_app', package: 'npm:x', urlPath: '/a/../b' })); - }); -}); diff --git a/utility/hdbTerms.ts b/utility/hdbTerms.ts index e83587715b..097396c81b 100644 --- a/utility/hdbTerms.ts +++ b/utility/hdbTerms.ts @@ -323,7 +323,10 @@ export const OPERATIONS_ENUM = { PACKAGE_CUSTOM_FUNCTION_PROJECT: 'package_custom_function_project', DEPLOY_CUSTOM_FUNCTION_PROJECT: 'deploy_custom_function_project', PACKAGE_COMPONENT: 'package_component', + // Peer phases get their own operation so an older node fails closed instead of reading an unknown + // phase field as a one-shot deploy. DEPLOY_COMPONENT: 'deploy_component', + COMPONENT_DEPLOY_PHASE: 'component_deploy_phase', // A public operation rather than a deploy phase: it fetches, resolves and installs nothing. REVERT_COMPONENT: 'revert_component', READ_TRANSACTION_LOG: 'read_transaction_log', diff --git a/utility/operation_authorization.ts b/utility/operation_authorization.ts index 538030ffcf..c3fa85352e 100644 --- a/utility/operation_authorization.ts +++ b/utility/operation_authorization.ts @@ -313,6 +313,10 @@ requiredPermissions.set(functionsOperations.addComponent.name, new (permission a requiredPermissions.set(functionsOperations.dropCustomFunctionProject.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.packageComponent.name, new (permission as any)(true, [])); requiredPermissions.set(functionsOperations.deployComponent.name, new (permission as any)(true, [])); +requiredPermissions.set( + functionsOperations.componentDeployPhase.name, + new (permission as any)(true, [], terms.OPERATIONS_ENUM.COMPONENT_DEPLOY_PHASE) +); requiredPermissions.set( functionsOperations.revertComponent.name, new (permission as any)(true, [], terms.OPERATIONS_ENUM.REVERT_COMPONENT)