From e332d229cbe57c5bb8aea304542daad43bab0ce0 Mon Sep 17 00:00:00 2001 From: wzb Date: Mon, 27 Jul 2026 12:14:37 +0800 Subject: [PATCH 01/16] fix(update): preserve proxy on npm cache failures --- bin/ocx.mjs | 7 ++ docs-site/src/content/docs/reference/cli.md | 5 +- docs/adr/0001-gui-update-worker.md | 13 ++ src/update/index.ts | 9 ++ src/update/job.ts | 46 ++++++++ src/update/npm-cache-preflight.d.mts | 44 +++++++ src/update/npm-cache-preflight.mjs | 124 ++++++++++++++++++++ tests/update-job.test.ts | 66 +++++++++++ tests/update-npm-cache-preflight.test.ts | 120 +++++++++++++++++++ tests/update-stop-first.test.ts | 13 ++ 10 files changed, 446 insertions(+), 1 deletion(-) create mode 100644 src/update/npm-cache-preflight.d.mts create mode 100644 src/update/npm-cache-preflight.mjs create mode 100644 tests/update-npm-cache-preflight.test.ts diff --git a/bin/ocx.mjs b/bin/ocx.mjs index 0218832de..42f4b6476 100755 --- a/bin/ocx.mjs +++ b/bin/ocx.mjs @@ -14,6 +14,7 @@ import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; import { homedir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { checkNpmCacheOwnership, formatNpmCacheOwnershipFailure } from "../src/update/npm-cache-preflight.mjs"; import { handoffWindowsTrayForUpdate, planWindowsTrayUpdate } from "../src/update/tray-update-plan.mjs"; const PKG = "@bitkyc08/opencodex"; @@ -133,6 +134,12 @@ function runNpmSelfUpdate() { process.exit(0); } + const cacheOwnership = checkNpmCacheOwnership({ npmBin: npm, shell: winShell }); + if (cacheOwnership.ok === false) { + console.error(`opencodex: ${formatNpmCacheOwnershipFailure(cacheOwnership)}`); + process.exit(1); + } + // Remember whether a background service manages the proxy BEFORE stopping — `ocx stop` // unloads it permanently, so a successful update must reinstall it afterwards. const serviceStatePath = join(configDir(), "service-state.json"); diff --git a/docs-site/src/content/docs/reference/cli.md b/docs-site/src/content/docs/reference/cli.md index 010f4b6bf..a29f19c50 100644 --- a/docs-site/src/content/docs/reference/cli.md +++ b/docs-site/src/content/docs/reference/cli.md @@ -499,7 +499,10 @@ Self-update opencodex from npm. Stable installs use `@latest`; preview installs unless you pass `--tag latest|preview`. It detects a source checkout and tells you to `git pull && bun install` instead, and is a no-op if you're already on the newest version for that tag. A running proxy is stopped before files are replaced; an installed service is rebuilt and -started automatically, while a foreground installation prints `ocx start` as the next step. +started automatically, while a foreground installation prints `ocx start` as the next step. On +Unix, the updater first checks that the configured npm cache is owned by the current user. It aborts +before stopping the proxy when it finds a foreign-owned cache entry, so you can correct the cache +ownership or configure a user-owned cache and retry without losing the running service. ```bash ocx update diff --git a/docs/adr/0001-gui-update-worker.md b/docs/adr/0001-gui-update-worker.md index 056db8e8a..1c1067487 100644 --- a/docs/adr/0001-gui-update-worker.md +++ b/docs/adr/0001-gui-update-worker.md @@ -22,6 +22,17 @@ the existing npm self-update guard is reused. For Bun global installs, it runs t global update command. Source checkouts remain manual-only and show `git pull && bun install && bun run build:gui`. +Before an npm updater stops the proxy, it resolves the configured npm cache and checks that every +entry is owned by the current Unix user. A foreign-owned entry (commonly left by an older `sudo npm` +invocation) aborts the update with an actionable error while the existing proxy and service remain +running. Failure to resolve or inspect the cache also aborts before shutdown; platforms without +Unix uid ownership skip this gate. + +If the installer still exits nonzero, the GUI worker first checks whether the pre-update proxy is +still healthy. It leaves that process alone when an early gate failed; when the updater already +stopped it, the worker makes a best-effort restart and verifies health. The update job remains +failed either way because restoring availability is not the same as installing the new version. + After an update requests a restart, the worker now waits for an identity-checked `/healthz` to return and remain healthy for a short stability window before marking the job successful. This keeps `update-job.json` honest on Windows cases where npm leaves the bundled Bun runtime in a bad @@ -45,6 +56,8 @@ restart path because `bun add -g` does not restart the proxy. - The GUI request handler stays responsive and does not overwrite its own running module graph. - Update status survives a proxy restart because it is stored in the opencodex config directory. - Restart handling can branch between service-managed installs and direct detached proxy starts. +- npm cache ownership failures are detected before service or proxy shutdown. +- A failed install best-effort restores a previously-active proxy without claiming update success. - A completed install can still finish with `status: "failed"` when the replacement proxy never becomes healthy or flaps during the stability window; the job log then points the user at `ocx start` and the Bun `--allow-scripts` reinstall path. diff --git a/src/update/index.ts b/src/update/index.ts index 7cd453b64..52d0b18a3 100644 --- a/src/update/index.ts +++ b/src/update/index.ts @@ -3,6 +3,7 @@ import { readFileSync, readdirSync } from "node:fs"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { getConfigDir, loadConfig, readPid, readRuntimePort } from "../config"; +import { checkNpmCacheOwnership, formatNpmCacheOwnershipFailure } from "./npm-cache-preflight.mjs"; import { handoffWindowsTrayForUpdate, planWindowsTrayUpdate } from "./tray-update-plan.mjs"; /** @@ -164,6 +165,14 @@ export async function runUpdate(): Promise { console.log(`Verified ${PKG}@${latest} integrity metadata ${integrity.integrity.slice(0, 24)}…`); } + if (installer === "npm") { + const cacheOwnership = checkNpmCacheOwnership(); + if (cacheOwnership.ok === false) { + console.error(`⚠️ ${formatNpmCacheOwnershipFailure(cacheOwnership)}`); + process.exit(1); + } + } + // Remember whether a background service manages the proxy BEFORE stopping — `ocx stop` // unloads it permanently, so a successful update must reinstall/restart it afterwards. let serviceWasInstalled = false; diff --git a/src/update/job.ts b/src/update/job.ts index 62a01716e..757ef764e 100644 --- a/src/update/job.ts +++ b/src/update/job.ts @@ -298,6 +298,8 @@ export interface RestartProxyIdentity { version?: string; } +export type FailedUpdateRecovery = "not-needed" | "still-running" | "restarted" | "failed"; + /** Test seam: the wait/spawn pair is injectable so the restart path is verifiable. */ export interface RestartIo { waitForPort?: typeof reclaimListenPort; @@ -524,6 +526,47 @@ async function defaultProbeProxyIdentity( } } +/** Keep a failed GUI install from also leaving a previously-active proxy offline. */ +async function recoverFailedGuiUpdate( + job: UpdateJobState, + captured: { port: number; hostname: string; oldPid?: number }, + proxyWasActive: boolean, + io: RestartIo = {}, +): Promise { + if (!proxyWasActive) return "not-needed"; + + const probeIdentity = io.probeProxyIdentity ?? defaultProbeProxyIdentity; + if (await probeIdentity(captured.port, captured.hostname)) { + updateJob(job, {}, `Update command failed, but the existing proxy remains healthy on ${captured.hostname}:${captured.port}.`); + return "still-running"; + } + + updateJob(job, {}, "Update command failed after the proxy stopped; attempting to restore the proxy..."); + try { + const restartFn = io.restartAfterUpdateFn ?? restartAfterUpdate; + await restartFn(job, captured, io); + const healthy = await awaitRestartedProxyHealthy(job, captured, io); + if (healthy.ok) { + updateJob(job, {}, `Previous proxy service restored on ${captured.hostname}:${captured.port}; the update itself still failed.`); + return "restarted"; + } + } catch (error) { + updateJob(job, {}, `Automatic proxy recovery threw: ${error instanceof Error ? error.message : String(error)}`); + } + + updateJob(job, {}, `Automatic proxy recovery failed. Restore the package, then run 'ocx service install' or 'ocx start --port ${captured.port}'.`); + return "failed"; +} + +export function recoverFailedGuiUpdateForTests( + job: UpdateJobState, + captured: { port: number; hostname: string; oldPid?: number }, + proxyWasActive: boolean, + io: RestartIo, +): Promise { + return recoverFailedGuiUpdate(job, captured, proxyWasActive, io); +} + /** * Health alone is not enough to skip the GUI worker restart: a surviving pre-update * process is still identity-healthy. Require update-correlated evidence — a new PID @@ -687,6 +730,7 @@ export async function runGuiUpdateWorker(jobId: string, channel: Channel, restar const livePid = readPid(); const preUpdateConfig = loadConfig(); const runtimeTrusted = !!(rt && livePid && rt.pid === livePid); + const proxyWasActive = isServiceInstalled() || runtimeTrusted; const configPort = typeof preUpdateConfig.port === "number" && preUpdateConfig.port > 0 ? preUpdateConfig.port : 10100; @@ -775,11 +819,13 @@ export async function runGuiUpdateWorker(jobId: string, channel: Channel, restar startWindowsTray(); } catch { /* retain the primary update failure */ } } + const recovery = await recoverFailedGuiUpdate(job, captured, proxyWasActive); updateJob(job, { status: "failed", exitCode: result.status, signal: result.signal, error: `update command failed (${result.status ?? "?"})`, + ...(recovery === "restarted" ? { restarted: true } : {}), }); return; } diff --git a/src/update/npm-cache-preflight.d.mts b/src/update/npm-cache-preflight.d.mts new file mode 100644 index 000000000..5529e1eb0 --- /dev/null +++ b/src/update/npm-cache-preflight.d.mts @@ -0,0 +1,44 @@ +export interface NpmCacheEntryStat { + uid?: number; + isDirectory(): boolean; +} + +export interface NpmCachePreflightOptions { + platform?: string; + getuid?: () => number; + npmBin?: string; + shell?: boolean; + spawn?: typeof import("node:child_process").spawnSync; + lstat?: (path: string) => NpmCacheEntryStat; + readdir?: (path: string) => string[]; +} + +export type NpmCacheOwnershipIssue = + | { kind: "foreign-owner"; path: string; actualUid: number } + | { kind: "error"; path: string; reason: string }; + +export type NpmCacheOwnershipResult = + | { ok: true; cachePath: string } + | { ok: "skipped"; reason: string } + | { + ok: false; + cachePath?: string; + entryPath?: string; + expectedUid: number; + actualUid?: number; + reason: string; + }; + +export declare function findForeignOwnedNpmCacheEntry( + cachePath: string, + expectedUid: number, + io?: Pick, +): NpmCacheOwnershipIssue | null; + +export declare function checkNpmCacheOwnership( + options?: NpmCachePreflightOptions, +): NpmCacheOwnershipResult; + +export declare function formatNpmCacheOwnershipFailure( + result: Extract, +): string; diff --git a/src/update/npm-cache-preflight.mjs b/src/update/npm-cache-preflight.mjs new file mode 100644 index 000000000..3a3e0fcdb --- /dev/null +++ b/src/update/npm-cache-preflight.mjs @@ -0,0 +1,124 @@ +import { spawnSync } from "node:child_process"; +import { lstatSync, readdirSync } from "node:fs"; +import { resolve } from "node:path"; + +function errorCode(error) { + return error && typeof error === "object" && "code" in error + ? String(error.code) + : "unknown error"; +} + +/** Find the first cache entry whose uid differs from the current user. */ +export function findForeignOwnedNpmCacheEntry(cachePath, expectedUid, io = {}) { + const lstat = io.lstat ?? lstatSync; + const readdir = io.readdir ?? (path => readdirSync(path, { encoding: "utf8" })); + const stack = [resolve(cachePath)]; + + while (stack.length > 0) { + const path = stack.pop(); + let stat; + try { + stat = lstat(path); + } catch (error) { + if (errorCode(error) === "ENOENT") continue; + return { + kind: "error", + path, + reason: `could not inspect npm cache entry (${errorCode(error)})`, + }; + } + + if (Number.isInteger(stat.uid) && stat.uid !== expectedUid) { + return { kind: "foreign-owner", path, actualUid: stat.uid }; + } + if (!stat.isDirectory()) continue; + + let entries; + try { + entries = readdir(path); + } catch (error) { + return { + kind: "error", + path, + reason: `could not read npm cache directory (${errorCode(error)})`, + }; + } + for (const entry of entries) stack.push(resolve(path, entry)); + } + + return null; +} + +/** + * npm can abort after it has retired the installed package when its cache contains + * root-owned entries. Detect that state before an updater stops the running proxy. + */ +export function checkNpmCacheOwnership(options = {}) { + const platform = options.platform ?? process.platform; + const getuid = options.getuid ?? process.getuid; + if (platform === "win32" || typeof getuid !== "function") { + return { ok: "skipped", reason: "uid ownership is unavailable on this platform" }; + } + + const expectedUid = getuid(); + const npmBin = options.npmBin ?? "npm"; + const spawn = options.spawn ?? spawnSync; + let result; + try { + result = spawn(npmBin, ["config", "get", "cache"], { + encoding: "utf8", + timeout: 12_000, + windowsHide: true, + shell: options.shell ?? false, + }); + } catch (error) { + return { + ok: false, + expectedUid, + reason: `could not resolve the npm cache (${errorCode(error)})`, + }; + } + if (result.status !== 0) { + const detail = result.error ? errorCode(result.error) : `status ${result.status ?? "timeout"}`; + return { + ok: false, + expectedUid, + reason: `could not resolve the npm cache (${detail})`, + }; + } + + const output = Buffer.isBuffer(result.stdout) ? result.stdout.toString("utf8") : String(result.stdout ?? ""); + const cachePath = output.split(/\r?\n/).map(line => line.trim()).filter(Boolean).at(-1); + if (!cachePath || cachePath === "undefined" || cachePath === "null") { + return { ok: false, expectedUid, reason: "npm did not report a cache path" }; + } + + const issue = findForeignOwnedNpmCacheEntry(cachePath, expectedUid, options); + if (!issue) return { ok: true, cachePath: resolve(cachePath) }; + if (issue.kind === "foreign-owner") { + return { + ok: false, + cachePath: resolve(cachePath), + entryPath: issue.path, + expectedUid, + actualUid: issue.actualUid, + reason: `npm cache entry is owned by uid ${issue.actualUid}; expected uid ${expectedUid}`, + }; + } + return { + ok: false, + cachePath: resolve(cachePath), + entryPath: issue.path, + expectedUid, + reason: issue.reason, + }; +} + +export function formatNpmCacheOwnershipFailure(result) { + return [ + "npm cache ownership pre-flight failed before stopping the proxy.", + result.entryPath ? `${result.reason}: ${result.entryPath}` : result.reason, + ...(result.cachePath ? [`Cache: ${result.cachePath}`] : []), + "Correct the cache ownership or configure a user-owned npm cache, then retry.", + ].join("\n"); +} diff --git a/tests/update-job.test.ts b/tests/update-job.test.ts index 82c85b1b9..f8c316294 100644 --- a/tests/update-job.test.ts +++ b/tests/update-job.test.ts @@ -8,6 +8,7 @@ import { finishGuiUpdateRestart, npmSelfUpdateRestartEvidence, readUpdateJob, + recoverFailedGuiUpdateForTests, restartCommand, restartAfterUpdateForTests, startUpdateJob, @@ -285,6 +286,71 @@ describe("GUI update execution decisions", () => { expect(spawned).toEqual([{ port: 19999 }]); }); + test("failed install leaves an already-healthy proxy untouched", async () => { + let restartCalls = 0; + const job: UpdateJobState = { + id: "failed-install-still-running", + status: "running", + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + currentVersion: "2.7.40", + latestVersion: "2.7.41", + channel: "latest", + installer: "npm", + restart: true, + command: "", + releaseNotesUrl: "", + log: [], + }; + writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + const recovery = await recoverFailedGuiUpdateForTests( + job, + { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, + true, + { + probeProxyIdentity: async () => ({ pid: 111, version: "2.7.40" }), + restartAfterUpdateFn: async () => { restartCalls += 1; }, + }, + ); + expect(recovery).toBe("still-running"); + expect(restartCalls).toBe(0); + expect(readUpdateJob(job.id)?.log.some(line => line.includes("existing proxy remains healthy"))).toBe(true); + }); + + test("failed install restores a proxy that the updater stopped", async () => { + let now = 0; + let restarted = false; + const job: UpdateJobState = { + id: "failed-install-recovery", + status: "running", + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + currentVersion: "2.7.40", + latestVersion: "2.7.41", + channel: "latest", + installer: "npm", + restart: true, + command: "", + releaseNotesUrl: "", + log: [], + }; + writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + const recovery = await recoverFailedGuiUpdateForTests( + job, + { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, + true, + { + probeProxyIdentity: async () => null, + probeProxy: async () => restarted, + restartAfterUpdateFn: async () => { restarted = true; }, + now: () => now, + sleepMs: async (ms) => { now += ms; }, + }, + ); + expect(recovery).toBe("restarted"); + expect(readUpdateJob(job.id)?.log.some(line => line.includes("update itself still failed"))).toBe(true); + }); + test("restart confirmation fails when the proxy never becomes healthy", async () => { let now = 0; const job: UpdateJobState = { diff --git a/tests/update-npm-cache-preflight.test.ts b/tests/update-npm-cache-preflight.test.ts new file mode 100644 index 000000000..f58f7dbec --- /dev/null +++ b/tests/update-npm-cache-preflight.test.ts @@ -0,0 +1,120 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { lstatSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { + checkNpmCacheOwnership, + findForeignOwnedNpmCacheEntry, + formatNpmCacheOwnershipFailure, +} from "../src/update/npm-cache-preflight.mjs"; + +let dir: string; + +beforeEach(() => { + dir = join(tmpdir(), `ocx-npm-cache-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(join(dir, "_cacache", "index-v5"), { recursive: true }); + writeFileSync(join(dir, "_cacache", "index-v5", "entry"), "cache"); +}); + +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +function cacheLookup(path: string): typeof import("node:child_process").spawnSync { + return (() => ({ + status: 0, + stdout: `${path}\n`, + stderr: "", + pid: 1, + output: [], + signal: null, + })) as never; +} + +describe("npm cache ownership pre-flight", () => { + test("accepts a cache owned by the current uid", () => { + const uid = process.getuid?.(); + if (uid === undefined) return; + expect(checkNpmCacheOwnership({ + getuid: () => uid, + spawn: cacheLookup(dir), + })).toEqual({ ok: true, cachePath: dir }); + }); + + test("finds a foreign-owned nested entry before package replacement", () => { + const uid = process.getuid?.(); + if (uid === undefined) return; + const foreign = join(dir, "_cacache", "index-v5", "entry"); + const issue = findForeignOwnedNpmCacheEntry(dir, uid, { + lstat: (path) => { + const stat = lstatSync(path); + return { + uid: path === foreign ? uid + 1 : stat.uid, + isDirectory: () => stat.isDirectory(), + }; + }, + readdir: path => readdirSync(path, { encoding: "utf8" }), + }); + expect(issue).toEqual({ kind: "foreign-owner", path: foreign, actualUid: uid + 1 }); + }); + + test("returns an actionable failure without changing the cache", () => { + const uid = process.getuid?.(); + if (uid === undefined) return; + const result = checkNpmCacheOwnership({ + getuid: () => uid + 1, + spawn: cacheLookup(dir), + }); + expect(result).toMatchObject({ + ok: false, + cachePath: dir, + entryPath: dir, + expectedUid: uid + 1, + actualUid: uid, + }); + if (result.ok !== false) throw new Error("expected ownership failure"); + expect(formatNpmCacheOwnershipFailure(result)).toContain("before stopping the proxy"); + expect(formatNpmCacheOwnershipFailure(result)).toContain("configure a user-owned npm cache"); + }); + + test("fails closed before shutdown when npm cannot resolve its cache", () => { + let inspected = false; + const result = checkNpmCacheOwnership({ + getuid: () => 501, + spawn: (() => ({ + status: null, + stdout: "", + stderr: "", + error: Object.assign(new Error("timed out"), { code: "ETIMEDOUT" }), + pid: 1, + output: [], + signal: "SIGTERM", + })) as never, + lstat: () => { + inspected = true; + throw new Error("must not inspect without a resolved cache path"); + }, + }); + expect(result).toMatchObject({ + ok: false, + expectedUid: 501, + reason: "could not resolve the npm cache (ETIMEDOUT)", + }); + expect(inspected).toBe(false); + if (result.ok !== false) throw new Error("expected lookup failure"); + expect(formatNpmCacheOwnershipFailure(result)).toContain("before stopping the proxy"); + }); + + test("skips uid checks on Windows without invoking npm", () => { + let spawned = false; + const result = checkNpmCacheOwnership({ + platform: "win32", + spawn: (() => { + spawned = true; + throw new Error("must not spawn"); + }) as never, + }); + expect(result.ok).toBe("skipped"); + expect(spawned).toBe(false); + }); +}); diff --git a/tests/update-stop-first.test.ts b/tests/update-stop-first.test.ts index 32ca3196c..ead85af05 100644 --- a/tests/update-stop-first.test.ts +++ b/tests/update-stop-first.test.ts @@ -27,6 +27,19 @@ describe("update stops the running proxy before replacing files", () => { expect(abortAt).toBeLessThan(stopAt); }); + test("npm cache ownership pre-flight runs before either updater stops the proxy", () => { + const cliGateAt = updateSource.indexOf("const cacheOwnership = checkNpmCacheOwnership()"); + const cliStopAt = updateSource.indexOf('[process.argv[1], "stop"]'); + const launcherGateAt = launcherSource.indexOf("const cacheOwnership = checkNpmCacheOwnership("); + const launcherStopAt = launcherSource.indexOf('[launcher, "stop"]'); + expect(cliGateAt).toBeGreaterThan(-1); + expect(launcherGateAt).toBeGreaterThan(-1); + expect(cliGateAt).toBeLessThan(cliStopAt); + expect(launcherGateAt).toBeLessThan(launcherStopAt); + expect(updateSource).toContain("formatNpmCacheOwnershipFailure(cacheOwnership)"); + expect(launcherSource).toContain("formatNpmCacheOwnershipFailure(cacheOwnership)"); + }); + test("npm launcher update path stops via its own launcher path before npm install", () => { expect(launcherSource).toContain('spawnSync(process.execPath, [launcher, "stop"]'); const stopAt = launcherSource.indexOf('[launcher, "stop"]'); From 7efed6c3a48697221535c72693fe75ac24797e45 Mon Sep 17 00:00:00 2001 From: wzb Date: Mon, 27 Jul 2026 12:57:08 +0800 Subject: [PATCH 02/16] fix(update): harden cache preflight and recovery --- .../src/content/docs/ja/reference/cli.md | 2 + .../src/content/docs/ko/reference/cli.md | 2 + docs-site/src/content/docs/reference/cli.md | 5 +- .../src/content/docs/ru/reference/cli.md | 2 + .../src/content/docs/zh-cn/reference/cli.md | 2 + src/update/job.ts | 65 ++++++++++--- src/update/npm-cache-preflight.d.mts | 3 +- src/update/npm-cache-preflight.mjs | 65 ++++++++++++- tests/update-job.test.ts | 97 +++++++++++++++++++ tests/update-npm-cache-preflight.test.ts | 53 +++++++++- tests/update-stop-first.test.ts | 9 +- .../windows-deploy-close-regressions.test.ts | 2 +- 12 files changed, 278 insertions(+), 29 deletions(-) diff --git a/docs-site/src/content/docs/ja/reference/cli.md b/docs-site/src/content/docs/ja/reference/cli.md index 442e21d53..a0e8d67af 100644 --- a/docs-site/src/content/docs/ja/reference/cli.md +++ b/docs-site/src/content/docs/ja/reference/cli.md @@ -423,6 +423,8 @@ npm から opencodex を自己更新します。安定版インストールは ` `git pull && bun install` を案内し、該当タグの最新版なら何もしません。ファイルを 差し替える前に実行中のプロキシを停止します。インストール済みのサービスは再ビルドして自動起動し、 フォアグラウンドインストールには次のステップとして `ocx start` を案内します。 +Unix では、更新前に設定済み npm キャッシュが現在のユーザー所有か確認します。所有者の異なる +エントリが見つかった場合やキャッシュを検査できない場合は、実行中のプロキシを停止する前に更新を中止します。 ```bash ocx update diff --git a/docs-site/src/content/docs/ko/reference/cli.md b/docs-site/src/content/docs/ko/reference/cli.md index 7ab091ef0..8708dfab7 100644 --- a/docs-site/src/content/docs/ko/reference/cli.md +++ b/docs-site/src/content/docs/ko/reference/cli.md @@ -442,6 +442,8 @@ npm에서 opencodex를 자체 업데이트합니다. 안정판 설치는 `@lates `git pull && bun install`을 안내하고, 해당 태그의 최신 버전이면 아무 작업도 하지 않습니다. 파일을 교체하기 전에 실행 중인 프록시를 중지합니다. 설치된 서비스는 다시 빌드해 자동으로 시작하고, 포그라운드 설치에는 다음 단계로 `ocx start`를 안내합니다. +Unix에서는 업데이트 전에 설정된 npm 캐시가 현재 사용자 소유인지 확인합니다. 다른 소유자의 항목이 +있거나 캐시를 검사할 수 없으면 실행 중인 프록시를 중지하기 전에 업데이트를 중단합니다. ```bash ocx update diff --git a/docs-site/src/content/docs/reference/cli.md b/docs-site/src/content/docs/reference/cli.md index a29f19c50..0c01b7693 100644 --- a/docs-site/src/content/docs/reference/cli.md +++ b/docs-site/src/content/docs/reference/cli.md @@ -501,8 +501,9 @@ unless you pass `--tag latest|preview`. It detects a source checkout and tells y tag. A running proxy is stopped before files are replaced; an installed service is rebuilt and started automatically, while a foreground installation prints `ocx start` as the next step. On Unix, the updater first checks that the configured npm cache is owned by the current user. It aborts -before stopping the proxy when it finds a foreign-owned cache entry, so you can correct the cache -ownership or configure a user-owned cache and retry without losing the running service. +before stopping the proxy when it finds a foreign-owned cache entry or cannot inspect the cache, so +you can correct the cache ownership or configure a user-owned cache and retry without losing the +running service. ```bash ocx update diff --git a/docs-site/src/content/docs/ru/reference/cli.md b/docs-site/src/content/docs/ru/reference/cli.md index 15a460279..655a8bbed 100644 --- a/docs-site/src/content/docs/ru/reference/cli.md +++ b/docs-site/src/content/docs/ru/reference/cli.md @@ -470,6 +470,8 @@ ocx debug usage logs [-f|--follow] если у вас уже новейшая версия для этого тега. Работающий прокси останавливается перед заменой файлов; установленный сервис пересобирается и запускается автоматически, а при установке, работающей на переднем плане, команда печатает `ocx start` как следующий шаг. +В Unix перед обновлением проверяется, что настроенный кеш npm принадлежит текущему пользователю. +Если найден элемент с другим владельцем или кеш невозможно проверить, обновление прерывается до остановки прокси. ```bash ocx update diff --git a/docs-site/src/content/docs/zh-cn/reference/cli.md b/docs-site/src/content/docs/zh-cn/reference/cli.md index 9b744afcb..999539854 100644 --- a/docs-site/src/content/docs/zh-cn/reference/cli.md +++ b/docs-site/src/content/docs/zh-cn/reference/cli.md @@ -424,6 +424,8 @@ provider debug 默认读取 `OCX_DEBUG=1`（旧的 `OCX_DEBUG_FRAMES=1` 仍可 `--tag latest|preview`。在源码 checkout 中，它会改为提示 `git pull && bun install`；如果已经是 相应 tag 的最新版，则不执行任何操作。替换文件前会停止正在运行的代理；已安装的服务会自动重建 并启动，而前台安装会把 `ocx start` 显示为下一步。 +在 Unix 上，更新器会先检查配置的 npm 缓存是否全部归当前用户所有。如果发现其他用户拥有的条目， +或无法检查缓存，更新会在停止代理前中止。 ```bash ocx update diff --git a/src/update/job.ts b/src/update/job.ts index 757ef764e..82257c268 100644 --- a/src/update/job.ts +++ b/src/update/job.ts @@ -2,10 +2,10 @@ import { spawn, spawnSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { atomicWriteFile, getConfigDir, loadConfig, readPid, readRuntimePort } from "../config"; +import { atomicWriteFile, getConfigDir, loadConfig, readPid } from "../config"; import { killProxy } from "../lib/process-control"; import { reclaimListenPort } from "../server/port-reclaim"; -import { isOpencodexHealthz, probeHostname, proxyIdentityAt, type HealthzIdentity } from "../server/proxy-liveness"; +import { findLiveProxy, isOpencodexHealthz, probeHostname, proxyIdentityAt, type HealthzIdentity } from "../server/proxy-liveness"; import { isServiceInstalled } from "../service"; import { type Channel, @@ -28,6 +28,8 @@ const UPDATE_TIMEOUT_MS = 180_000; const RESTART_TIMEOUT_MS = 60_000; const RESTART_HEALTH_TIMEOUT_MS = 15_000; const RESTART_STABILITY_WINDOW_MS = 15_000; +const FAILED_UPDATE_PROBE_ATTEMPTS = 3; +const FAILED_UPDATE_PROBE_DELAY_MS = 250; /** How long update restart waits for the captured port to become bindable after stop. */ export const RESTART_PORT_RECLAIM_MS = 30_000; @@ -308,6 +310,7 @@ export interface RestartIo { probeProxy?: (port: number, hostname?: string) => Promise; /** Richer /healthz read for update-correlated restart evidence (pid + version). */ probeProxyIdentity?: (port: number, hostname?: string) => Promise; + isProcessAlive?: (pid: number) => boolean; sleepMs?: (ms: number) => Promise; now?: () => number; /** Service-mode install/reinstall command (defaults to spawnSync via runLoggedCommand). */ @@ -526,6 +529,35 @@ async function defaultProbeProxyIdentity( } } +/** Check whether a captured PID still exists without signaling or replacing it. */ +function defaultIsProcessAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return !!(error && typeof error === "object" && "code" in error && error.code === "EPERM"); + } +} + +/** Retry identity-aware health checks before considering a recovery restart. */ +async function probeFailedUpdateProxy( + captured: { port: number; hostname: string }, + io: RestartIo, +): Promise { + const probeIdentity = io.probeProxyIdentity ?? defaultProbeProxyIdentity; + const sleep = io.sleepMs ?? (async (ms: number) => { + await new Promise(resolve => setTimeout(resolve, ms)); + }); + for (let attempt = 0; attempt < FAILED_UPDATE_PROBE_ATTEMPTS; attempt += 1) { + try { + const identity = await probeIdentity(captured.port, captured.hostname); + if (identity) return identity; + } catch { /* a custom probe failure is equivalent to no health response */ } + if (attempt + 1 < FAILED_UPDATE_PROBE_ATTEMPTS) await sleep(FAILED_UPDATE_PROBE_DELAY_MS); + } + return null; +} + /** Keep a failed GUI install from also leaving a previously-active proxy offline. */ async function recoverFailedGuiUpdate( job: UpdateJobState, @@ -535,19 +567,25 @@ async function recoverFailedGuiUpdate( ): Promise { if (!proxyWasActive) return "not-needed"; - const probeIdentity = io.probeProxyIdentity ?? defaultProbeProxyIdentity; - if (await probeIdentity(captured.port, captured.hostname)) { + if (await probeFailedUpdateProxy(captured, io)) { updateJob(job, {}, `Update command failed, but the existing proxy remains healthy on ${captured.hostname}:${captured.port}.`); return "still-running"; } + const oldPidStillAlive = captured.oldPid != null + && (io.isProcessAlive ?? defaultIsProcessAlive)(captured.oldPid); + if (oldPidStillAlive) { + updateJob(job, {}, `Update command failed and health probes timed out, but pre-update PID ${captured.oldPid} is still alive; refusing an automatic restart.`); + return "still-running"; + } + updateJob(job, {}, "Update command failed after the proxy stopped; attempting to restore the proxy..."); try { const restartFn = io.restartAfterUpdateFn ?? restartAfterUpdate; await restartFn(job, captured, io); const healthy = await awaitRestartedProxyHealthy(job, captured, io); if (healthy.ok) { - updateJob(job, {}, `Previous proxy service restored on ${captured.hostname}:${captured.port}; the update itself still failed.`); + updateJob(job, {}, `Previous proxy availability restored on ${captured.hostname}:${captured.port}; the update itself still failed.`); return "restarted"; } } catch (error) { @@ -723,21 +761,18 @@ export async function runGuiUpdateWorker(jobId: string, channel: Channel, restar let job = readUpdateJob(jobId); const check = checkForUpdate(channel); const now = new Date().toISOString(); - // Capture the live listen target BEFORE the update command runs: the stop-first update - // flow clears pid/runtime state, so this is the last moment the real port is knowable. - // Only trust runtime-port.json when its pid matches the live pidfile process. - const rt = readRuntimePort(); - const livePid = readPid(); + // Capture identity-checked liveness BEFORE the update command runs. Installation or + // runtime metadata alone must not turn an intentionally stopped service back on. const preUpdateConfig = loadConfig(); - const runtimeTrusted = !!(rt && livePid && rt.pid === livePid); - const proxyWasActive = isServiceInstalled() || runtimeTrusted; + const liveBeforeUpdate = await findLiveProxy(); + const proxyWasActive = liveBeforeUpdate !== null; const configPort = typeof preUpdateConfig.port === "number" && preUpdateConfig.port > 0 ? preUpdateConfig.port : 10100; const captured = { - port: runtimeTrusted ? rt.port : configPort, - hostname: (runtimeTrusted ? rt.hostname : undefined) ?? preUpdateConfig.hostname ?? "127.0.0.1", - ...(runtimeTrusted && livePid ? { oldPid: livePid } : {}), + port: liveBeforeUpdate?.port ?? configPort, + hostname: liveBeforeUpdate?.hostname ?? preUpdateConfig.hostname ?? "127.0.0.1", + ...(liveBeforeUpdate?.pid ? { oldPid: liveBeforeUpdate.pid } : {}), }; let trayWasInstalled = false; let trayWasRunning = false; diff --git a/src/update/npm-cache-preflight.d.mts b/src/update/npm-cache-preflight.d.mts index 5529e1eb0..c1d10a408 100644 --- a/src/update/npm-cache-preflight.d.mts +++ b/src/update/npm-cache-preflight.d.mts @@ -11,6 +11,7 @@ export interface NpmCachePreflightOptions { spawn?: typeof import("node:child_process").spawnSync; lstat?: (path: string) => NpmCacheEntryStat; readdir?: (path: string) => string[]; + realpath?: (path: string) => string; } export type NpmCacheOwnershipIssue = @@ -32,7 +33,7 @@ export type NpmCacheOwnershipResult = export declare function findForeignOwnedNpmCacheEntry( cachePath: string, expectedUid: number, - io?: Pick, + io?: Pick, ): NpmCacheOwnershipIssue | null; export declare function checkNpmCacheOwnership( diff --git a/src/update/npm-cache-preflight.mjs b/src/update/npm-cache-preflight.mjs index 3a3e0fcdb..829108796 100644 --- a/src/update/npm-cache-preflight.mjs +++ b/src/update/npm-cache-preflight.mjs @@ -1,7 +1,9 @@ import { spawnSync } from "node:child_process"; -import { lstatSync, readdirSync } from "node:fs"; -import { resolve } from "node:path"; +import { lstatSync, readdirSync, realpathSync } from "node:fs"; +import { homedir } from "node:os"; +import { isAbsolute, relative, resolve, sep } from "node:path"; +/** Extract a stable filesystem error code without serializing the full error. */ function errorCode(error) { return error && typeof error === "object" && "code" in error ? String(error.code) @@ -12,7 +14,21 @@ function errorCode(error) { export function findForeignOwnedNpmCacheEntry(cachePath, expectedUid, io = {}) { const lstat = io.lstat ?? lstatSync; const readdir = io.readdir ?? (path => readdirSync(path, { encoding: "utf8" })); - const stack = [resolve(cachePath)]; + const realpath = io.realpath ?? realpathSync; + let cacheRoot; + try { + // npm follows a configured cache-root symlink. Resolve only that root; nested + // symlinks remain lstat-only and are never traversed. + cacheRoot = realpath(resolve(cachePath)); + } catch (error) { + if (errorCode(error) === "ENOENT") return null; + return { + kind: "error", + path: resolve(cachePath), + reason: `could not resolve npm cache root (${errorCode(error)})`, + }; + } + const stack = [cacheRoot]; while (stack.length > 0) { const path = stack.pop(); @@ -31,6 +47,9 @@ export function findForeignOwnedNpmCacheEntry(cachePath, expectedUid, io = {}) { if (Number.isInteger(stat.uid) && stat.uid !== expectedUid) { return { kind: "foreign-owner", path, actualUid: stat.uid }; } + if (path === cacheRoot && !stat.isDirectory()) { + return { kind: "error", path, reason: "npm cache root is not a directory" }; + } if (!stat.isDirectory()) continue; let entries; @@ -49,6 +68,38 @@ export function findForeignOwnedNpmCacheEntry(cachePath, expectedUid, io = {}) { return null; } +/** Redact path segments that commonly carry secrets before they reach update logs. */ +function sanitizePathSegments(path) { + return path + .split(/[\\/]/) + .map(segment => /(?:secret|password|passwd|token|api[-_]?key|apikey|credential|email)/i.test(segment) + ? "[REDACTED]" + : segment) + .join("/"); +} + +/** Render paths home-relative, or hide absolute paths outside the current home. */ +function displayUserPath(path) { + const absolute = resolve(path); + const home = resolve(homedir()); + const fromHome = relative(home, absolute); + if (fromHome === "") return "~"; + if (fromHome !== ".." && !fromHome.startsWith(`..${sep}`) && !isAbsolute(fromHome)) { + return `~/${sanitizePathSegments(fromHome)}`; + } + return "[configured npm cache]"; +} + +/** Prefer a cache-relative entry while preserving the same account-name redaction. */ +function displayCacheEntry(cachePath, entryPath) { + const fromCache = relative(resolve(cachePath), resolve(entryPath)); + if (fromCache === "") return displayUserPath(cachePath); + if (fromCache !== ".." && !fromCache.startsWith(`..${sep}`) && !isAbsolute(fromCache)) { + return `${displayUserPath(cachePath)}/${sanitizePathSegments(fromCache)}`; + } + return displayUserPath(entryPath); +} + /** * npm can abort after it has retired the installed package when its cache contains * root-owned entries. Detect that state before an updater stops the running proxy. @@ -114,11 +165,15 @@ export function checkNpmCacheOwnership(options = {}) { }; } +/** Format an actionable update error without persisting an OS account name. */ export function formatNpmCacheOwnershipFailure(result) { + const entry = result.entryPath + ? displayCacheEntry(result.cachePath ?? result.entryPath, result.entryPath) + : null; return [ "npm cache ownership pre-flight failed before stopping the proxy.", - result.entryPath ? `${result.reason}: ${result.entryPath}` : result.reason, - ...(result.cachePath ? [`Cache: ${result.cachePath}`] : []), + entry ? `${result.reason}: ${entry}` : result.reason, + ...(result.cachePath ? [`Cache: ${displayUserPath(result.cachePath)}`] : []), "Correct the cache ownership or configure a user-owned npm cache, then retry.", ].join("\n"); } diff --git a/tests/update-job.test.ts b/tests/update-job.test.ts index f8c316294..618a035cc 100644 --- a/tests/update-job.test.ts +++ b/tests/update-job.test.ts @@ -317,6 +317,102 @@ describe("GUI update execution decisions", () => { expect(readUpdateJob(job.id)?.log.some(line => line.includes("existing proxy remains healthy"))).toBe(true); }); + test("failed install does not start a proxy that was inactive before the update", async () => { + let restartCalls = 0; + const job: UpdateJobState = { + id: "failed-install-inactive", + status: "running", + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + currentVersion: "2.7.40", + latestVersion: "2.7.41", + channel: "latest", + installer: "npm", + restart: true, + command: "", + releaseNotesUrl: "", + log: [], + }; + writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + const recovery = await recoverFailedGuiUpdateForTests( + job, + { port: 10100, hostname: "127.0.0.1" }, + false, + { + probeProxyIdentity: async () => { throw new Error("must not probe an inactive proxy"); }, + restartAfterUpdateFn: async () => { restartCalls += 1; }, + }, + ); + expect(recovery).toBe("not-needed"); + expect(restartCalls).toBe(0); + }); + + test("failed install retries a transient health miss before considering restart", async () => { + let probes = 0; + let restartCalls = 0; + const job: UpdateJobState = { + id: "failed-install-transient-probe", + status: "running", + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + currentVersion: "2.7.40", + latestVersion: "2.7.41", + channel: "latest", + installer: "npm", + restart: true, + command: "", + releaseNotesUrl: "", + log: [], + }; + writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + const recovery = await recoverFailedGuiUpdateForTests( + job, + { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, + true, + { + probeProxyIdentity: async () => (++probes === 1 ? null : { pid: 111, version: "2.7.40" }), + sleepMs: async () => {}, + restartAfterUpdateFn: async () => { restartCalls += 1; }, + }, + ); + expect(recovery).toBe("still-running"); + expect(probes).toBe(2); + expect(restartCalls).toBe(0); + }); + + test("failed install preserves a captured PID that remains alive after health timeouts", async () => { + let restartCalls = 0; + const job: UpdateJobState = { + id: "failed-install-live-pid", + status: "running", + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + currentVersion: "2.7.40", + latestVersion: "2.7.41", + channel: "latest", + installer: "npm", + restart: true, + command: "", + releaseNotesUrl: "", + log: [], + }; + writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + const recovery = await recoverFailedGuiUpdateForTests( + job, + { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, + true, + { + probeProxyIdentity: async () => null, + isProcessAlive: () => true, + sleepMs: async () => {}, + restartAfterUpdateFn: async () => { restartCalls += 1; }, + }, + ); + expect(recovery).toBe("still-running"); + expect(restartCalls).toBe(0); + expect(readUpdateJob(job.id)?.log.some(line => line.includes("refusing an automatic restart"))).toBe(true); + }); + test("failed install restores a proxy that the updater stopped", async () => { let now = 0; let restarted = false; @@ -341,6 +437,7 @@ describe("GUI update execution decisions", () => { true, { probeProxyIdentity: async () => null, + isProcessAlive: () => false, probeProxy: async () => restarted, restartAfterUpdateFn: async () => { restarted = true; }, now: () => now, diff --git a/tests/update-npm-cache-preflight.test.ts b/tests/update-npm-cache-preflight.test.ts index f58f7dbec..10017b479 100644 --- a/tests/update-npm-cache-preflight.test.ts +++ b/tests/update-npm-cache-preflight.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { lstatSync, mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; +import { lstatSync, mkdirSync, readdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; import { checkNpmCacheOwnership, @@ -9,15 +9,18 @@ import { } from "../src/update/npm-cache-preflight.mjs"; let dir: string; +let extraPaths: string[]; beforeEach(() => { dir = join(tmpdir(), `ocx-npm-cache-${process.pid}-${Date.now()}-${Math.random().toString(36).slice(2)}`); mkdirSync(join(dir, "_cacache", "index-v5"), { recursive: true }); writeFileSync(join(dir, "_cacache", "index-v5", "entry"), "cache"); + extraPaths = []; }); afterEach(() => { rmSync(dir, { recursive: true, force: true }); + for (const path of extraPaths) rmSync(path, { recursive: true, force: true }); }); function cacheLookup(path: string): typeof import("node:child_process").spawnSync { @@ -44,7 +47,7 @@ describe("npm cache ownership pre-flight", () => { test("finds a foreign-owned nested entry before package replacement", () => { const uid = process.getuid?.(); if (uid === undefined) return; - const foreign = join(dir, "_cacache", "index-v5", "entry"); + const foreign = join(realpathSync(dir), "_cacache", "index-v5", "entry"); const issue = findForeignOwnedNpmCacheEntry(dir, uid, { lstat: (path) => { const stat = lstatSync(path); @@ -58,6 +61,34 @@ describe("npm cache ownership pre-flight", () => { expect(issue).toEqual({ kind: "foreign-owner", path: foreign, actualUid: uid + 1 }); }); + test("follows a configured cache-root symlink but not nested symlinks", () => { + const uid = process.getuid?.(); + if (uid === undefined) return; + const cacheLink = `${dir}-link`; + const outside = `${dir}-outside`; + extraPaths.push(cacheLink, outside); + mkdirSync(outside, { recursive: true }); + writeFileSync(join(outside, "foreign"), "outside"); + symlinkSync(dir, cacheLink, "dir"); + symlinkSync(outside, join(dir, "nested-link"), "dir"); + const foreign = join(realpathSync(dir), "_cacache", "index-v5", "entry"); + const outsideRoot = realpathSync(outside); + let inspectedOutside = false; + const issue = findForeignOwnedNpmCacheEntry(cacheLink, uid, { + lstat: (path) => { + if (path.startsWith(outsideRoot)) inspectedOutside = true; + const stat = lstatSync(path); + return { + uid: path === foreign ? uid + 1 : stat.uid, + isDirectory: () => stat.isDirectory(), + }; + }, + readdir: path => readdirSync(path, { encoding: "utf8" }), + }); + expect(issue).toEqual({ kind: "foreign-owner", path: foreign, actualUid: uid + 1 }); + expect(inspectedOutside).toBe(false); + }); + test("returns an actionable failure without changing the cache", () => { const uid = process.getuid?.(); if (uid === undefined) return; @@ -68,7 +99,7 @@ describe("npm cache ownership pre-flight", () => { expect(result).toMatchObject({ ok: false, cachePath: dir, - entryPath: dir, + entryPath: realpathSync(dir), expectedUid: uid + 1, actualUid: uid, }); @@ -77,6 +108,20 @@ describe("npm cache ownership pre-flight", () => { expect(formatNpmCacheOwnershipFailure(result)).toContain("configure a user-owned npm cache"); }); + test("formats cache failures without persisting the OS account name", () => { + const cachePath = join(homedir(), ".npm"); + const output = formatNpmCacheOwnershipFailure({ + ok: false, + cachePath, + entryPath: join(cachePath, "_cacache", "foreign"), + expectedUid: 502, + actualUid: 0, + reason: "foreign owner", + }); + expect(output).toContain("~/.npm/_cacache/foreign"); + expect(output).not.toContain(homedir()); + }); + test("fails closed before shutdown when npm cannot resolve its cache", () => { let inspected = false; const result = checkNpmCacheOwnership({ diff --git a/tests/update-stop-first.test.ts b/tests/update-stop-first.test.ts index ead85af05..42b59c85f 100644 --- a/tests/update-stop-first.test.ts +++ b/tests/update-stop-first.test.ts @@ -3,6 +3,7 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; const updateSource = readFileSync(join(import.meta.dir, "..", "src", "update", "index.ts"), "utf8"); +const updateJobSource = readFileSync(join(import.meta.dir, "..", "src", "update", "job.ts"), "utf8"); const launcherSource = readFileSync(join(import.meta.dir, "..", "bin", "ocx.mjs"), "utf8"); const serverSource = readFileSync(join(import.meta.dir, "..", "src", "server", "index.ts"), "utf8"); const cliSource = readFileSync(join(import.meta.dir, "..", "src", "cli", "index.ts"), "utf8"); @@ -63,7 +64,7 @@ describe("update stops the running proxy before replacing files", () => { expect(launcherSource).toContain("OCX_BAKE_PORT"); // Live runtime port 10100 must not be discarded as a missing-port sentinel. expect(launcherSource).toContain("sawRuntimePort"); - expect(updateSource).toContain("runtimeTrusted"); + expect(updateJobSource).toContain("const liveBeforeUpdate = await findLiveProxy()"); }); test("both update paths surface a skipped history restore after the stop", () => { @@ -88,6 +89,12 @@ describe("update stops the running proxy before replacing files", () => { expect(launcherSource).toContain("stopRes.status !== 0 || stillHasRuntimeState"); }); + test("GUI failure recovery is gated by identity-checked pre-update liveness", () => { + expect(updateJobSource).toContain("const liveBeforeUpdate = await findLiveProxy()"); + expect(updateJobSource).toContain("const proxyWasActive = liveBeforeUpdate !== null"); + expect(updateJobSource).not.toContain("const proxyWasActive = isServiceInstalled() || runtimeTrusted"); + }); + test("GUI worker update children use pipe stdio so Windows npm.cmd does not open consoles", () => { expect(updateSource).toContain("function updateChildStdio()"); expect(updateSource).toContain('process.env.OCX_SERVICE === "1"'); diff --git a/tests/windows-deploy-close-regressions.test.ts b/tests/windows-deploy-close-regressions.test.ts index 2d9a89afb..10ebcad5f 100644 --- a/tests/windows-deploy-close-regressions.test.ts +++ b/tests/windows-deploy-close-regressions.test.ts @@ -27,7 +27,7 @@ describe("update-job restart avoids the shell-less .cmd EINVAL (Windows, bun/sou expect(src).toContain("OCX_BAKE_PORT"); // Service reinstall still runs (with bake) even when reclaim warns; direct start refuses to hop. expect(src).toContain("refusing to hop"); - expect(src).toContain("runtimeTrusted"); + expect(src).toContain("const liveBeforeUpdate = await findLiveProxy()"); expect(read("src/cli/index.ts")).toContain("allowEphemeralFallback: !hardPin"); expect(read("src/cli/index.ts")).toContain("preferRetryMs: hardPin ? 0 : 750"); expect(read("src/cli/index.ts")).toContain("Not opening the GUI"); From 6db4dc7f205c2b091dd48815a181d6af4f875808 Mon Sep 17 00:00:00 2001 From: wzb Date: Mon, 27 Jul 2026 13:33:47 +0800 Subject: [PATCH 03/16] fix(update): close failed-install recovery gaps --- bin/ocx.mjs | 3 +- docs/adr/0001-gui-update-worker.md | 17 +- src/update/job.ts | 172 ++++++++++++++---- src/update/npm-cache-preflight.d.mts | 8 +- src/update/npm-cache-preflight.mjs | 55 +++++- tests/update-job.test.ts | 88 +++++++-- tests/update-npm-cache-preflight.test.ts | 46 ++++- tests/update-stop-first.test.ts | 22 ++- .../windows-deploy-close-regressions.test.ts | 2 +- 9 files changed, 350 insertions(+), 63 deletions(-) diff --git a/bin/ocx.mjs b/bin/ocx.mjs index 42f4b6476..e66571b32 100755 --- a/bin/ocx.mjs +++ b/bin/ocx.mjs @@ -18,6 +18,7 @@ import { checkNpmCacheOwnership, formatNpmCacheOwnershipFailure } from "../src/u import { handoffWindowsTrayForUpdate, planWindowsTrayUpdate } from "../src/update/tray-update-plan.mjs"; const PKG = "@bitkyc08/opencodex"; +const NPM_INSTALL_TIMEOUT_MS = 180_000; const require = createRequire(import.meta.url); const here = dirname(fileURLToPath(import.meta.url)); const cliPath = join(here, "..", "src", "cli", "index.ts"); @@ -231,7 +232,7 @@ function runNpmSelfUpdate() { console.log(`Updating${latest ? ` to v${latest}` : ""}...\n$ ${npm} install -g ${PKG}@${tag}`); const res = spawnSync(npm, ["install", "-g", `${PKG}@${tag}`], { stdio: "inherit", - timeout: 180000, + timeout: NPM_INSTALL_TIMEOUT_MS, windowsHide: true, shell: winShell, }); diff --git a/docs/adr/0001-gui-update-worker.md b/docs/adr/0001-gui-update-worker.md index 1c1067487..4f08a3ce5 100644 --- a/docs/adr/0001-gui-update-worker.md +++ b/docs/adr/0001-gui-update-worker.md @@ -25,13 +25,18 @@ bun run build:gui`. Before an npm updater stops the proxy, it resolves the configured npm cache and checks that every entry is owned by the current Unix user. A foreign-owned entry (commonly left by an older `sudo npm` invocation) aborts the update with an actionable error while the existing proxy and service remain -running. Failure to resolve or inspect the cache also aborts before shutdown; platforms without -Unix uid ownership skip this gate. +running. The scan fails closed when the cache root is absent, an entry cannot be inspected, or its +50,000-entry / 10-second traversal budget is exhausted. Platforms without Unix uid ownership skip +this gate. -If the installer still exits nonzero, the GUI worker first checks whether the pre-update proxy is -still healthy. It leaves that process alone when an early gate failed; when the updater already -stopped it, the worker makes a best-effort restart and verifies health. The update job remains -failed either way because restoring availability is not the same as installing the new version. +The GUI worker retries its identity-checked pre-update liveness capture before deciding a service +was inactive. If the installer still exits nonzero, it probes again and leaves the old process alone +only when health or PID identity still proves it is OpenCodex. When npm already stopped the proxy +and retired the old package, the worker validates the hidden package's name, version, and launcher, +uses that launcher for a best-effort restart, and verifies health. The update job remains failed +either way because restoring availability is not the same as installing the new version. The GUI +worker's install timeout also exceeds the nested npm install deadline, so recovery cannot race a +still-running npm replacement child. After an update requests a restart, the worker now waits for an identity-checked `/healthz` to return and remain healthy for a short stability window before marking the job successful. This diff --git a/src/update/job.ts b/src/update/job.ts index 82257c268..308a5ef9b 100644 --- a/src/update/job.ts +++ b/src/update/job.ts @@ -1,11 +1,18 @@ import { spawn, spawnSync } from "node:child_process"; -import { existsSync, mkdirSync, readFileSync } from "node:fs"; -import { dirname, join } from "node:path"; +import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync } from "node:fs"; +import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; -import { atomicWriteFile, getConfigDir, loadConfig, readPid } from "../config"; +import { atomicWriteFile, getConfigDir, loadConfig, readPid, verifyPidIdentity } from "../config"; import { killProxy } from "../lib/process-control"; import { reclaimListenPort } from "../server/port-reclaim"; -import { findLiveProxy, isOpencodexHealthz, probeHostname, proxyIdentityAt, type HealthzIdentity } from "../server/proxy-liveness"; +import { + findLiveProxy, + isOpencodexHealthz, + probeHostname, + proxyIdentityAt, + type HealthzIdentity, + type LiveProxy, +} from "../server/proxy-liveness"; import { isServiceInstalled } from "../service"; import { type Channel, @@ -24,12 +31,12 @@ import { handoffWindowsTrayForUpdate, planWindowsTrayUpdate } from "./tray-updat const RELEASE_NOTES_URL = "https://github.com/lidge-jun/opencodex/releases/latest"; const UPDATE_JOB_FILENAME = "update-job.json"; -const UPDATE_TIMEOUT_MS = 180_000; +const UPDATE_TIMEOUT_MS = 360_000; const RESTART_TIMEOUT_MS = 60_000; const RESTART_HEALTH_TIMEOUT_MS = 15_000; const RESTART_STABILITY_WINDOW_MS = 15_000; -const FAILED_UPDATE_PROBE_ATTEMPTS = 3; -const FAILED_UPDATE_PROBE_DELAY_MS = 250; +const UPDATE_LIVENESS_PROBE_ATTEMPTS = 3; +const UPDATE_LIVENESS_PROBE_DELAY_MS = 250; /** How long update restart waits for the captured port to become bindable after stop. */ export const RESTART_PORT_RECLAIM_MS = 30_000; @@ -94,6 +101,70 @@ function packageLauncherPath(): string { return join(dirname(fileURLToPath(import.meta.url)), "..", "..", "bin", "ocx.mjs"); } +interface ValidatedNpmLauncher { + launcher: string; + mtimeMs: number; +} + +function isPathInside(parent: string, child: string): boolean { + const fromParent = relative(parent, child); + return fromParent !== "" + && fromParent !== ".." + && !fromParent.startsWith(`..${sep}`) + && !isAbsolute(fromParent); +} + +/** Validate that a package root still contains the exact pre-update package launcher. */ +function validateNpmRecoveryPackage(packageRoot: string, expectedVersion: string): ValidatedNpmLauncher | null { + try { + const rootStat = lstatSync(packageRoot); + if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) return null; + + const manifestPath = join(packageRoot, "package.json"); + const manifestStat = lstatSync(manifestPath); + if (!manifestStat.isFile() || manifestStat.isSymbolicLink()) return null; + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as { name?: unknown; version?: unknown }; + if (manifest.name !== PKG || manifest.version !== expectedVersion) return null; + + const launcherPath = join(packageRoot, "bin", "ocx.mjs"); + const launcherStat = lstatSync(launcherPath); + if (!launcherStat.isFile() || launcherStat.isSymbolicLink()) return null; + const canonicalRoot = realpathSync(packageRoot); + const canonicalLauncher = realpathSync(launcherPath); + if (!isPathInside(canonicalRoot, canonicalLauncher)) return null; + return { launcher: canonicalLauncher, mtimeMs: rootStat.mtimeMs }; + } catch { + return null; + } +} + +/** + * npm retires a package to a hidden sibling such as `.opencodex-Ab12Cd34` + * before replacing it. On a late install failure, prefer an intact current root; + * otherwise recover through the validated retired copy of the pre-update version. + */ +export function findNpmRecoveryLauncher( + currentLauncher: string, + expectedVersion: string, +): string | null { + const packageRoot = dirname(dirname(resolve(currentLauncher))); + const current = validateNpmRecoveryPackage(packageRoot, expectedVersion); + if (current) return current.launcher; + + const scopeRoot = dirname(packageRoot); + const prefix = `.${basename(packageRoot)}-`; + try { + const candidates = readdirSync(scopeRoot, { encoding: "utf8" }) + .filter(name => name.startsWith(prefix) && /^[A-Za-z0-9]{8}$/.test(name.slice(prefix.length))) + .map(name => validateNpmRecoveryPackage(join(scopeRoot, name), expectedVersion)) + .filter((candidate): candidate is ValidatedNpmLauncher => candidate !== null) + .sort((a, b) => b.mtimeMs - a.mtimeMs); + return candidates[0]?.launcher ?? null; + } catch { + return null; + } +} + function formatCommand(bin: string, args: string[]): string { return `${bin} ${args.join(" ")}`; } @@ -280,8 +351,13 @@ function runLoggedCommand(job: UpdateJobState, bin: string, args: string[], time return { status: result.status, signal: result.signal }; } -function spawnDetachedStart(job: UpdateJobState, installer: Installer, port?: number): void { - const cmd = restartCommand(false, installer, packageLauncherPath(), port); +function spawnDetachedStart( + job: UpdateJobState, + installer: Installer, + port?: number, + launcher = packageLauncherPath(), +): void { + const cmd = restartCommand(false, installer, launcher, port); const env = { ...process.env }; delete env.OCX_SERVICE; updateJob(job, {}, `$ ${cmd.display}`); @@ -305,12 +381,13 @@ export type FailedUpdateRecovery = "not-needed" | "still-running" | "restarted" /** Test seam: the wait/spawn pair is injectable so the restart path is verifiable. */ export interface RestartIo { waitForPort?: typeof reclaimListenPort; - spawnStart?: (job: UpdateJobState, installer: Installer, port?: number) => void; + spawnStart?: (job: UpdateJobState, installer: Installer, port?: number, launcher?: string) => void; serviceInstalledFn?: () => boolean; probeProxy?: (port: number, hostname?: string) => Promise; /** Richer /healthz read for update-correlated restart evidence (pid + version). */ probeProxyIdentity?: (port: number, hostname?: string) => Promise; - isProcessAlive?: (pid: number) => boolean; + verifyPidIdentityFn?: (pid: number) => number | null; + recoveryLauncherFn?: (currentLauncher: string, expectedVersion: string) => string | null; sleepMs?: (ms: number) => Promise; now?: () => number; /** Service-mode install/reinstall command (defaults to spawnSync via runLoggedCommand). */ @@ -322,14 +399,14 @@ export interface RestartIo { /** Override the explicit restart path (used by finishGuiUpdateRestart tests). */ restartAfterUpdateFn?: ( job: UpdateJobState, - captured?: { port: number; hostname: string; oldPid?: number }, + captured?: { port: number; hostname: string; oldPid?: number; recoveryLauncher?: string }, io?: RestartIo, ) => Promise; } async function restartAfterUpdate( job: UpdateJobState, - captured?: { port: number; hostname: string; oldPid?: number }, + captured?: { port: number; hostname: string; oldPid?: number; recoveryLauncher?: string }, io: RestartIo = {}, ): Promise { const serviceInstalled = (io.serviceInstalledFn ?? isServiceInstalled)(); @@ -349,7 +426,8 @@ async function restartAfterUpdate( svcArgs = serviceReinstallArgs(); } catch { /* fallback to default service install */ } } - const cmd = restartCommand(serviceInstalled, job.installer, packageLauncherPath(), port, svcArgs); + const launcher = captured?.recoveryLauncher ?? packageLauncherPath(); + const cmd = restartCommand(serviceInstalled, job.installer, launcher, port, svcArgs); const waitFn = io.waitForPort ?? reclaimListenPort; const reclaimOpts = { timeoutMs: RESTART_PORT_RECLAIM_MS, @@ -404,13 +482,13 @@ async function restartAfterUpdate( updateJob(job, {}, `Port ${port} still busy after ${Math.trunc(RESTART_PORT_RECLAIM_MS / 1000)}s (reclaim could not free the socket); not starting on another port. Retry 'ocx start --port ${port}'.`); return; } - (io.spawnStart ?? spawnDetachedStart)(job, job.installer, port); + (io.spawnStart ?? spawnDetachedStart)(job, job.installer, port, launcher); } /** Exposed for tests: drives the non-service restart path with injected io. */ export function restartAfterUpdateForTests( job: UpdateJobState, - captured: { port: number; hostname: string; oldPid?: number }, + captured: { port: number; hostname: string; oldPid?: number; recoveryLauncher?: string }, io: RestartIo, ): Promise { return restartAfterUpdate(job, captured, io); @@ -529,16 +607,6 @@ async function defaultProbeProxyIdentity( } } -/** Check whether a captured PID still exists without signaling or replacing it. */ -function defaultIsProcessAlive(pid: number): boolean { - try { - process.kill(pid, 0); - return true; - } catch (error) { - return !!(error && typeof error === "object" && "code" in error && error.code === "EPERM"); - } -} - /** Retry identity-aware health checks before considering a recovery restart. */ async function probeFailedUpdateProxy( captured: { port: number; hostname: string }, @@ -548,12 +616,29 @@ async function probeFailedUpdateProxy( const sleep = io.sleepMs ?? (async (ms: number) => { await new Promise(resolve => setTimeout(resolve, ms)); }); - for (let attempt = 0; attempt < FAILED_UPDATE_PROBE_ATTEMPTS; attempt += 1) { + for (let attempt = 0; attempt < UPDATE_LIVENESS_PROBE_ATTEMPTS; attempt += 1) { try { const identity = await probeIdentity(captured.port, captured.hostname); if (identity) return identity; } catch { /* a custom probe failure is equivalent to no health response */ } - if (attempt + 1 < FAILED_UPDATE_PROBE_ATTEMPTS) await sleep(FAILED_UPDATE_PROBE_DELAY_MS); + if (attempt + 1 < UPDATE_LIVENESS_PROBE_ATTEMPTS) await sleep(UPDATE_LIVENESS_PROBE_DELAY_MS); + } + return null; +} + +/** Retry the pre-update identity lookup before declaring an active proxy absent. */ +export async function findLiveProxyForUpdate( + probe: () => Promise = findLiveProxy, + sleep: (ms: number) => Promise = async (ms: number) => { + await new Promise(resolve => setTimeout(resolve, ms)); + }, +): Promise { + for (let attempt = 0; attempt < UPDATE_LIVENESS_PROBE_ATTEMPTS; attempt += 1) { + try { + const live = await probe(); + if (live) return live; + } catch { /* retry a transient liveness lookup failure */ } + if (attempt + 1 < UPDATE_LIVENESS_PROBE_ATTEMPTS) await sleep(UPDATE_LIVENESS_PROBE_DELAY_MS); } return null; } @@ -572,18 +657,35 @@ async function recoverFailedGuiUpdate( return "still-running"; } - const oldPidStillAlive = captured.oldPid != null - && (io.isProcessAlive ?? defaultIsProcessAlive)(captured.oldPid); - if (oldPidStillAlive) { - updateJob(job, {}, `Update command failed and health probes timed out, but pre-update PID ${captured.oldPid} is still alive; refusing an automatic restart.`); + const oldPidIdentityMatches = captured.oldPid != null + && (io.verifyPidIdentityFn ?? verifyPidIdentity)(captured.oldPid) === captured.oldPid; + if (oldPidIdentityMatches) { + updateJob(job, {}, `Update command failed and health probes timed out, but pre-update PID ${captured.oldPid} still identifies as OpenCodex; refusing an automatic restart.`); return "still-running"; } updateJob(job, {}, "Update command failed after the proxy stopped; attempting to restore the proxy..."); try { + let recoveryCaptured: { + port: number; + hostname: string; + oldPid?: number; + recoveryLauncher?: string; + } = captured; + if (job.installer === "npm") { + const currentLauncher = packageLauncherPath(); + const resolveRecoveryLauncher = io.recoveryLauncherFn ?? findNpmRecoveryLauncher; + const recoveryLauncher = resolveRecoveryLauncher(currentLauncher, job.currentVersion); + if (!recoveryLauncher) { + updateJob(job, {}, "Could not find a validated current or npm-retired launcher for the pre-update package."); + updateJob(job, {}, `Automatic proxy recovery failed. Restore the package, then run 'ocx service install' or 'ocx start --port ${captured.port}'.`); + return "failed"; + } + recoveryCaptured = { ...captured, recoveryLauncher }; + } const restartFn = io.restartAfterUpdateFn ?? restartAfterUpdate; - await restartFn(job, captured, io); - const healthy = await awaitRestartedProxyHealthy(job, captured, io); + await restartFn(job, recoveryCaptured, io); + const healthy = await awaitRestartedProxyHealthy(job, recoveryCaptured, io); if (healthy.ok) { updateJob(job, {}, `Previous proxy availability restored on ${captured.hostname}:${captured.port}; the update itself still failed.`); return "restarted"; @@ -764,7 +866,7 @@ export async function runGuiUpdateWorker(jobId: string, channel: Channel, restar // Capture identity-checked liveness BEFORE the update command runs. Installation or // runtime metadata alone must not turn an intentionally stopped service back on. const preUpdateConfig = loadConfig(); - const liveBeforeUpdate = await findLiveProxy(); + const liveBeforeUpdate = await findLiveProxyForUpdate(); const proxyWasActive = liveBeforeUpdate !== null; const configPort = typeof preUpdateConfig.port === "number" && preUpdateConfig.port > 0 ? preUpdateConfig.port diff --git a/src/update/npm-cache-preflight.d.mts b/src/update/npm-cache-preflight.d.mts index c1d10a408..955044978 100644 --- a/src/update/npm-cache-preflight.d.mts +++ b/src/update/npm-cache-preflight.d.mts @@ -12,6 +12,9 @@ export interface NpmCachePreflightOptions { lstat?: (path: string) => NpmCacheEntryStat; readdir?: (path: string) => string[]; realpath?: (path: string) => string; + now?: () => number; + maxEntries?: number; + maxDurationMs?: number; } export type NpmCacheOwnershipIssue = @@ -33,7 +36,10 @@ export type NpmCacheOwnershipResult = export declare function findForeignOwnedNpmCacheEntry( cachePath: string, expectedUid: number, - io?: Pick, + io?: Pick< + NpmCachePreflightOptions, + "lstat" | "readdir" | "realpath" | "now" | "maxEntries" | "maxDurationMs" + >, ): NpmCacheOwnershipIssue | null; export declare function checkNpmCacheOwnership( diff --git a/src/update/npm-cache-preflight.mjs b/src/update/npm-cache-preflight.mjs index 829108796..8106e9727 100644 --- a/src/update/npm-cache-preflight.mjs +++ b/src/update/npm-cache-preflight.mjs @@ -3,6 +3,9 @@ import { lstatSync, readdirSync, realpathSync } from "node:fs"; import { homedir } from "node:os"; import { isAbsolute, relative, resolve, sep } from "node:path"; +const DEFAULT_MAX_CACHE_ENTRIES = 50_000; +const DEFAULT_CACHE_SCAN_TIMEOUT_MS = 10_000; + /** Extract a stable filesystem error code without serializing the full error. */ function errorCode(error) { return error && typeof error === "object" && "code" in error @@ -15,13 +18,27 @@ export function findForeignOwnedNpmCacheEntry(cachePath, expectedUid, io = {}) { const lstat = io.lstat ?? lstatSync; const readdir = io.readdir ?? (path => readdirSync(path, { encoding: "utf8" })); const realpath = io.realpath ?? realpathSync; + const now = io.now ?? (() => Date.now()); + const maxEntries = Number.isFinite(io.maxEntries) && io.maxEntries > 0 + ? Math.trunc(io.maxEntries) + : DEFAULT_MAX_CACHE_ENTRIES; + const maxDurationMs = Number.isFinite(io.maxDurationMs) && io.maxDurationMs > 0 + ? Math.trunc(io.maxDurationMs) + : DEFAULT_CACHE_SCAN_TIMEOUT_MS; + const startedAt = now(); let cacheRoot; try { // npm follows a configured cache-root symlink. Resolve only that root; nested // symlinks remain lstat-only and are never traversed. cacheRoot = realpath(resolve(cachePath)); } catch (error) { - if (errorCode(error) === "ENOENT") return null; + if (errorCode(error) === "ENOENT") { + return { + kind: "error", + path: resolve(cachePath), + reason: "npm cache root does not exist", + }; + } return { kind: "error", path: resolve(cachePath), @@ -29,9 +46,22 @@ export function findForeignOwnedNpmCacheEntry(cachePath, expectedUid, io = {}) { }; } const stack = [cacheRoot]; + let discoveredEntries = 1; + + const elapsedBudgetIssue = path => ( + now() - startedAt > maxDurationMs + ? { + kind: "error", + path, + reason: `npm cache inspection exceeded its ${maxDurationMs}ms time budget`, + } + : null + ); while (stack.length > 0) { const path = stack.pop(); + const beforeStatBudget = elapsedBudgetIssue(path); + if (beforeStatBudget) return beforeStatBudget; let stat; try { stat = lstat(path); @@ -43,6 +73,8 @@ export function findForeignOwnedNpmCacheEntry(cachePath, expectedUid, io = {}) { reason: `could not inspect npm cache entry (${errorCode(error)})`, }; } + const afterStatBudget = elapsedBudgetIssue(path); + if (afterStatBudget) return afterStatBudget; if (Number.isInteger(stat.uid) && stat.uid !== expectedUid) { return { kind: "foreign-owner", path, actualUid: stat.uid }; @@ -52,6 +84,8 @@ export function findForeignOwnedNpmCacheEntry(cachePath, expectedUid, io = {}) { } if (!stat.isDirectory()) continue; + const beforeReadBudget = elapsedBudgetIssue(path); + if (beforeReadBudget) return beforeReadBudget; let entries; try { entries = readdir(path); @@ -62,7 +96,22 @@ export function findForeignOwnedNpmCacheEntry(cachePath, expectedUid, io = {}) { reason: `could not read npm cache directory (${errorCode(error)})`, }; } - for (const entry of entries) stack.push(resolve(path, entry)); + const afterReadBudget = elapsedBudgetIssue(path); + if (afterReadBudget) return afterReadBudget; + for (const entry of entries) { + const entryPath = resolve(path, entry); + const iterationBudget = elapsedBudgetIssue(entryPath); + if (iterationBudget) return iterationBudget; + if (discoveredEntries >= maxEntries) { + return { + kind: "error", + path: entryPath, + reason: `npm cache inspection exceeded its ${maxEntries}-entry budget`, + }; + } + discoveredEntries += 1; + stack.push(entryPath); + } } return null; @@ -153,7 +202,7 @@ export function checkNpmCacheOwnership(options = {}) { entryPath: issue.path, expectedUid, actualUid: issue.actualUid, - reason: `npm cache entry is owned by uid ${issue.actualUid}; expected uid ${expectedUid}`, + reason: "npm cache entry ownership does not match the current user", }; } return { diff --git a/tests/update-job.test.ts b/tests/update-job.test.ts index 618a035cc..0213dec0c 100644 --- a/tests/update-job.test.ts +++ b/tests/update-job.test.ts @@ -1,10 +1,12 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { checkForUpdate, confirmRestartAfterUpdateForTests, + findLiveProxyForUpdate, + findNpmRecoveryLauncher, finishGuiUpdateRestart, npmSelfUpdateRestartEvidence, readUpdateJob, @@ -104,6 +106,37 @@ describe("GUI update execution decisions", () => { }); }); + test("finds a validated npm-retired launcher when the current package is partial", () => { + const scopeRoot = join(dir, "global", "@bitkyc08"); + const currentRoot = join(scopeRoot, "opencodex"); + const retiredRoot = join(scopeRoot, ".opencodex-Ab12Cd34"); + for (const [root, name, version] of [ + [currentRoot, "@bitkyc08/opencodex", "2.7.41"], + [retiredRoot, "@bitkyc08/opencodex", "2.7.40"], + ] as const) { + mkdirSync(join(root, "bin"), { recursive: true }); + writeFileSync(join(root, "package.json"), JSON.stringify({ name, version })); + writeFileSync(join(root, "bin", "ocx.mjs"), "#!/usr/bin/env node\n"); + } + + expect(findNpmRecoveryLauncher(join(currentRoot, "bin", "ocx.mjs"), "2.7.40")) + .toBe(realpathSync(join(retiredRoot, "bin", "ocx.mjs"))); + }); + + test("rejects an npm-retired launcher with the wrong package identity", () => { + const scopeRoot = join(dir, "global", "@bitkyc08"); + const currentRoot = join(scopeRoot, "opencodex"); + const retiredRoot = join(scopeRoot, ".opencodex-Ab12Cd34"); + mkdirSync(join(retiredRoot, "bin"), { recursive: true }); + writeFileSync(join(retiredRoot, "package.json"), JSON.stringify({ + name: "untrusted-package", + version: "2.7.40", + })); + writeFileSync(join(retiredRoot, "bin", "ocx.mjs"), "#!/usr/bin/env node\n"); + + expect(findNpmRecoveryLauncher(join(currentRoot, "bin", "ocx.mjs"), "2.7.40")).toBeNull(); + }); + test("proxy restart pins --port so post-update start does not hop to an ephemeral port", () => { const proxy = restartCommand(false, "npm", "/pkg/bin/ocx.mjs", 10100); expect(proxy.mode).toBe("proxy"); @@ -119,7 +152,7 @@ describe("GUI update execution decisions", () => { // The stop-first update flow clears pid/runtime state before restartAfterUpdate runs, // so the wait must fire even with no readable pid — driven here via the io seam. const waited: Array<{ port: number; hostname: string; opts?: { killOcxHolders?: boolean; onlyKillPids?: number[] } }> = []; - const spawned: Array<{ port?: number }> = []; + const spawned: Array<{ port?: number; launcher?: string }> = []; const job: UpdateJobState = { id: "restart-io", status: "restarting", @@ -134,7 +167,11 @@ describe("GUI update execution decisions", () => { log: [], }; writeFileSync(updateJobPath(job.id), JSON.stringify(job)); - await restartAfterUpdateForTests(job, { port: 12345, hostname: "127.0.0.1" }, { + await restartAfterUpdateForTests(job, { + port: 12345, + hostname: "127.0.0.1", + recoveryLauncher: "/retired/bin/ocx.mjs", + }, { serviceInstalledFn: () => false, // drive the proxy-mode branch regardless of host state waitForPort: async (port, hostname, opts) => { waited.push({ @@ -147,8 +184,8 @@ describe("GUI update execution decisions", () => { }); return true; }, - spawnStart: (_job, _installer, port) => { - spawned.push({ port }); + spawnStart: (_job, _installer, port, launcher) => { + spawned.push({ port, launcher }); }, }); expect(waited).toEqual([{ @@ -156,7 +193,7 @@ describe("GUI update execution decisions", () => { hostname: "127.0.0.1", opts: { killOcxHolders: false, onlyKillPids: [] }, }]); - expect(spawned).toEqual([{ port: 12345 }]); + expect(spawned).toEqual([{ port: 12345, launcher: "/retired/bin/ocx.mjs" }]); }); test("restart reclaim allowlists only the trusted oldPid", async () => { @@ -220,6 +257,7 @@ describe("GUI update execution decisions", () => { test("service restart waits on the captured port and clears OCX_BAKE_PORT after install", async () => { const waited: Array<{ port: number; hostname: string }> = []; const bakeDuringInstall: string[] = []; + const launchersDuringInstall: string[] = []; const job: UpdateJobState = { id: "restart-svc", status: "restarting", @@ -237,20 +275,26 @@ describe("GUI update execution decisions", () => { const prev = process.env.OCX_BAKE_PORT; delete process.env.OCX_BAKE_PORT; try { - await restartAfterUpdateForTests(job, { port: 18765, hostname: "127.0.0.1" }, { + await restartAfterUpdateForTests(job, { + port: 18765, + hostname: "127.0.0.1", + recoveryLauncher: "/retired/bin/ocx.mjs", + }, { serviceInstalledFn: () => true, waitForPort: async (port, hostname) => { waited.push({ port, hostname: hostname ?? "" }); expect(process.env.OCX_BAKE_PORT).toBeUndefined(); return true; }, - runService: () => { + runService: (_job, _bin, args) => { bakeDuringInstall.push(process.env.OCX_BAKE_PORT ?? ""); + launchersDuringInstall.push(args[0] ?? ""); return { status: 0 }; }, }); expect(waited).toEqual([{ port: 18765, hostname: "127.0.0.1" }]); expect(bakeDuringInstall).toEqual(["18765"]); + expect(launchersDuringInstall).toEqual(["/retired/bin/ocx.mjs"]); expect(process.env.OCX_BAKE_PORT).toBeUndefined(); } finally { if (prev === undefined) delete process.env.OCX_BAKE_PORT; @@ -286,6 +330,18 @@ describe("GUI update execution decisions", () => { expect(spawned).toEqual([{ port: 19999 }]); }); + test("pre-update liveness retries a transient miss before classifying the proxy inactive", async () => { + let probes = 0; + const delays: number[] = []; + const live = await findLiveProxyForUpdate( + async () => (++probes === 1 ? null : { pid: 111, port: 15432, hostname: "127.0.0.1" }), + async ms => { delays.push(ms); }, + ); + expect(live).toEqual({ pid: 111, port: 15432, hostname: "127.0.0.1" }); + expect(probes).toBe(2); + expect(delays).toEqual([250]); + }); + test("failed install leaves an already-healthy proxy untouched", async () => { let restartCalls = 0; const job: UpdateJobState = { @@ -380,7 +436,7 @@ describe("GUI update execution decisions", () => { expect(restartCalls).toBe(0); }); - test("failed install preserves a captured PID that remains alive after health timeouts", async () => { + test("failed install preserves a captured PID only while it still identifies as OpenCodex", async () => { let restartCalls = 0; const job: UpdateJobState = { id: "failed-install-live-pid", @@ -403,7 +459,7 @@ describe("GUI update execution decisions", () => { true, { probeProxyIdentity: async () => null, - isProcessAlive: () => true, + verifyPidIdentityFn: pid => pid, sleepMs: async () => {}, restartAfterUpdateFn: async () => { restartCalls += 1; }, }, @@ -413,9 +469,10 @@ describe("GUI update execution decisions", () => { expect(readUpdateJob(job.id)?.log.some(line => line.includes("refusing an automatic restart"))).toBe(true); }); - test("failed install restores a proxy that the updater stopped", async () => { + test("failed install restores through the retired launcher after the old PID loses identity", async () => { let now = 0; let restarted = false; + let recoveryLauncher: string | undefined; const job: UpdateJobState = { id: "failed-install-recovery", status: "running", @@ -437,14 +494,19 @@ describe("GUI update execution decisions", () => { true, { probeProxyIdentity: async () => null, - isProcessAlive: () => false, + verifyPidIdentityFn: () => null, + recoveryLauncherFn: () => "/retired/bin/ocx.mjs", probeProxy: async () => restarted, - restartAfterUpdateFn: async () => { restarted = true; }, + restartAfterUpdateFn: async (_job, captured) => { + recoveryLauncher = captured?.recoveryLauncher; + restarted = true; + }, now: () => now, sleepMs: async (ms) => { now += ms; }, }, ); expect(recovery).toBe("restarted"); + expect(recoveryLauncher).toBe("/retired/bin/ocx.mjs"); expect(readUpdateJob(job.id)?.log.some(line => line.includes("update itself still failed"))).toBe(true); }); diff --git a/tests/update-npm-cache-preflight.test.ts b/tests/update-npm-cache-preflight.test.ts index 10017b479..a339d2874 100644 --- a/tests/update-npm-cache-preflight.test.ts +++ b/tests/update-npm-cache-preflight.test.ts @@ -102,6 +102,7 @@ describe("npm cache ownership pre-flight", () => { entryPath: realpathSync(dir), expectedUid: uid + 1, actualUid: uid, + reason: "npm cache entry ownership does not match the current user", }); if (result.ok !== false) throw new Error("expected ownership failure"); expect(formatNpmCacheOwnershipFailure(result)).toContain("before stopping the proxy"); @@ -116,10 +117,53 @@ describe("npm cache ownership pre-flight", () => { entryPath: join(cachePath, "_cacache", "foreign"), expectedUid: 502, actualUid: 0, - reason: "foreign owner", + reason: "npm cache entry ownership does not match the current user", }); expect(output).toContain("~/.npm/_cacache/foreign"); expect(output).not.toContain(homedir()); + expect(output).not.toContain("502"); + expect(output).not.toMatch(/\buid\b/i); + }); + + test("fails closed when the configured cache root does not exist", () => { + const missing = `${dir}-missing`; + const result = checkNpmCacheOwnership({ + getuid: () => 501, + spawn: cacheLookup(missing), + }); + expect(result).toMatchObject({ + ok: false, + cachePath: missing, + entryPath: missing, + reason: "npm cache root does not exist", + }); + }); + + test("fails closed when the cache exceeds the entry budget", () => { + const uid = process.getuid?.(); + if (uid === undefined) return; + const issue = findForeignOwnedNpmCacheEntry(dir, uid, { maxEntries: 2 }); + expect(issue).toMatchObject({ + kind: "error", + reason: "npm cache inspection exceeded its 2-entry budget", + }); + }); + + test("fails closed when the cache exceeds the elapsed-time budget", () => { + const uid = process.getuid?.(); + if (uid === undefined) return; + let now = 0; + const issue = findForeignOwnedNpmCacheEntry(dir, uid, { + maxDurationMs: 10, + now: () => { + now += 6; + return now; + }, + }); + expect(issue).toMatchObject({ + kind: "error", + reason: "npm cache inspection exceeded its 10ms time budget", + }); }); test("fails closed before shutdown when npm cannot resolve its cache", () => { diff --git a/tests/update-stop-first.test.ts b/tests/update-stop-first.test.ts index 42b59c85f..96e390c7b 100644 --- a/tests/update-stop-first.test.ts +++ b/tests/update-stop-first.test.ts @@ -64,7 +64,7 @@ describe("update stops the running proxy before replacing files", () => { expect(launcherSource).toContain("OCX_BAKE_PORT"); // Live runtime port 10100 must not be discarded as a missing-port sentinel. expect(launcherSource).toContain("sawRuntimePort"); - expect(updateJobSource).toContain("const liveBeforeUpdate = await findLiveProxy()"); + expect(updateJobSource).toContain("const liveBeforeUpdate = await findLiveProxyForUpdate()"); }); test("both update paths surface a skipped history restore after the stop", () => { @@ -90,11 +90,29 @@ describe("update stops the running proxy before replacing files", () => { }); test("GUI failure recovery is gated by identity-checked pre-update liveness", () => { - expect(updateJobSource).toContain("const liveBeforeUpdate = await findLiveProxy()"); + expect(updateJobSource).toContain("const liveBeforeUpdate = await findLiveProxyForUpdate()"); expect(updateJobSource).toContain("const proxyWasActive = liveBeforeUpdate !== null"); expect(updateJobSource).not.toContain("const proxyWasActive = isServiceInstalled() || runtimeTrusted"); }); + test("GUI failure recovery identity-checks the old PID and threads a validated launcher", () => { + expect(updateJobSource).toContain("(io.verifyPidIdentityFn ?? verifyPidIdentity)(captured.oldPid)"); + expect(updateJobSource).toContain("io.recoveryLauncherFn ?? findNpmRecoveryLauncher"); + expect(updateJobSource).toContain("recoveryCaptured = { ...captured, recoveryLauncher }"); + expect(updateJobSource).toContain("captured?.recoveryLauncher ?? packageLauncherPath()"); + }); + + test("GUI recovery waits beyond the nested npm install deadline", () => { + const outerRaw = /const UPDATE_TIMEOUT_MS = ([\d_]+);/.exec(updateJobSource)?.[1]; + const innerRaw = /const NPM_INSTALL_TIMEOUT_MS = ([\d_]+);/.exec(launcherSource)?.[1]; + expect(outerRaw).toBeDefined(); + expect(innerRaw).toBeDefined(); + const outerMs = Number(outerRaw?.replaceAll("_", "")); + const innerMs = Number(innerRaw?.replaceAll("_", "")); + expect(outerMs).toBeGreaterThanOrEqual(innerMs + 60_000); + expect(launcherSource).toContain("timeout: NPM_INSTALL_TIMEOUT_MS"); + }); + test("GUI worker update children use pipe stdio so Windows npm.cmd does not open consoles", () => { expect(updateSource).toContain("function updateChildStdio()"); expect(updateSource).toContain('process.env.OCX_SERVICE === "1"'); diff --git a/tests/windows-deploy-close-regressions.test.ts b/tests/windows-deploy-close-regressions.test.ts index 10ebcad5f..8b8d6f319 100644 --- a/tests/windows-deploy-close-regressions.test.ts +++ b/tests/windows-deploy-close-regressions.test.ts @@ -27,7 +27,7 @@ describe("update-job restart avoids the shell-less .cmd EINVAL (Windows, bun/sou expect(src).toContain("OCX_BAKE_PORT"); // Service reinstall still runs (with bake) even when reclaim warns; direct start refuses to hop. expect(src).toContain("refusing to hop"); - expect(src).toContain("const liveBeforeUpdate = await findLiveProxy()"); + expect(src).toContain("const liveBeforeUpdate = await findLiveProxyForUpdate()"); expect(read("src/cli/index.ts")).toContain("allowEphemeralFallback: !hardPin"); expect(read("src/cli/index.ts")).toContain("preferRetryMs: hardPin ? 0 : 750"); expect(read("src/cli/index.ts")).toContain("Not opening the GUI"); From 5c3586539b8ce2845a629c992d14b12ca0eb5c52 Mon Sep 17 00:00:00 2001 From: wzb Date: Mon, 27 Jul 2026 14:15:12 +0800 Subject: [PATCH 04/16] fix(update): harden preflight and recovery races --- docs/adr/0001-gui-update-worker.md | 21 ++-- src/update/job.ts | 60 ++++++++-- src/update/npm-cache-preflight.d.mts | 3 + src/update/npm-cache-preflight.mjs | 127 +++++++++++++++------- tests/update-job.test.ts | 133 ++++++++++++++++++++++- tests/update-npm-cache-preflight.test.ts | 34 +++++- tests/update-stop-first.test.ts | 4 + 7 files changed, 315 insertions(+), 67 deletions(-) diff --git a/docs/adr/0001-gui-update-worker.md b/docs/adr/0001-gui-update-worker.md index 4f08a3ce5..d49b6ed22 100644 --- a/docs/adr/0001-gui-update-worker.md +++ b/docs/adr/0001-gui-update-worker.md @@ -26,17 +26,22 @@ Before an npm updater stops the proxy, it resolves the configured npm cache and entry is owned by the current Unix user. A foreign-owned entry (commonly left by an older `sudo npm` invocation) aborts the update with an actionable error while the existing proxy and service remain running. The scan fails closed when the cache root is absent, an entry cannot be inspected, or its -50,000-entry / 10-second traversal budget is exhausted. Platforms without Unix uid ownership skip -this gate. +50,000-entry / 10-second traversal budget is exhausted. The blocking filesystem walk runs in a +child process so the parent can enforce the deadline even when a filesystem call itself stalls. +Failure logs omit both cache and entry paths; users can locate the cache explicitly with +`npm config get cache`. Platforms without Unix uid ownership skip this gate. The GUI worker retries its identity-checked pre-update liveness capture before deciding a service -was inactive. If the installer still exits nonzero, it probes again and leaves the old process alone -only when health or PID identity still proves it is OpenCodex. When npm already stopped the proxy -and retired the old package, the worker validates the hidden package's name, version, and launcher, -uses that launcher for a best-effort restart, and verifies health. The update job remains failed +was inactive. If health remains unavailable, a matching runtime-port record plus an identity-checked +live PID still preserves the pre-update activity evidence. If the installer exits nonzero, the worker +probes again immediately before recovery and leaves any old or concurrently replaced process alone +when health or PID identity proves it is OpenCodex. When npm already stopped the proxy and retired +the old package, the worker validates the hidden package's name, version, and launcher, uses that +launcher for a best-effort restart, and verifies health. Service and direct restart paths also stop +when the pidfile has changed to a different identity-checked proxy. The update job remains failed either way because restoring availability is not the same as installing the new version. The GUI -worker's install timeout also exceeds the nested npm install deadline, so recovery cannot race a -still-running npm replacement child. +worker's install timeout exceeds the nested npm install deadline, so recovery cannot race an npm +replacement child that is still running. After an update requests a restart, the worker now waits for an identity-checked `/healthz` to return and remain healthy for a short stability window before marking the job successful. This diff --git a/src/update/job.ts b/src/update/job.ts index 308a5ef9b..68c2d8081 100644 --- a/src/update/job.ts +++ b/src/update/job.ts @@ -2,7 +2,15 @@ import { spawn, spawnSync } from "node:child_process"; import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync } from "node:fs"; import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; -import { atomicWriteFile, getConfigDir, loadConfig, readPid, verifyPidIdentity } from "../config"; +import { + atomicWriteFile, + getConfigDir, + loadConfig, + readAlivePid, + readPid, + readRuntimePort, + verifyPidIdentity, +} from "../config"; import { killProxy } from "../lib/process-control"; import { reclaimListenPort } from "../server/port-reclaim"; import { @@ -383,6 +391,7 @@ export interface RestartIo { waitForPort?: typeof reclaimListenPort; spawnStart?: (job: UpdateJobState, installer: Installer, port?: number, launcher?: string) => void; serviceInstalledFn?: () => boolean; + readPidFn?: () => number | null; probeProxy?: (port: number, hostname?: string) => Promise; /** Richer /healthz read for update-correlated restart evidence (pid + version). */ probeProxyIdentity?: (port: number, hostname?: string) => Promise; @@ -419,6 +428,14 @@ async function restartAfterUpdate( const oldPid = typeof captured?.oldPid === "number" && captured.oldPid > 0 ? captured.oldPid : undefined; + const readCurrentPid = io.readPidFn ?? readPid; + const refuseForReplacementPid = (): boolean => { + const currentPid = readCurrentPid(); + if (currentPid === null || currentPid === oldPid) return false; + updateJob(job, {}, "A different identity-checked proxy PID appeared during restart handoff; leaving it untouched."); + return true; + }; + if (refuseForReplacementPid()) return; let svcArgs: string[] | undefined; if (serviceInstalled) { try { @@ -444,6 +461,7 @@ async function restartAfterUpdate( if (!freed) { updateJob(job, {}, `Port ${port} still busy after ${Math.trunc(RESTART_PORT_RECLAIM_MS / 1000)}s; refusing to hop — reinstall may fail until the port is free.`); } + if (refuseForReplacementPid()) return; const prevBake = process.env.OCX_BAKE_PORT; process.env.OCX_BAKE_PORT = String(Math.trunc(port)); let serviceOk = false; @@ -469,8 +487,12 @@ async function restartAfterUpdate( // proxy stopped when the service reinstall could not run. } - const pid = readPid(); + const pid = readCurrentPid(); if (pid) { + if (pid !== oldPid) { + updateJob(job, {}, "A different identity-checked proxy PID appeared before direct restart; leaving it untouched."); + return; + } updateJob(job, {}, `Stopping current proxy PID ${pid}.`); killProxy(pid); } @@ -482,6 +504,7 @@ async function restartAfterUpdate( updateJob(job, {}, `Port ${port} still busy after ${Math.trunc(RESTART_PORT_RECLAIM_MS / 1000)}s (reclaim could not free the socket); not starting on another port. Retry 'ocx start --port ${port}'.`); return; } + if (refuseForReplacementPid()) return; (io.spawnStart ?? spawnDetachedStart)(job, job.installer, port, launcher); } @@ -626,13 +649,20 @@ async function probeFailedUpdateProxy( return null; } -/** Retry the pre-update identity lookup before declaring an active proxy absent. */ -export async function findLiveProxyForUpdate( - probe: () => Promise = findLiveProxy, - sleep: (ms: number) => Promise = async (ms: number) => { +export interface UpdateLivenessIo { + findLiveProxyFn?: () => Promise; + sleepMs?: (ms: number) => Promise; + readAlivePidFn?: () => number | null; + verifyPidIdentityFn?: (pid: number) => number | null; + readRuntimePortFn?: (expectedPid?: number) => { pid?: number; port: number; hostname?: string } | null; +} + +/** Retry health first, then retain a PID-verified runtime target as active evidence. */ +export async function findLiveProxyForUpdate(io: UpdateLivenessIo = {}): Promise { + const probe = io.findLiveProxyFn ?? findLiveProxy; + const sleep = io.sleepMs ?? (async (ms: number) => { await new Promise(resolve => setTimeout(resolve, ms)); - }, -): Promise { + }); for (let attempt = 0; attempt < UPDATE_LIVENESS_PROBE_ATTEMPTS; attempt += 1) { try { const live = await probe(); @@ -640,7 +670,14 @@ export async function findLiveProxyForUpdate( } catch { /* retry a transient liveness lookup failure */ } if (attempt + 1 < UPDATE_LIVENESS_PROBE_ATTEMPTS) await sleep(UPDATE_LIVENESS_PROBE_DELAY_MS); } - return null; + + const candidatePid = (io.readAlivePidFn ?? readAlivePid)(); + if (candidatePid === null) return null; + const verifiedPid = (io.verifyPidIdentityFn ?? verifyPidIdentity)(candidatePid); + if (verifiedPid !== candidatePid) return null; + const runtime = (io.readRuntimePortFn ?? readRuntimePort)(candidatePid); + if (!runtime) return null; + return { pid: candidatePid, port: runtime.port, hostname: runtime.hostname }; } /** Keep a failed GUI install from also leaving a previously-active proxy offline. */ @@ -664,6 +701,11 @@ async function recoverFailedGuiUpdate( return "still-running"; } + if (await probeFailedUpdateProxy(captured, io)) { + updateJob(job, {}, `A replacement proxy became healthy on ${captured.hostname}:${captured.port} before recovery; leaving it untouched.`); + return "still-running"; + } + updateJob(job, {}, "Update command failed after the proxy stopped; attempting to restore the proxy..."); try { let recoveryCaptured: { diff --git a/src/update/npm-cache-preflight.d.mts b/src/update/npm-cache-preflight.d.mts index 955044978..b4d5fa17e 100644 --- a/src/update/npm-cache-preflight.d.mts +++ b/src/update/npm-cache-preflight.d.mts @@ -9,6 +9,9 @@ export interface NpmCachePreflightOptions { npmBin?: string; shell?: boolean; spawn?: typeof import("node:child_process").spawnSync; + scanSpawn?: typeof import("node:child_process").spawnSync; + scanBin?: string; + scanScript?: string; lstat?: (path: string) => NpmCacheEntryStat; readdir?: (path: string) => string[]; realpath?: (path: string) => string; diff --git a/src/update/npm-cache-preflight.mjs b/src/update/npm-cache-preflight.mjs index 8106e9727..e32641d7f 100644 --- a/src/update/npm-cache-preflight.mjs +++ b/src/update/npm-cache-preflight.mjs @@ -1,10 +1,15 @@ import { spawnSync } from "node:child_process"; -import { lstatSync, readdirSync, realpathSync } from "node:fs"; -import { homedir } from "node:os"; -import { isAbsolute, relative, resolve, sep } from "node:path"; +import { lstatSync, readFileSync, readdirSync, realpathSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; const DEFAULT_MAX_CACHE_ENTRIES = 50_000; const DEFAULT_CACHE_SCAN_TIMEOUT_MS = 10_000; +const CACHE_SCAN_WORKER_ARG = "__scan-npm-cache-ownership"; + +function positiveInteger(value, fallback) { + return Number.isFinite(value) && value > 0 ? Math.trunc(value) : fallback; +} /** Extract a stable filesystem error code without serializing the full error. */ function errorCode(error) { @@ -19,12 +24,8 @@ export function findForeignOwnedNpmCacheEntry(cachePath, expectedUid, io = {}) { const readdir = io.readdir ?? (path => readdirSync(path, { encoding: "utf8" })); const realpath = io.realpath ?? realpathSync; const now = io.now ?? (() => Date.now()); - const maxEntries = Number.isFinite(io.maxEntries) && io.maxEntries > 0 - ? Math.trunc(io.maxEntries) - : DEFAULT_MAX_CACHE_ENTRIES; - const maxDurationMs = Number.isFinite(io.maxDurationMs) && io.maxDurationMs > 0 - ? Math.trunc(io.maxDurationMs) - : DEFAULT_CACHE_SCAN_TIMEOUT_MS; + const maxEntries = positiveInteger(io.maxEntries, DEFAULT_MAX_CACHE_ENTRIES); + const maxDurationMs = positiveInteger(io.maxDurationMs, DEFAULT_CACHE_SCAN_TIMEOUT_MS); const startedAt = now(); let cacheRoot; try { @@ -117,36 +118,59 @@ export function findForeignOwnedNpmCacheEntry(cachePath, expectedUid, io = {}) { return null; } -/** Redact path segments that commonly carry secrets before they reach update logs. */ -function sanitizePathSegments(path) { - return path - .split(/[\\/]/) - .map(segment => /(?:secret|password|passwd|token|api[-_]?key|apikey|credential|email)/i.test(segment) - ? "[REDACTED]" - : segment) - .join("/"); +function parseOwnershipIssue(output, cachePath) { + try { + const parsed = JSON.parse(output); + if (parsed === null) return null; + if (!parsed || typeof parsed !== "object" || typeof parsed.path !== "string") throw new Error("invalid issue"); + if (parsed.kind === "foreign-owner" && Number.isInteger(parsed.actualUid)) return parsed; + if (parsed.kind === "error" && typeof parsed.reason === "string") return parsed; + } catch { /* convert malformed worker output into a fail-closed result below */ } + return { + kind: "error", + path: resolve(cachePath), + reason: "npm cache inspection returned invalid output", + }; } -/** Render paths home-relative, or hide absolute paths outside the current home. */ -function displayUserPath(path) { - const absolute = resolve(path); - const home = resolve(homedir()); - const fromHome = relative(home, absolute); - if (fromHome === "") return "~"; - if (fromHome !== ".." && !fromHome.startsWith(`..${sep}`) && !isAbsolute(fromHome)) { - return `~/${sanitizePathSegments(fromHome)}`; +/** Run the blocking filesystem walk out-of-process so its wall-clock deadline is enforceable. */ +function scanNpmCacheOwnership(cachePath, expectedUid, options) { + const maxEntries = positiveInteger(options.maxEntries, DEFAULT_MAX_CACHE_ENTRIES); + const maxDurationMs = positiveInteger(options.maxDurationMs, DEFAULT_CACHE_SCAN_TIMEOUT_MS); + const scanSpawn = options.scanSpawn ?? spawnSync; + let result; + try { + result = scanSpawn( + options.scanBin ?? process.execPath, + [options.scanScript ?? fileURLToPath(import.meta.url), CACHE_SCAN_WORKER_ARG], + { + encoding: "utf8", + input: JSON.stringify({ cachePath, expectedUid, maxEntries, maxDurationMs }), + timeout: maxDurationMs, + killSignal: "SIGKILL", + windowsHide: true, + shell: false, + }, + ); + } catch (error) { + return { + kind: "error", + path: resolve(cachePath), + reason: `could not inspect npm cache (${errorCode(error)})`, + }; } - return "[configured npm cache]"; -} - -/** Prefer a cache-relative entry while preserving the same account-name redaction. */ -function displayCacheEntry(cachePath, entryPath) { - const fromCache = relative(resolve(cachePath), resolve(entryPath)); - if (fromCache === "") return displayUserPath(cachePath); - if (fromCache !== ".." && !fromCache.startsWith(`..${sep}`) && !isAbsolute(fromCache)) { - return `${displayUserPath(cachePath)}/${sanitizePathSegments(fromCache)}`; + if (result.status !== 0) { + const timedOut = result.status === null || errorCode(result.error) === "ETIMEDOUT"; + return { + kind: "error", + path: resolve(cachePath), + reason: timedOut + ? `npm cache inspection exceeded its ${maxDurationMs}ms time budget` + : `could not inspect npm cache (${result.error ? errorCode(result.error) : `status ${result.status}`})`, + }; } - return displayUserPath(entryPath); + const output = Buffer.isBuffer(result.stdout) ? result.stdout.toString("utf8") : String(result.stdout ?? ""); + return parseOwnershipIssue(output, cachePath); } /** @@ -193,7 +217,7 @@ export function checkNpmCacheOwnership(options = {}) { return { ok: false, expectedUid, reason: "npm did not report a cache path" }; } - const issue = findForeignOwnedNpmCacheEntry(cachePath, expectedUid, options); + const issue = scanNpmCacheOwnership(cachePath, expectedUid, options); if (!issue) return { ok: true, cachePath: resolve(cachePath) }; if (issue.kind === "foreign-owner") { return { @@ -214,15 +238,34 @@ export function checkNpmCacheOwnership(options = {}) { }; } -/** Format an actionable update error without persisting an OS account name. */ +/** Format an actionable update error without persisting account ids or arbitrary local paths. */ export function formatNpmCacheOwnershipFailure(result) { - const entry = result.entryPath - ? displayCacheEntry(result.cachePath ?? result.entryPath, result.entryPath) - : null; return [ "npm cache ownership pre-flight failed before stopping the proxy.", - entry ? `${result.reason}: ${entry}` : result.reason, - ...(result.cachePath ? [`Cache: ${displayUserPath(result.cachePath)}`] : []), + result.reason, + "Run 'npm config get cache' to locate the configured cache.", "Correct the cache ownership or configure a user-owned npm cache, then retry.", ].join("\n"); } + +function runCacheScanWorker() { + try { + const payload = JSON.parse(readFileSync(0, "utf8")); + if (typeof payload?.cachePath !== "string" || !Number.isInteger(payload?.expectedUid)) { + process.exitCode = 2; + return; + } + const issue = findForeignOwnedNpmCacheEntry(payload.cachePath, payload.expectedUid, { + maxEntries: payload.maxEntries, + maxDurationMs: payload.maxDurationMs, + }); + process.stdout.write(JSON.stringify(issue)); + } catch { + process.exitCode = 2; + } +} + +const thisModulePath = fileURLToPath(import.meta.url); +if (process.argv[2] === CACHE_SCAN_WORKER_ARG && resolve(process.argv[1] ?? "") === resolve(thisModulePath)) { + runCacheScanWorker(); +} diff --git a/tests/update-job.test.ts b/tests/update-job.test.ts index 0213dec0c..41cb50742 100644 --- a/tests/update-job.test.ts +++ b/tests/update-job.test.ts @@ -226,6 +226,73 @@ describe("GUI update execution decisions", () => { expect(optsSeen).toEqual([{ killOcxHolders: true, onlyKillPids: [4242] }]); }); + test("service restart leaves a replacement PID untouched when it appears during port reclaim", async () => { + let pidReads = 0; + const job: UpdateJobState = { + id: "restart-replacement-pid", + status: "restarting", + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + currentVersion: "2.7.40", + latestVersion: "2.7.41", + channel: "latest", + installer: "npm", + restart: true, + command: "", + log: [], + }; + writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + await restartAfterUpdateForTests(job, { + port: 10100, + hostname: "127.0.0.1", + oldPid: 111, + recoveryLauncher: "/retired/bin/ocx.mjs", + }, { + serviceInstalledFn: () => true, + readPidFn: () => (++pidReads === 1 ? 111 : 222), + waitForPort: async () => true, + runService: () => { throw new Error("must not reinstall over a replacement PID"); }, + spawnStart: () => { throw new Error("must not start over a replacement PID"); }, + }); + expect(pidReads).toBe(2); + expect(readUpdateJob(job.id)?.log.some(line => + line.includes("different identity-checked proxy PID") && line.includes("leaving it untouched"), + )).toBe(true); + }); + + test("direct restart leaves a replacement PID untouched when it appears during port reclaim", async () => { + let pidReads = 0; + const job: UpdateJobState = { + id: "restart-direct-replacement-pid", + status: "restarting", + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + currentVersion: "2.7.40", + latestVersion: "2.7.41", + channel: "latest", + installer: "npm", + restart: true, + command: "", + log: [], + }; + writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + await restartAfterUpdateForTests(job, { + port: 10100, + hostname: "127.0.0.1", + oldPid: 111, + recoveryLauncher: "/retired/bin/ocx.mjs", + }, { + serviceInstalledFn: () => false, + readPidFn: () => (++pidReads < 3 ? null : 222), + waitForPort: async () => true, + spawnStart: () => { throw new Error("must not start over a replacement PID"); }, + }); + expect(pidReads).toBe(3); + expect(readUpdateJob(job.id)?.log.some(line => + line.includes("different identity-checked proxy PID") && line.includes("leaving it untouched"), + )).toBe(true); + }); + test("restart refuses to spawn when the captured port never becomes free", async () => { const spawned: Array<{ port?: number }> = []; const job: UpdateJobState = { @@ -333,15 +400,35 @@ describe("GUI update execution decisions", () => { test("pre-update liveness retries a transient miss before classifying the proxy inactive", async () => { let probes = 0; const delays: number[] = []; - const live = await findLiveProxyForUpdate( - async () => (++probes === 1 ? null : { pid: 111, port: 15432, hostname: "127.0.0.1" }), - async ms => { delays.push(ms); }, - ); + const live = await findLiveProxyForUpdate({ + findLiveProxyFn: async () => ( + ++probes === 1 ? null : { pid: 111, port: 15432, hostname: "127.0.0.1" } + ), + sleepMs: async ms => { delays.push(ms); }, + }); expect(live).toEqual({ pid: 111, port: 15432, hostname: "127.0.0.1" }); expect(probes).toBe(2); expect(delays).toEqual([250]); }); + test("pre-update liveness retains a PID-verified runtime target after health misses", async () => { + let probes = 0; + const live = await findLiveProxyForUpdate({ + findLiveProxyFn: async () => { + probes += 1; + return null; + }, + sleepMs: async () => {}, + readAlivePidFn: () => 111, + verifyPidIdentityFn: pid => pid, + readRuntimePortFn: expectedPid => ( + expectedPid === 111 ? { pid: 111, port: 16543, hostname: "127.0.0.1" } : null + ), + }); + expect(probes).toBe(3); + expect(live).toEqual({ pid: 111, port: 16543, hostname: "127.0.0.1" }); + }); + test("failed install leaves an already-healthy proxy untouched", async () => { let restartCalls = 0; const job: UpdateJobState = { @@ -469,6 +556,44 @@ describe("GUI update execution decisions", () => { expect(readUpdateJob(job.id)?.log.some(line => line.includes("refusing an automatic restart"))).toBe(true); }); + test("failed install leaves a concurrently restored replacement proxy untouched", async () => { + let probes = 0; + let restartCalls = 0; + const job: UpdateJobState = { + id: "failed-install-concurrent-replacement", + status: "running", + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + currentVersion: "2.7.40", + latestVersion: "2.7.41", + channel: "latest", + installer: "npm", + restart: true, + command: "", + releaseNotesUrl: "", + log: [], + }; + writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + const recovery = await recoverFailedGuiUpdateForTests( + job, + { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, + true, + { + probeProxyIdentity: async () => ( + ++probes <= 3 ? null : { pid: 222, version: "2.7.40" } + ), + verifyPidIdentityFn: () => null, + sleepMs: async () => {}, + recoveryLauncherFn: () => { throw new Error("must not resolve a launcher"); }, + restartAfterUpdateFn: async () => { restartCalls += 1; }, + }, + ); + expect(recovery).toBe("still-running"); + expect(probes).toBe(4); + expect(restartCalls).toBe(0); + expect(readUpdateJob(job.id)?.log.some(line => line.includes("replacement proxy became healthy"))).toBe(true); + }); + test("failed install restores through the retired launcher after the old PID loses identity", async () => { let now = 0; let restarted = false; diff --git a/tests/update-npm-cache-preflight.test.ts b/tests/update-npm-cache-preflight.test.ts index a339d2874..e4e2e3bc3 100644 --- a/tests/update-npm-cache-preflight.test.ts +++ b/tests/update-npm-cache-preflight.test.ts @@ -109,18 +109,22 @@ describe("npm cache ownership pre-flight", () => { expect(formatNpmCacheOwnershipFailure(result)).toContain("configure a user-owned npm cache"); }); - test("formats cache failures without persisting the OS account name", () => { - const cachePath = join(homedir(), ".npm"); + test("formats cache failures without persisting arbitrary path segments or account ids", () => { + const cachePath = join(homedir(), "customer-alice-cache"); + const entryPath = join(cachePath, "_npx", "node_modules", "@alice"); const output = formatNpmCacheOwnershipFailure({ ok: false, cachePath, - entryPath: join(cachePath, "_cacache", "foreign"), + entryPath, expectedUid: 502, actualUid: 0, reason: "npm cache entry ownership does not match the current user", }); - expect(output).toContain("~/.npm/_cacache/foreign"); + expect(output).toContain("npm config get cache"); expect(output).not.toContain(homedir()); + expect(output).not.toContain("customer-alice-cache"); + expect(output).not.toContain("@alice"); + expect(output).not.toContain("_npx"); expect(output).not.toContain("502"); expect(output).not.toMatch(/\buid\b/i); }); @@ -166,6 +170,28 @@ describe("npm cache ownership pre-flight", () => { }); }); + test("terminates a scan process that ignores SIGTERM at the wall-clock deadline", () => { + const uid = process.getuid?.(); + if (uid === undefined) return; + const blockingScan = join(dir, "blocking-scan.mjs"); + writeFileSync( + blockingScan, + "process.on('SIGTERM', () => {});\nsetInterval(() => {}, 60_000);\n", + ); + const startedAt = Date.now(); + const result = checkNpmCacheOwnership({ + getuid: () => uid, + spawn: cacheLookup(dir), + scanScript: blockingScan, + maxDurationMs: 250, + }); + expect(Date.now() - startedAt).toBeLessThan(2_000); + expect(result).toMatchObject({ + ok: false, + reason: "npm cache inspection exceeded its 250ms time budget", + }); + }); + test("fails closed before shutdown when npm cannot resolve its cache", () => { let inspected = false; const result = checkNpmCacheOwnership({ diff --git a/tests/update-stop-first.test.ts b/tests/update-stop-first.test.ts index 96e390c7b..569810aff 100644 --- a/tests/update-stop-first.test.ts +++ b/tests/update-stop-first.test.ts @@ -92,6 +92,9 @@ describe("update stops the running proxy before replacing files", () => { test("GUI failure recovery is gated by identity-checked pre-update liveness", () => { expect(updateJobSource).toContain("const liveBeforeUpdate = await findLiveProxyForUpdate()"); expect(updateJobSource).toContain("const proxyWasActive = liveBeforeUpdate !== null"); + expect(updateJobSource).toContain("(io.readAlivePidFn ?? readAlivePid)()"); + expect(updateJobSource).toContain("(io.verifyPidIdentityFn ?? verifyPidIdentity)(candidatePid)"); + expect(updateJobSource).toContain("(io.readRuntimePortFn ?? readRuntimePort)(candidatePid)"); expect(updateJobSource).not.toContain("const proxyWasActive = isServiceInstalled() || runtimeTrusted"); }); @@ -100,6 +103,7 @@ describe("update stops the running proxy before replacing files", () => { expect(updateJobSource).toContain("io.recoveryLauncherFn ?? findNpmRecoveryLauncher"); expect(updateJobSource).toContain("recoveryCaptured = { ...captured, recoveryLauncher }"); expect(updateJobSource).toContain("captured?.recoveryLauncher ?? packageLauncherPath()"); + expect(updateJobSource).toContain("if (refuseForReplacementPid()) return"); }); test("GUI recovery waits beyond the nested npm install deadline", () => { From 55d72e60b3aea8ac1e9f60387d64c41da848302a Mon Sep 17 00:00:00 2001 From: wzb Date: Mon, 27 Jul 2026 14:33:29 +0800 Subject: [PATCH 05/16] fix(update): consolidate replacement pid checks --- src/update/job.ts | 24 +++++++++++------------- tests/update-job.test.ts | 2 ++ tests/update-stop-first.test.ts | 5 ++++- 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/update/job.ts b/src/update/job.ts index 68c2d8081..2dcdcf678 100644 --- a/src/update/job.ts +++ b/src/update/job.ts @@ -429,13 +429,13 @@ async function restartAfterUpdate( ? captured.oldPid : undefined; const readCurrentPid = io.readPidFn ?? readPid; - const refuseForReplacementPid = (): boolean => { + const readPidForRestart = (context: string): { pid: number | null; refused: boolean } => { const currentPid = readCurrentPid(); - if (currentPid === null || currentPid === oldPid) return false; - updateJob(job, {}, "A different identity-checked proxy PID appeared during restart handoff; leaving it untouched."); - return true; + if (currentPid === null || currentPid === oldPid) return { pid: currentPid, refused: false }; + updateJob(job, {}, `A different identity-checked proxy PID appeared ${context}; leaving it untouched.`); + return { pid: null, refused: true }; }; - if (refuseForReplacementPid()) return; + if (readPidForRestart("during restart handoff").refused) return; let svcArgs: string[] | undefined; if (serviceInstalled) { try { @@ -461,7 +461,7 @@ async function restartAfterUpdate( if (!freed) { updateJob(job, {}, `Port ${port} still busy after ${Math.trunc(RESTART_PORT_RECLAIM_MS / 1000)}s; refusing to hop — reinstall may fail until the port is free.`); } - if (refuseForReplacementPid()) return; + if (readPidForRestart("after service port reclaim").refused) return; const prevBake = process.env.OCX_BAKE_PORT; process.env.OCX_BAKE_PORT = String(Math.trunc(port)); let serviceOk = false; @@ -487,12 +487,10 @@ async function restartAfterUpdate( // proxy stopped when the service reinstall could not run. } - const pid = readCurrentPid(); - if (pid) { - if (pid !== oldPid) { - updateJob(job, {}, "A different identity-checked proxy PID appeared before direct restart; leaving it untouched."); - return; - } + const directPid = readPidForRestart("before direct restart"); + if (directPid.refused) return; + const pid = directPid.pid; + if (pid !== null) { updateJob(job, {}, `Stopping current proxy PID ${pid}.`); killProxy(pid); } @@ -504,7 +502,7 @@ async function restartAfterUpdate( updateJob(job, {}, `Port ${port} still busy after ${Math.trunc(RESTART_PORT_RECLAIM_MS / 1000)}s (reclaim could not free the socket); not starting on another port. Retry 'ocx start --port ${port}'.`); return; } - if (refuseForReplacementPid()) return; + if (readPidForRestart("after direct port reclaim").refused) return; (io.spawnStart ?? spawnDetachedStart)(job, job.installer, port, launcher); } diff --git a/tests/update-job.test.ts b/tests/update-job.test.ts index 41cb50742..da08f11cb 100644 --- a/tests/update-job.test.ts +++ b/tests/update-job.test.ts @@ -239,6 +239,7 @@ describe("GUI update execution decisions", () => { installer: "npm", restart: true, command: "", + releaseNotesUrl: "", log: [], }; writeFileSync(updateJobPath(job.id), JSON.stringify(job)); @@ -273,6 +274,7 @@ describe("GUI update execution decisions", () => { installer: "npm", restart: true, command: "", + releaseNotesUrl: "", log: [], }; writeFileSync(updateJobPath(job.id), JSON.stringify(job)); diff --git a/tests/update-stop-first.test.ts b/tests/update-stop-first.test.ts index 569810aff..07fbd3757 100644 --- a/tests/update-stop-first.test.ts +++ b/tests/update-stop-first.test.ts @@ -103,7 +103,10 @@ describe("update stops the running proxy before replacing files", () => { expect(updateJobSource).toContain("io.recoveryLauncherFn ?? findNpmRecoveryLauncher"); expect(updateJobSource).toContain("recoveryCaptured = { ...captured, recoveryLauncher }"); expect(updateJobSource).toContain("captured?.recoveryLauncher ?? packageLauncherPath()"); - expect(updateJobSource).toContain("if (refuseForReplacementPid()) return"); + expect(updateJobSource).toContain("const readPidForRestart = (context: string)"); + expect(updateJobSource).toContain('if (readPidForRestart("after service port reclaim").refused) return'); + expect(updateJobSource).toContain('const directPid = readPidForRestart("before direct restart")'); + expect(updateJobSource).toContain('if (readPidForRestart("after direct port reclaim").refused) return'); }); test("GUI recovery waits beyond the nested npm install deadline", () => { From f1f75fa4d7ed4be9466cfcd1e18a82256029bb26 Mon Sep 17 00:00:00 2001 From: wzb Date: Mon, 27 Jul 2026 14:52:01 +0800 Subject: [PATCH 06/16] fix(update): revalidate restart pid identity --- src/update/job.ts | 6 ++++- tests/update-job.test.ts | 42 +++++++++++++++++++++++++++++++++ tests/update-stop-first.test.ts | 2 ++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/src/update/job.ts b/src/update/job.ts index 2dcdcf678..3ea0215f7 100644 --- a/src/update/job.ts +++ b/src/update/job.ts @@ -429,8 +429,12 @@ async function restartAfterUpdate( ? captured.oldPid : undefined; const readCurrentPid = io.readPidFn ?? readPid; + const verifyCurrentPid = io.verifyPidIdentityFn ?? verifyPidIdentity; const readPidForRestart = (context: string): { pid: number | null; refused: boolean } => { - const currentPid = readCurrentPid(); + const rawPid = readCurrentPid(); + const currentPid = rawPid !== null && verifyCurrentPid(rawPid) === rawPid + ? rawPid + : null; if (currentPid === null || currentPid === oldPid) return { pid: currentPid, refused: false }; updateJob(job, {}, `A different identity-checked proxy PID appeared ${context}; leaving it untouched.`); return { pid: null, refused: true }; diff --git a/tests/update-job.test.ts b/tests/update-job.test.ts index da08f11cb..c274e8210 100644 --- a/tests/update-job.test.ts +++ b/tests/update-job.test.ts @@ -251,6 +251,7 @@ describe("GUI update execution decisions", () => { }, { serviceInstalledFn: () => true, readPidFn: () => (++pidReads === 1 ? 111 : 222), + verifyPidIdentityFn: pid => pid, waitForPort: async () => true, runService: () => { throw new Error("must not reinstall over a replacement PID"); }, spawnStart: () => { throw new Error("must not start over a replacement PID"); }, @@ -286,6 +287,7 @@ describe("GUI update execution decisions", () => { }, { serviceInstalledFn: () => false, readPidFn: () => (++pidReads < 3 ? null : 222), + verifyPidIdentityFn: pid => pid, waitForPort: async () => true, spawnStart: () => { throw new Error("must not start over a replacement PID"); }, }); @@ -295,6 +297,46 @@ describe("GUI update execution decisions", () => { )).toBe(true); }); + test("direct restart treats an unverified pidfile PID as absent", async () => { + const verified: number[] = []; + let spawned = 0; + const job: UpdateJobState = { + id: "restart-unverified-pid", + status: "restarting", + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + currentVersion: "2.7.40", + latestVersion: "2.7.41", + channel: "latest", + installer: "npm", + restart: true, + command: "", + releaseNotesUrl: "", + log: [], + }; + writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + await restartAfterUpdateForTests(job, { + port: 10100, + hostname: "127.0.0.1", + oldPid: 111, + recoveryLauncher: "/retired/bin/ocx.mjs", + }, { + serviceInstalledFn: () => false, + readPidFn: () => 222, + verifyPidIdentityFn: pid => { + verified.push(pid); + return null; + }, + waitForPort: async () => true, + spawnStart: () => { spawned += 1; }, + }); + expect(verified).toEqual([222, 222, 222]); + expect(spawned).toBe(1); + const log = readUpdateJob(job.id)?.log ?? []; + expect(log.some(line => line.includes("Stopping current proxy PID"))).toBe(false); + expect(log.some(line => line.includes("different identity-checked proxy PID"))).toBe(false); + }); + test("restart refuses to spawn when the captured port never becomes free", async () => { const spawned: Array<{ port?: number }> = []; const job: UpdateJobState = { diff --git a/tests/update-stop-first.test.ts b/tests/update-stop-first.test.ts index 07fbd3757..e1621d0e9 100644 --- a/tests/update-stop-first.test.ts +++ b/tests/update-stop-first.test.ts @@ -104,6 +104,8 @@ describe("update stops the running proxy before replacing files", () => { expect(updateJobSource).toContain("recoveryCaptured = { ...captured, recoveryLauncher }"); expect(updateJobSource).toContain("captured?.recoveryLauncher ?? packageLauncherPath()"); expect(updateJobSource).toContain("const readPidForRestart = (context: string)"); + expect(updateJobSource).toContain("const verifyCurrentPid = io.verifyPidIdentityFn ?? verifyPidIdentity"); + expect(updateJobSource).toContain("verifyCurrentPid(rawPid) === rawPid"); expect(updateJobSource).toContain('if (readPidForRestart("after service port reclaim").refused) return'); expect(updateJobSource).toContain('const directPid = readPidForRestart("before direct restart")'); expect(updateJobSource).toContain('if (readPidForRestart("after direct port reclaim").refused) return'); From e3a837eeacf8ce3d92a36c8598ac98ca78ecfe25 Mon Sep 17 00:00:00 2001 From: wzb Date: Mon, 27 Jul 2026 15:38:45 +0800 Subject: [PATCH 07/16] fix(update): close failed-install recovery races --- bin/ocx.mjs | 27 +++- docs/adr/0001-gui-update-worker.md | 19 ++- src/config.ts | 23 +++- src/update/install-process.d.mts | 33 +++++ src/update/install-process.mjs | 189 +++++++++++++++++++++++++++ src/update/job.ts | 134 ++++++++++++++----- tests/config.test.ts | 7 + tests/ocx-launcher-source.test.ts | 9 +- tests/update-install-process.test.ts | 59 +++++++++ tests/update-job.test.ts | 82 +++++++++++- tests/update-stop-first.test.ts | 20 +-- 11 files changed, 533 insertions(+), 69 deletions(-) create mode 100644 src/update/install-process.d.mts create mode 100644 src/update/install-process.mjs create mode 100644 tests/update-install-process.test.ts diff --git a/bin/ocx.mjs b/bin/ocx.mjs index e66571b32..e638437d6 100755 --- a/bin/ocx.mjs +++ b/bin/ocx.mjs @@ -15,6 +15,10 @@ import { homedir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { checkNpmCacheOwnership, formatNpmCacheOwnershipFailure } from "../src/update/npm-cache-preflight.mjs"; +import { + INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE, + runProcessTreeCommand, +} from "../src/update/install-process.mjs"; import { handoffWindowsTrayForUpdate, planWindowsTrayUpdate } from "../src/update/tray-update-plan.mjs"; const PKG = "@bitkyc08/opencodex"; @@ -114,7 +118,7 @@ function runTrayLifecycle(launcher, action) { }); } -function runNpmSelfUpdate() { +async function runNpmSelfUpdate() { const current = currentPackageVersion(); const tag = updateTag(current); const npm = npmBin(); @@ -230,12 +234,24 @@ function runNpmSelfUpdate() { } console.log(`Updating${latest ? ` to v${latest}` : ""}...\n$ ${npm} install -g ${PKG}@${tag}`); - const res = spawnSync(npm, ["install", "-g", `${PKG}@${tag}`], { + const res = await runProcessTreeCommand(npm, ["install", "-g", `${PKG}@${tag}`], { stdio: "inherit", - timeout: NPM_INSTALL_TIMEOUT_MS, + timeoutMs: NPM_INSTALL_TIMEOUT_MS, windowsHide: true, shell: winShell, }); + if (!res.treeExited) { + if (trayBeforeUpdate.restoreOnFailure) runTrayLifecycle(launcher, "start"); + console.error("\nopencodex: installer process-tree cleanup could not be confirmed; automatic recovery is unsafe until those processes exit."); + process.exit(INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE); + } + if (res.interruptedSignal) { + if (process.platform === "win32") { + process.exit(res.interruptedSignal === "SIGINT" ? 130 : 143); + } + process.kill(process.pid, res.interruptedSignal); + return; + } if (res.status === 0) { console.log(`\nUpdated${latest ? ` to v${latest}` : ""}.`); repairCodexShimIfNeeded(); @@ -286,7 +302,8 @@ function runNpmSelfUpdate() { process.exit(0); } if (trayBeforeUpdate.restoreOnFailure) runTrayLifecycle(launcher, "start"); - console.error(`\nUpdate failed (${npm} exit ${res.status ?? "?"}). Try manually: ${npm} install -g ${PKG}@${tag}`); + const failure = res.timedOut ? `timed out after ${Math.trunc(NPM_INSTALL_TIMEOUT_MS / 1000)}s` : `exit ${res.status ?? "?"}`; + console.error(`\nUpdate failed (${npm} ${failure}). Try manually: ${npm} install -g ${PKG}@${tag}`); process.exit(1); } @@ -359,7 +376,7 @@ if (updateHelpRequested) { } if (process.argv[2] === "update" && isNodeModulesInstall() && !isBunGlobalInstall()) { - runNpmSelfUpdate(); + await runNpmSelfUpdate(); } const bun = resolveBun(); diff --git a/docs/adr/0001-gui-update-worker.md b/docs/adr/0001-gui-update-worker.md index d49b6ed22..35f2787eb 100644 --- a/docs/adr/0001-gui-update-worker.md +++ b/docs/adr/0001-gui-update-worker.md @@ -36,12 +36,18 @@ was inactive. If health remains unavailable, a matching runtime-port record plus live PID still preserves the pre-update activity evidence. If the installer exits nonzero, the worker probes again immediately before recovery and leaves any old or concurrently replaced process alone when health or PID identity proves it is OpenCodex. When npm already stopped the proxy and retired -the old package, the worker validates the hidden package's name, version, and launcher, uses that -launcher for a best-effort restart, and verifies health. Service and direct restart paths also stop -when the pidfile has changed to a different identity-checked proxy. The update job remains failed -either way because restoring availability is not the same as installing the new version. The GUI -worker's install timeout exceeds the nested npm install deadline, so recovery cannot race an npm -replacement child that is still running. +the old package, the worker validates each candidate's name, version, path containment, and complete +launcher runtime with a side-effect-free `--version` probe. It tries runnable candidates in order +until one restores health. Service and direct restart paths also stop when the pidfile has changed +to a different identity-checked proxy, and every such identity decision re-reads the process command +line instead of trusting a PID-only cache across the update boundary. The update job remains failed +either way because restoring availability is not the same as installing the new version. + +The npm installer runs in an isolated process tree. On timeout, the launcher terminates and awaits +the whole POSIX process group or Windows `taskkill /T /F` tree before returning failure. The outer +worker timeout remains longer than the nested deadline, and automatic recovery is skipped if either +layer cannot prove installer-tree shutdown, so recovery never races a lifecycle or replacement child +that may still be writing package files. After an update requests a restart, the worker now waits for an identity-checked `/healthz` to return and remain healthy for a short stability window before marking the job successful. This @@ -67,6 +73,7 @@ restart path because `bun add -g` does not restart the proxy. - Update status survives a proxy restart because it is stored in the opencodex config directory. - Restart handling can branch between service-managed installs and direct detached proxy starts. - npm cache ownership failures are detected before service or proxy shutdown. +- Failed installer process trees are terminated and confirmed gone before recovery starts. - A failed install best-effort restores a previously-active proxy without claiming update success. - A completed install can still finish with `status: "failed"` when the replacement proxy never becomes healthy or flaps during the stability window; the job log then points the user at diff --git a/src/config.ts b/src/config.ts index c056e1954..8c9f1cea2 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1225,15 +1225,19 @@ export function isOcxStartCommandLine(commandLine: string): boolean { return hasOcxEntrypoint && /(?:^|[\s"'])start(?:$|[\s"'])/.test(normalized); } +function isLikelyOcxStartProcessUncached(pid: number): boolean { + const commandLine = readProcessCommandLine(pid); + if (commandLine === undefined) return false; + return isOcxStartCommandLine(commandLine); +} + /** Per-process memo: waitForProxy/findLiveProxy used to spawn powershell on every 150ms poll. */ const ocxStartProcessCache = new Map(); function isLikelyOcxStartProcess(pid: number): boolean { const cached = ocxStartProcessCache.get(pid); if (cached !== undefined) return cached; - const commandLine = readProcessCommandLine(pid); - if (commandLine === undefined) return false; - const ok = isOcxStartCommandLine(commandLine); + const ok = isLikelyOcxStartProcessUncached(pid); ocxStartProcessCache.set(pid, ok); return ok; } @@ -1271,6 +1275,19 @@ export function verifyPidIdentity(candidatePid: number): number | null { return isLikelyOcxStartProcess(candidatePid) ? candidatePid : null; } +/** + * Re-read the process command line for update/recovery decisions that span enough + * time for the OS to reuse a PID. Never trust the per-process polling cache here. + */ +export function verifyPidIdentityFresh(candidatePid: number): number | null { + try { + process.kill(candidatePid, 0); + } catch (e: unknown) { + if ((e as NodeJS.ErrnoException).code !== "EPERM") return null; + } + return isLikelyOcxStartProcessUncached(candidatePid) ? candidatePid : null; +} + function readProcessCommandLine(pid: number): string | undefined { try { if (process.platform === "win32") { diff --git a/src/update/install-process.d.mts b/src/update/install-process.d.mts new file mode 100644 index 000000000..ce917242d --- /dev/null +++ b/src/update/install-process.d.mts @@ -0,0 +1,33 @@ +import type { StdioOptions } from "node:child_process"; + +export const INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE: number; + +export interface ProcessTreeCommandOptions { + timeoutMs?: number; + stdio?: StdioOptions; + windowsHide?: boolean; + shell?: boolean | string; + env?: NodeJS.ProcessEnv; + terminationGraceMs?: number; + forceWaitMs?: number; +} + +export interface ProcessTreeCommandResult { + status: number | null; + signal: NodeJS.Signals | null; + error?: Error; + interruptedSignal: NodeJS.Signals | null; + timedOut: boolean; + treeExited: boolean; +} + +export function terminateInstallerProcessTree( + pid: number | undefined, + options?: Pick, +): Promise; + +export function runProcessTreeCommand( + bin: string, + args: string[], + options?: ProcessTreeCommandOptions, +): Promise; diff --git a/src/update/install-process.mjs b/src/update/install-process.mjs new file mode 100644 index 000000000..c74141a02 --- /dev/null +++ b/src/update/install-process.mjs @@ -0,0 +1,189 @@ +import { spawn, spawnSync } from "node:child_process"; + +const TREE_POLL_INTERVAL_MS = 50; +const DEFAULT_TERMINATION_GRACE_MS = 5_000; +const DEFAULT_FORCE_WAIT_MS = 5_000; + +/** Exit code used when the updater cannot prove that its installer tree is gone. */ +export const INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE = 75; + +function isNoSuchProcess(error) { + return error && typeof error === "object" && "code" in error && error.code === "ESRCH"; +} + +function isProcessTreeAlive(pid) { + try { + process.kill(process.platform === "win32" ? pid : -pid, 0); + return true; + } catch (error) { + return !isNoSuchProcess(error); + } +} + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +async function waitForProcessTreeExit(pid, timeoutMs) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (!isProcessTreeAlive(pid)) return true; + await sleep(TREE_POLL_INTERVAL_MS); + } + return !isProcessTreeAlive(pid); +} + +/** + * Terminate a detached installer and every descendant before package recovery starts. + * POSIX uses the installer's process group; Windows snapshots and kills the tree while + * the root process is still alive via taskkill /T /F. + */ +export async function terminateInstallerProcessTree( + pid, + { + terminationGraceMs = DEFAULT_TERMINATION_GRACE_MS, + forceWaitMs = DEFAULT_FORCE_WAIT_MS, + } = {}, +) { + if (!Number.isSafeInteger(pid) || pid <= 0) return true; + if (!isProcessTreeAlive(pid)) return true; + + if (process.platform === "win32") { + const taskkill = `${process.env.SystemRoot ?? "C:\\Windows"}\\System32\\taskkill.exe`; + const result = spawnSync(taskkill, ["/PID", String(pid), "/T", "/F"], { + stdio: "ignore", + timeout: terminationGraceMs + forceWaitMs, + windowsHide: true, + }); + if (result.status !== 0) return false; + return waitForProcessTreeExit(pid, forceWaitMs); + } + + try { + process.kill(-pid, "SIGTERM"); + } catch (error) { + if (isNoSuchProcess(error)) return true; + return false; + } + if (await waitForProcessTreeExit(pid, terminationGraceMs)) return true; + + try { + process.kill(-pid, "SIGKILL"); + } catch (error) { + if (isNoSuchProcess(error)) return true; + return false; + } + return waitForProcessTreeExit(pid, forceWaitMs); +} + +/** + * Run a command in an isolated process tree. A timeout is not reported until the + * entire tree has been terminated or cleanup failure has been made explicit. + */ +export async function runProcessTreeCommand( + bin, + args, + { + timeoutMs, + stdio = "inherit", + windowsHide = true, + shell = false, + env = process.env, + terminationGraceMs = DEFAULT_TERMINATION_GRACE_MS, + forceWaitMs = DEFAULT_FORCE_WAIT_MS, + } = {}, +) { + let child; + try { + child = spawn(bin, args, { + detached: process.platform !== "win32", + env, + shell, + stdio, + windowsHide, + }); + } catch (error) { + return { + status: null, + signal: null, + error, + interruptedSignal: null, + timedOut: false, + treeExited: true, + }; + } + + const outcome = new Promise(resolve => { + let settled = false; + child.once("error", error => { + if (settled) return; + settled = true; + resolve({ status: null, signal: null, error }); + }); + child.once("exit", (status, signal) => { + if (settled) return; + settled = true; + resolve({ status, signal }); + }); + }); + + let timedOut = false; + let interruptedSignal = null; + let cleanupPromise = null; + let reportCleanupFailure; + const cleanupFailure = new Promise(resolve => { + reportCleanupFailure = resolve; + }); + const startCleanup = () => { + if (cleanupPromise) return; + cleanupPromise = terminateInstallerProcessTree(child.pid, { + terminationGraceMs, + forceWaitMs, + }); + void cleanupPromise.then(treeExited => { + if (treeExited) return; + try { child.kill("SIGKILL"); } catch { /* best-effort root cleanup */ } + reportCleanupFailure({ status: null, signal: null }); + }); + }; + const forwardedSignals = process.platform === "win32" + ? ["SIGINT", "SIGTERM"] + : ["SIGINT", "SIGTERM", "SIGHUP"]; + const signalHandlers = forwardedSignals.map(signal => { + const handler = () => { + interruptedSignal ??= signal; + startCleanup(); + }; + process.on(signal, handler); + return [signal, handler]; + }); + const timer = Number.isFinite(timeoutMs) && timeoutMs > 0 + ? setTimeout(() => { + timedOut = true; + startCleanup(); + }, timeoutMs) + : null; + + const result = await Promise.race([outcome, cleanupFailure]); + if (timer !== null) clearTimeout(timer); + + let treeExited = true; + if (cleanupPromise) { + treeExited = await cleanupPromise; + } else if (child.pid && isProcessTreeAlive(child.pid)) { + treeExited = await terminateInstallerProcessTree(child.pid, { + terminationGraceMs, + forceWaitMs, + }); + } + for (const [signal, handler] of signalHandlers) process.removeListener(signal, handler); + + return { + ...result, + signal: interruptedSignal ?? result.signal, + status: timedOut || interruptedSignal !== null ? null : result.status, + interruptedSignal, + timedOut, + treeExited, + }; +} diff --git a/src/update/job.ts b/src/update/job.ts index 3ea0215f7..5d11e66f0 100644 --- a/src/update/job.ts +++ b/src/update/job.ts @@ -9,7 +9,7 @@ import { readAlivePid, readPid, readRuntimePort, - verifyPidIdentity, + verifyPidIdentityFresh, } from "../config"; import { killProxy } from "../lib/process-control"; import { reclaimListenPort } from "../server/port-reclaim"; @@ -35,6 +35,10 @@ import { updateCommandStr, } from "./index"; import { isNewer } from "./notify"; +import { + INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE, + runProcessTreeCommand, +} from "./install-process.mjs"; import { handoffWindowsTrayForUpdate, planWindowsTrayUpdate } from "./tray-update-plan.mjs"; const RELEASE_NOTES_URL = "https://github.com/lidge-jun/opencodex/releases/latest"; @@ -114,6 +118,8 @@ interface ValidatedNpmLauncher { mtimeMs: number; } +type RecoveryLauncherProbe = (launcher: string) => Promise; + function isPathInside(parent: string, child: string): boolean { const fromParent = relative(parent, child); return fromParent !== "" @@ -122,8 +128,8 @@ function isPathInside(parent: string, child: string): boolean { && !isAbsolute(fromParent); } -/** Validate that a package root still contains the exact pre-update package launcher. */ -function validateNpmRecoveryPackage(packageRoot: string, expectedVersion: string): ValidatedNpmLauncher | null { +/** Validate the on-disk identity before any candidate code is executed. */ +function inspectNpmRecoveryPackage(packageRoot: string, expectedVersion: string): ValidatedNpmLauncher | null { try { const rootStat = lstatSync(packageRoot); if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) return null; @@ -146,31 +152,59 @@ function validateNpmRecoveryPackage(packageRoot: string, expectedVersion: string } } +async function probeNpmRecoveryLauncher(launcher: string): Promise { + const result = await runProcessTreeCommand(nodeBin(), [launcher, "--version"], { + stdio: "ignore", + timeoutMs: 10_000, + }); + return result.status === 0 && result.treeExited; +} + /** * npm retires a package to a hidden sibling such as `.opencodex-Ab12Cd34` * before replacing it. On a late install failure, prefer an intact current root; * otherwise recover through the validated retired copy of the pre-update version. */ -export function findNpmRecoveryLauncher( +export async function findNpmRecoveryLaunchers( currentLauncher: string, expectedVersion: string, -): string | null { + probeLauncher: RecoveryLauncherProbe = probeNpmRecoveryLauncher, +): Promise { const packageRoot = dirname(dirname(resolve(currentLauncher))); - const current = validateNpmRecoveryPackage(packageRoot, expectedVersion); - if (current) return current.launcher; - const scopeRoot = dirname(packageRoot); const prefix = `.${basename(packageRoot)}-`; + const candidates: ValidatedNpmLauncher[] = []; + const current = inspectNpmRecoveryPackage(packageRoot, expectedVersion); + if (current) candidates.push(current); try { - const candidates = readdirSync(scopeRoot, { encoding: "utf8" }) + const retired = readdirSync(scopeRoot, { encoding: "utf8" }) .filter(name => name.startsWith(prefix) && /^[A-Za-z0-9]{8}$/.test(name.slice(prefix.length))) - .map(name => validateNpmRecoveryPackage(join(scopeRoot, name), expectedVersion)) + .map(name => inspectNpmRecoveryPackage(join(scopeRoot, name), expectedVersion)) .filter((candidate): candidate is ValidatedNpmLauncher => candidate !== null) .sort((a, b) => b.mtimeMs - a.mtimeMs); - return candidates[0]?.launcher ?? null; + candidates.push(...retired); } catch { - return null; + // The current package can still be a valid recovery candidate when the scope + // directory itself cannot be enumerated. + } + + const launchers: string[] = []; + for (const candidate of candidates) { + try { + // `--version` loads the launcher's static imports, resolves the bundled Bun + // binary, and imports the complete CLI graph without starting or stopping a proxy. + if (await probeLauncher(candidate.launcher)) launchers.push(candidate.launcher); + } catch { /* an un-runnable candidate is equivalent to a partial package */ } } + return launchers; +} + +export async function findNpmRecoveryLauncher( + currentLauncher: string, + expectedVersion: string, + probeLauncher?: RecoveryLauncherProbe, +): Promise { + return (await findNpmRecoveryLaunchers(currentLauncher, expectedVersion, probeLauncher))[0] ?? null; } function formatCommand(bin: string, args: string[]): string { @@ -345,7 +379,13 @@ export function startUpdateJob(channel: Channel, restart: boolean): UpdateJobSta return { ...job, pid: child.pid }; } -function runLoggedCommand(job: UpdateJobState, bin: string, args: string[], timeout: number): { status: number | null; signal: NodeJS.Signals | null } { +interface LoggedCommandResult { + status: number | null; + signal: NodeJS.Signals | null; + timedOut: boolean; +} + +function runLoggedCommand(job: UpdateJobState, bin: string, args: string[], timeout: number): LoggedCommandResult { job = updateJob(job, {}, `$ ${formatCommand(bin, args)}`); const result = spawnSync(bin, args, { encoding: "utf8", @@ -356,7 +396,17 @@ function runLoggedCommand(job: UpdateJobState, bin: string, args: string[], time const stderr = typeof result.stderr === "string" ? result.stderr.trim() : ""; if (stdout) job = updateJob(job, {}, stdout.slice(-4000)); if (stderr) updateJob(job, {}, stderr.slice(-4000)); - return { status: result.status, signal: result.signal }; + return { + status: result.status, + signal: result.signal, + timedOut: (result.error as NodeJS.ErrnoException | undefined)?.code === "ETIMEDOUT", + }; +} + +export function installerFailureAllowsRecovery(result: LoggedCommandResult): boolean { + return !result.timedOut + && result.signal === null + && result.status !== INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE; } function spawnDetachedStart( @@ -396,7 +446,7 @@ export interface RestartIo { /** Richer /healthz read for update-correlated restart evidence (pid + version). */ probeProxyIdentity?: (port: number, hostname?: string) => Promise; verifyPidIdentityFn?: (pid: number) => number | null; - recoveryLauncherFn?: (currentLauncher: string, expectedVersion: string) => string | null; + recoveryLaunchersFn?: (currentLauncher: string, expectedVersion: string) => Promise | string[]; sleepMs?: (ms: number) => Promise; now?: () => number; /** Service-mode install/reinstall command (defaults to spawnSync via runLoggedCommand). */ @@ -429,7 +479,7 @@ async function restartAfterUpdate( ? captured.oldPid : undefined; const readCurrentPid = io.readPidFn ?? readPid; - const verifyCurrentPid = io.verifyPidIdentityFn ?? verifyPidIdentity; + const verifyCurrentPid = io.verifyPidIdentityFn ?? verifyPidIdentityFresh; const readPidForRestart = (context: string): { pid: number | null; refused: boolean } => { const rawPid = readCurrentPid(); const currentPid = rawPid !== null && verifyCurrentPid(rawPid) === rawPid @@ -675,7 +725,7 @@ export async function findLiveProxyForUpdate(io: UpdateLivenessIo = {}): Promise const candidatePid = (io.readAlivePidFn ?? readAlivePid)(); if (candidatePid === null) return null; - const verifiedPid = (io.verifyPidIdentityFn ?? verifyPidIdentity)(candidatePid); + const verifiedPid = (io.verifyPidIdentityFn ?? verifyPidIdentityFresh)(candidatePid); if (verifiedPid !== candidatePid) return null; const runtime = (io.readRuntimePortFn ?? readRuntimePort)(candidatePid); if (!runtime) return null; @@ -697,7 +747,7 @@ async function recoverFailedGuiUpdate( } const oldPidIdentityMatches = captured.oldPid != null - && (io.verifyPidIdentityFn ?? verifyPidIdentity)(captured.oldPid) === captured.oldPid; + && (io.verifyPidIdentityFn ?? verifyPidIdentityFresh)(captured.oldPid) === captured.oldPid; if (oldPidIdentityMatches) { updateJob(job, {}, `Update command failed and health probes timed out, but pre-update PID ${captured.oldPid} still identifies as OpenCodex; refusing an automatic restart.`); return "still-running"; @@ -710,29 +760,37 @@ async function recoverFailedGuiUpdate( updateJob(job, {}, "Update command failed after the proxy stopped; attempting to restore the proxy..."); try { - let recoveryCaptured: { - port: number; - hostname: string; - oldPid?: number; - recoveryLauncher?: string; - } = captured; + let recoveryLaunchers: Array = [undefined]; if (job.installer === "npm") { const currentLauncher = packageLauncherPath(); - const resolveRecoveryLauncher = io.recoveryLauncherFn ?? findNpmRecoveryLauncher; - const recoveryLauncher = resolveRecoveryLauncher(currentLauncher, job.currentVersion); - if (!recoveryLauncher) { - updateJob(job, {}, "Could not find a validated current or npm-retired launcher for the pre-update package."); + const resolveRecoveryLaunchers = io.recoveryLaunchersFn ?? findNpmRecoveryLaunchers; + recoveryLaunchers = await resolveRecoveryLaunchers(currentLauncher, job.currentVersion); + if (recoveryLaunchers.length === 0) { + updateJob(job, {}, "Could not find a runnable current or npm-retired launcher for the pre-update package."); updateJob(job, {}, `Automatic proxy recovery failed. Restore the package, then run 'ocx service install' or 'ocx start --port ${captured.port}'.`); return "failed"; } - recoveryCaptured = { ...captured, recoveryLauncher }; } + const restartFn = io.restartAfterUpdateFn ?? restartAfterUpdate; - await restartFn(job, recoveryCaptured, io); - const healthy = await awaitRestartedProxyHealthy(job, recoveryCaptured, io); - if (healthy.ok) { - updateJob(job, {}, `Previous proxy availability restored on ${captured.hostname}:${captured.port}; the update itself still failed.`); - return "restarted"; + for (let index = 0; index < recoveryLaunchers.length; index += 1) { + const recoveryLauncher = recoveryLaunchers[index]; + const recoveryCaptured = recoveryLauncher === undefined + ? captured + : { ...captured, recoveryLauncher }; + if (index > 0) { + updateJob(job, {}, `Recovery candidate ${index} did not restore the proxy; trying candidate ${index + 1} of ${recoveryLaunchers.length}.`); + } + try { + await restartFn(job, recoveryCaptured, io); + const healthy = await awaitRestartedProxyHealthy(job, recoveryCaptured, io); + if (healthy.ok) { + updateJob(job, {}, `Previous proxy availability restored on ${captured.hostname}:${captured.port}; the update itself still failed.`); + return "restarted"; + } + } catch { + updateJob(job, {}, `Recovery candidate ${index + 1} failed before health confirmation.`); + } } } catch (error) { updateJob(job, {}, `Automatic proxy recovery threw: ${error instanceof Error ? error.message : String(error)}`); @@ -1000,7 +1058,13 @@ export async function runGuiUpdateWorker(jobId: string, channel: Channel, restar startWindowsTray(); } catch { /* retain the primary update failure */ } } - const recovery = await recoverFailedGuiUpdate(job, captured, proxyWasActive); + const mayRecover = installerFailureAllowsRecovery(result); + const recovery = mayRecover + ? await recoverFailedGuiUpdate(job, captured, proxyWasActive) + : "failed"; + if (!mayRecover) { + updateJob(job, {}, "Automatic proxy recovery was skipped because installer process-tree shutdown was not confirmed. Wait for installer processes to exit before restoring the package or proxy."); + } updateJob(job, { status: "failed", exitCode: result.status, diff --git a/tests/config.test.ts b/tests/config.test.ts index 1224c15f4..7ef5c9b42 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -885,6 +885,13 @@ describe("opencodex config defaults", () => { expect(isOcxStartCommandLine("notepad.exe")).toBe(false); }); + test("exposes a fresh process identity path for lifecycle boundaries", () => { + const source = readFileSync(join(import.meta.dir, "..", "src", "config.ts"), "utf8"); + const freshCheck = /export function verifyPidIdentityFresh[\s\S]*?\n}/.exec(source)?.[0] ?? ""; + expect(freshCheck).toContain("isLikelyOcxStartProcessUncached(candidatePid)"); + expect(freshCheck).not.toContain("isLikelyOcxStartProcess(candidatePid)"); + }); + test("writes pid file as a numeric pid", () => { writePid(process.pid); diff --git a/tests/ocx-launcher-source.test.ts b/tests/ocx-launcher-source.test.ts index 5ecdcccb6..ff5a5d421 100644 --- a/tests/ocx-launcher-source.test.ts +++ b/tests/ocx-launcher-source.test.ts @@ -10,11 +10,10 @@ const source = readFileSync(join(import.meta.dir, "..", "bin", "ocx.mjs"), "utf8 describe("ocx.mjs npm launcher (source invariants)", () => { test("npm spawns go through a shell on Windows (Node ≥18.20 EINVALs shell-less .cmd spawns)", () => { - const spawnSites = source.match(/spawnSync\(npm,[\s\S]*?\}\)/g) ?? []; - expect(spawnSites.length).toBe(2); - for (const site of spawnSites) { - expect(site).toContain("shell: winShell"); - } + const viewSpawn = /spawnSync\(npm,[\s\S]*?\}\)/.exec(source)?.[0]; + const installSpawn = /runProcessTreeCommand\(npm,[\s\S]*?\}\)/.exec(source)?.[0]; + expect(viewSpawn).toContain("shell: winShell"); + expect(installSpawn).toContain("shell: winShell"); expect(source).toContain('const winShell = process.platform === "win32";'); }); diff --git a/tests/update-install-process.test.ts b/tests/update-install-process.test.ts new file mode 100644 index 000000000..c3598426e --- /dev/null +++ b/tests/update-install-process.test.ts @@ -0,0 +1,59 @@ +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runProcessTreeCommand } from "../src/update/install-process.mjs"; + +const cleanupPids = new Set(); +const cleanupDirs = new Set(); + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +afterEach(() => { + for (const pid of cleanupPids) { + try { process.kill(pid, "SIGKILL"); } catch { /* already gone */ } + } + cleanupPids.clear(); + for (const dir of cleanupDirs) rmSync(dir, { recursive: true, force: true }); + cleanupDirs.clear(); +}); + +describe("update installer process isolation", () => { + test("timeout kills and awaits the installer descendant tree", async () => { + const dir = join(tmpdir(), `ocx-installer-tree-${process.pid}-${Date.now()}`); + const fixture = join(dir, "installer-parent.mjs"); + const descendantPidPath = join(dir, "descendant.pid"); + mkdirSync(dir, { recursive: true }); + cleanupDirs.add(dir); + writeFileSync(fixture, [ + 'import { spawn } from "node:child_process";', + 'import { writeFileSync } from "node:fs";', + 'const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" });', + 'writeFileSync(process.argv[2], String(child.pid));', + 'setInterval(() => {}, 1000);', + "", + ].join("\n")); + + const result = await runProcessTreeCommand(process.execPath, [fixture, descendantPidPath], { + forceWaitMs: 2_000, + stdio: "ignore", + terminationGraceMs: 500, + timeoutMs: 1_000, + }); + + expect(existsSync(descendantPidPath)).toBe(true); + const descendantPid = Number.parseInt(readFileSync(descendantPidPath, "utf8"), 10); + const descendantAlive = isAlive(descendantPid); + if (descendantAlive) cleanupPids.add(descendantPid); + expect(result.timedOut).toBe(true); + expect(result.treeExited).toBe(true); + expect(descendantAlive).toBe(false); + }); +}); diff --git a/tests/update-job.test.ts b/tests/update-job.test.ts index c274e8210..4b7de3d0e 100644 --- a/tests/update-job.test.ts +++ b/tests/update-job.test.ts @@ -7,7 +7,9 @@ import { confirmRestartAfterUpdateForTests, findLiveProxyForUpdate, findNpmRecoveryLauncher, + findNpmRecoveryLaunchers, finishGuiUpdateRestart, + installerFailureAllowsRecovery, npmSelfUpdateRestartEvidence, readUpdateJob, recoverFailedGuiUpdateForTests, @@ -106,7 +108,7 @@ describe("GUI update execution decisions", () => { }); }); - test("finds a validated npm-retired launcher when the current package is partial", () => { + test("finds a validated npm-retired launcher when the current package is partial", async () => { const scopeRoot = join(dir, "global", "@bitkyc08"); const currentRoot = join(scopeRoot, "opencodex"); const retiredRoot = join(scopeRoot, ".opencodex-Ab12Cd34"); @@ -119,11 +121,11 @@ describe("GUI update execution decisions", () => { writeFileSync(join(root, "bin", "ocx.mjs"), "#!/usr/bin/env node\n"); } - expect(findNpmRecoveryLauncher(join(currentRoot, "bin", "ocx.mjs"), "2.7.40")) + expect(await findNpmRecoveryLauncher(join(currentRoot, "bin", "ocx.mjs"), "2.7.40")) .toBe(realpathSync(join(retiredRoot, "bin", "ocx.mjs"))); }); - test("rejects an npm-retired launcher with the wrong package identity", () => { + test("rejects an npm-retired launcher with the wrong package identity", async () => { const scopeRoot = join(dir, "global", "@bitkyc08"); const currentRoot = join(scopeRoot, "opencodex"); const retiredRoot = join(scopeRoot, ".opencodex-Ab12Cd34"); @@ -134,7 +136,31 @@ describe("GUI update execution decisions", () => { })); writeFileSync(join(retiredRoot, "bin", "ocx.mjs"), "#!/usr/bin/env node\n"); - expect(findNpmRecoveryLauncher(join(currentRoot, "bin", "ocx.mjs"), "2.7.40")).toBeNull(); + expect(await findNpmRecoveryLauncher(join(currentRoot, "bin", "ocx.mjs"), "2.7.40")).toBeNull(); + }); + + test("skips a matching current package whose complete launcher runtime cannot load", async () => { + const scopeRoot = join(dir, "global", "@bitkyc08"); + const currentRoot = join(scopeRoot, "opencodex"); + const retiredRoot = join(scopeRoot, ".opencodex-Ab12Cd34"); + for (const root of [currentRoot, retiredRoot]) { + mkdirSync(join(root, "bin"), { recursive: true }); + writeFileSync(join(root, "package.json"), JSON.stringify({ + name: "@bitkyc08/opencodex", + version: "2.7.40", + })); + } + writeFileSync(join(currentRoot, "bin", "ocx.mjs"), 'import "../src/cli/index.ts";\n'); + writeFileSync(join(retiredRoot, "bin", "ocx.mjs"), "process.exit(0);\n"); + + expect(await findNpmRecoveryLaunchers(join(currentRoot, "bin", "ocx.mjs"), "2.7.40")) + .toEqual([realpathSync(join(retiredRoot, "bin", "ocx.mjs"))]); + }); + + test("only recovers after a clean installer exit", () => { + expect(installerFailureAllowsRecovery({ status: 1, signal: null, timedOut: false })).toBe(true); + expect(installerFailureAllowsRecovery({ status: 75, signal: null, timedOut: false })).toBe(false); + expect(installerFailureAllowsRecovery({ status: null, signal: "SIGTERM", timedOut: true })).toBe(false); }); test("proxy restart pins --port so post-update start does not hop to an ephemeral port", () => { @@ -628,7 +654,7 @@ describe("GUI update execution decisions", () => { ), verifyPidIdentityFn: () => null, sleepMs: async () => {}, - recoveryLauncherFn: () => { throw new Error("must not resolve a launcher"); }, + recoveryLaunchersFn: () => { throw new Error("must not resolve a launcher"); }, restartAfterUpdateFn: async () => { restartCalls += 1; }, }, ); @@ -664,7 +690,7 @@ describe("GUI update execution decisions", () => { { probeProxyIdentity: async () => null, verifyPidIdentityFn: () => null, - recoveryLauncherFn: () => "/retired/bin/ocx.mjs", + recoveryLaunchersFn: () => ["/retired/bin/ocx.mjs"], probeProxy: async () => restarted, restartAfterUpdateFn: async (_job, captured) => { recoveryLauncher = captured?.recoveryLauncher; @@ -679,6 +705,50 @@ describe("GUI update execution decisions", () => { expect(readUpdateJob(job.id)?.log.some(line => line.includes("update itself still failed"))).toBe(true); }); + test("failed install tries the next runnable recovery package when the first does not start", async () => { + let now = 0; + let activeLauncher: string | undefined; + const attempted: Array = []; + const job: UpdateJobState = { + id: "failed-install-recovery-fallback", + status: "running", + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + currentVersion: "2.7.40", + latestVersion: "2.7.41", + channel: "latest", + installer: "npm", + restart: true, + command: "", + releaseNotesUrl: "", + log: [], + }; + writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + + const recovery = await recoverFailedGuiUpdateForTests( + job, + { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, + true, + { + probeProxyIdentity: async () => null, + verifyPidIdentityFn: () => null, + recoveryLaunchersFn: () => ["/current/bin/ocx.mjs", "/retired/bin/ocx.mjs"], + probeProxy: async () => activeLauncher === "/retired/bin/ocx.mjs", + restartAfterUpdateFn: async (_job, captured) => { + activeLauncher = captured?.recoveryLauncher; + attempted.push(activeLauncher); + if (activeLauncher === "/current/bin/ocx.mjs") throw new Error("partial runtime"); + }, + now: () => now, + sleepMs: async (ms) => { now += ms; }, + }, + ); + + expect(recovery).toBe("restarted"); + expect(attempted).toEqual(["/current/bin/ocx.mjs", "/retired/bin/ocx.mjs"]); + expect(readUpdateJob(job.id)?.log.some(line => line.includes("trying candidate 2 of 2"))).toBe(true); + }); + test("restart confirmation fails when the proxy never becomes healthy", async () => { let now = 0; const job: UpdateJobState = { diff --git a/tests/update-stop-first.test.ts b/tests/update-stop-first.test.ts index e1621d0e9..545d9c33e 100644 --- a/tests/update-stop-first.test.ts +++ b/tests/update-stop-first.test.ts @@ -44,7 +44,7 @@ describe("update stops the running proxy before replacing files", () => { test("npm launcher update path stops via its own launcher path before npm install", () => { expect(launcherSource).toContain('spawnSync(process.execPath, [launcher, "stop"]'); const stopAt = launcherSource.indexOf('[launcher, "stop"]'); - const installAt = launcherSource.indexOf('spawnSync(npm, ["install", "-g"'); + const installAt = launcherSource.indexOf('runProcessTreeCommand(npm, ["install", "-g"'); expect(stopAt).toBeGreaterThan(-1); expect(stopAt).toBeLessThan(installAt); expect(launcherSource).toContain('existsSync(join(configDir(), "ocx.pid"))'); @@ -78,7 +78,7 @@ describe("update stops the running proxy before replacing files", () => { expect(launcherSource).toContain('name.startsWith("codex-history-backup-") && name.endsWith(".json")'); expect(launcherSource).toContain("if (historyRestoreIncomplete())"); const warnAt = launcherSource.indexOf("Codex resume history was NOT restored"); - const installAt = launcherSource.indexOf('spawnSync(npm, ["install", "-g"'); + const installAt = launcherSource.indexOf('runProcessTreeCommand(npm, ["install", "-g"'); expect(warnAt).toBeGreaterThan(-1); expect(warnAt).toBeLessThan(installAt); }); @@ -93,18 +93,18 @@ describe("update stops the running proxy before replacing files", () => { expect(updateJobSource).toContain("const liveBeforeUpdate = await findLiveProxyForUpdate()"); expect(updateJobSource).toContain("const proxyWasActive = liveBeforeUpdate !== null"); expect(updateJobSource).toContain("(io.readAlivePidFn ?? readAlivePid)()"); - expect(updateJobSource).toContain("(io.verifyPidIdentityFn ?? verifyPidIdentity)(candidatePid)"); + expect(updateJobSource).toContain("(io.verifyPidIdentityFn ?? verifyPidIdentityFresh)(candidatePid)"); expect(updateJobSource).toContain("(io.readRuntimePortFn ?? readRuntimePort)(candidatePid)"); expect(updateJobSource).not.toContain("const proxyWasActive = isServiceInstalled() || runtimeTrusted"); }); test("GUI failure recovery identity-checks the old PID and threads a validated launcher", () => { - expect(updateJobSource).toContain("(io.verifyPidIdentityFn ?? verifyPidIdentity)(captured.oldPid)"); - expect(updateJobSource).toContain("io.recoveryLauncherFn ?? findNpmRecoveryLauncher"); - expect(updateJobSource).toContain("recoveryCaptured = { ...captured, recoveryLauncher }"); + expect(updateJobSource).toContain("(io.verifyPidIdentityFn ?? verifyPidIdentityFresh)(captured.oldPid)"); + expect(updateJobSource).toContain("io.recoveryLaunchersFn ?? findNpmRecoveryLaunchers"); + expect(updateJobSource).toContain("{ ...captured, recoveryLauncher }"); expect(updateJobSource).toContain("captured?.recoveryLauncher ?? packageLauncherPath()"); expect(updateJobSource).toContain("const readPidForRestart = (context: string)"); - expect(updateJobSource).toContain("const verifyCurrentPid = io.verifyPidIdentityFn ?? verifyPidIdentity"); + expect(updateJobSource).toContain("const verifyCurrentPid = io.verifyPidIdentityFn ?? verifyPidIdentityFresh"); expect(updateJobSource).toContain("verifyCurrentPid(rawPid) === rawPid"); expect(updateJobSource).toContain('if (readPidForRestart("after service port reclaim").refused) return'); expect(updateJobSource).toContain('const directPid = readPidForRestart("before direct restart")'); @@ -119,7 +119,9 @@ describe("update stops the running proxy before replacing files", () => { const outerMs = Number(outerRaw?.replaceAll("_", "")); const innerMs = Number(innerRaw?.replaceAll("_", "")); expect(outerMs).toBeGreaterThanOrEqual(innerMs + 60_000); - expect(launcherSource).toContain("timeout: NPM_INSTALL_TIMEOUT_MS"); + expect(launcherSource).toContain("timeoutMs: NPM_INSTALL_TIMEOUT_MS"); + expect(launcherSource).toContain("await runProcessTreeCommand(npm"); + expect(updateJobSource).toContain("installerFailureAllowsRecovery(result)"); }); test("GUI worker update children use pipe stdio so Windows npm.cmd does not open consoles", () => { @@ -146,7 +148,7 @@ describe("ocx update --help has no side effects (#168)", () => { test("the npm launcher intercepts update --help before the self-update path", () => { const helpAt = launcherSource.indexOf("updateHelpRequested"); - const updateAt = launcherSource.indexOf("runNpmSelfUpdate();"); + const updateAt = launcherSource.indexOf("await runNpmSelfUpdate();"); expect(helpAt).toBeGreaterThan(-1); expect(launcherSource).toContain('process.argv[2] === "update" &&'); // The guard that CALLS the self-update must come after the help exit. From 07f0748a42d08b7ce0ad7f982990be4a2cb05da5 Mon Sep 17 00:00:00 2001 From: wzb Date: Mon, 27 Jul 2026 16:26:06 +0800 Subject: [PATCH 08/16] fix(update): harden failed-install recovery --- bin/ocx.mjs | 19 +++- docs/adr/0001-gui-update-worker.md | 25 +++--- src/config.ts | 11 ++- src/update/install-process.mjs | 109 ++++++++++++++++++++++- src/update/job.ts | 38 ++++++-- tests/config.test.ts | 2 + tests/ocx-launcher-source.test.ts | 25 +++++- tests/update-install-process.test.ts | 66 ++++++++++++-- tests/update-job.test.ts | 82 +++++++++++------ tests/update-npm-cache-preflight.test.ts | 2 + tests/update-stop-first.test.ts | 4 + 11 files changed, 319 insertions(+), 64 deletions(-) diff --git a/bin/ocx.mjs b/bin/ocx.mjs index e638437d6..b0a2c0160 100755 --- a/bin/ocx.mjs +++ b/bin/ocx.mjs @@ -241,16 +241,23 @@ async function runNpmSelfUpdate() { shell: winShell, }); if (!res.treeExited) { - if (trayBeforeUpdate.restoreOnFailure) runTrayLifecycle(launcher, "start"); console.error("\nopencodex: installer process-tree cleanup could not be confirmed; automatic recovery is unsafe until those processes exit."); process.exit(INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE); } if (res.interruptedSignal) { + if (trayBeforeUpdate.restoreOnFailure) runTrayLifecycle(launcher, "start"); + console.error(`\nopencodex: update interrupted (${res.interruptedSignal}); the proxy is still stopped. Run 'ocx start' or re-run 'ocx update'.`); + const exitCode = res.interruptedSignal === "SIGINT" + ? 130 + : res.interruptedSignal === "SIGHUP" + ? 129 + : 143; if (process.platform === "win32") { - process.exit(res.interruptedSignal === "SIGINT" ? 130 : 143); + process.exit(exitCode); } process.kill(process.pid, res.interruptedSignal); - return; + // Do not return to the update call site while fatal signal delivery is pending. + process.exit(exitCode); } if (res.status === 0) { console.log(`\nUpdated${latest ? ` to v${latest}` : ""}.`); @@ -302,7 +309,11 @@ async function runNpmSelfUpdate() { process.exit(0); } if (trayBeforeUpdate.restoreOnFailure) runTrayLifecycle(launcher, "start"); - const failure = res.timedOut ? `timed out after ${Math.trunc(NPM_INSTALL_TIMEOUT_MS / 1000)}s` : `exit ${res.status ?? "?"}`; + const failure = res.timedOut + ? `timed out after ${Math.trunc(NPM_INSTALL_TIMEOUT_MS / 1000)}s` + : res.error + ? `could not run: ${res.error.message}` + : `exit ${res.status ?? "?"}`; console.error(`\nUpdate failed (${npm} ${failure}). Try manually: ${npm} install -g ${PKG}@${tag}`); process.exit(1); } diff --git a/docs/adr/0001-gui-update-worker.md b/docs/adr/0001-gui-update-worker.md index 35f2787eb..77c86ef00 100644 --- a/docs/adr/0001-gui-update-worker.md +++ b/docs/adr/0001-gui-update-worker.md @@ -36,18 +36,23 @@ was inactive. If health remains unavailable, a matching runtime-port record plus live PID still preserves the pre-update activity evidence. If the installer exits nonzero, the worker probes again immediately before recovery and leaves any old or concurrently replaced process alone when health or PID identity proves it is OpenCodex. When npm already stopped the proxy and retired -the old package, the worker validates each candidate's name, version, path containment, and complete -launcher runtime with a side-effect-free `--version` probe. It tries runnable candidates in order -until one restores health. Service and direct restart paths also stop when the pidfile has changed -to a different identity-checked proxy, and every such identity decision re-reads the process command -line instead of trusting a PID-only cache across the update boundary. The update job remains failed -either way because restoring availability is not the same as installing the new version. +the old package, the worker validates each candidate's trusted ownership, name, version, path +containment, and complete launcher runtime with a side-effect-free `--version` probe. Candidate +inspection and restart are bounded to at most two candidates, preferring the current package and then +the newest retired copies, and it tries those runnable candidates in order until one restores health. +Service and direct restart paths also stop when the pidfile has changed to a different identity-checked +proxy, and every such identity decision re-reads the process command line instead of trusting a +PID-only cache across the update boundary. The update job remains failed either way because restoring +availability is not the same as installing the new version. The npm installer runs in an isolated process tree. On timeout, the launcher terminates and awaits -the whole POSIX process group or Windows `taskkill /T /F` tree before returning failure. The outer -worker timeout remains longer than the nested deadline, and automatic recovery is skipped if either -layer cannot prove installer-tree shutdown, so recovery never races a lifecycle or replacement child -that may still be writing package files. +the whole POSIX process group or a Windows `taskkill /T /F` tree before returning failure. POSIX +cleanup treats zombie-only groups as stopped and refuses to signal a process-group ID that has been +reused by a new leader. Windows has no retained job-object handle after a normally failed installer +root exits, so that case remains explicitly unconfirmed and automatic recovery (including tray +restoration) stays disabled. The outer worker timeout remains longer than the nested deadline, and +recovery is skipped whenever either layer cannot prove installer-tree shutdown, so recovery never +races a lifecycle or replacement child that may still be writing package files. After an update requests a restart, the worker now waits for an identity-checked `/healthz` to return and remain healthy for a short stability window before marking the job successful. This diff --git a/src/config.ts b/src/config.ts index 8c9f1cea2..c0820868b 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1283,9 +1283,16 @@ export function verifyPidIdentityFresh(candidatePid: number): number | null { try { process.kill(candidatePid, 0); } catch (e: unknown) { - if ((e as NodeJS.ErrnoException).code !== "EPERM") return null; + if ((e as NodeJS.ErrnoException).code !== "EPERM") { + ocxStartProcessCache.delete(candidatePid); + return null; + } } - return isLikelyOcxStartProcessUncached(candidatePid) ? candidatePid : null; + const isOcx = isLikelyOcxStartProcessUncached(candidatePid); + // This read is authoritative across a PID-reuse boundary. Reconcile the + // polling memo so later short-lived checks cannot serve the old identity. + ocxStartProcessCache.set(candidatePid, isOcx); + return isOcx ? candidatePid : null; } function readProcessCommandLine(pid: number): string | undefined { diff --git a/src/update/install-process.mjs b/src/update/install-process.mjs index c74141a02..3b6f33966 100644 --- a/src/update/install-process.mjs +++ b/src/update/install-process.mjs @@ -1,6 +1,7 @@ import { spawn, spawnSync } from "node:child_process"; +import { readdirSync, readFileSync } from "node:fs"; -const TREE_POLL_INTERVAL_MS = 50; +const TREE_POLL_INTERVAL_MS = 100; const DEFAULT_TERMINATION_GRACE_MS = 5_000; const DEFAULT_FORCE_WAIT_MS = 5_000; @@ -11,10 +12,81 @@ function isNoSuchProcess(error) { return error && typeof error === "object" && "code" in error && error.code === "ESRCH"; } +function inspectLinuxProcessGroup(groupId) { + try { + let hasRunningMember = false; + let hasRunningLeader = false; + for (const entry of readdirSync("/proc", { withFileTypes: true })) { + if (!entry.isDirectory() || !/^\d+$/.test(entry.name)) continue; + let stat; + try { + stat = readFileSync(`/proc/${entry.name}/stat`, "utf8"); + } catch { + continue; + } + const commandEnd = stat.lastIndexOf(")"); + if (commandEnd === -1) continue; + const fields = stat.slice(commandEnd + 2).trim().split(/\s+/); + const state = fields[0]; + const processGroup = Number.parseInt(fields[2] ?? "", 10); + if (processGroup !== groupId || state === "Z" || state === "X") continue; + hasRunningMember = true; + if (Number.parseInt(entry.name, 10) === groupId) hasRunningLeader = true; + } + return { hasRunningMember, hasRunningLeader }; + } catch { + return null; + } +} + +function inspectPsProcessGroup(groupId) { + const result = spawnSync("/bin/ps", ["-axo", "pid=,pgid=,state="], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + timeout: 2_000, + windowsHide: true, + }); + if (result.status !== 0 || typeof result.stdout !== "string") return null; + let hasRunningMember = false; + let hasRunningLeader = false; + for (const line of result.stdout.split(/\r?\n/)) { + const match = /^\s*(\d+)\s+(\d+)\s+(\S+)/.exec(line); + if (!match || Number.parseInt(match[2], 10) !== groupId) continue; + if (match[3].startsWith("Z") || match[3].startsWith("X")) continue; + hasRunningMember = true; + if (Number.parseInt(match[1], 10) === groupId) hasRunningLeader = true; + } + return { hasRunningMember, hasRunningLeader }; +} + +function inspectProcessGroup(groupId) { + if (process.platform === "linux") return inspectLinuxProcessGroup(groupId); + if (process.platform === "darwin" || process.platform === "freebsd" || process.platform === "openbsd") { + return inspectPsProcessGroup(groupId); + } + return null; +} + +function processGroupHasRunningMember(groupId) { + return inspectProcessGroup(groupId)?.hasRunningMember ?? null; +} + +function inspectProcessGroupAfterLeaderExit(groupId) { + const inspection = inspectProcessGroup(groupId); + if (!inspection) return "unknown"; + // A new group cannot reuse this ID without a live leader whose PID equals the + // group ID. Never signal that group; it is unrelated to the completed child. + if (inspection.hasRunningLeader) return "reused"; + return inspection.hasRunningMember ? "running" : "exited"; +} + function isProcessTreeAlive(pid) { try { process.kill(process.platform === "win32" ? pid : -pid, 0); - return true; + if (process.platform === "win32") return true; + // Zombies keep a process group addressable by signal 0 but cannot mutate the + // package tree. Treat a zombie-only group as fully stopped. + return processGroupHasRunningMember(pid) ?? true; } catch (error) { return !isNoSuchProcess(error); } @@ -46,19 +118,25 @@ export async function terminateInstallerProcessTree( } = {}, ) { if (!Number.isSafeInteger(pid) || pid <= 0) return true; - if (!isProcessTreeAlive(pid)) return true; if (process.platform === "win32") { + // Once the root has exited, Node has no job-object handle with which to prove + // that background descendants also exited. Fail closed instead of treating a + // missing root PID as proof that the complete installer tree is gone. + if (!isProcessTreeAlive(pid)) return false; const taskkill = `${process.env.SystemRoot ?? "C:\\Windows"}\\System32\\taskkill.exe`; const result = spawnSync(taskkill, ["/PID", String(pid), "/T", "/F"], { stdio: "ignore", timeout: terminationGraceMs + forceWaitMs, windowsHide: true, }); + // A non-zero taskkill result cannot distinguish an already-gone root from a + // partially-killed tree. Recovery must remain disabled in either case. if (result.status !== 0) return false; return waitForProcessTreeExit(pid, forceWaitMs); } + if (!isProcessTreeAlive(pid)) return true; try { process.kill(-pid, "SIGTERM"); } catch (error) { @@ -113,16 +191,30 @@ export async function runProcessTreeCommand( }; } + let posixTreeAfterRootExit = "unknown"; + let windowsFailedExitTreeUnknown = false; + let spawnFailed = false; const outcome = new Promise(resolve => { let settled = false; child.once("error", error => { if (settled) return; settled = true; + spawnFailed = true; resolve({ status: null, signal: null, error }); }); child.once("exit", (status, signal) => { if (settled) return; settled = true; + if (process.platform === "win32") { + // A successful package-manager exit is its completion contract. On a + // failed exit, however, a background descendant cannot be ruled out once + // the root PID is gone, so failed-install recovery stays fail-closed. + windowsFailedExitTreeUnknown = status !== 0 || signal !== null; + } else if (child.pid) { + // Inspect group membership directly. A live process whose PID equals the + // old group ID proves reuse, in which case cleanup must not signal it. + posixTreeAfterRootExit = inspectProcessGroupAfterLeaderExit(child.pid); + } resolve({ status, signal }); }); }); @@ -136,6 +228,9 @@ export async function runProcessTreeCommand( }); const startCleanup = () => { if (cleanupPromise) return; + // If libuv already observed root exit, let the exit handler inspect the old + // group with PID-reuse protection instead of signaling by a freed PID here. + if (child.exitCode !== null || child.signalCode !== null) return; cleanupPromise = terminateInstallerProcessTree(child.pid, { terminationGraceMs, forceWaitMs, @@ -170,11 +265,17 @@ export async function runProcessTreeCommand( let treeExited = true; if (cleanupPromise) { treeExited = await cleanupPromise; - } else if (child.pid && isProcessTreeAlive(child.pid)) { + } else if (spawnFailed) { + treeExited = true; + } else if (process.platform === "win32") { + treeExited = !windowsFailedExitTreeUnknown; + } else if (child.pid && posixTreeAfterRootExit === "running") { treeExited = await terminateInstallerProcessTree(child.pid, { terminationGraceMs, forceWaitMs, }); + } else if (posixTreeAfterRootExit !== "exited") { + treeExited = false; } for (const [signal, handler] of signalHandlers) process.removeListener(signal, handler); diff --git a/src/update/job.ts b/src/update/job.ts index 5d11e66f0..18dce79c3 100644 --- a/src/update/job.ts +++ b/src/update/job.ts @@ -49,6 +49,7 @@ const RESTART_HEALTH_TIMEOUT_MS = 15_000; const RESTART_STABILITY_WINDOW_MS = 15_000; const UPDATE_LIVENESS_PROBE_ATTEMPTS = 3; const UPDATE_LIVENESS_PROBE_DELAY_MS = 250; +const MAX_NPM_RECOVERY_CANDIDATES = 2; /** How long update restart waits for the captured port to become bindable after stop. */ export const RESTART_PORT_RECLAIM_MS = 30_000; @@ -128,21 +129,40 @@ function isPathInside(parent: string, child: string): boolean { && !isAbsolute(fromParent); } +function hasTrustedRecoveryOwner(uid: number): boolean { + const currentUid = process.getuid?.(); + // Root-owned global installations are trusted and otherwise unrecoverable by + // an unprivileged GUI process; reject packages planted by every other account. + return currentUid === undefined || uid === currentUid || uid === 0; +} + /** Validate the on-disk identity before any candidate code is executed. */ function inspectNpmRecoveryPackage(packageRoot: string, expectedVersion: string): ValidatedNpmLauncher | null { try { const rootStat = lstatSync(packageRoot); - if (!rootStat.isDirectory() || rootStat.isSymbolicLink()) return null; + if ( + !rootStat.isDirectory() + || rootStat.isSymbolicLink() + || !hasTrustedRecoveryOwner(rootStat.uid) + ) return null; const manifestPath = join(packageRoot, "package.json"); const manifestStat = lstatSync(manifestPath); - if (!manifestStat.isFile() || manifestStat.isSymbolicLink()) return null; + if ( + !manifestStat.isFile() + || manifestStat.isSymbolicLink() + || !hasTrustedRecoveryOwner(manifestStat.uid) + ) return null; const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as { name?: unknown; version?: unknown }; if (manifest.name !== PKG || manifest.version !== expectedVersion) return null; const launcherPath = join(packageRoot, "bin", "ocx.mjs"); const launcherStat = lstatSync(launcherPath); - if (!launcherStat.isFile() || launcherStat.isSymbolicLink()) return null; + if ( + !launcherStat.isFile() + || launcherStat.isSymbolicLink() + || !hasTrustedRecoveryOwner(launcherStat.uid) + ) return null; const canonicalRoot = realpathSync(packageRoot); const canonicalLauncher = realpathSync(launcherPath); if (!isPathInside(canonicalRoot, canonicalLauncher)) return null; @@ -189,7 +209,7 @@ export async function findNpmRecoveryLaunchers( } const launchers: string[] = []; - for (const candidate of candidates) { + for (const candidate of candidates.slice(0, MAX_NPM_RECOVERY_CANDIDATES)) { try { // `--version` loads the launcher's static imports, resolves the bundled Bun // binary, and imports the complete CLI graph without starting or stopping a proxy. @@ -1052,18 +1072,18 @@ export async function runGuiUpdateWorker(jobId: string, channel: Channel, restar */ const result = runLoggedCommand(job, cmd.bin, cmd.args, UPDATE_TIMEOUT_MS); if (result.status !== 0) { - if (trayWasRunning) { + const mayRecover = installerFailureAllowsRecovery(result); + if (trayWasRunning && mayRecover) { try { const { startWindowsTray } = await import("../tray/windows"); startWindowsTray(); } catch { /* retain the primary update failure */ } } - const mayRecover = installerFailureAllowsRecovery(result); - const recovery = mayRecover - ? await recoverFailedGuiUpdate(job, captured, proxyWasActive) - : "failed"; + let recovery: FailedUpdateRecovery | null = null; if (!mayRecover) { updateJob(job, {}, "Automatic proxy recovery was skipped because installer process-tree shutdown was not confirmed. Wait for installer processes to exit before restoring the package or proxy."); + } else { + recovery = await recoverFailedGuiUpdate(job, captured, proxyWasActive); } updateJob(job, { status: "failed", diff --git a/tests/config.test.ts b/tests/config.test.ts index 7ef5c9b42..6ca101106 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -890,6 +890,8 @@ describe("opencodex config defaults", () => { const freshCheck = /export function verifyPidIdentityFresh[\s\S]*?\n}/.exec(source)?.[0] ?? ""; expect(freshCheck).toContain("isLikelyOcxStartProcessUncached(candidatePid)"); expect(freshCheck).not.toContain("isLikelyOcxStartProcess(candidatePid)"); + expect(freshCheck).toContain("ocxStartProcessCache.set(candidatePid, isOcx)"); + expect(freshCheck).toContain("ocxStartProcessCache.delete(candidatePid)"); }); test("writes pid file as a numeric pid", () => { diff --git a/tests/ocx-launcher-source.test.ts b/tests/ocx-launcher-source.test.ts index ff5a5d421..c74b1a647 100644 --- a/tests/ocx-launcher-source.test.ts +++ b/tests/ocx-launcher-source.test.ts @@ -10,13 +10,30 @@ const source = readFileSync(join(import.meta.dir, "..", "bin", "ocx.mjs"), "utf8 describe("ocx.mjs npm launcher (source invariants)", () => { test("npm spawns go through a shell on Windows (Node ≥18.20 EINVALs shell-less .cmd spawns)", () => { - const viewSpawn = /spawnSync\(npm,[\s\S]*?\}\)/.exec(source)?.[0]; - const installSpawn = /runProcessTreeCommand\(npm,[\s\S]*?\}\)/.exec(source)?.[0]; - expect(viewSpawn).toContain("shell: winShell"); - expect(installSpawn).toContain("shell: winShell"); + const npmCallSites = [ + ...source.matchAll(/spawnSync\(npm,[\s\S]*?\}\)/g), + ...source.matchAll(/runProcessTreeCommand\(npm,[\s\S]*?\}\)/g), + ].map(match => match[0]); + expect(npmCallSites).toHaveLength(2); + for (const callSite of npmCallSites) expect(callSite).toContain("shell: winShell"); expect(source).toContain('const winShell = process.platform === "win32";'); }); + test("unsafe installer cleanup never restarts the tray, while confirmed interruption does", () => { + const cleanupFailure = source.slice( + source.indexOf("if (!res.treeExited)"), + source.indexOf("if (res.interruptedSignal)"), + ); + const interruption = source.slice( + source.indexOf("if (res.interruptedSignal)"), + source.indexOf("if (res.status === 0)"), + ); + expect(cleanupFailure).not.toContain('runTrayLifecycle(launcher, "start")'); + expect(interruption).toContain('runTrayLifecycle(launcher, "start")'); + expect(interruption).toContain("process.exit(exitCode)"); + expect(source).toContain("res.error.message"); + }); + test("--tag is allowlisted before reaching shell-joined spawn args", () => { expect(source).toContain('if (explicit === "preview" || explicit === "latest") return explicit;'); expect(source).not.toMatch(/if \(tagIndex !== -1 && process\.argv\[tagIndex \+ 1\]\) return process\.argv/); diff --git a/tests/update-install-process.test.ts b/tests/update-install-process.test.ts index c3598426e..88851fffd 100644 --- a/tests/update-install-process.test.ts +++ b/tests/update-install-process.test.ts @@ -1,4 +1,5 @@ import { afterEach, describe, expect, test } from "bun:test"; +import { spawnSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -7,9 +8,16 @@ import { runProcessTreeCommand } from "../src/update/install-process.mjs"; const cleanupPids = new Set(); const cleanupDirs = new Set(); -function isAlive(pid: number): boolean { +function isRunning(pid: number): boolean { try { process.kill(pid, 0); + if (process.platform !== "win32") { + const state = spawnSync("/bin/ps", ["-o", "state=", "-p", String(pid)], { + encoding: "utf8", + stdio: ["ignore", "pipe", "ignore"], + }); + if (state.status === 0 && /^[ZX]/.test(state.stdout.trim())) return false; + } return true; } catch { return false; @@ -26,6 +34,17 @@ afterEach(() => { }); describe("update installer process isolation", () => { + test("spawn failures report their cause without inventing a live process tree", async () => { + const result = await runProcessTreeCommand(join(tmpdir(), "ocx-command-that-does-not-exist"), [], { + stdio: "ignore", + timeoutMs: 1_000, + }); + + expect(result.status).toBeNull(); + expect(result.error).toBeInstanceOf(Error); + expect(result.treeExited).toBe(true); + }); + test("timeout kills and awaits the installer descendant tree", async () => { const dir = join(tmpdir(), `ocx-installer-tree-${process.pid}-${Date.now()}`); const fixture = join(dir, "installer-parent.mjs"); @@ -45,15 +64,50 @@ describe("update installer process isolation", () => { forceWaitMs: 2_000, stdio: "ignore", terminationGraceMs: 500, - timeoutMs: 1_000, + timeoutMs: 3_000, }); expect(existsSync(descendantPidPath)).toBe(true); const descendantPid = Number.parseInt(readFileSync(descendantPidPath, "utf8"), 10); - const descendantAlive = isAlive(descendantPid); - if (descendantAlive) cleanupPids.add(descendantPid); + const descendantRunning = isRunning(descendantPid); + if (descendantRunning) cleanupPids.add(descendantPid); expect(result.timedOut).toBe(true); expect(result.treeExited).toBe(true); - expect(descendantAlive).toBe(false); - }); + expect(descendantRunning).toBe(false); + }, 15_000); + + test("a failed root exit cleans POSIX descendants and fails closed on Windows", async () => { + const dir = join(tmpdir(), `ocx-installer-exit-tree-${process.pid}-${Date.now()}`); + const fixture = join(dir, "installer-parent.mjs"); + const descendantPidPath = join(dir, "descendant.pid"); + mkdirSync(dir, { recursive: true }); + cleanupDirs.add(dir); + writeFileSync(fixture, [ + 'import { spawn } from "node:child_process";', + 'import { writeFileSync } from "node:fs";', + 'const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { stdio: "ignore" });', + 'writeFileSync(process.argv[2], String(child.pid));', + 'setTimeout(() => process.exit(1), 50);', + "", + ].join("\n")); + + const result = await runProcessTreeCommand(process.execPath, [fixture, descendantPidPath], { + forceWaitMs: 2_000, + stdio: "ignore", + terminationGraceMs: 500, + timeoutMs: 5_000, + }); + + expect(result.status).toBe(1); + expect(result.timedOut).toBe(false); + const descendantPid = Number.parseInt(readFileSync(descendantPidPath, "utf8"), 10); + const descendantRunning = isRunning(descendantPid); + if (descendantRunning) cleanupPids.add(descendantPid); + if (process.platform === "win32") { + expect(result.treeExited).toBe(false); + } else { + expect(result.treeExited).toBe(true); + expect(descendantRunning).toBe(false); + } + }, 15_000); }); diff --git a/tests/update-job.test.ts b/tests/update-job.test.ts index 4b7de3d0e..cc6c504ea 100644 --- a/tests/update-job.test.ts +++ b/tests/update-job.test.ts @@ -157,6 +157,38 @@ describe("GUI update execution decisions", () => { .toEqual([realpathSync(join(retiredRoot, "bin", "ocx.mjs"))]); }); + test("bounds npm recovery candidates before running launcher probes", async () => { + const scopeRoot = join(dir, "global", "@bitkyc08"); + const currentRoot = join(scopeRoot, "opencodex"); + const retiredRoots = [ + join(scopeRoot, ".opencodex-Ab12Cd34"), + join(scopeRoot, ".opencodex-Ef56Gh78"), + join(scopeRoot, ".opencodex-Ij90Kl12"), + ]; + for (const root of [currentRoot, ...retiredRoots]) { + mkdirSync(join(root, "bin"), { recursive: true }); + writeFileSync(join(root, "package.json"), JSON.stringify({ + name: "@bitkyc08/opencodex", + version: "2.7.40", + })); + writeFileSync(join(root, "bin", "ocx.mjs"), "process.exit(0);\n"); + } + const probed: string[] = []; + + const launchers = await findNpmRecoveryLaunchers( + join(currentRoot, "bin", "ocx.mjs"), + "2.7.40", + async launcher => { + probed.push(launcher); + return true; + }, + ); + + expect(probed).toHaveLength(2); + expect(launchers).toEqual(probed); + expect(launchers[0]).toBe(realpathSync(join(currentRoot, "bin", "ocx.mjs"))); + }); + test("only recovers after a clean installer exit", () => { expect(installerFailureAllowsRecovery({ status: 1, signal: null, timedOut: false })).toBe(true); expect(installerFailureAllowsRecovery({ status: 75, signal: null, timedOut: false })).toBe(false); @@ -192,7 +224,7 @@ describe("GUI update execution decisions", () => { command: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); await restartAfterUpdateForTests(job, { port: 12345, hostname: "127.0.0.1", @@ -237,7 +269,7 @@ describe("GUI update execution decisions", () => { command: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); await restartAfterUpdateForTests(job, { port: 10100, hostname: "127.0.0.1", oldPid: 4242 }, { serviceInstalledFn: () => false, waitForPort: async (_port, _hostname, opts) => { @@ -268,7 +300,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); await restartAfterUpdateForTests(job, { port: 10100, hostname: "127.0.0.1", @@ -304,7 +336,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); await restartAfterUpdateForTests(job, { port: 10100, hostname: "127.0.0.1", @@ -340,7 +372,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); await restartAfterUpdateForTests(job, { port: 10100, hostname: "127.0.0.1", @@ -378,7 +410,7 @@ describe("GUI update execution decisions", () => { command: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); await restartAfterUpdateForTests(job, { port: 10100, hostname: "127.0.0.1" }, { serviceInstalledFn: () => false, waitForPort: async () => false, @@ -408,7 +440,7 @@ describe("GUI update execution decisions", () => { command: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const prev = process.env.OCX_BAKE_PORT; delete process.env.OCX_BAKE_PORT; try { @@ -454,7 +486,7 @@ describe("GUI update execution decisions", () => { command: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); await restartAfterUpdateForTests(job, { port: 19999, hostname: "127.0.0.1" }, { serviceInstalledFn: () => true, waitForPort: async () => true, @@ -515,7 +547,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const recovery = await recoverFailedGuiUpdateForTests( job, { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, @@ -546,7 +578,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const recovery = await recoverFailedGuiUpdateForTests( job, { port: 10100, hostname: "127.0.0.1" }, @@ -577,7 +609,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const recovery = await recoverFailedGuiUpdateForTests( job, { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, @@ -609,7 +641,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const recovery = await recoverFailedGuiUpdateForTests( job, { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, @@ -643,7 +675,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const recovery = await recoverFailedGuiUpdateForTests( job, { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, @@ -682,7 +714,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const recovery = await recoverFailedGuiUpdateForTests( job, { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, @@ -723,7 +755,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const recovery = await recoverFailedGuiUpdateForTests( job, @@ -764,7 +796,7 @@ describe("GUI update execution decisions", () => { command: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const ok = await confirmRestartAfterUpdateForTests(job, { port: 10100, hostname: "127.0.0.1" }, { probeProxy: async () => false, now: () => now, @@ -793,7 +825,7 @@ describe("GUI update execution decisions", () => { command: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const ok = await confirmRestartAfterUpdateForTests(job, { port: 10100, hostname: "127.0.0.1" }, { probeProxy: async () => now < 12_000, now: () => now, @@ -823,7 +855,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const ok = await confirmRestartAfterUpdateForTests(job, { port: 10100, hostname: "127.0.0.1" }, { probeProxy: async () => now >= 1_000, now: () => now, @@ -850,7 +882,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const ok = await finishGuiUpdateRestart( job, { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, @@ -888,7 +920,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const ok = await finishGuiUpdateRestart( job, { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, @@ -937,7 +969,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const ok = await finishGuiUpdateRestart( job, { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, @@ -980,7 +1012,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const ok = await finishGuiUpdateRestart( job, { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, @@ -1027,7 +1059,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const ok = await finishGuiUpdateRestart(job, { port: 10100, hostname: "127.0.0.1" }, "npm", { serviceInstalledFn: () => false, probeProxy: async () => { @@ -1070,7 +1102,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const ok = await finishGuiUpdateRestart(job, { port: 10100, hostname: "127.0.0.1" }, "npm", { serviceInstalledFn: () => true, // Soft probe times out (proxy down after npm update); confirm after explicit restart succeeds. @@ -1109,7 +1141,7 @@ describe("GUI update execution decisions", () => { releaseNotesUrl: "", log: [], }; - writeFileSync(updateJobPath(job.id), JSON.stringify(job)); + writeFileSync(updateJobPath(), JSON.stringify(job)); const ok = await finishGuiUpdateRestart(job, { port: 10100, hostname: "127.0.0.1" }, "bun", { probeProxy: async () => true, now: () => now, diff --git a/tests/update-npm-cache-preflight.test.ts b/tests/update-npm-cache-preflight.test.ts index e4e2e3bc3..b15d6374d 100644 --- a/tests/update-npm-cache-preflight.test.ts +++ b/tests/update-npm-cache-preflight.test.ts @@ -132,6 +132,7 @@ describe("npm cache ownership pre-flight", () => { test("fails closed when the configured cache root does not exist", () => { const missing = `${dir}-missing`; const result = checkNpmCacheOwnership({ + platform: "linux", getuid: () => 501, spawn: cacheLookup(missing), }); @@ -195,6 +196,7 @@ describe("npm cache ownership pre-flight", () => { test("fails closed before shutdown when npm cannot resolve its cache", () => { let inspected = false; const result = checkNpmCacheOwnership({ + platform: "linux", getuid: () => 501, spawn: (() => ({ status: null, diff --git a/tests/update-stop-first.test.ts b/tests/update-stop-first.test.ts index 545d9c33e..9c4c1caac 100644 --- a/tests/update-stop-first.test.ts +++ b/tests/update-stop-first.test.ts @@ -109,6 +109,8 @@ describe("update stops the running proxy before replacing files", () => { expect(updateJobSource).toContain('if (readPidForRestart("after service port reclaim").refused) return'); expect(updateJobSource).toContain('const directPid = readPidForRestart("before direct restart")'); expect(updateJobSource).toContain('if (readPidForRestart("after direct port reclaim").refused) return'); + expect(updateJobSource).toContain("hasTrustedRecoveryOwner(rootStat.uid)"); + expect(updateJobSource).toContain("uid === currentUid || uid === 0"); }); test("GUI recovery waits beyond the nested npm install deadline", () => { @@ -122,6 +124,8 @@ describe("update stops the running proxy before replacing files", () => { expect(launcherSource).toContain("timeoutMs: NPM_INSTALL_TIMEOUT_MS"); expect(launcherSource).toContain("await runProcessTreeCommand(npm"); expect(updateJobSource).toContain("installerFailureAllowsRecovery(result)"); + expect(updateJobSource).toContain("if (trayWasRunning && mayRecover)"); + expect(updateJobSource).toContain("candidates.slice(0, MAX_NPM_RECOVERY_CANDIDATES)"); }); test("GUI worker update children use pipe stdio so Windows npm.cmd does not open consoles", () => { From ea27540a8f9bd7ddada9b76f6bbe14471bca0c7e Mon Sep 17 00:00:00 2001 From: wzb Date: Mon, 27 Jul 2026 17:09:30 +0800 Subject: [PATCH 09/16] fix(update): close final recovery trust gaps --- bin/ocx.mjs | 4 + docs/adr/0001-gui-update-worker.md | 12 ++- src/update/index.ts | 3 + src/update/install-process.d.mts | 16 +++- src/update/install-process.mjs | 26 ++++++ src/update/job.ts | 78 +++++++++++++++--- src/update/npm-cache-preflight.d.mts | 1 + src/update/npm-cache-preflight.mjs | 5 +- tests/ocx-launcher-source.test.ts | 18 ++-- tests/update-install-process.test.ts | 13 ++- tests/update-job.test.ts | 100 ++++++++++++++++++++++- tests/update-npm-cache-preflight.test.ts | 25 +++++- tests/update-stop-first.test.ts | 9 +- 13 files changed, 278 insertions(+), 32 deletions(-) diff --git a/bin/ocx.mjs b/bin/ocx.mjs index b0a2c0160..f0e1d2daa 100755 --- a/bin/ocx.mjs +++ b/bin/ocx.mjs @@ -144,6 +144,9 @@ async function runNpmSelfUpdate() { console.error(`opencodex: ${formatNpmCacheOwnershipFailure(cacheOwnership)}`); process.exit(1); } + if (cacheOwnership.ok === "skipped") { + console.warn(`opencodex: npm cache ownership pre-flight skipped: ${cacheOwnership.reason}. Proceeding best-effort.`); + } // Remember whether a background service manages the proxy BEFORE stopping — `ocx stop` // unloads it permanently, so a successful update must reinstall it afterwards. @@ -242,6 +245,7 @@ async function runNpmSelfUpdate() { }); if (!res.treeExited) { console.error("\nopencodex: installer process-tree cleanup could not be confirmed; automatic recovery is unsafe until those processes exit."); + console.error(` The proxy is stopped. Once no '${npm}' installer processes remain, run 'ocx start' or re-run 'ocx update'.${trayBeforeUpdate.restoreOnFailure ? " The Windows tray also remains stopped; run 'ocx tray start' after the installer processes exit." : ""}`); process.exit(INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE); } if (res.interruptedSignal) { diff --git a/docs/adr/0001-gui-update-worker.md b/docs/adr/0001-gui-update-worker.md index 77c86ef00..f470a5aca 100644 --- a/docs/adr/0001-gui-update-worker.md +++ b/docs/adr/0001-gui-update-worker.md @@ -37,9 +37,12 @@ live PID still preserves the pre-update activity evidence. If the installer exit probes again immediately before recovery and leaves any old or concurrently replaced process alone when health or PID identity proves it is OpenCodex. When npm already stopped the proxy and retired the old package, the worker validates each candidate's trusted ownership, name, version, path -containment, and complete launcher runtime with a side-effect-free `--version` probe. Candidate -inspection and restart are bounded to at most two candidates, preferring the current package and then -the newest retired copies, and it tries those runnable candidates in order until one restores health. +containment, and complete launcher runtime with a side-effect-free `--version` probe. On UID-capable +platforms, the scope and complete package tree must also reject symlinks, foreign owners, and +group/world-writable entries before any candidate code executes. The tree walk is bounded by entry +count and elapsed time. Candidate inspection and restart are bounded to at most two candidates, +preferring the current package and then the newest retired copies, and it tries those runnable +candidates in order until one restores health. Service and direct restart paths also stop when the pidfile has changed to a different identity-checked proxy, and every such identity decision re-reads the process command line instead of trusting a PID-only cache across the update boundary. The update job remains failed either way because restoring @@ -48,7 +51,8 @@ availability is not the same as installing the new version. The npm installer runs in an isolated process tree. On timeout, the launcher terminates and awaits the whole POSIX process group or a Windows `taskkill /T /F` tree before returning failure. POSIX cleanup treats zombie-only groups as stopped and refuses to signal a process-group ID that has been -reused by a new leader. Windows has no retained job-object handle after a normally failed installer +reused by a new leader, including a second identity check immediately before force-killing after the +grace period. Windows has no retained job-object handle after a normally failed installer root exits, so that case remains explicitly unconfirmed and automatic recovery (including tray restoration) stays disabled. The outer worker timeout remains longer than the nested deadline, and recovery is skipped whenever either layer cannot prove installer-tree shutdown, so recovery never diff --git a/src/update/index.ts b/src/update/index.ts index 52d0b18a3..7bd6fa8d8 100644 --- a/src/update/index.ts +++ b/src/update/index.ts @@ -171,6 +171,9 @@ export async function runUpdate(): Promise { console.error(`⚠️ ${formatNpmCacheOwnershipFailure(cacheOwnership)}`); process.exit(1); } + if (cacheOwnership.ok === "skipped") { + console.warn(`⚠️ npm cache ownership pre-flight skipped: ${cacheOwnership.reason}. Proceeding best-effort.`); + } } // Remember whether a background service manages the proxy BEFORE stopping — `ocx stop` diff --git a/src/update/install-process.d.mts b/src/update/install-process.d.mts index ce917242d..2056d3f5d 100644 --- a/src/update/install-process.d.mts +++ b/src/update/install-process.d.mts @@ -21,9 +21,23 @@ export interface ProcessTreeCommandResult { treeExited: boolean; } +export interface ProcessGroupInspection { + hasRunningMember: boolean; + hasRunningLeader: boolean; +} + +export type ProcessGroupForceDecision = "exited" | "signal" | "refuse"; + +export function processGroupForceDecision( + inspection: ProcessGroupInspection | null, + originalLeaderConfirmed: boolean, +): ProcessGroupForceDecision; + export function terminateInstallerProcessTree( pid: number | undefined, - options?: Pick, + options?: Pick & { + isOriginalLeader?: () => boolean; + }, ): Promise; export function runProcessTreeCommand( diff --git a/src/update/install-process.mjs b/src/update/install-process.mjs index 3b6f33966..aa9065836 100644 --- a/src/update/install-process.mjs +++ b/src/update/install-process.mjs @@ -67,6 +67,13 @@ function inspectProcessGroup(groupId) { return null; } +export function processGroupForceDecision(inspection, originalLeaderConfirmed) { + if (!inspection) return "refuse"; + if (!inspection.hasRunningMember) return "exited"; + if (inspection.hasRunningLeader && !originalLeaderConfirmed) return "refuse"; + return "signal"; +} + function processGroupHasRunningMember(groupId) { return inspectProcessGroup(groupId)?.hasRunningMember ?? null; } @@ -115,6 +122,7 @@ export async function terminateInstallerProcessTree( { terminationGraceMs = DEFAULT_TERMINATION_GRACE_MS, forceWaitMs = DEFAULT_FORCE_WAIT_MS, + isOriginalLeader, } = {}, ) { if (!Number.isSafeInteger(pid) || pid <= 0) return true; @@ -145,6 +153,23 @@ export async function terminateInstallerProcessTree( } if (await waitForProcessTreeExit(pid, terminationGraceMs)) return true; + const forceInspection = inspectProcessGroup(pid); + const originalLeaderConfirmed = forceInspection?.hasRunningLeader === true + && isOriginalLeader?.() === true; + const forceDecision = processGroupForceDecision(forceInspection, originalLeaderConfirmed); + if (forceDecision === "exited") return true; + if (forceDecision === "refuse") return false; + + // Reinspect at the force-signal boundary. The original group can exit and its + // ID can be reused after the grace-period inspection above; a replacement + // leader must never receive the pending SIGKILL. + const signalInspection = inspectProcessGroup(pid); + const signalLeaderConfirmed = signalInspection?.hasRunningLeader === true + && isOriginalLeader?.() === true; + const signalDecision = processGroupForceDecision(signalInspection, signalLeaderConfirmed); + if (signalDecision === "exited") return true; + if (signalDecision === "refuse") return false; + try { process.kill(-pid, "SIGKILL"); } catch (error) { @@ -234,6 +259,7 @@ export async function runProcessTreeCommand( cleanupPromise = terminateInstallerProcessTree(child.pid, { terminationGraceMs, forceWaitMs, + isOriginalLeader: () => child.exitCode === null && child.signalCode === null, }); void cleanupPromise.then(treeExited => { if (treeExited) return; diff --git a/src/update/job.ts b/src/update/job.ts index 18dce79c3..d61eb6383 100644 --- a/src/update/job.ts +++ b/src/update/job.ts @@ -50,6 +50,8 @@ const RESTART_STABILITY_WINDOW_MS = 15_000; const UPDATE_LIVENESS_PROBE_ATTEMPTS = 3; const UPDATE_LIVENESS_PROBE_DELAY_MS = 250; const MAX_NPM_RECOVERY_CANDIDATES = 2; +const MAX_NPM_RECOVERY_TREE_ENTRIES = 50_000; +const MAX_NPM_RECOVERY_TREE_SCAN_MS = 5_000; /** How long update restart waits for the captured port to become bindable after stop. */ export const RESTART_PORT_RECLAIM_MS = 30_000; @@ -136,6 +138,61 @@ function hasTrustedRecoveryOwner(uid: number): boolean { return currentUid === undefined || uid === currentUid || uid === 0; } +function hasTrustedRecoveryPermissions(stat: { uid: number; mode: number }): boolean { + if (!hasTrustedRecoveryOwner(stat.uid)) return false; + // POSIX group/other write bits make even a trusted-owner entry mutable by a + // different local account. Windows ACLs are not represented by these bits; + // npm failed-root recovery is already disabled there when tree exit is unknown. + return process.getuid === undefined || (stat.mode & 0o022) === 0; +} + +function hasTrustedRecoveryPath(packageRoot: string): boolean { + let path = packageRoot; + while (true) { + const stat = lstatSync(path); + if ( + !stat.isDirectory() + || stat.isSymbolicLink() + || !hasTrustedRecoveryOwner(stat.uid) + || ( + process.getuid !== undefined + && (stat.mode & 0o022) !== 0 + && (stat.mode & 0o1000) === 0 + ) + ) return false; + const parent = dirname(path); + if (parent === path) return true; + path = parent; + } +} + +function hasTrustedRecoveryTree(packageRoot: string): boolean { + const canonicalRoot = realpathSync(packageRoot); + const scopeRoot = dirname(canonicalRoot); + const scopeStat = lstatSync(scopeRoot); + if ( + !scopeStat.isDirectory() + || scopeStat.isSymbolicLink() + || !hasTrustedRecoveryPermissions(scopeStat) + || !hasTrustedRecoveryPath(canonicalRoot) + ) return false; + + const deadline = Date.now() + MAX_NPM_RECOVERY_TREE_SCAN_MS; + const pending = [canonicalRoot]; + let inspected = 0; + while (pending.length > 0) { + if (Date.now() > deadline || inspected >= MAX_NPM_RECOVERY_TREE_ENTRIES) return false; + const path = pending.pop()!; + inspected += 1; + const stat = lstatSync(path); + if (stat.isSymbolicLink() || !hasTrustedRecoveryPermissions(stat)) return false; + if (stat.isFile()) continue; + if (!stat.isDirectory()) return false; + for (const name of readdirSync(path, { encoding: "utf8" })) pending.push(join(path, name)); + } + return true; +} + /** Validate the on-disk identity before any candidate code is executed. */ function inspectNpmRecoveryPackage(packageRoot: string, expectedVersion: string): ValidatedNpmLauncher | null { try { @@ -143,27 +200,28 @@ function inspectNpmRecoveryPackage(packageRoot: string, expectedVersion: string) if ( !rootStat.isDirectory() || rootStat.isSymbolicLink() - || !hasTrustedRecoveryOwner(rootStat.uid) + || !hasTrustedRecoveryPermissions(rootStat) ) return null; + if (!hasTrustedRecoveryTree(packageRoot)) return null; - const manifestPath = join(packageRoot, "package.json"); + const canonicalRoot = realpathSync(packageRoot); + const manifestPath = join(canonicalRoot, "package.json"); const manifestStat = lstatSync(manifestPath); if ( !manifestStat.isFile() || manifestStat.isSymbolicLink() - || !hasTrustedRecoveryOwner(manifestStat.uid) + || !hasTrustedRecoveryPermissions(manifestStat) ) return null; const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as { name?: unknown; version?: unknown }; if (manifest.name !== PKG || manifest.version !== expectedVersion) return null; - const launcherPath = join(packageRoot, "bin", "ocx.mjs"); + const launcherPath = join(canonicalRoot, "bin", "ocx.mjs"); const launcherStat = lstatSync(launcherPath); if ( !launcherStat.isFile() || launcherStat.isSymbolicLink() - || !hasTrustedRecoveryOwner(launcherStat.uid) + || !hasTrustedRecoveryPermissions(launcherStat) ) return null; - const canonicalRoot = realpathSync(packageRoot); const canonicalLauncher = realpathSync(launcherPath); if (!isPathInside(canonicalRoot, canonicalLauncher)) return null; return { launcher: canonicalLauncher, mtimeMs: rootStat.mtimeMs }; @@ -423,10 +481,10 @@ function runLoggedCommand(job: UpdateJobState, bin: string, args: string[], time }; } -export function installerFailureAllowsRecovery(result: LoggedCommandResult): boolean { +export function installerFailureAllowsRecovery(installer: Installer, result: LoggedCommandResult): boolean { return !result.timedOut && result.signal === null - && result.status !== INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE; + && (installer !== "npm" || result.status !== INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE); } function spawnDetachedStart( @@ -1072,7 +1130,7 @@ export async function runGuiUpdateWorker(jobId: string, channel: Channel, restar */ const result = runLoggedCommand(job, cmd.bin, cmd.args, UPDATE_TIMEOUT_MS); if (result.status !== 0) { - const mayRecover = installerFailureAllowsRecovery(result); + const mayRecover = installerFailureAllowsRecovery(check.installer, result); if (trayWasRunning && mayRecover) { try { const { startWindowsTray } = await import("../tray/windows"); @@ -1081,7 +1139,7 @@ export async function runGuiUpdateWorker(jobId: string, channel: Channel, restar } let recovery: FailedUpdateRecovery | null = null; if (!mayRecover) { - updateJob(job, {}, "Automatic proxy recovery was skipped because installer process-tree shutdown was not confirmed. Wait for installer processes to exit before restoring the package or proxy."); + updateJob(job, {}, `Automatic proxy recovery was skipped because installer process-tree shutdown was not confirmed. Wait for installer processes to exit before restoring the package or proxy.${trayWasRunning ? " The Windows tray also remains stopped; run 'ocx tray start' after the installer processes exit." : ""}`); } else { recovery = await recoverFailedGuiUpdate(job, captured, proxyWasActive); } diff --git a/src/update/npm-cache-preflight.d.mts b/src/update/npm-cache-preflight.d.mts index b4d5fa17e..2a6eed893 100644 --- a/src/update/npm-cache-preflight.d.mts +++ b/src/update/npm-cache-preflight.d.mts @@ -8,6 +8,7 @@ export interface NpmCachePreflightOptions { getuid?: () => number; npmBin?: string; shell?: boolean; + lookupTimeoutMs?: number; spawn?: typeof import("node:child_process").spawnSync; scanSpawn?: typeof import("node:child_process").spawnSync; scanBin?: string; diff --git a/src/update/npm-cache-preflight.mjs b/src/update/npm-cache-preflight.mjs index e32641d7f..7cf51ebea 100644 --- a/src/update/npm-cache-preflight.mjs +++ b/src/update/npm-cache-preflight.mjs @@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url"; const DEFAULT_MAX_CACHE_ENTRIES = 50_000; const DEFAULT_CACHE_SCAN_TIMEOUT_MS = 10_000; +const DEFAULT_CACHE_LOOKUP_TIMEOUT_MS = 12_000; const CACHE_SCAN_WORKER_ARG = "__scan-npm-cache-ownership"; function positiveInteger(value, fallback) { @@ -187,11 +188,13 @@ export function checkNpmCacheOwnership(options = {}) { const expectedUid = getuid(); const npmBin = options.npmBin ?? "npm"; const spawn = options.spawn ?? spawnSync; + const lookupTimeoutMs = positiveInteger(options.lookupTimeoutMs, DEFAULT_CACHE_LOOKUP_TIMEOUT_MS); let result; try { result = spawn(npmBin, ["config", "get", "cache"], { encoding: "utf8", - timeout: 12_000, + timeout: lookupTimeoutMs, + killSignal: "SIGKILL", windowsHide: true, shell: options.shell ?? false, }); diff --git a/tests/ocx-launcher-source.test.ts b/tests/ocx-launcher-source.test.ts index c74b1a647..9954d9069 100644 --- a/tests/ocx-launcher-source.test.ts +++ b/tests/ocx-launcher-source.test.ts @@ -20,15 +20,17 @@ describe("ocx.mjs npm launcher (source invariants)", () => { }); test("unsafe installer cleanup never restarts the tray, while confirmed interruption does", () => { - const cleanupFailure = source.slice( - source.indexOf("if (!res.treeExited)"), - source.indexOf("if (res.interruptedSignal)"), - ); - const interruption = source.slice( - source.indexOf("if (res.interruptedSignal)"), - source.indexOf("if (res.status === 0)"), - ); + const cleanupAt = source.indexOf("if (!res.treeExited)"); + const interruptAt = source.indexOf("if (res.interruptedSignal)"); + const successAt = source.indexOf("if (res.status === 0)"); + expect(cleanupAt).toBeGreaterThan(-1); + expect(interruptAt).toBeGreaterThan(cleanupAt); + expect(successAt).toBeGreaterThan(interruptAt); + const cleanupFailure = source.slice(cleanupAt, interruptAt); + const interruption = source.slice(interruptAt, successAt); expect(cleanupFailure).not.toContain('runTrayLifecycle(launcher, "start")'); + expect(cleanupFailure).toContain("The proxy is stopped"); + expect(cleanupFailure).toContain("ocx tray start"); expect(interruption).toContain('runTrayLifecycle(launcher, "start")'); expect(interruption).toContain("process.exit(exitCode)"); expect(source).toContain("res.error.message"); diff --git a/tests/update-install-process.test.ts b/tests/update-install-process.test.ts index 88851fffd..9a0d8108d 100644 --- a/tests/update-install-process.test.ts +++ b/tests/update-install-process.test.ts @@ -3,7 +3,10 @@ import { spawnSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { runProcessTreeCommand } from "../src/update/install-process.mjs"; +import { + processGroupForceDecision, + runProcessTreeCommand, +} from "../src/update/install-process.mjs"; const cleanupPids = new Set(); const cleanupDirs = new Set(); @@ -34,6 +37,14 @@ afterEach(() => { }); describe("update installer process isolation", () => { + test("force cleanup refuses a reused or uninspectable process group", () => { + expect(processGroupForceDecision(null, false)).toBe("refuse"); + expect(processGroupForceDecision({ hasRunningMember: false, hasRunningLeader: false }, false)).toBe("exited"); + expect(processGroupForceDecision({ hasRunningMember: true, hasRunningLeader: false }, false)).toBe("signal"); + expect(processGroupForceDecision({ hasRunningMember: true, hasRunningLeader: true }, false)).toBe("refuse"); + expect(processGroupForceDecision({ hasRunningMember: true, hasRunningLeader: true }, true)).toBe("signal"); + }); + test("spawn failures report their cause without inventing a live process tree", async () => { const result = await runProcessTreeCommand(join(tmpdir(), "ocx-command-that-does-not-exist"), [], { stdio: "ignore", diff --git a/tests/update-job.test.ts b/tests/update-job.test.ts index cc6c504ea..9debacf1d 100644 --- a/tests/update-job.test.ts +++ b/tests/update-job.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { mkdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -139,6 +139,58 @@ describe("GUI update execution decisions", () => { expect(await findNpmRecoveryLauncher(join(currentRoot, "bin", "ocx.mjs"), "2.7.40")).toBeNull(); }); + test("rejects a recovery package with an untrusted-writable imported file", async () => { + if (process.getuid?.() === undefined) return; + const scopeRoot = join(dir, "global", "@bitkyc08"); + const currentRoot = join(scopeRoot, "opencodex"); + mkdirSync(join(currentRoot, "bin"), { recursive: true }); + mkdirSync(join(currentRoot, "src"), { recursive: true }); + writeFileSync(join(currentRoot, "package.json"), JSON.stringify({ + name: "@bitkyc08/opencodex", + version: "2.7.40", + })); + writeFileSync(join(currentRoot, "bin", "ocx.mjs"), 'import "../src/runtime.mjs";\n'); + const imported = join(currentRoot, "src", "runtime.mjs"); + writeFileSync(imported, "process.exit(0);\n"); + chmodSync(imported, 0o666); + let probed = false; + + expect(await findNpmRecoveryLaunchers( + join(currentRoot, "bin", "ocx.mjs"), + "2.7.40", + async () => { + probed = true; + return true; + }, + )).toEqual([]); + expect(probed).toBe(false); + }); + + test("rejects a recovery package below an untrusted-writable path component", async () => { + if (process.getuid?.() === undefined) return; + const unsafeParent = join(dir, "world-writable-global"); + const scopeRoot = join(unsafeParent, "@bitkyc08"); + const currentRoot = join(scopeRoot, "opencodex"); + mkdirSync(join(currentRoot, "bin"), { recursive: true }); + writeFileSync(join(currentRoot, "package.json"), JSON.stringify({ + name: "@bitkyc08/opencodex", + version: "2.7.40", + })); + writeFileSync(join(currentRoot, "bin", "ocx.mjs"), "process.exit(0);\n"); + chmodSync(unsafeParent, 0o777); + let probed = false; + + expect(await findNpmRecoveryLaunchers( + join(currentRoot, "bin", "ocx.mjs"), + "2.7.40", + async () => { + probed = true; + return true; + }, + )).toEqual([]); + expect(probed).toBe(false); + }); + test("skips a matching current package whose complete launcher runtime cannot load", async () => { const scopeRoot = join(dir, "global", "@bitkyc08"); const currentRoot = join(scopeRoot, "opencodex"); @@ -190,9 +242,11 @@ describe("GUI update execution decisions", () => { }); test("only recovers after a clean installer exit", () => { - expect(installerFailureAllowsRecovery({ status: 1, signal: null, timedOut: false })).toBe(true); - expect(installerFailureAllowsRecovery({ status: 75, signal: null, timedOut: false })).toBe(false); - expect(installerFailureAllowsRecovery({ status: null, signal: "SIGTERM", timedOut: true })).toBe(false); + expect(installerFailureAllowsRecovery("npm", { status: 1, signal: null, timedOut: false })).toBe(true); + expect(installerFailureAllowsRecovery("npm", { status: 75, signal: null, timedOut: false })).toBe(false); + expect(installerFailureAllowsRecovery("bun", { status: 75, signal: null, timedOut: false })).toBe(true); + expect(installerFailureAllowsRecovery("npm", { status: null, signal: "SIGTERM", timedOut: false })).toBe(false); + expect(installerFailureAllowsRecovery("npm", { status: 1, signal: null, timedOut: true })).toBe(false); }); test("proxy restart pins --port so post-update start does not hop to an ephemeral port", () => { @@ -781,6 +835,44 @@ describe("GUI update execution decisions", () => { expect(readUpdateJob(job.id)?.log.some(line => line.includes("trying candidate 2 of 2"))).toBe(true); }); + test("failed install reports remediation when no runnable recovery package remains", async () => { + let restartCalls = 0; + const job: UpdateJobState = { + id: "failed-install-no-candidate", + status: "running", + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + currentVersion: "2.7.40", + latestVersion: "2.7.41", + channel: "latest", + installer: "npm", + restart: true, + command: "", + releaseNotesUrl: "", + log: [], + }; + writeFileSync(updateJobPath(), JSON.stringify(job)); + + const recovery = await recoverFailedGuiUpdateForTests( + job, + { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, + true, + { + probeProxyIdentity: async () => null, + verifyPidIdentityFn: () => null, + sleepMs: async () => {}, + recoveryLaunchersFn: () => [], + restartAfterUpdateFn: async () => { restartCalls += 1; }, + }, + ); + + expect(recovery).toBe("failed"); + expect(restartCalls).toBe(0); + const log = readUpdateJob(job.id)?.log ?? []; + expect(log.some(line => line.includes("Could not find a runnable current or npm-retired launcher"))).toBe(true); + expect(log.some(line => line.includes("ocx start --port 10100"))).toBe(true); + }); + test("restart confirmation fails when the proxy never becomes healthy", async () => { let now = 0; const job: UpdateJobState = { diff --git a/tests/update-npm-cache-preflight.test.ts b/tests/update-npm-cache-preflight.test.ts index b15d6374d..dc79c78e2 100644 --- a/tests/update-npm-cache-preflight.test.ts +++ b/tests/update-npm-cache-preflight.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { lstatSync, mkdirSync, readdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { chmodSync, lstatSync, mkdirSync, readdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -193,6 +193,29 @@ describe("npm cache ownership pre-flight", () => { }); }); + test("force-kills an npm cache lookup that ignores SIGTERM", () => { + const uid = process.getuid?.(); + if (uid === undefined) return; + const blockingNpm = join(dir, "blocking-npm.mjs"); + writeFileSync( + blockingNpm, + "#!/usr/bin/env node\nprocess.on('SIGTERM', () => {});\nsetInterval(() => {}, 60_000);\n", + ); + chmodSync(blockingNpm, 0o755); + + const startedAt = Date.now(); + const result = checkNpmCacheOwnership({ + getuid: () => uid, + lookupTimeoutMs: 250, + npmBin: blockingNpm, + }); + expect(Date.now() - startedAt).toBeLessThan(2_000); + expect(result).toMatchObject({ + ok: false, + reason: "could not resolve the npm cache (ETIMEDOUT)", + }); + }, 5_000); + test("fails closed before shutdown when npm cannot resolve its cache", () => { let inspected = false; const result = checkNpmCacheOwnership({ diff --git a/tests/update-stop-first.test.ts b/tests/update-stop-first.test.ts index 9c4c1caac..2443f3cc9 100644 --- a/tests/update-stop-first.test.ts +++ b/tests/update-stop-first.test.ts @@ -39,6 +39,8 @@ describe("update stops the running proxy before replacing files", () => { expect(launcherGateAt).toBeLessThan(launcherStopAt); expect(updateSource).toContain("formatNpmCacheOwnershipFailure(cacheOwnership)"); expect(launcherSource).toContain("formatNpmCacheOwnershipFailure(cacheOwnership)"); + expect(updateSource).toContain("npm cache ownership pre-flight skipped"); + expect(launcherSource).toContain("npm cache ownership pre-flight skipped"); }); test("npm launcher update path stops via its own launcher path before npm install", () => { @@ -109,8 +111,10 @@ describe("update stops the running proxy before replacing files", () => { expect(updateJobSource).toContain('if (readPidForRestart("after service port reclaim").refused) return'); expect(updateJobSource).toContain('const directPid = readPidForRestart("before direct restart")'); expect(updateJobSource).toContain('if (readPidForRestart("after direct port reclaim").refused) return'); - expect(updateJobSource).toContain("hasTrustedRecoveryOwner(rootStat.uid)"); + expect(updateJobSource).toContain("hasTrustedRecoveryPermissions(rootStat)"); expect(updateJobSource).toContain("uid === currentUid || uid === 0"); + expect(updateJobSource).toContain("hasTrustedRecoveryTree(packageRoot)"); + expect(updateJobSource).toContain("(stat.mode & 0o022) === 0"); }); test("GUI recovery waits beyond the nested npm install deadline", () => { @@ -123,8 +127,9 @@ describe("update stops the running proxy before replacing files", () => { expect(outerMs).toBeGreaterThanOrEqual(innerMs + 60_000); expect(launcherSource).toContain("timeoutMs: NPM_INSTALL_TIMEOUT_MS"); expect(launcherSource).toContain("await runProcessTreeCommand(npm"); - expect(updateJobSource).toContain("installerFailureAllowsRecovery(result)"); + expect(updateJobSource).toContain("installerFailureAllowsRecovery(check.installer, result)"); expect(updateJobSource).toContain("if (trayWasRunning && mayRecover)"); + expect(updateJobSource).toContain("The Windows tray also remains stopped"); expect(updateJobSource).toContain("candidates.slice(0, MAX_NPM_RECOVERY_CANDIDATES)"); }); From 5564b0312d2c7942bbc449f5fe5d1b18ebf7edf1 Mon Sep 17 00:00:00 2001 From: wzb Date: Mon, 27 Jul 2026 17:31:31 +0800 Subject: [PATCH 10/16] fix(update): harden CLI installer cleanup --- docs/adr/0001-gui-update-worker.md | 5 ++- src/update/index.ts | 32 +++++++++++++-- src/update/install-process.d.mts | 4 ++ src/update/install-process.mjs | 58 ++++++++++++++++++++-------- tests/update-install-process.test.ts | 39 +++++++++++++++---- tests/update-stop-first.test.ts | 11 ++++++ 6 files changed, 118 insertions(+), 31 deletions(-) diff --git a/docs/adr/0001-gui-update-worker.md b/docs/adr/0001-gui-update-worker.md index f470a5aca..622466529 100644 --- a/docs/adr/0001-gui-update-worker.md +++ b/docs/adr/0001-gui-update-worker.md @@ -48,8 +48,9 @@ proxy, and every such identity decision re-reads the process command line instea PID-only cache across the update boundary. The update job remains failed either way because restoring availability is not the same as installing the new version. -The npm installer runs in an isolated process tree. On timeout, the launcher terminates and awaits -the whole POSIX process group or a Windows `taskkill /T /F` tree before returning failure. POSIX +The npm launcher and the Bun/source CLI installer paths run in an isolated process tree. On timeout, +the updater terminates and awaits the whole POSIX process group or a Windows `taskkill /T /F` tree +before returning failure; piped CLI output is drained and bounded before it is replayed. POSIX cleanup treats zombie-only groups as stopped and refuses to signal a process-group ID that has been reused by a new leader, including a second identity check immediately before force-killing after the grace period. Windows has no retained job-object handle after a normally failed installer diff --git a/src/update/index.ts b/src/update/index.ts index 7bd6fa8d8..9ac85db08 100644 --- a/src/update/index.ts +++ b/src/update/index.ts @@ -4,6 +4,10 @@ import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { getConfigDir, loadConfig, readPid, readRuntimePort } from "../config"; import { checkNpmCacheOwnership, formatNpmCacheOwnershipFailure } from "./npm-cache-preflight.mjs"; +import { + INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE, + runProcessTreeCommand, +} from "./install-process.mjs"; import { handoffWindowsTrayForUpdate, planWindowsTrayUpdate } from "./tray-update-plan.mjs"; /** @@ -257,14 +261,29 @@ export async function runUpdate(): Promise { const target = npmSpawnTarget(bin); const installStdio = updateChildStdio(); - const r = spawnSync(target.bin, cmdArgs, { + const r = await runProcessTreeCommand(target.bin, cmdArgs, { stdio: installStdio, - encoding: installStdio === "pipe" ? "utf8" : undefined, - timeout: 180000, + timeoutMs: 180000, windowsHide: true, shell: target.shell, }); if (installStdio === "pipe") logSpawnOutput("", r); + if (!r.treeExited) { + console.error("\nopencodex: installer process-tree cleanup could not be confirmed; automatic recovery is unsafe until those processes exit."); + console.error(` The proxy is stopped. Once no '${target.bin}' installer processes remain, run 'ocx start' or re-run 'ocx update'.${trayWasRunning ? " The Windows tray also remains stopped; run 'ocx tray start' after the installer processes exit." : ""}`); + process.exit(INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE); + } + if (r.interruptedSignal) { + if (trayWasRunning) { + try { + const { startWindowsTray } = await import("../tray/windows"); + startWindowsTray(); + } catch { /* preserve the primary interruption */ } + } + const signalExitCode = r.interruptedSignal === "SIGINT" ? 130 : r.interruptedSignal === "SIGHUP" ? 129 : 143; + console.error(`\nopencodex: update interrupted (${r.interruptedSignal}); the proxy is still stopped. Run 'ocx start' or re-run 'ocx update'.`); + process.exit(signalExitCode); + } if (r.status === 0) { console.log(`\n✅ Updated${latest ? ` to v${latest}` : ""}.`); // Re-bake the bundled Bun path into the Codex autostart shim on every @@ -352,7 +371,12 @@ export async function runUpdate(): Promise { startWindowsTray(); } catch { /* keep the primary update failure */ } } - console.error(`\n⚠️ Update failed (${bin} exit ${r.status ?? "?"}). Try manually: ${bin} ${cmdArgs.join(" ")}`); + const failure = r.timedOut + ? "timed out after 180s" + : r.error + ? `could not run: ${r.error.message}` + : `exit ${r.status ?? "?"}`; + console.error(`\n⚠️ Update failed (${bin} ${failure}). Try manually: ${bin} ${cmdArgs.join(" ")}`); process.exit(1); } } diff --git a/src/update/install-process.d.mts b/src/update/install-process.d.mts index 2056d3f5d..41d343456 100644 --- a/src/update/install-process.d.mts +++ b/src/update/install-process.d.mts @@ -10,6 +10,7 @@ export interface ProcessTreeCommandOptions { env?: NodeJS.ProcessEnv; terminationGraceMs?: number; forceWaitMs?: number; + inspectProcessGroup?: (groupId: number) => ProcessGroupInspection | null; } export interface ProcessTreeCommandResult { @@ -19,6 +20,8 @@ export interface ProcessTreeCommandResult { interruptedSignal: NodeJS.Signals | null; timedOut: boolean; treeExited: boolean; + stdout?: string; + stderr?: string; } export interface ProcessGroupInspection { @@ -37,6 +40,7 @@ export function terminateInstallerProcessTree( pid: number | undefined, options?: Pick & { isOriginalLeader?: () => boolean; + inspectProcessGroup?: (groupId: number) => ProcessGroupInspection | null; }, ): Promise; diff --git a/src/update/install-process.mjs b/src/update/install-process.mjs index aa9065836..5eab28b5f 100644 --- a/src/update/install-process.mjs +++ b/src/update/install-process.mjs @@ -4,6 +4,15 @@ import { readdirSync, readFileSync } from "node:fs"; const TREE_POLL_INTERVAL_MS = 100; const DEFAULT_TERMINATION_GRACE_MS = 5_000; const DEFAULT_FORCE_WAIT_MS = 5_000; +const MAX_CAPTURED_OUTPUT_CHARS = 4_000; + +function appendBoundedOutput(current, chunk) { + const text = typeof chunk === "string" ? chunk : Buffer.from(chunk).toString("utf8"); + const next = current + text; + return next.length > MAX_CAPTURED_OUTPUT_CHARS + ? next.slice(-MAX_CAPTURED_OUTPUT_CHARS) + : next; +} /** Exit code used when the updater cannot prove that its installer tree is gone. */ export const INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE = 75; @@ -74,12 +83,12 @@ export function processGroupForceDecision(inspection, originalLeaderConfirmed) { return "signal"; } -function processGroupHasRunningMember(groupId) { - return inspectProcessGroup(groupId)?.hasRunningMember ?? null; +function processGroupHasRunningMember(groupId, inspect = inspectProcessGroup) { + return inspect(groupId)?.hasRunningMember ?? null; } -function inspectProcessGroupAfterLeaderExit(groupId) { - const inspection = inspectProcessGroup(groupId); +function inspectProcessGroupAfterLeaderExit(groupId, inspect = inspectProcessGroup) { + const inspection = inspect(groupId); if (!inspection) return "unknown"; // A new group cannot reuse this ID without a live leader whose PID equals the // group ID. Never signal that group; it is unrelated to the completed child. @@ -87,13 +96,13 @@ function inspectProcessGroupAfterLeaderExit(groupId) { return inspection.hasRunningMember ? "running" : "exited"; } -function isProcessTreeAlive(pid) { +function isProcessTreeAlive(pid, inspect = inspectProcessGroup) { try { process.kill(process.platform === "win32" ? pid : -pid, 0); if (process.platform === "win32") return true; // Zombies keep a process group addressable by signal 0 but cannot mutate the // package tree. Treat a zombie-only group as fully stopped. - return processGroupHasRunningMember(pid) ?? true; + return processGroupHasRunningMember(pid, inspect) ?? true; } catch (error) { return !isNoSuchProcess(error); } @@ -103,13 +112,13 @@ function sleep(ms) { return new Promise(resolve => setTimeout(resolve, ms)); } -async function waitForProcessTreeExit(pid, timeoutMs) { +async function waitForProcessTreeExit(pid, timeoutMs, inspect = inspectProcessGroup) { const deadline = Date.now() + timeoutMs; while (Date.now() < deadline) { - if (!isProcessTreeAlive(pid)) return true; + if (!isProcessTreeAlive(pid, inspect)) return true; await sleep(TREE_POLL_INTERVAL_MS); } - return !isProcessTreeAlive(pid); + return !isProcessTreeAlive(pid, inspect); } /** @@ -123,15 +132,17 @@ export async function terminateInstallerProcessTree( terminationGraceMs = DEFAULT_TERMINATION_GRACE_MS, forceWaitMs = DEFAULT_FORCE_WAIT_MS, isOriginalLeader, + inspectProcessGroup: inspectOverride, } = {}, ) { if (!Number.isSafeInteger(pid) || pid <= 0) return true; + const inspect = inspectOverride ?? inspectProcessGroup; if (process.platform === "win32") { // Once the root has exited, Node has no job-object handle with which to prove // that background descendants also exited. Fail closed instead of treating a // missing root PID as proof that the complete installer tree is gone. - if (!isProcessTreeAlive(pid)) return false; + if (!isProcessTreeAlive(pid, inspect)) return false; const taskkill = `${process.env.SystemRoot ?? "C:\\Windows"}\\System32\\taskkill.exe`; const result = spawnSync(taskkill, ["/PID", String(pid), "/T", "/F"], { stdio: "ignore", @@ -141,19 +152,19 @@ export async function terminateInstallerProcessTree( // A non-zero taskkill result cannot distinguish an already-gone root from a // partially-killed tree. Recovery must remain disabled in either case. if (result.status !== 0) return false; - return waitForProcessTreeExit(pid, forceWaitMs); + return waitForProcessTreeExit(pid, forceWaitMs, inspect); } - if (!isProcessTreeAlive(pid)) return true; + if (!isProcessTreeAlive(pid, inspect)) return true; try { process.kill(-pid, "SIGTERM"); } catch (error) { if (isNoSuchProcess(error)) return true; return false; } - if (await waitForProcessTreeExit(pid, terminationGraceMs)) return true; + if (await waitForProcessTreeExit(pid, terminationGraceMs, inspect)) return true; - const forceInspection = inspectProcessGroup(pid); + const forceInspection = inspect(pid); const originalLeaderConfirmed = forceInspection?.hasRunningLeader === true && isOriginalLeader?.() === true; const forceDecision = processGroupForceDecision(forceInspection, originalLeaderConfirmed); @@ -163,7 +174,7 @@ export async function terminateInstallerProcessTree( // Reinspect at the force-signal boundary. The original group can exit and its // ID can be reused after the grace-period inspection above; a replacement // leader must never receive the pending SIGKILL. - const signalInspection = inspectProcessGroup(pid); + const signalInspection = inspect(pid); const signalLeaderConfirmed = signalInspection?.hasRunningLeader === true && isOriginalLeader?.() === true; const signalDecision = processGroupForceDecision(signalInspection, signalLeaderConfirmed); @@ -176,7 +187,7 @@ export async function terminateInstallerProcessTree( if (isNoSuchProcess(error)) return true; return false; } - return waitForProcessTreeExit(pid, forceWaitMs); + return waitForProcessTreeExit(pid, forceWaitMs, inspect); } /** @@ -194,8 +205,10 @@ export async function runProcessTreeCommand( env = process.env, terminationGraceMs = DEFAULT_TERMINATION_GRACE_MS, forceWaitMs = DEFAULT_FORCE_WAIT_MS, + inspectProcessGroup: inspectOverride, } = {}, ) { + const inspect = inspectOverride ?? inspectProcessGroup; let child; try { child = spawn(bin, args, { @@ -216,6 +229,13 @@ export async function runProcessTreeCommand( }; } + let stdout = ""; + let stderr = ""; + child.stdout?.setEncoding?.("utf8"); + child.stderr?.setEncoding?.("utf8"); + child.stdout?.on?.("data", chunk => { stdout = appendBoundedOutput(stdout, chunk); }); + child.stderr?.on?.("data", chunk => { stderr = appendBoundedOutput(stderr, chunk); }); + let posixTreeAfterRootExit = "unknown"; let windowsFailedExitTreeUnknown = false; let spawnFailed = false; @@ -238,7 +258,7 @@ export async function runProcessTreeCommand( } else if (child.pid) { // Inspect group membership directly. A live process whose PID equals the // old group ID proves reuse, in which case cleanup must not signal it. - posixTreeAfterRootExit = inspectProcessGroupAfterLeaderExit(child.pid); + posixTreeAfterRootExit = inspectProcessGroupAfterLeaderExit(child.pid, inspect); } resolve({ status, signal }); }); @@ -260,6 +280,7 @@ export async function runProcessTreeCommand( terminationGraceMs, forceWaitMs, isOriginalLeader: () => child.exitCode === null && child.signalCode === null, + inspectProcessGroup: inspect, }); void cleanupPromise.then(treeExited => { if (treeExited) return; @@ -299,6 +320,7 @@ export async function runProcessTreeCommand( treeExited = await terminateInstallerProcessTree(child.pid, { terminationGraceMs, forceWaitMs, + inspectProcessGroup: inspect, }); } else if (posixTreeAfterRootExit !== "exited") { treeExited = false; @@ -312,5 +334,7 @@ export async function runProcessTreeCommand( interruptedSignal, timedOut, treeExited, + stdout, + stderr, }; } diff --git a/tests/update-install-process.test.ts b/tests/update-install-process.test.ts index 9a0d8108d..cad72cc31 100644 --- a/tests/update-install-process.test.ts +++ b/tests/update-install-process.test.ts @@ -1,5 +1,4 @@ import { afterEach, describe, expect, test } from "bun:test"; -import { spawnSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -14,19 +13,29 @@ const cleanupDirs = new Set(); function isRunning(pid: number): boolean { try { process.kill(pid, 0); - if (process.platform !== "win32") { - const state = spawnSync("/bin/ps", ["-o", "state=", "-p", String(pid)], { - encoding: "utf8", - stdio: ["ignore", "pipe", "ignore"], - }); - if (state.status === 0 && /^[ZX]/.test(state.stdout.trim())) return false; - } return true; } catch { return false; } } +function processGroupInspector(descendantPidPath: string) { + return (groupId: number) => { + const hasRunningLeader = isRunning(groupId); + let hasRunningDescendant = false; + try { + const descendantPid = Number.parseInt(readFileSync(descendantPidPath, "utf8"), 10); + hasRunningDescendant = Number.isSafeInteger(descendantPid) && isRunning(descendantPid); + } catch { + // The child may not have written its pid before the root exits. + } + return { + hasRunningMember: hasRunningLeader || hasRunningDescendant, + hasRunningLeader, + }; + }; +} + afterEach(() => { for (const pid of cleanupPids) { try { process.kill(pid, "SIGKILL"); } catch { /* already gone */ } @@ -56,6 +65,18 @@ describe("update installer process isolation", () => { expect(result.treeExited).toBe(true); }); + test("drains and bounds piped installer output", async () => { + const result = await runProcessTreeCommand( + process.execPath, + ["-e", "console.log('installer stdout'); console.error('installer stderr')"], + { stdio: "pipe", timeoutMs: 1_000 }, + ); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("installer stdout"); + expect(result.stderr).toContain("installer stderr"); + }); + test("timeout kills and awaits the installer descendant tree", async () => { const dir = join(tmpdir(), `ocx-installer-tree-${process.pid}-${Date.now()}`); const fixture = join(dir, "installer-parent.mjs"); @@ -76,6 +97,7 @@ describe("update installer process isolation", () => { stdio: "ignore", terminationGraceMs: 500, timeoutMs: 3_000, + inspectProcessGroup: processGroupInspector(descendantPidPath), }); expect(existsSync(descendantPidPath)).toBe(true); @@ -107,6 +129,7 @@ describe("update installer process isolation", () => { stdio: "ignore", terminationGraceMs: 500, timeoutMs: 5_000, + inspectProcessGroup: processGroupInspector(descendantPidPath), }); expect(result.status).toBe(1); diff --git a/tests/update-stop-first.test.ts b/tests/update-stop-first.test.ts index 2443f3cc9..c627370cc 100644 --- a/tests/update-stop-first.test.ts +++ b/tests/update-stop-first.test.ts @@ -143,6 +143,17 @@ describe("update stops the running proxy before replacing files", () => { expect(updateSource).toContain("stdio: svcStdio"); expect(updateSource).toContain("windowsHide: true"); }); + + test("Bun/source installer cleanup is tree-aware before shim or service refresh", () => { + const installAt = updateSource.indexOf("await runProcessTreeCommand(target.bin, cmdArgs"); + const cleanupGateAt = updateSource.indexOf("if (!r.treeExited)"); + const successAt = updateSource.indexOf("if (r.status === 0)"); + expect(installAt).toBeGreaterThan(-1); + expect(cleanupGateAt).toBeGreaterThan(installAt); + expect(cleanupGateAt).toBeLessThan(successAt); + expect(updateSource).toContain("timeoutMs: 180000"); + expect(updateSource).toContain("INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE"); + }); }); describe("ocx update --help has no side effects (#168)", () => { From 1fdc443d26ed145aac5c791ce0cbf7e75d997df7 Mon Sep 17 00:00:00 2001 From: wzb Date: Mon, 27 Jul 2026 18:17:23 +0800 Subject: [PATCH 11/16] fix(update): close installer recovery review gaps --- docs/adr/0001-gui-update-worker.md | 28 +++++---- src/update/install-process.d.mts | 1 + src/update/install-process.mjs | 58 +++++++++++++------ src/update/job.ts | 57 +++++++++++++------ tests/update-install-process.test.ts | 85 ++++++++++++++++++++++++---- tests/update-job.test.ts | 76 +++++++++++++++++++++---- 6 files changed, 240 insertions(+), 65 deletions(-) diff --git a/docs/adr/0001-gui-update-worker.md b/docs/adr/0001-gui-update-worker.md index 622466529..aa1d8fefb 100644 --- a/docs/adr/0001-gui-update-worker.md +++ b/docs/adr/0001-gui-update-worker.md @@ -42,22 +42,27 @@ platforms, the scope and complete package tree must also reject symlinks, foreig group/world-writable entries before any candidate code executes. The tree walk is bounded by entry count and elapsed time. Candidate inspection and restart are bounded to at most two candidates, preferring the current package and then the newest retired copies, and it tries those runnable -candidates in order until one restores health. +candidates in order until one restores health. A recovery launcher is always started directly: it +may restore availability, but its potentially temporary path is never persisted into launchd, +systemd, or Task Scheduler. If a candidate starts but does not become healthy, the worker records +its PID, revalidates its OpenCodex identity, and stops it before trying the next candidate. Service and direct restart paths also stop when the pidfile has changed to a different identity-checked proxy, and every such identity decision re-reads the process command line instead of trusting a PID-only cache across the update boundary. The update job remains failed either way because restoring availability is not the same as installing the new version. The npm launcher and the Bun/source CLI installer paths run in an isolated process tree. On timeout, -the updater terminates and awaits the whole POSIX process group or a Windows `taskkill /T /F` tree -before returning failure; piped CLI output is drained and bounded before it is replayed. POSIX -cleanup treats zombie-only groups as stopped and refuses to signal a process-group ID that has been -reused by a new leader, including a second identity check immediately before force-killing after the -grace period. Windows has no retained job-object handle after a normally failed installer -root exits, so that case remains explicitly unconfirmed and automatic recovery (including tray -restoration) stays disabled. The outer worker timeout remains longer than the nested deadline, and -recovery is skipped whenever either layer cannot prove installer-tree shutdown, so recovery never -races a lifecycle or replacement child that may still be writing package files. +the updater terminates and awaits the known POSIX process group or a Windows `taskkill /T /F` tree; +piped CLI output is drained and bounded before it is replayed. A POSIX process group is not an OS +containment boundary because a lifecycle child can leave it with `setsid` or `setpgid`. Therefore a +timeout, interruption, or nonzero POSIX root exit is always reported as unconfirmed even when the +known group was successfully stopped, and automatic recovery stays disabled. Once a POSIX root has +exited, the updater also refuses to signal its leaderless group by a reusable numeric PGID. While the +root is live, cleanup treats zombie-only groups as stopped and rechecks the original leader twice at +the SIGTERM boundary; force-kill retains its own second group inspection. Windows has no retained +job-object handle after a normally failed installer root exits, so that case likewise remains +explicitly unconfirmed. The outer worker timeout remains longer than the nested deadline, and +recovery is skipped whenever either layer cannot prove installer-tree shutdown. After an update requests a restart, the worker now waits for an identity-checked `/healthz` to return and remain healthy for a short stability window before marking the job successful. This @@ -83,7 +88,8 @@ restart path because `bun add -g` does not restart the proxy. - Update status survives a proxy restart because it is stored in the opencodex config directory. - Restart handling can branch between service-managed installs and direct detached proxy starts. - npm cache ownership failures are detected before service or proxy shutdown. -- Failed installer process trees are terminated and confirmed gone before recovery starts. +- Automatic recovery starts only when installer shutdown is confirmed; ambiguous POSIX failures and + escaped descendants remain fail-closed with manual remediation. - A failed install best-effort restores a previously-active proxy without claiming update success. - A completed install can still finish with `status: "failed"` when the replacement proxy never becomes healthy or flaps during the stability window; the job log then points the user at diff --git a/src/update/install-process.d.mts b/src/update/install-process.d.mts index 41d343456..0408c5e64 100644 --- a/src/update/install-process.d.mts +++ b/src/update/install-process.d.mts @@ -1,6 +1,7 @@ import type { StdioOptions } from "node:child_process"; export const INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE: number; +export const MAX_CAPTURED_OUTPUT_CHARS: number; export interface ProcessTreeCommandOptions { timeoutMs?: number; diff --git a/src/update/install-process.mjs b/src/update/install-process.mjs index 5eab28b5f..fb74c4b75 100644 --- a/src/update/install-process.mjs +++ b/src/update/install-process.mjs @@ -4,7 +4,7 @@ import { readdirSync, readFileSync } from "node:fs"; const TREE_POLL_INTERVAL_MS = 100; const DEFAULT_TERMINATION_GRACE_MS = 5_000; const DEFAULT_FORCE_WAIT_MS = 5_000; -const MAX_CAPTURED_OUTPUT_CHARS = 4_000; +export const MAX_CAPTURED_OUTPUT_CHARS = 4_000; function appendBoundedOutput(current, chunk) { const text = typeof chunk === "string" ? chunk : Buffer.from(chunk).toString("utf8"); @@ -83,6 +83,15 @@ export function processGroupForceDecision(inspection, originalLeaderConfirmed) { return "signal"; } +function processGroupTerminationDecision(inspection, originalLeaderConfirmed) { + if (!inspection) return "refuse"; + if (!inspection.hasRunningMember) return "exited"; + // SIGTERM is sent only while the original root still anchors the group ID. + // Leaderless groups are handled fail-closed after root exit. + if (!inspection.hasRunningLeader || !originalLeaderConfirmed) return "refuse"; + return "signal"; +} + function processGroupHasRunningMember(groupId, inspect = inspectProcessGroup) { return inspect(groupId)?.hasRunningMember ?? null; } @@ -155,7 +164,21 @@ export async function terminateInstallerProcessTree( return waitForProcessTreeExit(pid, forceWaitMs, inspect); } - if (!isProcessTreeAlive(pid, inspect)) return true; + // Inspect twice at the signal boundary. SIGTERM is allowed only while the + // original root still anchors the group; a leaderless or replaced group is + // never signaled by its reusable numeric PGID. + const terminationInspection = inspect(pid); + const terminationLeaderConfirmed = terminationInspection?.hasRunningLeader === true + && isOriginalLeader?.() === true; + const terminationDecision = processGroupTerminationDecision(terminationInspection, terminationLeaderConfirmed); + if (terminationDecision === "exited") return true; + if (terminationDecision === "refuse") return false; + const signalInspection = inspect(pid); + const signalLeaderConfirmed = signalInspection?.hasRunningLeader === true + && isOriginalLeader?.() === true; + const signalDecision = processGroupTerminationDecision(signalInspection, signalLeaderConfirmed); + if (signalDecision === "exited") return true; + if (signalDecision === "refuse") return false; try { process.kill(-pid, "SIGTERM"); } catch (error) { @@ -174,12 +197,12 @@ export async function terminateInstallerProcessTree( // Reinspect at the force-signal boundary. The original group can exit and its // ID can be reused after the grace-period inspection above; a replacement // leader must never receive the pending SIGKILL. - const signalInspection = inspect(pid); - const signalLeaderConfirmed = signalInspection?.hasRunningLeader === true + const forceSignalInspection = inspect(pid); + const forceSignalLeaderConfirmed = forceSignalInspection?.hasRunningLeader === true && isOriginalLeader?.() === true; - const signalDecision = processGroupForceDecision(signalInspection, signalLeaderConfirmed); - if (signalDecision === "exited") return true; - if (signalDecision === "refuse") return false; + const forceSignalDecision = processGroupForceDecision(forceSignalInspection, forceSignalLeaderConfirmed); + if (forceSignalDecision === "exited") return true; + if (forceSignalDecision === "refuse") return false; try { process.kill(-pid, "SIGKILL"); @@ -311,19 +334,22 @@ export async function runProcessTreeCommand( let treeExited = true; if (cleanupPromise) { - treeExited = await cleanupPromise; + const knownGroupExited = await cleanupPromise; + // taskkill /T is a retained Windows tree operation. A POSIX process group is + // not containment: a lifecycle child can leave it with setsid/setpgid, so a + // timeout/interruption can never prove complete descendant shutdown. + treeExited = process.platform === "win32" && knownGroupExited; } else if (spawnFailed) { treeExited = true; } else if (process.platform === "win32") { treeExited = !windowsFailedExitTreeUnknown; - } else if (child.pid && posixTreeAfterRootExit === "running") { - treeExited = await terminateInstallerProcessTree(child.pid, { - terminationGraceMs, - forceWaitMs, - inspectProcessGroup: inspect, - }); - } else if (posixTreeAfterRootExit !== "exited") { - treeExited = false; + } else { + // A successful package-manager root exit is its completion contract. For a + // failed/signaled root, process groups cannot prove that a daemonized child + // did not escape, and signaling a leaderless group risks a reused PGID. + treeExited = result.status === 0 + && result.signal === null + && posixTreeAfterRootExit === "exited"; } for (const [signal, handler] of signalHandlers) process.removeListener(signal, handler); diff --git a/src/update/job.ts b/src/update/job.ts index d61eb6383..69125a9f7 100644 --- a/src/update/job.ts +++ b/src/update/job.ts @@ -492,7 +492,7 @@ function spawnDetachedStart( installer: Installer, port?: number, launcher = packageLauncherPath(), -): void { +): number | null { const cmd = restartCommand(false, installer, launcher, port); const env = { ...process.env }; delete env.OCX_SERVICE; @@ -504,6 +504,7 @@ function spawnDetachedStart( env, }); child.unref(); + return child.pid ?? null; } /** Identity snapshot used to prove an npm self-update actually replaced the pre-update process. */ @@ -517,7 +518,7 @@ export type FailedUpdateRecovery = "not-needed" | "still-running" | "restarted" /** Test seam: the wait/spawn pair is injectable so the restart path is verifiable. */ export interface RestartIo { waitForPort?: typeof reclaimListenPort; - spawnStart?: (job: UpdateJobState, installer: Installer, port?: number, launcher?: string) => void; + spawnStart?: (job: UpdateJobState, installer: Installer, port?: number, launcher?: string) => number | null | void; serviceInstalledFn?: () => boolean; readPidFn?: () => number | null; probeProxy?: (port: number, hostname?: string) => Promise; @@ -525,6 +526,7 @@ export interface RestartIo { probeProxyIdentity?: (port: number, hostname?: string) => Promise; verifyPidIdentityFn?: (pid: number) => number | null; recoveryLaunchersFn?: (currentLauncher: string, expectedVersion: string) => Promise | string[]; + killProxyFn?: (pid: number) => void; sleepMs?: (ms: number) => Promise; now?: () => number; /** Service-mode install/reinstall command (defaults to spawnSync via runLoggedCommand). */ @@ -538,15 +540,21 @@ export interface RestartIo { job: UpdateJobState, captured?: { port: number; hostname: string; oldPid?: number; recoveryLauncher?: string }, io?: RestartIo, - ) => Promise; + ) => Promise; } async function restartAfterUpdate( job: UpdateJobState, captured?: { port: number; hostname: string; oldPid?: number; recoveryLauncher?: string }, io: RestartIo = {}, -): Promise { +): Promise { const serviceInstalled = (io.serviceInstalledFn ?? isServiceInstalled)(); + // A failed npm install may leave only a hidden, npm-retired package runnable. + // It can restore availability, but must never be baked into launchd/systemd. + const persistentServiceRestart = serviceInstalled && captured?.recoveryLauncher === undefined; + if (serviceInstalled && !persistentServiceRestart) { + updateJob(job, {}, "Recovery launcher is temporary; starting the proxy directly and leaving the installed service stopped until the package is repaired."); + } const config = loadConfig(); // The stop-first update flow has already cleared pid/runtime state by the time we run, // so the pre-update capture (taken before the update command) is the authoritative @@ -567,16 +575,16 @@ async function restartAfterUpdate( updateJob(job, {}, `A different identity-checked proxy PID appeared ${context}; leaving it untouched.`); return { pid: null, refused: true }; }; - if (readPidForRestart("during restart handoff").refused) return; + if (readPidForRestart("during restart handoff").refused) return null; let svcArgs: string[] | undefined; - if (serviceInstalled) { + if (persistentServiceRestart) { try { const { serviceReinstallArgs } = await import("../service"); svcArgs = serviceReinstallArgs(); } catch { /* fallback to default service install */ } } const launcher = captured?.recoveryLauncher ?? packageLauncherPath(); - const cmd = restartCommand(serviceInstalled, job.installer, launcher, port, svcArgs); + const cmd = restartCommand(persistentServiceRestart, job.installer, launcher, port, svcArgs); const waitFn = io.waitForPort ?? reclaimListenPort; const reclaimOpts = { timeoutMs: RESTART_PORT_RECLAIM_MS, @@ -586,14 +594,14 @@ async function restartAfterUpdate( onlyKillPids: oldPid != null ? [oldPid] : [], }; - if (serviceInstalled) { + if (persistentServiceRestart) { // Stop-first update already unloaded the service; reclaim the socket (only the // captured old PID when trusted), then reinstall wrappers that bake `--port`. const freed = await waitFn(port, hostname, reclaimOpts); if (!freed) { updateJob(job, {}, `Port ${port} still busy after ${Math.trunc(RESTART_PORT_RECLAIM_MS / 1000)}s; refusing to hop — reinstall may fail until the port is free.`); } - if (readPidForRestart("after service port reclaim").refused) return; + if (readPidForRestart("after service port reclaim").refused) return null; const prevBake = process.env.OCX_BAKE_PORT; process.env.OCX_BAKE_PORT = String(Math.trunc(port)); let serviceOk = false; @@ -614,13 +622,13 @@ async function restartAfterUpdate( if (prevBake === undefined) delete process.env.OCX_BAKE_PORT; else process.env.OCX_BAKE_PORT = prevBake; } - if (serviceOk) return; + if (serviceOk) return null; // Fall through to the direct proxy start below so the update never leaves the // proxy stopped when the service reinstall could not run. } const directPid = readPidForRestart("before direct restart"); - if (directPid.refused) return; + if (directPid.refused) return null; const pid = directPid.pid; if (pid !== null) { updateJob(job, {}, `Stopping current proxy PID ${pid}.`); @@ -632,10 +640,13 @@ async function restartAfterUpdate( const freed = await waitFn(port, hostname, reclaimOpts); if (!freed) { updateJob(job, {}, `Port ${port} still busy after ${Math.trunc(RESTART_PORT_RECLAIM_MS / 1000)}s (reclaim could not free the socket); not starting on another port. Retry 'ocx start --port ${port}'.`); - return; + return null; } - if (readPidForRestart("after direct port reclaim").refused) return; - (io.spawnStart ?? spawnDetachedStart)(job, job.installer, port, launcher); + if (readPidForRestart("after direct port reclaim").refused) return null; + const startedPid = (io.spawnStart ?? spawnDetachedStart)(job, job.installer, port, launcher); + return typeof startedPid === "number" && Number.isSafeInteger(startedPid) && startedPid > 0 + ? startedPid + : null; } /** Exposed for tests: drives the non-service restart path with injected io. */ @@ -643,7 +654,7 @@ export function restartAfterUpdateForTests( job: UpdateJobState, captured: { port: number; hostname: string; oldPid?: number; recoveryLauncher?: string }, io: RestartIo, -): Promise { +): Promise { return restartAfterUpdate(job, captured, io); } @@ -860,12 +871,26 @@ async function recoverFailedGuiUpdate( updateJob(job, {}, `Recovery candidate ${index} did not restore the proxy; trying candidate ${index + 1} of ${recoveryLaunchers.length}.`); } try { - await restartFn(job, recoveryCaptured, io); + const startedPid = await restartFn(job, recoveryCaptured, io); const healthy = await awaitRestartedProxyHealthy(job, recoveryCaptured, io); if (healthy.ok) { updateJob(job, {}, `Previous proxy availability restored on ${captured.hostname}:${captured.port}; the update itself still failed.`); return "restarted"; } + if (typeof startedPid === "number" && Number.isSafeInteger(startedPid) && startedPid > 0) { + const verifyStartedPid = io.verifyPidIdentityFn ?? verifyPidIdentityFresh; + if (verifyStartedPid(startedPid) !== startedPid) { + updateJob(job, {}, `Recovery candidate ${index + 1} left PID ${startedPid} without a matching OpenCodex identity; refusing to kill or try another candidate.`); + break; + } + try { + (io.killProxyFn ?? killProxy)(startedPid); + updateJob(job, {}, `Stopped unhealthy recovery candidate ${index + 1} PID ${startedPid}.`); + } catch (error) { + updateJob(job, {}, `Could not stop unhealthy recovery candidate ${index + 1} PID ${startedPid}: ${error instanceof Error ? error.message : String(error)}`); + break; + } + } } catch { updateJob(job, {}, `Recovery candidate ${index + 1} failed before health confirmation.`); } diff --git a/tests/update-install-process.test.ts b/tests/update-install-process.test.ts index cad72cc31..32affc2ad 100644 --- a/tests/update-install-process.test.ts +++ b/tests/update-install-process.test.ts @@ -1,10 +1,13 @@ import { afterEach, describe, expect, test } from "bun:test"; +import { spawn } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { + MAX_CAPTURED_OUTPUT_CHARS, processGroupForceDecision, runProcessTreeCommand, + terminateInstallerProcessTree, } from "../src/update/install-process.mjs"; const cleanupPids = new Set(); @@ -54,6 +57,33 @@ describe("update installer process isolation", () => { expect(processGroupForceDecision({ hasRunningMember: true, hasRunningLeader: true }, true)).toBe("signal"); }); + test("rechecks the original leader immediately before SIGTERM", async () => { + if (process.platform === "win32") return; + const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { + detached: true, + stdio: "ignore", + }); + await new Promise((resolve, reject) => { + child.once("spawn", resolve); + child.once("error", reject); + }); + expect(child.pid).toBeDefined(); + cleanupPids.add(child.pid!); + let leaderChecks = 0; + + const treeExited = await terminateInstallerProcessTree(child.pid, { + inspectProcessGroup: () => ({ hasRunningMember: true, hasRunningLeader: true }), + isOriginalLeader: () => { + leaderChecks += 1; + return leaderChecks === 1; + }, + }); + + expect(treeExited).toBe(false); + expect(leaderChecks).toBe(2); + expect(isRunning(child.pid!)).toBe(true); + }); + test("spawn failures report their cause without inventing a live process tree", async () => { const result = await runProcessTreeCommand(join(tmpdir(), "ocx-command-that-does-not-exist"), [], { stdio: "ignore", @@ -66,15 +96,17 @@ describe("update installer process isolation", () => { }); test("drains and bounds piped installer output", async () => { + const stdoutPayload = "out-".repeat(MAX_CAPTURED_OUTPUT_CHARS); + const stderrPayload = "err-".repeat(MAX_CAPTURED_OUTPUT_CHARS); const result = await runProcessTreeCommand( process.execPath, - ["-e", "console.log('installer stdout'); console.error('installer stderr')"], + ["-e", `process.stdout.write(${JSON.stringify(stdoutPayload)}); process.stderr.write(${JSON.stringify(stderrPayload)})`], { stdio: "pipe", timeoutMs: 1_000 }, ); expect(result.status).toBe(0); - expect(result.stdout).toContain("installer stdout"); - expect(result.stderr).toContain("installer stderr"); + expect(result.stdout).toBe(stdoutPayload.slice(-MAX_CAPTURED_OUTPUT_CHARS)); + expect(result.stderr).toBe(stderrPayload.slice(-MAX_CAPTURED_OUTPUT_CHARS)); }); test("timeout kills and awaits the installer descendant tree", async () => { @@ -105,12 +137,13 @@ describe("update installer process isolation", () => { const descendantRunning = isRunning(descendantPid); if (descendantRunning) cleanupPids.add(descendantPid); expect(result.timedOut).toBe(true); - expect(result.treeExited).toBe(true); + expect(result.treeExited).toBe(process.platform === "win32"); expect(descendantRunning).toBe(false); }, 15_000); - test("a failed root exit cleans POSIX descendants and fails closed on Windows", async () => { - const dir = join(tmpdir(), `ocx-installer-exit-tree-${process.pid}-${Date.now()}`); + test("a failed root exit does not signal its remaining leaderless process group", async () => { + if (process.platform === "win32") return; + const dir = join(tmpdir(), `ocx-installer-leaderless-${process.pid}-${Date.now()}`); const fixture = join(dir, "installer-parent.mjs"); const descendantPidPath = join(dir, "descendant.pid"); mkdirSync(dir, { recursive: true }); @@ -132,16 +165,44 @@ describe("update installer process isolation", () => { inspectProcessGroup: processGroupInspector(descendantPidPath), }); + expect(result.status).toBe(1); + expect(result.treeExited).toBe(false); + const descendantPid = Number.parseInt(readFileSync(descendantPidPath, "utf8"), 10); + const descendantRunning = isRunning(descendantPid); + if (descendantRunning) cleanupPids.add(descendantPid); + expect(descendantRunning).toBe(true); + }, 15_000); + + test("a failed root exit fails closed when a descendant leaves its process group", async () => { + const dir = join(tmpdir(), `ocx-installer-exit-tree-${process.pid}-${Date.now()}`); + const fixture = join(dir, "installer-parent.mjs"); + const descendantPidPath = join(dir, "descendant.pid"); + mkdirSync(dir, { recursive: true }); + cleanupDirs.add(dir); + writeFileSync(fixture, [ + 'import { spawn } from "node:child_process";', + 'import { writeFileSync } from "node:fs";', + 'const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, stdio: "ignore" });', + 'child.unref();', + 'writeFileSync(process.argv[2], String(child.pid));', + 'setTimeout(() => process.exit(1), 50);', + "", + ].join("\n")); + + const result = await runProcessTreeCommand(process.execPath, [fixture, descendantPidPath], { + forceWaitMs: 2_000, + stdio: "ignore", + terminationGraceMs: 500, + timeoutMs: 5_000, + inspectProcessGroup: processGroupInspector(descendantPidPath), + }); + expect(result.status).toBe(1); expect(result.timedOut).toBe(false); const descendantPid = Number.parseInt(readFileSync(descendantPidPath, "utf8"), 10); const descendantRunning = isRunning(descendantPid); if (descendantRunning) cleanupPids.add(descendantPid); - if (process.platform === "win32") { - expect(result.treeExited).toBe(false); - } else { - expect(result.treeExited).toBe(true); - expect(descendantRunning).toBe(false); - } + expect(result.treeExited).toBe(false); + expect(descendantRunning).toBe(true); }, 15_000); }); diff --git a/tests/update-job.test.ts b/tests/update-job.test.ts index 9debacf1d..ef7648f8d 100644 --- a/tests/update-job.test.ts +++ b/tests/update-job.test.ts @@ -501,7 +501,6 @@ describe("GUI update execution decisions", () => { await restartAfterUpdateForTests(job, { port: 18765, hostname: "127.0.0.1", - recoveryLauncher: "/retired/bin/ocx.mjs", }, { serviceInstalledFn: () => true, waitForPort: async (port, hostname) => { @@ -517,7 +516,8 @@ describe("GUI update execution decisions", () => { }); expect(waited).toEqual([{ port: 18765, hostname: "127.0.0.1" }]); expect(bakeDuringInstall).toEqual(["18765"]); - expect(launchersDuringInstall).toEqual(["/retired/bin/ocx.mjs"]); + expect(launchersDuringInstall).toHaveLength(1); + expect(launchersDuringInstall[0]?.endsWith("/bin/ocx.mjs")).toBe(true); expect(process.env.OCX_BAKE_PORT).toBeUndefined(); } finally { if (prev === undefined) delete process.env.OCX_BAKE_PORT; @@ -525,6 +525,48 @@ describe("GUI update execution decisions", () => { } }); + test("retired recovery launchers never persist their path in an installed service", async () => { + const spawned: Array<{ port?: number; launcher?: string }> = []; + const job: UpdateJobState = { + id: "restart-retired-direct", + status: "restarting", + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + currentVersion: "2.7.40", + latestVersion: "2.7.41", + channel: "latest", + installer: "npm", + restart: true, + command: "", + releaseNotesUrl: "", + log: [], + }; + writeFileSync(updateJobPath(), JSON.stringify(job)); + + const startedPid = await restartAfterUpdateForTests(job, { + port: 10100, + hostname: "127.0.0.1", + recoveryLauncher: "/scope/.opencodex-Ab12Cd34/bin/ocx.mjs", + }, { + serviceInstalledFn: () => true, + waitForPort: async () => true, + runService: () => { throw new Error("must not persist a temporary recovery path"); }, + spawnStart: (_job, _installer, port, launcher) => { + spawned.push({ port, launcher }); + return 222; + }, + }); + + expect(startedPid).toBe(222); + expect(spawned).toEqual([{ + port: 10100, + launcher: "/scope/.opencodex-Ab12Cd34/bin/ocx.mjs", + }]); + expect(readUpdateJob(job.id)?.log.some(line => + line.includes("temporary") && line.includes("installed service stopped"), + )).toBe(true); + }); + test("service reinstall failure falls back to a direct proxy start", async () => { const spawned: Array<{ port: number }> = []; const job: UpdateJobState = { @@ -791,10 +833,12 @@ describe("GUI update execution decisions", () => { expect(readUpdateJob(job.id)?.log.some(line => line.includes("update itself still failed"))).toBe(true); }); - test("failed install tries the next runnable recovery package when the first does not start", async () => { + test("failed install stops an unhealthy recovery process before trying the next package", async () => { let now = 0; - let activeLauncher: string | undefined; + let activePid: number | null = null; const attempted: Array = []; + const killed: number[] = []; + const runningPids = new Set(); const job: UpdateJobState = { id: "failed-install-recovery-fallback", status: "running", @@ -817,13 +861,22 @@ describe("GUI update execution decisions", () => { true, { probeProxyIdentity: async () => null, - verifyPidIdentityFn: () => null, + serviceInstalledFn: () => false, + readPidFn: () => activePid, + verifyPidIdentityFn: pid => runningPids.has(pid) ? pid : null, recoveryLaunchersFn: () => ["/current/bin/ocx.mjs", "/retired/bin/ocx.mjs"], - probeProxy: async () => activeLauncher === "/retired/bin/ocx.mjs", - restartAfterUpdateFn: async (_job, captured) => { - activeLauncher = captured?.recoveryLauncher; - attempted.push(activeLauncher); - if (activeLauncher === "/current/bin/ocx.mjs") throw new Error("partial runtime"); + probeProxy: async () => activePid === 333 && runningPids.has(333), + waitForPort: async () => true, + spawnStart: (_job, _installer, _port, launcher) => { + attempted.push(launcher); + activePid = launcher === "/current/bin/ocx.mjs" ? 222 : 333; + runningPids.add(activePid); + return activePid; + }, + killProxyFn: pid => { + killed.push(pid); + runningPids.delete(pid); + if (activePid === pid) activePid = null; }, now: () => now, sleepMs: async (ms) => { now += ms; }, @@ -832,6 +885,9 @@ describe("GUI update execution decisions", () => { expect(recovery).toBe("restarted"); expect(attempted).toEqual(["/current/bin/ocx.mjs", "/retired/bin/ocx.mjs"]); + expect(killed).toEqual([222]); + expect(runningPids.has(222)).toBe(false); + expect(runningPids.has(333)).toBe(true); expect(readUpdateJob(job.id)?.log.some(line => line.includes("trying candidate 2 of 2"))).toBe(true); }); From a9daa127fc10607bd0e565df9f531fa8a9ab9906 Mon Sep 17 00:00:00 2001 From: wzb Date: Mon, 27 Jul 2026 18:57:53 +0800 Subject: [PATCH 12/16] fix(update): address recovery review feedback --- docs/adr/0001-gui-update-worker.md | 17 ++--- src/update/install-process.mjs | 29 ++------ src/update/job.ts | 29 ++++++-- tests/update-install-process.test.ts | 19 ++++- tests/update-job.test.ts | 101 ++++++++++++++++++++++++++- 5 files changed, 154 insertions(+), 41 deletions(-) diff --git a/docs/adr/0001-gui-update-worker.md b/docs/adr/0001-gui-update-worker.md index aa1d8fefb..e34a18951 100644 --- a/docs/adr/0001-gui-update-worker.md +++ b/docs/adr/0001-gui-update-worker.md @@ -38,10 +38,11 @@ probes again immediately before recovery and leaves any old or concurrently repl when health or PID identity proves it is OpenCodex. When npm already stopped the proxy and retired the old package, the worker validates each candidate's trusted ownership, name, version, path containment, and complete launcher runtime with a side-effect-free `--version` probe. On UID-capable -platforms, the scope and complete package tree must also reject symlinks, foreign owners, and -group/world-writable entries before any candidate code executes. The tree walk is bounded by entry -count and elapsed time. Candidate inspection and restart are bounded to at most two candidates, -preferring the current package and then the newest retired copies, and it tries those runnable +platforms, the scope and complete package tree must also reject foreign owners, group/world-writable +entries, and symlinks that leave the candidate tree before any code executes. Trusted npm-generated +links whose immediate and final targets remain inside the candidate tree are allowed. The tree walk +is bounded by entry count and elapsed time. Candidate inspection and restart are bounded to at most +two candidates, preferring the current package and then the newest retired copies, and it tries those runnable candidates in order until one restores health. A recovery launcher is always started directly: it may restore availability, but its potentially temporary path is never persisted into launchd, systemd, or Task Scheduler. If a candidate starts but does not become healthy, the worker records @@ -58,10 +59,10 @@ containment boundary because a lifecycle child can leave it with `setsid` or `se timeout, interruption, or nonzero POSIX root exit is always reported as unconfirmed even when the known group was successfully stopped, and automatic recovery stays disabled. Once a POSIX root has exited, the updater also refuses to signal its leaderless group by a reusable numeric PGID. While the -root is live, cleanup treats zombie-only groups as stopped and rechecks the original leader twice at -the SIGTERM boundary; force-kill retains its own second group inspection. Windows has no retained -job-object handle after a normally failed installer root exits, so that case likewise remains -explicitly unconfirmed. The outer worker timeout remains longer than the nested deadline, and +root is live, cleanup treats zombie-only groups as stopped and revalidates group membership and the +original leader immediately before each signal; uninspectable or replacement-led groups are refused. +Windows has no retained job-object handle after a normally failed installer root exits, so that case +likewise remains explicitly unconfirmed. The outer worker timeout remains longer than the nested deadline, and recovery is skipped whenever either layer cannot prove installer-tree shutdown. After an update requests a restart, the worker now waits for an identity-checked `/healthz` to diff --git a/src/update/install-process.mjs b/src/update/install-process.mjs index fb74c4b75..bb533cdc4 100644 --- a/src/update/install-process.mjs +++ b/src/update/install-process.mjs @@ -164,15 +164,8 @@ export async function terminateInstallerProcessTree( return waitForProcessTreeExit(pid, forceWaitMs, inspect); } - // Inspect twice at the signal boundary. SIGTERM is allowed only while the - // original root still anchors the group; a leaderless or replaced group is - // never signaled by its reusable numeric PGID. - const terminationInspection = inspect(pid); - const terminationLeaderConfirmed = terminationInspection?.hasRunningLeader === true - && isOriginalLeader?.() === true; - const terminationDecision = processGroupTerminationDecision(terminationInspection, terminationLeaderConfirmed); - if (terminationDecision === "exited") return true; - if (terminationDecision === "refuse") return false; + // An unreaped child retains its PID. Revalidate that leader identity at the + // signal boundary so a released or replacement-led group is never signaled. const signalInspection = inspect(pid); const signalLeaderConfirmed = signalInspection?.hasRunningLeader === true && isOriginalLeader?.() === true; @@ -194,16 +187,6 @@ export async function terminateInstallerProcessTree( if (forceDecision === "exited") return true; if (forceDecision === "refuse") return false; - // Reinspect at the force-signal boundary. The original group can exit and its - // ID can be reused after the grace-period inspection above; a replacement - // leader must never receive the pending SIGKILL. - const forceSignalInspection = inspect(pid); - const forceSignalLeaderConfirmed = forceSignalInspection?.hasRunningLeader === true - && isOriginalLeader?.() === true; - const forceSignalDecision = processGroupForceDecision(forceSignalInspection, forceSignalLeaderConfirmed); - if (forceSignalDecision === "exited") return true; - if (forceSignalDecision === "refuse") return false; - try { process.kill(-pid, "SIGKILL"); } catch (error) { @@ -344,12 +327,12 @@ export async function runProcessTreeCommand( } else if (process.platform === "win32") { treeExited = !windowsFailedExitTreeUnknown; } else { - // A successful package-manager root exit is its completion contract. For a - // failed/signaled root, process groups cannot prove that a daemonized child - // did not escape, and signaling a leaderless group risks a reused PGID. + // A successful package-manager root exit is its completion contract unless + // inspection positively finds a surviving member of the original group. + // Failed/signaled roots remain unconfirmed even when the group looks empty. treeExited = result.status === 0 && result.signal === null - && posixTreeAfterRootExit === "exited"; + && posixTreeAfterRootExit !== "running"; } for (const [signal, handler] of signalHandlers) process.removeListener(signal, handler); diff --git a/src/update/job.ts b/src/update/job.ts index 69125a9f7..b5dfe8e0d 100644 --- a/src/update/job.ts +++ b/src/update/job.ts @@ -1,5 +1,5 @@ import { spawn, spawnSync } from "node:child_process"; -import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync } from "node:fs"; +import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, readlinkSync, realpathSync } from "node:fs"; import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { @@ -179,16 +179,33 @@ function hasTrustedRecoveryTree(packageRoot: string): boolean { const deadline = Date.now() + MAX_NPM_RECOVERY_TREE_SCAN_MS; const pending = [canonicalRoot]; + const visited = new Set(); let inspected = 0; while (pending.length > 0) { if (Date.now() > deadline || inspected >= MAX_NPM_RECOVERY_TREE_ENTRIES) return false; const path = pending.pop()!; inspected += 1; - const stat = lstatSync(path); - if (stat.isSymbolicLink() || !hasTrustedRecoveryPermissions(stat)) return false; - if (stat.isFile()) continue; - if (!stat.isDirectory()) return false; - for (const name of readdirSync(path, { encoding: "utf8" })) pending.push(join(path, name)); + const entryStat = lstatSync(path); + let canonicalPath = path; + let canonicalStat = entryStat; + if (entryStat.isSymbolicLink()) { + // npm creates node_modules/.bin links. Permit only trusted-owner links whose + // immediate and final targets both remain inside this candidate package. + if (!hasTrustedRecoveryOwner(entryStat.uid)) return false; + const directTarget = resolve(dirname(path), readlinkSync(path)); + if (!isPathInside(canonicalRoot, directTarget)) return false; + canonicalPath = realpathSync(path); + if (!isPathInside(canonicalRoot, canonicalPath)) return false; + canonicalStat = lstatSync(canonicalPath); + } + if (!hasTrustedRecoveryPermissions(canonicalStat)) return false; + if (visited.has(canonicalPath)) continue; + visited.add(canonicalPath); + if (canonicalStat.isFile()) continue; + if (!canonicalStat.isDirectory()) return false; + for (const name of readdirSync(canonicalPath, { encoding: "utf8" })) { + pending.push(join(canonicalPath, name)); + } } return true; } diff --git a/tests/update-install-process.test.ts b/tests/update-install-process.test.ts index 32affc2ad..d209ee93d 100644 --- a/tests/update-install-process.test.ts +++ b/tests/update-install-process.test.ts @@ -57,7 +57,7 @@ describe("update installer process isolation", () => { expect(processGroupForceDecision({ hasRunningMember: true, hasRunningLeader: true }, true)).toBe("signal"); }); - test("rechecks the original leader immediately before SIGTERM", async () => { + test("checks the original leader immediately before SIGTERM", async () => { if (process.platform === "win32") return; const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], { detached: true, @@ -75,15 +75,28 @@ describe("update installer process isolation", () => { inspectProcessGroup: () => ({ hasRunningMember: true, hasRunningLeader: true }), isOriginalLeader: () => { leaderChecks += 1; - return leaderChecks === 1; + return false; }, }); expect(treeExited).toBe(false); - expect(leaderChecks).toBe(2); + expect(leaderChecks).toBe(1); expect(isRunning(child.pid!)).toBe(true); }); + test("accepts a clean POSIX exit when process-group inspection is unavailable", async () => { + if (process.platform === "win32") return; + const result = await runProcessTreeCommand(process.execPath, ["-e", ""], { + stdio: "ignore", + timeoutMs: 1_000, + inspectProcessGroup: () => null, + }); + + expect(result.status).toBe(0); + expect(result.signal).toBeNull(); + expect(result.treeExited).toBe(true); + }); + test("spawn failures report their cause without inventing a live process tree", async () => { const result = await runProcessTreeCommand(join(tmpdir(), "ocx-command-that-does-not-exist"), [], { stdio: "ignore", diff --git a/tests/update-job.test.ts b/tests/update-job.test.ts index ef7648f8d..4e19c6b5f 100644 --- a/tests/update-job.test.ts +++ b/tests/update-job.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { chmodSync, mkdirSync, realpathSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -191,6 +191,58 @@ describe("GUI update execution decisions", () => { expect(probed).toBe(false); }); + test("allows npm-generated symlinks that resolve inside a trusted recovery package", async () => { + if (process.platform === "win32") return; + const scopeRoot = join(dir, "global", "@bitkyc08"); + const currentRoot = join(scopeRoot, "opencodex"); + const dependencyBin = join(currentRoot, "node_modules", "bun", "bin", "bun.exe"); + const generatedBin = join(currentRoot, "node_modules", ".bin", "bun"); + mkdirSync(join(currentRoot, "bin"), { recursive: true }); + mkdirSync(join(currentRoot, "node_modules", ".bin"), { recursive: true }); + mkdirSync(join(currentRoot, "node_modules", "bun", "bin"), { recursive: true }); + writeFileSync(join(currentRoot, "package.json"), JSON.stringify({ + name: "@bitkyc08/opencodex", + version: "2.7.40", + })); + writeFileSync(join(currentRoot, "bin", "ocx.mjs"), "process.exit(0);\n"); + writeFileSync(dependencyBin, "#!/usr/bin/env node\n"); + symlinkSync("../bun/bin/bun.exe", generatedBin); + + const launcher = realpathSync(join(currentRoot, "bin", "ocx.mjs")); + expect(await findNpmRecoveryLaunchers( + launcher, + "2.7.40", + async candidate => candidate === launcher, + )).toEqual([launcher]); + }); + + test("rejects a recovery package symlink that leaves the candidate tree", async () => { + if (process.platform === "win32") return; + const scopeRoot = join(dir, "global", "@bitkyc08"); + const currentRoot = join(scopeRoot, "opencodex"); + const externalBin = join(dir, "external-tool"); + mkdirSync(join(currentRoot, "bin"), { recursive: true }); + mkdirSync(join(currentRoot, "node_modules", ".bin"), { recursive: true }); + writeFileSync(join(currentRoot, "package.json"), JSON.stringify({ + name: "@bitkyc08/opencodex", + version: "2.7.40", + })); + writeFileSync(join(currentRoot, "bin", "ocx.mjs"), "process.exit(0);\n"); + writeFileSync(externalBin, "#!/usr/bin/env node\n"); + symlinkSync(externalBin, join(currentRoot, "node_modules", ".bin", "external-tool")); + let probed = false; + + expect(await findNpmRecoveryLaunchers( + join(currentRoot, "bin", "ocx.mjs"), + "2.7.40", + async () => { + probed = true; + return true; + }, + )).toEqual([]); + expect(probed).toBe(false); + }); + test("skips a matching current package whose complete launcher runtime cannot load", async () => { const scopeRoot = join(dir, "global", "@bitkyc08"); const currentRoot = join(scopeRoot, "opencodex"); @@ -891,6 +943,53 @@ describe("GUI update execution decisions", () => { expect(readUpdateJob(job.id)?.log.some(line => line.includes("trying candidate 2 of 2"))).toBe(true); }); + test("failed install stops trying candidates when a started PID has no OpenCodex identity", async () => { + let now = 0; + const attempted: Array = []; + const killed: number[] = []; + const job: UpdateJobState = { + id: "failed-install-unverified-pid", + status: "running", + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + currentVersion: "2.7.40", + latestVersion: "2.7.41", + channel: "latest", + installer: "npm", + restart: true, + command: "", + releaseNotesUrl: "", + log: [], + }; + writeFileSync(updateJobPath(), JSON.stringify(job)); + + const recovery = await recoverFailedGuiUpdateForTests( + job, + { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, + true, + { + probeProxyIdentity: async () => null, + verifyPidIdentityFn: () => null, + recoveryLaunchersFn: () => ["/current/bin/ocx.mjs", "/retired/bin/ocx.mjs"], + probeProxy: async () => false, + restartAfterUpdateFn: async (_job, captured) => { + attempted.push(captured?.recoveryLauncher); + return 222; + }, + killProxyFn: pid => { killed.push(pid); }, + now: () => now, + sleepMs: async (ms) => { now += ms; }, + }, + ); + + expect(recovery).toBe("failed"); + expect(attempted).toEqual(["/current/bin/ocx.mjs"]); + expect(killed).toEqual([]); + expect(readUpdateJob(job.id)?.log.some(line => + line.includes("without a matching OpenCodex identity"), + )).toBe(true); + }); + test("failed install reports remediation when no runnable recovery package remains", async () => { let restartCalls = 0; const job: UpdateJobState = { From eb9f89fb7f1bccbda550eb70822c5fae59d8454d Mon Sep 17 00:00:00 2001 From: wzb Date: Mon, 27 Jul 2026 19:36:42 +0800 Subject: [PATCH 13/16] fix(update): harden recovery scan boundaries --- docs/adr/0001-gui-update-worker.md | 10 +- src/update/job.ts | 102 ++++++++----------- src/update/npm-cache-preflight.d.mts | 1 + src/update/npm-cache-preflight.mjs | 13 ++- src/update/recovery-tree-scan.mjs | 124 +++++++++++++++++++++++ tests/update-install-process.test.ts | 10 ++ tests/update-job.test.ts | 19 ++++ tests/update-npm-cache-preflight.test.ts | 13 ++- 8 files changed, 223 insertions(+), 69 deletions(-) create mode 100644 src/update/recovery-tree-scan.mjs diff --git a/docs/adr/0001-gui-update-worker.md b/docs/adr/0001-gui-update-worker.md index e34a18951..1bc6aa592 100644 --- a/docs/adr/0001-gui-update-worker.md +++ b/docs/adr/0001-gui-update-worker.md @@ -26,8 +26,10 @@ Before an npm updater stops the proxy, it resolves the configured npm cache and entry is owned by the current Unix user. A foreign-owned entry (commonly left by an older `sudo npm` invocation) aborts the update with an actionable error while the existing proxy and service remain running. The scan fails closed when the cache root is absent, an entry cannot be inspected, or its -50,000-entry / 10-second traversal budget is exhausted. The blocking filesystem walk runs in a -child process so the parent can enforce the deadline even when a filesystem call itself stalls. +50,000-entry / 10-second traversal budget is exhausted. Only the configured cache-root symlink is +canonicalized; nested cache symlinks are rejected fail-closed so a foreign-owned target cannot be +hidden behind one. The blocking filesystem walk runs in a child process so the parent can enforce +the deadline even when a filesystem call itself stalls. Failure logs omit both cache and entry paths; users can locate the cache explicitly with `npm config get cache`. Platforms without Unix uid ownership skip this gate. @@ -43,7 +45,9 @@ entries, and symlinks that leave the candidate tree before any code executes. Tr links whose immediate and final targets remain inside the candidate tree are allowed. The tree walk is bounded by entry count and elapsed time. Candidate inspection and restart are bounded to at most two candidates, preferring the current package and then the newest retired copies, and it tries those runnable -candidates in order until one restores health. A recovery launcher is always started directly: it +candidates in order until one restores health. The recovery-tree walk runs in its own short-lived +worker, and the GUI worker force-kills that child at the wall-clock deadline even if one filesystem +call blocks. A recovery launcher is always started directly: it may restore availability, but its potentially temporary path is never persisted into launchd, systemd, or Task Scheduler. If a candidate starts but does not become healthy, the worker records its PID, revalidates its OpenCodex identity, and stops it before trying the next candidate. diff --git a/src/update/job.ts b/src/update/job.ts index b5dfe8e0d..567d675ab 100644 --- a/src/update/job.ts +++ b/src/update/job.ts @@ -1,5 +1,5 @@ import { spawn, spawnSync } from "node:child_process"; -import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, readlinkSync, realpathSync } from "node:fs"; +import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync } from "node:fs"; import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { @@ -52,6 +52,7 @@ const UPDATE_LIVENESS_PROBE_DELAY_MS = 250; const MAX_NPM_RECOVERY_CANDIDATES = 2; const MAX_NPM_RECOVERY_TREE_ENTRIES = 50_000; const MAX_NPM_RECOVERY_TREE_SCAN_MS = 5_000; +const RECOVERY_TREE_SCAN_WORKER_ARG = "__scan-recovery-tree"; /** How long update restart waits for the captured port to become bindable after stop. */ export const RESTART_PORT_RECLAIM_MS = 30_000; @@ -146,68 +147,49 @@ function hasTrustedRecoveryPermissions(stat: { uid: number; mode: number }): boo return process.getuid === undefined || (stat.mode & 0o022) === 0; } -function hasTrustedRecoveryPath(packageRoot: string): boolean { - let path = packageRoot; - while (true) { - const stat = lstatSync(path); - if ( - !stat.isDirectory() - || stat.isSymbolicLink() - || !hasTrustedRecoveryOwner(stat.uid) - || ( - process.getuid !== undefined - && (stat.mode & 0o022) !== 0 - && (stat.mode & 0o1000) === 0 - ) - ) return false; - const parent = dirname(path); - if (parent === path) return true; - path = parent; - } +interface RecoveryTreeScanOptions { + scanSpawn?: typeof spawnSync; + scanBin?: string; + scanScript?: string; + timeoutMs?: number; } -function hasTrustedRecoveryTree(packageRoot: string): boolean { - const canonicalRoot = realpathSync(packageRoot); - const scopeRoot = dirname(canonicalRoot); - const scopeStat = lstatSync(scopeRoot); - if ( - !scopeStat.isDirectory() - || scopeStat.isSymbolicLink() - || !hasTrustedRecoveryPermissions(scopeStat) - || !hasTrustedRecoveryPath(canonicalRoot) - ) return false; - - const deadline = Date.now() + MAX_NPM_RECOVERY_TREE_SCAN_MS; - const pending = [canonicalRoot]; - const visited = new Set(); - let inspected = 0; - while (pending.length > 0) { - if (Date.now() > deadline || inspected >= MAX_NPM_RECOVERY_TREE_ENTRIES) return false; - const path = pending.pop()!; - inspected += 1; - const entryStat = lstatSync(path); - let canonicalPath = path; - let canonicalStat = entryStat; - if (entryStat.isSymbolicLink()) { - // npm creates node_modules/.bin links. Permit only trusted-owner links whose - // immediate and final targets both remain inside this candidate package. - if (!hasTrustedRecoveryOwner(entryStat.uid)) return false; - const directTarget = resolve(dirname(path), readlinkSync(path)); - if (!isPathInside(canonicalRoot, directTarget)) return false; - canonicalPath = realpathSync(path); - if (!isPathInside(canonicalRoot, canonicalPath)) return false; - canonicalStat = lstatSync(canonicalPath); - } - if (!hasTrustedRecoveryPermissions(canonicalStat)) return false; - if (visited.has(canonicalPath)) continue; - visited.add(canonicalPath); - if (canonicalStat.isFile()) continue; - if (!canonicalStat.isDirectory()) return false; - for (const name of readdirSync(canonicalPath, { encoding: "utf8" })) { - pending.push(join(canonicalPath, name)); - } +function hasTrustedRecoveryTree(packageRoot: string, options: RecoveryTreeScanOptions = {}): boolean { + const timeoutMs = Number.isFinite(options.timeoutMs) && options.timeoutMs! > 0 + ? Math.trunc(options.timeoutMs!) + : MAX_NPM_RECOVERY_TREE_SCAN_MS; + const scanSpawn = options.scanSpawn ?? spawnSync; + try { + const result = scanSpawn( + options.scanBin ?? process.execPath, + [ + options.scanScript ?? fileURLToPath(new URL("./recovery-tree-scan.mjs", import.meta.url)), + RECOVERY_TREE_SCAN_WORKER_ARG, + ], + { + encoding: "utf8", + input: JSON.stringify({ + packageRoot, + maxEntries: MAX_NPM_RECOVERY_TREE_ENTRIES, + maxDurationMs: timeoutMs, + }), + timeout: timeoutMs, + killSignal: "SIGKILL", + windowsHide: true, + shell: false, + }, + ); + return result.status === 0 && String(result.stdout ?? "").trim() === "1"; + } catch { + return false; } - return true; +} + +export function scanTrustedRecoveryTreeForTests( + packageRoot: string, + options: RecoveryTreeScanOptions = {}, +): boolean { + return hasTrustedRecoveryTree(packageRoot, options); } /** Validate the on-disk identity before any candidate code is executed. */ diff --git a/src/update/npm-cache-preflight.d.mts b/src/update/npm-cache-preflight.d.mts index 2a6eed893..2f8bdcd14 100644 --- a/src/update/npm-cache-preflight.d.mts +++ b/src/update/npm-cache-preflight.d.mts @@ -1,6 +1,7 @@ export interface NpmCacheEntryStat { uid?: number; isDirectory(): boolean; + isSymbolicLink(): boolean; } export interface NpmCachePreflightOptions { diff --git a/src/update/npm-cache-preflight.mjs b/src/update/npm-cache-preflight.mjs index 7cf51ebea..5cb7b4978 100644 --- a/src/update/npm-cache-preflight.mjs +++ b/src/update/npm-cache-preflight.mjs @@ -30,8 +30,8 @@ export function findForeignOwnedNpmCacheEntry(cachePath, expectedUid, io = {}) { const startedAt = now(); let cacheRoot; try { - // npm follows a configured cache-root symlink. Resolve only that root; nested - // symlinks remain lstat-only and are never traversed. + // npm follows a configured cache-root symlink. Resolve only that root; + // nested symlinks are rejected below and never traversed. cacheRoot = realpath(resolve(cachePath)); } catch (error) { if (errorCode(error) === "ENOENT") { @@ -81,6 +81,15 @@ export function findForeignOwnedNpmCacheEntry(cachePath, expectedUid, io = {}) { if (Number.isInteger(stat.uid) && stat.uid !== expectedUid) { return { kind: "foreign-owner", path, actualUid: stat.uid }; } + if (stat.isSymbolicLink()) { + // Only the configured cache root is canonicalized. Traversing a nested + // link could hide a foreign-owned target, so fail closed instead. + return { + kind: "error", + path, + reason: "npm cache contains a nested symbolic link", + }; + } if (path === cacheRoot && !stat.isDirectory()) { return { kind: "error", path, reason: "npm cache root is not a directory" }; } diff --git a/src/update/recovery-tree-scan.mjs b/src/update/recovery-tree-scan.mjs new file mode 100644 index 000000000..c8e7659b4 --- /dev/null +++ b/src/update/recovery-tree-scan.mjs @@ -0,0 +1,124 @@ +import { lstatSync, readFileSync, readdirSync, readlinkSync, realpathSync } from "node:fs"; +import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; + +export const DEFAULT_MAX_RECOVERY_TREE_ENTRIES = 50_000; +export const DEFAULT_MAX_RECOVERY_TREE_SCAN_MS = 5_000; +export const RECOVERY_TREE_SCAN_WORKER_ARG = "__scan-recovery-tree"; + +function isPathInside(parent, child) { + const fromParent = relative(parent, child); + return fromParent !== "" + && fromParent !== ".." + && !fromParent.startsWith(`..${sep}`) + && !isAbsolute(fromParent); +} + +function hasTrustedRecoveryOwner(uid) { + const currentUid = process.getuid?.(); + return currentUid === undefined || uid === currentUid || uid === 0; +} + +function hasTrustedRecoveryPermissions(stat) { + if (!hasTrustedRecoveryOwner(stat.uid)) return false; + return process.getuid === undefined || (stat.mode & 0o022) === 0; +} + +function hasTrustedRecoveryPath(packageRoot) { + let path = packageRoot; + while (true) { + const stat = lstatSync(path); + if ( + !stat.isDirectory() + || stat.isSymbolicLink() + || !hasTrustedRecoveryOwner(stat.uid) + || ( + process.getuid !== undefined + && (stat.mode & 0o022) !== 0 + && (stat.mode & 0o1000) === 0 + ) + ) return false; + const parent = dirname(path); + if (parent === path) return true; + path = parent; + } +} + +/** + * Scan a candidate package tree without executing any package code. The caller + * runs this in a short-lived worker so a stalled filesystem syscall is bounded + * by the parent process as well as by the between-entry budget below. + */ +export function scanTrustedRecoveryTree( + packageRoot, + { + maxEntries = DEFAULT_MAX_RECOVERY_TREE_ENTRIES, + maxDurationMs = DEFAULT_MAX_RECOVERY_TREE_SCAN_MS, + } = {}, +) { + try { + const canonicalRoot = realpathSync(packageRoot); + const scopeRoot = dirname(canonicalRoot); + const scopeStat = lstatSync(scopeRoot); + if ( + !scopeStat.isDirectory() + || scopeStat.isSymbolicLink() + || !hasTrustedRecoveryPermissions(scopeStat) + || !hasTrustedRecoveryPath(canonicalRoot) + ) return false; + + const entryLimit = Number.isFinite(maxEntries) && maxEntries > 0 + ? Math.trunc(maxEntries) + : DEFAULT_MAX_RECOVERY_TREE_ENTRIES; + const durationLimit = Number.isFinite(maxDurationMs) && maxDurationMs > 0 + ? Math.trunc(maxDurationMs) + : DEFAULT_MAX_RECOVERY_TREE_SCAN_MS; + const deadline = Date.now() + durationLimit; + const pending = [canonicalRoot]; + const visited = new Set(); + let inspected = 0; + while (pending.length > 0) { + if (Date.now() > deadline || inspected >= entryLimit) return false; + const path = pending.pop(); + inspected += 1; + const entryStat = lstatSync(path); + let canonicalPath = path; + let canonicalStat = entryStat; + if (entryStat.isSymbolicLink()) { + // npm creates node_modules/.bin links. Permit only trusted-owner links + // whose immediate and final targets remain inside this package. + if (!hasTrustedRecoveryOwner(entryStat.uid)) return false; + const directTarget = resolve(dirname(path), readlinkSync(path)); + if (!isPathInside(canonicalRoot, directTarget)) return false; + canonicalPath = realpathSync(path); + if (!isPathInside(canonicalRoot, canonicalPath)) return false; + canonicalStat = lstatSync(canonicalPath); + } + if (!hasTrustedRecoveryPermissions(canonicalStat)) return false; + if (visited.has(canonicalPath)) continue; + visited.add(canonicalPath); + if (canonicalStat.isFile()) continue; + if (!canonicalStat.isDirectory()) return false; + for (const name of readdirSync(canonicalPath, { encoding: "utf8" })) { + pending.push(join(canonicalPath, name)); + } + } + return true; + } catch { + return false; + } +} + +function runWorker() { + try { + const payload = JSON.parse(readFileSync(0, "utf8")); + if (typeof payload?.packageRoot !== "string") { + process.stdout.write("0\n"); + return; + } + process.stdout.write(scanTrustedRecoveryTree(payload.packageRoot, payload) ? "1\n" : "0\n"); + } catch { + process.stdout.write("0\n"); + } +} + +if (process.argv[2] === RECOVERY_TREE_SCAN_WORKER_ARG) runWorker(); diff --git a/tests/update-install-process.test.ts b/tests/update-install-process.test.ts index d209ee93d..8fafa3684 100644 --- a/tests/update-install-process.test.ts +++ b/tests/update-install-process.test.ts @@ -15,6 +15,16 @@ const cleanupDirs = new Set(); function isRunning(pid: number): boolean { try { + if (process.platform === "linux") { + // Linux keeps a killed orphan as a zombie until its parent (often PID 1) + // reaps it. `kill(pid, 0)` still succeeds for that interval, so inspect + // the proc state before treating the descendant as live. + const stat = readFileSync(`/proc/${pid}/stat`, "utf8"); + const closingParen = stat.lastIndexOf(")"); + if (closingParen >= 0 && stat.slice(closingParen + 2, closingParen + 3) === "Z") { + return false; + } + } process.kill(pid, 0); return true; } catch { diff --git a/tests/update-job.test.ts b/tests/update-job.test.ts index 4e19c6b5f..517f61a87 100644 --- a/tests/update-job.test.ts +++ b/tests/update-job.test.ts @@ -15,6 +15,7 @@ import { recoverFailedGuiUpdateForTests, restartCommand, restartAfterUpdateForTests, + scanTrustedRecoveryTreeForTests, startUpdateJob, updateExecutionCommand, updateJobPath, @@ -293,6 +294,24 @@ describe("GUI update execution decisions", () => { expect(launchers[0]).toBe(realpathSync(join(currentRoot, "bin", "ocx.mjs"))); }); + test("fails closed when the recovery-tree worker exceeds its hard deadline", () => { + const blockingScan = join(dir, "blocking-recovery-tree-scan.mjs"); + writeFileSync(blockingScan, [ + "process.on('SIGTERM', () => {});", + "setInterval(() => {}, 60_000);", + "", + ].join("\n")); + + const startedAt = Date.now(); + const result = scanTrustedRecoveryTreeForTests(join(dir, "candidate"), { + scanScript: blockingScan, + timeoutMs: 250, + }); + + expect(result).toBe(false); + expect(Date.now() - startedAt).toBeLessThan(2_000); + }, 5_000); + test("only recovers after a clean installer exit", () => { expect(installerFailureAllowsRecovery("npm", { status: 1, signal: null, timedOut: false })).toBe(true); expect(installerFailureAllowsRecovery("npm", { status: 75, signal: null, timedOut: false })).toBe(false); diff --git a/tests/update-npm-cache-preflight.test.ts b/tests/update-npm-cache-preflight.test.ts index dc79c78e2..a67f40feb 100644 --- a/tests/update-npm-cache-preflight.test.ts +++ b/tests/update-npm-cache-preflight.test.ts @@ -54,6 +54,7 @@ describe("npm cache ownership pre-flight", () => { return { uid: path === foreign ? uid + 1 : stat.uid, isDirectory: () => stat.isDirectory(), + isSymbolicLink: () => stat.isSymbolicLink(), }; }, readdir: path => readdirSync(path, { encoding: "utf8" }), @@ -61,7 +62,7 @@ describe("npm cache ownership pre-flight", () => { expect(issue).toEqual({ kind: "foreign-owner", path: foreign, actualUid: uid + 1 }); }); - test("follows a configured cache-root symlink but not nested symlinks", () => { + test("follows a configured cache-root symlink but rejects nested symlinks", () => { const uid = process.getuid?.(); if (uid === undefined) return; const cacheLink = `${dir}-link`; @@ -71,7 +72,6 @@ describe("npm cache ownership pre-flight", () => { writeFileSync(join(outside, "foreign"), "outside"); symlinkSync(dir, cacheLink, "dir"); symlinkSync(outside, join(dir, "nested-link"), "dir"); - const foreign = join(realpathSync(dir), "_cacache", "index-v5", "entry"); const outsideRoot = realpathSync(outside); let inspectedOutside = false; const issue = findForeignOwnedNpmCacheEntry(cacheLink, uid, { @@ -79,13 +79,18 @@ describe("npm cache ownership pre-flight", () => { if (path.startsWith(outsideRoot)) inspectedOutside = true; const stat = lstatSync(path); return { - uid: path === foreign ? uid + 1 : stat.uid, + uid: stat.uid, isDirectory: () => stat.isDirectory(), + isSymbolicLink: () => stat.isSymbolicLink(), }; }, readdir: path => readdirSync(path, { encoding: "utf8" }), }); - expect(issue).toEqual({ kind: "foreign-owner", path: foreign, actualUid: uid + 1 }); + expect(issue).toEqual({ + kind: "error", + path: join(realpathSync(dir), "nested-link"), + reason: "npm cache contains a nested symbolic link", + }); expect(inspectedOutside).toBe(false); }); From 8dfe46d31754bbf14aab2d166daca47f486ff764 Mon Sep 17 00:00:00 2001 From: wzb Date: Mon, 27 Jul 2026 20:23:19 +0800 Subject: [PATCH 14/16] fix(update): close recovery review gaps --- docs/adr/0001-gui-update-worker.md | 12 ++- src/config.ts | 36 ++++++-- src/update/job.ts | 122 ++++++++++++++++++++++------ src/update/recovery-tree-scan.d.mts | 13 +++ tests/config.test.ts | 13 +++ tests/update-job.test.ts | 90 ++++++++++++++++++-- tests/update-stop-first.test.ts | 8 ++ 7 files changed, 250 insertions(+), 44 deletions(-) create mode 100644 src/update/recovery-tree-scan.d.mts diff --git a/docs/adr/0001-gui-update-worker.md b/docs/adr/0001-gui-update-worker.md index 1bc6aa592..4b9d8e299 100644 --- a/docs/adr/0001-gui-update-worker.md +++ b/docs/adr/0001-gui-update-worker.md @@ -19,8 +19,9 @@ detached hidden CLI worker, and the worker performs the install command and opti For npm installs, the worker runs the Node launcher path (`node bin/ocx.mjs update --tag `) so the existing npm self-update guard is reused. For Bun global installs, it runs the existing Bun -global update command. Source checkouts remain manual-only and show `git pull && bun install && -bun run build:gui`. +global update command. The outer GUI installer command also runs through the shared process-tree +runner; a failed npm or Bun update is recoverable only when that runner confirms the installer tree +has exited. Source checkouts remain manual-only and show `git pull && bun install && bun run build:gui`. Before an npm updater stops the proxy, it resolves the configured npm cache and checks that every entry is owned by the current Unix user. A foreign-owned entry (commonly left by an older `sudo npm` @@ -49,8 +50,11 @@ candidates in order until one restores health. The recovery-tree walk runs in it worker, and the GUI worker force-kills that child at the wall-clock deadline even if one filesystem call blocks. A recovery launcher is always started directly: it may restore availability, but its potentially temporary path is never persisted into launchd, -systemd, or Task Scheduler. If a candidate starts but does not become healthy, the worker records -its PID, revalidates its OpenCodex identity, and stops it before trying the next candidate. +systemd, Task Scheduler, or the update job log; the log stores only a path-free candidate label. +If a candidate starts but does not become healthy, the worker retains the detached child handle, +then requires both the same process generation and a fresh OpenCodex identity before stopping that +PID and trying the next candidate. An exited child or missing generation proof fails closed so PID +reuse can never terminate a concurrent replacement proxy. Service and direct restart paths also stop when the pidfile has changed to a different identity-checked proxy, and every such identity decision re-reads the process command line instead of trusting a PID-only cache across the update boundary. The update job remains failed either way because restoring diff --git a/src/config.ts b/src/config.ts index c0820868b..73d8103df 100644 --- a/src/config.ts +++ b/src/config.ts @@ -1225,23 +1225,41 @@ export function isOcxStartCommandLine(commandLine: string): boolean { return hasOcxEntrypoint && /(?:^|[\s"'])start(?:$|[\s"'])/.test(normalized); } -function isLikelyOcxStartProcessUncached(pid: number): boolean { +function isLikelyOcxStartProcessUncached(pid: number): boolean | undefined { const commandLine = readProcessCommandLine(pid); - if (commandLine === undefined) return false; + if (commandLine === undefined) return undefined; return isOcxStartCommandLine(commandLine); } /** Per-process memo: waitForProxy/findLiveProxy used to spawn powershell on every 150ms poll. */ const ocxStartProcessCache = new Map(); -function isLikelyOcxStartProcess(pid: number): boolean { - const cached = ocxStartProcessCache.get(pid); +function lookupOcxStartProcess( + pid: number, + readCommandLine: (pid: number) => string | undefined, + cache: Map, +): boolean | undefined { + const cached = cache.get(pid); if (cached !== undefined) return cached; - const ok = isLikelyOcxStartProcessUncached(pid); - ocxStartProcessCache.set(pid, ok); + const commandLine = readCommandLine(pid); + if (commandLine === undefined) return undefined; + const ok = isOcxStartCommandLine(commandLine); + cache.set(pid, ok); return ok; } +function isLikelyOcxStartProcess(pid: number): boolean | undefined { + return lookupOcxStartProcess(pid, readProcessCommandLine, ocxStartProcessCache); +} + +export function lookupOcxStartProcessForTests( + pid: number, + readCommandLine: (pid: number) => string | undefined, + cache = new Map(), +): boolean | undefined { + return lookupOcxStartProcess(pid, readCommandLine, cache); +} + /** * Alive pid from the pid file without the expensive Windows command-line probe. * Safe for liveness polls: callers still identity-check /healthz before trusting the proxy. @@ -1272,7 +1290,7 @@ export function verifyPidIdentity(candidatePid: number): number | null { } catch (e: unknown) { if ((e as NodeJS.ErrnoException).code !== "EPERM") return null; } - return isLikelyOcxStartProcess(candidatePid) ? candidatePid : null; + return isLikelyOcxStartProcess(candidatePid) === true ? candidatePid : null; } /** @@ -1289,6 +1307,10 @@ export function verifyPidIdentityFresh(candidatePid: number): number | null { } } const isOcx = isLikelyOcxStartProcessUncached(candidatePid); + if (isOcx === undefined) { + ocxStartProcessCache.delete(candidatePid); + return null; + } // This read is authoritative across a PID-reuse boundary. Reconcile the // polling memo so later short-lived checks cannot serve the old identity. ocxStartProcessCache.set(candidatePid, isOcx); diff --git a/src/update/job.ts b/src/update/job.ts index 567d675ab..929eb25b8 100644 --- a/src/update/job.ts +++ b/src/update/job.ts @@ -1,4 +1,4 @@ -import { spawn, spawnSync } from "node:child_process"; +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; import { existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, realpathSync } from "node:fs"; import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; @@ -39,6 +39,7 @@ import { INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE, runProcessTreeCommand, } from "./install-process.mjs"; +import { RECOVERY_TREE_SCAN_WORKER_ARG } from "./recovery-tree-scan.mjs"; import { handoffWindowsTrayForUpdate, planWindowsTrayUpdate } from "./tray-update-plan.mjs"; const RELEASE_NOTES_URL = "https://github.com/lidge-jun/opencodex/releases/latest"; @@ -52,7 +53,6 @@ const UPDATE_LIVENESS_PROBE_DELAY_MS = 250; const MAX_NPM_RECOVERY_CANDIDATES = 2; const MAX_NPM_RECOVERY_TREE_ENTRIES = 50_000; const MAX_NPM_RECOVERY_TREE_SCAN_MS = 5_000; -const RECOVERY_TREE_SCAN_WORKER_ARG = "__scan-recovery-tree"; /** How long update restart waits for the captured port to become bindable after stop. */ export const RESTART_PORT_RECLAIM_MS = 30_000; @@ -462,6 +462,10 @@ interface LoggedCommandResult { timedOut: boolean; } +interface InstallerCommandResult extends LoggedCommandResult { + treeExited: boolean; +} + function runLoggedCommand(job: UpdateJobState, bin: string, args: string[], timeout: number): LoggedCommandResult { job = updateJob(job, {}, `$ ${formatCommand(bin, args)}`); const result = spawnSync(bin, args, { @@ -480,30 +484,93 @@ function runLoggedCommand(job: UpdateJobState, bin: string, args: string[], time }; } -export function installerFailureAllowsRecovery(installer: Installer, result: LoggedCommandResult): boolean { - return !result.timedOut +async function runLoggedProcessTreeCommand( + job: UpdateJobState, + bin: string, + args: string[], + timeout: number, +): Promise { + job = updateJob(job, {}, `$ ${formatCommand(bin, args)}`); + const result = await runProcessTreeCommand(bin, args, { + stdio: "pipe", + timeoutMs: timeout, + windowsHide: true, + shell: false, + }); + const stdout = typeof result.stdout === "string" ? result.stdout.trim() : ""; + const stderr = typeof result.stderr === "string" ? result.stderr.trim() : ""; + if (stdout) job = updateJob(job, {}, stdout.slice(-4000)); + if (stderr) job = updateJob(job, {}, stderr.slice(-4000)); + return { + status: result.status, + signal: result.signal, + timedOut: result.timedOut, + treeExited: result.treeExited, + }; +} + +export function installerFailureAllowsRecovery(installer: Installer, result: InstallerCommandResult): boolean { + return result.treeExited + && !result.timedOut && result.signal === null && (installer !== "npm" || result.status !== INSTALLER_TREE_CLEANUP_FAILED_EXIT_CODE); } +interface StartedRecoveryProcess { + pid: number; + /** True only while the detached ChildProcess handle still represents this start. */ + sameGeneration: () => boolean; +} + +type RestartStartResult = StartedRecoveryProcess | number | null | void; + +function normalizeStartedRecoveryProcess(result: RestartStartResult): StartedRecoveryProcess | null { + if (typeof result === "number") { + if (!Number.isSafeInteger(result) || result <= 0) return null; + // Numeric results are retained for test seams and older callers, but carry no + // generation proof. Recovery therefore fails closed before attempting a kill. + return { pid: result, sameGeneration: () => false }; + } + if (!result || !Number.isSafeInteger(result.pid) || result.pid <= 0) return null; + return typeof result.sameGeneration === "function" ? result : null; +} + +export function formatProxyStartLog(installer: Installer, launcher: string, port?: number): string { + const candidateLabel = launcher === packageLauncherPath() ? "current package" : "validated recovery candidate"; + const portLabel = typeof port === "number" && Number.isFinite(port) && port > 0 + ? ` on port ${Math.trunc(port)}` + : ""; + return `Starting ${installer} proxy from ${candidateLabel}${portLabel}.`; +} + function spawnDetachedStart( job: UpdateJobState, installer: Installer, port?: number, launcher = packageLauncherPath(), -): number | null { +): StartedRecoveryProcess | null { const cmd = restartCommand(false, installer, launcher, port); const env = { ...process.env }; delete env.OCX_SERVICE; - updateJob(job, {}, `$ ${cmd.display}`); - const child = spawn(cmd.bin, cmd.args, { + // Do not persist cmd.display here: a retired npm launcher contains the absolute + // user home path and update-job.json is returned by the management API. + updateJob(job, {}, formatProxyStartLog(installer, launcher, port)); + const child: ChildProcess = spawn(cmd.bin, cmd.args, { detached: true, stdio: "ignore", windowsHide: true, env, }); + const pid = child.pid; + if (typeof pid !== "number" || !Number.isSafeInteger(pid) || pid <= 0) return null; + let exited = false; + child.once("exit", () => { exited = true; }); + child.once("error", () => { exited = true; }); child.unref(); - return child.pid ?? null; + return { + pid, + sameGeneration: () => !exited && child.exitCode === null && child.signalCode === null, + }; } /** Identity snapshot used to prove an npm self-update actually replaced the pre-update process. */ @@ -517,7 +584,7 @@ export type FailedUpdateRecovery = "not-needed" | "still-running" | "restarted" /** Test seam: the wait/spawn pair is injectable so the restart path is verifiable. */ export interface RestartIo { waitForPort?: typeof reclaimListenPort; - spawnStart?: (job: UpdateJobState, installer: Installer, port?: number, launcher?: string) => number | null | void; + spawnStart?: (job: UpdateJobState, installer: Installer, port?: number, launcher?: string) => RestartStartResult; serviceInstalledFn?: () => boolean; readPidFn?: () => number | null; probeProxy?: (port: number, hostname?: string) => Promise; @@ -539,14 +606,14 @@ export interface RestartIo { job: UpdateJobState, captured?: { port: number; hostname: string; oldPid?: number; recoveryLauncher?: string }, io?: RestartIo, - ) => Promise; + ) => Promise; } async function restartAfterUpdate( job: UpdateJobState, captured?: { port: number; hostname: string; oldPid?: number; recoveryLauncher?: string }, io: RestartIo = {}, -): Promise { +): Promise { const serviceInstalled = (io.serviceInstalledFn ?? isServiceInstalled)(); // A failed npm install may leave only a hidden, npm-retired package runnable. // It can restore availability, but must never be baked into launchd/systemd. @@ -642,10 +709,9 @@ async function restartAfterUpdate( return null; } if (readPidForRestart("after direct port reclaim").refused) return null; - const startedPid = (io.spawnStart ?? spawnDetachedStart)(job, job.installer, port, launcher); - return typeof startedPid === "number" && Number.isSafeInteger(startedPid) && startedPid > 0 - ? startedPid - : null; + return normalizeStartedRecoveryProcess( + (io.spawnStart ?? spawnDetachedStart)(job, job.installer, port, launcher), + ); } /** Exposed for tests: drives the non-service restart path with injected io. */ @@ -654,7 +720,7 @@ export function restartAfterUpdateForTests( captured: { port: number; hostname: string; oldPid?: number; recoveryLauncher?: string }, io: RestartIo, ): Promise { - return restartAfterUpdate(job, captured, io); + return restartAfterUpdate(job, captured, io).then(started => started?.pid ?? null); } function restartFailureHint(port: number): string { @@ -870,23 +936,29 @@ async function recoverFailedGuiUpdate( updateJob(job, {}, `Recovery candidate ${index} did not restore the proxy; trying candidate ${index + 1} of ${recoveryLaunchers.length}.`); } try { - const startedPid = await restartFn(job, recoveryCaptured, io); + const started = normalizeStartedRecoveryProcess(await restartFn(job, recoveryCaptured, io)); const healthy = await awaitRestartedProxyHealthy(job, recoveryCaptured, io); if (healthy.ok) { updateJob(job, {}, `Previous proxy availability restored on ${captured.hostname}:${captured.port}; the update itself still failed.`); return "restarted"; } - if (typeof startedPid === "number" && Number.isSafeInteger(startedPid) && startedPid > 0) { + if (started) { + let sameGeneration = false; + try { sameGeneration = started.sameGeneration(); } catch { /* fail closed */ } + if (!sameGeneration) { + updateJob(job, {}, `Recovery candidate ${index + 1} PID ${started.pid} no longer matches the spawned process generation; refusing to kill or try another candidate.`); + break; + } const verifyStartedPid = io.verifyPidIdentityFn ?? verifyPidIdentityFresh; - if (verifyStartedPid(startedPid) !== startedPid) { - updateJob(job, {}, `Recovery candidate ${index + 1} left PID ${startedPid} without a matching OpenCodex identity; refusing to kill or try another candidate.`); + if (verifyStartedPid(started.pid) !== started.pid) { + updateJob(job, {}, `Recovery candidate ${index + 1} left PID ${started.pid} without a matching OpenCodex identity; refusing to kill or try another candidate.`); break; } try { - (io.killProxyFn ?? killProxy)(startedPid); - updateJob(job, {}, `Stopped unhealthy recovery candidate ${index + 1} PID ${startedPid}.`); + (io.killProxyFn ?? killProxy)(started.pid); + updateJob(job, {}, `Stopped unhealthy recovery candidate ${index + 1} PID ${started.pid}.`); } catch (error) { - updateJob(job, {}, `Could not stop unhealthy recovery candidate ${index + 1} PID ${startedPid}: ${error instanceof Error ? error.message : String(error)}`); + updateJob(job, {}, `Could not stop unhealthy recovery candidate ${index + 1} PID ${started.pid}: ${error instanceof Error ? error.message : String(error)}`); break; } } @@ -1152,8 +1224,8 @@ export async function runGuiUpdateWorker(jobId: string, channel: Channel, restar - 대안 분석: (1) 서버에서 runUpdate 직접 호출: process.exit/stdio/실행 파일 교체 위험. (2) GUI에서 CLI 명령 안내만 제공: 자동 업데이트 UX 부족. (3) 숨은 worker가 Node launcher/Bun 전역 명령을 실행: 상태 추적과 안전한 재시작이 가능. - 선택 근거: 현재 CLI의 npm self-update 우회를 재사용하면서도 GUI 서버 요청 생명주기와 설치 작업을 분리할 수 있어 가장 안정적이다. */ - const result = runLoggedCommand(job, cmd.bin, cmd.args, UPDATE_TIMEOUT_MS); - if (result.status !== 0) { + const result = await runLoggedProcessTreeCommand(job, cmd.bin, cmd.args, UPDATE_TIMEOUT_MS); + if (result.status !== 0 || !result.treeExited) { const mayRecover = installerFailureAllowsRecovery(check.installer, result); if (trayWasRunning && mayRecover) { try { diff --git a/src/update/recovery-tree-scan.d.mts b/src/update/recovery-tree-scan.d.mts new file mode 100644 index 000000000..db784da8b --- /dev/null +++ b/src/update/recovery-tree-scan.d.mts @@ -0,0 +1,13 @@ +export const DEFAULT_MAX_RECOVERY_TREE_ENTRIES: number; +export const DEFAULT_MAX_RECOVERY_TREE_SCAN_MS: number; +export const RECOVERY_TREE_SCAN_WORKER_ARG: string; + +export interface RecoveryTreeScanOptions { + maxEntries?: number; + maxDurationMs?: number; +} + +export function scanTrustedRecoveryTree( + packageRoot: string, + options?: RecoveryTreeScanOptions, +): boolean; diff --git a/tests/config.test.ts b/tests/config.test.ts index 6ca101106..bbbd0b656 100644 --- a/tests/config.test.ts +++ b/tests/config.test.ts @@ -13,6 +13,7 @@ import { isValidProviderName, isOcxStartCommandLine, loadConfig, + lookupOcxStartProcessForTests, multiAgentGuidanceEnabled, parsePidFile, positiveIntegerConfigError, @@ -894,6 +895,18 @@ describe("opencodex config defaults", () => { expect(freshCheck).toContain("ocxStartProcessCache.delete(candidatePid)"); }); + test("retries a transient command-line lookup instead of caching unknown as false", () => { + const cache = new Map(); + let reads = 0; + const readCommandLine = () => (++reads === 1 ? undefined : "opencodex start"); + + expect(lookupOcxStartProcessForTests(4242, readCommandLine, cache)).toBeUndefined(); + expect(cache.has(4242)).toBe(false); + expect(lookupOcxStartProcessForTests(4242, readCommandLine, cache)).toBe(true); + expect(reads).toBe(2); + expect(cache.get(4242)).toBe(true); + }); + test("writes pid file as a numeric pid", () => { writePid(process.pid); diff --git a/tests/update-job.test.ts b/tests/update-job.test.ts index 517f61a87..780a4f510 100644 --- a/tests/update-job.test.ts +++ b/tests/update-job.test.ts @@ -9,6 +9,7 @@ import { findNpmRecoveryLauncher, findNpmRecoveryLaunchers, finishGuiUpdateRestart, + formatProxyStartLog, installerFailureAllowsRecovery, npmSelfUpdateRestartEvidence, readUpdateJob, @@ -109,6 +110,15 @@ describe("GUI update execution decisions", () => { }); }); + test("recovery start logs label candidates without persisting launcher paths", () => { + const retiredLauncher = "/Users/test/.npm-global/lib/node_modules/@bitkyc08/.opencodex-Ab12Cd34/bin/ocx.mjs"; + const line = formatProxyStartLog("npm", retiredLauncher, 10100); + + expect(line).toBe("Starting npm proxy from validated recovery candidate on port 10100."); + expect(line).not.toContain(retiredLauncher); + expect(line).not.toContain("/Users/test/"); + }); + test("finds a validated npm-retired launcher when the current package is partial", async () => { const scopeRoot = join(dir, "global", "@bitkyc08"); const currentRoot = join(scopeRoot, "opencodex"); @@ -313,11 +323,24 @@ describe("GUI update execution decisions", () => { }, 5_000); test("only recovers after a clean installer exit", () => { - expect(installerFailureAllowsRecovery("npm", { status: 1, signal: null, timedOut: false })).toBe(true); - expect(installerFailureAllowsRecovery("npm", { status: 75, signal: null, timedOut: false })).toBe(false); - expect(installerFailureAllowsRecovery("bun", { status: 75, signal: null, timedOut: false })).toBe(true); - expect(installerFailureAllowsRecovery("npm", { status: null, signal: "SIGTERM", timedOut: false })).toBe(false); - expect(installerFailureAllowsRecovery("npm", { status: 1, signal: null, timedOut: true })).toBe(false); + expect(installerFailureAllowsRecovery("npm", { + status: 1, signal: null, timedOut: false, treeExited: true, + })).toBe(true); + expect(installerFailureAllowsRecovery("npm", { + status: 75, signal: null, timedOut: false, treeExited: true, + })).toBe(false); + expect(installerFailureAllowsRecovery("bun", { + status: 75, signal: null, timedOut: false, treeExited: true, + })).toBe(true); + expect(installerFailureAllowsRecovery("bun", { + status: 1, signal: null, timedOut: false, treeExited: false, + })).toBe(false); + expect(installerFailureAllowsRecovery("npm", { + status: null, signal: "SIGTERM", timedOut: false, treeExited: true, + })).toBe(false); + expect(installerFailureAllowsRecovery("npm", { + status: 1, signal: null, timedOut: true, treeExited: true, + })).toBe(false); }); test("proxy restart pins --port so post-update start does not hop to an ephemeral port", () => { @@ -942,7 +965,11 @@ describe("GUI update execution decisions", () => { attempted.push(launcher); activePid = launcher === "/current/bin/ocx.mjs" ? 222 : 333; runningPids.add(activePid); - return activePid; + const startedPid = activePid; + return { + pid: startedPid, + sameGeneration: () => runningPids.has(startedPid), + }; }, killProxyFn: pid => { killed.push(pid); @@ -993,7 +1020,7 @@ describe("GUI update execution decisions", () => { probeProxy: async () => false, restartAfterUpdateFn: async (_job, captured) => { attempted.push(captured?.recoveryLauncher); - return 222; + return { pid: 222, sameGeneration: () => true }; }, killProxyFn: pid => { killed.push(pid); }, now: () => now, @@ -1009,6 +1036,53 @@ describe("GUI update execution decisions", () => { )).toBe(true); }); + test("failed install never kills a reused PID after the spawned process generation exits", async () => { + let now = 0; + const attempted: Array = []; + const killed: number[] = []; + const job: UpdateJobState = { + id: "failed-install-reused-generation", + status: "running", + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + currentVersion: "2.7.40", + latestVersion: "2.7.41", + channel: "latest", + installer: "npm", + restart: true, + command: "", + releaseNotesUrl: "", + log: [], + }; + writeFileSync(updateJobPath(), JSON.stringify(job)); + + const recovery = await recoverFailedGuiUpdateForTests( + job, + { port: 10100, hostname: "127.0.0.1", oldPid: 111 }, + true, + { + probeProxyIdentity: async () => null, + verifyPidIdentityFn: pid => pid === 222 ? pid : null, + recoveryLaunchersFn: () => ["/current/bin/ocx.mjs", "/retired/bin/ocx.mjs"], + probeProxy: async () => false, + restartAfterUpdateFn: async (_job, captured) => { + attempted.push(captured?.recoveryLauncher); + return { pid: 222, sameGeneration: () => false }; + }, + killProxyFn: pid => { killed.push(pid); }, + now: () => now, + sleepMs: async (ms) => { now += ms; }, + }, + ); + + expect(recovery).toBe("failed"); + expect(attempted).toEqual(["/current/bin/ocx.mjs"]); + expect(killed).toEqual([]); + expect(readUpdateJob(job.id)?.log.some(line => + line.includes("no longer matches the spawned process generation"), + )).toBe(true); + }); + test("failed install reports remediation when no runnable recovery package remains", async () => { let restartCalls = 0; const job: UpdateJobState = { @@ -1513,7 +1587,7 @@ describe("immutable update target (WP160)", () => { const gateAt = source.indexOf("const integrity = checkUpdatePackageIntegrity(check.latestVersion);"); const failAt = source.indexOf('updateJob(job, { status: "failed", error: integrity.reason });'); - const spawnAt = source.indexOf("const result = runLoggedCommand(job, cmd.bin, cmd.args, UPDATE_TIMEOUT_MS);"); + const spawnAt = source.indexOf("const result = await runLoggedProcessTreeCommand(job, cmd.bin, cmd.args, UPDATE_TIMEOUT_MS);"); expect(gateAt).toBeGreaterThan(-1); expect(failAt).toBeGreaterThan(-1); expect(spawnAt).toBeGreaterThan(-1); diff --git a/tests/update-stop-first.test.ts b/tests/update-stop-first.test.ts index c627370cc..d40d69d87 100644 --- a/tests/update-stop-first.test.ts +++ b/tests/update-stop-first.test.ts @@ -127,12 +127,20 @@ describe("update stops the running proxy before replacing files", () => { expect(outerMs).toBeGreaterThanOrEqual(innerMs + 60_000); expect(launcherSource).toContain("timeoutMs: NPM_INSTALL_TIMEOUT_MS"); expect(launcherSource).toContain("await runProcessTreeCommand(npm"); + expect(updateJobSource).toContain("await runLoggedProcessTreeCommand(job, cmd.bin, cmd.args, UPDATE_TIMEOUT_MS)"); + expect(updateJobSource).toContain("if (result.status !== 0 || !result.treeExited)"); + expect(updateJobSource).toContain("return result.treeExited"); expect(updateJobSource).toContain("installerFailureAllowsRecovery(check.installer, result)"); expect(updateJobSource).toContain("if (trayWasRunning && mayRecover)"); expect(updateJobSource).toContain("The Windows tray also remains stopped"); expect(updateJobSource).toContain("candidates.slice(0, MAX_NPM_RECOVERY_CANDIDATES)"); }); + test("GUI recovery scan imports the worker argument contract from the worker module", () => { + expect(updateJobSource).toContain('import { RECOVERY_TREE_SCAN_WORKER_ARG } from "./recovery-tree-scan.mjs"'); + expect(updateJobSource).not.toContain('const RECOVERY_TREE_SCAN_WORKER_ARG = "__scan-recovery-tree"'); + }); + test("GUI worker update children use pipe stdio so Windows npm.cmd does not open consoles", () => { expect(updateSource).toContain("function updateChildStdio()"); expect(updateSource).toContain('process.env.OCX_SERVICE === "1"'); From d215cbeb9d712e123e3eebadd0ee5ef471747b03 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Mon, 27 Jul 2026 23:48:02 +0900 Subject: [PATCH 15/16] fix(update): harden npm recovery preflight logs --- src/update/job.ts | 28 ++++++++++--- src/update/npm-cache-preflight.d.mts | 3 +- src/update/npm-cache-preflight.mjs | 17 +++++++- tests/update-job.test.ts | 33 ++++++++++++++++ tests/update-npm-cache-preflight.test.ts | 50 +++++++++++++++++++++++- 5 files changed, 123 insertions(+), 8 deletions(-) diff --git a/src/update/job.ts b/src/update/job.ts index 929eb25b8..23f79adf1 100644 --- a/src/update/job.ts +++ b/src/update/job.ts @@ -307,7 +307,7 @@ function ensureJobDir(): void { function writeJob(job: UpdateJobState): void { ensureJobDir(); - atomicWriteFile(updateJobPath(), `${JSON.stringify(job, null, 2)}\n`); + atomicWriteFile(updateJobPath(), `${JSON.stringify(sanitizeUpdateJobState(job), null, 2)}\n`); } export function readUpdateJob(jobId?: string | null): UpdateJobState | null { @@ -321,14 +321,32 @@ export function readUpdateJob(jobId?: string | null): UpdateJobState | null { } } +function sanitizeUpdateJobText(value: string): string { + return value + .replace(/(?:[A-Za-z]:\\|\/)[^\s"'<>]*ocx\.mjs\b/g, "ocx.mjs") + .replace(/(?:[A-Za-z]:\\|\/)[^\s"'<>]*(?:\.npm|_cacache|_npx|\.opencodex-)[^\s"'<>]*/g, "[redacted npm path]") + .replace(/\/(?:Users|home)\/[^/\s"'<>]+(?:\/[^\s"'<>]*)*/g, "[redacted user path]") + .replace(/\b(uid|gid)\s*[:=]\s*\d+\b/gi, "$1=[redacted]") + .replace(/\b(UID|GID)\s+\d+\b/g, "$1 [redacted]"); +} + +function sanitizeUpdateJobState(job: UpdateJobState): UpdateJobState { + return { + ...job, + command: sanitizeUpdateJobText(job.command), + log: job.log.map(sanitizeUpdateJobText), + ...(job.error ? { error: sanitizeUpdateJobText(job.error) } : {}), + }; +} + function updateJob(job: UpdateJobState, patch: Partial, logLine?: string): UpdateJobState { const current = readUpdateJob(job.id) ?? job; - const next = { + const next = sanitizeUpdateJobState({ ...current, ...patch, updatedAt: new Date().toISOString(), - log: logLine ? [...current.log, logLine] : current.log, - }; + log: logLine ? [...current.log, sanitizeUpdateJobText(logLine)] : current.log, + }); writeJob(next); return next; } @@ -407,7 +425,7 @@ export function checkForUpdate( installer, updateAvailable, canUpdate: installer !== "source" && updateAvailable, - command, + command: sanitizeUpdateJobText(command), releaseNotesUrl: RELEASE_NOTES_URL, ...(reason ? { reason } : {}), }; diff --git a/src/update/npm-cache-preflight.d.mts b/src/update/npm-cache-preflight.d.mts index 2f8bdcd14..158624860 100644 --- a/src/update/npm-cache-preflight.d.mts +++ b/src/update/npm-cache-preflight.d.mts @@ -14,6 +14,7 @@ export interface NpmCachePreflightOptions { scanSpawn?: typeof import("node:child_process").spawnSync; scanBin?: string; scanScript?: string; + access?: (path: string, mode: number) => void; lstat?: (path: string) => NpmCacheEntryStat; readdir?: (path: string) => string[]; realpath?: (path: string) => string; @@ -43,7 +44,7 @@ export declare function findForeignOwnedNpmCacheEntry( expectedUid: number, io?: Pick< NpmCachePreflightOptions, - "lstat" | "readdir" | "realpath" | "now" | "maxEntries" | "maxDurationMs" + "access" | "lstat" | "readdir" | "realpath" | "now" | "maxEntries" | "maxDurationMs" >, ): NpmCacheOwnershipIssue | null; diff --git a/src/update/npm-cache-preflight.mjs b/src/update/npm-cache-preflight.mjs index 5cb7b4978..bfc63fd1b 100644 --- a/src/update/npm-cache-preflight.mjs +++ b/src/update/npm-cache-preflight.mjs @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { lstatSync, readFileSync, readdirSync, realpathSync } from "node:fs"; +import { accessSync, constants, lstatSync, readFileSync, readdirSync, realpathSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; @@ -22,6 +22,7 @@ function errorCode(error) { /** Find the first cache entry whose uid differs from the current user. */ export function findForeignOwnedNpmCacheEntry(cachePath, expectedUid, io = {}) { const lstat = io.lstat ?? lstatSync; + const access = io.access ?? accessSync; const readdir = io.readdir ?? (path => readdirSync(path, { encoding: "utf8" })); const realpath = io.realpath ?? realpathSync; const now = io.now ?? (() => Date.now()); @@ -93,6 +94,20 @@ export function findForeignOwnedNpmCacheEntry(cachePath, expectedUid, io = {}) { if (path === cacheRoot && !stat.isDirectory()) { return { kind: "error", path, reason: "npm cache root is not a directory" }; } + try { + const accessMode = stat.isDirectory() + ? constants.R_OK | constants.W_OK | constants.X_OK + : constants.R_OK | constants.W_OK; + access(path, accessMode); + } catch (error) { + return { + kind: "error", + path, + reason: stat.isDirectory() + ? `npm cache directory is not readable, writable, and searchable (${errorCode(error)})` + : `npm cache entry is not readable and writable (${errorCode(error)})`, + }; + } if (!stat.isDirectory()) continue; const beforeReadBudget = elapsedBudgetIssue(path); diff --git a/tests/update-job.test.ts b/tests/update-job.test.ts index 780a4f510..7f0119acc 100644 --- a/tests/update-job.test.ts +++ b/tests/update-job.test.ts @@ -110,6 +110,39 @@ describe("GUI update execution decisions", () => { }); }); + test("persists installer-derived job fields without raw cache paths or uid values", async () => { + const rawPath = "/home/alice/.npm/_cacache/tmp/entry"; + const rawUid = "uid=1001"; + const job: UpdateJobState = { + id: "sanitize-installer-output", + status: "restarting", + startedAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + currentVersion: "2.7.40", + latestVersion: "2.7.41", + channel: "latest", + installer: "npm", + restart: true, + command: `node ${rawPath}/bin/ocx.mjs update --tag latest`, + releaseNotesUrl: "", + log: [`npm failed at ${rawPath} ${rawUid}`], + error: `installer stderr ${rawPath} ${rawUid}`, + }; + writeFileSync(updateJobPath(), JSON.stringify(job)); + await restartAfterUpdateForTests(job, { port: 10100, hostname: "127.0.0.1" }, { + serviceInstalledFn: () => false, + readPidFn: () => null, + waitForPort: async () => false, + spawnStart: () => { throw new Error("must not spawn"); }, + }); + const persisted = JSON.stringify(readUpdateJob(job.id)); + expect(persisted).not.toContain("/home/alice"); + expect(persisted).not.toContain("_cacache"); + expect(persisted).not.toContain("uid=1001"); + expect(persisted).not.toContain("alice"); + expect(readUpdateJob(job.id)?.command).toContain("ocx.mjs update --tag latest"); + }); + test("recovery start logs label candidates without persisting launcher paths", () => { const retiredLauncher = "/Users/test/.npm-global/lib/node_modules/@bitkyc08/.opencodex-Ab12Cd34/bin/ocx.mjs"; const line = formatProxyStartLog("npm", retiredLauncher, 10100); diff --git a/tests/update-npm-cache-preflight.test.ts b/tests/update-npm-cache-preflight.test.ts index a67f40feb..7c2ae83e0 100644 --- a/tests/update-npm-cache-preflight.test.ts +++ b/tests/update-npm-cache-preflight.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { chmodSync, lstatSync, mkdirSync, readdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { chmodSync, constants, lstatSync, mkdirSync, readdirSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; import { @@ -114,6 +114,54 @@ describe("npm cache ownership pre-flight", () => { expect(formatNpmCacheOwnershipFailure(result)).toContain("configure a user-owned npm cache"); }); + test("fails closed when the cache is owned by the user but not effectively writable", () => { + const uid = process.getuid?.() ?? 501; + const cacheRoot = realpathSync(dir); + const accessDenied = Object.assign(new Error("permission denied"), { code: "EACCES" }); + const issue = findForeignOwnedNpmCacheEntry(dir, uid, { + lstat: (path) => { + const stat = lstatSync(path); + return { + uid, + isDirectory: () => stat.isDirectory(), + isSymbolicLink: () => stat.isSymbolicLink(), + }; + }, + readdir: path => readdirSync(path, { encoding: "utf8" }), + access: (path, mode) => { + if (path === cacheRoot && (mode & constants.W_OK) !== 0) throw accessDenied; + }, + }); + expect(issue).toEqual({ + kind: "error", + path: cacheRoot, + reason: "npm cache directory is not readable, writable, and searchable (EACCES)", + }); + + const result = checkNpmCacheOwnership({ + platform: "linux", + getuid: () => uid, + spawn: cacheLookup(dir), + scanSpawn: (() => ({ + status: 0, + stdout: `${JSON.stringify(issue)}\n`, + stderr: "", + pid: 1, + output: [], + signal: null, + })) as never, + }); + expect(result).toMatchObject({ + ok: false, + cachePath: dir, + entryPath: cacheRoot, + expectedUid: uid, + reason: "npm cache directory is not readable, writable, and searchable (EACCES)", + }); + if (result.ok !== false) throw new Error("expected access failure"); + expect(formatNpmCacheOwnershipFailure(result)).toContain("before stopping the proxy"); + }); + test("formats cache failures without persisting arbitrary path segments or account ids", () => { const cachePath = join(homedir(), "customer-alice-cache"); const entryPath = join(cachePath, "_npx", "node_modules", "@alice"); From b0434ea584c4b51d60facd6a40ba34414ab12c16 Mon Sep 17 00:00:00 2001 From: bitkyc08-arch Date: Tue, 28 Jul 2026 00:04:51 +0900 Subject: [PATCH 16/16] test(update): assert launcher suffix with platform separator --- tests/update-job.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/update-job.test.ts b/tests/update-job.test.ts index 7f0119acc..8d44c1ca0 100644 --- a/tests/update-job.test.ts +++ b/tests/update-job.test.ts @@ -644,7 +644,7 @@ describe("GUI update execution decisions", () => { expect(waited).toEqual([{ port: 18765, hostname: "127.0.0.1" }]); expect(bakeDuringInstall).toEqual(["18765"]); expect(launchersDuringInstall).toHaveLength(1); - expect(launchersDuringInstall[0]?.endsWith("/bin/ocx.mjs")).toBe(true); + expect(launchersDuringInstall[0]?.endsWith(join("bin", "ocx.mjs"))).toBe(true); expect(process.env.OCX_BAKE_PORT).toBeUndefined(); } finally { if (prev === undefined) delete process.env.OCX_BAKE_PORT;