diff --git a/docs-site/docs/en/cli-commands.md b/docs-site/docs/en/cli-commands.md index d829a5124..4c01e21d1 100644 --- a/docs-site/docs/en/cli-commands.md +++ b/docs-site/docs/en/cli-commands.md @@ -7,7 +7,7 @@ Manage the daemon and sessions from the terminal. | `botmux setup` | Interactive configuration (first run / add / edit / delete a bot) | | `botmux start` | Start the daemon (managed by PM2) | | `botmux stop` | Stop the daemon | -| `botmux restart [--include-pm2]` | Restart the daemon (automatically restores active sessions); `--include-pm2` also restarts botmux's PM2 God daemon | +| `botmux restart [--include-pm2]` | Restart the daemon (automatically restores active sessions); `--include-pm2` additionally retires botmux's PM2 God daemon after the fleet is safely shut down, so the whole process tree restarts from the invoking shell's clean environment (plugin services are gracefully stopped first; auto ones come back after the restart) | | `botmux logs [--lines N]` | View logs | | `botmux status` | View daemon status | | `botmux upgrade` | Upgrade to the latest version | diff --git a/docs-site/docs/zh/cli-commands.md b/docs-site/docs/zh/cli-commands.md index 6d995a388..f7ce116ff 100644 --- a/docs-site/docs/zh/cli-commands.md +++ b/docs-site/docs/zh/cli-commands.md @@ -7,7 +7,7 @@ | `botmux setup` | 交互式配置(首次 / 添加 / 编辑 / 删除机器人) | | `botmux start` | 启动 daemon(PM2 管理) | | `botmux stop` | 停止 daemon | -| `botmux restart [--include-pm2]` | 重启 daemon(自动恢复活跃会话);`--include-pm2` 会同时重启 botmux 专用 PM2 God daemon | +| `botmux restart [--include-pm2]` | 重启 daemon(自动恢复活跃会话);`--include-pm2` 会在 fleet 安全关停并验证后同时退役 botmux 专用 PM2 God daemon,让整棵进程树以当前 shell 的干净环境全新启动(插件 service 会先优雅停止,auto 的重启后自动恢复) | | `botmux logs [--lines N]` | 查看日志 | | `botmux status` | 查看 daemon 状态 | | `botmux upgrade` | 升级到最新版本 | diff --git a/src/adapters/backend/zmx-backend.ts b/src/adapters/backend/zmx-backend.ts index 39b00566e..870264d37 100644 --- a/src/adapters/backend/zmx-backend.ts +++ b/src/adapters/backend/zmx-backend.ts @@ -1159,7 +1159,9 @@ export class ZmxBackend implements SessionBackend { // fake terminal leader and never controls the backing PTY dimensions. stdio: ['ignore', 'ignore', 'pipe'], timeout: ZMX_COMMAND_TIMEOUT_MS, - env: zmxControlEnv(opts), + // The session PTY inherits THIS env — zmxFreshSessionEnv pins TERM, + // which the pm2-boundary scrub removed from the daemon/worker env. + env: zmxFreshSessionEnv(opts), }); const ready = this.waitForFreshReady(launch); @@ -2517,6 +2519,20 @@ function createZmxLaunchPayload(bin: string, args: string[], opts: SpawnOpts): Z } } +/** + * Env for the one-shot client that CREATES a fresh session. Unlike the + * node-pty backends (which force TERM via `name: 'xterm-256color'`), zmx sets + * no TERM of its own: the forkpty child inherits the create client's env + * verbatim, and the daemon/worker env arrives here with TERM scrubbed at the + * pm2 boundary (INVOKER_TERMINAL_ENV_KEYS). Left absent, every CLI in the + * session fails supports-color detection and renders colorless — pin the same + * constant every other backend PTY already forces. Control clients (get/set/ + * list/kill) and a user's own `zmx attach` from a real terminal are untouched. + */ +export function zmxFreshSessionEnv(opts: SpawnOpts): NodeJS.ProcessEnv { + return { ...zmxControlEnv(opts), TERM: 'xterm-256color' }; +} + /** Strip every payload-delivered key from ZMX control subprocesses. */ export function zmxControlEnv(opts: SpawnOpts): NodeJS.ProcessEnv { const env = zmxEnv(opts.env); diff --git a/src/cli.ts b/src/cli.ts index c8c7570b2..8ea534ccf 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -9,7 +9,8 @@ * botmux start — start daemon and auto plugin services * botmux stop [--with-plugin] — stop daemon (optionally stop auto plugin services) * botmux restart [--include-pm2] [--with-plugin] — restart daemon, then ensure auto plugin services; - * --include-pm2 is a zero-live-God admission fence, not authority to signal an existing PM2 God + * --include-pm2 additionally retires the PM2 God after the fleet is verified retired + * (socket-addressed `pm2 kill`, never a PID signal) so the whole tree restarts from a fresh env * botmux restart --bootstrap-shutdown-protocol --yes — operator-approved one-time retirement * of a pre-protocol fleet after independently confirming all Session/Riff work is idle * botmux logs [--lines] — view daemon logs @@ -150,7 +151,13 @@ import { restartFailurePathIn, } from './cli/restart-failure-notification.js'; import { resolveRestartFailureOwner } from './cli/restart-failure-owner.js'; -import { assertIncludePm2RestartAdmission } from './cli/pm2-god-admission.js'; +import { + assertNoReplacementPm2God, + assertPm2RegistryQuiescentForGodRetirement, + retireSoleLivePm2God, +} from './cli/pm2-god-retirement.js'; +import { pm2FleetMutationLockTarget, withPm2FleetMutationLock } from './cli/pm2-fleet-lock.js'; +import { LogFileFollower, type LogTailSource } from './cli/log-tail.js'; import { requestAttestedDaemonShutdown, requestAttestedDaemonShutdownBatch, @@ -318,8 +325,12 @@ const PM2_NAME = 'botmux'; * when those external pm2 installations get moved or removed. */ const PM2_HOME = join(CONFIG_DIR, 'pm2'); -const PM2_FLEET_MUTATION_LOCK_TARGET = join(CONFIG_DIR, 'pm2-fleet-mutation'); +// Shared with plugin service-manager (src/cli/pm2-fleet-lock.ts): one lock +// serializes every botmux-internal mutation of the shared PM2_HOME. +const PM2_FLEET_MUTATION_LOCK_TARGET = pm2FleetMutationLockTarget(); const PM2_START_COMMAND_TIMEOUT_MS = 30_000; +// `pm2 kill` with an already-empty fleet only tears down the God + socket. +const PM2_GOD_KILL_COMMAND_TIMEOUT_MS = 30_000; const PM2_START_VERIFY_MIN_TIMEOUT_MS = 60_000; const PM2_START_VERIFY_PER_PROCESS_MS = 2_000; const PM2_START_LATE_PUBLICATION_SETTLE_MS = 10_000; @@ -361,8 +372,9 @@ function pm2Bin(): string { } /** Env for pm2 invocations with an isolated PM2_HOME. The scrub set (session - * CLI homes, Claude/workflow markers, Dashboard H5 credentials) lives in - * cli/pm2-env.ts so it stays assertable in a test. */ + * CLI homes, Claude/workflow markers, Dashboard H5 credentials, invoker + * terminal fingerprints, turn-scoped session identity — plus the TERM + * re-pin) lives in cli/pm2-env.ts so it stays assertable in a test. */ function pm2Env(home: string = PM2_HOME): NodeJS.ProcessEnv { return pm2CallerEnv(process.env, home); } @@ -2785,7 +2797,7 @@ async function cmdStart(): Promise { } } - await withFileLock(PM2_FLEET_MUTATION_LOCK_TARGET, async () => { + await withPm2FleetMutationLock(async () => { await withFileLock(BOTS_JSON_FILE, async () => { const lockedBots = loadBotsJson(); if (JSON.stringify(lockedBots) !== JSON.stringify(botsForCheck)) { @@ -3596,7 +3608,7 @@ async function cmdStop(): Promise { ); } ensureConfigDir(); - await withFileLock(PM2_FLEET_MUTATION_LOCK_TARGET, async () => { + await withPm2FleetMutationLock(async () => { assertNoDuplicatePm2GodDaemons(); cleanupLegacyPm2(bootstrapShutdownProtocol ? 'stop' : undefined); if (bootstrapShutdownProtocol) { @@ -3699,7 +3711,7 @@ async function cmdRestart(): Promise { process.exit(1); } ensureConfigDir(); - await withFileLock(PM2_FLEET_MUTATION_LOCK_TARGET, async () => { + await withPm2FleetMutationLock(async () => { const includePm2 = process.argv.includes('--include-pm2'); const includePluginServices = process.argv.includes('--with-plugin'); const bootstrapShutdownProtocol = process.argv.includes('--bootstrap-shutdown-protocol'); @@ -3712,9 +3724,6 @@ async function cmdRestart(): Promise { if (bootstrapShutdownProtocol && includePm2) { throw new Error('[restart] --bootstrap-shutdown-protocol cannot be combined with --include-pm2'); } - if (includePm2) { - assertIncludePm2RestartAdmission(listPm2GodDaemonPids()); - } const restartIntentDir = resolveDataDir(); let stagedRestartIntent: RestartIntent | null = null; @@ -3726,17 +3735,16 @@ async function cmdRestart(): Promise { preflightNodeSanity(); await ensureSystemDependencies(); cleanupLegacyPm2(bootstrapShutdownProtocol ? 'restart' : undefined); - if (bootstrapShutdownProtocol || includePm2) { - // An include-pm2 restart was admitted only when no live PM2 God existed; - // a read-only jlist probe would start one and invalidate that admission. - // Keep the existing include-pm2 clean-start path unchanged. - if (bootstrapShutdownProtocol) bootstrapDeleteAllBotmuxProcesses('restart'); - else deleteAllBotmuxProcesses(); + if (bootstrapShutdownProtocol) { + bootstrapDeleteAllBotmuxProcesses('restart'); } else { // This process is the newly installed code generation even when the // Dashboard that spawned it is still the old in-memory generation. Do // the policy probe here, before the generic retirement path throws, so // the first-upgrade failure becomes durable and reaches the owner. + // --include-pm2 takes this path too: the probe's lazy jlist may start a + // PM2 God, which is harmless now — God retirement below runs after the + // fleet is verified retired, against whichever God owns the home. const preflight = evaluateRestartShutdownPreflight(); if (preflight.bootstrapRequired) { const detail = 'current daemon PM2 policy requires the one-time shutdown-protocol bootstrap'; @@ -3756,7 +3764,25 @@ async function cmdRestart(): Promise { } deleteAllBotmuxProcesses(); } - if (includePluginServices) await stopPluginServicesForCli(undefined, { autoOnly: true }); + // --include-pm2 tears down the whole PM2_HOME, so ALL plugin services get + // a graceful stop first instead of dying with the God; auto ones are + // re-ensured after the restart, manually-started ones stay down. Stop + // errors are collected into reports rather than thrown (service-manager + // contract), so this path re-checks them: a service that failed to stop + // must block the God retirement below, not die with the God. + if (includePm2) { + const pluginStopReports = await stopPluginServicesForCli(undefined, {}); + const failedStops = pluginStopReports.filter(report => report.action === 'failed'); + if (failedStops.length > 0) { + throw new Error( + `[restart --include-pm2] plugin service(s) failed to stop gracefully: ` + + `${failedStops.map(report => report.pluginId).join(', ')}; ` + + 'the PM2 God and every remaining process were left untouched — fix or delete these services, then rerun', + ); + } + } else if (includePluginServices) { + await stopPluginServicesForCli(undefined, { autoOnly: true }); + } cleanupStaleDaemonDescriptors(); const retiredProjection = readVerifiedBotmuxPm2Projection('restart-start'); @@ -3768,6 +3794,33 @@ async function cmdRestart(): Promise { ); } + // Only now — with the core fleet verified retired and plugin services + // stopped — is the God a stateless supervisor of nothing, safe to retire + // via its own control socket. The fresh `pm2 start` below then births a + // new God from this CLI's (pm2Env-scrubbed) environment, which is what + // makes --include-pm2 a genuinely complete restart. + if (includePm2) { + // Independent whole-registry proof, not just the core projection above: + // every row still known to the God (a plugin stop that failed, an + // orphaned row from an uninstalled plugin) blocks the kill fail-closed. + assertPm2RegistryQuiescentForGodRetirement( + parsePm2JlistOutputStrict(pm2Capture(['jlist'])).map(row => ({ + name: typeof row?.name === 'string' && row.name ? row.name : 'unknown', + status: typeof row?.pm2_env?.status === 'string' ? row.pm2_env.status : undefined, + pid: parsePm2Integer(row?.pid, { nonNegative: true }), + })), + ); + const retired = await retireSoleLivePm2God({ + listGodPids: () => listPm2GodDaemonPids(), + readStartIdentity: pid => readSupervisorProcessStartIdentity(pid), + isAlive: pid => { try { process.kill(pid, 0); return true; } catch { return false; } }, + pm2Kill: () => runPm2(['kill'], true, PM2_HOME, PM2_GOD_KILL_COMMAND_TIMEOUT_MS), + sleep: ms => new Promise(resolve => setTimeout(resolve, ms)), + now: () => Date.now(), + }); + if (retired) console.log(`已退役 PM2 God (pid ${retired.pid}),fleet 将以当前干净环境全新启动`); + } + await withFileLock(BOTS_JSON_FILE, async () => { const restartBots = loadBotsJson(); const cfg = ecosystemConfig(restartBots); @@ -3795,6 +3848,10 @@ async function cmdRestart(): Promise { start: timeoutMs => { assertBotsConfigSnapshotUnchanged('restart-start', restartBots); assertNoDuplicatePm2GodDaemons(); + // After retiring the God this start must be the one to birth + // its successor; a God that appeared in between came from some + // other client's environment and is refused. + if (includePm2) assertNoReplacementPm2God(listPm2GodDaemonPids()); preflightNodeSanity(); runPm2(['start', cfg], true, PM2_HOME, timeoutMs); }, @@ -3878,7 +3935,7 @@ async function ensureBotDaemonStopped( ): Promise { ensureConfigDir(); try { - return await withFileLock(PM2_FLEET_MUTATION_LOCK_TARGET, async () => ( + return await withPm2FleetMutationLock(async () => ( withFileLock(BOTS_JSON_FILE, async () => { assertNoDuplicatePm2GodDaemons(); preflightNodeSanity(); @@ -3947,7 +4004,7 @@ async function ensureBotDaemonStarted( ): Promise { ensureConfigDir(); try { - return await withFileLock(PM2_FLEET_MUTATION_LOCK_TARGET, async () => ( + return await withPm2FleetMutationLock(async () => ( withFileLock(BOTS_JSON_FILE, async () => { assertNoDuplicatePm2GodDaemons(); preflightNodeSanity(); @@ -4252,18 +4309,32 @@ function warnIfLegacyBotmuxAlive(): void { let legacyPid = 0; try { legacyPid = parseInt(readFileSync(legacyPidFile, 'utf-8').trim(), 10); } catch { return; } if (!legacyPid) return; - try { process.kill(legacyPid, 0); } catch { return; } + // Deliberately NO PM2 client here: this helper runs at the top of read-only + // commands (status/logs), and a pm2 jlist against the legacy home would + // lazily REVIVE a legacy God the moment the recorded one exits between the + // kill(0) probe and the client's connect — a read command must never create + // one. The process-table marker scan also closes the stale-pid-file hole: + // kill(pid, 0) alone would accept an unrelated process that reused the pid. + try { + if (!listPm2GodDaemonPids(legacyHome).includes(legacyPid)) return; + } catch { return; } + // "Still has botmux processes" from PM2's own pid files, not from a client: + // the legacy God maintains -.pid under /pids while an app + // process runs. try { - const output = pm2Capture(['jlist'], legacyHome); - const apps = parsePm2JlistOutput(output); - const hasBotmux = apps.some(a => a.name === PM2_NAME || a.name.startsWith(`${PM2_NAME}-`)); - if (hasBotmux) { + for (const entry of readdirSync(join(legacyHome, 'pids'))) { + if (!entry.startsWith(PM2_NAME) || !entry.endsWith('.pid')) continue; + let appPid = 0; + try { appPid = parseInt(readFileSync(join(legacyHome, 'pids', entry), 'utf-8').trim(), 10); } catch { continue; } + if (!appPid) continue; + try { process.kill(appPid, 0); } catch { continue; } console.warn('⚠️ 检测到旧版 PM2_HOME (~/.pm2) 下仍有 botmux 进程,运行 `botmux restart` 完成迁移。\n'); + return; } - } catch { /* ignore */ } + } catch { /* no pids dir */ } } -function cmdLogs(): void { +async function cmdLogs(): Promise { warnIfLegacyBotmuxAlive(); const lines = process.argv.includes('--lines') ? process.argv[process.argv.indexOf('--lines') + 1] || '50' @@ -4275,7 +4346,16 @@ function cmdLogs(): void { ? process.argv[process.argv.indexOf('--bot') + 1] : undefined; - let target: string; + // No PM2 client at all: `pm2 logs` lazily births a God when none is alive + // (Client.start → pingDaemon false → launchDaemon), which made this read + // command a check/use race against `restart --include-pm2` — a God observed + // under the fleet lock could be retired before the spawned client connected, + // and the connecting client would then create a replacement God inside the + // kill→start window. Tailing the log files pm2 itself writes (the exact + // out_file/error_file paths ecosystemConfig pins) removes the raced resource + // entirely: no interleaving of `botmux logs` can create a God, no fleet lock + // is needed, and logs keep working while the fleet is stopped. + let sources: LogTailSource[]; if (botIdx !== undefined) { const numericIdx = /^\d+$/.test(botIdx) ? Number(botIdx) : undefined; const selectedIdx = numericIdx === undefined @@ -4283,30 +4363,86 @@ function cmdLogs(): void { : numericIdx >= 0 && numericIdx < bots.length ? numericIdx : undefined; - target = selectedIdx !== undefined - ? botProcessName(bots[selectedIdx], selectedIdx, PM2_NAME) - : numericIdx !== undefined - ? `${PM2_NAME}-${botIdx}` - : botIdx; + if (selectedIdx !== undefined) { + sources = coreBotLogSources(bots, selectedIdx); + } else { + const wanted = numericIdx !== undefined ? `${PM2_NAME}-${botIdx}` : botIdx; + sources = allBotmuxLogSources(bots).filter(source => source.label === wanted); + if (sources.length === 0) { + console.error(`✗ 未找到 ${wanted} 的日志文件(可用:不带 --bot 查看全部,或用 0-based index / pm2 名 / appId)`); + process.exitCode = 1; + return; + } + } } else { - // Show all botmux logs via pm2 regex match - target = `/^${PM2_NAME}/`; + sources = allBotmuxLogSources(bots); } - // Use spawn for streaming output. Windows cannot spawn a .js CLI script - // directly, so run the bundled pm2 script through the current node.exe. - const pm2 = buildPm2SpawnCommand(pm2Bin(), ['logs', target, '--lines', lines]); - const child = spawn(pm2.command, pm2.args, { - stdio: 'inherit', - env: pm2Env(), - shell: pm2.shell ?? false, - }); - child.on('exit', code => process.exit(code ?? 0)); + // Not `|| 50`: an explicit --lines 0 (follow-only) must stay 0. + const rawLines = Number.parseInt(lines, 10); + const parsedLines = Number.isSafeInteger(rawLines) && rawLines >= 0 ? rawLines : 50; + console.log(`跟踪 ${sources.length} 个日志文件(Ctrl+C 退出;fleet 停止时也能查看历史并等待新日志)`); + const follower = new LogFileFollower({ sources, writeLine: line => console.log(line) }); + follower.printInitialTail(parsedLines); + follower.start(); } -function cmdStatus(): void { +/** Log files of one core bot daemon — the exact paths ecosystemConfig pins. */ +function coreBotLogSources(bots: any[], index: number): LogTailSource[] { + const label = botProcessName(bots[index], index, PM2_NAME); + return [ + { label, stream: 'out', file: join(LOG_DIR, `daemon-${index}-out.log`) }, + { label, stream: 'err', file: join(LOG_DIR, `daemon-${index}-error.log`) }, + ]; +} + +/** Every botmux log file: core daemons + dashboard (ecosystemConfig paths) + * plus plugin services (PM2 default logs dir under the shared PM2_HOME). */ +function allBotmuxLogSources(bots: any[]): LogTailSource[] { + const sources = bots.flatMap((_bot: any, i: number) => coreBotLogSources(bots, i)); + sources.push( + { label: 'botmux-dashboard', stream: 'out', file: join(LOG_DIR, 'dashboard-out.log') }, + { label: 'botmux-dashboard', stream: 'err', file: join(LOG_DIR, 'dashboard-error.log') }, + ); + const pluginLogsDir = join(PM2_HOME, 'logs'); + try { + for (const entry of readdirSync(pluginLogsDir)) { + const match = entry.match(/^(botmux-plugin-.+?)-(out|error)(?:-\d+)?\.log$/); + if (!match) continue; + sources.push({ + label: match[1], + stream: match[2] === 'error' ? 'err' : 'out', + file: join(pluginLogsDir, entry), + }); + } + } catch { /* no plugin logs yet */ } + return sources; +} + +async function cmdStatus(): Promise { warnIfLegacyBotmuxAlive(); - runPm2(['status']); + // A pm2 client with no live God lazily births one from this process's env. + // During an include-pm2 restart's kill→start window that replacement God + // would abort the restart with the whole fleet offline, and in a normally + // stopped state it would silently resurrect a God nobody asked for — so a + // read-only status must check for a God under the fleet lock and never + // create one. + let entered = false; + try { + await withPm2FleetMutationLock(async () => { + entered = true; + if (listPm2GodDaemonPids().length === 0) { + console.log('PM2 God 未运行:fleet 已停止(status 不会隐式启动它;需要时运行 botmux start)。'); + return; + } + runPm2(['status']); + }, { maxWaitMs: 3_000 }); + } catch (err) { + if (entered) throw err; + // Non-zero so automation doesn't mistake "couldn't observe" for success. + console.log('fleet 停启操作进行中(fleet mutation 锁被占用),请稍后重试 botmux status。'); + process.exitCode = 1; + } } function cmdUpgrade(): void { @@ -6961,7 +7097,7 @@ botmux v${getVersion()} — IM ↔ AI 编程 CLI 桥接 start 启动 daemon,并启动 mode=auto 的插件 service stop 停止 daemon(默认不停止插件 service;--with-plugin 显式停止 mode=auto 的插件 service) restart 重启 daemon(默认不停止插件 service,core 启动后确保 mode=auto 正在运行;--with-plugin 显式先停再启动 auto service) - --include-pm2 仅允许“入场时没有 live PM2 God”的干净启动;若已有 live God,整条命令会在 fleet/breadcrumb 零改动处拒绝,且不会信号或重启现存 God + --include-pm2 在 fleet 安全退役并验证后,经 PM2_HOME socket 退役 PM2 God(绝不按 PID 发信号),再以当前干净环境全新启动——彻底重启整棵进程树;会先优雅停止全部插件 service(auto 的启动后自动恢复) 首次升级若旧 daemon 缺少 shutdown protocol:先独立确认所有 Session/Riff 工作均 idle,再一次性运行 botmux restart --bootstrap-shutdown-protocol --yes;普通 stop/restart 仍保持 fail-closed logs 查看 daemon 日志(--lines N, --bot <0-based-index|pm2-name|appId>) @@ -12823,13 +12959,14 @@ async function reconcilePluginServicesForCli( async function stopPluginServicesForCli( pluginIds?: string[], options: { autoOnly?: boolean } = {}, -): Promise { +): Promise { const { stopPluginServices } = await import('./core/plugins/service-manager.js'); const reports = await stopPluginServices(pluginIds, options); if (reports.length > 0) { console.log('\n插件 host service:'); console.log(formatPluginServiceReports(reports)); } + return reports; } function requirePluginId(raw: string | undefined): string { @@ -13396,8 +13533,8 @@ switch (command) { case 'stop-bot': await cmdStopBot(process.argv.slice(3)); break; case 'stop': await cmdStop(); break; case 'restart': await cmdRestart(); break; - case 'logs': cmdLogs(); break; - case 'status': cmdStatus(); break; + case 'logs': await cmdLogs(); break; + case 'status': await cmdStatus(); break; case 'upgrade': case 'update': cmdUpgrade(); break; case 'dashboard': await cmdDashboard(process.argv.slice(3)); break; diff --git a/src/cli/log-tail.ts b/src/cli/log-tail.ts new file mode 100644 index 000000000..3b0811517 --- /dev/null +++ b/src/cli/log-tail.ts @@ -0,0 +1,190 @@ +import { closeSync, existsSync, openSync, readSync, statSync, type Stats } from 'node:fs'; + +/** + * PM2-client-free log tailing for `botmux logs`. + * + * `pm2 logs` is a PM2 client, and any PM2 client invoked with no live God + * lazily births one (pm2 Client.start → pingDaemon false → launchDaemon). + * That made the read command a check/use race against `restart + * --include-pm2`: a God observed under the fleet lock could be retired after + * the lock released but before the spawned client connected, and the + * connecting client would then create a replacement God inside the + * kill→start window. Tailing the log FILES pm2 itself writes removes the PM2 + * client entirely — no interleaving of this command can create a God, and + * logs keep working while the fleet is stopped. + */ + +export interface LogTailSource { + /** Display label, e.g. the PM2 process name. */ + label: string; + stream: 'out' | 'err'; + file: string; +} + +export function formatLogLine(source: LogTailSource, line: string): string { + return `${source.label}${source.stream === 'err' ? ' (err)' : ''} | ${line}`; +} + +/** Last `n` complete lines of a text chunk (trailing newline ignored). */ +export function lastLinesOfChunk(chunk: string, n: number): string[] { + if (n <= 0) return []; + const all = chunk.split('\n'); + if (all.length > 0 && all[all.length - 1] === '') all.pop(); + return all.slice(Math.max(0, all.length - n)); +} + +// Initial `--lines` window: start small and grow until the window holds +// enough complete lines (or the whole file / a hard cap), so a large N is +// honored instead of silently clipped by a fixed window. +const INITIAL_TAIL_WINDOW_BYTES = 64 * 1024; +const MAX_TAIL_WINDOW_BYTES = 8 * 1024 * 1024; + +function readFileRange(file: string, start: number, end: number): string { + const length = end - start; + if (length <= 0) return ''; + const fd = openSync(file, 'r'); + try { + const buffer = Buffer.alloc(length); + const read = readSync(fd, buffer, 0, length, start); + return buffer.subarray(0, read).toString('utf8'); + } finally { + closeSync(fd); + } +} + +/** Tail window ending at `size`, aligned to a line start (or BOF). */ +function readAlignedTailWindow(file: string, size: number, wantLines: number): string { + for (let window = INITIAL_TAIL_WINDOW_BYTES; ; window *= 2) { + const start = Math.max(0, size - window); + const chunk = readFileRange(file, start, size); + if (start === 0) return chunk; + const newlineCount = chunk.split('\n').length - 1; + // Need wantLines complete lines AFTER dropping the leading partial line a + // mid-file window start produces (plus a possible unterminated tail). + if (newlineCount > wantLines + 1 || window >= MAX_TAIL_WINDOW_BYTES) { + return chunk.slice(chunk.indexOf('\n') + 1); + } + } +} + +interface FollowState { + source: LogTailSource; + /** Next byte offset to read from; undefined until the file first appears. */ + offset: number | undefined; + /** Trailing partial (unterminated) line carried between polls. */ + partial: string; + /** File generation identity; a change means the path was atomically + * replaced (rotation) even when the new size passes every offset check. */ + dev: number | undefined; + ino: number | undefined; +} + +export interface LogFileFollowerOptions { + sources: readonly LogTailSource[]; + writeLine: (formatted: string) => void; + pollIntervalMs?: number; +} + +/** + * Print the last N lines of every existing source, then follow appends. + * Correctness boundaries this models explicitly: + * - An unterminated trailing line is NEVER emitted as a line; it is carried + * as `partial` (from the initial tail too) and emitted once completed, so + * "abc" + later "def\n" prints one "abcdef" line. + * - Atomic rotation (rename + recreate at the same path) is detected by + * dev+ino generation identity, not by size heuristics — a new file whose + * size already exceeds the old offset would otherwise be read mid-stream. + * (On filesystems reporting ino=0 this degrades to the size checks.) + * - In-place truncation (`pm2 flush`) resets to the top of the new content. + * - Files may appear later (fleet starts while tailing) or disappear. + */ +export class LogFileFollower { + private readonly states: FollowState[]; + private readonly writeLine: (formatted: string) => void; + private readonly pollIntervalMs: number; + private timer: NodeJS.Timeout | undefined; + + constructor(opts: LogFileFollowerOptions) { + this.states = opts.sources.map(source => ({ + source, offset: undefined, partial: '', dev: undefined, ino: undefined, + })); + this.writeLine = opts.writeLine; + this.pollIntervalMs = opts.pollIntervalMs ?? 300; + } + + /** Emit the initial `--lines` window and position offsets at end-of-file. */ + printInitialTail(lines: number): void { + for (const state of this.states) { + if (!existsSync(state.source.file)) continue; + let stats: Stats; + try { stats = statSync(state.source.file); } catch { continue; } + let chunk: string; + try { chunk = readAlignedTailWindow(state.source.file, stats.size, lines); } catch { continue; } + let complete = chunk; + if (chunk.length > 0 && !chunk.endsWith('\n')) { + const cut = chunk.lastIndexOf('\n'); + state.partial = chunk.slice(cut + 1); + complete = cut >= 0 ? chunk.slice(0, cut + 1) : ''; + } + for (const line of lastLinesOfChunk(complete, lines)) { + this.writeLine(formatLogLine(state.source, line)); + } + state.offset = stats.size; + state.dev = stats.dev; + state.ino = stats.ino; + } + } + + start(): void { + if (this.timer) return; + this.timer = setInterval(() => this.pollOnce(), this.pollIntervalMs); + } + + stop(): void { + if (this.timer) clearInterval(this.timer); + this.timer = undefined; + } + + /** One poll pass; exposed so tests can drive it deterministically. */ + pollOnce(): void { + for (const state of this.states) { + let stats: Stats; + try { + stats = statSync(state.source.file); + } catch { + // Absent (not yet created, or rotated away): forget position so the + // file is picked up from its start when it (re)appears. + state.offset = undefined; + state.partial = ''; + state.dev = undefined; + state.ino = undefined; + continue; + } + const sameGeneration = state.dev === stats.dev && state.ino === stats.ino; + if (state.offset === undefined || !sameGeneration) { + // Newly appeared file, or an atomic replace under the same path: + // this is a different byte stream, so any carried position/partial + // belongs to the OLD generation and must be dropped unconditionally. + state.offset = 0; + state.partial = ''; + } else if (stats.size < state.offset) { + // Truncated in place (pm2 flush): restart from the top of the new + // content instead of replaying or reading past EOF. + state.offset = 0; + state.partial = ''; + } + state.dev = stats.dev; + state.ino = stats.ino; + if (stats.size === state.offset) continue; + let chunk: string; + try { chunk = readFileRange(state.source.file, state.offset, stats.size); } catch { continue; } + state.offset = stats.size; + const combined = state.partial + chunk; + const parts = combined.split('\n'); + state.partial = parts.pop() ?? ''; + for (const line of parts) { + this.writeLine(formatLogLine(state.source, line)); + } + } + } +} diff --git a/src/cli/pm2-env.ts b/src/cli/pm2-env.ts index c235d007c..77dcbd262 100644 --- a/src/cli/pm2-env.ts +++ b/src/cli/pm2-env.ts @@ -1,6 +1,8 @@ import { scrubClaudeSessionMarkerEnv, + scrubInvokerTerminalEnv, scrubSessionCliHomeEnv, + scrubSessionTurnMarkerEnv, scrubWorkflowWorkerEnv, stripDashboardH5Env, } from '../utils/child-env.js'; @@ -45,6 +47,20 @@ export function scrubPm2CallerEnv(env: NodeJS.ProcessEnv): void { // legitimate consumer and loads the family itself (index-dashboard.ts), so // nothing needs it here. stripDashboardH5Env(env); + // Invoker-terminal fingerprints (NO_COLOR=1 / CODEX_CI=1 / PAGER=cat from + // an agent's non-interactive shell) ride the same persistence vector; baked + // in they turn every session PTY on the machine colorless. Same class, same + // boundary — see INVOKER_TERMINAL_ENV_KEYS. + scrubInvokerTerminalEnv(env); + // Turn-scoped session identity and capabilities of the invoking bot session + // must not become fleet-wide daemon env — the daemon stays session-agnostic + // (see SESSION_TURN_MARKER_ENV_KEYS for the per-key rationale). + scrubSessionTurnMarkerEnv(env); + // Re-pin TERM to the constant every botmux PTY already forces. Deleting it + // outright would make pm2 CLIENT output (e.g. `botmux status` on a real + // TTY) fail supports-color detection and render colorless; pinning keeps + // the baked value deterministic and invoker-independent instead of absent. + env.TERM = 'xterm-256color'; } /** Build the env for a pm2 invocation with an isolated PM2_HOME. */ diff --git a/src/cli/pm2-fleet-lock.ts b/src/cli/pm2-fleet-lock.ts new file mode 100644 index 000000000..5c79f403d --- /dev/null +++ b/src/cli/pm2-fleet-lock.ts @@ -0,0 +1,66 @@ +import { join } from 'node:path'; +import { homedir } from 'node:os'; +import { AsyncLocalStorage } from 'node:async_hooks'; +import { withFileLock, withFileLockSync } from '../utils/file-lock.js'; + +/** + * One lock for every botmux-internal mutation of the shared PM2_HOME + * (~/.botmux/pm2): core fleet stop/start/restart AND plugin service + * lifecycle. Core and plugin apps live under the same God, so two separate + * locks would let a concurrent plugin start slip between an include-pm2 + * restart's plugin stop and its `pm2 kill`, or between the kill and the fresh + * fleet start. Computed lazily so tests can repoint HOME. + */ +export function pm2FleetMutationLockTarget(): string { + return join(homedir(), '.botmux', 'pm2-fleet-mutation'); +} + +/** + * Re-entrancy ownership is bound to the ASYNC CALL CHAIN, not the process: a + * module-level counter would let an unrelated concurrent flow in the same + * process (the dashboard serves POST /api/plugins/:id/services/* handlers + * concurrently) mistake "someone in this process holds the lock" for "I hold + * the lock" and skip the file lock entirely. AsyncLocalStorage context only + * flows into calls awaited UNDER the holder's callback, so nested calls + * (cmdRestart stopping plugin services while holding the lock) short-circuit + * while independent concurrent flows queue on the file lock like any other + * process. + */ +const lockOwnership = new AsyncLocalStorage<{ held: true }>(); + +/** True only within the async call chain that currently holds the lock. */ +export function pm2FleetMutationLockHeld(): boolean { + return lockOwnership.getStore() !== undefined; +} + +/** + * Serialize a PM2_HOME mutation against every other botmux flow — other + * processes via the file lock, other async chains in THIS process via the + * same file lock (file-lock treats a live same-pid holder as held, not + * stale). Lock order is fixed: fleet lock FIRST, then the plugin service + * lock — never the reverse. + */ +export async function withPm2FleetMutationLock( + fn: () => Promise | T, + opts: { maxWaitMs?: number } = {}, +): Promise { + if (lockOwnership.getStore()) return await fn(); + return withFileLock( + pm2FleetMutationLockTarget(), + () => lockOwnership.run({ held: true }, async () => fn()), + opts, + ); +} + +/** Sync variant for sync call sites (plugin service lock wrapper). */ +export function withPm2FleetMutationLockSync( + fn: () => T, + opts: { maxWaitMs?: number } = {}, +): T { + if (lockOwnership.getStore()) return fn(); + return withFileLockSync( + pm2FleetMutationLockTarget(), + () => lockOwnership.run({ held: true }, fn), + opts, + ); +} diff --git a/src/cli/pm2-god-admission.ts b/src/cli/pm2-god-admission.ts deleted file mode 100644 index ddaa81fca..000000000 --- a/src/cli/pm2-god-admission.ts +++ /dev/null @@ -1,27 +0,0 @@ -/** - * Node's process.kill is PID-addressed and PM2's `kill` RPC is PM2_HOME/socket - * addressed; neither binds a signal to a PID+birth generation. Therefore a - * live God cannot be safely replaced automatically. This admission check must - * run before any fleet or breadcrumb mutation. - */ -export function assertIncludePm2RestartAdmission(pids: readonly number[]): void { - const canonical = [...new Set(pids)] - .filter(pid => Number.isSafeInteger(pid) && pid > 1) - .sort((a, b) => a - b); - if (canonical.length !== pids.length) { - throw new Error('[restart --include-pm2] PM2 God scan returned invalid/duplicate PIDs'); - } - if (canonical.length === 0) return; - if (canonical.length > 1) { - throw new Error( - `[restart --include-pm2] multiple PM2 God daemons are visible ` - + `(pids: ${canonical.join(', ')}); no process or breadcrumb was changed`, - ); - } - throw new Error( - `[restart --include-pm2] refusing before fleet mutation: live PM2 God pid ${canonical[0]} ` - + 'cannot be signalled with generation-bound authority on this platform; ' - + 'this option does not signal or restart an existing God; ' - + 'no process or breadcrumb was changed', - ); -} diff --git a/src/cli/pm2-god-retirement.ts b/src/cli/pm2-god-retirement.ts new file mode 100644 index 000000000..4183a45fb --- /dev/null +++ b/src/cli/pm2-god-retirement.ts @@ -0,0 +1,133 @@ +/** + * `restart --include-pm2` God retirement. + * + * A PID-addressed kill(2) cannot bind a signal to a PID+birth generation, so + * this module never signals the God by PID. The kill is addressed to the + * PM2_HOME control socket (`pm2 kill`) — i.e. to + * "whichever God owns this home", which is exactly the object being retired — + * and the recorded pid+birth identity is used only to VERIFY disappearance + * afterwards. Callers must run this strictly AFTER the managed core fleet has + * been retired and verified gone (and plugin services stopped): with an empty + * fleet the God holds no session state, so retiring it cannot interrupt a + * Riff prepare/persist/commit handshake. + */ + +export interface Pm2GodRetirementRuntime { + /** Scan for God processes owning this PM2_HOME (cmdline-marker based). */ + listGodPids(): number[]; + /** Birth identity for verification only — never signalling authority. */ + readStartIdentity(pid: number): string | undefined; + isAlive(pid: number): boolean; + /** Bounded `pm2 kill` against this PM2_HOME's control socket. */ + pm2Kill(): void; + sleep(ms: number): Promise; + now(): number; +} + +export const PM2_GOD_RETIREMENT_VERIFY_TIMEOUT_MS = 15_000; +const VERIFY_POLL_INTERVAL_MS = 200; + +export interface RetiredPm2God { + pid: number; + startIdentity: string | undefined; +} + +export interface Pm2RegistryRowLiveness { + name: string; + status: string | undefined; + pid: number | undefined; +} + +/** Registry statuses that prove no process is running for the row. */ +const TERMINAL_PM2_ROW_STATUSES: ReadonlySet = new Set(['stopped', 'errored']); + +/** + * `pm2 kill` slaughters every process the God still manages without any + * graceful handshake, so the God may only be retired once the WHOLE registry + * is quiescent: every row — core, plugin, or an orphaned row a plugin + * uninstall left behind — must be in a terminal status with no live pid. A + * plugin stop that failed (stop errors are collected into reports, not + * rethrown) or a leftover running `botmux-plugin-*` row therefore blocks the + * kill here, fail-closed, instead of being silently killed with the God. + */ +export function assertPm2RegistryQuiescentForGodRetirement( + rows: readonly Pm2RegistryRowLiveness[], +): void { + const live = rows.filter(row => { + const status = (row.status ?? '').trim(); + const pidLive = typeof row.pid === 'number' && Number.isSafeInteger(row.pid) && row.pid > 0; + return pidLive || !TERMINAL_PM2_ROW_STATUSES.has(status); + }); + if (live.length === 0) return; + const detail = live + .map(row => `${row.name}:${row.status ?? 'unknown'}${row.pid ? `:pid ${row.pid}` : ''}`) + .join(', '); + throw new Error( + `[restart --include-pm2] refusing pm2 kill: PM2 registry still has live/unproven row(s): ${detail}; ` + + 'stop or delete them first (e.g. a plugin service that failed to stop, or a leftover ' + + 'botmux-plugin-* row from an uninstalled plugin); the God and every remaining process were left untouched', + ); +} + +/** + * Between God retirement and the fresh `pm2 start`, any pm2 client invocation + * (even a read-only jlist from another shell) lazily births a God from ITS + * environment, not from this restart's cleaned one. Accepting it would defeat + * the whole point of --include-pm2, so the start transaction refuses. + */ +export function assertNoReplacementPm2God(pids: readonly number[]): void { + if (pids.length === 0) return; + throw new Error( + `[restart --include-pm2] a replacement PM2 God (pid ${pids.join(', ')}) appeared between God ` + + 'retirement and the fleet start; it was not born from this restart\'s cleaned environment — ' + + 'rerun `botmux restart --include-pm2`', + ); +} + +/** + * Retire the sole live PM2 God, or return null when none is alive. Fails + * closed — without mutating anything — on an invalid scan or multiple visible + * Gods, and fails closed after `pm2 kill` if the God's disappearance cannot + * be proven within the timeout. + */ +export async function retireSoleLivePm2God( + rt: Pm2GodRetirementRuntime, + timeoutMs: number = PM2_GOD_RETIREMENT_VERIFY_TIMEOUT_MS, +): Promise { + const scanned = rt.listGodPids(); + const canonical = [...new Set(scanned)] + .filter(pid => Number.isSafeInteger(pid) && pid > 1) + .sort((a, b) => a - b); + if (canonical.length !== scanned.length) { + throw new Error('[restart --include-pm2] PM2 God scan returned invalid/duplicate PIDs; no process was signalled'); + } + if (canonical.length === 0) return null; + if (canonical.length > 1) { + throw new Error( + `[restart --include-pm2] multiple PM2 God daemons are visible ` + + `(pids: ${canonical.join(', ')}); no process was signalled`, + ); + } + + const pid = canonical[0]; + const startIdentity = rt.readStartIdentity(pid); + rt.pm2Kill(); + + const deadline = rt.now() + timeoutMs; + for (;;) { + // Two independent proofs: the marker scan finds no God for this home, and + // the original pid is gone (or its slot was reused by a different birth). + const scanEmpty = rt.listGodPids().length === 0; + const originalGone = !rt.isAlive(pid) + || (startIdentity !== undefined && rt.readStartIdentity(pid) !== startIdentity); + if (scanEmpty && originalGone) return { pid, startIdentity }; + if (rt.now() >= deadline) { + throw new Error( + `[restart --include-pm2] PM2 God pid ${pid} is still observable after pm2 kill; ` + + 'the core fleet is already retired and nothing further was mutated — ' + + 'inspect the God process, then rerun `botmux restart` (with or without --include-pm2) to bring the fleet back', + ); + } + await rt.sleep(VERIFY_POLL_INTERVAL_MS); + } +} diff --git a/src/core/plugins/pm2.ts b/src/core/plugins/pm2.ts index b2c1748dd..3c8044c73 100644 --- a/src/core/plugins/pm2.ts +++ b/src/core/plugins/pm2.ts @@ -4,8 +4,8 @@ import { join } from 'node:path'; import { createRequire } from 'node:module'; import { spawnSync } from 'node:child_process'; import { buildPm2SpawnCommand } from '../../cli/pm2-command.js'; +import { scrubPm2CallerEnv } from '../../cli/pm2-env.js'; import { stripPm2GracefulExitMarker } from '../../pm2-graceful-exit.js'; -import { stripDashboardH5Env } from '../../utils/child-env.js'; const require = createRequire(import.meta.url); const BOTMUX_HOME = join(homedir(), '.botmux'); @@ -35,16 +35,21 @@ function pm2Env(extra?: Record): NodeJS.ProcessEnv { // the plugin service is an arbitrary long-lived process that could launch a // foreground botmux, which would then exit 90 on a clean stop. See // stripPm2GracefulExitMarker. - const inherited = stripPm2GracefulExitMarker(process.env); - delete inherited.kill_timeout; - // The dashboard starts/stops plugin services in-process, and it is the one - // machine-wide holder of the Feishu H5 login family (BOTMUX_DASHBOARD_FEISHU_H5_*, - // APP_SECRET included). A raw process.env copy would hand that credential to - // an arbitrary third-party plugin service AND persist it in the plugin PM2 - // home's metadata/dump. No plugin consumes it — the dashboard is the only - // consumer in the fleet. - stripDashboardH5Env(inherited); - return { ...inherited, ...(extra ?? {}), PM2_HOME: PLUGIN_PM2_HOME }; + const merged = stripPm2GracefulExitMarker({ ...process.env, ...(extra ?? {}) }); + delete merged.kill_timeout; + // Plugin PM2 shares the God's PM2_HOME, so this boundary both persists the + // caller's env into plugin apps AND can create the shared God itself. It + // therefore applies the SAME caller hygiene as the core pm2 entry + // (scrubPm2CallerEnv: CLI home pointers, Claude session markers, workflow + // identity, dashboard H5 credentials — a raw copy would hand that secret to + // an arbitrary third-party plugin service — plus invoker terminal + // fingerprints and turn-scoped session identity, with the TERM re-pin), and + // applies it AFTER the manifest env merge, so a plugin manifest cannot + // revive a scrubbed key (a service needing its own data root must resolve + // it internally, not via CLAUDE_CONFIG_DIR/CODEX_HOME). + scrubPm2CallerEnv(merged); + merged.PM2_HOME = PLUGIN_PM2_HOME; + return merged; } export function runPluginPm2(args: string[], opts: { inherit?: boolean; timeoutMs?: number; env?: Record } = {}): void { diff --git a/src/core/plugins/service-manager.ts b/src/core/plugins/service-manager.ts index 23b511246..d9c56024d 100644 --- a/src/core/plugins/service-manager.ts +++ b/src/core/plugins/service-manager.ts @@ -5,6 +5,7 @@ import { config } from '../../config.js'; import { formatUrlHost } from '../dashboard-url.js'; import { atomicWriteFileSync } from '../../utils/atomic-write.js'; import { withFileLock, withFileLockSync } from '../../utils/file-lock.js'; +import { withPm2FleetMutationLock, withPm2FleetMutationLockSync } from '../../cli/pm2-fleet-lock.js'; import { readPluginRegistry } from '../../services/plugin-registry-store.js'; import { pluginHome, @@ -71,12 +72,27 @@ function serviceLockTarget(): string { return `${pluginsHome()}/service-manager`; } +/** + * Plugin PM2 shares the God's PM2_HOME with the core fleet, so every plugin + * lifecycle mutation must ALSO hold the shared fleet-mutation lock — without + * it, a concurrent plugin start could slip between an include-pm2 restart's + * plugin stop and its `pm2 kill`. Lock order is fixed everywhere: fleet lock + * FIRST, then the plugin service lock (never the reverse). Same-process + * nesting (cmdRestart already holds the fleet lock when it stops plugin + * services) is handled by the fleet lock's re-entrancy counter. + */ export function withPluginServiceLockSync(fn: () => T): T { - return withFileLockSync(serviceLockTarget(), fn, { maxWaitMs: 30_000 }); + return withPm2FleetMutationLockSync( + () => withFileLockSync(serviceLockTarget(), fn, { maxWaitMs: 30_000 }), + { maxWaitMs: 30_000 }, + ); } export function withPluginServiceLock(fn: () => Promise | T): Promise { - return withFileLock(serviceLockTarget(), async () => fn(), { maxWaitMs: 30_000 }); + return withPm2FleetMutationLock( + () => withFileLock(serviceLockTarget(), async () => fn(), { maxWaitMs: 30_000 }), + { maxWaitMs: 30_000 }, + ); } function definitionEnv(record: InstalledPluginRecord, definition: PluginServiceDefinition): Record { @@ -446,17 +462,24 @@ export async function deletePluginServices(pluginIds?: readonly string[]): Promi } export async function listPluginServiceStatus(): Promise { - const reports: PluginServiceReport[] = []; - for (const record of selectedRecords()) { - try { - const definition = await loadPluginServiceDefinition(record); - if (!definition) continue; - const app = findPm2App(pluginPm2AppName(record.id)); - const state = writeServiceState(record, definition, app); - reports.push(reportFromState(record, 'status', state)); - } catch (err: any) { - reports.push(reportFromState(record, 'failed', readServiceState(record.id), err?.message ?? String(err))); + // Also serialized: a "read-only" status probe runs pm2 jlist, and a pm2 + // client with no live God lazily births one from THIS process's env — + // during an include-pm2 restart's kill→start window that would insert a + // replacement God. Holding the shared locks parks the probe until the + // mutation completes. + return withPluginServiceLock(async () => { + const reports: PluginServiceReport[] = []; + for (const record of selectedRecords()) { + try { + const definition = await loadPluginServiceDefinition(record); + if (!definition) continue; + const app = findPm2App(pluginPm2AppName(record.id)); + const state = writeServiceState(record, definition, app); + reports.push(reportFromState(record, 'status', state)); + } catch (err: any) { + reports.push(reportFromState(record, 'failed', readServiceState(record.id), err?.message ?? String(err))); + } } - } - return reports; + return reports; + }); } diff --git a/src/index-daemon.ts b/src/index-daemon.ts index 06ef60611..e9e05b39a 100644 --- a/src/index-daemon.ts +++ b/src/index-daemon.ts @@ -4,7 +4,7 @@ import { join } from 'node:path'; import { homedir } from 'node:os'; import { existsSync } from 'node:fs'; import { installStdioEpipeGuard } from './utils/stdio-epipe-guard.js'; -import { scrubClaudeSessionMarkerEnv, scrubSessionCliHomeEnv, scrubWorkflowWorkerEnv, stripDashboardH5Env } from './utils/child-env.js'; +import { scrubClaudeSessionMarkerEnv, scrubInvokerTerminalEnv, scrubSessionCliHomeEnv, scrubSessionTurnMarkerEnv, scrubWorkflowWorkerEnv, stripDashboardH5Env } from './utils/child-env.js'; // Under pm2 the daemon's stdout/stderr are pipes to the God daemon. A broken // pipe (log streaming detaches, God daemon restart) would otherwise emit an @@ -37,9 +37,9 @@ stripDashboardH5Env(process.env); // v3 workflow workers spread this process's env into their spawn env, so a // restart issued from a bot session would otherwise pin that session's owner // onto every workflow CLI child. -for (const k of ['BOTMUX_SESSION_ID', 'BOTMUX_LARK_APP_ID', 'BOTMUX_CHAT_ID', 'BOTMUX_CHAT_TYPE', 'BOTMUX_ROOT_MESSAGE_ID', 'BOTMUX_OWNER_OPEN_ID', '__OWNER_OPEN_ID']) { - delete process.env[k]; -} +// (Covers BOTMUX_LARK_APP_ID too: the daemon resolves its own bot via +// BOTMUX_BOT_INDEX and must not trust an inherited app id.) +scrubSessionTurnMarkerEnv(process.env); // Same vector, session-level CLI data-root pointers (CLAUDE_CONFIG_DIR / // CODEX_HOME): a value baked into pm2's saved app env — or resurrected from a // stale dump.pm2, which bypasses the pm2Env() strip in cli.ts — would make @@ -57,6 +57,19 @@ scrubClaudeSessionMarkerEnv(process.env); // code or forking ordinary chat workers. The ephemeral pool re-adds the exact // markers only to genuine workflow workers. scrubWorkflowWorkerEnv(process.env); +// Same vector once more, invoker-terminal fingerprints: NO_COLOR=1 / +// CODEX_CI=1 / PAGER=cat baked by a restart issued from an agent's +// non-interactive shell (or resurrected from a stale dump) would flow into +// every worker and session PTY and render every bot TUI colorless. Scrubbing +// here also heals an already-poisoned fleet on its next daemon boot without +// waiting for a clean-shell restart. See INVOKER_TERMINAL_ENV_KEYS. +scrubInvokerTerminalEnv(process.env); +// Re-pin TERM after the scrub, same constant as both pm2Env() entries: +// deterministic instead of absent. Workers fork from this env, and the zmx +// backend's fresh sessions inherit the create client's env verbatim (zmx has +// no node-pty `name` forcing TERM) — left absent, every CLI in a zmx session +// fails supports-color detection and renders colorless. +process.env.TERM = 'xterm-256color'; async function main() { // Resolve global UI locale from ~/.botmux/config.json BEFORE loading diff --git a/src/utils/child-env.ts b/src/utils/child-env.ts index be9dbcc58..4c9f739f2 100644 --- a/src/utils/child-env.ts +++ b/src/utils/child-env.ts @@ -143,6 +143,54 @@ export function stripDashboardH5Env(env: NodeJS.ProcessEnv): void { } } +/** + * Terminal/interactivity fingerprints of whichever process invoked a pm2 + * mutation. pm2 persists the caller's env into every managed app (and into + * dump.pm2 for resurrect), so a `botmux restart` issued from an agent's + * non-interactive shell — Claude Code / Codex tool shells export NO_COLOR=1, + * CODEX_CI=1, PAGER=cat, plus the terminal-app identity of whatever terminal + * hosted them — bakes "you have no colors, you are in CI" into every daemon, + * which every worker and session PTY then inherits: all bot CLI TUIs render + * colorless, and TERMINFO can even point at a terminal app's private terminfo + * dir. A daemon is a headless service: every key here describes the invoker's + * terminal or harness, never the machine, so deleting them at the pm2 + * boundary is always correct. Session PTYs set their own TERM (the backends + * spawn with name 'xterm-256color'), and a user who wants genuinely colorless + * bots keeps the per-bot `env` channel — like CLAUDE_EFFORT, the ambient + * "export it in the shell that runs `botmux restart`" channel is sacrificed + * because at this boundary it cannot be told apart from contamination. + */ +export const INVOKER_TERMINAL_ENV_KEYS = [ + // Color semantics + 'NO_COLOR', + 'FORCE_COLOR', + 'CLICOLOR', + 'CLICOLOR_FORCE', + // Terminal identity. TERMINFO is a single-directory override that terminal + // apps export for their own private terminfo bundle — invoker-scoped. + // TERMINFO_DIRS is deliberately ABSENT: it is a search-path list that + // NixOS/home-manager and custom-ncurses setups configure machine-wide, and + // deleting it would break terminfo resolution for every PTY on such hosts. + 'TERM', + 'COLORTERM', + 'TERMINFO', + 'TERM_PROGRAM', + 'TERM_PROGRAM_VERSION', + 'TERM_SESSION_ID', + // CI / agent-harness flags + 'CI', + 'CODEX_CI', + // Non-interactive pager pins agent shells export for their own subcommands + 'PAGER', + 'GIT_PAGER', + 'GH_PAGER', +] as const; + +/** Delete inherited invoker-terminal fingerprints from `env` in place. */ +export function scrubInvokerTerminalEnv(env: NodeJS.ProcessEnv): void { + for (const key of INVOKER_TERMINAL_ENV_KEYS) delete env[key]; +} + /** * Env vars that must never reach a spawned CLI child. The bot's IM-app creds * (a child CLI's own Lark OAuth reads `process.env.LARK_APP_ID` as the app to @@ -370,6 +418,82 @@ export const BOTMUX_INJECTED_ENV_KEYS = [ 'CJADK_INTERACTIVE', ] as const; +/** + * Session-only botmux identity and capabilities of the process that invoked a + * pm2 mutation. A `botmux restart` issued from inside a bot session carries + * that session's routing identity and capabilities; persisted by pm2 into the + * fleet, every daemon then carries a stale foreign turn identity, and a + * plugin service started from the same env would misroute its own + * `botmux send` to a long-dead thread. + * + * This is a DELIBERATELY hand-maintained list. It is NOT derived from + * BOTMUX_INJECTED_ENV_KEYS: that list is the tmux/pane TRANSPORT whitelist + * and mixes session-only keys with ambient/daemon config that merely needs + * pane delivery (CLAUDE_CODE_RESUME_TOKEN_THRESHOLD is ambient-only — worker + * reads it from its own process.env and per-bot env REJECTS it, so a boot + * scrub would silently kill the user's setting; HERMES_HOME and the two + * HERMES_BOTMUX_* roots are ambient install-location config nothing in + * botmux ever sets, same contract as GROK_HOME; BOTS_CONFIG / + * SESSION_DATA_DIR / BOTMUX_LARK_LIST_BOTS_API_* are documented ambient or + * ecosystem-block config). The reverse also holds: session/sandbox routing + * keys the pane transport never carries (BOTMUX_SESSION_SCOPE, + * BOTMUX_SEND_RELAY) still need scrubbing here. Every entry below is + * session-scoped BY CONSTRUCTION: the daemon/worker computes and injects it + * per session AFTER every boundary scrub, and no ambient/env-file channel for + * it exists. + */ +export const SESSION_TURN_MARKER_ENV_KEYS = [ + // "Runs inside botmux" pane marker; pane wrapper injects it per session. + 'BOTMUX', + // Turn/session routing identity (worker-injected per pane/turn). + 'BOTMUX_SESSION_ID', + 'BOTMUX_CHAT_ID', + 'BOTMUX_CHAT_TYPE', + 'BOTMUX_ROOT_MESSAGE_ID', + 'BOTMUX_TURN_ID', + 'BOTMUX_DISPATCH_ATTEMPT', + // thread|chat scope, computed per session from rootMessageId (worker.ts). + 'BOTMUX_SESSION_SCOPE', + // Daemon-authenticated session owner, both channels (applySessionOwnerEnv). + 'BOTMUX_OWNER_OPEN_ID', + '__OWNER_OPEN_ID', + // Unguessable pane/profile authority channel (daemon-rotated capability). + 'BOTMUX_ORIGIN_CHANNEL_ID', + // Sandbox send-relay directory — a per-session capability path. + 'BOTMUX_SEND_RELAY', + // Session-scoped MCP gateway capability + fail-closed marker. + 'BOTMUX_MCP_GATEWAY_SOCKET', + 'BOTMUX_MCP_GATEWAY_REQUIRED', + // Owning daemon's per-boot IPC port; the daemon self-sets the real value at + // boot (daemon.ts), so an inherited copy is always a stale foreign port. + 'BOTMUX_DAEMON_IPC_PORT', + // Per-session read-isolation / apiOnly / sandbox verdicts (worker-owned). + 'BOTMUX_READ_ISOLATION', + 'BOTMUX_READ_ISOLATED', + 'BOTMUX_API_ONLY', + 'IS_SANDBOX', + // Per-session display footer resolved from bots.json (never env-configured). + 'BOTMUX_BRAND_LABEL', + // Per-bot usage display, resolved via resolveUsageDisplay(cfg) per session; + // env is never a config input for it. + 'BOTMUX_USAGE_DISPLAY', + // One-shot per-session artifacts/paths. + 'BOTMUX_PI_INITIAL_PROMPT_FILE', + 'BOTMUX_CODEX_APP_CONTROL_BOOTSTRAP', + // Ready-gate hook command, sessionReadyHookCommand() per session. + 'BOTMUX_READY_COMMAND', + // Per-app value pinned by the ecosystemConfig env block; an inherited copy + // is untrusted (the daemon resolves its bot via BOTMUX_BOT_INDEX). + 'BOTMUX_LARK_APP_ID', + // cjadk wrapper-branch knob, set/deleted per spawn by the worker. + 'CJADK_INTERACTIVE', +] as const; + +/** Delete inherited session-only identity/capabilities from `env` in place. */ +export function scrubSessionTurnMarkerEnv(env: NodeJS.ProcessEnv): void { + for (const key of SESSION_TURN_MARKER_ENV_KEYS) delete env[key]; +} + /** Proxy env vars that must reach the CLI child process so it can dial the * upstream API on hosts without direct internet access. Forwarded explicitly * by buildBotmuxEnvAssignments (tmux/tmux-pipe/zellij backends) and diff --git a/test/child-env.test.ts b/test/child-env.test.ts index 25060c16c..e68f46d22 100644 --- a/test/child-env.test.ts +++ b/test/child-env.test.ts @@ -6,12 +6,16 @@ import { CLAUDE_SESSION_MARKER_ENV_KEYS, DASHBOARD_H5_ENV_KEYS, DASHBOARD_H5_ENV_PREFIX, + INVOKER_TERMINAL_ENV_KEYS, redactChildEnv, REDACTED_CHILD_ENV_KEYS, scrubClaudeSessionMarkerEnv, + scrubInvokerTerminalEnv, scrubSessionCliHomeEnv, + scrubSessionTurnMarkerEnv, scrubWorkflowWorkerEnv, SESSION_CLI_HOME_ENV_KEYS, + SESSION_TURN_MARKER_ENV_KEYS, stripDashboardH5Env, WORKFLOW_WORKER_ENV_KEYS, } from '../src/utils/child-env.js'; @@ -345,6 +349,157 @@ describe('scrubWorkflowWorkerEnv()', () => { }); }); +describe('scrubInvokerTerminalEnv()', () => { + it('removes every invoker-terminal fingerprint in place, leaving machine env alone', () => { + const env: NodeJS.ProcessEnv = { + ...Object.fromEntries(INVOKER_TERMINAL_ENV_KEYS.map((key) => [key, 'fingerprint'])), + PATH: '/usr/bin', + LANG: 'en_US.UTF-8', + SSH_AUTH_SOCK: '/tmp/agent.sock', + HTTPS_PROXY: 'http://proxy:8080', + }; + + scrubInvokerTerminalEnv(env); + + for (const key of INVOKER_TERMINAL_ENV_KEYS) { + expect(key in env, key).toBe(false); + } + // Machine/user env that legitimately flows into the fleet must survive. + expect(env.PATH).toBe('/usr/bin'); + expect(env.LANG).toBe('en_US.UTF-8'); + expect(env.SSH_AUTH_SOCK).toBe('/tmp/agent.sock'); + expect(env.HTTPS_PROXY).toBe('http://proxy:8080'); + }); + + it('pins the observed agent-shell fingerprints that turned the fleet colorless', () => { + // The 2026-08 incident baked exactly these from a Codex tool shell into + // every daemon: NO_COLOR killed all session TUI colors, CODEX_CI marked + // every child as CI, PAGER=cat + TERMINFO pointed at a terminal app's + // private dir. Keep them pinned so a list refactor cannot drop them. + for (const key of ['NO_COLOR', 'FORCE_COLOR', 'CODEX_CI', 'CI', 'TERM', 'TERMINFO', 'PAGER', 'GIT_PAGER', 'GH_PAGER']) { + expect(INVOKER_TERMINAL_ENV_KEYS).toContain(key); + } + // TERMINFO_DIRS is machine-level terminfo search-path config on + // NixOS/custom-ncurses hosts, not an invoker fingerprint — never scrub it. + expect(INVOKER_TERMINAL_ENV_KEYS).not.toContain('TERMINFO_DIRS'); + }); +}); + +describe('scrubSessionTurnMarkerEnv()', () => { + it('removes turn-scoped session identity, leaving documented ambient config alone', () => { + const env: NodeJS.ProcessEnv = { + ...Object.fromEntries(SESSION_TURN_MARKER_ENV_KEYS.map((key) => [key, 'stale-turn'])), + // Documented ambient daemon config channels must NOT be swept by this + // scrub (they are handled by resolveDaemonEnv / registry precedence). + BOTS_CONFIG: '/alt/bots.json', + BOTMUX_PUBLIC_URL: 'https://botmux.example', + KEEP: 'v', + }; + + scrubSessionTurnMarkerEnv(env); + + for (const key of SESSION_TURN_MARKER_ENV_KEYS) { + expect(key in env, key).toBe(false); + } + expect(env.BOTS_CONFIG).toBe('/alt/bots.json'); + expect(env.BOTMUX_PUBLIC_URL).toBe('https://botmux.example'); + expect(env.KEEP).toBe('v'); + }); + + it('covers both owner channels so a stale owner can never be baked fleet-wide', () => { + expect(SESSION_TURN_MARKER_ENV_KEYS).toContain('BOTMUX_OWNER_OPEN_ID'); + expect(SESSION_TURN_MARKER_ENV_KEYS).toContain('__OWNER_OPEN_ID'); + expect(SESSION_TURN_MARKER_ENV_KEYS).toContain('BOTMUX_SESSION_ID'); + }); + + it('covers session-only capabilities AND routing keys the pane transport never carries', () => { + // The list is hand-maintained on the "session-scoped by construction" + // criterion, NOT derived from BOTMUX_INJECTED_ENV_KEYS — that list is the + // pane TRANSPORT whitelist and mixes in ambient config. Both directions + // must hold: capabilities that ARE transported, and routing keys that are + // NOT (BOTMUX_SESSION_SCOPE / BOTMUX_SEND_RELAY reach children outside + // the pane injection list). + for (const key of [ + 'BOTMUX_MCP_GATEWAY_SOCKET', + 'BOTMUX_MCP_GATEWAY_REQUIRED', + 'BOTMUX_DAEMON_IPC_PORT', + 'BOTMUX_READ_ISOLATION', + 'BOTMUX_READ_ISOLATED', + 'BOTMUX_API_ONLY', + 'IS_SANDBOX', + 'BOTMUX_ORIGIN_CHANNEL_ID', + 'BOTMUX_LARK_APP_ID', + 'BOTMUX_SESSION_SCOPE', + 'BOTMUX_SEND_RELAY', + ]) { + expect(SESSION_TURN_MARKER_ENV_KEYS, key).toContain(key); + } + }); + + it('legitimate ambient/daemon config survives the FULL pm2/daemon-boot scrub stack', () => { + // The regression this pins: CLAUDE_CODE_RESUME_TOKEN_THRESHOLD's only + // legitimate channel is daemon ambient env (worker.ts reads process.env, + // per-bot env rejects the key) — index-daemon loads ~/.botmux/.env via + // dotenv and THEN runs these scrubs, so including it in any scrub family + // silently kills the user's setting. Same for the HERMES install-location + // roots (nothing in botmux sets them; ambient-only, like GROK_HOME) and + // the documented ambient/ecosystem config keys. + const env: NodeJS.ProcessEnv = { + CLAUDE_CODE_RESUME_TOKEN_THRESHOLD: '150000', + HERMES_HOME: '/opt/hermes', + HERMES_BOTMUX_SOURCE_HOME: '/opt/hermes-src', + HERMES_BOTMUX_PROFILES_ROOT: '/opt/hermes-profiles', + BOTS_CONFIG: '/alt/bots.json', + SESSION_DATA_DIR: '/data/botmux', + BOTMUX_LARK_LIST_BOTS_API_ENABLED: 'true', + BOTMUX_LARK_LIST_BOTS_API_TIMEOUT_MS: '3000', + GROK_HOME: '/opt/grok', + }; + + // The full boundary stack, in the index-daemon boot order. + scrubSessionTurnMarkerEnv(env); + scrubSessionCliHomeEnv(env); + scrubClaudeSessionMarkerEnv(env); + scrubWorkflowWorkerEnv(env); + scrubInvokerTerminalEnv(env); + + expect(env.CLAUDE_CODE_RESUME_TOKEN_THRESHOLD).toBe('150000'); + expect(env.HERMES_HOME).toBe('/opt/hermes'); + expect(env.HERMES_BOTMUX_SOURCE_HOME).toBe('/opt/hermes-src'); + expect(env.HERMES_BOTMUX_PROFILES_ROOT).toBe('/opt/hermes-profiles'); + expect(env.BOTS_CONFIG).toBe('/alt/bots.json'); + expect(env.SESSION_DATA_DIR).toBe('/data/botmux'); + expect(env.BOTMUX_LARK_LIST_BOTS_API_ENABLED).toBe('true'); + expect(env.BOTMUX_LARK_LIST_BOTS_API_TIMEOUT_MS).toBe('3000'); + expect(env.GROK_HOME).toBe('/opt/grok'); + }); + + it('true session-only values are deleted by the same stack', () => { + const env: NodeJS.ProcessEnv = { + BOTMUX_SESSION_ID: 's-1', + BOTMUX_SESSION_SCOPE: 'thread', + BOTMUX_SEND_RELAY: '/tmp/relay', + BOTMUX_MCP_GATEWAY_SOCKET: '/tmp/mcp.sock', + BOTMUX_DAEMON_IPC_PORT: '7951', + BOTMUX_OWNER_OPEN_ID: 'ou_x', + IS_SANDBOX: '1', + CLAUDE_CONFIG_DIR: '/leak/claude', + CLAUDECODE: '1', + NO_COLOR: '1', + }; + + scrubSessionTurnMarkerEnv(env); + scrubSessionCliHomeEnv(env); + scrubClaudeSessionMarkerEnv(env); + scrubWorkflowWorkerEnv(env); + scrubInvokerTerminalEnv(env); + + for (const key of Object.keys(env)) { + expect.fail(`expected every key deleted, found ${key}`); + } + }); +}); + describe('session CLI home scrub call sites', () => { // The scrub only works if every process boundary actually invokes it. These // source-level pins keep a refactor from silently dropping a boundary: @@ -419,6 +574,33 @@ describe('session CLI home scrub call sites', () => { expect(worker).toContain('zellijEnv(redactChildEnv(process.env))'); }); + it('pm2 boundaries and daemon boot scrub invoker-terminal fingerprints and turn markers', () => { + // Same persistence vector as the scrubs above, fourth and fifth key + // families: agent-shell fingerprints (NO_COLOR/CODEX_CI/PAGER — colorless + // fleet TUIs) and turn-scoped session identity. Both pm2 client boundaries + // (core cli.ts pm2Env → cli/pm2-env.ts, and plugin pm2.ts, which share + // the God's PM2_HOME and both route through scrubPm2CallerEnv) must bake + // clean env; daemon boot additionally heals a fleet already poisoned by + // an earlier restart or a stale dump.pm2. + const pm2EnvSrc = read('cli/pm2-env.ts'); + const fn = pm2EnvSrc.slice(pm2EnvSrc.indexOf('export function scrubPm2CallerEnv(')); + const fnBody = fn.slice(0, fn.indexOf('\n}')); + expect(fnBody).toContain('scrubInvokerTerminalEnv('); + expect(fnBody).toContain('scrubSessionTurnMarkerEnv('); + // TERM is re-pinned (not left absent) inside the shared scrub so pm2 + // CLIENT output on a real TTY keeps supports-color detection. + expect(fnBody).toContain("env.TERM = 'xterm-256color'"); + const pluginPm2 = read('core/plugins/pm2.ts'); + expect(pluginPm2).toContain('scrubPm2CallerEnv('); + expect(read('index-daemon.ts')).toContain('scrubInvokerTerminalEnv(process.env)'); + expect(read('index-daemon.ts')).toContain('scrubSessionTurnMarkerEnv(process.env)'); + // Daemon boot must re-pin too: the boot scrub runs AFTER pm2Env() baked + // its snapshot, so without this the daemon (and every forked worker) runs + // TERM-less — the zmx backend's sessions inherit that env verbatim (no + // node-pty `name` to force TERM) and their CLIs render colorless. + expect(read('index-daemon.ts')).toContain("process.env.TERM = 'xterm-256color'"); + }); + it('worker-pool strips the PM2 sentinel when forking a worker (source pin)', () => { // WORKER_REDACTED_ENV_KEYS is a private const in worker-pool.ts (worker fork // boundary, not importable without side effects), so pin at the source that diff --git a/test/log-tail.test.ts b/test/log-tail.test.ts new file mode 100644 index 000000000..07f50cd94 --- /dev/null +++ b/test/log-tail.test.ts @@ -0,0 +1,162 @@ +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import { appendFileSync, mkdtempSync, renameSync, rmSync, unlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + formatLogLine, + lastLinesOfChunk, + LogFileFollower, + type LogTailSource, +} from '../src/cli/log-tail.js'; + +describe('lastLinesOfChunk', () => { + it('takes the last N complete lines, ignoring the trailing newline', () => { + expect(lastLinesOfChunk('a\nb\nc\n', 2)).toEqual(['b', 'c']); + expect(lastLinesOfChunk('a\nb\nc', 2)).toEqual(['b', 'c']); + expect(lastLinesOfChunk('a\n', 5)).toEqual(['a']); + expect(lastLinesOfChunk('', 3)).toEqual([]); + expect(lastLinesOfChunk('a\nb', 0)).toEqual([]); + }); +}); + +describe('LogFileFollower', () => { + let dir: string; + let out: string[]; + + const source = (label: string, stream: 'out' | 'err', file: string): LogTailSource => + ({ label, stream, file }); + + beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), 'botmux-log-tail-')); + out = []; + }); + + afterEach(() => { + rmSync(dir, { recursive: true, force: true }); + }); + + const follower = (sources: LogTailSource[]) => + new LogFileFollower({ sources, writeLine: line => out.push(line) }); + + it('initial tail prints only the last N lines with labels', () => { + const file = join(dir, 'daemon-0-out.log'); + writeFileSync(file, 'l1\nl2\nl3\nl4\n'); + const f = follower([source('botmux-0', 'out', file)]); + f.printInitialTail(2); + expect(out).toEqual(['botmux-0 | l3', 'botmux-0 | l4']); + }); + + it('follows appends and keeps partial lines until completed', () => { + const file = join(dir, 'daemon-0-out.log'); + writeFileSync(file, 'old\n'); + const f = follower([source('botmux-0', 'out', file)]); + f.printInitialTail(10); + out.length = 0; + + appendFileSync(file, 'new-1\npart'); + f.pollOnce(); + expect(out).toEqual(['botmux-0 | new-1']); + + appendFileSync(file, 'ial\n'); + f.pollOnce(); + expect(out).toEqual(['botmux-0 | new-1', 'botmux-0 | partial']); + }); + + it('in-place truncation (pm2 flush) resets and reads the new content', () => { + const file = join(dir, 'daemon-0-out.log'); + writeFileSync(file, 'before-1\nbefore-2\n'); + const f = follower([source('botmux-0', 'out', file)]); + f.printInitialTail(10); + out.length = 0; + + writeFileSync(file, 'after\n'); // shorter than the old offset + f.pollOnce(); + expect(out).toEqual(['botmux-0 | after']); + }); + + it('a file that appears after start is picked up from its beginning', () => { + // The fleet-stopped case: `botmux logs` keeps running while a later + // `botmux start` creates the log files. + const file = join(dir, 'daemon-1-out.log'); + const f = follower([source('botmux-1', 'out', file)]); + f.printInitialTail(10); + expect(out).toEqual([]); + + f.pollOnce(); // still absent + writeFileSync(file, 'born\n'); + f.pollOnce(); + expect(out).toEqual(['botmux-1 | born']); + }); + + it('a file that disappears and returns is re-read from the start', () => { + const file = join(dir, 'daemon-0-out.log'); + writeFileSync(file, 'gen1\n'); + const f = follower([source('botmux-0', 'out', file)]); + f.printInitialTail(10); + out.length = 0; + + unlinkSync(file); + f.pollOnce(); + writeFileSync(file, 'gen2\n'); + f.pollOnce(); + expect(out).toEqual(['botmux-0 | gen2']); + }); + + it('an unterminated trailing line at startup is carried, not emitted, and completes as ONE line', () => { + // The split-line bug this pins: initial tail must not print "abc" as a + // finished line when the file ends without a newline — the later "def\n" + // append belongs to the SAME line. + const file = join(dir, 'daemon-0-out.log'); + writeFileSync(file, 'done-1\nabc'); + const f = follower([source('botmux-0', 'out', file)]); + f.printInitialTail(10); + expect(out).toEqual(['botmux-0 | done-1']); + + appendFileSync(file, 'def\n'); + f.pollOnce(); + expect(out).toEqual(['botmux-0 | done-1', 'botmux-0 | abcdef']); + }); + + it('atomic rotation (rename + recreate, new size ≥ old offset) is detected by file identity', () => { + // The skipped-prefix bug this pins: with offset-only state, a same-path + // replacement whose size already exceeds the old offset passes the + // size { + const file = join(dir, 'daemon-0-out.log'); + // ~130KiB of 26-byte lines so the last 3 lines sit beyond one window + // boundary and the window must grow to stay line-aligned. + const line = (i: number) => `line-${String(i).padStart(6, '0')}-xxxxxxxxxx`; + const total = 5200; + writeFileSync(file, Array.from({ length: total }, (_, i) => line(i)).join('\n') + '\n'); + const f = follower([source('botmux-0', 'out', file)]); + f.printInitialTail(3); + expect(out).toEqual([ + `botmux-0 | ${line(total - 3)}`, + `botmux-0 | ${line(total - 2)}`, + `botmux-0 | ${line(total - 1)}`, + ]); + }); + + it('error-stream lines carry the (err) marker', () => { + const file = join(dir, 'daemon-0-error.log'); + writeFileSync(file, 'boom\n'); + const f = follower([source('botmux-0', 'err', file)]); + f.printInitialTail(1); + expect(out).toEqual(['botmux-0 (err) | boom']); + expect(formatLogLine(source('x', 'err', file), 'y')).toBe('x (err) | y'); + }); +}); diff --git a/test/plugin-pm2-env.test.ts b/test/plugin-pm2-env.test.ts index ca81b5056..96422daa3 100644 --- a/test/plugin-pm2-env.test.ts +++ b/test/plugin-pm2-env.test.ts @@ -84,4 +84,64 @@ describe('plugin PM2 environment', () => { expect(options.env.PLUGIN_VALUE).toBe('preserved'); expect(options.env.PATH).toBe(process.env.PATH); }); + + it('applies the same five scrub families as the core pm2 boundary', async () => { + // Plugin PM2 shares the God's PM2_HOME, so this entry both persists env + // into plugin apps and can birth the shared God — a plugin start issued + // from a bot/workflow session must not carry the session's CLI home, + // Claude markers, workflow identity, agent-shell fingerprints, or turn + // identity into either. + vi.stubEnv('CLAUDE_CONFIG_DIR', '/leak/claude'); + vi.stubEnv('CODEX_HOME', '/leak/codex'); + vi.stubEnv('CLAUDECODE', '1'); + vi.stubEnv('BOTMUX_WORKFLOW', 'wf-1'); + vi.stubEnv('NO_COLOR', '1'); + vi.stubEnv('CODEX_CI', '1'); + vi.stubEnv('BOTMUX_SESSION_ID', 'session-leak'); + vi.stubEnv('BOTMUX_OWNER_OPEN_ID', 'ou_leak'); + vi.resetModules(); + const { runPluginPm2 } = await import('../src/core/plugins/pm2.js'); + + runPluginPm2(['start', 'fixture'], { inherit: false }); + + const options = childProcess.spawnSync.mock.calls[0]?.[2] as { env: NodeJS.ProcessEnv }; + for (const key of [ + 'CLAUDE_CONFIG_DIR', 'CODEX_HOME', 'CLAUDECODE', 'BOTMUX_WORKFLOW', + 'NO_COLOR', 'CODEX_CI', 'BOTMUX_SESSION_ID', 'BOTMUX_OWNER_OPEN_ID', + ]) { + expect(options.env[key], key).toBeUndefined(); + } + // Deterministic TERM instead of absent (pm2 client color detection). + expect(options.env.TERM).toBe('xterm-256color'); + }); + + it('freezes the scrubs over the manifest env merge — extras cannot revive scrubbed keys', async () => { + const { PM2_GRACEFUL_EXIT_CODE_ENV } = await import('../src/pm2-graceful-exit.js'); + vi.resetModules(); + const { runPluginPm2 } = await import('../src/core/plugins/pm2.js'); + + runPluginPm2(['start', 'fixture'], { + inherit: false, + env: { + BOTMUX_SESSION_ID: 'manifest-forged', + CLAUDECODE: '1', + NO_COLOR: '1', + TERM: 'dumb', + BOTMUX_DASHBOARD_FEISHU_H5_APP_SECRET: 'manifest-forged-secret', + [PM2_GRACEFUL_EXIT_CODE_ENV]: '90', + PM2_HOME: '/forged/pm2-home', + PLUGIN_VALUE: 'preserved', + }, + }); + + const options = childProcess.spawnSync.mock.calls[0]?.[2] as { env: NodeJS.ProcessEnv }; + expect(options.env.BOTMUX_SESSION_ID).toBeUndefined(); + expect(options.env.CLAUDECODE).toBeUndefined(); + expect(options.env.NO_COLOR).toBeUndefined(); + expect(options.env[PM2_GRACEFUL_EXIT_CODE_ENV]).toBeUndefined(); + expect(options.env.BOTMUX_DASHBOARD_FEISHU_H5_APP_SECRET).toBeUndefined(); + expect(options.env.TERM).toBe('xterm-256color'); + expect(options.env.PM2_HOME).toBe(join(home, '.botmux', 'pm2')); + expect(options.env.PLUGIN_VALUE).toBe('preserved'); + }); }); diff --git a/test/plugin-service-restart-lifecycle.test.ts b/test/plugin-service-restart-lifecycle.test.ts index c567d5b61..b109d6a9d 100644 --- a/test/plugin-service-restart-lifecycle.test.ts +++ b/test/plugin-service-restart-lifecycle.test.ts @@ -15,12 +15,15 @@ function restartFunctionSource(): string { describe('plugin service restart lifecycle', () => { it('preserves auto services by default and always ensures them after core starts', () => { const source = restartFunctionSource(); - const stop = 'if (includePluginServices) await stopPluginServicesForCli(undefined, { autoOnly: true });'; + const stop = 'await stopPluginServicesForCli(undefined, { autoOnly: true });'; const transaction = 'runBoundedPm2StartTransaction('; const coreStart = "runPm2(['start', cfg], true, PM2_HOME, timeoutMs);"; const ensure = 'await reconcilePluginServicesForCli(undefined, { autoOnly: true });'; expect(source).toContain(stop); + // Default restart still stops nothing: the autoOnly stop stays behind the + // explicit --with-plugin flag (include-pm2 has its own all-services stop). + expect(source).toContain('else if (includePluginServices) {'); expect(source).toContain(transaction); expect(source).toContain(coreStart); expect(source).toContain(ensure); diff --git a/test/pm2-fleet-lock.test.ts b/test/pm2-fleet-lock.test.ts new file mode 100644 index 000000000..cb4549772 --- /dev/null +++ b/test/pm2-fleet-lock.test.ts @@ -0,0 +1,80 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + pm2FleetMutationLockTarget, + withPm2FleetMutationLock, + withPm2FleetMutationLockSync, +} from '../src/cli/pm2-fleet-lock.js'; + +/** + * BEHAVIOR tests (not source pins) for the ownership model: re-entrancy must + * be scoped to the async call chain that actually holds the lock. A + * process-global "held" flag would let an unrelated concurrent flow in the + * same process (dashboard HTTP handlers) skip the file lock while another + * flow holds it. + */ +describe('withPm2FleetMutationLock ownership', () => { + let home: string; + + beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'botmux-fleet-lock-')); + vi.stubEnv('HOME', home); + mkdirSync(join(home, '.botmux'), { recursive: true }); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + rmSync(home, { recursive: true, force: true }); + }); + + it('nested calls within the holding chain short-circuit (async and sync)', async () => { + const result = await withPm2FleetMutationLock(async () => { + const inner = await withPm2FleetMutationLock(async () => 'inner-async'); + const innerSync = withPm2FleetMutationLockSync(() => 'inner-sync'); + return `${inner}/${innerSync}`; + }, { maxWaitMs: 2_000 }); + expect(result).toBe('inner-async/inner-sync'); + }); + + it('two INDEPENDENT concurrent chains in one process serialize on the file lock', async () => { + // The dashboard scenario: chain A holds the lock and is suspended at an + // await; chain B starts concurrently. B must NOT treat A's ownership as + // its own — it has to queue on the file lock until A releases. + const events: string[] = []; + let releaseA!: () => void; + const aInside = new Promise(resolveInside => { + void withPm2FleetMutationLock(async () => { + events.push('A-enter'); + resolveInside(); + await new Promise(resolve => { releaseA = resolve; }); + events.push('A-exit'); + }, { maxWaitMs: 5_000 }); + }); + await aInside; + + const b = withPm2FleetMutationLock(async () => { + events.push('B-enter'); + }, { maxWaitMs: 5_000 }); + + // Give B ample time to (incorrectly) enter if ownership were process-global. + await new Promise(resolve => setTimeout(resolve, 300)); + expect(events).toEqual(['A-enter']); + + releaseA(); + await b; + expect(events).toEqual(['A-enter', 'A-exit', 'B-enter']); + }); + + it('a lock held by another live process makes an unrelated chain wait, then time out', async () => { + // Simulate an external holder: a fresh lock file whose recorded pid is + // alive (our own pid — file-lock treats a live same-pid holder as HELD, + // never stale). A chain with no ownership context must queue and + // eventually time out instead of stealing or skipping the lock. + writeFileSync(`${pm2FleetMutationLockTarget()}.lock`, String(process.pid)); + await expect( + withPm2FleetMutationLock(async () => 'must-not-run', { maxWaitMs: 400 }), + ).rejects.toThrow(/file-lock timeout/); + }); +}); diff --git a/test/pm2-god-admission.test.ts b/test/pm2-god-admission.test.ts deleted file mode 100644 index 2b252ffc0..000000000 --- a/test/pm2-god-admission.test.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { describe, expect, it, vi } from 'vitest'; -import { assertIncludePm2RestartAdmission } from '../src/cli/pm2-god-admission.js'; - -describe('restart --include-pm2 admission', () => { - it('admits only an initially zero-God state', () => { - expect(() => assertIncludePm2RestartAdmission([])).not.toThrow(); - }); - - it('rejects a live God before any caller mutation', () => { - const mutate = vi.fn(); - expect(() => { - assertIncludePm2RestartAdmission([101]); - mutate(); - }).toThrow(/cannot be signalled with generation-bound authority.*does not signal or restart.*no process or breadcrumb was changed/); - expect(mutate).not.toHaveBeenCalled(); - }); - - it('rejects duplicate Gods without selecting either generation', () => { - expect(() => assertIncludePm2RestartAdmission([101, 202])) - .toThrow(/multiple PM2 God daemons.*no process or breadcrumb was changed/); - }); -}); diff --git a/test/pm2-god-retirement.test.ts b/test/pm2-god-retirement.test.ts new file mode 100644 index 000000000..a296cb5bb --- /dev/null +++ b/test/pm2-god-retirement.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it } from 'vitest'; +import { + assertNoReplacementPm2God, + assertPm2RegistryQuiescentForGodRetirement, + retireSoleLivePm2God, + type Pm2GodRetirementRuntime, +} from '../src/cli/pm2-god-retirement.js'; + +interface HarnessOptions { + scans: number[][]; + identities?: Record>; + alive?: Record; + killError?: Error; +} + +function harness(opts: HarnessOptions) { + const calls = { kill: 0, sleeps: 0 }; + let clock = 0; + const scanQueue = [...opts.scans]; + const identityQueues = new Map>( + Object.entries(opts.identities ?? {}).map(([pid, q]) => [Number(pid), [...q]]), + ); + const aliveQueues = new Map( + Object.entries(opts.alive ?? {}).map(([pid, q]) => [Number(pid), [...q]]), + ); + const shift = (queue: T[] | undefined, fallback: T): T => + queue === undefined || queue.length === 0 ? fallback : queue.length === 1 ? queue[0] : queue.shift()!; + const rt: Pm2GodRetirementRuntime = { + listGodPids: () => shift(scanQueue.length > 0 ? scanQueue : undefined, []), + readStartIdentity: pid => shift(identityQueues.get(pid), undefined), + isAlive: pid => shift(aliveQueues.get(pid), false), + pm2Kill: () => { + calls.kill++; + if (opts.killError) throw opts.killError; + }, + sleep: async () => { calls.sleeps++; clock += 1_000; }, + now: () => clock, + }; + return { rt, calls }; +} + +describe('retireSoleLivePm2God', () => { + it('no live God → returns null without touching pm2', async () => { + const { rt, calls } = harness({ scans: [[]] }); + await expect(retireSoleLivePm2God(rt)).resolves.toBeNull(); + expect(calls.kill).toBe(0); + }); + + it('multiple visible Gods → fails closed before any kill', async () => { + const { rt, calls } = harness({ scans: [[101, 202]] }); + await expect(retireSoleLivePm2God(rt)).rejects.toThrow(/multiple PM2 God daemons/); + expect(calls.kill).toBe(0); + }); + + it('invalid or duplicate scan rows → fails closed before any kill', async () => { + for (const scan of [[0], [-3], [7046, 7046]]) { + const { rt, calls } = harness({ scans: [scan] }); + await expect(retireSoleLivePm2God(rt)).rejects.toThrow(/invalid\/duplicate PIDs/); + expect(calls.kill).toBe(0); + } + }); + + it('sole God → socket kill, then verified gone by scan + pid death', async () => { + const { rt, calls } = harness({ + scans: [[7046], [7046], []], + identities: { 7046: ['birth-A'] }, + alive: { 7046: [true, false] }, + }); + await expect(retireSoleLivePm2God(rt)).resolves.toEqual({ pid: 7046, startIdentity: 'birth-A' }); + expect(calls.kill).toBe(1); + }); + + it('pid reused by a different birth counts as gone once the scan is empty', async () => { + const { rt } = harness({ + scans: [[7046], []], + identities: { 7046: ['birth-A', 'birth-B'] }, + alive: { 7046: [true] }, + }); + await expect(retireSoleLivePm2God(rt)).resolves.toEqual({ pid: 7046, startIdentity: 'birth-A' }); + }); + + it('God still observable at the deadline → fails closed with recovery guidance', async () => { + const { rt } = harness({ + scans: [[7046], [7046]], + identities: { 7046: ['birth-A'] }, + alive: { 7046: [true] }, + }); + await expect(retireSoleLivePm2God(rt, 3_000)) + .rejects.toThrow(/still observable after pm2 kill/); + }); + + it('pm2 kill failure propagates', async () => { + const { rt } = harness({ + scans: [[7046]], + killError: new Error('pm2 kill failed: status 1'), + }); + await expect(retireSoleLivePm2God(rt)).rejects.toThrow(/pm2 kill failed/); + }); +}); + +describe('assertPm2RegistryQuiescentForGodRetirement', () => { + it('accepts an empty registry and terminal rows without live pids', () => { + expect(() => assertPm2RegistryQuiescentForGodRetirement([])).not.toThrow(); + expect(() => assertPm2RegistryQuiescentForGodRetirement([ + { name: 'botmux-plugin-a', status: 'stopped', pid: undefined }, + { name: 'botmux-plugin-b', status: 'stopped', pid: 0 }, + { name: 'botmux-plugin-c', status: 'errored', pid: undefined }, + ])).not.toThrow(); + }); + + it('refuses when a plugin service failed to stop and is still online', () => { + expect(() => assertPm2RegistryQuiescentForGodRetirement([ + { name: 'botmux-plugin-hung', status: 'online', pid: 4321 }, + ])).toThrow(/still has live\/unproven row\(s\).*botmux-plugin-hung:online:pid 4321/s); + }); + + it('refuses an orphaned running row left behind by an uninstalled plugin', () => { + // stopPluginServices only iterates registry records that still carry a + // service definition; an uninstalled plugin's leftover PM2 row is invisible + // to it and MUST be caught here instead of dying silently with the God. + expect(() => assertPm2RegistryQuiescentForGodRetirement([ + { name: 'botmux-plugin-uninstalled-leftover', status: 'online', pid: 555 }, + { name: 'botmux-plugin-ok', status: 'stopped', pid: undefined }, + ])).toThrow(/botmux-plugin-uninstalled-leftover/); + }); + + it('refuses non-terminal statuses even without a pid, and live pids even when "stopped"', () => { + for (const row of [ + { name: 'r1', status: 'launching', pid: undefined }, + { name: 'r2', status: 'stopping', pid: undefined }, + { name: 'r3', status: undefined, pid: undefined }, + { name: 'r4', status: 'stopped', pid: 999 }, + ]) { + expect(() => assertPm2RegistryQuiescentForGodRetirement([row]), row.name).toThrow(); + } + }); +}); + +describe('assertNoReplacementPm2God', () => { + it('passes when no God exists right before the fresh start', () => { + expect(() => assertNoReplacementPm2God([])).not.toThrow(); + }); + + it('refuses a God inserted between retirement and start', () => { + expect(() => assertNoReplacementPm2God([8123])) + .toThrow(/replacement PM2 God \(pid 8123\)/); + }); +}); diff --git a/test/riff-keychain-jwt.test.ts b/test/riff-keychain-jwt.test.ts index 85cd7c260..72a8e4d91 100644 --- a/test/riff-keychain-jwt.test.ts +++ b/test/riff-keychain-jwt.test.ts @@ -198,7 +198,7 @@ describe('readBytecloudKeychainJwt — token extraction', () => { writeKeychain(join('.config', 'kaboo-cli'), { access_token: 'a', bytecloud_jwt: 'JWT-KABOO', refresh_token: 'r', }); - expect(readBytecloudKeychainJwt(home, bare)).toBe('JWT-KABOO'); + expect(readBytecloudKeychainJwt(home, bare, Date.now(), 'linux')).toBe('JWT-KABOO'); }); it('reads bytecloud_jwt from bytedcli data dir (the Mac-verified layout)', () => { @@ -222,14 +222,14 @@ describe('readBytecloudKeychainJwt — token extraction', () => { writeFileSync(join(dir, 'credentials.json'), JSON.stringify({ app_id: 'x', expires_at: 123, user: 'u', // note: NO bytecloud_jwt }), 'utf-8'); - expect(readBytecloudKeychainJwt(home, bare)).toBeNull(); + expect(readBytecloudKeychainJwt(home, bare, Date.now(), 'linux')).toBeNull(); }); it('skips a keychain file whose bytecloud_jwt is empty and keeps scanning', () => { // kaboo has an empty token; bytedcli has a real one — later candidate wins. writeKeychain(join('.config', 'kaboo-cli'), { bytecloud_jwt: '' }); writeKeychain(join('.local', 'share', 'bytedcli', 'data'), { bytecloud_jwt: 'JWT-REAL' }); - expect(readBytecloudKeychainJwt(home, bare)).toBe('JWT-REAL'); + expect(readBytecloudKeychainJwt(home, bare, Date.now(), 'linux')).toBe('JWT-REAL'); }); it('skips a malformed (non-JSON) keychain file without throwing', () => { @@ -237,7 +237,7 @@ describe('readBytecloudKeychainJwt — token extraction', () => { mkdirSync(dir, { recursive: true }); writeFileSync(join(dir, 'default'), 'not-json{{{', 'utf-8'); writeKeychain(join('.cjadk'), { bytecloud_jwt: 'JWT-CJADK' }); - expect(readBytecloudKeychainJwt(home, bare)).toBe('JWT-CJADK'); + expect(readBytecloudKeychainJwt(home, bare, Date.now(), 'linux')).toBe('JWT-CJADK'); }); it('reads the bytedcli keychain from the AIME workspace path (both AIME vars set)', () => { @@ -281,13 +281,13 @@ describe('readBytecloudKeychainJwt — expiry-aware selection (stale must not sh const freshest = makeJwt(NOW + 9999); writeKeychain('.cjadk', { bytecloud_jwt: freshest }); writeKeychain(join('.local', 'share', 'bytedcli', 'data'), { bytecloud_jwt: makeJwt(NOW + 500) }); - expect(readBytecloudKeychainJwt(home, bare, nowMs)).toBe(freshest); + expect(readBytecloudKeychainJwt(home, bare, nowMs, 'linux')).toBe(freshest); }); it('returns null when every parseable token is expired (and no opaque fallback exists)', () => { writeKeychain(join('.config', 'kaboo-cli'), { bytecloud_jwt: makeJwt(NOW - 1) }); writeKeychain(join('.local', 'share', 'bytedcli', 'data'), { bytecloud_jwt: makeJwt(NOW - 999) }); - expect(readBytecloudKeychainJwt(home, bare, nowMs)).toBeNull(); + expect(readBytecloudKeychainJwt(home, bare, nowMs, 'linux')).toBeNull(); }); it('an opaque (exp-less) token is used only as fallback, never over a parseable live token', () => { @@ -295,20 +295,20 @@ describe('readBytecloudKeychainJwt — expiry-aware selection (stale must not sh writeKeychain(join('.config', 'kaboo-cli'), { bytecloud_jwt: 'opaque-no-exp' }); const live = makeJwt(NOW + 3600); writeKeychain(join('.local', 'share', 'bytedcli', 'data'), { bytecloud_jwt: live }); - expect(readBytecloudKeychainJwt(home, bare, nowMs)).toBe(live); + expect(readBytecloudKeychainJwt(home, bare, nowMs, 'linux')).toBe(live); }); it('falls back to an opaque token when no parseable-live token exists', () => { // only an opaque token present → use it (better than nothing; we cannot judge its exp). writeKeychain(join('.config', 'kaboo-cli'), { bytecloud_jwt: 'opaque-only' }); - expect(readBytecloudKeychainJwt(home, bare, nowMs)).toBe('opaque-only'); + expect(readBytecloudKeychainJwt(home, bare, nowMs, 'linux')).toBe('opaque-only'); }); it('prefers a parseable-live token over an opaque one even when the opaque is listed later', () => { const live = makeJwt(NOW + 3600); writeKeychain(join('.config', 'kaboo-cli'), { bytecloud_jwt: live }); writeKeychain(join('.local', 'share', 'bytedcli', 'data'), { bytecloud_jwt: 'opaque-later' }); - expect(readBytecloudKeychainJwt(home, bare, nowMs)).toBe(live); + expect(readBytecloudKeychainJwt(home, bare, nowMs, 'linux')).toBe(live); }); it('a non-3-segment fake token (2 or 4 segments) must NOT shadow a real JWT via a decodable exp', () => { @@ -319,7 +319,7 @@ describe('readBytecloudKeychainJwt — expiry-aware selection (stale must not sh const real = makeJwt(NOW + 3600); writeKeychain(join('.config', 'kaboo-cli'), { bytecloud_jwt: fake2 }); writeKeychain(join('.local', 'share', 'bytedcli', 'data'), { bytecloud_jwt: real }); - expect(readBytecloudKeychainJwt(home, bare, nowMs)).toBe(real); + expect(readBytecloudKeychainJwt(home, bare, nowMs, 'linux')).toBe(real); }); }); diff --git a/test/shutdown-supervisor-contract.test.ts b/test/shutdown-supervisor-contract.test.ts index 264aa50c1..bafca74a6 100644 --- a/test/shutdown-supervisor-contract.test.ts +++ b/test/shutdown-supervisor-contract.test.ts @@ -226,10 +226,27 @@ describe('graceful shutdown supervisor contract', () => { const region = cli.slice(start, end); expect(start, label).toBeGreaterThanOrEqual(0); expect(end, label).toBeGreaterThan(start); - expect(region, label).toContain('withFileLock(PM2_FLEET_MUTATION_LOCK_TARGET'); - expect(region, label).not.toContain('withFileLockSync(PM2_FLEET_MUTATION_LOCK_TARGET'); + expect(region, label).toContain('withPm2FleetMutationLock('); + expect(region, label).not.toContain('withPm2FleetMutationLockSync('); } + // Plugin services live under the SAME God, so their lifecycle must share + // the same fleet lock — in fixed order (fleet first, service second), and + // even for the status probe, whose pm2 jlist can lazily birth a God. + const serviceManager = readFileSync( + new URL('../src/core/plugins/service-manager.ts', import.meta.url), 'utf8', + ); + expect(serviceManager).toContain( + 'withPm2FleetMutationLockSync(\n () => withFileLockSync(serviceLockTarget()', + ); + expect(serviceManager).toContain( + 'withPm2FleetMutationLock(\n () => withFileLock(serviceLockTarget()', + ); + const statusFn = serviceManager.slice( + serviceManager.indexOf('export async function listPluginServiceStatus('), + ); + expect(statusFn.slice(0, statusFn.indexOf('\n}'))).toContain('withPluginServiceLock('); + const exactHelper = cli.slice( cli.indexOf('async function cmdInternalPm2StartExact('), cli.indexOf('function runExactPm2Starts(', cli.indexOf('async function cmdInternalPm2StartExact(')), @@ -239,6 +256,49 @@ describe('graceful shutdown supervisor contract', () => { expect(exactHelper).toContain('lockPid !== process.ppid'); }); + it('read-only status/logs cannot lazily birth a PM2 God', () => { + // Any pm2 client invocation with no live God daemonizes one from its own + // env (pm2 Client.start → pingDaemon false → launchDaemon). + // + // cmdStatus keeps a REAL atomic gate: the God scan and the pm2 call both + // run inside the fleet lock (runPm2 is synchronous), and God retirement + // needs that same lock, so no interleaving can retire the observed God + // before the client connects. On lock-wait timeout it exits non-zero. + const statusStart = cli.indexOf('async function cmdStatus('); + const status = cli.slice(statusStart, cli.indexOf('function cmdUpgrade(', statusStart)); + expect(status).toContain('withPm2FleetMutationLock('); + const statusGate = status.indexOf('listPm2GodDaemonPids().length === 0'); + const statusRun = status.indexOf("runPm2(['status'])"); + expect(statusGate).toBeGreaterThanOrEqual(0); + expect(statusRun).toBeGreaterThan(statusGate); + expect(status).toContain('process.exitCode = 1'); + + // cmdLogs cannot get the same shape — its PM2 client would outlive any + // held lock, and "gate, release, then spawn" is a check/use race (the God + // can be retired between release and the client's connect). So logs runs + // NO pm2 client at all: it tails the log files pm2 writes. Structurally + // there is nothing left to race — pin the absence of every pm2 entry + // point across the TRANSITIVE call chain, not just the function body + // (warnIfLegacyBotmuxAlive once hid a legacy-home jlist client that a + // body-only scan missed). + const logsStart = cli.indexOf('async function cmdLogs('); + const logs = cli.slice(logsStart, statusStart); + expect(logs).toContain('LogFileFollower'); + const legacyWarnStart = cli.indexOf('function warnIfLegacyBotmuxAlive('); + const legacyWarn = cli.slice(legacyWarnStart, cli.indexOf('\n}', legacyWarnStart)); + const logTail = readFileSync(new URL('../src/cli/log-tail.ts', import.meta.url), 'utf8'); + for (const banned of ['runPm2(', 'pm2Capture(', 'pm2Bin(', 'buildPm2SpawnCommand(', 'pm2Env(']) { + expect(logs, `cmdLogs: ${banned}`).not.toContain(banned); + expect(legacyWarn, `warnIfLegacyBotmuxAlive: ${banned}`).not.toContain(banned); + expect(logTail, `log-tail: ${banned}`).not.toContain(banned); + } + // The legacy warning verifies the recorded pid against the process-table + // God marker scan (a stale pid file must not match a reused pid) and + // reads PM2's own pid files instead of asking a client. + expect(legacyWarn).toContain('listPm2GodDaemonPids(legacyHome).includes(legacyPid)'); + expect(legacyWarn).toContain("join(legacyHome, 'pids')"); + }); + it('fails closed before PM2 mutation on duplicate Gods, stale preflight, or unregistered descriptors', () => { const duplicateStart = cli.indexOf('function listSingletonPm2GodDaemonPidsForMutation('); const duplicateEnd = cli.indexOf('function runPm2(', duplicateStart); @@ -384,7 +444,12 @@ describe('graceful shutdown supervisor contract', () => { expect(legacy).toContain('assertNoDuplicatePm2GodDaemons(legacyHome)'); expect(legacy).toContain('preflightNodeSanity(legacyHome)'); - expect(cli).not.toContain("runPm2(['kill']"); + // `pm2 kill` slaughters every managed app without the safe shutdown + // handshake, so the ONLY permitted call site is the include-pm2 God + // retirement inside cmdRestart, which runs strictly after the fleet is + // verified retired (pinned by the God-retirement contract test below). + expect(legacy).not.toContain("runPm2(['kill']"); + expect(cli.split("runPm2(['kill']").length - 1).toBe(1); }); it('exposes an explicit double-confirmed first-upgrade bootstrap without weakening normal shutdown', () => { @@ -407,21 +472,39 @@ describe('graceful shutdown supervisor contract', () => { expect(cli).toContain('botmux restart --bootstrap-shutdown-protocol --yes'); }); - it('rejects include-pm2 before breadcrumb/fleet mutation when a live God exists', () => { + it('retires the God only after the fleet is verified retired, and never by PID signal', () => { const start = cli.indexOf('async function cmdRestart()'); const end = cli.indexOf('/**\n * Bring a SINGLE bot', start); const restart = cli.slice(start, end); - const admission = restart.indexOf( - 'assertIncludePm2RestartAdmission(listPm2GodDaemonPids())', - ); - const consume = restart.indexOf('consumeRestartIntentTo('); - const retire = restart.indexOf('deleteAllBotmuxProcesses()'); - expect(admission).toBeGreaterThanOrEqual(0); - expect(consume).toBeGreaterThan(admission); - expect(retire).toBeGreaterThan(consume); + const coreRetire = restart.indexOf('deleteAllBotmuxProcesses()'); + const pluginStop = restart.indexOf('stopPluginServicesForCli(undefined, {})'); + const strictStops = restart.indexOf("report.action === 'failed'"); + const verifyEmpty = restart.indexOf("readVerifiedBotmuxPm2Projection('restart-start')"); + const quiescentGate = restart.indexOf('assertPm2RegistryQuiescentForGodRetirement('); + const godRetire = restart.indexOf('retireSoleLivePm2God('); + const freshStart = restart.indexOf('runBoundedPm2StartTransaction('); + expect(coreRetire).toBeGreaterThanOrEqual(0); + expect(pluginStop).toBeGreaterThan(coreRetire); + // A plugin stop failure is a collected report, not a thrown error — the + // include-pm2 path must re-check reports and refuse before touching the God. + expect(strictStops).toBeGreaterThan(pluginStop); + expect(verifyEmpty).toBeGreaterThan(strictStops); + // Whole-registry quiescence proof (plugin rows and orphans included) + // strictly between the core projection check and the kill. + expect(quiescentGate).toBeGreaterThan(verifyEmpty); + expect(godRetire).toBeGreaterThan(quiescentGate); + expect(freshStart).toBeGreaterThan(godRetire); + // A God that appears between retirement and the fresh start was born from + // some other client's environment — the start transaction refuses it. + const replacementGuard = restart.indexOf('assertNoReplacementPm2God(listPm2GodDaemonPids())'); + expect(replacementGuard).toBeGreaterThan(godRetire); + expect(replacementGuard).toBeLessThan(restart.indexOf("runPm2(['start', cfg], true, PM2_HOME, timeoutMs)")); + // Socket-addressed kill only — a raw PID signal cannot be generation-bound. + expect(restart).toContain("runPm2(['kill']"); expect(restart).not.toContain('killPm2GodDaemon'); - expect(cli).toContain('--include-pm2 仅允许“入场时没有 live PM2 God”的干净启动'); - expect(cli).not.toContain('--include-pm2 同时重启 PM2 God'); + const godRetirement = readFileSync(new URL('../src/cli/pm2-god-retirement.ts', import.meta.url), 'utf8'); + expect(godRetirement).not.toContain('process.kill'); + expect(cli).toContain('cannot be combined with --include-pm2'); }); it('attests the whole daemon fleet then uses exact IPC batch/successor requests', () => { diff --git a/test/zmx-backend-helpers.test.ts b/test/zmx-backend-helpers.test.ts index 7445be878..424f3a379 100644 --- a/test/zmx-backend-helpers.test.ts +++ b/test/zmx-backend-helpers.test.ts @@ -21,6 +21,7 @@ import { parseZmxShortList, tmuxKeyToBytes, zmxControlEnv, + zmxFreshSessionEnv, ZmxBackend, } from '../src/adapters/backend/zmx-backend.js'; import { @@ -502,6 +503,34 @@ describe('zmx backend pure helpers', () => { expect(controlEnv.PATH).toContain('/bin'); }); + it('pins TERM in the fresh-session create client env (zmx sessions inherit it verbatim)', () => { + const opts = { + cwd: '/tmp/work', + cols: 80, + rows: 24, + env: { PATH: '/bin', BOTMUX_SESSION_ID: 'session-secret' }, + }; + // No TERM inherited (the pm2-boundary scrub removed it from the + // daemon/worker env): the pin supplies the constant instead of leaving + // the whole session TERM-less — CLIs would fail supports-color detection + // and render colorless. + const env = zmxFreshSessionEnv(opts); + expect(env.TERM).toBe('xterm-256color'); + // Still a control-env superset: payload-delivered keys stay stripped. + expect(env.BOTMUX_SESSION_ID).toBeUndefined(); + // An inherited invoker TERM is overridden by the same constant every + // other backend PTY already forces — deterministic, invoker-independent. + expect(zmxFreshSessionEnv({ ...opts, env: { PATH: '/bin', TERM: 'xterm-ghostty' } }).TERM) + .toBe('xterm-256color'); + // Source pin: only the one-shot create client (whose env becomes the + // session env) spawns with the pinned env; control clients (get/set/ + // list/kill) keep the unpinned zmxControlEnv. + const src = readFileSync(new URL('../src/adapters/backend/zmx-backend.ts', import.meta.url), 'utf-8'); + const spawnAt = src.indexOf("spawnSync('zmx', buildFreshAttachArgs("); + expect(spawnAt).toBeGreaterThan(-1); + expect(src.slice(spawnAt, src.indexOf('});', spawnAt))).toContain('env: zmxFreshSessionEnv(opts)'); + }); + it('keeps the POSIX ZMX payload path sourced through the user shell with the argv sentinel', () => { const shShell = makeExecutableShell('sh'); const opts = {