fix(pm2): 恢复 restart --include-pm2 彻底重启并阻断发起方环境泄漏进 fleet

docs-site/docs/en/cli-commands.md

@@ -7,7 +7,7 @@ Manage the daemon and sessions from the terminal.
 | `botmux setup` | Interactive configuration (first run / add / edit / delete a bot) |
 | `botmux start` | Start the daemon (managed by PM2) |
 | `botmux stop` | Stop the daemon |
-| `botmux restart [--include-pm2]` | Restart the daemon (automatically restores active sessions); `--include-pm2` also restarts botmux's PM2 God daemon |
+| `botmux restart [--include-pm2]` | Restart the daemon (automatically restores active sessions); `--include-pm2` additionally retires botmux's PM2 God daemon after the fleet is safely shut down, so the whole process tree restarts from the invoking shell's clean environment (plugin services are gracefully stopped first; auto ones come back after the restart) |
 | `botmux logs [--lines N]` | View logs |
 | `botmux status` | View daemon status |
 | `botmux upgrade` | Upgrade to the latest version |

docs-site/docs/zh/cli-commands.md

@@ -7,7 +7,7 @@
 | `botmux setup` | 交互式配置（首次 / 添加 / 编辑 / 删除机器人） |
 | `botmux start` | 启动 daemon（PM2 管理） |
 | `botmux stop` | 停止 daemon |
-| `botmux restart [--include-pm2]` | 重启 daemon（自动恢复活跃会话）；`--include-pm2` 会同时重启 botmux 专用 PM2 God daemon |
+| `botmux restart [--include-pm2]` | 重启 daemon（自动恢复活跃会话）；`--include-pm2` 会在 fleet 安全关停并验证后同时退役 botmux 专用 PM2 God daemon，让整棵进程树以当前 shell 的干净环境全新启动（插件 service 会先优雅停止，auto 的重启后自动恢复） |
 | `botmux logs [--lines N]` | 查看日志 |
 | `botmux status` | 查看 daemon 状态 |
 | `botmux upgrade` | 升级到最新版本 |

src/adapters/backend/zmx-backend.ts

@@ -1159,7 +1159,9 @@ export class ZmxBackend implements SessionBackend {
         // fake terminal leader and never controls the backing PTY dimensions.
         stdio: ['ignore', 'ignore', 'pipe'],
         timeout: ZMX_COMMAND_TIMEOUT_MS,
-        env: zmxControlEnv(opts),
+        // The session PTY inherits THIS env — zmxFreshSessionEnv pins TERM,
+        // which the pm2-boundary scrub removed from the daemon/worker env.
+        env: zmxFreshSessionEnv(opts),
       });
 
       const ready = this.waitForFreshReady(launch);
@@ -2517,6 +2519,20 @@ function createZmxLaunchPayload(bin: string, args: string[], opts: SpawnOpts): Z
   }
 }
 
+/**
+ * Env for the one-shot client that CREATES a fresh session. Unlike the
+ * node-pty backends (which force TERM via `name: 'xterm-256color'`), zmx sets
+ * no TERM of its own: the forkpty child inherits the create client's env
+ * verbatim, and the daemon/worker env arrives here with TERM scrubbed at the
+ * pm2 boundary (INVOKER_TERMINAL_ENV_KEYS). Left absent, every CLI in the
+ * session fails supports-color detection and renders colorless — pin the same
+ * constant every other backend PTY already forces. Control clients (get/set/
+ * list/kill) and a user's own `zmx attach` from a real terminal are untouched.
+ */
+export function zmxFreshSessionEnv(opts: SpawnOpts): NodeJS.ProcessEnv {
+  return { ...zmxControlEnv(opts), TERM: 'xterm-256color' };
+}
+
 /** Strip every payload-delivered key from ZMX control subprocesses. */
 export function zmxControlEnv(opts: SpawnOpts): NodeJS.ProcessEnv {
   const env = zmxEnv(opts.env);

src/cli/log-tail.ts

@@ -0,0 +1,190 @@
+import { closeSync, existsSync, openSync, readSync, statSync, type Stats } from 'node:fs';
+
+/**
+ * PM2-client-free log tailing for `botmux logs`.
+ *
+ * `pm2 logs` is a PM2 client, and any PM2 client invoked with no live God
+ * lazily births one (pm2 Client.start → pingDaemon false → launchDaemon).
+ * That made the read command a check/use race against `restart
+ * --include-pm2`: a God observed under the fleet lock could be retired after
+ * the lock released but before the spawned client connected, and the
+ * connecting client would then create a replacement God inside the
+ * kill→start window. Tailing the log FILES pm2 itself writes removes the PM2
+ * client entirely — no interleaving of this command can create a God, and
+ * logs keep working while the fleet is stopped.
+ */
+
+export interface LogTailSource {
+  /** Display label, e.g. the PM2 process name. */
+  label: string;
+  stream: 'out' | 'err';
+  file: string;
+}
+
+export function formatLogLine(source: LogTailSource, line: string): string {
+  return `${source.label}${source.stream === 'err' ? ' (err)' : ''} | ${line}`;
+}
+
+/** Last `n` complete lines of a text chunk (trailing newline ignored). */
+export function lastLinesOfChunk(chunk: string, n: number): string[] {
+  if (n <= 0) return [];
+  const all = chunk.split('\n');
+  if (all.length > 0 && all[all.length - 1] === '') all.pop();
+  return all.slice(Math.max(0, all.length - n));
+}
+
+// Initial `--lines` window: start small and grow until the window holds
+// enough complete lines (or the whole file / a hard cap), so a large N is
+// honored instead of silently clipped by a fixed window.
+const INITIAL_TAIL_WINDOW_BYTES = 64 * 1024;
+const MAX_TAIL_WINDOW_BYTES = 8 * 1024 * 1024;
+
+function readFileRange(file: string, start: number, end: number): string {
+  const length = end - start;
+  if (length <= 0) return '';
+  const fd = openSync(file, 'r');
+  try {
+    const buffer = Buffer.alloc(length);
+    const read = readSync(fd, buffer, 0, length, start);
+    return buffer.subarray(0, read).toString('utf8');
+  } finally {
+    closeSync(fd);
+  }
+}
+
+/** Tail window ending at `size`, aligned to a line start (or BOF). */
+function readAlignedTailWindow(file: string, size: number, wantLines: number): string {
+  for (let window = INITIAL_TAIL_WINDOW_BYTES; ; window *= 2) {
+    const start = Math.max(0, size - window);
+    const chunk = readFileRange(file, start, size);
+    if (start === 0) return chunk;
+    const newlineCount = chunk.split('\n').length - 1;
+    // Need wantLines complete lines AFTER dropping the leading partial line a
+    // mid-file window start produces (plus a possible unterminated tail).
+    if (newlineCount > wantLines + 1 || window >= MAX_TAIL_WINDOW_BYTES) {
+      return chunk.slice(chunk.indexOf('\n') + 1);
+    }
+  }
+}
+
+interface FollowState {
+  source: LogTailSource;
+  /** Next byte offset to read from; undefined until the file first appears. */
+  offset: number | undefined;
+  /** Trailing partial (unterminated) line carried between polls. */
+  partial: string;
+  /** File generation identity; a change means the path was atomically
+   *  replaced (rotation) even when the new size passes every offset check. */
+  dev: number | undefined;
+  ino: number | undefined;
+}
+
+export interface LogFileFollowerOptions {
+  sources: readonly LogTailSource[];
+  writeLine: (formatted: string) => void;
+  pollIntervalMs?: number;
+}
+
+/**
+ * Print the last N lines of every existing source, then follow appends.
+ * Correctness boundaries this models explicitly:
+ *  - An unterminated trailing line is NEVER emitted as a line; it is carried
+ *    as `partial` (from the initial tail too) and emitted once completed, so
+ *    "abc" + later "def\n" prints one "abcdef" line.
+ *  - Atomic rotation (rename + recreate at the same path) is detected by
+ *    dev+ino generation identity, not by size heuristics — a new file whose
+ *    size already exceeds the old offset would otherwise be read mid-stream.
+ *    (On filesystems reporting ino=0 this degrades to the size checks.)
+ *  - In-place truncation (`pm2 flush`) resets to the top of the new content.
+ *  - Files may appear later (fleet starts while tailing) or disappear.
+ */
+export class LogFileFollower {
+  private readonly states: FollowState[];
+  private readonly writeLine: (formatted: string) => void;
+  private readonly pollIntervalMs: number;
+  private timer: NodeJS.Timeout | undefined;
+
+  constructor(opts: LogFileFollowerOptions) {
+    this.states = opts.sources.map(source => ({
+      source, offset: undefined, partial: '', dev: undefined, ino: undefined,
+    }));
+    this.writeLine = opts.writeLine;
+    this.pollIntervalMs = opts.pollIntervalMs ?? 300;
+  }
+
+  /** Emit the initial `--lines` window and position offsets at end-of-file. */
+  printInitialTail(lines: number): void {
+    for (const state of this.states) {
+      if (!existsSync(state.source.file)) continue;
+      let stats: Stats;
+      try { stats = statSync(state.source.file); } catch { continue; }
+      let chunk: string;
+      try { chunk = readAlignedTailWindow(state.source.file, stats.size, lines); } catch { continue; }
+      let complete = chunk;
+      if (chunk.length > 0 && !chunk.endsWith('\n')) {
+        const cut = chunk.lastIndexOf('\n');
+        state.partial = chunk.slice(cut + 1);
+        complete = cut >= 0 ? chunk.slice(0, cut + 1) : '';
+      }
+      for (const line of lastLinesOfChunk(complete, lines)) {
+        this.writeLine(formatLogLine(state.source, line));
+      }
+      state.offset = stats.size;
+      state.dev = stats.dev;
+      state.ino = stats.ino;
+    }
+  }
+
+  start(): void {
+    if (this.timer) return;
+    this.timer = setInterval(() => this.pollOnce(), this.pollIntervalMs);
+  }
+
+  stop(): void {
+    if (this.timer) clearInterval(this.timer);
+    this.timer = undefined;
+  }
+
+  /** One poll pass; exposed so tests can drive it deterministically. */
+  pollOnce(): void {
+    for (const state of this.states) {
+      let stats: Stats;
+      try {
+        stats = statSync(state.source.file);
+      } catch {
+        // Absent (not yet created, or rotated away): forget position so the
+        // file is picked up from its start when it (re)appears.
+        state.offset = undefined;
+        state.partial = '';
+        state.dev = undefined;
+        state.ino = undefined;
+        continue;
+      }
+      const sameGeneration = state.dev === stats.dev && state.ino === stats.ino;
+      if (state.offset === undefined || !sameGeneration) {
+        // Newly appeared file, or an atomic replace under the same path:
+        // this is a different byte stream, so any carried position/partial
+        // belongs to the OLD generation and must be dropped unconditionally.
+        state.offset = 0;
+        state.partial = '';
+      } else if (stats.size < state.offset) {
+        // Truncated in place (pm2 flush): restart from the top of the new
+        // content instead of replaying or reading past EOF.
+        state.offset = 0;
+        state.partial = '';
+      }
+      state.dev = stats.dev;
+      state.ino = stats.ino;
+      if (stats.size === state.offset) continue;
+      let chunk: string;
+      try { chunk = readFileRange(state.source.file, state.offset, stats.size); } catch { continue; }
+      state.offset = stats.size;
+      const combined = state.partial + chunk;
+      const parts = combined.split('\n');
+      state.partial = parts.pop() ?? '';
+      for (const line of parts) {
+        this.writeLine(formatLogLine(state.source, line));
+      }
+    }
+  }
+}

src/cli/pm2-env.ts

@@ -1,6 +1,8 @@
 import {
   scrubClaudeSessionMarkerEnv,
+  scrubInvokerTerminalEnv,
   scrubSessionCliHomeEnv,
+  scrubSessionTurnMarkerEnv,
   scrubWorkflowWorkerEnv,
   stripDashboardH5Env,
 } from '../utils/child-env.js';
@@ -45,6 +47,20 @@ export function scrubPm2CallerEnv(env: NodeJS.ProcessEnv): void {
   // legitimate consumer and loads the family itself (index-dashboard.ts), so
   // nothing needs it here.
   stripDashboardH5Env(env);
+  // Invoker-terminal fingerprints (NO_COLOR=1 / CODEX_CI=1 / PAGER=cat from
+  // an agent's non-interactive shell) ride the same persistence vector; baked
+  // in they turn every session PTY on the machine colorless. Same class, same
+  // boundary — see INVOKER_TERMINAL_ENV_KEYS.
+  scrubInvokerTerminalEnv(env);
+  // Turn-scoped session identity and capabilities of the invoking bot session
+  // must not become fleet-wide daemon env — the daemon stays session-agnostic
+  // (see SESSION_TURN_MARKER_ENV_KEYS for the per-key rationale).
+  scrubSessionTurnMarkerEnv(env);
+  // Re-pin TERM to the constant every botmux PTY already forces. Deleting it
+  // outright would make pm2 CLIENT output (e.g. `botmux status` on a real
+  // TTY) fail supports-color detection and render colorless; pinning keeps
+  // the baked value deterministic and invoker-independent instead of absent.
+  env.TERM = 'xterm-256color';
 }
 
 /** Build the env for a pm2 invocation with an isolated PM2_HOME. */

src/cli/pm2-fleet-lock.ts

@@ -0,0 +1,66 @@
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+import { AsyncLocalStorage } from 'node:async_hooks';
+import { withFileLock, withFileLockSync } from '../utils/file-lock.js';
+
+/**
+ * One lock for every botmux-internal mutation of the shared PM2_HOME
+ * (~/.botmux/pm2): core fleet stop/start/restart AND plugin service
+ * lifecycle. Core and plugin apps live under the same God, so two separate
+ * locks would let a concurrent plugin start slip between an include-pm2
+ * restart's plugin stop and its `pm2 kill`, or between the kill and the fresh
+ * fleet start. Computed lazily so tests can repoint HOME.
+ */
+export function pm2FleetMutationLockTarget(): string {
+  return join(homedir(), '.botmux', 'pm2-fleet-mutation');
+}
+
+/**
+ * Re-entrancy ownership is bound to the ASYNC CALL CHAIN, not the process: a
+ * module-level counter would let an unrelated concurrent flow in the same
+ * process (the dashboard serves POST /api/plugins/:id/services/* handlers
+ * concurrently) mistake "someone in this process holds the lock" for "I hold
+ * the lock" and skip the file lock entirely. AsyncLocalStorage context only
+ * flows into calls awaited UNDER the holder's callback, so nested calls
+ * (cmdRestart stopping plugin services while holding the lock) short-circuit
+ * while independent concurrent flows queue on the file lock like any other
+ * process.
+ */
+const lockOwnership = new AsyncLocalStorage<{ held: true }>();
+
+/** True only within the async call chain that currently holds the lock. */
+export function pm2FleetMutationLockHeld(): boolean {
+  return lockOwnership.getStore() !== undefined;
+}
+
+/**
+ * Serialize a PM2_HOME mutation against every other botmux flow — other
+ * processes via the file lock, other async chains in THIS process via the
+ * same file lock (file-lock treats a live same-pid holder as held, not
+ * stale). Lock order is fixed: fleet lock FIRST, then the plugin service
+ * lock — never the reverse.
+ */
+export async function withPm2FleetMutationLock<T>(
+  fn: () => Promise<T> | T,
+  opts: { maxWaitMs?: number } = {},
+): Promise<T> {
+  if (lockOwnership.getStore()) return await fn();
+  return withFileLock(
+    pm2FleetMutationLockTarget(),
+    () => lockOwnership.run({ held: true }, async () => fn()),
+    opts,
+  );
+}
+
+/** Sync variant for sync call sites (plugin service lock wrapper). */
+export function withPm2FleetMutationLockSync<T>(
+  fn: () => T,
+  opts: { maxWaitMs?: number } = {},
+): T {
+  if (lockOwnership.getStore()) return fn();
+  return withFileLockSync(
+    pm2FleetMutationLockTarget(),
+    () => lockOwnership.run({ held: true }, fn),
+    opts,
+  );
+}