diff --git a/src/cli.ts b/src/cli.ts index 9dd5bc1..a006224 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -3,7 +3,7 @@ import { readFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { down, up, type DownOptions, type UpOptions } from "./up.ts"; +import { down, restart, up, type DownOptions, type RestartOptions, type UpOptions } from "./up.ts"; import { cmdAdd, cmdApp, cmdCos, cmdDoctor, cmdEnv, cmdInit, cmdInstallCli, cmdJob, cmdLs, cmdPersonas, cmdPretrust, cmdReload, cmdRemove, cmdRename, cmdRender, cmdRun, cmdShell, hasFlag, optValue, positionals, unknownFlag } from "./commands.ts"; import { cmdCompletions } from "./completions.ts"; import { cmdEval } from "./eval.ts"; @@ -80,6 +80,7 @@ export async function main(argv: string[]): Promise { case "run": code = await cmdRun(rest); break; case "up": code = await cmdUp(rest); break; case "down": code = await cmdDown(rest); break; + case "restart": code = await cmdRestart(rest); break; case "env": code = cmdEnv(rest); break; case "shell": code = await cmdShell(rest); break; case "personas": code = await cmdPersonas(rest); break; @@ -128,6 +129,15 @@ async function cmdDown(args: string[]): Promise { return down(opts); } +async function cmdRestart(args: string[]): Promise { + const bad = rejectUnknown("restart", args); + if (bad !== null) return bad; + const opts: RestartOptions = {}; + opts.json = hasFlag(args, "--json"); + opts.network = positionals(args)[0]; + return restart(opts); +} + // NOTE: `app ` (the Convoy.app menubar host manager) is intentionally hidden from this help // output + the README Commands list until the macOS app is dailyable (Nathan's call). The subcommand // still dispatches (see the "app" case + cmdApp); un-hide by re-adding its line here + in the README. @@ -144,6 +154,7 @@ function printHelp(): void { " run [role] launch an AD-HOC session — NOT declared, NOT reconciled, NOT respawned, no durable context (declare it with `add` if it should survive) [--identity --harness claude|codex --model --transport ding|mcp --mcp --network --dir --persona --prefix --config-dir --dry-run --force]\n" + " up host a network in the foreground (TCC anchor + supervisor + flapping-cap) [--once = one-shot reconcile-and-exit (adopts live sessions, no daemon) --json]\n" + " down [network] tear down the network — the ONLY path that kills sessions [--dry-run --force --json]\n" + + " restart [network] SAFE restart — stop the up process (agents survive) then re-adopt them; use this, never `down` + `up` [--json]\n" + " env [network] print eval-safe exports for a network's env — `eval \"$(convoy env )\"` sets ST_ROOT+PTY_ROOT [--identity ]\n" + " shell [network] open an interactive subshell with a network's env exported (pty ls / st just work); exit to leave [--identity ]\n" + " remove remove an agent\n" + diff --git a/src/command-table.ts b/src/command-table.ts index a983674..79cba42 100644 --- a/src/command-table.ts +++ b/src/command-table.ts @@ -198,6 +198,13 @@ export const COMMANDS: readonly CommandSpec[] = [ flags: [DRY_RUN_FLAG, { name: "force", desc: "Tear down without confirmation", kind: "bool" }, JSON_FLAG], positional: NETWORK_POSITIONAL, }, + { + name: "restart", + desc: "Safely restart the host — stop the up process (agents survive) then re-adopt them (NOT `down`)", + // No --network: the network is the positional, and --network is rejected (rc=2). + flags: [JSON_FLAG], + positional: NETWORK_POSITIONAL, + }, { name: "eval", desc: "Run a batch/eval cell end-to-end (spin → wait for the done-signal → grade) → machine verdict", diff --git a/src/completions.test.ts b/src/completions.test.ts index ea3ce05..544b42f 100644 --- a/src/completions.test.ts +++ b/src/completions.test.ts @@ -125,9 +125,9 @@ describe("flags are scoped to the subcommand that honors them", () => { } }); - it("offers --json only to ls/init/up/down and the batch verbs eval/job", () => { + it("offers --json only to ls/init/up/down/restart and the batch verbs eval/job", () => { const withJson = COMMANDS.filter((c) => (c.flags ?? []).some((f) => f.name === "json")).map((c) => c.name); - expect(withJson.sort()).toEqual(["down", "eval", "init", "job", "ls", "up"]); + expect(withJson.sort()).toEqual(["down", "eval", "init", "job", "ls", "restart", "up"]); }); it("matches the CLI: a scoped flag is rejected where it does not apply", () => { diff --git a/src/flapping-cap.test.ts b/src/flapping-cap.test.ts index 935bfae..c892abb 100644 --- a/src/flapping-cap.test.ts +++ b/src/flapping-cap.test.ts @@ -5,6 +5,7 @@ import { describe, it, expect } from "vitest"; import { classify, classifyFailedAttempt, + clearParkForFreshSupervisor, commandFingerprint, FLAPPING_STATUS, effectiveLimit, @@ -197,3 +198,42 @@ describe("classifyFailedAttempt — a recovery attempt that never produced a lea if (parked.kind === "flap") expect(writtenTags(parked.tags)[TAG.status]).toBe(FLAPPING_STATUS); }); }); + +describe("clearParkForFreshSupervisor — a fresh foreground `convoy up` restores the FULL fleet (parking-recovery)", () => { + it("ACCEPTANCE: a PARKED member is un-parked — status cleared AND the counter zeroed (relaunchable again)", () => { + // The reproduced bug: an outage drives the cap to its limit → the agent parks → and a fresh supervisor, + // reading the persisted `status=flapping`, would `skip` it forever. A deliberate bring-up must not inherit + // that. Both fields reset: clearing status alone is not enough — a counter still at the cap re-parks on + // the very next fast fail. + const cleared = clearParkForFreshSupervisor(tags({ status: FLAPPING_STATUS, consecutiveFastFails: LIMIT })); + expect(cleared).not.toBeNull(); + expect(cleared?.status).toBeNull(); + expect(cleared?.consecutiveFastFails).toBe(0); + }); + + it("resets a NON-parked member with prior fails too — 'regardless of prior fail count' (Nathan mandate)", () => { + const cleared = clearParkForFreshSupervisor(tags({ status: null, consecutiveFastFails: LIMIT - 1 })); + expect(cleared?.consecutiveFastFails).toBe(0); + expect(cleared?.status).toBeNull(); + }); + + it("is a NO-OP for a clean member (no park, counter 0) — the caller writes no tag needlessly", () => { + expect(clearParkForFreshSupervisor(tags({ status: null, consecutiveFastFails: 0 }))).toBeNull(); + }); + + it("preserves the rest of the strategy state — only status + counter are touched", () => { + const before = tags({ status: FLAPPING_STATUS, consecutiveFastFails: LIMIT, commandHash: HASH_A, lastRespawnAt: at(500), fastFailLimitOverride: 5, fastFailWindowOverride: 120 }); + const cleared = clearParkForFreshSupervisor(before); + expect(cleared?.commandHash).toBe(HASH_A); + expect(cleared?.lastRespawnAt).toEqual(at(500)); + expect(cleared?.fastFailLimitOverride).toBe(5); + expect(cleared?.fastFailWindowOverride).toBe(120); + }); + + it("the written tags drop the park status and carry a zeroed counter (what up() persists to disk)", () => { + const cleared = clearParkForFreshSupervisor(tags({ status: FLAPPING_STATUS, consecutiveFastFails: LIMIT })); + const written = writtenTags(cleared!); + expect(written[TAG.status]).toBeUndefined(); // no park written — up() also REMOVES the on-disk status tag + expect(written[TAG.consecutive]).toBe("0"); + }); +}); diff --git a/src/flapping-cap.ts b/src/flapping-cap.ts index 1f6056c..e88af97 100644 --- a/src/flapping-cap.ts +++ b/src/flapping-cap.ts @@ -151,6 +151,29 @@ export function classifyFailedAttempt(input: { return { kind: "respawn", tags: { ...tags, lastRespawnAt: now, consecutiveFastFails: nextCounter, commandHash: currentHash, status: null } }; } +/** A fresh FOREGROUND supervisor gives every member a clean cap budget (convoy parking-recovery, 2026-07-22). + * + * The bug this fixes: `strategy.status=flapping` and the fast-fail counter PERSIST to the session's tags + * (the on-disk supervision contract), so they outlive the supervisor that wrote them. A mass outage drives + * the cap to its limit → the agents park → and then a FRESH `convoy up`, reading those stale tags, hits the + * `isFlapping(...) → skip` gate in `classify` and NEVER relaunches them. The reconstructed incident: a + * bring-up after an outage brought back only some of the fleet; the rest stayed parked from a prior + * supervisor's give-up and had to be hand-launched. + * + * A foreground `convoy up` is a DELIBERATE bring-up — the operator gesture that says "restore the fleet" — + * so it must not inherit a prior supervisor's verdict. This clears the park (status) AND zeroes the counter, + * regardless of prior fail count, giving each member a fresh budget; the cap still re-accrues tick-to-tick + * WITHIN this supervisor's watch (the real crash-loop protection). Returns the reset tags, or null when + * nothing needs clearing (not parked, counter already 0) so the caller writes no tag needlessly. + * + * The `--once` shepherd cron does NOT call this: it runs every few minutes, so un-parking there would + * relaunch a genuinely broken agent on every tick — parking MUST stay durable across `--once`. This + * reset is scoped to the rare, intentional foreground bring-up. Pure → unit-testable. */ +export function clearParkForFreshSupervisor(tags: StrategyTags): StrategyTags | null { + if (tags.status !== FLAPPING_STATUS && tags.consecutiveFastFails === 0) return null; + return { ...tags, status: null, consecutiveFastFails: 0 }; +} + /** Classify one permanent-and-gone session (spec §5.3). Pure: same inputs → same decision. */ export function classify(input: { session: string; diff --git a/src/up-parking-recovery.test.ts b/src/up-parking-recovery.test.ts new file mode 100644 index 0000000..2b2343f --- /dev/null +++ b/src/up-parking-recovery.test.ts @@ -0,0 +1,186 @@ +// PARKING-RECOVERY (Nathan mandate, convoy incident 2026-07-22) — a supervisor bring-up after a mass +// outage MUST restore the FULL fleet. The bug: `strategy.status=flapping` + the fast-fail counter persist +// to a session's tags, so an outage that drives the cap to its limit PARKS the agents, and then a fresh +// `convoy up`, reading those stale tags, `skip`s them forever (classify: `isFlapping → skip`). The +// reconstructed incident: a bring-up brought back only part of the fleet; the rest stayed parked from a +// prior supervisor's give-up and had to be hand-launched. +// +// The fix (see up.ts FRESH-SUPERVISOR UN-PARK + flapping-cap.ts clearParkForFreshSupervisor): a foreground +// `convoy up` is a DELIBERATE bring-up, so at startup it clears the park + zeroes the counter for permanent +// members (regardless of prior fail count); the cap re-accrues tick-to-tick within THIS supervisor's watch. +// The `--once` shepherd cron does NOT un-park (it runs every few minutes — un-parking there would relaunch +// a genuinely broken agent every tick), so parking stays durable for it. +// +// This proves it end to end: it stands up a PARKED, gone-but-recorded agent (via convoy's own spawn path +// plus the same `strategy.*` tags a prior supervisor would have written), then asserts a fresh FOREGROUND +// up UN-PARKS and RELAUNCHES it, while a fresh `--once` up leaves it parked. Process-level (real daemons + +// a real `convoy up`), scoped to a throwaway XDG_STATE_HOME. Lives in the vitest gate (test.yml), not the +// hermetic nix flake check. + +import { afterEach, describe, expect, it } from "vitest"; +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { updateTags } from "@compoundingtech/pty/client"; +import { gone, PtyHost, processAlive, spawnFromPtyFile } from "./host.ts"; +import { FLAPPING_STATUS, TAG } from "./flapping-cap.ts"; + +const repoRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); +const bin = join(repoRoot, "bin", "convoy"); +const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); + +let home = ""; +let net = ""; +let host: ChildProcess | null = null; +const savedPtyRoot = process.env["PTY_ROOT"]; + +function childEnv(): NodeJS.ProcessEnv { + return { ...process.env, XDG_STATE_HOME: home, ST_ROOT: "", PTY_ROOT: "" }; +} + +function freshNet(): void { + home = mkdtempSync(join(tmpdir(), "cvy-park-")); + net = join(home, "convoy", "default"); + mkdirSync(join(net, "catalog"), { recursive: true }); + mkdirSync(join(net, "smalltalk"), { recursive: true }); +} + +/** Stand up one permanent agent whose harness EXITS quickly (`sleep 1`), so it lands in the gone-but- + * recorded state a real crashed agent occupies — the shape the park tags attach to and a replay relaunches. */ +async function spawnAgent(id: string): Promise { + const workspace = join(net, "agents", id); + mkdirSync(join(workspace, ".convoy"), { recursive: true }); + writeFileSync( + join(workspace, ".convoy", "pty.toml"), + `prefix = "${id}"\n\n[sessions.claude]\nid = "${id}"\ncommand = "sleep 1"\n\n[sessions.claude.tags]\nstrategy = "permanent"\nrole = "agent"\n\n[sessions.claude.env]\nST_AGENT = "${id}"\n`, + ); + const { spawned, failed } = await spawnFromPtyFile(workspace, net); + if (failed.length > 0 || spawned.length === 0) throw new Error(`spawn ${id} failed: ${JSON.stringify({ spawned, failed })}`); +} + +/** Poll until the agent is in the real CRASHED shape: gone-but-recorded with a DEAD pid. The harness + * exits, and ~0.5s later the pty daemon writes its exit record and shuts down (pid clears) — only then is + * the pid dead, so `convoy up` treats it as a genuine death to RESPAWN rather than a transient-gone to + * ADOPT (the adopt-alive guard: reported-gone but pid-alive → adopt, never respawn). */ +async function waitCrashed(id: string, timeoutMs = 12000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const s = (await new PtyHost(net).sessions()).find((x) => x.name === id); + if (s && gone(s) && !processAlive(s.pid)) return; + await sleep(150); + } + throw new Error(`${id} never reached the crashed (gone + dead-pid) state`); +} + +/** Write the park a prior supervisor would have left: status=flapping at the cap. */ +function park(id: string): void { + updateTags(id, { [TAG.status]: FLAPPING_STATUS, [TAG.consecutive]: "3" }); +} + +/** The persisted strategy.status tag for an agent (undefined once cleared). */ +async function statusTag(id: string): Promise { + const s = (await new PtyHost(net).sessions()).find((x) => x.name === id); + return s?.tags[TAG.status]; +} + +function lockedHostPid(): number | null { + try { + const pid = Number.parseInt(readFileSync(join(net, "convoy.pid"), "utf8").trim(), 10); + return Number.isInteger(pid) && processAlive(pid) ? pid : null; + } catch { + return null; + } +} + +/** Start a foreground `convoy up --json`, wait until it is hosting, run for `runMs` (long enough for the + * startup un-park + the immediate first reconcile), then kill it. Returns the captured JSONL stdout. */ +async function runForegroundUp(runMs: number): Promise { + const child = spawn(process.execPath, [bin, "up", net, "--json"], { env: childEnv(), stdio: ["ignore", "pipe", "pipe"] }); + host = child; + let stdout = ""; + child.stdout?.on("data", (d: Buffer) => (stdout += d.toString())); + const deadline = Date.now() + 15000; + while (Date.now() < deadline && lockedHostPid() !== child.pid) { + if (child.exitCode !== null) throw new Error(`convoy up exited early (code ${child.exitCode})`); + await sleep(50); + } + await sleep(runMs); + const exited = new Promise((r) => child.once("exit", () => r())); + child.kill("SIGTERM"); + await exited; + host = null; + return stdout; +} + +/** Parse a JSONL stream into records. */ +function records(stream: string): Array<{ type?: string; session?: string; spawned?: string[] }> { + const out: Array<{ type?: string; session?: string; spawned?: string[] }> = []; + for (const line of stream.split("\n")) { + if (!line.trim()) continue; + try { + out.push(JSON.parse(line)); + } catch { + /* human line leaked to stdout? ignore */ + } + } + return out; +} + +afterEach(() => { + if (host) { + try { + host.kill("SIGKILL"); + } catch { + /* ignore */ + } + host = null; + } + try { + spawnSync(process.execPath, [bin, "down", net, "--force"], { env: childEnv() }); + } catch { + /* ignore */ + } + if (home) rmSync(home, { recursive: true, force: true }); + if (savedPtyRoot === undefined) delete process.env["PTY_ROOT"]; + else process.env["PTY_ROOT"] = savedPtyRoot; +}); + +describe("parking recovery — a fresh foreground `convoy up` restores a PARKED agent (Nathan mandate)", () => { + it("ACCEPTANCE: a fresh FOREGROUND up UN-PARKS and RELAUNCHES a parked gone agent (regardless of fail count)", async () => { + freshNet(); + const id = "prk-alpha"; + await spawnAgent(id); + await waitCrashed(id); + park(id); + expect(await statusTag(id), "the agent must be parked before the bring-up").toBe(FLAPPING_STATUS); + + const out = records(await runForegroundUp(2500)); + + // It was UN-PARKED (the startup pass cleared the persisted park)... + expect(out.some((r) => r.type === "unpark" && r.session === id), "a fresh foreground up must emit an unpark for the parked agent").toBe(true); + // ...and then RELAUNCHED (the reconcile respawned it once un-parking made it eligible — a still-parked + // agent would have been skipped and never respawned/replayed). + const relaunched = out.some((r) => (r.type === "respawn" && r.session === id) || (r.type === "replay" && (r.spawned ?? []).includes(id))); + expect(relaunched, "a fresh foreground up must relaunch the un-parked agent").toBe(true); + // The persisted park is gone from disk. + expect(await statusTag(id), "the flapping status tag must be cleared on disk after the bring-up").not.toBe(FLAPPING_STATUS); + }, 45000); + + it("CONTROL: a `--once` bring-up does NOT un-park — parking stays durable for the shepherd cron", async () => { + freshNet(); + const id = "prk-beta"; + await spawnAgent(id); + await waitCrashed(id); + park(id); + + const r = spawnSync(process.execPath, [bin, "up", net, "--once", "--json"], { env: childEnv(), encoding: "utf8" }); + expect(r.status, `--once should exit 0\nstderr:\n${r.stderr}`).toBe(0); + + // No un-park was emitted, and the park is still on disk — `--once` must respect it (else it would + // relaunch a genuinely broken agent every few minutes). + expect(records(r.stdout).some((rec) => rec.type === "unpark"), "--once must NOT un-park anything").toBe(false); + expect(await statusTag(id), "--once must leave the park intact").toBe(FLAPPING_STATUS); + }, 45000); +}); diff --git a/src/up-restart.test.ts b/src/up-restart.test.ts new file mode 100644 index 0000000..5c3c542 --- /dev/null +++ b/src/up-restart.test.ts @@ -0,0 +1,129 @@ +// SAFE RESTART (Nathan mandate, convoy incident 2026-07-22) — `convoy restart` exists so nobody reaches +// for `convoy down` + `convoy up` to restart a live network. `down` KILLS every agent (it is the only +// teardown), so restarting that way is the mass-outage footgun. `restart` instead STOPS the host PROCESS +// (SIGTERM — agents keep running, the Nomad decoupling) and becomes a fresh `convoy up` that RE-ADOPTS +// the still-running agents. +// +// This proves it end to end: a real agent daemon is supervised by a real `convoy up` (host A); a real +// `convoy restart` then stops host A and takes over as host B — and the agent must survive at the SAME +// pid throughout, with host B now holding the lock. Process-level, scoped to a throwaway XDG_STATE_HOME. +// Lives in the vitest gate (test.yml), not the hermetic nix flake check. + +import { afterEach, describe, expect, it } from "vitest"; +import { spawn, spawnSync, type ChildProcess } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { PtyHost, processAlive, spawnFromPtyFile } from "./host.ts"; + +const repoRoot = join(dirname(fileURLToPath(import.meta.url)), ".."); +const bin = join(repoRoot, "bin", "convoy"); +const sleep = (ms: number): Promise => new Promise((r) => setTimeout(r, ms)); + +let home = ""; +let net = ""; +const live: ChildProcess[] = []; +const savedPtyRoot = process.env["PTY_ROOT"]; + +function childEnv(): NodeJS.ProcessEnv { + return { ...process.env, XDG_STATE_HOME: home, ST_ROOT: "", PTY_ROOT: "" }; +} + +function freshNet(): void { + home = mkdtempSync(join(tmpdir(), "cvy-restart-")); + net = join(home, "convoy", "default"); + mkdirSync(join(net, "catalog"), { recursive: true }); + mkdirSync(join(net, "smalltalk"), { recursive: true }); +} + +async function spawnAgent(id: string): Promise { + const workspace = join(net, "agents", id); + mkdirSync(join(workspace, ".convoy"), { recursive: true }); + writeFileSync( + join(workspace, ".convoy", "pty.toml"), + `prefix = "${id}"\n\n[sessions.claude]\nid = "${id}"\ncommand = "exec sleep 2000000"\n\n[sessions.claude.tags]\nstrategy = "permanent"\nrole = "agent"\n\n[sessions.claude.env]\nST_AGENT = "${id}"\n`, + ); + const { spawned, failed } = await spawnFromPtyFile(workspace, net); + if (failed.length > 0 || spawned.length === 0) throw new Error(`spawn ${id} failed: ${JSON.stringify({ spawned, failed })}`); + const s = (await new PtyHost(net).sessions()).find((x) => x.name === id); + if (!s?.pid) throw new Error(`no pid for ${id}`); + return s.pid; +} + +function lockedHostPid(): number | null { + try { + const pid = Number.parseInt(readFileSync(join(net, "convoy.pid"), "utf8").trim(), 10); + return Number.isInteger(pid) && processAlive(pid) ? pid : null; + } catch { + return null; + } +} + +function startHost(cmd: "up" | "restart"): ChildProcess { + const child = spawn(process.execPath, [bin, cmd, net, "--json"], { env: childEnv(), stdio: ["ignore", "pipe", "pipe"] }); + live.push(child); + return child; +} + +/** Poll until `pred()` holds or we time out. */ +async function until(pred: () => boolean, timeoutMs = 20000): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (pred()) return true; + await sleep(75); + } + return false; +} + +async function stop(child: ChildProcess): Promise { + if (child.exitCode !== null) return; + const exited = new Promise((r) => child.once("exit", () => r())); + child.kill("SIGKILL"); + await exited; +} + +afterEach(async () => { + for (const c of live) await stop(c).catch(() => {}); + live.length = 0; + try { + spawnSync(process.execPath, [bin, "down", net, "--force"], { env: childEnv() }); + } catch { + /* ignore */ + } + if (home) rmSync(home, { recursive: true, force: true }); + if (savedPtyRoot === undefined) delete process.env["PTY_ROOT"]; + else process.env["PTY_ROOT"] = savedPtyRoot; +}); + +describe("convoy restart — the SAFE restart (stop the process, agents survive, new host re-adopts)", () => { + it("ACCEPTANCE: stops the running host, the agent SURVIVES at the same pid, and restart becomes the new host", async () => { + freshNet(); + const agentPid = await spawnAgent("rst-alpha"); + + // Host A takes over. + const hostA = startHost("up"); + expect(await until(() => lockedHostPid() === hostA.pid), "host A should acquire the lock").toBe(true); + + // `convoy restart` stops host A and takes over as host B. + const restart = startHost("restart"); + const flipped = await until(() => hostA.exitCode !== null && lockedHostPid() === restart.pid); + expect(flipped, "restart must stop host A and become the new lock owner").toBe(true); + + // Host A is gone; the restart process is the host; and — the whole point — the agent never died. + expect(hostA.exitCode, "the old host process must have exited").not.toBeNull(); + expect(lockedHostPid(), "the restart process now hosts the network").toBe(restart.pid); + const after = (await new PtyHost(net).sessions()).find((x) => x.name === "rst-alpha"); + expect(after?.pid, "the agent must keep its exact pid across the restart").toBe(agentPid); + expect(processAlive(agentPid), "the agent must still be ALIVE after the restart").toBe(true); + }, 60000); + + it("with NO host running, restart simply STARTS one (it does not error)", async () => { + freshNet(); + await spawnAgent("rst-beta"); + expect(lockedHostPid(), "precondition: nothing is hosting yet").toBeNull(); + + const restart = startHost("restart"); + expect(await until(() => lockedHostPid() === restart.pid), "restart with no prior host must just start hosting").toBe(true); + }, 45000); +}); diff --git a/src/up.ts b/src/up.ts index f73944c..1ae0138 100644 --- a/src/up.ts +++ b/src/up.ts @@ -11,6 +11,7 @@ import { defaultConvoyNetwork, isNetworkName, networkDirForName, networkDirOfStR import { classify, classifyFailedAttempt, + clearParkForFreshSupervisor, effectiveLimit, effectiveWindow, isFlapping, @@ -359,6 +360,33 @@ export async function up(opts: UpOptions): Promise { const notify = opts.notify ?? []; const dingTargets = (crashed: SupervisedSession, sessions: readonly SupervisedSession[]): string[] => crashDingTargets(crashed, sessions, notify, busIdOf); + // FRESH-SUPERVISOR UN-PARK (parking-recovery, 2026-07-22). A foreground `convoy up` is a DELIBERATE + // bring-up — after a mass outage it MUST restore the FULL fleet, not inherit a prior supervisor's + // give-up. `strategy.status=flapping` + the fast-fail counter persist to each session's tags, so a + // parked agent stays parked across a restart (classify's `isFlapping → skip`), and a bring-up brought + // back only part of the fleet — the rest had to be hand-launched. So, ONCE at startup, clear the park + // and zero the counter for permanent members (regardless of prior fail count); the cap re-accrues + // tick-to-tick within THIS supervisor's watch. A fully-gone parked agent (no session record left) is + // relaunched by the catalog pass instead — this handles the gone-but-recorded ones the cap would skip. + // + // `--once` (the shepherd cron) SKIPS this: it runs every few minutes, so un-parking there would + // relaunch a genuinely broken agent every tick. Parking must stay durable across `--once`. + if (opts.once !== true) { + const startupNow = new Date(); + for (const s of await host.sessions()) { + if (!isPermanent(s)) continue; + const cleared = clearParkForFreshSupervisor(parseStrategyTags(s.tags)); + if (!cleared) continue; + host.removeTag(s.name, TAG.status); // updateTags MERGES — the park must be removed, not just overwritten + host.setTags(s.name, writtenTags(cleared)); // consecutive-fast-fails → 0 + state.set(s.name, cleared); + emit( + { type: "unpark", identity: logicalId(s), session: s.name, ts: isoString(startupNow) }, + `[convoy-up] fresh supervisor — cleared parked/flapping state for ${logicalId(s)} session=${s.name}; giving it a fresh cap budget`, + ); + } + } + const tick = async (): Promise => { const now = new Date(); // Manifest replay relaunches EVERY limb of an agent, so it must happen at most once per agent per @@ -734,3 +762,49 @@ export async function down(opts: DownOptions): Promise { if (acquired) lock.release(); } } + +export interface RestartOptions { + network?: string | undefined; + json?: boolean; + /** How long to wait for the running host to release the lock after the stop signal (ms). */ + stopTimeoutMs?: number; +} + +/** `convoy restart []` — the SAFE restart, and the whole reason it exists: it is NOT `convoy + * down` + `convoy up`. `down` KILLS every agent (it is the only teardown), so restarting a live network + * that way is the mass-outage footgun — the exact shape of the 2026-07-22 incident. `restart` instead + * STOPS the host PROCESS with SIGTERM (agents keep running — the Nomad decoupling: up's signal handler + * just sets `stop`, and its exit path leaves every session up), waits for it to release the host lock, + * then becomes a fresh `convoy up` that RE-ADOPTS the still-running agents (reconcile skips live ones). + * If no host is running, it simply starts one. */ +export async function restart(opts: RestartOptions): Promise { + const root = resolveRoot(opts.network); + const lock = new HostLock(root); + const json = opts.json === true; + + const owner = lock.liveOwner(); + if (owner !== null) { + process.stderr.write(`convoy restart: stopping host pid ${owner} (agents keep running — this is NOT convoy down)…\n`); + try { + process.kill(owner, "SIGTERM"); // graceful stop → up sets `stop`, exits, keeps every session, releases the lock + } catch { + // already gone between the read and the signal — fall through to the wait, which will see it released + } + // Wait for the old host to release the lock; a fresh `up` would otherwise refuse (single-owner guard), + // or worse, two hosts would briefly double-supervise. Poll the lock rather than the pid so we key on + // the same signal `up`'s guard does. + const deadline = Date.now() + (opts.stopTimeoutMs ?? 15000); + while (Date.now() < deadline && lock.liveOwner() !== null) await sleep(100); + if (lock.liveOwner() !== null) { + process.stderr.write(`convoy restart: host pid ${owner} did not stop within the timeout — aborting so we never double-host. Stop it by hand, then \`convoy up\`.\n`); + return 1; + } + process.stderr.write(`convoy restart: host stopped; re-adopting the running agents…\n`); + } else { + process.stderr.write(`convoy restart: no host is running — starting a fresh one.\n`); + } + + // Become the new foreground host. It re-adopts the still-running sessions (the reconcile skips live + // ones — `if (!gone(s)) continue`) and, per the fresh-supervisor un-park, restores any parked members. + return up({ network: opts.network, json }); +}