From a6c3d827e7ab58f95bacee2f03ebe2aeb8376df4 Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:16:52 -0400 Subject: [PATCH 1/7] fix(sync): negotiate adopt-channel AEAD so packaged Electron can account-connect Electron's BoringSSL lacks chacha20-poly1305 in createCipheriv, so v1.2.35's sealed account adoption failed with "Unknown cipher" on every route. Clients now offer supportedAeads in account_challenge; hosts intersect, echo the chosen aead (covered by the challenge signature when negotiated), and both sides seal/unseal with it. Clients without the field get the byte-identical legacy chacha path, keeping iOS and older desktops untouched. Co-Authored-By: Claude Fable 5 --- .../src/services/sync/syncHostService.test.ts | 109 ++++++++++++ .../src/services/sync/syncHostService.ts | 36 ++++ .../syncPairedMachineStore.test.ts | 156 ++++++++++++++++++ .../remoteRuntime/syncPairedMachineStore.ts | 59 ++++++- .../shared/sync/adoptChannelCrypto.test.ts | 48 +++++- .../src/shared/sync/adoptChannelCrypto.ts | 62 ++++++- apps/desktop/src/shared/types/sync.ts | 2 + 7 files changed, 454 insertions(+), 18 deletions(-) diff --git a/apps/ade-cli/src/services/sync/syncHostService.test.ts b/apps/ade-cli/src/services/sync/syncHostService.test.ts index 11e7b1e95..f5df6d1c0 100644 --- a/apps/ade-cli/src/services/sync/syncHostService.test.ts +++ b/apps/ade-cli/src/services/sync/syncHostService.test.ts @@ -1803,6 +1803,7 @@ describe("sync host account authentication", () => { const issueChallenge = async ( client: Awaited>, requestId: string, + supportedAeads?: string[], ) => { const nonce = Buffer.alloc(32, requestId.length); const ephemeral = generateX25519EphemeralKeyPair(); @@ -1813,6 +1814,7 @@ describe("sync host account authentication", () => { v: 1, nonce: nonce.toString("base64"), clientEphemeralPublicKey: ephemeral.publicKeyRaw.toString("base64"), + ...(supportedAeads ? { supportedAeads } : {}), }, })); const envelope = await waitForValue( @@ -1827,6 +1829,7 @@ describe("sync host account authentication", () => { ts: number; hostEphemeralPublicKey: string; signature: string; + aead?: string; }; const canonical = buildAdoptChallengeSignatureInput({ hostDeviceId: payload.hostDeviceId, @@ -1834,6 +1837,7 @@ describe("sync host account authentication", () => { clientEphemeralPublicKey: ephemeral.publicKeyRaw.toString("base64"), hostEphemeralPublicKey: payload.hostEphemeralPublicKey, ts: payload.ts, + ...(payload.aead ? { aead: payload.aead } : {}), }); expect(verifyEd25519( Buffer.from( @@ -1863,6 +1867,7 @@ describe("sync host account authentication", () => { hostDeviceId: "host-device-1", ts: adoptNow, }); + expect(challenge.payload).not.toHaveProperty("aead"); const duplicateEphemeral = generateX25519EphemeralKeyPair(); client.ws.send(encodeSyncEnvelope({ type: "account_challenge", @@ -1957,6 +1962,81 @@ describe("sync host account authentication", () => { ([message]) => message === "sync_host.account_auth_requires_relay", )).toBe(false); + const aesClient = await openAccountClient(port); + clients.push(aesClient); + const aesChallenge = await issueChallenge( + aesClient, + "aes-challenge", + ["aes-256-gcm"], + ); + expect(aesChallenge.payload.aead).toBe("aes-256-gcm"); + const aesPeer = { + ...peer, + deviceId: "sealed-aes-device", + siteId: "sealed-aes-site", + }; + const aesAccountAuth = { + deviceId: aesPeer.deviceId, + accountToken, + dpop: signAccountDpop({ + privateKey: dpopKey.privateKey, + publicKeyX963: dpopKey.publicKeyX963, + deviceId: aesPeer.deviceId, + accountToken, + }), + }; + aesClient.ws.send(encodeSyncEnvelope({ + type: "hello", + requestId: "sealed-aes-hello", + payload: { + peer: aesPeer, + auth: { + kind: "account_sealed", + v: 1, + deviceId: aesPeer.deviceId, + sealed: seal( + aesChallenge.sessionKey, + buildAdoptHelloAad( + aesChallenge.payload.hostDeviceId, + aesPeer.deviceId, + ), + Buffer.from(JSON.stringify(aesAccountAuth)), + undefined, + "aes-256-gcm", + ), + }, + }, + })); + const aesSealedHelloOk = await waitForValue( + () => aesClient.envelopes.find((entry) => + entry.type === "hello_ok" && entry.requestId === "sealed-aes-hello" + ), + "AES sealed direct hello_ok", + ); + const aesOpenedHelloOk = JSON.parse(unseal( + aesChallenge.sessionKey, + buildAdoptHelloOkAad( + aesChallenge.payload.hostDeviceId, + aesPeer.deviceId, + ), + (aesSealedHelloOk.payload as { sealed: string }).sealed, + "aes-256-gcm", + ).toString("utf8")) as { + brain: { deviceId: string }; + accountPairing: { deviceId: string; secret: string }; + }; + expect(aesOpenedHelloOk).toMatchObject({ + brain: { deviceId: "host-device-1" }, + accountPairing: { + deviceId: aesPeer.deviceId, + secret: expect.any(String), + }, + }); + expect(pairingStore.authenticate( + aesPeer.deviceId, + aesOpenedHelloOk.accountPairing.secret, + )).toBe(true); + const plainClient = await openAccountClient(port); clients.push(plainClient); const plainPeer = { @@ -2037,6 +2117,35 @@ describe("sync host account authentication", () => { message: expect.stringMatching(/expired/i), }); + const incompatibleClient = await openAccountClient(port); + clients.push(incompatibleClient); + const incompatibleEphemeral = generateX25519EphemeralKeyPair(); + incompatibleClient.ws.send(encodeSyncEnvelope({ + type: "account_challenge", + requestId: "incompatible-aead", + payload: { + v: 1, + nonce: Buffer.alloc(32, 11).toString("base64"), + clientEphemeralPublicKey: + incompatibleEphemeral.publicKeyRaw.toString("base64"), + supportedAeads: ["future-aead"], + }, + })); + const incompatible = await waitForValue( + () => incompatibleClient.envelopes.find((entry) => + entry.type === "account_challenge_error" + && entry.requestId === "incompatible-aead" + ), + "incompatible AEAD rejection", + ); + expect(incompatible.payload).toEqual({ + message: + "No compatible account adoption cipher is available. Update ADE on both Macs.", + }); + expect(incompatibleClient.envelopes.some( + (entry) => entry.type === "account_challenge_ok", + )).toBe(false); + // Well-formed challenges only trigger a public signature and are the // normal adoption operation, so they must NEVER trip the abuse throttle — // otherwise a few normal relay adoptions (all sharing the loopback origin) diff --git a/apps/ade-cli/src/services/sync/syncHostService.ts b/apps/ade-cli/src/services/sync/syncHostService.ts index b89a30af6..da241c521 100644 --- a/apps/ade-cli/src/services/sync/syncHostService.ts +++ b/apps/ade-cli/src/services/sync/syncHostService.ts @@ -158,7 +158,9 @@ import { generateX25519EphemeralKeyPair, seal, signEd25519, + supportedAdoptChannelAeads, unseal, + type AdoptChannelAead, } from "../../../../desktop/src/shared/sync/adoptChannelCrypto"; import { createMachineIdentitySigningStore, @@ -528,6 +530,7 @@ type PeerState = { nonce: string; hostDeviceId: string; expiresAtMs: number; + aead: AdoptChannelAead; } | null; subscribedSessionIds: Set; pendingTerminalSnapshots: Map; @@ -1383,6 +1386,7 @@ type ParsedAccountChallenge = { nonceBytes: Buffer; clientEphemeralPublicKey: string; clientEphemeralPublicKeyBytes: Buffer; + supportedAeads: string[] | null; }; function parseAccountChallengePayload(payload: unknown): ParsedAccountChallenge | null { @@ -1394,11 +1398,21 @@ function parseAccountChallengePayload(payload: unknown): ParsedAccountChallenge value.clientEphemeralPublicKey, 32, ); + const supportedAeads = value.supportedAeads === undefined + ? null + : value.supportedAeads; if ( !nonceBytes || !clientEphemeralPublicKeyBytes || typeof value.nonce !== "string" || typeof value.clientEphemeralPublicKey !== "string" + || ( + supportedAeads !== null + && ( + !Array.isArray(supportedAeads) + || supportedAeads.some((entry) => typeof entry !== "string") + ) + ) ) { return null; } @@ -1407,6 +1421,7 @@ function parseAccountChallengePayload(payload: unknown): ParsedAccountChallenge nonceBytes, clientEphemeralPublicKey: value.clientEphemeralPublicKey, clientEphemeralPublicKeyBytes, + supportedAeads, }; } @@ -6245,6 +6260,19 @@ export function createSyncHostService(args: SyncHostServiceArgs) { }, envelope.requestId); return; } + const hostSupportedAeads = supportedAdoptChannelAeads(); + const negotiatedAead = challenge.supportedAeads?.find( + (aead): aead is AdoptChannelAead => + hostSupportedAeads.includes(aead as AdoptChannelAead), + ); + if (challenge.supportedAeads !== null && !negotiatedAead) { + send(peer.ws, "account_challenge_error", { + message: + "No compatible account adoption cipher is available. Update ADE on both Macs.", + }, envelope.requestId); + return; + } + const adoptAead = negotiatedAead ?? "chacha20-poly1305"; // A well-formed challenge only triggers one public signature and is the // normal adoption operation, so it feeds no cooldown at all. Signing // load is already bounded by the per-connection single-active-challenge @@ -6267,6 +6295,7 @@ export function createSyncHostService(args: SyncHostServiceArgs) { clientEphemeralPublicKey: challenge.clientEphemeralPublicKey, hostEphemeralPublicKey, ts, + ...(negotiatedAead ? { aead: negotiatedAead } : {}), }); const sessionKey = deriveAdoptSessionKey({ privateKey: hostEphemeral.privateKey, @@ -6278,6 +6307,7 @@ export function createSyncHostService(args: SyncHostServiceArgs) { nonce: challenge.nonce, hostDeviceId, expiresAtMs: ts + ADOPT_CHANNEL_CHALLENGE_TTL_MS, + aead: adoptAead, }; send(peer.ws, "account_challenge_ok", { v: 1, @@ -6285,6 +6315,7 @@ export function createSyncHostService(args: SyncHostServiceArgs) { ts, hostEphemeralPublicKey, signature: signEd25519(identity.privateKey, canonical).toString("base64"), + ...(negotiatedAead ? { aead: negotiatedAead } : {}), }, envelope.requestId); } catch (error) { peer.adoptChallenge = null; @@ -6452,6 +6483,7 @@ export function createSyncHostService(args: SyncHostServiceArgs) { sessionKey: Buffer; hostDeviceId: string; clientDeviceId: string; + aead: AdoptChannelAead; } | null = null; if (hello.auth?.kind === "account_sealed") { const sealedAuth = hello.auth; @@ -6492,6 +6524,7 @@ export function createSyncHostService(args: SyncHostServiceArgs) { sealedAuth.deviceId, ), sealedAuth.sealed, + challenge.aead, ); const accountAuth = parseUnsealedAccountAuth( JSON.parse(plaintext.toString("utf8")), @@ -6508,6 +6541,7 @@ export function createSyncHostService(args: SyncHostServiceArgs) { sessionKey: challenge.sessionKey, hostDeviceId: challenge.hostDeviceId, clientDeviceId: sealedAuth.deviceId, + aead: challenge.aead, }; } catch { rejectSealedHello("The sealed account credentials could not be opened."); @@ -6904,6 +6938,8 @@ export function createSyncHostService(args: SyncHostServiceArgs) { sealedAdoption.clientDeviceId, ), Buffer.from(JSON.stringify(helloOkPayload), "utf8"), + undefined, + sealedAdoption.aead, ), }, envelope.requestId); } else { diff --git a/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.test.ts b/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.test.ts index 04c5660d6..233934dda 100644 --- a/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.test.ts +++ b/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.test.ts @@ -15,6 +15,7 @@ import type { AdeAccountMachine } from "../../../shared/types/account"; import type { DesktopPairedMachineCredentials } from "../../../shared/types/pairedRuntime"; import { encodeSyncEnvelope, parseSyncEnvelope, wsDataToText } from "../sync/syncProtocol"; import { DesktopPairedMachineStore } from "./syncPairedMachineStore"; +import * as adoptChannelCrypto from "../../../shared/sync/adoptChannelCrypto"; import { buildAdoptChallengeSignatureInput, buildAdoptHelloAad, @@ -25,11 +26,13 @@ import { seal, signEd25519, unseal, + type AdoptChannelAead, } from "../../../shared/sync/adoptChannelCrypto"; const originalAdeHome = process.env.ADE_HOME; afterEach(() => { + vi.restoreAllMocks(); if (originalAdeHome === undefined) delete process.env.ADE_HOME; else process.env.ADE_HOME = originalAdeHome; }); @@ -71,6 +74,8 @@ function successfulSealedAdoptionSocket(args: { hostDeviceId: string; hostName: string; pairedSecret?: string; + aead?: AdoptChannelAead; + onSupportedAeads?: (aeads: string[] | undefined) => void; }): FakeWebSocket { let hostSessionKey: Buffer | null = null; return new FakeWebSocket((text, ws) => { @@ -79,7 +84,9 @@ function successfulSealedAdoptionSocket(args: { const request = envelope.payload as { nonce: string; clientEphemeralPublicKey: string; + supportedAeads?: string[]; }; + args.onSupportedAeads?.(request.supportedAeads); const hostEphemeral = generateX25519EphemeralKeyPair(); const hostEphemeralPublicKey = hostEphemeral.publicKeyRaw.toString("base64"); const ts = Date.now(); @@ -89,6 +96,7 @@ function successfulSealedAdoptionSocket(args: { clientEphemeralPublicKey: request.clientEphemeralPublicKey, hostEphemeralPublicKey, ts, + ...(args.aead ? { aead: args.aead } : {}), }); hostSessionKey = deriveAdoptSessionKey({ privateKey: hostEphemeral.privateKey, @@ -107,6 +115,7 @@ function successfulSealedAdoptionSocket(args: { args.signingPrivateKey, canonical, ).toString("base64"), + ...(args.aead ? { aead: args.aead } : {}), }, })); return; @@ -117,6 +126,17 @@ function successfulSealedAdoptionSocket(args: { auth: { kind: string; deviceId: string; sealed: string }; }; expect(payload.auth.kind).toBe("account_sealed"); + const accountAuth = JSON.parse(unseal( + hostSessionKey, + buildAdoptHelloAad(args.hostDeviceId, payload.auth.deviceId), + payload.auth.sealed, + args.aead, + ).toString("utf8")) as { + deviceId: string; + accountToken: string; + }; + expect(accountAuth.deviceId).toBe(payload.auth.deviceId); + expect(accountAuth.accountToken).toBeTruthy(); const helloOk = { peer: payload.peer, brain: { @@ -145,6 +165,8 @@ function successfulSealedAdoptionSocket(args: { hostSessionKey, buildAdoptHelloOkAad(args.hostDeviceId, payload.auth.deviceId), Buffer.from(JSON.stringify(helloOk)), + undefined, + args.aead, ), }, })); @@ -649,6 +671,140 @@ describe("DesktopPairedMachineStore", () => { ]); }); + it("offers only runtime-supported AEADs and completes adoption with negotiated AES-256-GCM", async () => { + process.env.ADE_HOME = fs.mkdtempSync( + path.join(os.tmpdir(), "ade-desktop-aes-adoption-"), + ); + vi.spyOn(adoptChannelCrypto, "supportedAdoptChannelAeads") + .mockReturnValue(["aes-256-gcm"]); + const signing = generateKeyPairSync("ed25519"); + let offeredAeads: string[] | undefined; + const machine: AdeAccountMachine = { + machineKey: "machine-aes-adoption", + deviceId: "host-aes-adoption", + name: "AES Studio", + platform: "macOS", + deviceType: "desktop", + pubkey: `ed25519:${rawPublicKeyFromSpki(signing.publicKey).toString("base64")}`, + online: true, + lastSeenAt: Date.now(), + reachableEndpoints: [ + { kind: "lan", host: "aes-studio.local", port: 8787 }, + ], + }; + + const adopted = await new DesktopPairedMachineStore().pairWithAccountMachine( + machine, + "aes-account-token", + "AES client", + { + accountOwnerUserId: "aes-account-user", + pairingTimeoutMs: 2_000, + createWebSocket: () => successfulSealedAdoptionSocket({ + signingPrivateKey: signing.privateKey, + hostDeviceId: "host-aes-adoption", + hostName: machine.name ?? "AES Studio", + pairedSecret: "aes-paired-secret", + aead: "aes-256-gcm", + onSupportedAeads: (aeads) => { + offeredAeads = aeads; + }, + }) as unknown as WebSocket, + }, + ); + + expect(offeredAeads).toEqual(["aes-256-gcm"]); + expect(adopted.hostIdentity.deviceId).toBe(machine.deviceId); + expect(adopted.secret).toBe("aes-paired-secret"); + }); + + it.each([ + { + name: "requires an update when an old host omits AEAD negotiation", + responseAead: undefined, + expectedError: + "The other Mac is running an older ADE that can't negotiate a compatible cipher — update it to the latest version.", + }, + { + name: "rejects a host AEAD that the client did not offer", + responseAead: "chacha20-poly1305" as const, + expectedError: + "Host identity verification failed — the machine may be running an older ADE.", + }, + ])("$name", async ({ responseAead, expectedError }) => { + process.env.ADE_HOME = fs.mkdtempSync( + path.join(os.tmpdir(), "ade-desktop-aead-rejection-"), + ); + vi.spyOn(adoptChannelCrypto, "supportedAdoptChannelAeads") + .mockReturnValue(["aes-256-gcm"]); + const signing = generateKeyPairSync("ed25519"); + const sentTypes: string[] = []; + const machine: AdeAccountMachine = { + machineKey: "machine-aead-rejection", + deviceId: "host-aead-rejection", + name: "Cipher Studio", + platform: "macOS", + deviceType: "desktop", + pubkey: `ed25519:${rawPublicKeyFromSpki(signing.publicKey).toString("base64")}`, + online: true, + lastSeenAt: Date.now(), + reachableEndpoints: [ + { kind: "lan", host: "cipher-studio.local", port: 8787 }, + ], + }; + + const pairing = new DesktopPairedMachineStore().pairWithAccountMachine( + machine, + "must-remain-local", + "AES-only client", + { + accountOwnerUserId: "aes-only-account-user", + pairingTimeoutMs: 500, + createWebSocket: () => new FakeWebSocket((text, ws) => { + const envelope = parseSyncEnvelope(wsDataToText(text)); + sentTypes.push(envelope.type); + if (envelope.type !== "account_challenge") return; + const request = envelope.payload as { + nonce: string; + clientEphemeralPublicKey: string; + supportedAeads?: string[]; + }; + expect(request.supportedAeads).toEqual(["aes-256-gcm"]); + const hostEphemeral = generateX25519EphemeralKeyPair(); + const hostEphemeralPublicKey = + hostEphemeral.publicKeyRaw.toString("base64"); + const ts = Date.now(); + const canonical = buildAdoptChallengeSignatureInput({ + hostDeviceId: "host-aead-rejection", + nonce: request.nonce, + clientEphemeralPublicKey: request.clientEphemeralPublicKey, + hostEphemeralPublicKey, + ts, + ...(responseAead ? { aead: responseAead } : {}), + }); + ws.receive(encodeSyncEnvelope({ + type: "account_challenge_ok", + requestId: envelope.requestId, + payload: { + v: 1, + hostDeviceId: "host-aead-rejection", + ts, + hostEphemeralPublicKey, + signature: signEd25519( + signing.privateKey, + canonical, + ).toString("base64"), + ...(responseAead ? { aead: responseAead } : {}), + }, + })); + }) as unknown as WebSocket, + }, + ); + + await expect(pairing).rejects.toThrow(expectedError); + expect(sentTypes).toEqual(["account_challenge"]); + }); + it("falls through a closed relay to sealed tailnet adoption with exact stages", async () => { process.env.ADE_HOME = fs.mkdtempSync( path.join(os.tmpdir(), "ade-desktop-tailnet-fallback-"), diff --git a/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.ts b/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.ts index 3813f00f2..5496337d8 100644 --- a/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.ts +++ b/apps/desktop/src/main/services/remoteRuntime/syncPairedMachineStore.ts @@ -29,8 +29,10 @@ import { deriveAdoptSessionKey, generateX25519EphemeralKeyPair, seal, + supportedAdoptChannelAeads, unseal, verifyEd25519, + type AdoptChannelAead, } from "../../../shared/sync/adoptChannelCrypto"; import type { AdeAccountMachine } from "../../../shared/types/account"; import { @@ -85,6 +87,8 @@ class AccountPairingAuthorizationError extends Error { const HOST_IDENTITY_VERIFICATION_ERROR = "Host identity verification failed — the machine may be running an older ADE."; +const ADOPT_CHANNEL_UPDATE_REQUIRED_ERROR = + "The other Mac is running an older ADE that can't negotiate a compatible cipher — update it to the latest version."; export class AccountHostIdentityVerificationError extends Error { readonly code = "account_host_identity_verification_failed"; @@ -750,6 +754,8 @@ export class DesktopPairedMachineStore { } try { let adoptSessionKey: Buffer | null = null; + let adoptAead: AdoptChannelAead = "chacha20-poly1305"; + let clientSupportedAeads: AdoptChannelAead[] = []; if (hostSigningPublicKey) { const challengeRequestId = `challenge-${randomUUID()}`; const nonce = randomBytes(32); @@ -757,6 +763,7 @@ export class DesktopPairedMachineStore { const clientEphemeral = generateX25519EphemeralKeyPair(); const clientEphemeralPublicKey = clientEphemeral.publicKeyRaw.toString("base64"); + clientSupportedAeads = supportedAdoptChannelAeads(); const challengeResponse = waitForSyncEnvelope( connection, (envelope) => envelope.requestId === challengeRequestId @@ -775,6 +782,7 @@ export class DesktopPairedMachineStore { v: 1, nonce: nonceBase64, clientEphemeralPublicKey, + supportedAeads: clientSupportedAeads, }, challengeRequestId); } catch (error) { void challengeResponse.catch(() => {}); @@ -802,6 +810,19 @@ export class DesktopPairedMachineStore { 32, ); const signature = decodeCanonicalBase64(challenge?.signature, 64); + const challengeHasAead = challenge?.aead !== undefined; + if ( + challengeHasAead + && ( + typeof challenge?.aead !== "string" + || !clientSupportedAeads.includes(challenge.aead as AdoptChannelAead) + ) + ) { + throw hostIdentityVerificationError(); + } + if (challengeHasAead) { + adoptAead = challenge!.aead as AdoptChannelAead; + } if ( challenge?.v !== 1 || challenge.hostDeviceId !== expectedHostDeviceId @@ -819,6 +840,7 @@ export class DesktopPairedMachineStore { clientEphemeralPublicKey, hostEphemeralPublicKey: challenge.hostEphemeralPublicKey!, ts: challenge.ts!, + ...(challengeHasAead ? { aead: challenge.aead! } : {}), }); if (!verifyEd25519(hostSigningPublicKey, canonical, signature)) { throw hostIdentityVerificationError(); @@ -845,6 +867,33 @@ export class DesktopPairedMachineStore { accountToken, dpop: accountDpop, }; + let sealedAccountAuth: string | null = null; + if (adoptSessionKey) { + try { + if (!clientSupportedAeads.includes(adoptAead)) { + throw new Error(`Unsupported adoption cipher: ${adoptAead}`); + } + sealedAccountAuth = seal( + adoptSessionKey, + buildAdoptHelloAad( + expectedHostDeviceId, + localDeviceId, + ), + Buffer.from(JSON.stringify(legacyAccountAuth), "utf8"), + undefined, + adoptAead, + ); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + if ( + !clientSupportedAeads.includes(adoptAead) + || /unknown cipher|unsupported/i.test(reason) + ) { + throw new Error(ADOPT_CHANNEL_UPDATE_REQUIRED_ERROR); + } + throw error; + } + } const hello: SyncHelloPayload = { peer, auth: adoptSessionKey @@ -852,14 +901,7 @@ export class DesktopPairedMachineStore { kind: "account_sealed", v: 1, deviceId: localDeviceId, - sealed: seal( - adoptSessionKey, - buildAdoptHelloAad( - expectedHostDeviceId, - localDeviceId, - ), - Buffer.from(JSON.stringify(legacyAccountAuth), "utf8"), - ), + sealed: sealedAccountAuth!, } : { kind: "account", @@ -904,6 +946,7 @@ export class DesktopPairedMachineStore { localDeviceId, ), sealedHelloOk.sealed, + adoptAead, ).toString("utf8")) as PairedRuntimeHelloOkPayload; } catch { throw hostIdentityVerificationError(); diff --git a/apps/desktop/src/shared/sync/adoptChannelCrypto.test.ts b/apps/desktop/src/shared/sync/adoptChannelCrypto.test.ts index 996d1d706..5bb7d2331 100644 --- a/apps/desktop/src/shared/sync/adoptChannelCrypto.test.ts +++ b/apps/desktop/src/shared/sync/adoptChannelCrypto.test.ts @@ -51,15 +51,55 @@ describe("adoptChannelCrypto", () => { expect(() => unseal(randomBytes(32), aad, sealed)).toThrow(); }); - it("builds the exact canonical challenge signature string", () => { - expect(buildAdoptChallengeSignatureInput({ + it("seals with AES-256-GCM using AAD and rejects tampering", () => { + const key = randomBytes(32); + const nonce = Buffer.alloc(12, 7); + const aad = Buffer.from("ade-adopt-v1|aes-host|aes-client"); + const plaintext = Buffer.from('{"accountToken":"aes-secret"}'); + const sealed = seal(key, aad, plaintext, nonce, "aes-256-gcm"); + const tampered = Buffer.from(sealed, "base64"); + tampered[nonce.byteLength + 1] ^= 0x01; + + expect(Buffer.from(sealed, "base64").subarray(0, nonce.byteLength)) + .toEqual(nonce); + expect(unseal(key, aad, sealed, "aes-256-gcm")).toEqual(plaintext); + expect(() => unseal( + key, + aad, + tampered.toString("base64"), + "aes-256-gcm", + )).toThrow(); + expect(() => unseal( + key, + Buffer.from("wrong aad"), + sealed, + "aes-256-gcm", + )).toThrow(); + }); + + it("keeps the legacy signature input byte-identical and appends a negotiated AEAD", () => { + const args = { hostDeviceId: "host-device", nonce: "bm9uY2U=", clientEphemeralPublicKey: "Y2xpZW50", hostEphemeralPublicKey: "aG9zdA==", ts: 1_783_500_123_456, - })).toMatchInlineSnapshot( - `"ade-adopt-v1|host-device|bm9uY2U=|Y2xpZW50|aG9zdA==|1783500123456"`, + }; + const legacyInput = buildAdoptChallengeSignatureInput(args); + const negotiatedInput = buildAdoptChallengeSignatureInput({ + ...args, + aead: "aes-256-gcm", + }); + + expect(legacyInput).toBe( + "ade-adopt-v1|host-device|bm9uY2U=|Y2xpZW50|aG9zdA==|1783500123456", + ); + expect(Buffer.from(legacyInput, "utf8")).toEqual(Buffer.from( + "ade-adopt-v1|host-device|bm9uY2U=|Y2xpZW50|aG9zdA==|1783500123456", + "utf8", + )); + expect(negotiatedInput).toBe( + "ade-adopt-v1|host-device|bm9uY2U=|Y2xpZW50|aG9zdA==|1783500123456|aes-256-gcm", ); }); diff --git a/apps/desktop/src/shared/sync/adoptChannelCrypto.ts b/apps/desktop/src/shared/sync/adoptChannelCrypto.ts index fcf50f578..44e5749f8 100644 --- a/apps/desktop/src/shared/sync/adoptChannelCrypto.ts +++ b/apps/desktop/src/shared/sync/adoptChannelCrypto.ts @@ -21,10 +21,16 @@ export const ADOPT_CHANNEL_NONCE_BYTES = 32; export const ADOPT_CHANNEL_KEY_BYTES = 32; export const ADOPT_CHANNEL_AEAD_NONCE_BYTES = 12; export const ADOPT_CHANNEL_PUBLIC_KEY_BYTES = 32; +export const ADOPT_CHANNEL_AEADS = [ + "chacha20-poly1305", + "aes-256-gcm", +] as const; +export type AdoptChannelAead = (typeof ADOPT_CHANNEL_AEADS)[number]; const X25519_SPKI_PREFIX = Buffer.from("302a300506032b656e032100", "hex"); const ED25519_SPKI_PREFIX = Buffer.from("302a300506032b6570032100", "hex"); const AEAD_TAG_BYTES = 16; +let cachedSupportedAdoptChannelAeads: AdoptChannelAead[] | null = null; function assertLength(value: Buffer, expected: number, label: string): void { if (value.byteLength !== expected) { @@ -81,6 +87,7 @@ export function buildAdoptChallengeSignatureInput(args: { clientEphemeralPublicKey: string; hostEphemeralPublicKey: string; ts: number; + aead?: string; }): string { return [ ADOPT_CHANNEL_CONTEXT, @@ -89,6 +96,7 @@ export function buildAdoptChallengeSignatureInput(args: { args.clientEphemeralPublicKey, args.hostEphemeralPublicKey, String(args.ts), + ...(args.aead === undefined ? [] : [args.aead]), ].join("|"); } @@ -136,17 +144,60 @@ export function deriveAdoptSessionKey(args: { )); } +function createAdoptChannelCipher( + aead: AdoptChannelAead, + key: Buffer, + nonce: Buffer, +) { + return aead === "chacha20-poly1305" + ? createCipheriv("chacha20-poly1305", key, nonce, { + authTagLength: AEAD_TAG_BYTES, + }) + : createCipheriv("aes-256-gcm", key, nonce, { + authTagLength: AEAD_TAG_BYTES, + }); +} + +function createAdoptChannelDecipher( + aead: AdoptChannelAead, + key: Buffer, + nonce: Buffer, +) { + return aead === "chacha20-poly1305" + ? createDecipheriv("chacha20-poly1305", key, nonce, { + authTagLength: AEAD_TAG_BYTES, + }) + : createDecipheriv("aes-256-gcm", key, nonce, { + authTagLength: AEAD_TAG_BYTES, + }); +} + +export function supportedAdoptChannelAeads(): AdoptChannelAead[] { + if (!cachedSupportedAdoptChannelAeads) { + const key = Buffer.alloc(ADOPT_CHANNEL_KEY_BYTES); + const nonce = Buffer.alloc(ADOPT_CHANNEL_AEAD_NONCE_BYTES); + cachedSupportedAdoptChannelAeads = ADOPT_CHANNEL_AEADS.filter((aead) => { + try { + createAdoptChannelCipher(aead, key, nonce); + return true; + } catch { + return false; + } + }); + } + return [...cachedSupportedAdoptChannelAeads]; +} + export function seal( key: Buffer, aad: Buffer, plaintext: Buffer, nonce: Buffer = randomBytes(ADOPT_CHANNEL_AEAD_NONCE_BYTES), + aead: AdoptChannelAead = "chacha20-poly1305", ): string { assertLength(key, ADOPT_CHANNEL_KEY_BYTES, "Adoption session key"); assertLength(nonce, ADOPT_CHANNEL_AEAD_NONCE_BYTES, "AEAD nonce"); - const cipher = createCipheriv("chacha20-poly1305", key, nonce, { - authTagLength: AEAD_TAG_BYTES, - }); + const cipher = createAdoptChannelCipher(aead, key, nonce); cipher.setAAD(aad, { plaintextLength: plaintext.byteLength }); const ciphertext = Buffer.concat([cipher.update(plaintext), cipher.final()]); return Buffer.concat([nonce, ciphertext, cipher.getAuthTag()]).toString("base64"); @@ -156,6 +207,7 @@ export function unseal( key: Buffer, aad: Buffer, blobBase64: string, + aead: AdoptChannelAead = "chacha20-poly1305", ): Buffer { assertLength(key, ADOPT_CHANNEL_KEY_BYTES, "Adoption session key"); const blob = decodeCanonicalBase64(blobBase64); @@ -171,9 +223,7 @@ export function unseal( ADOPT_CHANNEL_AEAD_NONCE_BYTES, blob.byteLength - AEAD_TAG_BYTES, ); - const decipher = createDecipheriv("chacha20-poly1305", key, nonce, { - authTagLength: AEAD_TAG_BYTES, - }); + const decipher = createAdoptChannelDecipher(aead, key, nonce); decipher.setAAD(aad, { plaintextLength: ciphertext.byteLength }); decipher.setAuthTag(tag); return Buffer.concat([decipher.update(ciphertext), decipher.final()]); diff --git a/apps/desktop/src/shared/types/sync.ts b/apps/desktop/src/shared/types/sync.ts index 076b65731..1bb185c23 100644 --- a/apps/desktop/src/shared/types/sync.ts +++ b/apps/desktop/src/shared/types/sync.ts @@ -807,6 +807,7 @@ export type SyncAccountChallengePayload = { v: 1; nonce: string; clientEphemeralPublicKey: string; + supportedAeads?: string[]; }; export type SyncAccountChallengeOkPayload = { @@ -815,6 +816,7 @@ export type SyncAccountChallengeOkPayload = { ts: number; hostEphemeralPublicKey: string; signature: string; + aead?: string; }; export type SyncAccountChallengeErrorPayload = { From d89f28fa1dba9594c339bda7828cd118ae9a46ee Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:18:16 -0400 Subject: [PATCH 2/7] Reliability: guaranteed brain updates, wedge-proofing, verified handover, publish health, truthful version UI Brain lifecycle: freshness self-monitor (stat-gated, idle-drained self-restart), wedge watchdog (worker-thread loop monitor, breadcrumbed SIGKILL recovery), responsiveness-probed launchd repair with verified PID handover, sync-listener zombie reaping with PID-reuse guards, port-drift republish, and a protocol compatibility window so newer brains connect instead of being quarantined. Updates: transactional install (rollback + parked state on abort, 10s quit escalation after consent), idle auto-apply with cancelable countdown, unified latest-version source, and truthful Running/Installed/Latest UI for both the app and the brain. Diagnostics: per-leg publish budgets with honest token/http timeout classification, cancellable Clerk refresh, rotating timestamped brain JSONL logs, ade doctor, machine-card publish-health line, wedge recovery notice, and reliability telemetry events. Co-Authored-By: Claude Fable 5 --- apps/account-directory/wrangler.jsonc | 1 + apps/ade-cli/src/adeRpcServer.ts | 3 + apps/ade-cli/src/bootstrap.ts | 1 + apps/ade-cli/src/cli.test.ts | 150 +---- apps/ade-cli/src/cli.ts | 559 ++++++++++++++++-- apps/ade-cli/src/commands/brainUpdate.ts | 34 +- apps/ade-cli/src/commands/doctor.test.ts | 144 +++++ apps/ade-cli/src/commands/doctor.ts | 335 +++++++++++ .../ade-cli/src/multiProjectRpcServer.test.ts | 58 ++ apps/ade-cli/src/multiProjectRpcServer.ts | 90 ++- .../ade-cli/src/serviceManager/common.test.ts | 132 ++++- apps/ade-cli/src/serviceManager/common.ts | 29 + apps/ade-cli/src/serviceManager/index.ts | 2 +- .../src/serviceManager/installLaunchd.ts | 160 ++++- .../account/accountAuthService.test.ts | 34 ++ .../services/account/accountAuthService.ts | 125 +++- .../accountMachinePublisherService.test.ts | 222 ++++++- .../account/accountMachinePublisherService.ts | 365 ++++++++++-- .../personalChats/personalChatScope.ts | 26 + .../src/services/projects/projectScope.ts | 15 +- .../runtime/brainFreshnessMonitor.test.ts | 98 +++ .../services/runtime/brainFreshnessMonitor.ts | 175 ++++++ .../src/services/runtime/brainLogger.test.ts | 68 +++ .../src/services/runtime/brainLogger.ts | 53 ++ .../runtime/brainLoopWatchdog.test.ts | 119 ++++ .../src/services/runtime/brainLoopWatchdog.ts | 299 ++++++++++ .../services/runtime/runtimeBuildIdentity.ts | 22 + .../src/services/sync/sharedSyncListener.ts | 144 ++++- .../src/services/sync/syncHostService.test.ts | 178 +++++- .../src/services/sync/syncHostService.ts | 175 +++++- .../src/services/sync/syncHostSingleton.ts | 4 +- .../sync/syncLoopbackCollision.test.ts | 68 ++- .../services/sync/syncPairingConnectInfo.ts | 5 +- .../ade-cli/src/services/sync/syncProtocol.ts | 1 + .../sync/syncRemoteCommandService.test.ts | 42 ++ .../services/sync/syncRemoteCommandService.ts | 121 +++- apps/ade-cli/src/services/sync/syncService.ts | 19 +- apps/desktop/src/main/main.ts | 60 +- .../analytics/productAnalyticsPolicy.ts | 28 +- .../analytics/productAnalyticsService.test.ts | 70 +++ .../services/github/githubService.test.ts | 40 +- .../src/main/services/github/githubService.ts | 127 +++- .../src/main/services/ipc/registerIpc.ts | 22 +- .../localRuntimeConnectionPool.test.ts | 137 ++++- .../localRuntimeConnectionPool.ts | 230 ++++++- .../src/main/services/logging/logger.ts | 32 +- .../updates/autoUpdateService.test.ts | 314 ++++++---- .../services/updates/autoUpdateService.ts | 498 ++++++++++++---- apps/desktop/src/preload/global.d.ts | 1 + apps/desktop/src/preload/preload.ts | 2 + apps/desktop/src/renderer/browserMock.ts | 11 + .../src/renderer/components/app/AppShell.tsx | 6 + .../components/app/AutoUpdateBanner.test.tsx | 159 +++++ .../components/app/AutoUpdateBanner.tsx | 144 +++++ .../components/app/AutoUpdateControl.test.tsx | 4 + .../components/app/AutoUpdateControl.tsx | 17 +- .../app/BrainRecoveryNotice.test.ts | 62 ++ .../components/app/BrainRecoveryNotice.tsx | 106 ++++ .../components/app/useAutoUpdateSnapshot.ts | 53 ++ .../remoteTargets/RemoteTargetList.tsx | 61 ++ .../remoteMachineModel.publishHealth.test.ts | 67 +++ .../remoteTargets/remoteMachineModel.ts | 53 ++ .../components/settings/AboutSection.tsx | 101 +++- .../src/renderer/webclient/adapter/app.ts | 19 + .../src/renderer/webclient/adapter/index.ts | 1 + apps/desktop/src/shared/adeRuntimeProtocol.ts | 20 + apps/desktop/src/shared/ipc.ts | 1 + apps/desktop/src/shared/types/core.ts | 44 ++ .../src/shared/types/productAnalytics.ts | 6 + apps/desktop/src/shared/types/sync.ts | 16 + docs/ARCHITECTURE.md | 11 + docs/features/remote-runtime/README.md | 10 +- docs/logging.md | 2 + 73 files changed, 5934 insertions(+), 677 deletions(-) create mode 100644 apps/ade-cli/src/commands/doctor.test.ts create mode 100644 apps/ade-cli/src/commands/doctor.ts create mode 100644 apps/ade-cli/src/services/runtime/brainFreshnessMonitor.test.ts create mode 100644 apps/ade-cli/src/services/runtime/brainFreshnessMonitor.ts create mode 100644 apps/ade-cli/src/services/runtime/brainLogger.test.ts create mode 100644 apps/ade-cli/src/services/runtime/brainLogger.ts create mode 100644 apps/ade-cli/src/services/runtime/brainLoopWatchdog.test.ts create mode 100644 apps/ade-cli/src/services/runtime/brainLoopWatchdog.ts create mode 100644 apps/ade-cli/src/services/runtime/runtimeBuildIdentity.ts create mode 100644 apps/desktop/src/renderer/components/app/AutoUpdateBanner.test.tsx create mode 100644 apps/desktop/src/renderer/components/app/AutoUpdateBanner.tsx create mode 100644 apps/desktop/src/renderer/components/app/BrainRecoveryNotice.test.ts create mode 100644 apps/desktop/src/renderer/components/app/BrainRecoveryNotice.tsx create mode 100644 apps/desktop/src/renderer/components/app/useAutoUpdateSnapshot.ts create mode 100644 apps/desktop/src/renderer/components/remoteTargets/remoteMachineModel.publishHealth.test.ts create mode 100644 apps/desktop/src/shared/adeRuntimeProtocol.ts diff --git a/apps/account-directory/wrangler.jsonc b/apps/account-directory/wrangler.jsonc index af20b7728..ce03084c0 100644 --- a/apps/account-directory/wrangler.jsonc +++ b/apps/account-directory/wrangler.jsonc @@ -5,6 +5,7 @@ "compatibility_date": "2026-06-30", "workers_dev": true, "preview_urls": false, + "observability": { "enabled": true, "head_sampling_rate": 1 }, "triggers": { "crons": ["* * * * *"] }, "vars": { "ONLINE_WINDOW_MS": "90000", diff --git a/apps/ade-cli/src/adeRpcServer.ts b/apps/ade-cli/src/adeRpcServer.ts index 9b7aee152..ae132ad6d 100644 --- a/apps/ade-cli/src/adeRpcServer.ts +++ b/apps/ade-cli/src/adeRpcServer.ts @@ -73,6 +73,7 @@ import { resolveLaneCreateRemoteBase } from "./services/laneCreateRemoteBase"; import { BUILT_IN_BROWSER_ACTOR_CAPABILITY_PARAM } from "./services/builtInBrowser/desktopBridgeMethods"; import { resolveCodexComputerUseMcpConfig } from "../../desktop/src/main/utils/codexComputerUse"; import { parseTrackedCliLaunchConfig } from "../../desktop/src/main/utils/terminalSessionSignals"; +import { RUNTIME_COMPAT_LEVEL } from "../../desktop/src/shared/adeRuntimeProtocol"; // Cross-surface (desktop + TUI + iOS) model picker favorites & recents. // Backed by the per-project cr-sqlite CRR DB (runtime.db) so the three surfaces @@ -5177,6 +5178,8 @@ export function createAdeRpcRequestHandler(args: { runtimeInfo: { name: "ade-rpc", version: serverVersion, + minCompatibleProtocol: RUNTIME_COMPAT_LEVEL, + protocolVersion: RUNTIME_COMPAT_LEVEL, buildHash: typeof process.env.ADE_RUNTIME_BUILD_HASH === "string" && process.env.ADE_RUNTIME_BUILD_HASH.trim() diff --git a/apps/ade-cli/src/bootstrap.ts b/apps/ade-cli/src/bootstrap.ts index 426ed099e..23dca2215 100644 --- a/apps/ade-cli/src/bootstrap.ts +++ b/apps/ade-cli/src/bootstrap.ts @@ -1600,6 +1600,7 @@ export async function createAdeRuntime(args: { productAnalyticsService, logger, getAccountDirectoryHealth: resolvedArgs.syncRuntime.getAccountDirectoryHealth, + requestAccountMachinePublish: resolvedArgs.syncRuntime.requestAccountMachinePublish, accountAuthService, projectId: resolvedArgs.syncRuntime.registryProjectId ?? projectId, runtimeProjectId: projectId, diff --git a/apps/ade-cli/src/cli.test.ts b/apps/ade-cli/src/cli.test.ts index 4bff9b35a..1b8fa9bdb 100644 --- a/apps/ade-cli/src/cli.test.ts +++ b/apps/ade-cli/src/cli.test.ts @@ -1,6 +1,7 @@ import { spawn } from "node:child_process"; import fs from "node:fs"; import { createServer } from "node:http"; +import net from "node:net"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; @@ -4992,130 +4993,41 @@ describe("ADE CLI", () => { expect(output).toContain("Git repository detected"); }); - it("adds sync route health to doctor and names a loopback listener mismatch", () => { - const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), "ade-cli-doctor-sync-")); - fs.mkdirSync(path.join(projectRoot, ".ade"), { recursive: true }); - try { - const plan = expectExecutePlan(buildCliPlan(["doctor"])); - expect(plan.steps).toContainEqual({ - key: "syncStatus", - method: "sync.getStatus", - params: { includeTransferReadiness: false }, - optional: true, - }); - expect(plan.steps).toContainEqual({ - key: "relaySelfProbe", - method: "sync.runSelfProbe", - optional: true, - }); - const summary = summarizeExecution({ - plan, - connection: { - mode: "runtime-socket", - projectRoot, - workspaceRoot: projectRoot, - socketPath: path.join(projectRoot, ".ade", "ade.sock"), - }, - values: { - rpcActions: { actions: [{}] }, - actions: { actions: [{}] }, - syncStatus: { - pairingConnectInfo: { port: 8787 }, - routeHealth: { - listener: { - listenerBound: true, - loopbackAdeValidated: false, - reason: "Expected ADE 426 Upgrade Required; received 404 Not Found.", - }, - tailscale: { - enabled: true, - tailscaleReachable: false, - reason: "Tailscale route points at the listener mismatch.", - }, - relay: { - enabled: true, - relayControlConnected: false, - relayBridgeValidated: false, - skipReason: "Relay control upgrade failed with HTTP 401: token expired", - lastControlError: "Relay control closed (1006).", - }, - }, - }, - relaySelfProbe: { - ok: false, - detail: "Relay self-probe closed before ready (4501): host offline.", - }, - }, - } as any) as Record; - - expect(summary.sync).toMatchObject({ - enabled: true, - usable: false, - status: "warning", - }); - expect(summary.sync.failingRoutes).toEqual([ - expect.stringContaining("listener"), - expect.stringContaining("tailscale"), - "relay: Relay control upgrade failed with HTTP 401: token expired", - ]); - expect(summary.sync.message).toContain("404 Not Found"); - expect(summary.sync.message).toContain("HTTP 401: token expired"); - expect(summary.relaySelfProbe).toMatchObject({ - ready: false, - status: "warning", - message: expect.stringContaining("FAILED"), - }); - const output = formatOutput(summary, { - projectRoot, - workspaceRoot: projectRoot, - role: "agent", - headless: false, - requireSocket: false, - socketPath: null, - pretty: true, - text: true, - timeoutMs: 1000, - }, "doctor"); - expect(output).toContain("Sync route failure"); - expect(output).toContain("listener"); - expect(output).toContain("relay self-probe"); - expect(output).toContain("host offline"); - } finally { - fs.rmSync(projectRoot, { recursive: true, force: true }); - } + it("builds doctor as a read-only local command and keeps online checks explicit", () => { + expect(buildCliPlan(["doctor"])).toEqual({ kind: "doctor", online: false }); + expect(buildCliPlan(["doctor", "--online"])).toEqual({ kind: "doctor", online: true }); }); - it("marks the doctor Relay self-probe skipped when no brain is running", () => { - const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), "ade-cli-doctor-no-brain-")); - fs.mkdirSync(path.join(projectRoot, ".ade"), { recursive: true }); + it("bounds doctor when a dead socket accepts a connection but never responds", async () => { + if (process.platform === "win32") return; + const root = fs.mkdtempSync(path.join(os.tmpdir(), "ade-cli-doctor-dead-sock-")); + const socketPath = path.join(root, "ade.sock"); + const acceptedSockets = new Set(); + const server = net.createServer((socket) => { + acceptedSockets.add(socket); + socket.once("close", () => acceptedSockets.delete(socket)); + }); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(socketPath, resolve); + }); + const startedAt = Date.now(); try { - const summary = summarizeExecution({ - plan: expectExecutePlan(buildCliPlan(["doctor"])), - connection: { - mode: "headless", - projectRoot, - workspaceRoot: projectRoot, - socketPath: path.join(projectRoot, ".ade", "ade.sock"), - }, - values: { - rpcActions: { actions: [{}] }, - actions: { actions: [{}] }, - relaySelfProbe: { - ok: false, - detail: "Relay self-probe skipped because the control socket is not connected.", - }, - }, - } as any) as Record; - - expect(summary.relaySelfProbe).toEqual(expect.objectContaining({ - ready: true, - status: "unavailable", - message: "Skipped — no running ADE brain.", - })); + const result = await runCli(["--socket", socketPath, "doctor", "--json"]); + expect(result.exitCode).toBe(1); + expect(JSON.parse(result.output)).toMatchObject({ + ok: false, + rows: expect.arrayContaining([ + expect.objectContaining({ key: "brain", status: "fail" }), + ]), + }); + expect(Date.now() - startedAt).toBeLessThan(2_750); } finally { - fs.rmSync(projectRoot, { recursive: true, force: true }); + for (const socket of acceptedSockets) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + fs.rmSync(root, { recursive: true, force: true }); } - }); + }, 5_000); it("detects project-local Linear credentials in doctor readiness", () => { const previousAdeLinearApi = process.env.ADE_LINEAR_API; diff --git a/apps/ade-cli/src/cli.ts b/apps/ade-cli/src/cli.ts index b029e6d18..cf517b01b 100644 --- a/apps/ade-cli/src/cli.ts +++ b/apps/ade-cli/src/cli.ts @@ -1,12 +1,12 @@ #!/usr/bin/env node import { Buffer } from "node:buffer"; import { spawn, spawnSync } from "node:child_process"; -import { createHash } from "node:crypto"; import fs from "node:fs"; import net from "node:net"; import os from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; +import { computeRuntimeBuildHash } from "./services/runtime/runtimeBuildIdentity"; import YAML from "yaml"; import { CURSOR_CLOUD_HELP, @@ -23,6 +23,11 @@ import { CliSkillUsageError, runSkillCommand, } from "./commands/skill"; +import { + evaluateDoctorRows, + type DoctorInput, + type DoctorRow, +} from "./commands/doctor"; import { buildDeeplink, type DeeplinkEnvelope } from "../../desktop/src/shared/deeplinks"; import { buildPairingQrPayload } from "../../desktop/src/shared/pairingQr"; import { buildWebClientPairUrl } from "../../desktop/src/shared/webClientUrl"; @@ -42,6 +47,7 @@ import { browseProjectDirectories } from "../../desktop/src/main/services/projec import { createProjectScaffoldService } from "../../desktop/src/main/services/projects/projectScaffoldService"; import { resolveRepoRoot } from "../../desktop/src/main/services/projects/projectService"; import type { Logger } from "../../desktop/src/main/services/logging/logger"; +import { createBrainLogger } from "./services/runtime/brainLogger"; import type { CloneProjectInput, CreateProjectInput, @@ -99,6 +105,7 @@ import { } from "../../desktop/src/shared/types/sync"; import { isCurrentProcessDescendantOfPid, + resolveAdeServeCommand, type AdeServiceCommand, } from "./serviceManager/common"; import { normalizeAdeRuntimeRole, resolveAdeDefaultRole } from "./runtimeRoles"; @@ -124,6 +131,10 @@ import type { AdeLastFailureReport, AdeRecoveryErrorCode } from "../../desktop/s import { createDiskPressureMonitor } from "../../desktop/src/main/services/storage/diskPressure"; import { isUrgentDiskPressure } from "../../desktop/src/shared/types/storage"; import { boundLaunchdLogs } from "./services/runtime/runtimeLogMaintenance"; +import { + readBrainLoopWatchdogLastWedge, + startBrainLoopWatchdog, +} from "./services/runtime/brainLoopWatchdog"; type JsonObject = Record; @@ -288,6 +299,7 @@ type CliPlan = | { kind: "desktop"; rest: string[] } | { kind: "runtime"; rest: string[] } | { kind: "brain"; rest: string[] } + | { kind: "doctor"; online: boolean } | { kind: "serve"; rest: string[] } | { kind: "rpc-stdio"; rest: string[] } | { kind: "pty-host-worker" } @@ -572,7 +584,7 @@ const TOP_LEVEL_HELP = `${ADE_BANNER} $ ade projects list List projects registered on this machine $ ade sync web [--open] [--no-clipboard] Print (and copy) the web client pairing link + code $ ade sync status | pin generate Manage machine sync and phone pairing - $ ade doctor Inspect project, brain, runtime, and tool availability + $ ade doctor [--online] Inspect installed app and machine-brain health $ ade lanes list | show | create | child Work with lanes and lane stacks $ ade git status | commit | push | stash Run ADE-aware git operations $ ade operations status | wait Poll operation/test/chat/run status @@ -11886,29 +11898,8 @@ function buildCliPlan( } if (primary === "doctor") { return { - kind: "execute", - label: "doctor", - summary: "doctor", - steps: [ - { key: "ping", method: "ping" }, - { key: "rpcActions", method: "ade/actions/list" }, - listActionsStep("actions"), - { - ...actionStep("projectConfig", "project_config", "get"), - optional: true, - }, - { - key: "syncStatus", - method: "sync.getStatus", - params: { includeTransferReadiness: false }, - optional: true, - }, - { - key: "relaySelfProbe", - method: "sync.runSelfProbe", - optional: true, - }, - ], + kind: "doctor", + online: readFlag(args, ["--online"]), }; } if (primary === "auth") { @@ -13085,6 +13076,10 @@ class SocketJsonRpcClient { this.socket.end(); } + destroy(): void { + this.socket.destroy(); + } + private onData(chunk: Buffer): void { this.buffer = this.buffer.length ? Buffer.concat([this.buffer, chunk]) @@ -14202,6 +14197,7 @@ type MachineRuntimeInfo = { packageChannel: string | null; projectRoot: string | null; pid: number | null; + uptimeMs?: number | null; }; function readMachineRuntimeInfo(value: unknown): MachineRuntimeInfo { @@ -14213,9 +14209,11 @@ function readMachineRuntimeInfo(value: unknown): MachineRuntimeInfo { packageChannel: null, projectRoot: null, pid: null, + uptimeMs: null, }; } const pid = value.runtimeInfo.pid; + const uptimeMs = value.runtimeInfo.uptimeMs; return { version: asString(value.runtimeInfo.version), buildHash: asString(value.runtimeInfo.buildHash), @@ -14226,6 +14224,10 @@ function readMachineRuntimeInfo(value: unknown): MachineRuntimeInfo { typeof pid === "number" && Number.isFinite(pid) && pid > 0 ? Math.floor(pid) : null, + uptimeMs: + typeof uptimeMs === "number" && Number.isFinite(uptimeMs) && uptimeMs >= 0 + ? Math.floor(uptimeMs) + : null, }; } @@ -14287,20 +14289,14 @@ export function shouldEnforceMachineRuntimeBuildCompatibility( return !socketPathOverride?.trim(); } -function computeRuntimeBuildHash(filePath: string): string | null { - try { - return createHash("sha256").update(fs.readFileSync(filePath)).digest("hex"); - } catch { - return null; - } -} - function prepareMachineRuntimeDaemonCommand(serviceCommand: AdeServiceCommand): { args: string[]; buildHash: string | null; + filePath: string | null; } { const args = [...serviceCommand.args]; let buildHash: string | null = null; + let filePath: string | null = null; if ( serviceCommand.command === process.execPath && args.length === 1 && @@ -14308,13 +14304,16 @@ function prepareMachineRuntimeDaemonCommand(serviceCommand: AdeServiceCommand): fs.existsSync(CLI_DIST_PATH) ) { args.splice(0, 1, CLI_DIST_PATH, "serve"); - buildHash = computeRuntimeBuildHash(CLI_DIST_PATH); + filePath = CLI_DIST_PATH; + buildHash = computeRuntimeBuildHash(filePath); } else if (serviceCommand.command === process.execPath && args[0]) { - buildHash = computeRuntimeBuildHash(path.resolve(args[0])); + filePath = path.resolve(args[0]); + buildHash = computeRuntimeBuildHash(filePath); } else if (fs.existsSync(serviceCommand.command)) { - buildHash = computeRuntimeBuildHash(path.resolve(serviceCommand.command)); + filePath = path.resolve(serviceCommand.command); + buildHash = computeRuntimeBuildHash(filePath); } - return { args, buildHash }; + return { args, buildHash, filePath }; } async function resolveExpectedMachineRuntimeBuildHash(): Promise { @@ -14952,6 +14951,360 @@ async function runBrainCommand( ); } +type DoctorBrainProbe = { + brain: DoctorInput["brain"]; + syncStatus: JsonObject | null; + account: DoctorInput["account"]; + runtimeSyncPort: number | null; + runtimePublishHealth: DoctorInput["publishHealth"]; + runtimeLastWedge: DoctorInput["wedge"]; +}; + +const DOCTOR_BRAIN_DEADLINE_MS = 2_000; +const DOCTOR_ONLINE_DEADLINE_MS = 1_200; + +function doctorTimeout( + promise: Promise, + deadlineMs: number, + label: string, +): Promise { + let timer: ReturnType | null = null; + return Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error(`${label} timed out.`)), Math.max(1, deadlineMs)); + }), + ]).finally(() => { + if (timer) clearTimeout(timer); + }); +} + +function doctorRuntimeStatusFromInitialize(value: unknown): { + syncPort: number | null; + publishHealth: DoctorInput["publishHealth"]; + lastWedge: DoctorInput["wedge"]; +} { + const runtimeInfo = isRecord(value) && isRecord(value.runtimeInfo) + ? value.runtimeInfo + : {}; + const syncPort = typeof runtimeInfo.syncPort === "number" + && Number.isInteger(runtimeInfo.syncPort) + && runtimeInfo.syncPort > 0 + && runtimeInfo.syncPort <= 65_535 + ? runtimeInfo.syncPort + : null; + const rawPublish = isRecord(runtimeInfo.publishHealth) ? runtimeInfo.publishHealth : null; + const rawDurations = rawPublish && isRecord(rawPublish.lastLegDurations) + ? rawPublish.lastLegDurations + : null; + const publishHealth = rawPublish && rawDurations && asString(rawPublish.state) + ? { + state: asString(rawPublish.state)! as NonNullable["state"], + failingSinceMs: + typeof rawPublish.failingSinceMs === "number" && Number.isFinite(rawPublish.failingSinceMs) + ? Math.max(0, rawPublish.failingSinceMs) + : null, + lastLegDurations: { + snapshot: finiteDoctorDuration(rawDurations.snapshot), + token: finiteDoctorDuration(rawDurations.token), + http: finiteDoctorDuration(rawDurations.http), + }, + lastSuccessAt: + typeof rawPublish.lastSuccessAt === "number" && Number.isFinite(rawPublish.lastSuccessAt) + ? Math.max(0, rawPublish.lastSuccessAt) + : null, + skipReason: asString(rawPublish.skipReason), + } + : null; + const rawWedge = isRecord(runtimeInfo.lastWedge) ? runtimeInfo.lastWedge : null; + const lastWedge = rawWedge + && asString(rawWedge.lastCommand) + && typeof rawWedge.blockedMs === "number" + && Number.isFinite(rawWedge.blockedMs) + && asString(rawWedge.ts) + ? { + lastCommand: asString(rawWedge.lastCommand)!, + blockedMs: Math.max(0, Math.floor(rawWedge.blockedMs)), + ts: asString(rawWedge.ts)!, + } + : null; + return { syncPort, publishHealth, lastWedge }; +} + +function finiteDoctorDuration(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? Math.round(value) + : null; +} + +async function probeDoctorBrain( + options: GlobalOptions, +): Promise { + const startedAt = Date.now(); + const deadlineAt = startedAt + DOCTOR_BRAIN_DEADLINE_MS; + const socketPath = await resolveMachineRuntimeSocketPath(options.socketPath); + let client: SocketJsonRpcClient | null = null; + try { + const remaining = () => Math.max(1, deadlineAt - Date.now()); + client = await doctorTimeout( + SocketJsonRpcClient.connect(socketPath, remaining(), "ADE brain"), + remaining(), + "ADE brain connection", + ); + const initializeResult = await doctorTimeout( + client.request( + "ade/initialize", + buildInitializeParams(options, "ade-doctor"), + ), + remaining(), + "ADE brain initialize", + ); + const runtimeInfo = readMachineRuntimeInfo(initializeResult); + const runtimeStatus = doctorRuntimeStatusFromInitialize(initializeResult); + const expectedBuildHash = await doctorTimeout( + resolveExpectedMachineRuntimeBuildHash(), + remaining(), + "ADE runtime build identity", + ).catch(() => null); + const mismatchReason = machineRuntimeMismatchReason( + runtimeInfo, + expectedBuildHash, + options.role, + ); + const [syncResult, accountResult] = await doctorTimeout( + Promise.allSettled([ + client.request("sync.getStatus", { includeTransferReadiness: false }), + client.request("account.call", { action: "status", args: {} }), + ]), + remaining(), + "ADE brain health reads", + ); + const syncStatus = syncResult.status === "fulfilled" && isRecord(syncResult.value) + ? syncResult.value + : null; + const accountValue = accountResult.status === "fulfilled" + ? unwrapActionEnvelope(accountResult.value) + : null; + const accountStatus = isRecord(accountValue) ? accountValue : null; + return { + brain: { + running: true, + version: runtimeInfo.version, + buildHash: runtimeInfo.buildHash, + pid: runtimeInfo.pid, + uptimeMs: runtimeInfo.uptimeMs ?? null, + mismatchReason, + error: null, + }, + syncStatus, + account: { + signedIn: typeof accountStatus?.signedIn === "boolean" ? accountStatus.signedIn : null, + source: asString(accountStatus?.source), + error: accountResult.status === "rejected" + ? accountResult.reason instanceof Error + ? accountResult.reason.message + : String(accountResult.reason) + : accountStatus + ? null + : "Account status unavailable.", + }, + runtimeSyncPort: runtimeStatus.syncPort, + runtimePublishHealth: runtimeStatus.publishHealth, + runtimeLastWedge: runtimeStatus.lastWedge, + }; + } catch (error) { + return { + brain: { + running: false, + version: null, + buildHash: null, + pid: null, + uptimeMs: null, + mismatchReason: null, + error: error instanceof Error ? error.message : String(error), + }, + syncStatus: null, + account: { + signedIn: null, + source: null, + error: "Brain unavailable.", + }, + runtimeSyncPort: null, + runtimePublishHealth: null, + runtimeLastWedge: null, + }; + } finally { + try { + client?.destroy(); + } catch {} + } +} + +export function readInstalledDesktopVersion( + appPaths?: string[], +): { version: string | null; path: string | null } { + if (process.platform !== "darwin" && !appPaths) { + return { version: null, path: null }; + } + const explicitPath = process.env.ADE_DESKTOP_APP_PATH?.trim(); + const preferredName = resolveDefaultDesktopAppName(); + const candidates = appPaths ?? [ + ...(explicitPath ? [explicitPath] : []), + `/Applications/${preferredName}.app`, + "/Applications/ADE.app", + "/Applications/ADE Beta.app", + "/Applications/ADE Alpha.app", + ]; + for (const appPath of Array.from(new Set(candidates))) { + const infoPlist = path.join(appPath, "Contents", "Info.plist"); + if (!fs.existsSync(infoPlist)) continue; + try { + const raw = fs.readFileSync(infoPlist, "utf8"); + const xmlMatch = /\s*CFBundleShortVersionString\s*<\/key>\s*\s*([^<]+?)\s*<\/string>/i.exec(raw); + if (xmlMatch?.[1]?.trim()) { + return { version: xmlMatch[1].trim(), path: appPath }; + } + } catch { + // Binary plists are handled by plutil below. + } + const result = spawnSync( + "/usr/bin/plutil", + ["-extract", "CFBundleShortVersionString", "raw", "-o", "-", infoPlist], + { encoding: "utf8", timeout: 500 }, + ); + const version = typeof result.stdout === "string" ? result.stdout.trim() : ""; + if (result.status === 0 && version) { + return { version, path: appPath }; + } + } + return { version: null, path: null }; +} + +async function readLatestDesktopVersionOnline(): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), DOCTOR_ONLINE_DEADLINE_MS); + try { + const { fetchAdeLatestRelease } = await import( + "../../desktop/src/main/services/github/githubService" + ); + const release = await doctorTimeout( + fetchAdeLatestRelease({ + fetchImpl: ((input: string, init?: RequestInit) => + fetch(input, { ...init, signal: controller.signal })) as never, + }), + DOCTOR_ONLINE_DEADLINE_MS, + "Latest release check", + ); + return release?.version ?? null; + } catch { + return null; + } finally { + clearTimeout(timer); + } +} + +function readLatestKnownDesktopVersionFromDisk(runtimeDir: string): string | null { + try { + const parsed = JSON.parse( + fs.readFileSync(path.join(runtimeDir, "update-status.json"), "utf8"), + ) as Record; + return asString(parsed.version); + } catch { + return null; + } +} + +async function runDoctorCommand( + online: boolean, + options: GlobalOptions, +): Promise<{ + ok: boolean; + checkedAt: string; + online: boolean; + rows: DoctorRow[]; + app: DoctorInput["app"]; + brain: DoctorInput["brain"]; + wedge: DoctorInput["wedge"]; + syncPort: number | null; + portDiagnoses: DoctorInput["portDiagnoses"]; + publishHealth: DoctorInput["publishHealth"]; + relayHealth: DoctorInput["relayHealth"]; + account: DoctorInput["account"]; +}> { + const nowMs = Date.now(); + const layout = resolveMachineAdeLayout(); + const diskLatestKnownVersion = readLatestKnownDesktopVersionFromDisk(layout.runtimeDir); + const [installedApp, latestKnownVersion, brainProbe] = await Promise.all([ + Promise.resolve(readInstalledDesktopVersion()), + online + ? readLatestDesktopVersionOnline().then((version) => version ?? diskLatestKnownVersion) + : Promise.resolve(diskLatestKnownVersion), + probeDoctorBrain(options), + ]); + const syncRouteHealth = brainProbe.syncStatus && isRecord(brainProbe.syncStatus.routeHealth) + ? brainProbe.syncStatus.routeHealth + : null; + const listener = syncRouteHealth && isRecord(syncRouteHealth.listener) + ? syncRouteHealth.listener + : null; + const syncPort = typeof listener?.port === "number" && Number.isInteger(listener.port) + ? listener.port + : brainProbe.runtimeSyncPort; + const rawPublishHealth = syncRouteHealth && isRecord(syncRouteHealth.accountDirectory) + ? syncRouteHealth.accountDirectory + : null; + const publishHealth = rawPublishHealth + ? doctorRuntimeStatusFromInitialize({ + runtimeInfo: { publishHealth: rawPublishHealth }, + }).publishHealth + : brainProbe.runtimePublishHealth; + const relayHealth = syncRouteHealth && isRecord(syncRouteHealth.relay) + ? syncRouteHealth.relay as DoctorInput["relayHealth"] + : null; + let portDiagnoses: DoctorInput["portDiagnoses"] = []; + if (brainProbe.brain.running && syncPort != null && syncPort !== DEFAULT_SYNC_HOST_PORT) { + const { inspectSyncListenerPort } = await import("./services/sync/sharedSyncListener"); + portDiagnoses = [ + inspectSyncListenerPort(DEFAULT_SYNC_HOST_PORT), + inspectSyncListenerPort(DEFAULT_SYNC_HOST_PORT + 1), + inspectSyncListenerPort(DEFAULT_SYNC_HOST_PORT + 2), + ]; + } + const wedge = readBrainLoopWatchdogLastWedge(layout.runtimeDir) + ?? brainProbe.runtimeLastWedge; + const input: DoctorInput = { + nowMs, + app: { + installedVersion: installedApp.version, + latestKnownVersion, + path: installedApp.path, + online, + }, + brain: brainProbe.brain, + wedge, + syncPort, + portDiagnoses, + publishHealth, + relayHealth, + account: brainProbe.account, + }; + const rows = evaluateDoctorRows(input); + return { + ok: !rows.some((row) => row.status === "fail"), + checkedAt: new Date(nowMs).toISOString(), + online, + rows, + app: input.app, + brain: input.brain, + wedge, + syncPort, + portDiagnoses, + publishHealth, + relayHealth, + account: input.account, + }; +} + async function runDesktopCommand(rest: string[]): Promise { const args = [...rest]; const sub = firstPositional(args) ?? "open"; @@ -15102,6 +15455,8 @@ async function runServe( await new Promise((resolve) => setTimeout(resolve, startupBackoffMs)); } let serveStarted = false; + let stopBrainLoopWatchdog: (() => void) | null = null; + let stopBrainFreshnessMonitor: (() => void) | null = null; try { const removeRuntimeProcessErrorBoundary = installRuntimeProcessErrorBoundary("ADE brain"); const [ @@ -15131,6 +15486,40 @@ async function runServe( ]); const layout = resolveMachineAdeLayout(); + const headlessProjectLogger: Logger = createBrainLogger( + path.join(layout.runtimeDir, "brain.jsonl"), + ); + const { + createProductAnalyticsService, + defaultProductAnalyticsStateFile, + getSharedProductAnalyticsService, + } = await import("../../desktop/src/main/services/analytics/productAnalyticsService"); + const brainAnalyticsStateFile = defaultProductAnalyticsStateFile(layout.adeDir); + const brainProductAnalytics = getSharedProductAnalyticsService( + brainAnalyticsStateFile, + () => createProductAnalyticsService({ + stateFilePath: brainAnalyticsStateFile, + logger: headlessProjectLogger, + appVersion: VERSION, + runtimeMode: "brain", + }), + ); + stopBrainLoopWatchdog = startBrainLoopWatchdog({ + runtimeDir: layout.runtimeDir, + warn: (event, meta) => headlessProjectLogger.warn(event, meta), + onRecovered: (breadcrumb) => { + brainProductAnalytics.captureInternal({ + event: "ade_brain_recovered", + surface: "api", + properties: { + blocked_ms: breadcrumb.blockedMs, + last_command: breadcrumb.lastCommand, + }, + dedupeKey: `brain-recovered:${breadcrumb.ts}`, + minimumIntervalMs: 24 * 60 * 60 * 1_000, + }); + }, + }); const rawSocketPath = readValue(args, ["--socket"]) ?? process.env.ADE_RPC_SOCKET_PATH?.trim() ?? @@ -15171,14 +15560,6 @@ async function runServe( overrides, ); let scopeRegistry: InstanceType; - const headlessProjectLogger: Logger = { - debug: () => {}, - info: () => {}, - warn: (event, meta) => - process.stderr.write(`${event} ${JSON.stringify(meta ?? {})}\n`), - error: (event, meta) => - process.stderr.write(`${event} ${JSON.stringify(meta ?? {})}\n`), - }; const createHeadlessProjectScaffoldService = () => { const githubService = createHeadlessGitHubService( process.cwd(), @@ -15458,7 +15839,7 @@ async function runServe( ? createSharedSyncListener({ logger: { warn: (message, fields) => - process.stderr.write(`${message} ${JSON.stringify(fields ?? {})}\n`), + headlessProjectLogger.warn(message, fields), }, }) : null; @@ -15548,6 +15929,18 @@ async function runServe( scopeRegistry, personalChatScope, getAccountDirectoryHealth, + getRuntimeStatus: () => { + const publishHealth = getAccountDirectoryHealth(); + return { + syncPort: sharedSyncListener?.getPort() ?? null, + publishHealth: { + state: publishHealth.state, + failingSinceMs: publishHealth.failingSinceMs, + lastLegDurations: { ...publishHealth.lastLegDurations }, + }, + lastWedge: readBrainLoopWatchdogLastWedge(layout.runtimeDir), + }; + }, disposeScopesOnDispose: false, onShutdown: finish, }); @@ -15729,6 +16122,9 @@ async function runServe( }, getMachineKey: () => machineCloudRelayStore.getMachineIdentity().machineKey, directoryBaseUrl: () => process.env.ADE_ACCOUNT_DIRECTORY_URL?.trim() || undefined, + captureAnalytics: (input) => { + brainProductAnalytics.captureInternal(input); + }, }); accountMachinePublisher.start(); } @@ -15739,6 +16135,49 @@ async function runServe( clearLastFailure({ kind: "machine" }); serveStarted = true; + const serviceCommand = resolveAdeServeCommand(); + const preparedServiceCommand = prepareMachineRuntimeDaemonCommand(serviceCommand); + const isPrimaryBrain = + syncEnabled + && socketPath === layout.socketPath + && process.env.ADE_DISABLE_RUNTIME_SERVICE_INSTALL !== "1"; + if (isPrimaryBrain && preparedServiceCommand.filePath) { + const [{ createBrainFreshnessMonitor }, { requestBrainServiceRestart }] = await Promise.all([ + import("./services/runtime/brainFreshnessMonitor"), + import("./commands/brainUpdate"), + ]); + const freshnessMonitor = createBrainFreshnessMonitor({ + filePath: preparedServiceCommand.filePath, + runningHash: + process.env.ADE_RUNTIME_BUILD_HASH?.trim() + || preparedServiceCommand.buildHash, + isIdle: () => !hasActiveHeadlessConnections(states), + logger: { + warn: (event, fields) => + headlessProjectLogger.warn(event, fields), + }, + restart: () => { + const result = requestBrainServiceRestart({ + command: serviceCommand.command, + commandArgs: preparedServiceCommand.args, + env: { + ...process.env, + ...(serviceCommand.env ?? {}), + }, + }); + if (result.status !== 0) { + headlessProjectLogger.error("brain.freshness_restart_failed", { + status: result.status, + error: result.stderr || result.stdout || "service restart failed", + }); + throw new Error(result.stderr || result.stdout || "service restart failed"); + } + }, + }); + freshnessMonitor.start(); + stopBrainFreshnessMonitor = () => freshnessMonitor.stop(); + } + const stopParentMonitor = monitorRuntimeParentProcess(finish); const stopIdleMonitor = monitorRuntimeIdleExit(states, finish); try { @@ -15815,6 +16254,9 @@ async function runServe( }); } throw error; + } finally { + stopBrainFreshnessMonitor?.(); + stopBrainLoopWatchdog?.(); } } @@ -17987,6 +18429,24 @@ function formatTextOutput( ["socket", isRecord(value) ? value.socketPath : null], ]); case "doctor": { + const doctorRows = isRecord(value) && Array.isArray(value.rows) + ? value.rows.filter(isRecord) + : []; + if (doctorRows.length > 0) { + return renderTable( + ["check", "status", "detail"], + doctorRows.map((row) => [ + asString(row.label) ?? asString(row.key) ?? "Unknown", + row.status === "ok" + ? "green(ok)" + : row.status === "warn" + ? "yellow(warn)" + : "red(fail)", + asString(row.detail) ?? "", + ]), + "No health checks were returned.", + ); + } const project = isRecord(value) && isRecord(value.project) ? value.project : {}; const desktop = @@ -19417,6 +19877,13 @@ async function runCli( throw error; } } + if (plan.kind === "doctor") { + const result = await runDoctorCommand(plan.online, parsed.options); + return { + output: formatOutput(result, parsed.options, "doctor"), + exitCode: result.ok ? 0 : 1, + }; + } if (plan.kind === "runtime") { const result = await runRuntimeCommand(plan.rest, parsed.options); return { diff --git a/apps/ade-cli/src/commands/brainUpdate.ts b/apps/ade-cli/src/commands/brainUpdate.ts index d4109abbd..85308f794 100644 --- a/apps/ade-cli/src/commands/brainUpdate.ts +++ b/apps/ade-cli/src/commands/brainUpdate.ts @@ -372,6 +372,28 @@ function runCommand( }; } +export type BrainServiceRestartResult = ReturnType; + +export function requestBrainServiceRestart( + args: { + command: string; + commandArgs: string[]; + env?: NodeJS.ProcessEnv; + }, + run: typeof runCommand = runCommand, +): BrainServiceRestartResult { + return run( + args.command, + [...args.commandArgs, "--install-service"], + { + env: { + ...(args.env ?? process.env), + ADE_ALLOW_RUNTIME_SERVICE_SELF_MUTATION: "1", + }, + }, + ); +} + function runtimeNodeModulesPath(runtimeRoot: string): string { return path.join(runtimeRoot, "node_modules"); } @@ -599,12 +621,14 @@ async function applyStagedBrainUpdate( const env = { ...runtimeSidecarEnv(process.env, manifest.runtimeTargetDir), ADE_HOME: manifest.adeHome, - ADE_ALLOW_RUNTIME_SERVICE_SELF_MUTATION: "1", }; - const result = (deps.runCommand ?? runCommand)( - manifest.binaryPath, - ["serve", "--install-service"], - { env }, + const result = requestBrainServiceRestart( + { + command: manifest.binaryPath, + commandArgs: ["serve"], + env, + }, + deps.runCommand ?? runCommand, ); if (result.status !== 0) { const message = result.stderr || result.stdout || "ADE brain service restart failed."; diff --git a/apps/ade-cli/src/commands/doctor.test.ts b/apps/ade-cli/src/commands/doctor.test.ts new file mode 100644 index 000000000..18bfe98fd --- /dev/null +++ b/apps/ade-cli/src/commands/doctor.test.ts @@ -0,0 +1,144 @@ +import { describe, expect, it } from "vitest"; +import { + compareDoctorVersions, + evaluateDoctorRows, + type DoctorInput, +} from "./doctor"; +import { createSyncAccountDirectoryHealth } from "../../../desktop/src/shared/types/sync"; + +const NOW = Date.parse("2026-07-23T12:00:00.000Z"); + +function healthyInput(): DoctorInput { + return { + nowMs: NOW, + app: { + installedVersion: "1.2.35", + latestKnownVersion: "1.2.35", + path: "/Applications/ADE.app", + online: false, + }, + brain: { + running: true, + version: "1.2.35", + buildHash: "build", + pid: 123, + uptimeMs: 90_000, + mismatchReason: null, + error: null, + }, + wedge: null, + syncPort: 8787, + portDiagnoses: [], + publishHealth: createSyncAccountDirectoryHealth("published", null, { + lastSuccessAt: NOW - 10_000, + lastLegDurations: { snapshot: 20, token: 40, http: 80 }, + }), + relayHealth: { + enabled: true, + relayControlConnected: true, + relayBridgeValidated: true, + relayEndToEndVerifiedAt: "2026-07-23T11:59:50.000Z", + relayEndToEndFailure: null, + lastFailureAt: null, + skipReason: null, + lastControlError: null, + lastControlOpenAt: "2026-07-23T11:59:00.000Z", + lastBridgeValidationAt: "2026-07-23T11:59:10.000Z", + }, + account: { + signedIn: true, + source: "loopback", + error: null, + }, + }; +} + +describe("doctor row evaluation", () => { + it("marks a healthy machine with no red rows", () => { + const rows = evaluateDoctorRows(healthyInput()); + + expect(rows.map((row) => [row.key, row.status])).toEqual([ + ["app", "ok"], + ["brain", "ok"], + ["wedge", "ok"], + ["sync_port", "ok"], + ["publish", "ok"], + ["relay", "ok"], + ["account", "ok"], + ]); + }); + + it("warns for a recent wedge and alternate port with diagnosed holders", () => { + const input = healthyInput(); + input.wedge = { + lastCommand: "chat.send", + blockedMs: 16_500, + ts: new Date(NOW - 60_000).toISOString(), + }; + input.syncPort = 8789; + input.portDiagnoses = [{ + port: 8787, + holders: [{ pid: 456, command: "old ade serve", startTime: null }], + }]; + + const rows = evaluateDoctorRows(input); + + expect(rows.find((row) => row.key === "wedge")).toMatchObject({ + status: "warn", + detail: expect.stringContaining("chat.send"), + }); + expect(rows.find((row) => row.key === "sync_port")).toMatchObject({ + status: "warn", + detail: expect.stringContaining("8787: pid 456"), + }); + }); + + it("fails a mismatched brain and a publish episode over two minutes", () => { + const input = healthyInput(); + input.brain.mismatchReason = "build hash changed"; + input.publishHealth = createSyncAccountDirectoryHealth("http_timeout", "timed out", { + failingSinceMs: NOW - 5 * 60_000, + lastLegDurations: { snapshot: 12, token: 80, http: 9_200 }, + }); + + const rows = evaluateDoctorRows(input); + + expect(rows.find((row) => row.key === "brain")).toMatchObject({ + status: "fail", + detail: expect.stringContaining("build hash changed"), + }); + expect(rows.find((row) => row.key === "publish")).toEqual(expect.objectContaining({ + status: "fail", + detail: expect.stringMatching(/failing for 5m .* slow leg: http \(9\.2s\)/), + })); + }); + + it("fails only the brain while dependent checks degrade when the socket is dead", () => { + const input = healthyInput(); + input.brain = { + running: false, + version: null, + buildHash: null, + pid: null, + uptimeMs: null, + mismatchReason: null, + error: "Timed out waiting for ade/initialize.", + }; + input.syncPort = null; + input.publishHealth = null; + input.relayHealth = null; + input.account = { signedIn: null, source: null, error: "brain unavailable" }; + + const rows = evaluateDoctorRows(input); + + expect(rows.filter((row) => row.status === "fail").map((row) => row.key)).toEqual(["brain"]); + }); + + it("compares release versions without depending on tag formatting", () => { + expect(compareDoctorVersions("v1.2.36", "1.2.35")).toBe(1); + expect(compareDoctorVersions("1.2.35", "v1.2.35")).toBe(0); + expect(compareDoctorVersions("1.2.34", "1.2.35")).toBe(-1); + expect(compareDoctorVersions("next", "1.2.35")).toBeNull(); + }); +}); + diff --git a/apps/ade-cli/src/commands/doctor.ts b/apps/ade-cli/src/commands/doctor.ts new file mode 100644 index 000000000..ed1b19e9f --- /dev/null +++ b/apps/ade-cli/src/commands/doctor.ts @@ -0,0 +1,335 @@ +import type { + SyncAccountDirectoryHealth, + SyncRouteHealth, +} from "../../../desktop/src/shared/types/sync"; +import type { + BrainLoopWatchdogBreadcrumb, +} from "../services/runtime/brainLoopWatchdog"; +import type { + SyncListenerPortDiagnosis, +} from "../services/sync/sharedSyncListener"; + +export type DoctorRowStatus = "ok" | "warn" | "fail"; + +export type DoctorRow = { + key: + | "app" + | "brain" + | "wedge" + | "sync_port" + | "publish" + | "relay" + | "account"; + label: string; + status: DoctorRowStatus; + detail: string; +}; + +export type DoctorBrainInput = { + running: boolean; + version: string | null; + buildHash: string | null; + pid: number | null; + uptimeMs: number | null; + mismatchReason: string | null; + error: string | null; +}; + +export type DoctorPublishHealth = Pick< + SyncAccountDirectoryHealth, + "state" | "failingSinceMs" | "lastLegDurations" +> & Partial>; + +export type DoctorInput = { + nowMs: number; + app: { + installedVersion: string | null; + latestKnownVersion: string | null; + path: string | null; + online: boolean; + }; + brain: DoctorBrainInput; + wedge: BrainLoopWatchdogBreadcrumb | null; + syncPort: number | null; + portDiagnoses: SyncListenerPortDiagnosis[]; + publishHealth: DoctorPublishHealth | null; + relayHealth: (SyncRouteHealth["relay"] & { + relayEndToEndVerifiedAt?: string | null; + relayEndToEndFailure?: string | null; + }) | null; + account: { + signedIn: boolean | null; + source: string | null; + error: string | null; + }; +}; + +const DAY_MS = 24 * 60 * 60 * 1_000; +const RECENT_PUBLISH_MS = 2 * 60 * 1_000; +const PUBLISH_FAILURE_RED_MS = 2 * 60 * 1_000; + +function normalizedVersionParts(value: string): number[] | null { + const match = /^v?(\d+(?:\.\d+){0,3})/.exec(value.trim()); + return match ? match[1]!.split(".").map((part) => Number.parseInt(part, 10)) : null; +} + +export function compareDoctorVersions(left: string, right: string): number | null { + const leftParts = normalizedVersionParts(left); + const rightParts = normalizedVersionParts(right); + if (!leftParts || !rightParts) return null; + const count = Math.max(leftParts.length, rightParts.length); + for (let index = 0; index < count; index += 1) { + const delta = (leftParts[index] ?? 0) - (rightParts[index] ?? 0); + if (delta !== 0) return delta > 0 ? 1 : -1; + } + return 0; +} + +function compactDuration(ms: number): string { + const seconds = Math.max(0, Math.round(ms / 1_000)); + if (seconds < 60) return `${seconds}s`; + const minutes = Math.floor(seconds / 60); + if (minutes < 60) return `${minutes}m`; + const hours = Math.floor(minutes / 60); + if (hours < 24) return `${hours}h`; + return `${Math.floor(hours / 24)}d`; +} + +function formatUptime(ms: number | null): string { + return ms == null ? "" : ` · uptime ${compactDuration(ms)}`; +} + +function slowestPublishLeg( + health: DoctorPublishHealth, +): { leg: keyof DoctorPublishHealth["lastLegDurations"]; durationMs: number } | null { + let slowest: ReturnType = null; + for (const [leg, rawDuration] of Object.entries(health.lastLegDurations) as Array< + [keyof DoctorPublishHealth["lastLegDurations"], number | null] + >) { + if (rawDuration == null || (slowest && rawDuration <= slowest.durationMs)) continue; + slowest = { leg, durationMs: rawDuration }; + } + return slowest; +} + +function publishLegDetail(health: DoctorPublishHealth): string { + const slowest = slowestPublishLeg(health); + return slowest + ? ` · slow leg: ${slowest.leg} (${(slowest.durationMs / 1_000).toFixed(1)}s)` + : ""; +} + +function appRow(input: DoctorInput["app"]): DoctorRow { + if (!input.installedVersion) { + return { + key: "app", + label: "App", + status: "warn", + detail: "ADE desktop was not found on disk.", + }; + } + if (!input.latestKnownVersion) { + return { + key: "app", + label: "App", + status: "ok", + detail: `installed ${input.installedVersion} · latest not checked${input.online ? " (unavailable)" : " (use --online)"}`, + }; + } + const comparison = compareDoctorVersions(input.installedVersion, input.latestKnownVersion); + return { + key: "app", + label: "App", + status: comparison != null && comparison < 0 ? "warn" : "ok", + detail: `installed ${input.installedVersion} · latest ${input.latestKnownVersion}`, + }; +} + +function brainRow(input: DoctorBrainInput): DoctorRow { + if (!input.running) { + return { + key: "brain", + label: "Brain", + status: "fail", + detail: input.error ? `not responding · ${input.error}` : "not responding", + }; + } + const identity = [ + input.version ? `version ${input.version}` : "version unknown", + input.pid ? `pid ${input.pid}` : "pid unknown", + ].join(" · "); + return { + key: "brain", + label: "Brain", + status: input.mismatchReason ? "fail" : "ok", + detail: input.mismatchReason + ? `${identity} · ${input.mismatchReason}${formatUptime(input.uptimeMs)}` + : `${identity}${formatUptime(input.uptimeMs)}`, + }; +} + +function wedgeRow( + wedge: BrainLoopWatchdogBreadcrumb | null, + nowMs: number, +): DoctorRow { + if (!wedge) { + return { + key: "wedge", + label: "Wedge history", + status: "ok", + detail: "no recovered wedge recorded", + }; + } + const timestampMs = Date.parse(wedge.ts); + const ageMs = Number.isFinite(timestampMs) ? Math.max(0, nowMs - timestampMs) : null; + return { + key: "wedge", + label: "Wedge history", + status: ageMs != null && ageMs <= DAY_MS ? "warn" : "ok", + detail: `${wedge.lastCommand} blocked ${compactDuration(wedge.blockedMs)} · ${ + ageMs == null ? wedge.ts : `${compactDuration(ageMs)} ago` + }`, + }; +} + +function syncPortRow(input: DoctorInput): DoctorRow { + if (input.syncPort == null) { + return { + key: "sync_port", + label: "Sync port", + status: input.brain.running ? "fail" : "warn", + detail: input.brain.running ? "brain did not report a bound sync port" : "unavailable while brain is down", + }; + } + if (input.syncPort === 8787) { + return { + key: "sync_port", + label: "Sync port", + status: "ok", + detail: "bound on 8787", + }; + } + const holders = input.portDiagnoses.flatMap((diagnosis) => + diagnosis.holders.map((holder) => + `${diagnosis.port}: pid ${holder.pid}${holder.command ? ` ${holder.command}` : ""}`, + ), + ); + return { + key: "sync_port", + label: "Sync port", + status: "warn", + detail: `bound on ${input.syncPort} instead of 8787${ + holders.length ? ` · base holders: ${holders.join("; ")}` : " · first three base ports have no visible holders" + }`, + }; +} + +function publishRow( + health: DoctorPublishHealth | null, + nowMs: number, +): DoctorRow { + if (!health) { + return { + key: "publish", + label: "Publish health", + status: "warn", + detail: "account-directory health unavailable", + }; + } + const failingForMs = health.failingSinceMs == null + ? null + : Math.max(0, nowMs - health.failingSinceMs); + if (failingForMs != null && failingForMs >= PUBLISH_FAILURE_RED_MS) { + return { + key: "publish", + label: "Publish health", + status: "fail", + detail: `failing for ${compactDuration(failingForMs)} · ${health.state}${publishLegDetail(health)}`, + }; + } + if (health.state === "published") { + const successAgeMs = health.lastSuccessAt == null + ? null + : Math.max(0, nowMs - health.lastSuccessAt); + if (successAgeMs != null && successAgeMs <= RECENT_PUBLISH_MS) { + return { + key: "publish", + label: "Publish health", + status: "ok", + detail: `published ${compactDuration(successAgeMs)} ago${publishLegDetail(health)}`, + }; + } + } + return { + key: "publish", + label: "Publish health", + status: "warn", + detail: failingForMs == null + ? `${health.state}${health.skipReason ? ` · ${health.skipReason}` : ""}` + : `failing for ${compactDuration(failingForMs)} · ${health.state}${publishLegDetail(health)}`, + }; +} + +function relayRow(relay: DoctorInput["relayHealth"]): DoctorRow { + if (!relay) { + return { + key: "relay", + label: "Relay", + status: "warn", + detail: "route health unavailable", + }; + } + if (relay.enabled !== true) { + return { + key: "relay", + label: "Relay", + status: "warn", + detail: relay.skipReason ?? "disabled", + }; + } + const failure = relay.relayEndToEndFailure + ?? relay.skipReason + ?? relay.lastControlError + ?? null; + const healthy = relay.relayControlConnected === true + && relay.relayBridgeValidated === true + && Boolean(relay.relayEndToEndVerifiedAt) + && !failure; + return { + key: "relay", + label: "Relay", + status: healthy ? "ok" : "fail", + detail: healthy + ? `reachable · verified ${relay.relayEndToEndVerifiedAt}` + : failure ?? "relay route is not fully validated", + }; +} + +function accountRow(account: DoctorInput["account"]): DoctorRow { + if (account.signedIn === true) { + return { + key: "account", + label: "Account", + status: "ok", + detail: `signed in${account.source ? ` · ${account.source}` : ""}`, + }; + } + return { + key: "account", + label: "Account", + status: "warn", + detail: account.error ?? (account.signedIn === false ? "signed out" : "status unavailable"), + }; +} + +export function evaluateDoctorRows(input: DoctorInput): DoctorRow[] { + return [ + appRow(input.app), + brainRow(input.brain), + wedgeRow(input.wedge, input.nowMs), + syncPortRow(input), + publishRow(input.publishHealth, input.nowMs), + relayRow(input.relayHealth), + accountRow(input.account), + ]; +} diff --git a/apps/ade-cli/src/multiProjectRpcServer.test.ts b/apps/ade-cli/src/multiProjectRpcServer.test.ts index 329cd2ed7..4bbf1a32b 100644 --- a/apps/ade-cli/src/multiProjectRpcServer.test.ts +++ b/apps/ade-cli/src/multiProjectRpcServer.test.ts @@ -12,6 +12,7 @@ import * as gitModule from "../../desktop/src/main/services/git/git"; import { ProjectRegistry } from "./services/projects/projectRegistry"; import { ProjectScopeRegistry } from "./services/projects/projectScope"; import type { SyncRoleSnapshot } from "../../desktop/src/shared/types"; +import { RUNTIME_COMPAT_LEVEL } from "../../desktop/src/shared/adeRuntimeProtocol"; function createRegistry() { const root = fs.mkdtempSync(path.join(os.tmpdir(), "ade-multi-project-rpc-")); @@ -590,6 +591,8 @@ describe("multi-project RPC server", () => { expect(init).toMatchObject({ runtimeInfo: { buildHash: expectedHash, + minCompatibleProtocol: RUNTIME_COMPAT_LEVEL, + protocolVersion: RUNTIME_COMPAT_LEVEL, multiProject: true, }, }); @@ -604,6 +607,55 @@ describe("multi-project RPC server", () => { } }); + it("advertises machine recovery health in runtimeInfo", async () => { + const { registry } = createRegistry(); + const getRuntimeStatus = vi.fn(() => ({ + syncPort: 8789, + publishHealth: { + state: "http_timeout" as const, + failingSinceMs: 120_000, + lastLegDurations: { snapshot: 10, token: 20, http: 9_200 }, + }, + lastWedge: { + lastCommand: "chat.send", + blockedMs: 16_500, + ts: "2026-07-23T12:00:00.000Z", + }, + })); + const handler = createMultiProjectRpcRequestHandler({ + serverVersion: "test", + projectRegistry: registry, + getRuntimeStatus, + }); + + const initialized = await handler({ + jsonrpc: "2.0", + id: 1, + method: "ade/initialize", + params: {}, + }); + + expect(initialized).toMatchObject({ + runtimeInfo: { + pid: process.pid, + uptimeMs: expect.any(Number), + syncPort: 8789, + publishHealth: { + state: "http_timeout", + failingSinceMs: 120_000, + lastLegDurations: { snapshot: 10, token: 20, http: 9_200 }, + }, + lastWedge: { + lastCommand: "chat.send", + blockedMs: 16_500, + ts: "2026-07-23T12:00:00.000Z", + }, + }, + }); + expect(getRuntimeStatus).toHaveBeenCalledTimes(1); + handler.dispose(); + }); + it("exposes machine personal chats without a project id", async () => { const { registry } = createRegistry(); const personalChatScope = { @@ -1174,6 +1226,12 @@ describe("multi-project RPC server", () => { lastHttpStatus: 401, lastHttpReason: "invalid audience", reachableEndpointCount: 2, + lastLegDurations: { + snapshot: 3, + token: 12, + http: 250, + }, + failingSinceMs: 123, }; const handler = createMultiProjectRpcRequestHandler({ serverVersion: "test", diff --git a/apps/ade-cli/src/multiProjectRpcServer.ts b/apps/ade-cli/src/multiProjectRpcServer.ts index 119e96c09..8502b3433 100644 --- a/apps/ade-cli/src/multiProjectRpcServer.ts +++ b/apps/ade-cli/src/multiProjectRpcServer.ts @@ -1,5 +1,5 @@ import { createAdeRpcRequestHandler } from "./adeRpcServer"; -import { createHash, randomUUID } from "node:crypto"; +import { randomUUID } from "node:crypto"; import { execFile } from "node:child_process"; import fs from "node:fs"; import os from "node:os"; @@ -18,8 +18,10 @@ import type { CreateProjectInput, ListMyGitHubReposInput, ProjectBrowseInput, + RuntimeActivityCounts, } from "../../desktop/src/shared/types"; import type { BufferedEvent } from "./eventBuffer"; +import { computeRuntimeBuildHash as hashRuntimeBuild } from "./services/runtime/runtimeBuildIdentity"; import { JsonRpcError, JsonRpcErrorCode, @@ -39,7 +41,10 @@ import { type ProjectRegistrationSource, } from "./services/projects/projectRegistry"; import { ProjectScopeRegistry } from "./services/projects/projectScope"; -import { PersonalChatScope } from "./services/personalChats/personalChatScope"; +import { + PersonalChatScope, + summarizeRuntimeActivity, +} from "./services/personalChats/personalChatScope"; import { createHeadlessGitHubService } from "./headlessLinearServices"; import { callerHasRoleAtLeast, @@ -68,6 +73,7 @@ import { reconcileAccountOwnedMachineTrust, } from "./services/account/accountMachineDirectoryService"; import { mapPlatform } from "./services/sync/syncProtocol"; +import { RUNTIME_COMPAT_LEVEL } from "../../desktop/src/shared/adeRuntimeProtocol"; type HandlerEntry = { handler: JsonRpcHandler & { dispose?: () => void }; @@ -87,9 +93,24 @@ export type MultiProjectRpcHandlerOptions = { scopeRegistry?: ProjectScopeRegistry; disposeScopesOnDispose?: boolean; onShutdown?: (() => void) | null; - personalChatScope?: Pick; + personalChatScope?: Pick< + PersonalChatScope, + "capabilities" | "call" | "streamEvents" | "dispose" + > & Partial>; accountAuthService?: AccountAuthService; getAccountDirectoryHealth?: () => SyncAccountDirectoryHealth; + getRuntimeStatus?: () => { + syncPort: number | null; + publishHealth: Pick< + SyncAccountDirectoryHealth, + "state" | "failingSinceMs" | "lastLegDurations" + > | null; + lastWedge: { + lastCommand: string; + blockedMs: number; + ts: string; + } | null; + }; registerAccountConfigRoot?: typeof registerAccountConfigProjectRoot; reconcileAccountOwnership?: typeof reconcileAccountOwnedMachineTrust; }; @@ -101,6 +122,7 @@ const RUNTIME_METHODS = new Set([ "shutdown", "exit", "runtime/info", + "runtime.activitySummary", "account.call", "personalChats.call", "personalChats.streamEvents", @@ -946,9 +968,7 @@ export function createMultiProjectRpcRequestHandler( cachedRuntimeBuildHash = null; return null; } - cachedRuntimeBuildHash = createHash("sha256") - .update(fs.readFileSync(resolved)) - .digest("hex"); + cachedRuntimeBuildHash = hashRuntimeBuild(resolved); return cachedRuntimeBuildHash; } catch { cachedRuntimeBuildHash = null; @@ -967,6 +987,28 @@ export function createMultiProjectRpcRequestHandler( }; }; + const resolveRuntimeStatus = () => { + try { + const status = options.getRuntimeStatus?.(); + return { + syncPort: status?.syncPort ?? null, + publishHealth: status?.publishHealth + ? { + ...status.publishHealth, + lastLegDurations: { ...status.publishHealth.lastLegDurations }, + } + : null, + lastWedge: status?.lastWedge ? { ...status.lastWedge } : null, + }; + } catch { + return { + syncPort: null, + publishHealth: null, + lastWedge: null, + }; + } + }; + const handler = (async (request: JsonRpcRequest): Promise => { const method = typeof request.method === "string" ? request.method : ""; const params = safeParams(request.params); @@ -982,8 +1024,12 @@ export function createMultiProjectRpcRequestHandler( name: "ade-rpc", version: options.serverVersion, ...resolveRuntimeEnvInfo(), + ...resolveRuntimeStatus(), + minCompatibleProtocol: RUNTIME_COMPAT_LEVEL, + protocolVersion: RUNTIME_COMPAT_LEVEL, multiProject: true, pid: process.pid, + uptimeMs: Math.max(0, Math.round(process.uptime() * 1_000)), }, capabilities: { actions: { @@ -1034,8 +1080,12 @@ export function createMultiProjectRpcRequestHandler( name: "ade-rpc", version: options.serverVersion, ...envInfo, + ...resolveRuntimeStatus(), + minCompatibleProtocol: RUNTIME_COMPAT_LEVEL, + protocolVersion: RUNTIME_COMPAT_LEVEL, multiProject: true, pid: process.pid, + uptimeMs: Math.max(0, Math.round(process.uptime() * 1_000)), }, adeDir: layout.adeDir, socketPath: layout.socketPath, @@ -1142,6 +1192,34 @@ export function createMultiProjectRpcRequestHandler( return await personalChatScope.streamEvents(params); } + if (method === "runtime.activitySummary") { + const projectActivity = await Promise.all( + projectRegistry.list().map(async (record) => { + const pendingScope = scopeRegistry.getIfBooted(record.projectId); + if (!pendingScope) return { activeAgentTurns: 0, activeWorkSessions: 0 }; + const scope = await pendingScope.catch(() => null); + if (!scope) return { activeAgentTurns: 0, activeWorkSessions: 0 }; + return summarizeRuntimeActivity(scope.runtime); + }), + ); + const personalActivity = typeof personalChatScope.activitySummary === "function" + ? await personalChatScope.activitySummary() + : { activeAgentTurns: 0, activeWorkSessions: 0 }; + const activeAgentTurns = projectActivity.reduce( + (total, activity) => total + activity.activeAgentTurns, + personalActivity.activeAgentTurns, + ); + const activeWorkSessions = projectActivity.reduce( + (total, activity) => total + activity.activeWorkSessions, + personalActivity.activeWorkSessions, + ); + return { + idle: activeAgentTurns === 0 && activeWorkSessions === 0, + activeAgentTurns, + activeWorkSessions, + }; + } + if (method === "projects.list") { return await decorateProjectListWithIcons(projectRegistry.list()); } diff --git a/apps/ade-cli/src/serviceManager/common.test.ts b/apps/ade-cli/src/serviceManager/common.test.ts index 2e6ee48c4..1255d4b9e 100644 --- a/apps/ade-cli/src/serviceManager/common.test.ts +++ b/apps/ade-cli/src/serviceManager/common.test.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import { createHash } from "node:crypto"; import os from "node:os"; import path from "node:path"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { ADE_RUNTIME_SERVICE_NAME, isCurrentProcessDescendantOfPid, @@ -374,8 +374,14 @@ describe("launchd service install", () => { args: ["serve"], env: { NODE_PATH: "/opt/ade/node_modules" }, }; + const install = ( + deps: NonNullable[0]>, + ) => installLaunchdService({ + responsivenessProbe: () => true, + ...deps, + }); - it("writes the plist and loads the launch agent", () => { + it("writes the plist and loads the launch agent", async () => { const homeDir = makeTempHome("ade-launchd-install-"); const servicePath = launchAgentPath(homeDir); const calls: Array<{ command: string; args: string[] }> = []; @@ -386,7 +392,7 @@ describe("launchd service install", () => { { status: 0, stdout: "", stderr: "" }, ]); - const result = installLaunchdService({ command: serviceCommand, spawnSync, homeDir }); + const result = await install({ command: serviceCommand, spawnSync, homeDir }); expect(result).toMatchObject({ ok: true, @@ -403,7 +409,7 @@ describe("launchd service install", () => { ]); }); - it("leaves an unchanged running launch agent loaded", () => { + it("leaves an unchanged running launch agent loaded", async () => { const homeDir = makeTempHome("ade-launchd-running-"); const servicePath = launchAgentPath(homeDir); fs.mkdirSync(path.dirname(servicePath), { recursive: true }); @@ -413,7 +419,7 @@ describe("launchd service install", () => { { status: 0, stdout: "state = running\n", stderr: "" }, ]); - const result = installLaunchdService({ command: serviceCommand, spawnSync, homeDir }); + const result = await install({ command: serviceCommand, spawnSync, homeDir }); expect(result).toMatchObject({ ok: true, @@ -427,7 +433,70 @@ describe("launchd service install", () => { ]); }); - it("reloads an unchanged running launch agent when a packaged trust reset requests it", () => { + it("reinstalls an unchanged running launch agent when its runtime socket is wedged", async () => { + const homeDir = makeTempHome("ade-launchd-wedged-"); + const servicePath = launchAgentPath(homeDir); + fs.mkdirSync(path.dirname(servicePath), { recursive: true }); + fs.writeFileSync(servicePath, renderLaunchdPlist(serviceCommand, homeDir), "utf8"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = spawnSequence(calls, [ + { status: 0, stdout: "state = running\npid = 1234\n", stderr: "" }, + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: "", stderr: "" }, + ]); + const responsivenessProbe = vi.fn() + .mockReturnValueOnce(false) + .mockReturnValueOnce(true); + + const result = await install({ + command: serviceCommand, + spawnSync, + homeDir, + responsivenessProbe, + currentPid: 9999, + parentPid: () => null, + handoverPidAlive: () => false, + terminateDeps: { kill: () => {}, pidAlive: () => false }, + }); + + expect(result.ok).toBe(true); + expect(responsivenessProbe).toHaveBeenCalledTimes(2); + expect(calls.map((call) => call.args[0])).toEqual([ + "print", + "unload", + "-axo", + "load", + ]); + }); + + it("returns a typed failure when the replacement never becomes responsive", async () => { + const homeDir = makeTempHome("ade-launchd-handover-fail-"); + const calls: Array<{ command: string; args: string[] }> = []; + const spawnSync = spawnSequence(calls, [ + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: "", stderr: "" }, + { status: 0, stdout: "", stderr: "" }, + ]); + + const result = await install({ + command: serviceCommand, + spawnSync, + homeDir, + responsivenessProbe: () => false, + handoverPidAlive: () => false, + handoverTimeoutMs: 0, + }); + + expect(result).toMatchObject({ + ok: false, + action: "install", + failureStep: "replacement_responsive", + }); + }); + + it("reloads an unchanged running launch agent when a packaged trust reset requests it", async () => { const homeDir = makeTempHome("ade-launchd-trust-reset-"); const servicePath = launchAgentPath(homeDir); fs.mkdirSync(path.dirname(servicePath), { recursive: true }); @@ -440,7 +509,7 @@ describe("launchd service install", () => { { status: 0, stdout: "", stderr: "" }, ]); - const result = installLaunchdService({ + const result = await install({ command: serviceCommand, spawnSync, homeDir, @@ -459,7 +528,7 @@ describe("launchd service install", () => { ]); }); - it("reloads an unchanged launch agent when it is loaded but stopped", () => { + it("reloads an unchanged launch agent when it is loaded but stopped", async () => { const homeDir = makeTempHome("ade-launchd-stopped-"); const servicePath = launchAgentPath(homeDir); fs.mkdirSync(path.dirname(servicePath), { recursive: true }); @@ -471,7 +540,7 @@ describe("launchd service install", () => { { status: 0, stdout: "", stderr: "" }, ]); - const result = installLaunchdService({ command: serviceCommand, spawnSync, homeDir }); + const result = await install({ command: serviceCommand, spawnSync, homeDir }); expect(result).toMatchObject({ ok: true, @@ -487,7 +556,7 @@ describe("launchd service install", () => { ]); }); - it("does not load a launch agent when another channel's brain hosts sync", () => { + it("does not load a launch agent when another channel's brain hosts sync", async () => { const homeDir = makeTempHome("ade-launchd-conflict-"); const servicePath = launchAgentPath(homeDir); const lockPath = path.join(homeDir, "sync-host-lock.json"); @@ -506,7 +575,7 @@ describe("launchd service install", () => { const killed: Array<{ pid: number; signal: NodeJS.Signals | number }> = []; const spawnSync = spawnSequence(calls, []); - const result = installLaunchdService({ + const result = await install({ command: serviceCommand, spawnSync, homeDir, @@ -538,7 +607,7 @@ describe("launchd service install", () => { expect(fs.readFileSync(servicePath, "utf8")).toBe(renderLaunchdPlist(serviceCommand, homeDir)); }); - it("reaps a stale same-channel sync brain and installs anyway", () => { + it("reaps a stale same-channel sync brain and installs anyway", async () => { const homeDir = makeTempHome("ade-launchd-reap-"); const adeHome = path.join(homeDir, ".ade"); const servicePath = launchAgentPath(homeDir); @@ -556,7 +625,7 @@ describe("launchd service install", () => { const killed: Array<{ pid: number; signal: NodeJS.Signals | number }> = []; const spawnSync = spawnSequence(calls, []); - const result = installLaunchdService({ + const result = await install({ command: serviceCommand, spawnSync, homeDir, @@ -587,7 +656,7 @@ describe("launchd service install", () => { ]); }); - it("terminates the previous service child and stale serve processes on restart", () => { + it("terminates the previous service child and stale serve processes on restart", async () => { const homeDir = makeTempHome("ade-launchd-sweep-"); const adeHome = path.join(homeDir, ".ade"); const staleServeLine = ` 4242 ${serviceCommand.command} serve --socket ${path.join(adeHome, "sock", "ade.sock")}`; @@ -600,7 +669,7 @@ describe("launchd service install", () => { { status: 0, stdout: "", stderr: "" }, ]); - const result = installLaunchdService({ + const result = await install({ command: serviceCommand, spawnSync, homeDir, @@ -619,7 +688,7 @@ describe("launchd service install", () => { ]); }); - it("refuses to restart a launch agent from a descendant of the loaded service", () => { + it("refuses to restart a launch agent from a descendant of the loaded service", async () => { const homeDir = makeTempHome("ade-launchd-self-install-"); const servicePath = launchAgentPath(homeDir); const calls: Array<{ command: string; args: string[] }> = []; @@ -632,12 +701,16 @@ describe("launchd service install", () => { 100: 1, })[pid] ?? null; - const result = installLaunchdService({ + const result = await install({ command: serviceCommand, spawnSync, homeDir, currentPid: 400, parentPid, + terminateDeps: { + kill: () => {}, + pidAlive: () => false, + }, }); expect(result).toMatchObject({ @@ -653,7 +726,7 @@ describe("launchd service install", () => { expect(fs.existsSync(servicePath)).toBe(false); }); - it("allows launch agent restart from a descendant when self-mutation is explicitly enabled", () => { + it("allows launch agent restart from a descendant when self-mutation is explicitly enabled", async () => { const homeDir = makeTempHome("ade-launchd-self-install-override-"); const servicePath = launchAgentPath(homeDir); const calls: Array<{ command: string; args: string[] }> = []; @@ -670,12 +743,16 @@ describe("launchd service install", () => { })[pid] ?? null; process.env.ADE_ALLOW_RUNTIME_SERVICE_SELF_MUTATION = "1"; - const result = installLaunchdService({ + const result = await install({ command: serviceCommand, spawnSync, homeDir, currentPid: 400, parentPid, + terminateDeps: { + kill: () => {}, + pidAlive: () => false, + }, }); expect(result).toMatchObject({ @@ -765,7 +842,7 @@ describe("launchd service install", () => { expect(fs.existsSync(servicePath)).toBe(false); }); - it("surfaces launchctl load failures", () => { + it("surfaces launchctl load failures", async () => { const homeDir = makeTempHome("ade-launchd-fail-"); const calls: Array<{ command: string; args: string[] }> = []; const spawnSync = spawnSequence(calls, [ @@ -775,7 +852,7 @@ describe("launchd service install", () => { { status: 5, stdout: "", stderr: "Load failed" }, ]); - const result = installLaunchdService({ command: serviceCommand, spawnSync, homeDir }); + const result = await install({ command: serviceCommand, spawnSync, homeDir }); expect(result.ok).toBe(false); expect(result.message).toBe("Load failed"); @@ -1044,8 +1121,19 @@ function spawnSequence( calls: Array<{ command: string; args: string[] }>, results: ServiceManagerProcessResult[], ): ServiceManagerSpawnSync { + let loadSeen = false; return (command, args) => { + const next = results.shift(); + if ( + !next + && loadSeen + && command === "launchctl" + && args[0] === "print" + ) { + return { status: 0, stdout: "state = running\npid = 7777\n", stderr: "" }; + } calls.push({ command, args }); - return results.shift() ?? { status: 0, stdout: "", stderr: "" }; + if (command === "launchctl" && args[0] === "load") loadSeen = true; + return next ?? { status: 0, stdout: "", stderr: "" }; }; } diff --git a/apps/ade-cli/src/serviceManager/common.ts b/apps/ade-cli/src/serviceManager/common.ts index 048a4a2e3..23c231b05 100644 --- a/apps/ade-cli/src/serviceManager/common.ts +++ b/apps/ade-cli/src/serviceManager/common.ts @@ -13,6 +13,8 @@ export type ServiceManagerResult = { // very brain it tried to mutate. Consumers must branch on this typed flag, // never on the human-readable `message` text. selfMutationBlocked?: boolean; + /** Typed install verification stage for callers that need repair diagnostics. */ + failureStep?: "predecessor_exit" | "replacement_pid" | "replacement_responsive"; }; export type ServiceManagerStatusResult = { @@ -244,6 +246,33 @@ export function terminatePidGracefully(pid: number | null, deps: TerminatePidDep } } +export async function terminatePidGracefullyAsync( + pid: number | null, + deps: TerminatePidDeps = {}, +): Promise { + if (!pid || pid <= 0 || pid === process.pid) return; + const kill = deps.kill ?? defaultKill; + const pidAlive = deps.pidAlive ?? defaultPidAlive; + try { + kill(pid, "SIGTERM"); + } catch { + return; + } + const deadline = Date.now() + (deps.graceTimeoutMs ?? 1_500); + while (Date.now() < deadline) { + if (!pidAlive(pid)) return; + await new Promise((resolve) => { + const timer = setTimeout(resolve, 50); + timer.unref?.(); + }); + } + try { + kill(pid, "SIGKILL"); + } catch { + // best effort + } +} + export function resolveAdeServeCliScriptPath(command: AdeServiceCommand): string { const first = command.args[0]?.trim(); if (first && first !== "serve") return first; diff --git a/apps/ade-cli/src/serviceManager/index.ts b/apps/ade-cli/src/serviceManager/index.ts index 12397a19b..5642b3de5 100644 --- a/apps/ade-cli/src/serviceManager/index.ts +++ b/apps/ade-cli/src/serviceManager/index.ts @@ -30,7 +30,7 @@ export function getRuntimeServiceMainPid(): number | null { } } -export function installRuntimeService(): ServiceManagerResult { +export async function installRuntimeService(): Promise { switch (process.platform) { case "darwin": return installLaunchdService(); diff --git a/apps/ade-cli/src/serviceManager/installLaunchd.ts b/apps/ade-cli/src/serviceManager/installLaunchd.ts index fda70f2ba..7da721e50 100644 --- a/apps/ade-cli/src/serviceManager/installLaunchd.ts +++ b/apps/ade-cli/src/serviceManager/installLaunchd.ts @@ -14,6 +14,7 @@ import { type ServiceManagerSpawnSync, type ServiceManagerStatusResult, terminatePidGracefully, + terminatePidGracefullyAsync, type TerminatePidDeps, } from "./common"; import { @@ -32,6 +33,17 @@ type LaunchdServiceManagerDeps = { env?: NodeJS.ProcessEnv; currentPid?: number; parentPid?: (pid: number) => number | null; + /** Defaults on. Repair-only callers may skip the preflight RPC probe. */ + probeResponsiveness?: boolean; + responsivenessProbe?: (args: { + socketPath: string; + timeoutMs: number; + command: AdeServiceCommand; + }) => boolean; + handoverTimeoutMs?: number; + handoverPollMs?: number; + handoverPidAlive?: (pid: number) => boolean; + sleep?: (ms: number) => Promise; }; type LaunchdServiceUninstallDeps = { @@ -189,19 +201,89 @@ function getLoadedLaunchdState( }; } +function pidAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} + +async function sleepAsync(ms: number): Promise { + await new Promise((resolve) => { + const timer = setTimeout(resolve, ms); + timer.unref?.(); + }); +} + +function runtimeStatusArgs(command: AdeServiceCommand, socketPath: string): string[] { + const args = [...command.args]; + const serveIndex = args.lastIndexOf("serve"); + if (serveIndex >= 0) { + args.splice(serveIndex, 1, "runtime", "status"); + } else { + args.push("runtime", "status"); + } + args.push("--socket", socketPath, "--timeout", "1500", "--text"); + return args; +} + +function defaultResponsivenessProbe(args: { + socketPath: string; + timeoutMs: number; + command: AdeServiceCommand; +}): boolean { + const env = { + ...process.env, + ...(args.command.env ?? {}), + ADE_DISABLE_RUNTIME_SERVICE_INSTALL: "1", + }; + const result = spawnSync( + args.command.command, + runtimeStatusArgs(args.command, args.socketPath), + { + encoding: "utf8", + env, + timeout: args.timeoutMs, + stdio: "ignore", + }, + ); + return result.status === 0 && !result.error; +} + +function handoverFailure( + servicePath: string, + failureStep: NonNullable, + message: string, +): ServiceManagerResult { + return { + ok: false, + serviceName: ADE_RUNTIME_SERVICE_NAME, + action: "install", + path: servicePath, + failureStep, + message, + }; +} + export function getLaunchdServiceMainPid( run: ServiceManagerSpawnSync = spawnSync, ): number | null { return getLoadedLaunchdState(run)?.pid ?? null; } -export function installLaunchdService(deps: LaunchdServiceManagerDeps = {}): ServiceManagerResult { +export async function installLaunchdService( + deps: LaunchdServiceManagerDeps = {}, +): Promise { const run = deps.spawnSync ?? spawnSync; const env = deps.env ?? process.env; const homeDir = deps.homeDir ?? os.homedir(); const servicePath = launchAgentPath(homeDir); const command = deps.command ?? resolveAdeServeCommand(); const adeHome = command.env?.ADE_HOME?.trim() || env.ADE_HOME?.trim() || path.join(homeDir, ".ade"); + const socketPath = path.join(adeHome, "sock", "ade.sock"); + const probeResponsiveness = deps.responsivenessProbe ?? defaultResponsivenessProbe; fs.mkdirSync(path.dirname(servicePath), { recursive: true }); const plist = renderLaunchdPlist(command, homeDir); fs.mkdirSync(path.join(adeHome, "runtime"), { recursive: true }); @@ -212,13 +294,18 @@ export function installLaunchdService(deps: LaunchdServiceManagerDeps = {}): Ser const forceRestart = env.ADE_FORCE_RUNTIME_SERVICE_RESTART === "1"; const loaded = getLoadedLaunchdState(run); if (!forceRestart && plistUnchanged && loaded?.running === true) { - return { - ok: true, - serviceName: ADE_RUNTIME_SERVICE_NAME, - action: "install", - path: servicePath, - message: "ADE service launchd service is already installed and running.", - }; + if ( + deps.probeResponsiveness === false + || probeResponsiveness({ socketPath, timeoutMs: 1_500, command }) + ) { + return { + ok: true, + serviceName: ADE_RUNTIME_SERVICE_NAME, + action: "install", + path: servicePath, + message: "ADE service launchd service is already installed and running.", + }; + } } const selfBlock = selfServiceMutationBlock({ action: "install", @@ -249,7 +336,7 @@ export function installLaunchdService(deps: LaunchdServiceManagerDeps = {}): Ser fs.writeFileSync(servicePath, plist, "utf8"); if (loaded?.running === true) { run("launchctl", ["unload", servicePath], { stdio: "ignore" }); - terminatePidGracefully(loaded.pid, deps.terminateDeps); + await terminatePidGracefullyAsync(loaded.pid, deps.terminateDeps); } } return { @@ -260,7 +347,7 @@ export function installLaunchdService(deps: LaunchdServiceManagerDeps = {}): Ser message: `${formatSyncHostSingletonConflictMessage(conflict)} The ${ADE_RUNTIME_SERVICE_NAME} launch agent was updated for this ADE channel and will start when mobile sync is available.`, }; } - terminatePidGracefully(conflict.owner.pid, deps.terminateDeps); + await terminatePidGracefullyAsync(conflict.owner.pid, deps.terminateDeps); } if (!plistUnchanged) { @@ -271,14 +358,14 @@ export function installLaunchdService(deps: LaunchdServiceManagerDeps = {}): Ser // orphaned brain keeps the channel socket and sync lock hostage. Reap the // previous service child plus any stale same-channel serve processes // before loading the replacement. - terminatePidGracefully(loaded?.pid ?? null, deps.terminateDeps); + await terminatePidGracefullyAsync(loaded?.pid ?? null, deps.terminateDeps); const stalePids = listStaleChannelServePids(run, { cliScriptPath: resolveAdeServeCliScriptPath(command), - primarySocketPath: path.join(adeHome, "sock", "ade.sock"), + primarySocketPath: socketPath, excludePids: loaded?.pid ? [loaded.pid] : [], }); for (const pid of stalePids) { - terminatePidGracefully(pid, deps.terminateDeps); + await terminatePidGracefullyAsync(pid, deps.terminateDeps); } const load = run("launchctl", ["load", servicePath], { encoding: "utf8" }); if (load.status !== 0) { @@ -290,6 +377,53 @@ export function installLaunchdService(deps: LaunchdServiceManagerDeps = {}): Ser message: serviceManagerResultText(load) || "launchctl load failed.", }; } + const oldPid = loaded?.pid ?? null; + const isAlive = deps.handoverPidAlive ?? deps.terminateDeps?.pidAlive ?? pidAlive; + const sleep = deps.sleep ?? sleepAsync; + const timeoutMs = Math.max(0, deps.handoverTimeoutMs ?? 10_000); + const pollMs = Math.max(10, deps.handoverPollMs ?? 100); + const deadline = Date.now() + timeoutMs; + let predecessorGone = oldPid == null || !isAlive(oldPid); + let replacementPid: number | null = null; + let replacementResponsive = false; + do { + predecessorGone = oldPid == null || !isAlive(oldPid); + const replacement = getLoadedLaunchdState(run); + replacementPid = replacement?.running === true ? replacement.pid : null; + const replacementDiffers = replacementPid != null && replacementPid !== oldPid; + if (predecessorGone && replacementDiffers) { + replacementResponsive = probeResponsiveness({ + socketPath, + timeoutMs: Math.min(1_500, Math.max(1, deadline - Date.now())), + command, + }); + if (replacementResponsive) break; + } + if (Date.now() >= deadline) break; + await sleep(Math.min(pollMs, Math.max(1, deadline - Date.now()))); + } while (Date.now() <= deadline); + + if (!predecessorGone) { + return handoverFailure( + servicePath, + "predecessor_exit", + `ADE service handover failed because predecessor pid ${oldPid} is still alive.`, + ); + } + if (replacementPid == null || replacementPid === oldPid) { + return handoverFailure( + servicePath, + "replacement_pid", + `ADE service handover failed because launchd did not report a distinct replacement pid (old ${oldPid ?? "none"}, new ${replacementPid ?? "none"}).`, + ); + } + if (!replacementResponsive) { + return handoverFailure( + servicePath, + "replacement_responsive", + `ADE service handover failed because replacement pid ${replacementPid} did not initialize over ${socketPath} within ${timeoutMs}ms.`, + ); + } return { ok: true, serviceName: ADE_RUNTIME_SERVICE_NAME, diff --git a/apps/ade-cli/src/services/account/accountAuthService.test.ts b/apps/ade-cli/src/services/account/accountAuthService.test.ts index f7f533cc7..f274549b9 100644 --- a/apps/ade-cli/src/services/account/accountAuthService.test.ts +++ b/apps/ade-cli/src/services/account/accountAuthService.test.ts @@ -1822,6 +1822,40 @@ describe("AccountAuthService refresh and sign-out", () => { }); }); + it("aborts a refresh while waiting for cross-process token rotation", async () => { + vi.useFakeTimers(); + const nowMs = Date.parse("2026-07-14T12:00:00.000Z"); + const store = new MemoryCredentialStore(); + store.setSync(ACCOUNT_SESSION_CREDENTIAL_KEY, JSON.stringify(storedSession({ + accessToken: jwt({ sub: "user_old", exp: Math.floor((nowMs - 60_000) / 1000) }), + }))); + const fetchImpl = vi.fn(async () => jsonResponse({ + error: "invalid_grant", + error_description: "refresh token was already consumed", + }, 400)); + const service = createAccountAuthService({ + credentialStore: store, + getOAuthConfig: () => ({ issuer: "https://clerk.example.test", clientId: "client-public" }), + fetchImpl, + refreshRotationWaitMs: 6_000, + refreshRotationPollMs: 1_000, + now: () => nowMs, + }); + activeServices.push(service); + const controller = new AbortController(); + + const token = service.getAccessToken({ signal: controller.signal }); + await vi.advanceTimersByTimeAsync(0); + expect(fetchImpl).toHaveBeenCalledOnce(); + controller.abort(new DOMException("publisher stopped", "AbortError")); + + await expect(token).rejects.toMatchObject({ + name: "AbortError", + message: "publisher stopped", + }); + expect(store.getSync(ACCOUNT_SESSION_CREDENTIAL_KEY)).not.toBeNull(); + }); + it("keeps the shared grant while a default-window peer refresh is still being persisted", async () => { vi.useFakeTimers(); const nowMs = Date.parse("2026-07-14T12:00:00.000Z"); diff --git a/apps/ade-cli/src/services/account/accountAuthService.ts b/apps/ade-cli/src/services/account/accountAuthService.ts index 7d1e5ad95..c20fc11e0 100644 --- a/apps/ade-cli/src/services/account/accountAuthService.ts +++ b/apps/ade-cli/src/services/account/accountAuthService.ts @@ -183,7 +183,7 @@ export type AccountAuthService = { getStatus(): AccountAuthStatus; /** Last persisted-session read result, refreshed by getStatus/getAccessToken. */ getSessionReadState(): AccountSessionReadState; - getAccessToken(options?: { forceRefresh?: boolean }): Promise; + getAccessToken(options?: AccountAccessTokenOptions): Promise; createToken(): Promise; cancelLogin(sessionId: string): void; signOut(): AccountAuthStatus; @@ -192,6 +192,12 @@ export type AccountAuthService = { dispose(): void; }; +export type AccountAccessTokenOptions = { + forceRefresh?: boolean; + signal?: AbortSignal; + timeoutMs?: number; +}; + export type AccountActionDomainService = { startLogin(): Promise; pollLogin(args: { sessionId?: string }): Promise; @@ -206,7 +212,7 @@ export type AccountActionDomainService = { export async function getSignedInAccountAccessToken( service: Pick, - options?: { forceRefresh?: boolean }, + options?: AccountAccessTokenOptions, ): Promise { const status = service.getStatus(); if (!status.signedIn && status.source !== "env-token") return null; @@ -585,6 +591,7 @@ async function postTokenForm(args: { fetchImpl: (input: string, init?: RequestInit) => Promise; tokenUrl: string; body: Record; + signal?: AbortSignal; }): Promise { const response = await args.fetchImpl(args.tokenUrl, { method: "POST", @@ -593,6 +600,7 @@ async function postTokenForm(args: { "content-type": "application/x-www-form-urlencoded", }, body: new URLSearchParams(args.body).toString(), + signal: args.signal, }); const payload = asRecord(await response.json().catch(() => ({}))); if (!response.ok) { @@ -945,6 +953,7 @@ export function createAccountAuthService(args: { const waitForRefreshRotation = async ( rejected: { raw: string; session: AccountSessionRecord }, + signal?: AbortSignal, ): Promise< | { kind: "rotated"; snapshot: { raw: string; session: AccountSessionRecord } } | { kind: "superseded" } @@ -952,6 +961,7 @@ export function createAccountAuthService(args: { > => { let waitedMs = 0; while (true) { + signal?.throwIfAborted(); const latest = readSessionSnapshot(); if (latest.raw !== rejected.raw) { if ( @@ -969,7 +979,21 @@ export function createAccountAuthService(args: { } if (waitedMs >= refreshRotationWaitMs) return { kind: "unchanged" }; const delayMs = Math.min(refreshRotationPollMs, refreshRotationWaitMs - waitedMs); - await new Promise((resolve) => setTimeout(resolve, delayMs)); + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal?.removeEventListener("abort", onAbort); + resolve(); + }, delayMs); + const onAbort = () => { + clearTimeout(timer); + reject( + signal?.reason instanceof Error + ? signal.reason + : new DOMException("The account token request was aborted.", "AbortError"), + ); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + }); waitedMs += delayMs; } }; @@ -1037,6 +1061,7 @@ export function createAccountAuthService(args: { const fetchUserinfoProfile = async ( token: TokenResponse, oauthConfig: AccountOAuthConfig | null, + signal?: AbortSignal, ): Promise | null> => { if (!oauthConfig) return null; if (shouldRejectDevelopmentAccountMaterial({ @@ -1048,7 +1073,12 @@ export function createAccountAuthService(args: { return null; } const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), userinfoRequestTimeoutMs); + const abortFromCaller = () => controller.abort(signal?.reason); + signal?.throwIfAborted(); + signal?.addEventListener("abort", abortFromCaller, { once: true }); + const timer = setTimeout(() => { + controller.abort(new DOMException("The account userinfo request timed out.", "TimeoutError")); + }, userinfoRequestTimeoutMs); timer.unref?.(); try { const response = await fetchImpl(`${oauthConfig.issuer}/oauth/userinfo`, { @@ -1081,10 +1111,12 @@ export function createAccountAuthService(args: { profile.picture ?? profile.image_url ?? profile.avatar_url, ), }; - } catch { + } catch (error) { + if (signal?.aborted) throw error; return null; } finally { clearTimeout(timer); + signal?.removeEventListener("abort", abortFromCaller); } }; @@ -1093,7 +1125,11 @@ export function createAccountAuthService(args: { previous?: AccountSessionRecord | null, authSource: AccountSessionRecord["authSource"] = previous?.authSource ?? "loopback", oauthConfig: AccountOAuthConfig | null = previous?.oauthConfig ?? null, - options: { fetchUserinfo?: boolean; obtainedAtMs?: number } = {}, + options: { + fetchUserinfo?: boolean; + obtainedAtMs?: number; + signal?: AbortSignal; + } = {}, ): Promise => { if (shouldRejectDevelopmentAccountMaterial({ env, @@ -1117,7 +1153,7 @@ export function createAccountAuthService(args: { } const userinfo = options.fetchUserinfo === false ? null - : await fetchUserinfoProfile(token, oauthConfig); + : await fetchUserinfoProfile(token, oauthConfig, options.signal); if ( userinfo?.userId && ( @@ -1665,9 +1701,11 @@ export function createAccountAuthService(args: { return envCredential ? envCredentialStatus(envCredential.inspected) : toStatus(record); }; - const getAccessToken = async ( - options: { forceRefresh?: boolean } = {}, + const getAccessTokenWithSignal = async ( + options: Pick, + signal?: AbortSignal, ): Promise => { + signal?.throwIfAborted(); const sessionSnapshot = readSessionSnapshot(); const record = sessionSnapshot.session; const acceptedEnvCredential = readAcceptedEnvCredential(); @@ -1729,6 +1767,7 @@ export function createAccountAuthService(args: { token = await postTokenForm({ fetchImpl, tokenUrl: `${config.issuer}/oauth/token`, + signal, body: { grant_type: "refresh_token", refresh_token: refreshTokenAtRefresh, @@ -1736,6 +1775,7 @@ export function createAccountAuthService(args: { }, }); } catch { + signal?.throwIfAborted(); throw new Error( "ADE_ACCOUNT_TOKEN refresh failed. Replace it with a newly provisioned token from `ade account token create`.", ); @@ -1749,10 +1789,16 @@ export function createAccountAuthService(args: { // The credential changed while this process was refreshing. The // replacement is already newer than the request we started, so // consume it normally instead of force-rotating it again. - return getAccessToken(); + return getAccessTokenWithSignal({}, signal); } envRefreshToken = token.refreshToken ?? refreshTokenAtRefresh; - const refreshed = await buildSessionRecord(token, envSession, "loopback", config); + const refreshed = await buildSessionRecord( + token, + envSession, + "loopback", + config, + { signal }, + ); envSession = refreshed; return refreshed.accessToken; })(); @@ -1801,6 +1847,7 @@ export function createAccountAuthService(args: { token = await postTokenForm({ fetchImpl, tokenUrl: `${config.issuer}/oauth/token`, + signal, body: { grant_type: "refresh_token", refresh_token: refreshRecord.refreshToken!, @@ -1819,7 +1866,7 @@ export function createAccountAuthService(args: { // rotating refresh exchange may not have persisted its replacement // by the time Clerk rejects our old token, so briefly poll before // declaring the grant dead. - const rotation = await waitForRefreshRotation(refreshSnapshot); + const rotation = await waitForRefreshRotation(refreshSnapshot, signal); if (rotation.kind === "rotated" && attempt === 0) { refreshSnapshot = rotation.snapshot; continue; @@ -1842,7 +1889,7 @@ export function createAccountAuthService(args: { refreshSnapshot.session, undefined, config, - { fetchUserinfo: false, obtainedAtMs }, + { fetchUserinfo: false, obtainedAtMs, signal }, ); if (authEpoch !== epochAtJoin) return null; if (!persistRefreshedSessionIfCurrent(refreshed, refreshSnapshot.raw)) { @@ -1860,9 +1907,10 @@ export function createAccountAuthService(args: { refreshSnapshot.session, undefined, config, - { obtainedAtMs }, + { obtainedAtMs, signal }, ); - } catch { + } catch (error) { + if (signal?.aborted) throw error; // The rotated credential and verified prior subject are already // durable. Optional profile enrichment must not make that successful // refresh unusable. @@ -1884,11 +1932,56 @@ export function createAccountAuthService(args: { // satisfies this forced refresh; forcing another exchange here would // immediately consume the newly rotated refresh token and recreate the // cross-process race this path is designed to avoid. - return getAccessToken(); + return getAccessTokenWithSignal({}, signal); } return refreshed.accessToken; }; + const getAccessToken = async ( + options: AccountAccessTokenOptions = {}, + ): Promise => { + const controller = new AbortController(); + const abortFromCaller = () => controller.abort(options.signal?.reason); + options.signal?.throwIfAborted(); + options.signal?.addEventListener("abort", abortFromCaller, { once: true }); + const requestedTimeoutMs = options.timeoutMs; + const timeoutMs = typeof requestedTimeoutMs === "number" + && Number.isFinite(requestedTimeoutMs) + && requestedTimeoutMs > 0 + ? Math.trunc(requestedTimeoutMs) + : null; + const timeout = timeoutMs == null + ? null + : setTimeout(() => { + controller.abort(new DOMException("The account token request timed out.", "TimeoutError")); + }, timeoutMs); + timeout?.unref?.(); + let rejectOnAbort: (() => void) | null = null; + const aborted = new Promise((_resolve, reject) => { + rejectOnAbort = () => reject( + controller.signal.reason instanceof Error + ? controller.signal.reason + : new DOMException("The account token request was aborted.", "AbortError"), + ); + controller.signal.addEventListener("abort", rejectOnAbort, { once: true }); + }); + try { + return await Promise.race([ + getAccessTokenWithSignal( + { ...(options.forceRefresh ? { forceRefresh: true } : {}) }, + controller.signal, + ), + aborted, + ]); + } finally { + if (timeout) clearTimeout(timeout); + if (rejectOnAbort) { + controller.signal.removeEventListener("abort", rejectOnAbort); + } + options.signal?.removeEventListener("abort", abortFromCaller); + } + }; + const createToken = async (): Promise => { const record = readSession(); if (readAcceptedEnvCredential() && !record?.suppressEnvCredential) { diff --git a/apps/ade-cli/src/services/account/accountMachinePublisherService.test.ts b/apps/ade-cli/src/services/account/accountMachinePublisherService.test.ts index c531a040e..861df172d 100644 --- a/apps/ade-cli/src/services/account/accountMachinePublisherService.test.ts +++ b/apps/ade-cli/src/services/account/accountMachinePublisherService.test.ts @@ -163,6 +163,53 @@ describe("account machine publisher health", () => { lastHttpStatus: 204, lastHttpReason: null, reachableEndpointCount: 1, + lastLegDurations: { + snapshot: 0, + token: 0, + http: 25, + }, + failingSinceMs: null, + }); + }); + + it("samples successful leg durations and escalates slow legs to warn", async () => { + let clock = 0; + let tokenDelayMs = 2; + const debug = vi.fn(); + const info = vi.fn(); + const warn = vi.fn(); + const service = createAccountMachinePublisherService({ + getAccessToken: async () => { + clock += tokenDelayMs; + return "account-token"; + }, + getAccountStatus: () => ({ signedIn: true, sessionReadState: "available" as const }), + getSnapshot: async () => { + clock += 1; + return snapshot(); + }, + getMachineKey: () => "machine-studio", + directoryBaseUrl: () => "https://directory.example", + fetchImpl: vi.fn(async () => { + clock += 3; + return new Response(null, { status: 204 }); + }), + now: () => clock, + logger: { debug, info, warn }, + }); + + for (let attempt = 0; attempt < 10; attempt += 1) { + await service.publishNow(); + } + + expect(debug).toHaveBeenCalledTimes(9); + expect(info).toHaveBeenCalledWith("account.machine_publish_ok", { + legDurationsMs: { snapshot: 1, token: 2, http: 3 }, + }); + tokenDelayMs = 2_001; + await service.publishNow(); + expect(warn).toHaveBeenCalledWith("account.machine_publish_ok", { + legDurationsMs: { snapshot: 1, token: 2_001, http: 3 }, }); }); @@ -264,7 +311,7 @@ describe("account machine publisher health", () => { }); }); - it("distinguishes HTTP, timeout, and transport failures", async () => { + it("distinguishes HTTP timeouts from transport failures", async () => { const baseOptions = { getAccessToken: async () => "account-token", getAccountStatus: () => ({ signedIn: true, sessionReadState: "available" as const }), @@ -299,20 +346,124 @@ describe("account machine publisher health", () => { }); vi.useFakeTimers(); + const warn = vi.fn(); const timeout = createAccountMachinePublisherService({ ...baseOptions, requestTimeoutMs: 250, fetchImpl: vi.fn((_input, init) => new Promise((_resolve, reject) => { - init?.signal?.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError"))); + init?.signal?.addEventListener("abort", () => reject(new DOMException("aborted", "AbortError"))); })), + logger: { warn }, }); const publishing = timeout.publishNow(); await vi.advanceTimersByTimeAsync(250); await publishing; expect(timeout.getPublisherHealth()).toMatchObject({ - state: "timeout", + state: "http_timeout", + lastHttpStatus: null, + failingSinceMs: expect.any(Number), + lastLegDurations: { + snapshot: expect.any(Number), + token: expect.any(Number), + http: 250, + }, + }); + expect(warn).toHaveBeenCalledWith("account.machine_publish_failed", expect.objectContaining({ + leg: "http", + code: "http_timeout", + legDurationsMs: expect.objectContaining({ http: 250 }), + })); + }); + + it("captures one publish-failing event per failure episode after two minutes", async () => { + let clock = 0; + let succeeds = false; + const captureAnalytics = vi.fn(); + const service = createAccountMachinePublisherService({ + getAccessToken: async () => "account-token", + getAccountStatus: () => ({ signedIn: true, sessionReadState: "available" as const }), + getSnapshot: async () => snapshot(), + getMachineKey: () => "machine-studio", + directoryBaseUrl: () => "https://directory.example", + fetchImpl: vi.fn(async () => succeeds + ? new Response(null, { status: 204 }) + : new Response(null, { status: 503 })), + now: () => clock, + captureAnalytics, + }); + + await service.publishNow(); + clock = 121_000; + await service.publishNow(); + clock = 180_000; + await service.publishNow(); + + expect(captureAnalytics).toHaveBeenCalledTimes(1); + expect(captureAnalytics).toHaveBeenCalledWith({ + event: "ade_publish_failing", + surface: "api", + properties: { + failing_minutes: 2, + leg: "http", + code: "http_error", + }, + dedupeKey: "publish-failing:0", + minimumIntervalMs: 24 * 60 * 60 * 1_000, + }); + + succeeds = true; + clock = 200_000; + await service.publishNow(); + succeeds = false; + clock = 300_000; + await service.publishNow(); + clock = 421_000; + await service.publishNow(); + + expect(captureAnalytics).toHaveBeenCalledTimes(2); + }); + + it("bounds a stalled token leg and reports token_timeout without starting HTTP", async () => { + vi.useFakeTimers(); + const warn = vi.fn(); + const fetchImpl = vi.fn(); + const getAccessToken = vi.fn((options?: { signal?: AbortSignal }) => + new Promise((_resolve, reject) => { + options?.signal?.addEventListener("abort", () => { + reject(options.signal?.reason); + }); + })); + const service = createAccountMachinePublisherService({ + getAccessToken, + getAccountStatus: () => ({ signedIn: true, sessionReadState: "available" as const }), + getSnapshot: async () => snapshot(), + getMachineKey: () => "machine-studio", + directoryBaseUrl: () => "https://directory.example", + fetchImpl, + tokenTimeoutMs: 250, + logger: { warn }, + }); + + const publishing = service.publishNow(); + await vi.advanceTimersByTimeAsync(250); + await publishing; + + expect(fetchImpl).not.toHaveBeenCalled(); + expect(service.getPublisherHealth()).toMatchObject({ + state: "token_timeout", lastHttpStatus: null, + failingSinceMs: expect.any(Number), + lastLegDurations: { + snapshot: expect.any(Number), + token: 250, + http: null, + }, }); + expect(warn).toHaveBeenCalledWith("account.machine_publish_failed", expect.objectContaining({ + leg: "token", + code: "token_timeout", + legDurationsMs: expect.objectContaining({ token: 250, http: null }), + })); }); it("force-refreshes once after the directory rejects the first bearer", async () => { @@ -338,8 +489,15 @@ describe("account machine publisher health", () => { await service.publishNow(); - expect(getAccessToken).toHaveBeenNthCalledWith(1); - expect(getAccessToken).toHaveBeenNthCalledWith(2, { forceRefresh: true }); + expect(getAccessToken).toHaveBeenNthCalledWith(1, expect.objectContaining({ + signal: expect.any(AbortSignal), + timeoutMs: 10_000, + })); + expect(getAccessToken).toHaveBeenNthCalledWith(2, expect.objectContaining({ + forceRefresh: true, + signal: expect.any(AbortSignal), + timeoutMs: 10_000, + })); expect(fetchImpl).toHaveBeenCalledTimes(2); expect(service.getPublisherHealth()).toMatchObject({ state: "published", @@ -347,6 +505,60 @@ describe("account machine publisher health", () => { }); }); + it("classifies a stalled 401 refresh as token_timeout, outside the HTTP budget", async () => { + vi.useFakeTimers(); + const warn = vi.fn(); + const getAccessToken = vi.fn((options?: { + forceRefresh?: boolean; + signal?: AbortSignal; + }): Promise => { + if (!options?.forceRefresh) return Promise.resolve("stale-token"); + return new Promise((_resolve, reject) => { + options.signal?.addEventListener("abort", () => { + reject(options.signal?.reason); + }); + }); + }); + const fetchImpl = vi.fn(async () => new Response( + JSON.stringify({ error: "expired bearer" }), + { + status: 401, + headers: { "content-type": "application/json" }, + }, + )); + const service = createAccountMachinePublisherService({ + getAccessToken, + getAccountStatus: () => ({ signedIn: true, sessionReadState: "available" as const }), + getSnapshot: async () => snapshot(), + getMachineKey: () => "machine-studio", + directoryBaseUrl: () => "https://directory.example", + fetchImpl, + tokenTimeoutMs: 250, + requestTimeoutMs: 250, + logger: { warn }, + }); + + const publishing = service.publishNow(); + await vi.advanceTimersByTimeAsync(250); + await publishing; + + expect(fetchImpl).toHaveBeenCalledOnce(); + expect(service.getPublisherHealth()).toMatchObject({ + state: "token_timeout", + lastHttpStatus: 401, + lastHttpReason: "expired bearer", + lastLegDurations: { + snapshot: expect.any(Number), + token: 250, + http: expect.any(Number), + }, + }); + expect(warn).toHaveBeenCalledWith("account.machine_publish_failed", expect.objectContaining({ + leg: "token_refresh_401", + code: "token_timeout", + })); + }); + it("accelerates transient retries with bounded backoff and resets after success", async () => { vi.useFakeTimers(); const fetchImpl = vi.fn() diff --git a/apps/ade-cli/src/services/account/accountMachinePublisherService.ts b/apps/ade-cli/src/services/account/accountMachinePublisherService.ts index 68f313e2d..a06a3de6e 100644 --- a/apps/ade-cli/src/services/account/accountMachinePublisherService.ts +++ b/apps/ade-cli/src/services/account/accountMachinePublisherService.ts @@ -3,9 +3,11 @@ import { createSyncAccountDirectoryHealth, type AdeAccountMachineEndpoint, type SyncAccountDirectoryHealth, + type SyncAccountDirectoryLegDurations, type SyncRoleSnapshot, type SyncRouteHealth, } from "../../../../desktop/src/shared/types"; +import type { ProductAnalyticsCapture } from "../../../../desktop/src/shared/types/productAnalytics"; import { readAccountDirectoryHttpReason, resolveTrustedAccountDirectoryBaseUrl, @@ -32,6 +34,10 @@ export const ACCOUNT_MACHINE_RELAY_STATE_POLL_MS = 2_000; // window while bounding sustained failures at a 20-second request cadence. export const ACCOUNT_MACHINE_RETRY_BACKOFF_MS = [1_000, 2_000, 5_000, 10_000, 20_000] as const; const DEFAULT_REQUEST_TIMEOUT_MS = 8_000; +const DEFAULT_TOKEN_TIMEOUT_MS = 10_000; +const SLOW_PUBLISH_LEG_MS = 2_000; +const PUBLISH_INFO_INTERVAL = 10; +export const PUBLISH_FAILURE_ANALYTICS_THRESHOLD_MS = 120_000; export type AccountMachineRegistration = { machineKey: string; @@ -52,6 +58,8 @@ export type AccountMachineRegistration = { }; type AccountMachinePublisherLogger = { + debug?(message: string, meta?: Record): void; + info?(message: string, meta?: Record): void; warn(message: string, meta?: Record): void; }; @@ -59,6 +67,42 @@ type BrainAccountMachinePublisherLogger = AccountMachinePublisherLogger & { info(message: string, meta?: Record): void; }; +type AccountMachinePublishLeg = + | "setup" + | "snapshot" + | "token" + | "token_refresh_401" + | "http"; + +function failureLegForState( + state: SyncAccountDirectoryHealth["state"], +): AccountMachinePublishLeg { + if (state === "snapshot_failed" || state === "missing_pairing_connect_info") return "snapshot"; + if (state === "token_unreadable" || state === "token_timeout") return "token"; + if ( + state === "http_error" + || state === "http_timeout" + || state === "timeout" + || state === "transport_error" + ) return "http"; + return "setup"; +} + +class AccountMachinePublishLegTimeoutError extends Error { + constructor(readonly leg: Extract) { + super(`Account machine publish ${leg} timed out.`); + this.name = "AccountMachinePublishLegTimeoutError"; + } +} + +function emptyLegDurations(): SyncAccountDirectoryLegDurations { + return { + snapshot: null, + token: null, + http: null, + }; +} + export type AccountMachineRegistrationSnapshot = Pick< SyncRoleSnapshot, "role" | "runtimeRole" | "runtimeName" | "pairingConnectInfo" @@ -239,7 +283,11 @@ function relayPublishStateSignature( } export function createAccountMachinePublisherService(options: { - getAccessToken: (options?: { forceRefresh?: boolean }) => Promise; + getAccessToken: (options?: { + forceRefresh?: boolean; + signal?: AbortSignal; + timeoutMs?: number; + }) => Promise; getAccountStatus?: () => PublisherAccountStatus; getSnapshot: () => Promise; getMachineKey: () => string; @@ -251,8 +299,10 @@ export function createAccountMachinePublisherService(options: { heartbeatMs?: number; relayStatePollMs?: number; requestTimeoutMs?: number; + tokenTimeoutMs?: number; now?: () => number; logger?: AccountMachinePublisherLogger; + captureAnalytics?: (input: ProductAnalyticsCapture) => void; }) { const heartbeatMs = Math.max(1_000, Math.floor(options.heartbeatMs ?? ACCOUNT_MACHINE_HEARTBEAT_MS)); const relayStatePollMs = Math.max( @@ -260,12 +310,13 @@ export function createAccountMachinePublisherService(options: { Math.floor(options.relayStatePollMs ?? ACCOUNT_MACHINE_RELAY_STATE_POLL_MS), ); const requestTimeoutMs = Math.max(250, Math.floor(options.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS)); + const tokenTimeoutMs = Math.max(250, Math.floor(options.tokenTimeoutMs ?? DEFAULT_TOKEN_TIMEOUT_MS)); const now = options.now ?? Date.now; let started = false; let disposed = false; let heartbeatTimer: ReturnType | null = null; let relayStatePollTimer: ReturnType | null = null; - let activeController: AbortController | null = null; + const activeControllers = new Set(); let inFlight: Promise | null = null; let triggeredPublishPending = false; let lastRelayPublishStateSignature: string | null = null; @@ -276,6 +327,8 @@ export function createAccountMachinePublisherService(options: { } | null = null; let lastWarning: string | null = null; let transientFailureCount = 0; + let successfulPublishCount = 0; + let publishFailureAnalyticsEmitted = false; let unsubscribeSignIn: (() => void) | null = null; let health = createSyncAccountDirectoryHealth( "sync_disabled", @@ -296,10 +349,20 @@ export function createAccountMachinePublisherService(options: { return changed; }; - const warnOnce = (code: string, meta: Record = {}): void => { + const warnOnce = ( + code: string, + leg: AccountMachinePublishLeg, + legDurationsMs: SyncAccountDirectoryLegDurations, + meta: Record = {}, + ): void => { if (lastWarning === code) return; lastWarning = code; - options.logger?.warn("account.machine_publish_failed", { code, ...meta }); + options.logger?.warn("account.machine_publish_failed", { + leg, + legDurationsMs, + code, + ...meta, + }); }; const recordTransientFailure = (): void => { @@ -350,8 +413,21 @@ export function createAccountMachinePublisherService(options: { lastHttpReason?: string | null; reachableEndpointCount?: number; succeededAt?: number; + legDurations?: SyncAccountDirectoryLegDurations; }, ): void => { + const failureStates = new Set([ + "snapshot_failed", + "machine_key_unavailable", + "missing_pairing_connect_info", + "token_unreadable", + "invalid_directory_url", + "http_error", + "token_timeout", + "http_timeout", + "timeout", + "transport_error", + ]); health = { state, skipReason: args.skipReason, @@ -361,14 +437,153 @@ export function createAccountMachinePublisherService(options: { lastHttpStatus: args.lastHttpStatus ?? null, lastHttpReason: args.lastHttpReason ?? null, reachableEndpointCount: args.reachableEndpointCount ?? 0, + lastLegDurations: args.legDurations + ? { ...args.legDurations } + : health.lastLegDurations, + failingSinceMs: state === "published" + ? null + : failureStates.has(state) + ? health.failingSinceMs ?? args.attemptAt + : null, }; + if (state === "published") { + publishFailureAnalyticsEmitted = false; + } else if ( + health.failingSinceMs != null + && !publishFailureAnalyticsEmitted + && args.attemptAt - health.failingSinceMs >= PUBLISH_FAILURE_ANALYTICS_THRESHOLD_MS + ) { + publishFailureAnalyticsEmitted = true; + const leg = failureLegForState(state); + options.captureAnalytics?.({ + event: "ade_publish_failing", + surface: "api", + properties: { + failing_minutes: Math.max(2, Math.floor((args.attemptAt - health.failingSinceMs) / 60_000)), + leg, + code: state, + }, + dedupeKey: `publish-failing:${health.failingSinceMs}`, + minimumIntervalMs: 24 * 60 * 60 * 1_000, + }); + } }; const publish = async (): Promise => { if (disposed) return; const attemptAt = now(); + const legDurations = emptyLegDurations(); + const addLegDuration = ( + leg: keyof SyncAccountDirectoryLegDurations, + startedAt: number, + ): void => { + const elapsed = Math.max(0, Math.round(now() - startedAt)); + legDurations[leg] = (legDurations[leg] ?? 0) + elapsed; + }; + const outcome = ( + state: SyncAccountDirectoryHealth["state"], + args: Omit[1], "legDurations">, + ): void => { + recordOutcome(state, { + ...args, + legDurations, + }); + }; + const runTokenLeg = async ( + leg: Extract, + forceRefresh: boolean, + ): Promise => { + const controller = new AbortController(); + activeControllers.add(controller); + const startedAt = now(); + let timeout: ReturnType | null = null; + let rejectOnAbort: (() => void) | null = null; + const aborted = new Promise((_resolve, reject) => { + rejectOnAbort = () => reject( + controller.signal.reason instanceof Error + ? controller.signal.reason + : new DOMException("The account token request was aborted.", "AbortError"), + ); + controller.signal.addEventListener("abort", rejectOnAbort, { once: true }); + timeout = setTimeout(() => { + controller.abort(new AccountMachinePublishLegTimeoutError(leg)); + }, tokenTimeoutMs); + timeout.unref?.(); + }); + try { + return await Promise.race([ + options.getAccessToken({ + ...(forceRefresh ? { forceRefresh: true } : {}), + signal: controller.signal, + timeoutMs: tokenTimeoutMs, + }), + aborted, + ]); + } finally { + if (timeout) clearTimeout(timeout); + if (rejectOnAbort) { + controller.signal.removeEventListener("abort", rejectOnAbort); + } + activeControllers.delete(controller); + addLegDuration("token", startedAt); + } + }; + const sendRegistration = async ( + token: string, + registration: AccountMachineRegistration, + baseUrl: string, + ): Promise => { + const remainingHttpBudgetMs = requestTimeoutMs - (legDurations.http ?? 0); + if (remainingHttpBudgetMs <= 0) { + throw new AccountMachinePublishLegTimeoutError("http"); + } + const controller = new AbortController(); + activeControllers.add(controller); + const startedAt = now(); + let timeout: ReturnType | null = null; + let rejectOnAbort: (() => void) | null = null; + const aborted = new Promise((_resolve, reject) => { + rejectOnAbort = () => reject( + controller.signal.reason instanceof Error + ? controller.signal.reason + : new DOMException("The account-directory request was aborted.", "AbortError"), + ); + controller.signal.addEventListener("abort", rejectOnAbort, { once: true }); + timeout = setTimeout(() => { + controller.abort(new AccountMachinePublishLegTimeoutError("http")); + }, remainingHttpBudgetMs); + timeout.unref?.(); + }); + try { + return await Promise.race([ + (options.fetchImpl ?? fetch)(`${baseUrl}/account/machines/register`, { + method: "POST", + headers: { + accept: "application/json", + authorization: `Bearer ${token}`, + "content-type": "application/json", + }, + body: JSON.stringify(registration), + credentials: "omit", + referrerPolicy: "no-referrer", + cache: "no-store", + redirect: "error", + signal: controller.signal, + }), + aborted, + ]); + } finally { + if (timeout) clearTimeout(timeout); + if (rejectOnAbort) { + controller.signal.removeEventListener("abort", rejectOnAbort); + } + activeControllers.delete(controller); + addLegDuration("http", startedAt); + } + }; + if (options.isSyncEnabled?.() === false) { - recordOutcome("sync_disabled", { + outcome("sync_disabled", { attemptAt, skipReason: "Account-directory publishing is disabled because sync is disabled.", directoryOrigin: null, @@ -394,31 +609,34 @@ export function createAccountMachinePublisherService(options: { baseUrl = null; } if (!baseUrl) { - recordOutcome("invalid_directory_url", { + outcome("invalid_directory_url", { attemptAt, skipReason: "The configured account-directory URL is invalid or untrusted.", directoryOrigin: null, }); - warnOnce("invalid_directory_url"); + warnOnce("invalid_directory_url", "setup", legDurations); return; } const directoryOrigin = new URL(baseUrl).origin; let snapshot: AccountMachineRegistrationSnapshot | null; + const snapshotStartedAt = now(); try { snapshot = await options.getSnapshot(); } catch { if (disposed) return; - recordOutcome("snapshot_failed", { + outcome("snapshot_failed", { attemptAt, skipReason: "The active sync snapshot could not be read.", directoryOrigin, }); return; + } finally { + addLegDuration("snapshot", snapshotStartedAt); } if (disposed) return; if (!snapshot) { - recordOutcome("no_active_sync_scope", { + outcome("no_active_sync_scope", { attemptAt, skipReason: "No active sync scope is available.", directoryOrigin, @@ -433,7 +651,7 @@ export function createAccountMachinePublisherService(options: { // The typed outcome below is the public diagnostic; no identity detail is logged. } if (!machineKey) { - recordOutcome("machine_key_unavailable", { + outcome("machine_key_unavailable", { attemptAt, skipReason: "The machine directory key is unavailable.", directoryOrigin, @@ -441,7 +659,7 @@ export function createAccountMachinePublisherService(options: { return; } if (!snapshot.pairingConnectInfo) { - recordOutcome("missing_pairing_connect_info", { + outcome("missing_pairing_connect_info", { attemptAt, skipReason: "Pairing connection information is unavailable.", directoryOrigin, @@ -452,7 +670,7 @@ export function createAccountMachinePublisherService(options: { ? snapshot.runtimeRole === "host" : snapshot.role === "brain"; if (!isHost) { - recordOutcome("not_host", { + outcome("not_host", { attemptAt, skipReason: "This runtime is not the active sync host.", directoryOrigin, @@ -462,7 +680,7 @@ export function createAccountMachinePublisherService(options: { const publicKeyRawBase64 = readSigningPublicKey(); if (options.getMachineIdentitySigningPublicKey && !publicKeyRawBase64) { - recordOutcome("machine_key_unavailable", { + outcome("machine_key_unavailable", { attemptAt, skipReason: "The machine identity signing key is unavailable.", directoryOrigin, @@ -476,7 +694,7 @@ export function createAccountMachinePublisherService(options: { publicKeyRawBase64, }); if (!observedRegistration) { - recordOutcome("machine_key_unavailable", { + outcome("machine_key_unavailable", { attemptAt, skipReason: "The machine registration could not be built.", directoryOrigin, @@ -490,7 +708,7 @@ export function createAccountMachinePublisherService(options: { try { accountStatus = options.getAccountStatus?.() ?? null; } catch { - recordOutcome("token_unreadable", { + outcome("token_unreadable", { attemptAt, skipReason: "The ADE brain could not read account status.", directoryOrigin, @@ -501,7 +719,7 @@ export function createAccountMachinePublisherService(options: { if (isPublisherSignedOut(accountStatus)) { clearRetainedRelayState(); const unreadable = accountStatus?.sessionReadState === "unreadable"; - recordOutcome(unreadable ? "token_unreadable" : "account_signed_out", { + outcome(unreadable ? "token_unreadable" : "account_signed_out", { attemptAt, skipReason: unreadable ? "The ADE brain could not read the stored account session." @@ -549,13 +767,25 @@ export function createAccountMachinePublisherService(options: { let accessToken: string | null = null; try { - accessToken = (await options.getAccessToken())?.trim() || null; - } catch { + accessToken = (await runTokenLeg("token", false))?.trim() || null; + } catch (error) { + if (disposed) return; + if (error instanceof AccountMachinePublishLegTimeoutError) { + recordTransientFailure(); + outcome("token_timeout", { + attemptAt, + skipReason: "The ADE account token refresh timed out.", + directoryOrigin, + reachableEndpointCount, + }); + warnOnce("token_timeout", "token", legDurations); + return; + } accessToken = null; } if (disposed) return; if (!accessToken) { - recordOutcome("token_unreadable", { + outcome("token_unreadable", { attemptAt, skipReason: "The ADE brain could not read or refresh the account token.", directoryOrigin, @@ -564,39 +794,40 @@ export function createAccountMachinePublisherService(options: { return; } - const controller = new AbortController(); - activeController = controller; - const timeout = setTimeout(() => controller.abort(), requestTimeoutMs); - timeout.unref?.(); try { - const sendRegistration = (token: string): Promise => - (options.fetchImpl ?? fetch)(`${baseUrl}/account/machines/register`, { - method: "POST", - headers: { - accept: "application/json", - authorization: `Bearer ${token}`, - "content-type": "application/json", - }, - body: JSON.stringify(registration), - credentials: "omit", - referrerPolicy: "no-referrer", - cache: "no-store", - redirect: "error", - signal: controller.signal, - }); - - let response = await sendRegistration(accessToken); + let response = await sendRegistration(accessToken, registration, baseUrl); let firstUnauthorizedReason: string | null = null; if (response.status === 401) { firstUnauthorizedReason = await readAccountDirectoryHttpReason(response).catch(() => null); let refreshedToken: string | null = null; try { - refreshedToken = (await options.getAccessToken({ forceRefresh: true }))?.trim() || null; - } catch { + refreshedToken = (await runTokenLeg("token_refresh_401", true))?.trim() || null; + } catch (error) { + if (disposed) return; + if (error instanceof AccountMachinePublishLegTimeoutError) { + recordTransientFailure(); + outcome("token_timeout", { + attemptAt, + skipReason: "The ADE account token refresh after HTTP 401 timed out.", + directoryOrigin, + lastHttpStatus: 401, + lastHttpReason: firstUnauthorizedReason, + reachableEndpointCount, + }); + warnOnce( + "token_timeout", + "token_refresh_401", + legDurations, + { status: 401 }, + ); + return; + } // Preserve the first unauthorized response and its parsed reason. } if (disposed) return; - if (refreshedToken) response = await sendRegistration(refreshedToken); + if (refreshedToken) { + response = await sendRegistration(refreshedToken, registration, baseUrl); + } } if (!response.ok) { // Always drain the final response, even when the first 401 already @@ -611,7 +842,7 @@ export function createAccountMachinePublisherService(options: { if (response.status === 401 || response.status === 403) { clearRetainedRelayState(); } - recordOutcome("http_error", { + outcome("http_error", { attemptAt, skipReason: httpReason ? `The account directory returned HTTP ${response.status}: ${httpReason}` @@ -621,7 +852,7 @@ export function createAccountMachinePublisherService(options: { lastHttpReason: httpReason, reachableEndpointCount, }); - warnOnce("http_error", { status: response.status }); + warnOnce("http_error", "http", legDurations, { status: response.status }); return; } await response.body?.cancel().catch(() => {}); @@ -635,7 +866,7 @@ export function createAccountMachinePublisherService(options: { endpoints: publishedRelayEndpoints, } : null; - recordOutcome("published", { + outcome("published", { attemptAt, skipReason: null, directoryOrigin, @@ -643,24 +874,35 @@ export function createAccountMachinePublisherService(options: { reachableEndpointCount, succeededAt: now(), }); + successfulPublishCount += 1; + const logMeta = { legDurationsMs: { ...legDurations } }; + const slow = Object.values(legDurations).some( + (duration) => duration != null && duration > SLOW_PUBLISH_LEG_MS, + ); + if (slow) { + options.logger?.warn("account.machine_publish_ok", logMeta); + } else if (successfulPublishCount % PUBLISH_INFO_INTERVAL === 0) { + options.logger?.info?.("account.machine_publish_ok", logMeta); + } else { + options.logger?.debug?.("account.machine_publish_ok", logMeta); + } } catch (error) { - if (disposed && controller.signal.aborted) return; - const state = controller.signal.aborted ? "timeout" : "transport_error"; + if (disposed) return; + const isHttpTimeout = error instanceof AccountMachinePublishLegTimeoutError + && error.leg === "http"; + const state = isHttpTimeout ? "http_timeout" : "transport_error"; recordTransientFailure(); - recordOutcome(state, { + outcome(state, { attemptAt, - skipReason: state === "timeout" + skipReason: state === "http_timeout" ? "The account-directory publish request timed out." : "The account-directory publish request could not reach the service.", directoryOrigin, reachableEndpointCount, }); - warnOnce(state, { + warnOnce(state, "http", legDurations, { errorKind: error instanceof Error ? error.name : "unknown", }); - } finally { - clearTimeout(timeout); - if (activeController === controller) activeController = null; } }; @@ -676,7 +918,7 @@ export function createAccountMachinePublisherService(options: { directoryOrigin: health.directoryOrigin, reachableEndpointCount: health.reachableEndpointCount, }); - warnOnce("transport_error", { + warnOnce("transport_error", "setup", health.lastLegDurations, { errorKind: error instanceof Error ? error.name : "unknown", }); }) @@ -804,7 +1046,10 @@ export function createAccountMachinePublisherService(options: { }, getPublisherHealth(): SyncAccountDirectoryHealth { - return { ...health }; + return { + ...health, + lastLegDurations: { ...health.lastLegDurations }, + }; }, dispose(): void { @@ -814,8 +1059,10 @@ export function createAccountMachinePublisherService(options: { clearHeartbeatTimer(); if (relayStatePollTimer) clearTimeout(relayStatePollTimer); relayStatePollTimer = null; - activeController?.abort(); - activeController = null; + for (const controller of activeControllers) { + controller.abort(new DOMException("The account publisher was disposed.", "AbortError")); + } + activeControllers.clear(); unsubscribeSignIn?.(); unsubscribeSignIn = null; }, @@ -838,6 +1085,7 @@ export function createBrainAccountMachinePublisherService(options: { getMachineKey: () => string; directoryBaseUrl?: () => string | null | undefined; logger: BrainAccountMachinePublisherLogger; + captureAnalytics?: (input: ProductAnalyticsCapture) => void; }): AccountMachinePublisherService { const accountAuthService = getSharedAccountAuthService({ secretsDir: options.secretsDir, @@ -882,5 +1130,6 @@ export function createBrainAccountMachinePublisherService(options: { }, subscribeToSignIn: (listener) => accountAuthService.onSignedIn(listener), logger: options.logger, + captureAnalytics: options.captureAnalytics, }); } diff --git a/apps/ade-cli/src/services/personalChats/personalChatScope.ts b/apps/ade-cli/src/services/personalChats/personalChatScope.ts index 1706c7fb9..401cb9ba6 100644 --- a/apps/ade-cli/src/services/personalChats/personalChatScope.ts +++ b/apps/ade-cli/src/services/personalChats/personalChatScope.ts @@ -6,6 +6,7 @@ import type { PersonalChatAction, PersonalChatCallResponse, PersonalChatCapabilities, + RuntimeActivityCounts, } from "../../../../desktop/src/shared/types"; import { PERSONAL_CHAT_ACTIONS } from "../../../../desktop/src/shared/types"; import { resolveAdeLayout } from "../../../../desktop/src/shared/adeLayout"; @@ -19,6 +20,18 @@ type PersonalChatScopeOptions = { type ObjectArgs = Record; +export function summarizeRuntimeActivity(runtime: AdeRuntime): RuntimeActivityCounts { + return { + activeAgentTurns: runtime.agentChatService?.hasActiveWorkloads() ? 1 : 0, + activeWorkSessions: runtime.ptyService.list({ status: "running", limit: 500 }) + .filter( + (session) => + session.runtimeState === "running" + || session.runtimeState === "waiting-input", + ).length, + }; +} + function asObject(value: unknown): ObjectArgs { return value && typeof value === "object" && !Array.isArray(value) ? value as ObjectArgs @@ -72,6 +85,19 @@ export class PersonalChatScope { return { version: 1, actions: [...PERSONAL_CHAT_ACTIONS] }; } + /** + * Read activity only when the machine chat runtime is already booted. Update + * idleness probes must never create the personal-chat runtime as a side + * effect. + */ + async activitySummary(): Promise { + const pending = this.runtimePromise; + if (!pending) return { activeAgentTurns: 0, activeWorkSessions: 0 }; + const runtime = await pending.catch(() => null); + if (!runtime) return { activeAgentTurns: 0, activeWorkSessions: 0 }; + return summarizeRuntimeActivity(runtime); + } + async call(actionValue: unknown, argsValue: unknown): Promise { if (!isPersonalChatAction(actionValue)) { throw new Error(`Unsupported personal chat action: ${String(actionValue ?? "")}.`); diff --git a/apps/ade-cli/src/services/projects/projectScope.ts b/apps/ade-cli/src/services/projects/projectScope.ts index 7a9f3a874..2bf72d31b 100644 --- a/apps/ade-cli/src/services/projects/projectScope.ts +++ b/apps/ade-cli/src/services/projects/projectScope.ts @@ -1,5 +1,6 @@ import type { AdeRuntime, AdeRuntimeSyncOptions } from "../../bootstrap"; import type { SyncCommandPayload } from "../../../../desktop/src/shared/types"; +import type { SyncRemoteCommandExecutionContext } from "../sync/syncRemoteCommandService"; import type { ProjectId, ProjectRecord, ProjectRegistry } from "./projectRegistry"; type SwitchSyncHostOptions = { @@ -63,8 +64,11 @@ export class ProjectScopeRegistry { private latestSyncHostTransitionId = 0; private latestSyncHostTransitionProjectId: ProjectId | null = null; private readonly remoteCommandExecutor = { - execute: async (payload: SyncCommandPayload): Promise => { - return await this.executeRemoteCommand(payload); + execute: async ( + payload: SyncCommandPayload, + context?: SyncRemoteCommandExecutionContext, + ): Promise => { + return await this.executeRemoteCommand(payload, context); }, }; @@ -432,7 +436,10 @@ export class ProjectScopeRegistry { }; } - private async executeRemoteCommand(payload: SyncCommandPayload): Promise { + private async executeRemoteCommand( + payload: SyncCommandPayload, + context?: SyncRemoteCommandExecutionContext, + ): Promise { const projectId = typeof payload.projectId === "string" && payload.projectId.trim() ? payload.projectId.trim() : null; @@ -444,6 +451,6 @@ export class ProjectScopeRegistry { if (!syncService) { throw new Error(`Phone sync is not available for project ${projectId}.`); } - return await syncService.executeRemoteCommand(payload); + return await syncService.executeRemoteCommand(payload, context); } } diff --git a/apps/ade-cli/src/services/runtime/brainFreshnessMonitor.test.ts b/apps/ade-cli/src/services/runtime/brainFreshnessMonitor.test.ts new file mode 100644 index 000000000..512d0ebb4 --- /dev/null +++ b/apps/ade-cli/src/services/runtime/brainFreshnessMonitor.test.ts @@ -0,0 +1,98 @@ +import { describe, expect, it, vi } from "vitest"; +import { createBrainFreshnessMonitor } from "./brainFreshnessMonitor"; + +describe("brain freshness monitor", () => { + it("does not hash or restart while the entrypoint stat is unchanged", async () => { + const stat = vi.fn(async () => ({ mtimeMs: 10, size: 20 })); + const computeHash = vi.fn(async () => "disk-hash"); + const restart = vi.fn(); + const monitor = createBrainFreshnessMonitor({ + filePath: "/tmp/cli.cjs", + runningHash: "running-hash", + isIdle: () => true, + restart, + logger: { warn: vi.fn() }, + stat, + computeHash, + }); + + await monitor.probeNow(); + await monitor.probeNow(); + + expect(stat).toHaveBeenCalledTimes(2); + expect(computeHash).not.toHaveBeenCalled(); + expect(restart).not.toHaveBeenCalled(); + }); + + it("waits for idle and restarts after a changed entrypoint hashes differently", async () => { + const stats = [ + { mtimeMs: 10, size: 20 }, + { mtimeMs: 11, size: 21 }, + ]; + const stat = vi.fn(async () => stats.shift() ?? { mtimeMs: 11, size: 21 }); + const computeHash = vi.fn(async () => "disk-hash"); + const restart = vi.fn(); + const warn = vi.fn(); + let idle = false; + const sleep = vi.fn(async () => { + idle = true; + }); + const monitor = createBrainFreshnessMonitor({ + filePath: "/tmp/cli.cjs", + runningHash: "running-hash", + isIdle: () => idle, + restart, + logger: { warn }, + stat, + computeHash, + sleep, + now: (() => { + let value = 0; + return () => value += 10; + })(), + idlePollMs: 10, + maxIdleWaitMs: 100, + }); + + await monitor.probeNow(); + await monitor.probeNow(); + + expect(computeHash).toHaveBeenCalledTimes(1); + expect(sleep).toHaveBeenCalledTimes(1); + expect(restart).toHaveBeenCalledTimes(1); + expect(warn).toHaveBeenCalledWith("brain.freshness_mismatch", { + runningHash: "running-hash", + diskHash: "disk-hash", + }); + expect(warn).not.toHaveBeenCalledWith( + "brain.freshness_restart_forced", + expect.anything(), + ); + }); + + it("retries a changed entrypoint after a transient restart failure", async () => { + const stat = vi.fn() + .mockResolvedValueOnce({ mtimeMs: 10, size: 20 }) + .mockResolvedValue({ mtimeMs: 11, size: 21 }); + const restart = vi.fn() + .mockRejectedValueOnce(new Error("launchctl failed")) + .mockResolvedValueOnce(undefined); + const monitor = createBrainFreshnessMonitor({ + filePath: "/tmp/cli.cjs", + runningHash: "running-hash", + isIdle: () => true, + restart, + logger: { warn: vi.fn() }, + stat, + computeHash: async () => "disk-hash", + setInterval: vi.fn(() => ({ unref: vi.fn() })) as never, + clearInterval: vi.fn(), + }); + + await monitor.probeNow(); + await monitor.probeNow(); + await monitor.probeNow(); + + expect(restart).toHaveBeenCalledTimes(2); + }); +}); diff --git a/apps/ade-cli/src/services/runtime/brainFreshnessMonitor.ts b/apps/ade-cli/src/services/runtime/brainFreshnessMonitor.ts new file mode 100644 index 000000000..ba5f9f56c --- /dev/null +++ b/apps/ade-cli/src/services/runtime/brainFreshnessMonitor.ts @@ -0,0 +1,175 @@ +import fs from "node:fs"; +import { computeRuntimeBuildHashAsync } from "./runtimeBuildIdentity"; + +const DEFAULT_FRESHNESS_INTERVAL_MS = 300_000; +const DEFAULT_IDLE_POLL_MS = 5_000; +const DEFAULT_MAX_IDLE_WAIT_MS = 30 * 60_000; + +type BrainFreshnessLogger = { + warn: (event: string, fields: Record) => void; +}; + +type BrainFileStat = { + mtimeMs: number; + size: number; +}; + +export type BrainFreshnessMonitorOptions = { + filePath: string; + runningHash: string | null; + isIdle: () => boolean; + restart: () => void | Promise; + logger: BrainFreshnessLogger; + env?: NodeJS.ProcessEnv; + stat?: (filePath: string) => Promise; + computeHash?: (filePath: string) => Promise | string | null; + now?: () => number; + sleep?: (ms: number) => Promise; + intervalMs?: number; + idlePollMs?: number; + maxIdleWaitMs?: number; + setInterval?: typeof globalThis.setInterval; + clearInterval?: typeof globalThis.clearInterval; +}; + +export type BrainFreshnessMonitor = { + start(): void; + stop(): void; + probeNow(): Promise; +}; + +function parseIntervalMs(env: NodeJS.ProcessEnv): number { + const raw = env.ADE_BRAIN_FRESHNESS_INTERVAL_MS?.trim(); + if (!raw) return DEFAULT_FRESHNESS_INTERVAL_MS; + const parsed = Number.parseInt(raw, 10); + return Number.isFinite(parsed) && parsed > 0 + ? Math.max(1_000, parsed) + : DEFAULT_FRESHNESS_INTERVAL_MS; +} + +async function statFile(filePath: string): Promise { + const stat = await fs.promises.stat(filePath); + return { mtimeMs: stat.mtimeMs, size: stat.size }; +} + +function sameStat(left: BrainFileStat, right: BrainFileStat): boolean { + return left.mtimeMs === right.mtimeMs && left.size === right.size; +} + +export function createBrainFreshnessMonitor( + options: BrainFreshnessMonitorOptions, +): BrainFreshnessMonitor { + const env = options.env ?? process.env; + const stat = options.stat ?? statFile; + const computeHash = options.computeHash ?? computeRuntimeBuildHashAsync; + const now = options.now ?? Date.now; + const intervalMs = options.intervalMs ?? parseIntervalMs(env); + const idlePollMs = Math.max(1, options.idlePollMs ?? DEFAULT_IDLE_POLL_MS); + const maxIdleWaitMs = Math.max(0, options.maxIdleWaitMs ?? DEFAULT_MAX_IDLE_WAIT_MS); + const setIntervalFn = options.setInterval ?? globalThis.setInterval; + const clearIntervalFn = options.clearInterval ?? globalThis.clearInterval; + const sleep = options.sleep ?? (async (ms: number) => { + await new Promise((resolve) => { + const timer = setTimeout(resolve, ms); + timer.unref?.(); + }); + }); + + let stopped = false; + let started = false; + let timer: ReturnType | null = null; + let lastStat: BrainFileStat | null = null; + let probeInFlight: Promise | null = null; + let restartRequested = false; + + const waitForIdleAndRestart = async ( + runningHash: string, + diskHash: string, + ): Promise => { + const deadline = now() + maxIdleWaitMs; + while (!stopped && !options.isIdle() && now() < deadline) { + await sleep(Math.min(idlePollMs, Math.max(1, deadline - now()))); + } + if (stopped) return; + if (!options.isIdle()) { + options.logger.warn("brain.freshness_restart_forced", { + runningHash, + diskHash, + maxWaitMs: maxIdleWaitMs, + }); + } + await options.restart(); + }; + + const probe = async (): Promise => { + if (stopped || restartRequested) return; + let nextStat: BrainFileStat; + try { + nextStat = await stat(options.filePath); + } catch { + return; + } + if (lastStat == null) { + lastStat = nextStat; + return; + } + if (sameStat(lastStat, nextStat)) return; + const previousStat = lastStat; + lastStat = nextStat; + const diskHash = await computeHash(options.filePath); + const runningHash = options.runningHash; + if (!diskHash || !runningHash || diskHash === runningHash) return; + + restartRequested = true; + if (timer) { + clearIntervalFn(timer); + timer = null; + } + options.logger.warn("brain.freshness_mismatch", { + runningHash, + diskHash, + }); + try { + await waitForIdleAndRestart(runningHash, diskHash); + } catch { + // The service handover can fail transiently. Keep the old stat baseline + // so the unchanged replacement file is re-hashed on the next scheduled + // probe instead of disabling freshness checks for this brain forever. + lastStat = previousStat; + restartRequested = false; + if (started && !stopped && !timer) { + timer = setIntervalFn(() => { + void probe(); + }, intervalMs); + timer.unref?.(); + } + } + }; + + return { + start(): void { + if (stopped || timer || env.ADE_DISABLE_BRAIN_FRESHNESS === "1") return; + started = true; + void this.probeNow(); + timer = setIntervalFn(() => { + void this.probeNow(); + }, intervalMs); + timer.unref?.(); + }, + + stop(): void { + stopped = true; + if (timer) clearIntervalFn(timer); + timer = null; + }, + + probeNow(): Promise { + if (probeInFlight) return probeInFlight; + const current = probe().finally(() => { + if (probeInFlight === current) probeInFlight = null; + }); + probeInFlight = current; + return current; + }, + }; +} diff --git a/apps/ade-cli/src/services/runtime/brainLogger.test.ts b/apps/ade-cli/src/services/runtime/brainLogger.test.ts new file mode 100644 index 000000000..382b5cfe9 --- /dev/null +++ b/apps/ade-cli/src/services/runtime/brainLogger.test.ts @@ -0,0 +1,68 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { createBrainLogger } from "./brainLogger"; + +const tempDirs: string[] = []; + +async function waitForFile(filePath: string): Promise { + await vi.waitFor(() => { + expect(fs.existsSync(filePath)).toBe(true); + expect(fs.statSync(filePath).size).toBeGreaterThan(0); + }); +} + +afterEach(() => { + vi.unstubAllEnvs(); + for (const tempDir of tempDirs.splice(0)) { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + +describe("createBrainLogger", () => { + it("writes timestamped JSONL, rotates at the configured cap, and prefixes stderr mirrors", async () => { + vi.stubEnv("ADE_LOG_LEVEL", "info"); + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "ade-brain-logger-")); + tempDirs.push(tempDir); + const logPath = path.join(tempDir, "brain.jsonl"); + const rotatedPath = path.join(tempDir, "brain.1.jsonl"); + const stderrWrite = vi.fn(() => true); + const logger = createBrainLogger(logPath, { + maxFileBytes: 180, + flushBatchSize: 1, + flushIntervalMs: 5, + rotationCheckWriteInterval: 1, + now: () => new Date("2026-07-23T16:00:00.000Z"), + stderr: { + write: stderrWrite, + } as unknown as Pick, + }); + + logger.info("brain.started", { detail: "x".repeat(160) }); + await waitForFile(logPath); + logger.warn("account.machine_publish_failed", { code: "token_timeout" }); + await waitForFile(rotatedPath); + await vi.waitFor(() => { + expect(fs.readFileSync(logPath, "utf8")).toContain( + "account.machine_publish_failed", + ); + }); + + const current = JSON.parse(fs.readFileSync(logPath, "utf8").trim()); + const rotated = JSON.parse(fs.readFileSync(rotatedPath, "utf8").trim()); + expect(current).toMatchObject({ + level: "warn", + event: "account.machine_publish_failed", + meta: { code: "token_timeout" }, + }); + expect(Date.parse(current.ts)).not.toBeNaN(); + expect(rotated).toMatchObject({ + level: "info", + event: "brain.started", + }); + expect(stderrWrite).toHaveBeenCalledWith( + "[2026-07-23T16:00:00.000Z] WARN account.machine_publish_failed {\"code\":\"token_timeout\"}\n", + ); + }); +}); diff --git a/apps/ade-cli/src/services/runtime/brainLogger.ts b/apps/ade-cli/src/services/runtime/brainLogger.ts new file mode 100644 index 000000000..6efa0d7f6 --- /dev/null +++ b/apps/ade-cli/src/services/runtime/brainLogger.ts @@ -0,0 +1,53 @@ +import { + createFileLogger, + type FileLoggerOptions, + type Logger, +} from "../../../../desktop/src/main/services/logging/logger"; + +type BrainLoggerOptions = FileLoggerOptions & { + now?: () => Date; + stderr?: Pick; +}; + +function serializedMeta(meta: Record | undefined): string { + if (!meta) return ""; + try { + return ` ${JSON.stringify(meta)}`; + } catch { + return " {\"serializationError\":true}"; + } +} + +export function createBrainLogger( + logFilePath: string, + options: BrainLoggerOptions = {}, +): Logger { + const { + now = () => new Date(), + stderr = process.stderr, + ...fileLoggerOptions + } = options; + const fileLogger = createFileLogger(logFilePath, fileLoggerOptions); + const mirror = ( + level: "warn" | "error", + event: string, + meta?: Record, + ): void => { + stderr.write( + `[${now().toISOString()}] ${level.toUpperCase()} ${event}${serializedMeta(meta)}\n`, + ); + }; + + return { + debug: (event, meta) => fileLogger.debug(event, meta), + info: (event, meta) => fileLogger.info(event, meta), + warn: (event, meta) => { + fileLogger.warn(event, meta); + mirror("warn", event, meta); + }, + error: (event, meta) => { + fileLogger.error(event, meta); + mirror("error", event, meta); + }, + }; +} diff --git a/apps/ade-cli/src/services/runtime/brainLoopWatchdog.test.ts b/apps/ade-cli/src/services/runtime/brainLoopWatchdog.test.ts new file mode 100644 index 000000000..b6b435b59 --- /dev/null +++ b/apps/ade-cli/src/services/runtime/brainLoopWatchdog.test.ts @@ -0,0 +1,119 @@ +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it, vi } from "vitest"; +import { + BRAIN_LOOP_WATCHDOG_BREADCRUMB_FILE, + BRAIN_LOOP_WATCHDOG_LAST_WEDGE_FILE, + evaluateBrainLoopWatchdog, + readBrainLoopWatchdogLastWedge, + recoverBrainLoopWatchdogBreadcrumb, + startBrainLoopWatchdog, +} from "./brainLoopWatchdog"; + +describe("brainLoopWatchdog", () => { + it("detects a main-thread heartbeat gap while the worker check cadence stays healthy", () => { + expect(evaluateBrainLoopWatchdog({ + nowWallMs: 20_000, + nowMonotonicMs: 20_000, + lastHeartbeatWallMs: 4_000, + lastHeartbeatMonotonicMs: 4_000, + previousCheckWallMs: 19_000, + previousCheckMonotonicMs: 19_000, + thresholdMs: 15_000, + checkIntervalMs: 1_000, + })).toEqual({ + blocked: true, + blockedMs: 16_000, + slept: false, + }); + }); + + it("skips a suspend/resume gap when worker wall and monotonic clocks jump together", () => { + expect(evaluateBrainLoopWatchdog({ + nowWallMs: 120_000, + nowMonotonicMs: 120_000, + lastHeartbeatWallMs: 1_000, + lastHeartbeatMonotonicMs: 1_000, + previousCheckWallMs: 2_000, + previousCheckMonotonicMs: 2_000, + thresholdMs: 15_000, + checkIntervalMs: 1_000, + })).toEqual({ + blocked: false, + blockedMs: 119_000, + slept: true, + }); + }); + + it("uses both clocks so a wall-clock adjustment alone cannot trigger a kill", () => { + expect(evaluateBrainLoopWatchdog({ + nowWallMs: 80_000, + nowMonotonicMs: 5_000, + lastHeartbeatWallMs: 1_000, + lastHeartbeatMonotonicMs: 4_000, + previousCheckWallMs: 79_000, + previousCheckMonotonicMs: 4_000, + thresholdMs: 15_000, + checkIntervalMs: 1_000, + })).toMatchObject({ + blocked: false, + blockedMs: 1_000, + slept: false, + }); + }); + + it("renames a crash breadcrumb to last-wedge and emits the recovery warning", () => { + const runtimeDir = fs.mkdtempSync(path.join(os.tmpdir(), "ade-loop-watchdog-")); + const breadcrumb = { + lastCommand: "ai.getStatus", + blockedMs: 16_250, + ts: "2026-07-23T12:00:00.000Z", + }; + fs.writeFileSync( + path.join(runtimeDir, BRAIN_LOOP_WATCHDOG_BREADCRUMB_FILE), + JSON.stringify(breadcrumb), + "utf8", + ); + const warn = vi.fn(); + try { + expect(recoverBrainLoopWatchdogBreadcrumb({ runtimeDir, warn })).toEqual(breadcrumb); + expect(warn).toHaveBeenCalledWith("brain.recovered_from_wedge", breadcrumb); + expect(fs.existsSync(path.join(runtimeDir, BRAIN_LOOP_WATCHDOG_BREADCRUMB_FILE))).toBe(false); + expect(JSON.parse( + fs.readFileSync(path.join(runtimeDir, BRAIN_LOOP_WATCHDOG_LAST_WEDGE_FILE), "utf8"), + )).toEqual(breadcrumb); + expect(readBrainLoopWatchdogLastWedge(runtimeDir)).toEqual(breadcrumb); + } finally { + fs.rmSync(runtimeDir, { recursive: true, force: true }); + } + }); + + it("reports a recovered breadcrumb once before a disabled watchdog returns", () => { + const runtimeDir = fs.mkdtempSync(path.join(os.tmpdir(), "ade-loop-watchdog-capture-")); + const breadcrumb = { + lastCommand: "chat.send", + blockedMs: 18_000, + ts: "2026-07-23T12:00:00.000Z", + }; + fs.writeFileSync( + path.join(runtimeDir, BRAIN_LOOP_WATCHDOG_BREADCRUMB_FILE), + JSON.stringify(breadcrumb), + "utf8", + ); + const onRecovered = vi.fn(); + try { + const stop = startBrainLoopWatchdog({ + runtimeDir, + env: { ADE_DISABLE_LOOP_WATCHDOG: "1" }, + warn: vi.fn(), + onRecovered, + }); + stop(); + expect(onRecovered).toHaveBeenCalledTimes(1); + expect(onRecovered).toHaveBeenCalledWith(breadcrumb); + } finally { + fs.rmSync(runtimeDir, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/ade-cli/src/services/runtime/brainLoopWatchdog.ts b/apps/ade-cli/src/services/runtime/brainLoopWatchdog.ts new file mode 100644 index 000000000..ecfc8ae58 --- /dev/null +++ b/apps/ade-cli/src/services/runtime/brainLoopWatchdog.ts @@ -0,0 +1,299 @@ +import fs from "node:fs"; +import path from "node:path"; +import { Worker } from "node:worker_threads"; + +export const DEFAULT_BRAIN_LOOP_WATCHDOG_MS = 15_000; +export const BRAIN_LOOP_WATCHDOG_HEARTBEAT_MS = 1_000; +export const BRAIN_LOOP_WATCHDOG_BREADCRUMB_FILE = "event-loop-wedge.json"; +export const BRAIN_LOOP_WATCHDOG_LAST_WEDGE_FILE = "last-wedge.json"; + +export type BrainLoopWatchdogBreadcrumb = { + lastCommand: string; + blockedMs: number; + ts: string; +}; + +export type BrainLoopWatchdogEvaluation = { + blocked: boolean; + blockedMs: number; + slept: boolean; +}; + +export function evaluateBrainLoopWatchdog(args: { + nowWallMs: number; + nowMonotonicMs: number; + lastHeartbeatWallMs: number; + lastHeartbeatMonotonicMs: number; + previousCheckWallMs: number; + previousCheckMonotonicMs: number; + thresholdMs: number; + checkIntervalMs?: number; +}): BrainLoopWatchdogEvaluation { + const wallSinceHeartbeat = Math.max(0, args.nowWallMs - args.lastHeartbeatWallMs); + const monotonicSinceHeartbeat = Math.max( + 0, + args.nowMonotonicMs - args.lastHeartbeatMonotonicMs, + ); + const blockedMs = Math.floor(Math.min(wallSinceHeartbeat, monotonicSinceHeartbeat)); + const wallSinceCheck = Math.max(0, args.nowWallMs - args.previousCheckWallMs); + const monotonicSinceCheck = Math.max( + 0, + args.nowMonotonicMs - args.previousCheckMonotonicMs, + ); + const sleepGapFloorMs = Math.max( + args.thresholdMs, + (args.checkIntervalMs ?? BRAIN_LOOP_WATCHDOG_HEARTBEAT_MS) * 4, + ); + const checkGapDeltaMs = Math.abs(wallSinceCheck - monotonicSinceCheck); + const slept = wallSinceCheck > sleepGapFloorMs + && monotonicSinceCheck > sleepGapFloorMs + && checkGapDeltaMs <= Math.max(2_000, Math.max(wallSinceCheck, monotonicSinceCheck) * 0.25); + return { + blocked: !slept && blockedMs > args.thresholdMs, + blockedMs, + slept, + }; +} + +function parseWatchdogThresholdMs(raw: string | undefined): number { + const parsed = Number.parseInt(raw?.trim() ?? "", 10); + if (!Number.isFinite(parsed)) return DEFAULT_BRAIN_LOOP_WATCHDOG_MS; + return Math.max(BRAIN_LOOP_WATCHDOG_HEARTBEAT_MS, parsed); +} + +function readBreadcrumb(filePath: string): BrainLoopWatchdogBreadcrumb | null { + try { + const parsed = JSON.parse(fs.readFileSync(filePath, "utf8")) as Partial; + if ( + typeof parsed.lastCommand !== "string" + || typeof parsed.blockedMs !== "number" + || !Number.isFinite(parsed.blockedMs) + || typeof parsed.ts !== "string" + ) { + return null; + } + return { + lastCommand: parsed.lastCommand, + blockedMs: Math.max(0, Math.floor(parsed.blockedMs)), + ts: parsed.ts, + }; + } catch { + return null; + } +} + +export function readBrainLoopWatchdogLastWedge( + runtimeDir: string, +): BrainLoopWatchdogBreadcrumb | null { + return readBreadcrumb(path.join(runtimeDir, BRAIN_LOOP_WATCHDOG_LAST_WEDGE_FILE)); +} + +export function recoverBrainLoopWatchdogBreadcrumb(args: { + runtimeDir: string; + warn?: (event: string, meta: Record) => void; +}): BrainLoopWatchdogBreadcrumb | null { + const breadcrumbPath = path.join(args.runtimeDir, BRAIN_LOOP_WATCHDOG_BREADCRUMB_FILE); + if (!fs.existsSync(breadcrumbPath)) return null; + const breadcrumb = readBreadcrumb(breadcrumbPath) ?? { + lastCommand: "unknown", + blockedMs: 0, + ts: new Date().toISOString(), + }; + fs.mkdirSync(args.runtimeDir, { recursive: true, mode: 0o700 }); + const lastWedgePath = path.join(args.runtimeDir, BRAIN_LOOP_WATCHDOG_LAST_WEDGE_FILE); + try { + if (process.platform === "win32" && fs.existsSync(lastWedgePath)) { + fs.unlinkSync(lastWedgePath); + } + fs.renameSync(breadcrumbPath, lastWedgePath); + } catch { + fs.copyFileSync(breadcrumbPath, lastWedgePath); + fs.unlinkSync(breadcrumbPath); + } + if (args.warn) { + args.warn("brain.recovered_from_wedge", breadcrumb); + } else { + process.stderr.write(`brain.recovered_from_wedge ${JSON.stringify(breadcrumb)}\n`); + } + return breadcrumb; +} + +let nextCommandToken = 0; +const activeCommands = new Map(); + +function currentCommandName(): string { + let current = "idle"; + for (const action of activeCommands.values()) current = action; + return current; +} + +export function trackBrainLoopWatchdogCommand(action: string): () => void { + const token = ++nextCommandToken; + activeCommands.set(token, action.trim() || "unknown"); + return () => { + activeCommands.delete(token); + }; +} + +function buildWorkerSource(): string { + return String.raw` + const fs = require("node:fs"); + const path = require("node:path"); + const { parentPort, workerData } = require("node:worker_threads"); + + const monotonicMs = () => Number(process.hrtime.bigint() / 1000000n); + let lastHeartbeatWallMs = null; + let lastHeartbeatMonotonicMs = null; + let previousCheckWallMs = Date.now(); + let previousCheckMonotonicMs = monotonicMs(); + let lastCommand = "idle"; + + const writeBreadcrumbAndKill = (blockedMs) => { + const breadcrumb = { + lastCommand, + blockedMs: Math.max(0, Math.floor(blockedMs)), + ts: new Date().toISOString(), + }; + fs.mkdirSync(workerData.runtimeDir, { recursive: true, mode: 0o700 }); + const breadcrumbPath = path.join(workerData.runtimeDir, workerData.breadcrumbFile); + const tempPath = breadcrumbPath + "." + process.pid + "." + Date.now() + ".tmp"; + fs.writeFileSync(tempPath, JSON.stringify(breadcrumb) + "\n", { + encoding: "utf8", + mode: 0o600, + }); + try { + if (process.platform === "win32" && fs.existsSync(breadcrumbPath)) { + fs.unlinkSync(breadcrumbPath); + } + fs.renameSync(tempPath, breadcrumbPath); + } catch (error) { + try { fs.unlinkSync(tempPath); } catch {} + throw error; + } + fs.writeSync(2, "brain.event_loop_blocked " + JSON.stringify(breadcrumb) + "\n"); + process.kill(process.pid, "SIGKILL"); + }; + + parentPort.on("message", (message) => { + if (!message || message.type !== "heartbeat") return; + lastHeartbeatWallMs = Date.now(); + lastHeartbeatMonotonicMs = monotonicMs(); + if (typeof message.lastCommand === "string" && message.lastCommand.length > 0) { + lastCommand = message.lastCommand; + } + }); + + setInterval(() => { + const nowWallMs = Date.now(); + const nowMonotonicMs = monotonicMs(); + if (lastHeartbeatWallMs == null || lastHeartbeatMonotonicMs == null) { + previousCheckWallMs = nowWallMs; + previousCheckMonotonicMs = nowMonotonicMs; + return; + } + + const wallSinceCheck = Math.max(0, nowWallMs - previousCheckWallMs); + const monotonicSinceCheck = Math.max(0, nowMonotonicMs - previousCheckMonotonicMs); + const sleepGapFloorMs = Math.max( + workerData.thresholdMs, + workerData.checkIntervalMs * 4, + ); + const checkGapDeltaMs = Math.abs(wallSinceCheck - monotonicSinceCheck); + const slept = wallSinceCheck > sleepGapFloorMs + && monotonicSinceCheck > sleepGapFloorMs + && checkGapDeltaMs <= Math.max( + 2000, + Math.max(wallSinceCheck, monotonicSinceCheck) * 0.25, + ); + previousCheckWallMs = nowWallMs; + previousCheckMonotonicMs = nowMonotonicMs; + if (slept) { + lastHeartbeatWallMs = nowWallMs; + lastHeartbeatMonotonicMs = nowMonotonicMs; + return; + } + + const blockedMs = Math.floor(Math.min( + Math.max(0, nowWallMs - lastHeartbeatWallMs), + Math.max(0, nowMonotonicMs - lastHeartbeatMonotonicMs), + )); + if (blockedMs > workerData.thresholdMs) { + writeBreadcrumbAndKill(blockedMs); + } + }, workerData.checkIntervalMs); + `; +} + +export function startBrainLoopWatchdog(args: { + runtimeDir: string; + env?: NodeJS.ProcessEnv; + forceInTests?: boolean; + warn?: (event: string, meta: Record) => void; + onRecovered?: (breadcrumb: BrainLoopWatchdogBreadcrumb) => void; +}): () => void { + const recovered = recoverBrainLoopWatchdogBreadcrumb({ + runtimeDir: args.runtimeDir, + warn: args.warn, + }); + if (recovered) { + try { + args.onRecovered?.(recovered); + } catch { + // Recovery telemetry is best-effort and must not block watchdog startup. + } + } + const env = args.env ?? process.env; + if (env.ADE_DISABLE_LOOP_WATCHDOG === "1") return () => {}; + if (!args.forceInTests && (env.VITEST || env.NODE_ENV === "test")) return () => {}; + + const thresholdMs = parseWatchdogThresholdMs(env.ADE_LOOP_WATCHDOG_MS); + let disposed = false; + let worker: Worker; + try { + worker = new Worker(buildWorkerSource(), { + eval: true, + workerData: { + runtimeDir: args.runtimeDir, + thresholdMs, + checkIntervalMs: BRAIN_LOOP_WATCHDOG_HEARTBEAT_MS, + breadcrumbFile: BRAIN_LOOP_WATCHDOG_BREADCRUMB_FILE, + }, + }); + } catch (error) { + args.warn?.("brain.loop_watchdog_start_failed", { + error: error instanceof Error ? error.message : String(error), + }); + return () => {}; + } + worker.unref(); + + const heartbeat = (): void => { + try { + worker.postMessage({ + type: "heartbeat", + lastCommand: currentCommandName(), + }); + } catch { + // The worker's error/exit handlers report a failed watchdog once. + } + }; + heartbeat(); + const interval = setInterval(heartbeat, BRAIN_LOOP_WATCHDOG_HEARTBEAT_MS); + interval.unref?.(); + + worker.on("error", (error) => { + if (disposed) return; + args.warn?.("brain.loop_watchdog_failed", { error: error.message }); + }); + worker.on("exit", (code) => { + if (disposed || code === 0) return; + args.warn?.("brain.loop_watchdog_exited", { code }); + }); + + return () => { + if (disposed) return; + disposed = true; + clearInterval(interval); + void worker.terminate(); + }; +} diff --git a/apps/ade-cli/src/services/runtime/runtimeBuildIdentity.ts b/apps/ade-cli/src/services/runtime/runtimeBuildIdentity.ts new file mode 100644 index 000000000..0b12c55e9 --- /dev/null +++ b/apps/ade-cli/src/services/runtime/runtimeBuildIdentity.ts @@ -0,0 +1,22 @@ +import { createHash } from "node:crypto"; +import fs from "node:fs"; + +export function computeRuntimeBuildHash(filePath: string): string | null { + try { + return createHash("sha256").update(fs.readFileSync(filePath)).digest("hex"); + } catch { + return null; + } +} + +export async function computeRuntimeBuildHashAsync( + filePath: string, +): Promise { + try { + return createHash("sha256") + .update(await fs.promises.readFile(filePath)) + .digest("hex"); + } catch { + return null; + } +} diff --git a/apps/ade-cli/src/services/sync/sharedSyncListener.ts b/apps/ade-cli/src/services/sync/sharedSyncListener.ts index 0c2013ede..6d747b762 100644 --- a/apps/ade-cli/src/services/sync/sharedSyncListener.ts +++ b/apps/ade-cli/src/services/sync/sharedSyncListener.ts @@ -1,8 +1,9 @@ import http from "node:http"; +import { spawnSync } from "node:child_process"; import { randomBytes, timingSafeEqual } from "node:crypto"; import { WebSocketServer, WebSocket, type RawData } from "ws"; import type { SyncPeerMetadata } from "../../../../desktop/src/shared/types"; -import { DEFAULT_SYNC_HOST_PORT } from "./syncProtocol"; +import { DEFAULT_SYNC_HOST_PORT, SYNC_HOST_MAX_PORT } from "./syncProtocol"; import type { RelayAuthorizationSnapshot } from "./relayAuthorization"; import { assertAdeLoopbackListener, @@ -13,6 +14,14 @@ import { type SyncLoopbackProbeResult, type SyncLoopbackValidationStatus, } from "./syncLoopbackProbe"; +import { + isStaleChannelServeCommandLine, + resolveAdeServeCliScriptPath, + resolveAdeServeCommand, + terminatePidGracefullyAsync, +} from "../../serviceManager/common"; +import { getRuntimeServiceMainPid } from "../../serviceManager"; +import { resolveMachineAdeLayout } from "../projects/machineLayout"; // Bind the sync host on all interfaces by default so phones on the same // wifi/LAN can reach it without Tailscale. 0.0.0.0 is a superset of loopback, @@ -56,6 +65,16 @@ type SharedSyncListenerLogger = { warn?: (message: string, fields?: Record) => void; }; +export type SyncListenerPortDiagnosis = { + port: number; + holders: Array<{ + pid: number; + command: string | null; + /** Stable process birth identity used to guard against PID reuse. */ + startTime: string | null; + }>; +}; + export type SharedSyncListenerConnection = { ws: WebSocket; remoteAddress: string | null; @@ -241,12 +260,36 @@ export function createSharedSyncListener(options: { maxPayloadBytes?: number; parkedPeerGraceMs?: number; loopbackProbe?: (port: number, expectedNonce: string) => Promise; + inspectPort?: (port: number) => SyncListenerPortDiagnosis | Promise; + activeServicePid?: () => number | null; + terminatePid?: (pid: number) => Promise; } = {}): SharedSyncListener { const logger = options.logger ?? {}; const bindHost = options.bindHost ?? SYNC_HOST_BIND_HOST; const maxPayloadBytes = options.maxPayloadBytes ?? SYNC_HOST_MAX_PAYLOAD_BYTES; const parkedPeerGraceMs = Math.max(50, Math.floor(options.parkedPeerGraceMs ?? DEFAULT_PARKED_PEER_GRACE_MS)); const loopbackProbe = options.loopbackProbe ?? probeAdeLoopbackListener; + const inspectPort = options.inspectPort ?? inspectSyncListenerPort; + const activeServicePid = options.activeServicePid ?? getRuntimeServiceMainPid; + const terminatePid = options.terminatePid ?? ((pid) => terminatePidGracefullyAsync(pid)); + const serviceCommand = resolveAdeServeCommand(); + const staleServeMatch = { + cliScriptPath: resolveAdeServeCliScriptPath(serviceCommand), + primarySocketPath: resolveMachineAdeLayout().socketPath, + }; + const findStaleHolder = ( + diagnosis: SyncListenerPortDiagnosis, + ): SyncListenerPortDiagnosis["holders"][number] | null => { + const excludedPids = new Set([ + process.pid, + activeServicePid() ?? -1, + ]); + return diagnosis.holders.find((holder) => + !excludedPids.has(holder.pid) + && holder.command != null + && isStaleChannelServeCommandLine(holder.command, staleServeMatch), + ) ?? null; + }; // One identity per listener instance. Every candidate bind for this // instance emits the same nonce, and only in-process validators receive the // expected value they must compare against. @@ -434,7 +477,22 @@ export function createSharedSyncListener(options: { // (each re-bind resolves a different port), so a shadow on one ephemeral // port does not short-circuit the remaining fresh-port attempts. const shadowedPorts = new Set(); - for (const attemptedPort of attemptPlan) { + const zombieReapedPorts = new Set(); + const portDiagnoses = new Map(); + const diagnosePort = async ( + port: number, + refresh = false, + ): Promise => { + if (!refresh) { + const cached = portDiagnoses.get(port); + if (cached) return cached; + } + const diagnosis = await inspectPort(port); + portDiagnoses.set(port, diagnosis); + return diagnosis; + }; + for (let attemptIndex = 0; attemptIndex < attemptPlan.length; attemptIndex += 1) { + const attemptedPort = attemptPlan[attemptIndex]!; if (closed) throw new Error("The shared sync listener has been closed."); if (attemptedPort !== 0 && shadowedPorts.has(attemptedPort)) continue; // The retry delay lets a dying listener free a FIXED port; an ephemeral @@ -527,6 +585,36 @@ export function createSharedSyncListener(options: { } const retryable = isRetryableListenerBindError(error) && (attemptedPort !== 0 || isLoopbackShadowedError(error)); + if ( + (error as NodeJS.ErrnoException | null | undefined)?.code === "EADDRINUSE" + && attemptedPort >= DEFAULT_SYNC_HOST_PORT + && attemptedPort <= SYNC_HOST_MAX_PORT + && !zombieReapedPorts.has(attemptedPort) + ) { + const diagnosis = await diagnosePort(attemptedPort); + const staleHolder = findStaleHolder(diagnosis); + if (staleHolder) { + // Reconfirm ownership immediately before signaling. The original + // holder may have exited and its pid may have been reused while + // the first lsof/ps diagnosis was being assembled. + const confirmedHolder = findStaleHolder(await diagnosePort(attemptedPort, true)); + if ( + staleHolder.startTime != null + && confirmedHolder?.pid === staleHolder.pid + && confirmedHolder.startTime === staleHolder.startTime + ) { + zombieReapedPorts.add(attemptedPort); + await terminatePid(staleHolder.pid); + logger.info?.("sync_listener.zombie_reaped", { + port: attemptedPort, + pid: staleHolder.pid, + }); + // Non-preferred candidates occur only once in the normal plan. + // Insert exactly one immediate retry for the newly-freed port. + attemptPlan.splice(attemptIndex + 1, 0, attemptedPort); + } + } + } logger.warn?.( isLoopbackShadowedError(error) ? "sync_listener.loopback_shadowed" @@ -540,6 +628,16 @@ export function createSharedSyncListener(options: { if (!retryable) throw error; } } + const candidateDiagnosis = await Promise.all( + [...new Set(candidates)] + .filter((port) => port > 0) + .slice(0, 5) + .map((port) => diagnosePort(port)), + ); + logger.warn?.("sync_listener.bind_exhausted", { + candidates: candidateDiagnosis, + error: lastError instanceof Error ? lastError.message : String(lastError), + }); throw lastError instanceof Error ? lastError : new Error("Unable to bind the shared sync listener."); @@ -697,3 +795,45 @@ export function createSharedSyncListener(options: { }, }; } + +export function inspectSyncListenerPort(port: number): SyncListenerPortDiagnosis { + const lsof = spawnSync( + "lsof", + ["-nP", `-iTCP:${port}`, "-sTCP:LISTEN", "-Fp"], + { encoding: "utf8", timeout: 200 }, + ); + if (lsof.status !== 0) return { port, holders: [] }; + const pids = [...new Set( + String(lsof.stdout ?? "") + .split(/\r?\n/) + .filter((line) => /^p\d+$/.test(line)) + .map((line) => Number(line.slice(1))) + .filter((pid) => Number.isFinite(pid) && pid > 0), + )]; + return { + port, + holders: pids.map((pid) => { + const commandResult = spawnSync( + "ps", + ["-p", String(pid), "-o", "command="], + { encoding: "utf8", timeout: 200 }, + ); + const startResult = spawnSync( + "ps", + ["-p", String(pid), "-o", "lstart="], + { encoding: "utf8", timeout: 200 }, + ); + const command = commandResult.status === 0 + ? String(commandResult.stdout ?? "").trim() + : ""; + const startTime = startResult.status === 0 + ? String(startResult.stdout ?? "").trim() + : ""; + return { + pid, + command: command || null, + startTime: startTime || null, + }; + }), + }; +} diff --git a/apps/ade-cli/src/services/sync/syncHostService.test.ts b/apps/ade-cli/src/services/sync/syncHostService.test.ts index f5df6d1c0..ec619ee13 100644 --- a/apps/ade-cli/src/services/sync/syncHostService.test.ts +++ b/apps/ade-cli/src/services/sync/syncHostService.test.ts @@ -14,6 +14,7 @@ import type { PersonalChatScopeContract, SyncChangesetAckPayload, SyncChangesetBatchPayload, + SyncCommandPayload, SyncInvalidationBatchPayload, SyncMobileProjectSummary, SyncPeerMetadata, @@ -6023,16 +6024,19 @@ describe("paired-client product analytics consent", () => { ok: true, result: { accepted: true, reason: "accepted" }, }); - expect(execute).toHaveBeenCalledWith(expect.objectContaining({ - action: "analytics.capture", - projectId: "project-1", - args: expect.objectContaining({ - event: "ade_project_opened", - surface: "web", + expect(execute).toHaveBeenCalledWith( + expect.objectContaining({ + action: "analytics.capture", projectId: "project-1", - dedupeKey: "web_project_opened:project-1", + args: expect.objectContaining({ + event: "ade_project_opened", + surface: "web", + projectId: "project-1", + dedupeKey: "web_project_opened:project-1", + }), }), - })); + expect.objectContaining({ signal: expect.any(AbortSignal) }), + ); execute.mockClear(); const foreignProjectCapture = await sendCommand("analytics.capture", { @@ -6069,7 +6073,10 @@ describe("paired-client product analytics consent", () => { await expect(sendCommand("lanes.create", { name: "private-lane" }, "mutation-while-disabled")) .resolves.toMatchObject({ payload: { ok: true, result: { ok: true } } }); - expect(execute).toHaveBeenCalledWith(expect.objectContaining({ action: "lanes.create" })); + expect(execute).toHaveBeenCalledWith( + expect.objectContaining({ action: "lanes.create" }), + expect.objectContaining({ signal: expect.any(AbortSignal) }), + ); const suppressedUsageInsert = dbRun.mock.calls.find(([sql]) => String(sql).includes("insert into usage_events")); expect(suppressedUsageInsert?.[1]).toEqual([ expect.any(String), @@ -7504,6 +7511,159 @@ describe("sync host reliability guards", () => { } }); + it("aborts an opt-in remote command executor when the message timeout fires", async () => { + const { projectRoot, cleanup } = createTempProjectRoot(); + const logger = createDiscoveryLogger(); + let markAborted!: () => void; + const aborted = new Promise((resolve) => { markAborted = resolve; }); + const execute = vi.fn(( + _payload: SyncCommandPayload, + context?: { signal?: AbortSignal }, + ): Promise => new Promise((_resolve, reject) => { + const onAbort = () => { + markAborted(); + reject(context?.signal?.reason ?? new Error("aborted")); + }; + if (context?.signal?.aborted) onAbort(); + else context?.signal?.addEventListener("abort", onAbort, { once: true }); + })); + const host = createReliabilityHost(projectRoot, { + logger, + messageTimeoutMs: 100, + remoteCommandExecutor: { execute }, + }); + let peer: Awaited> | null = null; + try { + peer = await connectPeer( + await host.waitUntilListening(), + host.getBootstrapToken(), + "ios-command-timeout-abort", + ); + peer.ws.send(encodeSyncEnvelope({ + type: "command", + requestId: "timeout-abort", + projectId: "project-2", + payload: { + commandId: "timeout-abort", + action: "lanes.list", + projectId: "project-2", + args: {}, + }, + })); + await waitForEnvelope(peer.envelopes, "command_ack", "timeout-abort"); + await aborted; + expect(execute.mock.calls[0]?.[1]?.signal?.aborted).toBe(true); + expect(logger.warn).toHaveBeenCalledWith("sync_host.command_timed_out", { + action: "lanes.list", + durationMs: expect.any(Number), + peerKind: "mobile", + timedOut: true, + }); + } finally { + peer?.ws.close(); + await host.dispose(); + cleanup(); + } + }); + + it("aborts an opt-in remote command executor when its peer closes", async () => { + const { projectRoot, cleanup } = createTempProjectRoot(); + let markAborted!: () => void; + const aborted = new Promise((resolve) => { markAborted = resolve; }); + const execute = vi.fn(( + _payload: SyncCommandPayload, + context?: { signal?: AbortSignal }, + ): Promise => new Promise((_resolve, reject) => { + const onAbort = () => { + markAborted(); + reject(context?.signal?.reason ?? new Error("aborted")); + }; + if (context?.signal?.aborted) onAbort(); + else context?.signal?.addEventListener("abort", onAbort, { once: true }); + })); + const host = createReliabilityHost(projectRoot, { + messageTimeoutMs: 5_000, + remoteCommandExecutor: { execute }, + }); + let peer: Awaited> | null = null; + try { + peer = await connectPeer( + await host.waitUntilListening(), + host.getBootstrapToken(), + "ios-command-close-abort", + ); + peer.ws.send(encodeSyncEnvelope({ + type: "command", + requestId: "close-abort", + projectId: "project-2", + payload: { + commandId: "close-abort", + action: "lanes.list", + projectId: "project-2", + args: {}, + }, + })); + await waitForEnvelope(peer.envelopes, "command_ack", "close-abort"); + peer.ws.close(); + await aborted; + expect(execute).toHaveBeenCalledTimes(1); + } finally { + peer?.ws.close(); + await host.dispose(); + cleanup(); + } + }); + + it("warns when a completed command exceeds the slow-command threshold", async () => { + const { projectRoot, cleanup } = createTempProjectRoot(); + const logger = createDiscoveryLogger(); + const execute = vi.fn(async () => { + await new Promise((resolve) => setTimeout(resolve, 25)); + return { ok: true }; + }); + const host = createReliabilityHost(projectRoot, { + logger, + slowCommandThresholdMs: 10, + remoteCommandExecutor: { execute }, + }); + let peer: Awaited> | null = null; + try { + peer = await connectPeer( + await host.waitUntilListening(), + host.getBootstrapToken(), + "desktop-slow-command", + { platform: "macOS", deviceType: "desktop" }, + ); + peer.ws.send(encodeSyncEnvelope({ + type: "command", + requestId: "slow-command", + projectId: "project-2", + payload: { + commandId: "slow-command", + action: "lanes.list", + projectId: "project-2", + args: {}, + }, + })); + await waitForEnvelope(peer.envelopes, "command_result", "slow-command"); + expect(logger.warn).toHaveBeenCalledWith("sync_host.command_slow", { + action: "lanes.list", + durationMs: expect.any(Number), + peerKind: "desktop", + }); + const slowFields = logger.warn.mock.calls.find( + ([event]) => event === "sync_host.command_slow", + )?.[1]; + expect(slowFields?.durationMs).toBeGreaterThanOrEqual(10); + expect(slowFields).not.toHaveProperty("args"); + expect(slowFields).not.toHaveProperty("payload"); + } finally { + peer?.ws.close(); + await host.dispose(); + cleanup(); + } + }); + it("rejects oversized artifact reads with a clear file response", async () => { const { projectRoot, cleanup } = createTempProjectRoot(); const artifactPath = path.join(projectRoot, ".ade", "artifacts", "large.bin"); diff --git a/apps/ade-cli/src/services/sync/syncHostService.ts b/apps/ade-cli/src/services/sync/syncHostService.ts index da241c521..64bf8925a 100644 --- a/apps/ade-cli/src/services/sync/syncHostService.ts +++ b/apps/ade-cli/src/services/sync/syncHostService.ts @@ -172,6 +172,7 @@ import { createSyncRemoteCommandService, type SyncRemoteCommandService } from ". import { prepareProductAnalyticsRemoteCommand } from "./productAnalyticsRemoteCommand"; import { buildPairingConnectInfo } from "./syncPairingConnectInfo"; import type { PushPublisherService } from "../push/pushPublisherService"; +import { trackBrainLoopWatchdogCommand } from "../runtime/brainLoopWatchdog"; import { buildChangesetBatchPayload, DEFAULT_MAX_CHANGESET_BATCH_BYTES, @@ -299,6 +300,7 @@ const PEER_BACKPRESSURE_BYTES = 4 * 1024 * 1024; const REQUIRED_SEND_MAX_BUFFERED_BYTES = 16 * 1024 * 1024; const SEND_AND_WAIT_TIMEOUT_MS = 15_000; const DEFAULT_SYNC_MESSAGE_TIMEOUT_MS = 60_000; +const DEFAULT_SYNC_SLOW_COMMAND_MS = 5_000; const MAX_SYNC_ARTIFACT_BYTES = 8 * 1024 * 1024; export const SYNC_HOST_CHAT_ACTIVE_BACKGROUND_BACKPRESSURE_BYTES = 512 * 1024; export const SYNC_HOST_CHAT_ACTIVE_CHANGESET_BATCH_BYTES = 64 * 1024; @@ -560,6 +562,7 @@ type PeerState = { queuedMessageCount: number; terminalInputQueue: Promise; pendingTerminalOwnershipChanges: number; + inFlightOperationControllers: Set; /** Local consent for this browser/phone; never mutates machine-wide consent. */ productAnalyticsEnabled: boolean; }; @@ -820,6 +823,8 @@ type SyncHostServiceArgs = { pollIntervalMs?: number; authTimeoutMs?: number; messageTimeoutMs?: number; + /** Test seam; production warns for commands taking at least five seconds. */ + slowCommandThresholdMs?: number; brainStatusIntervalMs?: number; compressionThresholdBytes?: number; deviceRegistryService?: DeviceRegistryService; @@ -853,6 +858,41 @@ function sanitizeRemoteAddress(remoteAddress: string | null | undefined): string return value.startsWith("::ffff:") ? value.slice("::ffff:".length) : value; } +function syncOperationAbortError(signal: AbortSignal): Error { + if (signal.reason instanceof Error) return signal.reason; + const error = new Error( + typeof signal.reason === "string" && signal.reason.trim() + ? signal.reason + : "Sync operation aborted.", + ); + error.name = "AbortError"; + return error; +} + +async function runWithSyncOperationSignal( + run: () => Promise | T, + signal: AbortSignal, +): Promise { + if (signal.aborted) throw syncOperationAbortError(signal); + return await new Promise((resolve, reject) => { + let settled = false; + const settle = (complete: () => void): void => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", onAbort); + complete(); + }; + const onAbort = (): void => settle(() => reject(syncOperationAbortError(signal))); + signal.addEventListener("abort", onAbort, { once: true }); + Promise.resolve() + .then(run) + .then( + (value) => settle(() => resolve(value)), + (error) => settle(() => reject(error)), + ); + }); +} + function ensureBootstrapToken(filePath: string): string { fs.mkdirSync(path.dirname(filePath), { recursive: true }); if (!fs.existsSync(filePath)) { @@ -2049,6 +2089,10 @@ export function createSyncHostService(args: SyncHostServiceArgs) { const brainStatusIntervalMs = Math.max(1_000, Math.floor(args.brainStatusIntervalMs ?? DEFAULT_BRAIN_STATUS_INTERVAL_MS)); const authTimeoutMs = Math.max(1_000, Math.floor(args.authTimeoutMs ?? SYNC_HOST_AUTH_TIMEOUT_MS)); const messageTimeoutMs = Math.max(100, Math.floor(args.messageTimeoutMs ?? DEFAULT_SYNC_MESSAGE_TIMEOUT_MS)); + const slowCommandThresholdMs = Math.max( + 1, + Math.floor(args.slowCommandThresholdMs ?? DEFAULT_SYNC_SLOW_COMMAND_MS), + ); const compressionThresholdBytes = Math.max(256, Math.floor(args.compressionThresholdBytes ?? DEFAULT_SYNC_COMPRESSION_THRESHOLD_BYTES)); const maxChangesetBatchBytes = DEFAULT_MAX_CHANGESET_BATCH_BYTES; const maxChangesetBatchRows = DEFAULT_MAX_CHANGESET_BATCH_ROWS; @@ -2934,6 +2978,13 @@ export function createSyncHostService(args: SyncHostServiceArgs) { peer.authTimeout = null; } + function abortPeerOperations(peer: PeerState, reason: string): void { + for (const controller of peer.inFlightOperationControllers) { + if (!controller.signal.aborted) controller.abort(new Error(reason)); + } + peer.inFlightOperationControllers.clear(); + } + function isPeerLifecycleCurrent(peer: PeerState, generation: number): boolean { return !disposed && peers.has(peer) @@ -3187,6 +3238,7 @@ export function createSyncHostService(args: SyncHostServiceArgs) { queuedMessageCount: 0, terminalInputQueue: Promise.resolve(), pendingTerminalOwnershipChanges: 0, + inFlightOperationControllers: new Set(), // Paired clients own their local preference. Fail closed on every new // connection until that client explicitly reasserts consent, so an // opted-out reconnect cannot leak an exportable first mutation. @@ -3245,6 +3297,7 @@ export function createSyncHostService(args: SyncHostServiceArgs) { }); ws.on("close", (code, reason) => { peer.lifecycleGeneration += 1; + abortPeerOperations(peer, "Sync peer closed."); peer.pendingTerminalSnapshots.clear(); clearPeerAuthTimeout(peer); releaseConnectionAttemptWinner(peer); @@ -5622,7 +5675,12 @@ export function createSyncHostService(args: SyncHostServiceArgs) { } } - async function handleCommand(peer: PeerState, requestId: string | null, payload: SyncCommandPayload): Promise { + async function handleCommand( + peer: PeerState, + requestId: string | null, + payload: SyncCommandPayload, + signal: AbortSignal, + ): Promise { const commandId = toOptionalString(payload.commandId) ?? requestId ?? `cmd-${Date.now()}`; const requestedProjectId = toOptionalString(payload.projectId); const hostProjectId = toOptionalString(args.projectId); @@ -5828,9 +5886,24 @@ export function createSyncHostService(args: SyncHostServiceArgs) { const routedPayload = matchesHostProject && hostProjectId ? { ...surfaceBoundPayload, projectId: hostProjectId } : surfaceBoundPayload; - const created = preparedAnalytics.captureDisabled - ? { accepted: false, reason: "disabled" } - : await executor.execute(routedPayload); + const executionStartedAtMs = Date.now(); + const stopTrackingCommand = trackBrainLoopWatchdogCommand(payload.action); + let created: unknown; + try { + created = preparedAnalytics.captureDisabled + ? { accepted: false, reason: "disabled" } + : await executor.execute(routedPayload, { signal }); + } finally { + stopTrackingCommand(); + const durationMs = Math.max(0, Date.now() - executionStartedAtMs); + if (durationMs >= slowCommandThresholdMs) { + args.logger.warn("sync_host.command_slow", { + action: payload.action, + durationMs, + peerKind: surface, + }); + } + } if ( matchesHostProject && !payload.action.startsWith("analytics.") @@ -6183,15 +6256,35 @@ export function createSyncHostService(args: SyncHostServiceArgs) { function handleMessageWithTimeout(peer: PeerState, envelope: ParsedSyncEnvelope): Promise { const operationGeneration = peer.lifecycleGeneration; + const operationController = new AbortController(); + peer.inFlightOperationControllers.add(operationController); + const operationStartedAtMs = Date.now(); let timer: ReturnType | null = null; const timeout = new Promise((_resolve, reject) => { timer = setTimeout(() => { + const durationMs = Math.max(messageTimeoutMs, Date.now() - operationStartedAtMs); + if (envelope.type === "command") { + const action = toOptionalString( + (envelope.payload as Partial | null)?.action, + ) ?? "unknown"; + args.logger.warn("sync_host.command_timed_out", { + action, + durationMs, + peerKind: usageClientSurfaceFromPeer( + peer.metadata?.deviceType, + peer.metadata?.platform, + ), + timedOut: true, + }); + } + operationController.abort( + new Error(`Timed out handling sync message ${envelope.type} after ${messageTimeoutMs}ms.`), + ); if (peer.lifecycleGeneration === operationGeneration) { - // Promise.race does not cancel the handler. Invalidate its operation - // generation and close the peer so lifecycle-guarded auth, pairing, - // and host-state work cannot resume. Accepted commands intentionally - // keep running: their commandId ledger must capture the eventual - // result so a replacement peer can retry without executing twice. + // Invalidate the operation generation and close the peer. Known-slow + // handlers observe operationController.signal and stop waiting; + // handlers that cannot safely cancel keep the existing exactly-once + // ledger behavior and may still publish an eventual replay result. peer.lifecycleGeneration += 1; try { peer.ws.close(4002, "Sync message timed out"); @@ -6203,12 +6296,20 @@ export function createSyncHostService(args: SyncHostServiceArgs) { }, messageTimeoutMs); timer.unref?.(); }); - return Promise.race([handleMessage(peer, envelope), timeout]).finally(() => { + return Promise.race([ + handleMessage(peer, envelope, operationController.signal), + timeout, + ]).finally(() => { if (timer) clearTimeout(timer); + peer.inFlightOperationControllers.delete(operationController); }); } - async function handleMessage(peer: PeerState, envelope: ParsedSyncEnvelope): Promise { + async function handleMessage( + peer: PeerState, + envelope: ParsedSyncEnvelope, + signal: AbortSignal, + ): Promise { const lifecycleGeneration = peer.lifecycleGeneration; if (!isPeerLifecycleCurrent(peer, lifecycleGeneration)) return; const heartbeatAwaitedAt = markPeerMessageSeen(peer); @@ -7154,11 +7255,14 @@ export function createSyncHostService(args: SyncHostServiceArgs) { barrier.captureAttempt += 1; const session = args.sessionService.get(sessionId); const transcriptSnapshot = session - ? await args.ptyService.readTranscriptSnapshot({ - sessionId, - maxBytes, - alignStartToSafeBoundary: true, - }) + ? await runWithSyncOperationSignal( + () => args.ptyService.readTranscriptSnapshot({ + sessionId, + maxBytes, + alignStartToSafeBoundary: true, + }), + signal, + ) : null; if (!isCurrentTerminalSnapshotBarrier(peer, sessionId, barrier, lifecycleGeneration)) break; @@ -7301,12 +7405,15 @@ export function createSyncHostService(args: SyncHostServiceArgs) { Math.min(beforeOffset, transcriptWindow.endOffset), ); const requestedStartOffset = Math.max(transcriptWindow.startOffset, endOffset - pageBytes); - const range = await args.ptyService.readTranscriptRange({ - sessionId, - startOffset: requestedStartOffset, - endOffset, - alignStartToSafeBoundary: true, - }); + const range = await runWithSyncOperationSignal( + () => args.ptyService.readTranscriptRange({ + sessionId, + startOffset: requestedStartOffset, + endOffset, + alignStartToSafeBoundary: true, + }), + signal, + ); if (!range) { sendRequired(peer, "terminal_history", refused, envelope.requestId); break; @@ -7361,7 +7468,10 @@ export function createSyncHostService(args: SyncHostServiceArgs) { // subscription at all, or the periodic pump would stream the ACTIVE // project's transcript for the same session id after the empty ack. const personalTranscriptPath = personalChatRequested - ? await args.personalChatScope?.transcriptPath?.(sessionId).catch(() => null) ?? null + ? await runWithSyncOperationSignal( + () => args.personalChatScope?.transcriptPath?.(sessionId).catch(() => null) ?? null, + signal, + ) : null; const foreignScope = personalChatRequested ? personalTranscriptPath @@ -7403,11 +7513,17 @@ export function createSyncHostService(args: SyncHostServiceArgs) { // derive turn state from the streamed status events instead. const resolveLiveStatusFields = async (): Promise<{ turnActive?: boolean }> => { if (personalChatRequested) { - const turnActive = await args.personalChatScope?.isTurnActive?.(sessionId).catch(() => false); + const turnActive = await runWithSyncOperationSignal( + () => args.personalChatScope?.isTurnActive?.(sessionId).catch(() => false) ?? false, + signal, + ); return typeof turnActive === "boolean" ? { turnActive } : {}; } if (foreignScope.kind !== "local") return {}; - const liveSummary = await args.agentChatService?.getSessionSummary(sessionId).catch(() => null); + const liveSummary = await runWithSyncOperationSignal( + () => args.agentChatService?.getSessionSummary(sessionId).catch(() => null) ?? null, + signal, + ); return liveSummary ? { turnActive: liveSummary.status === "active" } : {}; }; // Replay buffers hold the ACTIVE project's live events — a foreign @@ -7456,7 +7572,10 @@ export function createSyncHostService(args: SyncHostServiceArgs) { let truncated: boolean; let transcriptSize: number; if (foreignTranscriptPath) { - const foreignSnapshot = await readForeignChatSnapshot(foreignTranscriptPath, maxBytes); + const foreignSnapshot = await runWithSyncOperationSignal( + () => readForeignChatSnapshot(foreignTranscriptPath, maxBytes), + signal, + ); events = foreignSnapshot.events; truncated = foreignSnapshot.truncated; transcriptSize = foreignSnapshot.transcriptSize; @@ -7520,7 +7639,7 @@ export function createSyncHostService(args: SyncHostServiceArgs) { ...(!toOptionalString((envelope.payload as SyncCommandPayload | null)?.projectId) && envelope.projectId ? { projectId: envelope.projectId } : {}), - }); + }, signal); break; default: break; @@ -7863,6 +7982,7 @@ export function createSyncHostService(args: SyncHostServiceArgs) { detachSharedListener = null; const snapshots: SyncPeerHandoffSnapshot[] = []; for (const peer of peers) { + abortPeerOperations(peer, "Sync host changed."); pairedChannelService.closePeer(peer.ws, "Sync host changed.", true); clearPeerAuthTimeout(peer); const relayAuthorizationSnapshot = peer.relayAuthorization?.snapshot() ?? null; @@ -7932,6 +8052,7 @@ export function createSyncHostService(args: SyncHostServiceArgs) { await new Promise((resolve) => { const finish = () => resolve(); for (const peer of peers) { + abortPeerOperations(peer, "Sync host stopped."); pairedChannelService.closePeer(peer.ws, "Sync host stopped.", false); clearPeerAuthTimeout(peer); try { diff --git a/apps/ade-cli/src/services/sync/syncHostSingleton.ts b/apps/ade-cli/src/services/sync/syncHostSingleton.ts index d8019a2cb..7fed53171 100644 --- a/apps/ade-cli/src/services/sync/syncHostSingleton.ts +++ b/apps/ade-cli/src/services/sync/syncHostSingleton.ts @@ -3,9 +3,7 @@ import { randomUUID } from "node:crypto"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; -import { DEFAULT_SYNC_HOST_PORT } from "./syncProtocol"; - -const SYNC_HOST_MAX_PORT = 8999; +import { DEFAULT_SYNC_HOST_PORT, SYNC_HOST_MAX_PORT } from "./syncProtocol"; const LOCK_VERSION = 1; export type SyncHostSingletonOwner = { diff --git a/apps/ade-cli/src/services/sync/syncLoopbackCollision.test.ts b/apps/ade-cli/src/services/sync/syncLoopbackCollision.test.ts index f3bedadeb..760346dee 100644 --- a/apps/ade-cli/src/services/sync/syncLoopbackCollision.test.ts +++ b/apps/ade-cli/src/services/sync/syncLoopbackCollision.test.ts @@ -1,4 +1,6 @@ import fs from "node:fs"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; import http from "node:http"; import os from "node:os"; import path from "node:path"; @@ -9,6 +11,7 @@ import { createSyncCloudRelayStore } from "./syncCloudRelayStore"; import { createSyncService, type SyncService } from "./syncService"; import { probeAdeLoopbackListener, type SyncLoopbackProbeResult } from "./syncLoopbackProbe"; import type { SyncTunnelClientStatus } from "./syncTunnelClientService"; +import { resolveMachineAdeLayout } from "../projects/machineLayout"; const ORIGINAL_BIND_HOST = vi.hoisted(() => process.env.ADE_SYNC_BIND_HOST); vi.hoisted(() => { @@ -163,6 +166,58 @@ describe("sync loopback collision recovery", () => { })); }); + it("reaps a stale ADE holder and retries the occupied candidate", async () => { + const port = await findFreeLegacyPort(); + const holder = spawn(process.execPath, [ + "-e", + [ + "const http=require('node:http');", + "const port=Number(process.argv[1]);", + "const server=http.createServer();", + "server.listen(port,'127.0.0.1',()=>process.stdout.write('ready\\n'));", + "process.on('SIGTERM',()=>server.close(()=>process.exit(0)));", + ].join(""), + String(port), + ], { + stdio: ["ignore", "pipe", "ignore"], + }); + await once(holder.stdout!, "data"); + const logger = { info: vi.fn(), warn: vi.fn() }; + const adeHome = fs.mkdtempSync(path.join(os.tmpdir(), "ade-sync-zombie-")); + const previousAdeHome = process.env.ADE_HOME; + process.env.ADE_HOME = adeHome; + const listener = createSharedSyncListener({ + bindHost: "127.0.0.1", + logger, + activeServicePid: () => null, + inspectPort: (candidate) => ({ + port: candidate, + holders: [{ + pid: holder.pid!, + command: + `apps/ade-cli/dist/cli.cjs serve --socket ${resolveMachineAdeLayout().socketPath}`, + startTime: "test-holder-birth", + }], + }), + }); + + try { + await expect(listener.ensureListening([port])).resolves.toBe(port); + expect(holder.pid).toBeGreaterThan(0); + expect(logger.info).toHaveBeenCalledWith("sync_listener.zombie_reaped", { + port, + pid: holder.pid, + }); + if (holder.exitCode == null) await once(holder, "exit"); + } finally { + await listener.close(); + if (!holder.killed && holder.exitCode == null) holder.kill("SIGKILL"); + if (previousAdeHome === undefined) delete process.env.ADE_HOME; + else process.env.ADE_HOME = previousAdeHome; + fs.rmSync(adeHome, { recursive: true, force: true }); + } + }); + it.runIf(process.platform === "darwin")( "scans past a foreign 127.0.0.1 listener and publishes only the ADE-validated port", async () => { @@ -172,9 +227,15 @@ describe("sync loopback collision recovery", () => { process.env.ADE_SYNC_HOST_LOCK_PATH = lockPath; const foreign = await bindForeignLegacyListener(); const listener = createSharedSyncListener({ bindHost: "0.0.0.0" }); + const logger = createLogger(); + const requestAccountMachinePublish = vi.fn(); const db = await openKvDb(path.join(projectRoot, ".ade", "kv.sqlite"), createLogger() as any); (db.sync as { isAvailable?: () => boolean }).isAvailable = () => true; - const service = createService(db, projectRoot, { sharedSyncListener: listener }); + const service = createService(db, projectRoot, { + logger: logger as any, + sharedSyncListener: listener, + requestAccountMachinePublish, + }); service.getDeviceRegistryService().touchLocalDevice({ lastPort: foreign.port }); try { @@ -196,6 +257,11 @@ describe("sync loopback collision recovery", () => { }); expect(publishedPorts()).not.toContain(foreign.port); expect(new Set(publishedPorts())).toEqual(new Set([resolvedPort!])); + expect(logger.warn).toHaveBeenCalledWith("sync_listener.port_drifted", { + from: foreign.port, + to: resolvedPort, + }); + expect(requestAccountMachinePublish).toHaveBeenCalledTimes(1); await expect(probeAdeLoopbackListener( foreign.port, listener.getExpectedLoopbackNonce(), diff --git a/apps/ade-cli/src/services/sync/syncPairingConnectInfo.ts b/apps/ade-cli/src/services/sync/syncPairingConnectInfo.ts index 82b6e87cd..2a2656af9 100644 --- a/apps/ade-cli/src/services/sync/syncPairingConnectInfo.ts +++ b/apps/ade-cli/src/services/sync/syncPairingConnectInfo.ts @@ -3,10 +3,7 @@ import type { SyncDeviceRecord, SyncPairingConnectInfo, } from "../../../../desktop/src/shared/types"; -import { DEFAULT_SYNC_HOST_PORT } from "./syncProtocol"; - -const SYNC_HOST_PORT_RETRY_WINDOW = 8999 - DEFAULT_SYNC_HOST_PORT; -const SYNC_HOST_MAX_PORT = DEFAULT_SYNC_HOST_PORT + SYNC_HOST_PORT_RETRY_WINDOW; +import { DEFAULT_SYNC_HOST_PORT, SYNC_HOST_MAX_PORT } from "./syncProtocol"; function normalizeHost(host: string | null | undefined): string | null { if (!host) return null; diff --git a/apps/ade-cli/src/services/sync/syncProtocol.ts b/apps/ade-cli/src/services/sync/syncProtocol.ts index 846a88842..09423c10d 100644 --- a/apps/ade-cli/src/services/sync/syncProtocol.ts +++ b/apps/ade-cli/src/services/sync/syncProtocol.ts @@ -5,6 +5,7 @@ import { safeJsonParse } from "../../../../desktop/src/main/services/shared/util export const SYNC_PROTOCOL_VERSION: SyncProtocolVersion = 1; export const DEFAULT_SYNC_HOST_PORT = 8787; +export const SYNC_HOST_MAX_PORT = 8999; export const DEFAULT_SYNC_COMPRESSION_THRESHOLD_BYTES = 4 * 1024; export const MAX_UNCOMPRESSED_SYNC_ENVELOPE_BYTES = 25 * 1024 * 1024; export const RPC_DATA_CHUNK_BYTES = 256 * 1024; diff --git a/apps/ade-cli/src/services/sync/syncRemoteCommandService.test.ts b/apps/ade-cli/src/services/sync/syncRemoteCommandService.test.ts index 7163a328d..515f752eb 100644 --- a/apps/ade-cli/src/services/sync/syncRemoteCommandService.test.ts +++ b/apps/ade-cli/src/services/sync/syncRemoteCommandService.test.ts @@ -16,6 +16,7 @@ function makePayload( function createService(options?: { agentChatService?: Record; + aiIntegrationService?: Record; conflictService?: Record; diffService?: Record; externalSessionsService?: Record; @@ -83,6 +84,7 @@ function createService(options?: { ...(options?.operationService ? { operationService: options.operationService } : {}), ...(options?.projectConfigService ? { projectConfigService: options.projectConfigService } : {}), ...(options?.agentChatService ? { agentChatService: options.agentChatService } : {}), + ...(options?.aiIntegrationService ? { aiIntegrationService: options.aiIntegrationService } : {}), ...(options?.externalSessionsService ? { externalSessionsService: options.externalSessionsService } : {}), ...(options?.syncPinStore ? { syncPinStore: options.syncPinStore } : {}), ...(options?.getPairingConnectInfo ? { getPairingConnectInfo: options.getPairingConnectInfo } : {}), @@ -118,6 +120,46 @@ function makePairingConnectInfo( } describe("createSyncRemoteCommandService", () => { + it("caps ai.getStatus probes at 30 seconds", async () => { + vi.useFakeTimers(); + const getStatus = vi.fn(() => new Promise(() => {})); + const { service } = createService({ + aiIntegrationService: { + getStatus, + getDailyUsageBatch: vi.fn(() => new Map()), + getFeatureFlag: vi.fn(() => false), + getDailyBudgetLimit: vi.fn(() => null), + }, + }); + try { + const result = expect( + service.execute(makePayload("ai.getStatus", { force: true })), + ).rejects.toThrow("ai.getStatus timed out after 30000ms"); + await vi.advanceTimersByTimeAsync(30_000); + await result; + expect(getStatus).toHaveBeenCalledWith({ + force: true, + refreshOpenCodeInventory: false, + }); + } finally { + vi.useRealTimers(); + } + }); + + it("stops waiting for transcript pulls when the execution signal aborts", async () => { + const getChatTranscript = vi.fn(() => new Promise(() => {})); + const { service } = createService({ + agentChatService: { getChatTranscript }, + }); + const controller = new AbortController(); + const pending = service.execute( + makePayload("chat.getTranscript", { sessionId: "chat-1" }), + { signal: controller.signal }, + ); + controller.abort(new Error("peer closed")); + await expect(pending).rejects.toThrow("peer closed"); + }); + it("forwards bounded GitHub history pagination for mobile PR lists", async () => { const getGithubSnapshot = vi.fn().mockResolvedValue({ repoPullRequests: [] }); const { service } = createService({ prService: { getGithubSnapshot } }); diff --git a/apps/ade-cli/src/services/sync/syncRemoteCommandService.ts b/apps/ade-cli/src/services/sync/syncRemoteCommandService.ts index fefbf1d3e..6ce70c5e8 100644 --- a/apps/ade-cli/src/services/sync/syncRemoteCommandService.ts +++ b/apps/ade-cli/src/services/sync/syncRemoteCommandService.ts @@ -383,11 +383,97 @@ type SyncRemoteCommandServiceArgs = { logger: Logger; }; +export type SyncRemoteCommandExecutionContext = { + signal?: AbortSignal; +}; + type RegisteredRemoteCommand = { descriptor: SyncRemoteCommandDescriptor; - handler: (args: Record) => Promise; + handler: ( + args: Record, + context: SyncRemoteCommandExecutionContext, + ) => Promise; }; +export const AI_STATUS_REMOTE_COMMAND_TIMEOUT_MS = 30_000; + +function remoteCommandAbortError(signal: AbortSignal): Error { + if (signal.reason instanceof Error) return signal.reason; + const error = new Error( + typeof signal.reason === "string" && signal.reason.trim() + ? signal.reason + : "Remote command aborted.", + ); + error.name = "AbortError"; + return error; +} + +async function runWithRemoteCommandSignal( + run: () => Promise | T, + signal?: AbortSignal, +): Promise { + if (!signal) return await run(); + if (signal.aborted) throw remoteCommandAbortError(signal); + return await new Promise((resolve, reject) => { + let settled = false; + const settle = (complete: () => void): void => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", onAbort); + complete(); + }; + const onAbort = (): void => settle(() => reject(remoteCommandAbortError(signal))); + signal.addEventListener("abort", onAbort, { once: true }); + Promise.resolve() + .then(run) + .then( + (value) => settle(() => resolve(value)), + (error) => settle(() => reject(error)), + ); + }); +} + +function commandHandlerObservesAbort(action: string): boolean { + return action === "ai.getStatus" + || action === "chat.resolveSmartLinkPreview" + || action === "chat.getTranscript" + || action === "chat.getChatEventHistory" + || action === "chat.getSubagentTranscript" + || action === "chat.getMainTranscript" + || action === "chat.getChatEventHistoryPage" + || action === "agentChat.getEventHistoryPage" + || action === "github.getStatus" + || action === "github.getRemoteStatus" + || action === "prs.refresh" + || action === "prs.preflightCreateLaneFromPrBranch" + || action.startsWith("prs.get") + || action.startsWith("prs.list"); +} + +async function runAiStatusWithTimeout( + run: () => Promise, + parentSignal?: AbortSignal, +): Promise { + const controller = new AbortController(); + const onParentAbort = (): void => controller.abort(parentSignal?.reason); + if (parentSignal?.aborted) onParentAbort(); + else parentSignal?.addEventListener("abort", onParentAbort, { once: true }); + const timer = setTimeout(() => { + const error = new Error( + `ai.getStatus timed out after ${AI_STATUS_REMOTE_COMMAND_TIMEOUT_MS}ms.`, + ); + error.name = "TimeoutError"; + controller.abort(error); + }, AI_STATUS_REMOTE_COMMAND_TIMEOUT_MS); + timer.unref?.(); + try { + return await runWithRemoteCommandSignal(run, controller.signal); + } finally { + clearTimeout(timer); + parentSignal?.removeEventListener("abort", onParentAbort); + } +} + function isRecord(value: unknown): value is Record { return Boolean(value) && typeof value === "object" && !Array.isArray(value); } @@ -3479,7 +3565,10 @@ async function buildLaneDetailPayload(args: SyncRemoteCommandServiceArgs, laneId type RemoteCommandRegistrar = ( action: SyncRemoteCommandAction, policy: SyncRemoteCommandPolicy, - handler: (payload: Record) => Promise, + handler: ( + payload: Record, + context: SyncRemoteCommandExecutionContext, + ) => Promise, scope?: SyncRemoteCommandDescriptor["scope"], ) => void; @@ -4712,12 +4801,15 @@ function registerMiscRemoteCommands({ args, register }: RemoteCommandRegistratio requireService(args.projectConfigService, "Project config service not available.").save( parseProjectConfigSaveArgs(payload).candidate, )); - register("ai.getStatus", { viewerAllowed: true }, async (payload) => { + register("ai.getStatus", { viewerAllowed: true }, async (payload, context) => { try { - return await buildAiSettingsStatus(args.aiIntegrationService, { - force: payload.force === true, - refreshOpenCodeInventory: payload.refreshOpenCodeInventory === true, - }); + return await runAiStatusWithTimeout( + () => buildAiSettingsStatus(args.aiIntegrationService, { + force: payload.force === true, + refreshOpenCodeInventory: payload.refreshOpenCodeInventory === true, + }), + context.signal, + ); } catch (error) { if (isDatabaseClosedError(error)) return getUnavailableAiStatus(); throw error; @@ -4977,7 +5069,10 @@ export function createSyncRemoteCommandService(args: SyncRemoteCommandServiceArg const register = ( action: SyncRemoteCommandAction, policy: SyncRemoteCommandPolicy, - handler: (payload: Record) => Promise, + handler: ( + payload: Record, + context: SyncRemoteCommandExecutionContext, + ) => Promise, scope: SyncRemoteCommandDescriptor["scope"] = "project", ) => { registry.set(action, { @@ -5096,7 +5191,10 @@ export function createSyncRemoteCommandService(args: SyncRemoteCommandServiceArg return registry.get(action as SyncRemoteCommandAction)?.descriptor ?? null; }, - async execute(payload: SyncCommandPayload): Promise { + async execute( + payload: SyncCommandPayload, + context: SyncRemoteCommandExecutionContext = {}, + ): Promise { const handler = registry.get(payload.action as SyncRemoteCommandAction); if (!handler) { throw new Error(`Unsupported remote command: ${payload.action}`); @@ -5107,7 +5205,10 @@ export function createSyncRemoteCommandService(args: SyncRemoteCommandServiceArg scope: handler.descriptor.scope, policy: handler.descriptor.policy, }); - return await handler.handler(commandArgs); + const run = () => handler.handler(commandArgs, context); + return commandHandlerObservesAbort(payload.action) + ? await runWithRemoteCommandSignal(run, context.signal) + : await run(); }, }; } diff --git a/apps/ade-cli/src/services/sync/syncService.ts b/apps/ade-cli/src/services/sync/syncService.ts index 973a75785..d7974b950 100644 --- a/apps/ade-cli/src/services/sync/syncService.ts +++ b/apps/ade-cli/src/services/sync/syncService.ts @@ -70,7 +70,7 @@ import { createSyncCloudRelayStore, type SyncCloudRelayStore } from "./syncCloud import { createSyncPeerService } from "./syncPeerService"; import { createSyncPinStore } from "./syncPinStore"; import { createSyncRuntimeNameStore } from "./syncRuntimeNameStore"; -import { DEFAULT_SYNC_HOST_PORT } from "./syncProtocol"; +import { DEFAULT_SYNC_HOST_PORT, SYNC_HOST_MAX_PORT } from "./syncProtocol"; import { createSyncRemoteCommandService, type ExternalSessionsRemoteService, type SyncRemoteCommandService } from "./syncRemoteCommandService"; import { buildAddressCandidates, @@ -102,6 +102,7 @@ type SyncServiceArgs = { productAnalyticsService?: ProductAnalyticsService | null; logger: Logger; getAccountDirectoryHealth?: () => SyncAccountDirectoryHealth; + requestAccountMachinePublish?: () => void | Promise; accountAuthService?: Pick; getAccountAttestationConfig?: () => AccountAttestationConfig; projectId?: string | null; @@ -370,9 +371,7 @@ function isChatToolType(toolType: string | null | undefined): boolean { return normalized === "cursor" || normalized.endsWith("-chat"); } const LEGACY_SYNC_HOST_PORT_RETRY_WINDOW = 13; -const SYNC_HOST_PORT_RETRY_WINDOW = 8999 - DEFAULT_SYNC_HOST_PORT; const LEGACY_SYNC_HOST_MAX_PORT = DEFAULT_SYNC_HOST_PORT + LEGACY_SYNC_HOST_PORT_RETRY_WINDOW; -const SYNC_HOST_MAX_PORT = DEFAULT_SYNC_HOST_PORT + SYNC_HOST_PORT_RETRY_WINDOW; const LOCAL_LANE_PRESENCE_HEARTBEAT_MS = 30_000; const TRANSFER_READINESS_CACHE_MS = 15_000; const STALE_BRAIN_LAST_SEEN_MS = 5 * 60_000; @@ -884,6 +883,13 @@ export function createSyncService(args: SyncServiceArgs) { lastHost: localDevice.ipAddresses[0] ?? localDevice.tailscaleIp ?? localDevice.lastHost, lastPort: resolvedPort, }); + if (localDevice.lastPort != null && localDevice.lastPort !== resolvedPort) { + args.logger.warn("sync_listener.port_drifted", { + from: localDevice.lastPort, + to: resolvedPort, + }); + void args.requestAccountMachinePublish?.(); + } }; try { const portCandidates = buildHostPortCandidates(preferredPort); @@ -1763,8 +1769,11 @@ export function createSyncService(args: SyncServiceArgs) { return remoteCommandService.getDescriptor(action); }, - async executeRemoteCommand(payload: Parameters[0]): Promise { - return await remoteCommandService.execute(payload); + async executeRemoteCommand( + payload: Parameters[0], + context?: Parameters[1], + ): Promise { + return await remoteCommandService.execute(payload, context); }, getDeviceRegistryService() { diff --git a/apps/desktop/src/main/main.ts b/apps/desktop/src/main/main.ts index 981698694..70bfb82c1 100644 --- a/apps/desktop/src/main/main.ts +++ b/apps/desktop/src/main/main.ts @@ -152,7 +152,7 @@ import { getSharedAccountAuthService, registerAccountConfigProjectRoot, } from "../../../ade-cli/src/services/account/sharedAccountAuthService"; -import { uninstallRuntimeService } from "../../../ade-cli/src/serviceManager"; +import { installRuntimeService, uninstallRuntimeService } from "../../../ade-cli/src/serviceManager"; import { ElectronSafeStorageCredentialStore, EncryptedFileCredentialStore, @@ -2116,17 +2116,32 @@ app.whenReady().then(async () => { tempRoot: app.getPath("temp"), logger: updateLogger, }); + let runtimeServiceUninstalledForUpdate = false; + let autoUpdateInstallRollbackReason: string | null = null; + const reinstallRuntimeServiceAfterUpdateAbort = async (reason: string): Promise => { + if (!runtimeServiceUninstalledForUpdate) return; + const result = await installRuntimeService(); + const payload = { + reason, + ok: result.ok, + serviceName: result.serviceName, + path: result.path, + message: result.message, + selfMutationBlocked: result.selfMutationBlocked === true, + }; + if (!result.ok) { + updateLogger.error("autoUpdate.runtime_service_reinstall_after_abort_failed", payload); + throw new Error(result.message); + } + runtimeServiceUninstalledForUpdate = false; + updateLogger.info("autoUpdate.runtime_service_reinstalled_after_abort", payload); + }; const prepareAutoUpdateInstall = async (): Promise => { updateLogger.info("autoUpdate.prepare_quit_and_install_start", { serviceManaged: shouldRepairRuntimeServiceOnFallback, }); - try { - localRuntimePool.dispose(); - } catch (error) { - updateLogger.warn("autoUpdate.local_runtime_dispose_before_install_failed", { - error: error instanceof Error ? error.message : String(error), - }); - } + runtimeServiceUninstalledForUpdate = false; + autoUpdateInstallRollbackReason = null; if (!shouldRepairRuntimeServiceOnFallback) { updateLogger.info("autoUpdate.prepare_quit_and_install_done", { serviceManaged: false, @@ -2146,10 +2161,19 @@ app.whenReady().then(async () => { throw new Error(result.message); } updateLogger.info("autoUpdate.runtime_service_uninstalled_before_install", payload); + runtimeServiceUninstalledForUpdate = true; + if (autoUpdateInstallRollbackReason) { + await reinstallRuntimeServiceAfterUpdateAbort(autoUpdateInstallRollbackReason); + return; + } updateLogger.info("autoUpdate.prepare_quit_and_install_done", { serviceManaged: true, }); }; + const rollbackAutoUpdateInstall = async (reason: string): Promise => { + autoUpdateInstallRollbackReason = reason; + await reinstallRuntimeServiceAfterUpdateAbort(reason); + }; const autoUpdateService = createAutoUpdateService({ logger: updateLogger, currentVersion: app.getVersion(), @@ -2158,6 +2182,19 @@ app.whenReady().then(async () => { installTargetPath: process.execPath, autoCheckEnabled: app.isPackaged, beforeQuitAndInstall: prepareAutoUpdateInstall, + rollbackQuitAndInstall: rollbackAutoUpdateInstall, + getRuntimeActivitySummary: () => localRuntimePool.activitySummary(), + productAnalyticsService, + forceQuit: () => { + for (const win of BrowserWindow.getAllWindows()) { + try { + win.destroy(); + } catch { + // Keep escalating through every window; app.exit is the hard bound. + } + } + app.exit(0); + }, }); const shouldRefreshRuntimeServiceAfterUpdate = app.isPackaged @@ -6063,7 +6100,7 @@ app.whenReady().then(async () => { win: BrowserWindow, event: Electron.Event, ): void => { - if (shutdownRequested) return; + if (shutdownRequested || autoUpdateService.isInstallQuitArmed()) return; if (closeWindowWithoutQuitPrompt.delete(win.id)) return; event.preventDefault(); if (BrowserWindow.getAllWindows().filter((openWindow) => !openWindow.isDestroyed()).length > 1) { @@ -6134,6 +6171,7 @@ app.whenReady().then(async () => { runImmediateProcessCleanup("process_exit"); }); app.on("will-quit", () => { + autoUpdateService.notifyQuitHandoffStarted(); runImmediateProcessCleanup("will_quit"); disposeSharedTranscriptionService(); }); @@ -6645,6 +6683,10 @@ app.whenReady().then(async () => { app.on("before-quit", (event) => { if (shutdownFinalized) return; + if (autoUpdateService.isInstallQuitArmed()) { + quitWarningAcknowledged = true; + return; + } event.preventDefault(); if (shutdownRequested) return; requestQuitAfterWarnings(null, "before_quit"); diff --git a/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts b/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts index 3fc1d3454..44f33440d 100644 --- a/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts +++ b/apps/desktop/src/main/services/analytics/productAnalyticsPolicy.ts @@ -1,4 +1,5 @@ import { isMeaningfulUsageAction } from "../usage/usageStatsStore"; +import { AUTO_UPDATE_INSTALL_ABORT_REASONS } from "../../../shared/types"; import type { ProductAnalyticsCapture, ProductAnalyticsEventName, @@ -14,6 +15,9 @@ export const MAX_LOCAL_IDENTIFIER_LENGTH = 512; export const INTERNAL_ONLY_EVENTS = new Set([ "ade_work_session_started", "ade_work_session_completed", "ade_daily_usage_summary", "ade_analytics_budget", + "ade_update_install_aborted", "ade_update_quit_escalated", "ade_update_auto_applied", + "ade_update_auto_apply_cancelled", + "ade_brain_recovered", "ade_publish_failing", ]); export const EVENT_DAILY_BUDGETS: Record = { @@ -26,6 +30,12 @@ export const EVENT_DAILY_BUDGETS: Record = { ade_error: 20, ade_daily_usage_summary: 60, ade_analytics_budget: 2, + ade_update_install_aborted: 20, + ade_update_quit_escalated: 10, + ade_update_auto_applied: 10, + ade_update_auto_apply_cancelled: 10, + ade_brain_recovered: 10, + ade_publish_failing: 10, }; export const EVENT_MINUTE_BUDGETS: Record = { @@ -38,19 +48,26 @@ export const EVENT_MINUTE_BUDGETS: Record = { ade_error: 6, ade_daily_usage_summary: 60, ade_analytics_budget: 2, + ade_update_install_aborted: 6, + ade_update_quit_escalated: 3, + ade_update_auto_applied: 3, + ade_update_auto_apply_cancelled: 3, + ade_brain_recovered: 3, + ade_publish_failing: 3, }; const STRING_PROPERTIES = new Set([ "screen", "feature", "action", "outcome", "app_version", "runtime_mode", "provider", "model_family", "duration_bucket", "error_kind", "route_kind", "connection_state", "drop_reason", "source", "mode", - "entry_point", "release_channel", "summary_kind", + "entry_point", "release_channel", "summary_kind", "reason", "last_command", "leg", "code", ]); const NUMBER_PROPERTIES = new Set([ "sent_count", "dropped_count", "interaction_count", "session_count", "chat_session_count", "terminal_session_count", "active_lane_count", "lanes_created", "lanes_archived", "commits_created", "push_operations", "pr_landings", "files_changed", "artifacts_captured", "automation_runs", "worker_runs", "active_days", "current_streak_days", "token_count", "input_token_count", "output_token_count", "call_count", - "duration_ms", "provider_count", "model_count", "error_count", "bytes_freed", "files_compressed", + "duration_ms", "provider_count", "model_count", "error_count", "bytes_freed", "files_compressed", "blocked_ms", + "failing_minutes", ]); const BOOLEAN_PROPERTIES = new Set(["recoverable", "paired", "cached_data", "is_packaged"]); @@ -82,6 +99,12 @@ const EVENT_PROPERTY_KEYS: Record "duration_ms", "provider_count", "model_count", "error_count", ]), ade_analytics_budget: new Set(["sent_count", "dropped_count", "drop_reason"]), + ade_update_install_aborted: new Set(["reason"]), + ade_update_quit_escalated: new Set(["blocked_ms"]), + ade_update_auto_applied: new Set(), + ade_update_auto_apply_cancelled: new Set(), + ade_brain_recovered: new Set(["blocked_ms", "last_command"]), + ade_publish_failing: new Set(["failing_minutes", "leg", "code"]), }; const SLUG_VALUE = /^[a-z0-9][a-z0-9._+-]*$/i; @@ -122,6 +145,7 @@ const SAFE_STRING_VALUES: Partial>> = { ]), release_channel: new Set(["stable", "beta", "development", "unknown"]), summary_kind: new Set(["overall", "client", "provider", "model"]), + reason: new Set(AUTO_UPDATE_INSTALL_ABORT_REASONS), }; export function safeProductAnalyticsString(value: ProductAnalyticsPropertyValue): string | null { diff --git a/apps/desktop/src/main/services/analytics/productAnalyticsService.test.ts b/apps/desktop/src/main/services/analytics/productAnalyticsService.test.ts index 6243478b6..b51139c07 100644 --- a/apps/desktop/src/main/services/analytics/productAnalyticsService.test.ts +++ b/apps/desktop/src/main/services/analytics/productAnalyticsService.test.ts @@ -136,6 +136,39 @@ describe("productAnalyticsService", () => { fs.rmSync(harness.root, { recursive: true, force: true }); }); + it("accepts only coarse transactional update telemetry properties", () => { + const harness = makeHarness(); + + expect(harness.service.captureInternal({ + event: "ade_update_install_aborted", + surface: "desktop", + properties: { reason: "prepare_failed", error: "private runtime message" }, + })).toEqual({ accepted: true, reason: "accepted" }); + expect(harness.service.captureInternal({ + event: "ade_update_quit_escalated", + surface: "desktop", + properties: { blocked_ms: 10_000, blocked_phase: "private phase detail" }, + })).toEqual({ accepted: true, reason: "accepted" }); + expect(harness.service.captureInternal({ + event: "ade_update_auto_applied", + surface: "desktop", + })).toEqual({ accepted: true, reason: "accepted" }); + expect(harness.service.captureInternal({ + event: "ade_update_auto_apply_cancelled", + surface: "desktop", + })).toEqual({ accepted: true, reason: "accepted" }); + + expect(harness.messages.map((message) => message.event)).toEqual([ + "ade_update_install_aborted", + "ade_update_quit_escalated", + "ade_update_auto_applied", + "ade_update_auto_apply_cancelled", + ]); + expect(harness.messages[0]?.properties).toMatchObject({ reason: "prepare_failed" }); + expect(harness.messages[1]?.properties).toMatchObject({ blocked_ms: 10_000 }); + expect(JSON.stringify(harness.messages)).not.toContain("private"); + }); + it("accepts the storage-doctor maintenance event with numeric aggregates and coarse outcome", () => { const harness = makeHarness(); const result = harness.service.capture({ @@ -166,6 +199,43 @@ describe("productAnalyticsService", () => { fs.rmSync(harness.root, { recursive: true, force: true }); }); + it("accepts only coarse reliability telemetry properties", () => { + const harness = makeHarness(); + + expect(harness.service.captureInternal({ + event: "ade_brain_recovered", + surface: "api", + properties: { + blocked_ms: 125_000, + last_command: "sync.refresh", + payload: "/Users/alice/private-project", + }, + })).toEqual({ accepted: true, reason: "accepted" }); + expect(harness.service.captureInternal({ + event: "ade_publish_failing", + surface: "api", + properties: { + failing_minutes: 3, + leg: "token", + code: "token_timeout", + endpoint: "https://private.example.test", + }, + })).toEqual({ accepted: true, reason: "accepted" }); + + expect(harness.messages).toHaveLength(2); + expect(harness.messages[0]?.properties).toMatchObject({ + blocked_ms: 125_000, + last_command: "sync.refresh", + }); + expect(harness.messages[1]?.properties).toMatchObject({ + failing_minutes: 3, + leg: "token", + code: "token_timeout", + }); + expect(JSON.stringify(harness.messages)).not.toContain("private"); + fs.rmSync(harness.root, { recursive: true, force: true }); + }); + it("does not forward arbitrary build-controlled version text", () => { const harness = makeHarness({ appVersion: "../../private/project\nsecret" }); expect(harness.service.capture({ diff --git a/apps/desktop/src/main/services/github/githubService.test.ts b/apps/desktop/src/main/services/github/githubService.test.ts index 5329d193f..9a713f913 100644 --- a/apps/desktop/src/main/services/github/githubService.test.ts +++ b/apps/desktop/src/main/services/github/githubService.test.ts @@ -50,7 +50,11 @@ vi.mock("../git/git", () => ({ // Replace global fetch vi.stubGlobal("fetch", mockFetch); -import { createGithubService, fetchAdeLatestRelease } from "./githubService"; +import { + GITHUB_API_BODY_TIMEOUT_MS, + createGithubService, + fetchAdeLatestRelease, +} from "./githubService"; // --------------------------------------------------------------------------- // Helpers @@ -177,6 +181,40 @@ describe("githubService.apiRequest", () => { expect(result.response!.status).toBe(200); }); + it("aborts and rejects when the response body never finishes", async () => { + vi.useFakeTimers(); + let markBodyStarted!: () => void; + const bodyStarted = new Promise((resolve) => { markBodyStarted = resolve; }); + mockFetch.mockResolvedValueOnce({ + ok: true, + status: 200, + headers: new Headers({ "content-type": "application/json" }), + text: vi.fn(() => { + markBodyStarted(); + return new Promise(() => {}); + }), + json: vi.fn(), + } as unknown as Response); + const service = makeService(); + try { + const pending = service.apiRequest({ + method: "GET", + path: "/repos/owner/repo", + token: "ghp_test123", + }); + await bodyStarted; + const result = expect(pending).rejects.toThrow( + "GitHub API response body timed out", + ); + await vi.advanceTimersByTimeAsync(GITHUB_API_BODY_TIMEOUT_MS); + await result; + const fetchSignal = mockFetch.mock.calls[0]?.[1]?.signal as AbortSignal; + expect(fetchSignal.aborted).toBe(true); + } finally { + vi.useRealTimers(); + } + }); + it("throws with message from response when errors array is absent", async () => { mockFetch.mockResolvedValueOnce(jsonResponse(404, { message: "Not Found" })); diff --git a/apps/desktop/src/main/services/github/githubService.ts b/apps/desktop/src/main/services/github/githubService.ts index e4e377185..22adffa59 100644 --- a/apps/desktop/src/main/services/github/githubService.ts +++ b/apps/desktop/src/main/services/github/githubService.ts @@ -29,6 +29,7 @@ import { nowIso, asString } from "../shared/utils"; const AUTH_STORE_FILE_NAME = "github-token.v1.bin"; const MACHINE_TOKEN_KEY = "github.token.v1"; const GITHUB_API_TIMEOUT_MS = 20_000; +export const GITHUB_API_BODY_TIMEOUT_MS = 30_000; const GH_AUTH_TOKEN_CACHE_TTL_MS = 30_000; const GH_HOSTS_TOKEN_CACHE_MAX_ENTRIES = 32; const GITHUB_STATUS_FAILURE_COOLDOWN_MS = 30_000; @@ -207,14 +208,121 @@ async function readGitHubCliAuthToken(): Promise { } } +const githubResponseCleanup = new WeakMap void>(); + +function githubRequestTimeoutError(phase: "request" | "response body"): Error { + return new Error( + `GitHub API ${phase} timed out. Check network access on this machine.`, + ); +} + +function wrapGitHubResponseBody(args: { + response: Response; + controller: AbortController; + cleanupUpstreamAbort: () => void; +}): Response { + let settled = false; + let timeoutError: Error | null = null; + const pendingRejects = new Set<(error: Error) => void>(); + const finish = (): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + args.controller.signal.removeEventListener("abort", onAbort); + args.cleanupUpstreamAbort(); + githubResponseCleanup.delete(args.response); + }; + const onAbort = (): void => { + const error = timeoutError + ?? (args.controller.signal.reason instanceof Error + ? args.controller.signal.reason + : new Error("GitHub API request aborted.")); + for (const reject of pendingRejects) reject(error); + pendingRejects.clear(); + finish(); + }; + const timer = setTimeout(() => { + timeoutError = githubRequestTimeoutError("response body"); + args.controller.abort(timeoutError); + }, GITHUB_API_BODY_TIMEOUT_MS); + timer.unref?.(); + args.controller.signal.addEventListener("abort", onAbort, { once: true }); + + const wrapBodyReader = (read: () => Promise): (() => Promise) => + async (): Promise => { + if (timeoutError) throw timeoutError; + if (args.controller.signal.aborted) { + throw args.controller.signal.reason instanceof Error + ? args.controller.signal.reason + : new Error("GitHub API request aborted."); + } + return await new Promise((resolve, reject) => { + const rejectPending = (error: Error): void => reject(error); + pendingRejects.add(rejectPending); + Promise.resolve() + .then(read) + .then( + (value) => { + pendingRejects.delete(rejectPending); + finish(); + resolve(value); + }, + (error) => { + pendingRejects.delete(rejectPending); + finish(); + reject(error); + }, + ); + }); + }; + + for (const method of ["arrayBuffer", "blob", "formData", "json", "text"] as const) { + const original = args.response[method]; + if (typeof original !== "function") continue; + Object.defineProperty(args.response, method, { + configurable: true, + value: wrapBodyReader(original.bind(args.response) as () => Promise), + }); + } + githubResponseCleanup.set(args.response, finish); + return args.response; +} + +function releaseGitHubResponse(response: Response): void { + githubResponseCleanup.get(response)?.(); + void response.body?.cancel().catch(() => {}); +} + async function fetchGitHub(input: string | URL, init: RequestInit): Promise { const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), GITHUB_API_TIMEOUT_MS); + const upstreamSignal = init.signal; + const onUpstreamAbort = (): void => controller.abort(upstreamSignal?.reason); + if (upstreamSignal?.aborted) onUpstreamAbort(); + else upstreamSignal?.addEventListener("abort", onUpstreamAbort, { once: true }); + const cleanupUpstreamAbort = (): void => { + upstreamSignal?.removeEventListener("abort", onUpstreamAbort); + }; + let headerTimedOut = false; + const timer = setTimeout(() => { + headerTimedOut = true; + controller.abort(githubRequestTimeoutError("request")); + }, GITHUB_API_TIMEOUT_MS); + timer.unref?.(); try { - return await fetch(input, { ...init, signal: controller.signal }); + const response = await fetch(input, { ...init, signal: controller.signal }); + clearTimeout(timer); + return wrapGitHubResponseBody({ + response, + controller, + cleanupUpstreamAbort, + }); } catch (error) { - if (error instanceof Error && error.name === "AbortError") { - throw new Error("GitHub API request timed out. Check network access on this machine."); + cleanupUpstreamAbort(); + if (headerTimedOut) { + throw githubRequestTimeoutError("request"); + } + if (controller.signal.aborted && controller.signal.reason instanceof Error) { + throw controller.signal.reason; } throw error; } finally { @@ -373,7 +481,10 @@ export async function fetchAdeLatestRelease(options?: { } catch { return null; } - if (!response.ok) return null; + if (!response.ok) { + releaseGitHubResponse(response); + return null; + } const payload = (await response.json().catch(() => null)) as Record | null; if (!payload) return null; @@ -758,7 +869,10 @@ export function createGithubService({ }, }, ); - if (response.ok) return { ok: true, error: null }; + if (response.ok) { + releaseGitHubResponse(response); + return { ok: true, error: null }; + } const payload = (await response.json().catch(() => ({}))) as Record; const message = asString(payload.message) || `HTTP ${response.status}`; return { ok: false, error: `${response.status}: ${message}` }; @@ -916,6 +1030,7 @@ export function createGithubService({ if (response.status === 304) { const cached = etagCache.get(urlKey); if (cached) { + releaseGitHubResponse(response); return { data: cached.data as T, response, linkHeader: cached.linkHeader }; } } diff --git a/apps/desktop/src/main/services/ipc/registerIpc.ts b/apps/desktop/src/main/services/ipc/registerIpc.ts index 1c6cef604..1b1f972ab 100644 --- a/apps/desktop/src/main/services/ipc/registerIpc.ts +++ b/apps/desktop/src/main/services/ipc/registerIpc.ts @@ -1,6 +1,11 @@ import { app, BrowserWindow, clipboard, desktopCapturer, dialog, ipcMain, nativeImage, shell, systemPreferences } from "electron"; import type { IpcMainInvokeEvent } from "electron"; -import { compareUpdateVersions, createEmptyAutoUpdateSnapshot, type createAutoUpdateService } from "../updates/autoUpdateService"; +import { + buildGithubReleaseUrl, + compareUpdateVersions, + createEmptyAutoUpdateSnapshot, + type createAutoUpdateService, +} from "../updates/autoUpdateService"; import { spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; import fs from "node:fs"; @@ -3762,6 +3767,17 @@ export function registerIpc({ }); ipcMain.handle(IPC.appGetLatestRelease, async (): Promise => { + if (app.isPackaged) { + const snapshot = getCtx().autoUpdateService?.getSnapshot(); + const version = snapshot?.latestKnownVersion?.trim() ?? ""; + if (!version) return null; + return { + version, + htmlUrl: buildGithubReleaseUrl(version), + publishedAt: null, + updateAvailable: compareUpdateVersions(version, app.getVersion()) > 0, + }; + } let token: string | null = null; try { // ADE's release repository is public. Use only an immediately available @@ -10210,6 +10226,10 @@ export function registerIpc({ return getCtx().autoUpdateService?.quitAndInstall() ?? false; }); + ipcMain.handle(IPC.updateCancelAutoApply, () => { + return getCtx().autoUpdateService?.cancelAutoApply() ?? false; + }); + ipcMain.handle(IPC.updateDismissInstalledNotice, () => { getCtx().autoUpdateService?.dismissInstalledNotice(); }); diff --git a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts index e66b01917..4e0bfda8f 100644 --- a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts +++ b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts @@ -25,6 +25,7 @@ import { localReleaseBuildOutputRuntimeBlock, LocalRuntimeConnectionPool, parseRuntimeServiceManagerOutput, + readLocalRuntimeInfo, shouldAutoInstallRuntimeServiceFromPath, } from "./localRuntimeConnectionPool"; import { @@ -219,6 +220,56 @@ describe("local runtime connection pool", () => { expect(compareRuntimeVersionStrings("next", "1.2.13")).toBeNull(); }); + it("quarantines a newer brain whose protocol window excludes this desktop", async () => { + const logger = { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() }; + const pool = new LocalRuntimeConnectionPool("1.0.0", logger as never); + const internals = pool as unknown as { + runtimeCompatibilityError: (socketPath: string, runtimeInfo: { + version: string; + buildHash: string; + defaultRole: string; + pid: number; + minCompatibleProtocol: number; + protocolVersion: number; + }) => Error & { skewState: string }; + connectClient: (socketPath: string) => Promise; + startIsolatedRuntime: (socketPath: string, error: Error) => Promise<{ + client: unknown; + child: null; + socketPath: string; + }>; + tryConnect: (socketPath: string) => Promise<{ socketPath: string } | null>; + }; + const compatibilityError = internals.runtimeCompatibilityError("/tmp/ade.sock", { + version: "2.0.0", + buildHash: "newer-build", + defaultRole: "cto", + pid: 4321, + minCompatibleProtocol: 2, + protocolVersion: 2, + }); + vi.spyOn(internals, "connectClient").mockRejectedValue(compatibilityError); + const startIsolatedRuntime = vi.spyOn(internals, "startIsolatedRuntime") + .mockResolvedValue({ + client: {}, + child: null, + socketPath: "/tmp/ade-isolated.sock", + }); + + try { + await expect(internals.tryConnect("/tmp/ade.sock")).resolves.toMatchObject({ + socketPath: "/tmp/ade-isolated.sock", + }); + expect(compatibilityError.skewState).toBe("runtime_newer"); + expect(startIsolatedRuntime).toHaveBeenCalledWith( + "/tmp/ade.sock", + compatibilityError, + ); + } finally { + pool.dispose(); + } + }); + it("starts fallback runtimes with sync enabled by default", () => { const args = buildLocalRuntimeServeArgs("/opt/ade/cli.cjs", "/tmp/ade.sock"); @@ -328,7 +379,7 @@ describe("local runtime connection pool", () => { } }); - it("skips service install when a newer brain is already running", async () => { + it("skips service install when a newer compatible brain is already running", async () => { const adeCliRoot = path.resolve(process.cwd(), "../ade-cli"); const cliPath = path.join(adeCliRoot, "src", "cli.ts"); const tsxLoaderPath = path.join(adeCliRoot, "node_modules", "tsx", "dist", "loader.mjs"); @@ -375,9 +426,9 @@ describe("local runtime connection pool", () => { expect(pool.getStatus()).toMatchObject({ versionSkew: { - state: "runtime_newer", + state: "none", appVersion: "1.0.0", - runtimeVersion: "2.0.0", + runtimeVersion: null, }, serviceInstall: { state: "skipped", @@ -385,11 +436,10 @@ describe("local runtime connection pool", () => { path: cliPath, }, }); - expect(logger.warn).toHaveBeenCalledWith( + expect(logger.info).toHaveBeenCalledWith( "local_runtime.service_install_skipped", expect.objectContaining({ - reason: "preserve_running_runtime", - skewState: "runtime_newer", + reason: "compatible_newer_runtime", runtimeVersion: "2.0.0", appVersion: "1.0.0", }), @@ -486,6 +536,10 @@ describe("local runtime connection pool", () => { expect(pool.getStatus()).toMatchObject({ connectionState: "idle", + pid: null, + syncPort: null, + publishHealth: null, + lastWedge: null, versionSkew: { state: "none", }, @@ -517,6 +571,65 @@ describe("local runtime connection pool", () => { }); }); + it("parses and carries runtime recovery health onto LocalRuntimeStatus", () => { + const parsed = readLocalRuntimeInfo({ + runtimeInfo: { + version: "1.2.35", + buildHash: "build", + defaultRole: "cto", + pid: 4321, + syncPort: 8789, + publishHealth: { + state: "http_timeout", + failingSinceMs: 123_000, + lastLegDurations: { snapshot: 12, token: 34, http: 9_200 }, + }, + lastWedge: { + lastCommand: "chat.send", + blockedMs: 16_500, + ts: "2026-07-23T12:00:00.000Z", + }, + minCompatibleProtocol: 1, + protocolVersion: 1, + }, + }); + const pool = new LocalRuntimeConnectionPool("1.2.35", { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + } as never); + const internals = pool as unknown as { + activeClient: unknown; + activeRuntimePid: number | null; + activeRuntimeSyncPort: number | null; + activeRuntimePublishHealth: typeof parsed.publishHealth; + activeRuntimeLastWedge: typeof parsed.lastWedge; + }; + internals.activeClient = {}; + internals.activeRuntimePid = parsed.pid; + internals.activeRuntimeSyncPort = parsed.syncPort; + internals.activeRuntimePublishHealth = parsed.publishHealth; + internals.activeRuntimeLastWedge = parsed.lastWedge; + + expect(pool.getStatus()).toMatchObject({ + connectionState: "connected", + pid: 4321, + syncPort: 8789, + publishHealth: { + state: "http_timeout", + failingSinceMs: 123_000, + lastLegDurations: { snapshot: 12, token: 34, http: 9_200 }, + }, + lastWedge: { + lastCommand: "chat.send", + blockedMs: 16_500, + ts: "2026-07-23T12:00:00.000Z", + }, + }); + pool.dispose(); + }); + it("retries the service install from isolated recovery and reports runtime mode transitions", async () => { const modeChanges: string[] = []; const pool = new LocalRuntimeConnectionPool("1.2.3", { @@ -1948,7 +2061,7 @@ describe("local runtime connection pool", () => { } }, 45_000); - it("does not repair over a newer local brain", async () => { + it("connects normally to a newer protocol-compatible local brain", async () => { const adeCliRoot = path.resolve(process.cwd(), "../ade-cli"); const cliPath = path.join(adeCliRoot, "src", "cli.ts"); const tsxLoaderPath = path.join(adeCliRoot, "node_modules", "tsx", "dist", "loader.mjs"); @@ -2005,20 +2118,18 @@ describe("local runtime connection pool", () => { const tryRepair = vi.spyOn(internals, "tryRepairServiceConnection"); const connection = await internals.tryConnect(socketPath); - expect(connection?.socketPath).toBeTruthy(); - expect(connection?.socketPath).not.toBe(socketPath); + expect(connection?.socketPath).toBe(socketPath); expect(tryRepair).not.toHaveBeenCalled(); - expect(logger.warn).toHaveBeenCalledWith("local_runtime.newer_brain_preserved", expect.objectContaining({ + expect(logger.info).toHaveBeenCalledWith("local_runtime.newer_brain_accepted", expect.objectContaining({ appVersion: "1.0.0", runtimeVersion: "2.0.0", runtimePid: daemonPid, })); expect(pool.getStatus()).toMatchObject({ - runtimeMode: "isolated", versionSkew: { - state: "runtime_newer", + state: "none", appVersion: "1.0.0", - runtimeVersion: "2.0.0", + runtimeVersion: null, }, }); expect(() => process.kill(daemonPid, 0)).not.toThrow(); diff --git a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts index 842c3c15d..a16bad7ee 100644 --- a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts +++ b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts @@ -6,6 +6,7 @@ import os from "node:os"; import path from "node:path"; import { app } from "electron"; import { isAdeRuntimeNamedPipePath } from "../../../shared/adeRuntimeIpc"; +import { isRuntimeProtocolCompatible } from "../../../shared/adeRuntimeProtocol"; import type { RemoteRuntimeActionRequest, RemoteRuntimeActionResult, @@ -18,6 +19,7 @@ import type { import type { AdeActionRegistryEntry, LocalRuntimeStatus, + RuntimeActivitySummary, SyncCloudRelayStatus, SyncDeviceRecord, SyncDeviceRuntimeState, @@ -429,32 +431,121 @@ function openSocketTransport(socketPath: string, timeoutMs = 3_000): Promise; if (!value || typeof value !== "object" || Array.isArray(value)) { - return { version: null, buildHash: null, defaultRole: null, pid: null }; + return empty; } const runtimeInfo = (value as { runtimeInfo?: unknown }).runtimeInfo; if (!runtimeInfo || typeof runtimeInfo !== "object" || Array.isArray(runtimeInfo)) { - return { version: null, buildHash: null, defaultRole: null, pid: null }; + return empty; } const info = runtimeInfo as Record; const version = info.version; const buildHash = info.buildHash; const defaultRole = info.defaultRole; const pid = info.pid; + const syncPort = info.syncPort; + const rawPublishHealth = info.publishHealth; + const publishHealthRecord = rawPublishHealth && typeof rawPublishHealth === "object" && !Array.isArray(rawPublishHealth) + ? rawPublishHealth as Record + : null; + const rawLegDurations = publishHealthRecord?.lastLegDurations; + const legDurationsRecord = rawLegDurations && typeof rawLegDurations === "object" && !Array.isArray(rawLegDurations) + ? rawLegDurations as Record + : null; + const publishState = publishHealthRecord?.state; + const rawLastWedge = info.lastWedge; + const lastWedgeRecord = rawLastWedge && typeof rawLastWedge === "object" && !Array.isArray(rawLastWedge) + ? rawLastWedge as Record + : null; + const lastWedgeCommand = lastWedgeRecord?.lastCommand; + const lastWedgeBlockedMs = lastWedgeRecord?.blockedMs; + const lastWedgeTs = lastWedgeRecord?.ts; + const minCompatibleProtocol = info.minCompatibleProtocol; + const protocolVersion = info.protocolVersion; return { version: typeof version === "string" && version.trim() ? version.trim() : null, buildHash: typeof buildHash === "string" && buildHash.trim() ? buildHash.trim() : null, defaultRole: typeof defaultRole === "string" && defaultRole.trim() ? defaultRole.trim() : null, pid: typeof pid === "number" && Number.isFinite(pid) && pid > 0 ? Math.floor(pid) : null, + syncPort: + typeof syncPort === "number" + && Number.isInteger(syncPort) + && syncPort > 0 + && syncPort <= 65_535 + ? syncPort + : null, + publishHealth: + typeof publishState === "string" && publishState.trim() && legDurationsRecord + ? { + state: publishState as NonNullable["state"], + failingSinceMs: + typeof publishHealthRecord?.failingSinceMs === "number" + && Number.isFinite(publishHealthRecord.failingSinceMs) + ? Math.max(0, publishHealthRecord.failingSinceMs) + : null, + lastLegDurations: { + snapshot: finiteDuration(legDurationsRecord.snapshot), + token: finiteDuration(legDurationsRecord.token), + http: finiteDuration(legDurationsRecord.http), + }, + } + : null, + lastWedge: + typeof lastWedgeCommand === "string" + && lastWedgeCommand.trim() + && typeof lastWedgeBlockedMs === "number" + && Number.isFinite(lastWedgeBlockedMs) + && typeof lastWedgeTs === "string" + && lastWedgeTs.trim() + ? { + lastCommand: lastWedgeCommand.trim(), + blockedMs: Math.max(0, Math.floor(lastWedgeBlockedMs)), + ts: lastWedgeTs.trim(), + } + : null, + minCompatibleProtocol: + typeof minCompatibleProtocol === "number" + && Number.isInteger(minCompatibleProtocol) + && minCompatibleProtocol > 0 + ? minCompatibleProtocol + : null, + protocolVersion: + typeof protocolVersion === "number" + && Number.isInteger(protocolVersion) + && protocolVersion > 0 + ? protocolVersion + : null, }; } +function finiteDuration(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? Math.round(value) + : null; +} + export function computeLocalRuntimeBuildHash(cliPath = resolveCliScriptPath()): string | null { try { const content = fs.readFileSync(cliPath); @@ -699,6 +790,9 @@ export class LocalRuntimeConnectionPool { private activeConnection: LocalRuntimeConnection | null = null; private activeClient: RuntimeRpcClient | null = null; private activeRuntimePid: number | null = null; + private activeRuntimeSyncPort: number | null = null; + private activeRuntimePublishHealth: LocalRuntimeStatus["publishHealth"] = null; + private activeRuntimeLastWedge: LocalRuntimeStatus["lastWedge"] = null; private ownedRuntimeChild: ChildProcess | null = null; private isolatedRecoveryTimer: NodeJS.Timeout | null = null; private isolatedModeActive = false; @@ -757,6 +851,15 @@ export class LocalRuntimeConnectionPool { : this.connection ? "connecting" : "idle", + pid: this.activeRuntimePid, + syncPort: this.activeRuntimeSyncPort, + publishHealth: this.activeRuntimePublishHealth + ? { + ...this.activeRuntimePublishHealth, + lastLegDurations: { ...this.activeRuntimePublishHealth.lastLegDurations }, + } + : null, + lastWedge: this.activeRuntimeLastWedge ? { ...this.activeRuntimeLastWedge } : null, runtimeMode: this.isolatedModeActive ? "isolated" : "primary", versionSkew: { ...this.versionSkewStatus }, serviceInstall: { ...this.serviceInstallStatus }, @@ -945,7 +1048,32 @@ export class LocalRuntimeConnectionPool { return; } const socketPath = process.env.ADE_RUNTIME_SOCKET_PATH?.trim() || resolveMachineAdeLayout().socketPath; - const runningCompatibilityError = await this.probeRuntimeCompatibilityError(socketPath); + const runningProbe = await this.probeRuntimeCompatibility(socketPath); + const runningCompatibilityError = runningProbe?.error ?? null; + if ( + !runningCompatibilityError + && runningProbe + && runningProbe.compatibleNewer + ) { + this.clearVersionSkewStatus(); + this.serviceInstallStatus = { + state: "skipped", + attempted: false, + path: cliPath, + message: "Skipped ADE service install because the newer running brain is protocol-compatible.", + exitCode: null, + updatedAt: new Date().toISOString(), + }; + this.logger.info("local_runtime.service_install_skipped", { + cliPath, + socketPath, + reason: "compatible_newer_runtime", + runtimeVersion: runningProbe.runtimeInfo.version, + appVersion: this.appVersion, + runtimePid: runningProbe.runtimeInfo.pid, + }); + return; + } if (runningCompatibilityError?.skewState === "runtime_newer") { this.noteCompatibilityError(runningCompatibilityError); this.serviceInstallStatus = { @@ -1175,6 +1303,10 @@ export class LocalRuntimeConnectionPool { return coerceProjects(await entry.client.call("projects.list", {})); } + async activitySummary(): Promise { + return await this.callSync("runtime.activitySummary"); + } + async syncStatusForRoot(rootPath: string, args: SyncGetStatusArgs = {}): Promise { return await this.callSyncForRoot(rootPath, "sync.getStatus", { includeTransferReadiness: args.includeTransferReadiness === true, @@ -1480,6 +1612,9 @@ export class LocalRuntimeConnectionPool { this.activeConnection = null; this.activeClient = null; this.activeRuntimePid = null; + this.activeRuntimeSyncPort = null; + this.activeRuntimePublishHealth = null; + this.activeRuntimeLastWedge = null; this.ownedRuntimeChild = null; this.projectsByRoot.clear(); this.projectRegistrationsByRoot.clear(); @@ -1507,6 +1642,9 @@ export class LocalRuntimeConnectionPool { this.activeConnection = null; this.activeClient = null; this.activeRuntimePid = null; + this.activeRuntimeSyncPort = null; + this.activeRuntimePublishHealth = null; + this.activeRuntimeLastWedge = null; } throw error; }); @@ -1532,6 +1670,9 @@ export class LocalRuntimeConnectionPool { this.activeConnection = null; this.activeClient = null; this.activeRuntimePid = null; + this.activeRuntimeSyncPort = null; + this.activeRuntimePublishHealth = null; + this.activeRuntimeLastWedge = null; this.projectsByRoot.clear(); return true; } @@ -1556,11 +1697,19 @@ export class LocalRuntimeConnectionPool { }; } + private isCompatibleNewerRuntime( + runtimeInfo: ReturnType, + ): boolean { + return compareRuntimeVersionStrings(runtimeInfo.version, this.appVersion) === 1 + && isRuntimeProtocolCompatible(runtimeInfo); + } + private runtimeCompatibilityError( socketPath: string, - runtimeInfo: ReturnType, + runtimeInfo: ReturnType, ): LocalRuntimeCompatibilityError | null { const expectedBuildHash = computeLocalRuntimeBuildHash(); + let acceptedNewerRuntime = false; if (!isCompatibleRuntimeVersion({ runtimeVersion: runtimeInfo.version, appVersion: this.appVersion, @@ -1576,17 +1725,29 @@ export class LocalRuntimeConnectionPool { runtimePid: runtimeInfo.pid, }); const comparison = compareRuntimeVersionStrings(runtimeInfo.version, this.appVersion); - return new LocalRuntimeCompatibilityError( - `ADE service version ${runtimeInfo.version} does not match desktop version ${this.appVersion}.`, - runtimeInfo, - comparison == null || comparison === 0 - ? "unknown" - : comparison > 0 - ? "runtime_newer" - : "runtime_older", - ); + if (this.isCompatibleNewerRuntime(runtimeInfo)) { + acceptedNewerRuntime = true; + this.logger.info("local_runtime.newer_brain_accepted", { + socketPath, + runtimeVersion: runtimeInfo.version, + appVersion: this.appVersion, + runtimePid: runtimeInfo.pid, + minCompatibleProtocol: runtimeInfo.minCompatibleProtocol, + protocolVersion: runtimeInfo.protocolVersion, + }); + } else { + return new LocalRuntimeCompatibilityError( + `ADE service version ${runtimeInfo.version} does not match desktop version ${this.appVersion}.`, + runtimeInfo, + comparison == null || comparison === 0 + ? "unknown" + : comparison > 0 + ? "runtime_newer" + : "runtime_older", + ); + } } - if (expectedBuildHash && runtimeInfo.buildHash !== expectedBuildHash) { + if (!acceptedNewerRuntime && expectedBuildHash && runtimeInfo.buildHash !== expectedBuildHash) { this.logger.info("local_runtime.build_mismatch_detected", { socketPath, runtimeBuildHash: runtimeInfo.buildHash, @@ -1615,13 +1776,22 @@ export class LocalRuntimeConnectionPool { return null; } - private async probeRuntimeCompatibilityError(socketPath: string): Promise { + private async probeRuntimeCompatibility(socketPath: string): Promise<{ + error: LocalRuntimeCompatibilityError | null; + runtimeInfo: ReturnType; + compatibleNewer: boolean; + } | null> { let client: RuntimeRpcClient | null = null; try { const transport = await openSocketTransport(socketPath); client = new RuntimeRpcClient(transport); const initializeResult = await client.initialize("ade-desktop-service-install-probe", this.appVersion); - return this.runtimeCompatibilityError(socketPath, readRuntimeInfo(initializeResult)); + const runtimeInfo = readLocalRuntimeInfo(initializeResult); + return { + error: this.runtimeCompatibilityError(socketPath, runtimeInfo), + runtimeInfo, + compatibleNewer: this.isCompatibleNewerRuntime(runtimeInfo), + }; } catch { return null; } finally { @@ -1977,16 +2147,8 @@ export class LocalRuntimeConnectionPool { const transport = await openSocketTransport(socketPath); client = new RuntimeRpcClient(transport); const initializeResult = await client.initialize("ade-desktop-local-probe", this.appVersion); - const runtimeInfo = readRuntimeInfo(initializeResult); - const expectedBuildHash = computeLocalRuntimeBuildHash(); - return isCompatibleRuntimeVersion({ - runtimeVersion: runtimeInfo.version, - appVersion: this.appVersion, - runtimeBuildHash: runtimeInfo.buildHash, - expectedBuildHash, - }) - && (!expectedBuildHash || runtimeInfo.buildHash === expectedBuildHash) - && runtimeInfo.defaultRole === "cto"; + const runtimeInfo = readLocalRuntimeInfo(initializeResult); + return this.runtimeCompatibilityError(socketPath, runtimeInfo) == null; } catch { return false; } finally { @@ -2013,7 +2175,7 @@ export class LocalRuntimeConnectionPool { closeRuntimeClient(client); throw error; } - const runtimeInfo = readRuntimeInfo(initializeResult); + const runtimeInfo = readLocalRuntimeInfo(initializeResult); const compatibilityError = this.runtimeCompatibilityError(socketPath, runtimeInfo); if (compatibilityError) { closeRuntimeClient(client); @@ -2024,6 +2186,9 @@ export class LocalRuntimeConnectionPool { } this.activeClient = client; this.activeRuntimePid = runtimeInfo.pid; + this.activeRuntimeSyncPort = runtimeInfo.syncPort; + this.activeRuntimePublishHealth = runtimeInfo.publishHealth; + this.activeRuntimeLastWedge = runtimeInfo.lastWedge; client.onDisconnect((error) => { if (this.activeClient !== client && this.activeConnection?.client !== client) return; this.logger.warn("local_runtime.disconnected", { @@ -2034,6 +2199,12 @@ export class LocalRuntimeConnectionPool { this.activeConnection = null; this.activeClient = null; this.activeRuntimePid = null; + this.activeRuntimeSyncPort = null; + this.activeRuntimePublishHealth = null; + this.activeRuntimeLastWedge = null; + this.activeRuntimeSyncPort = null; + this.activeRuntimePublishHealth = null; + this.activeRuntimeLastWedge = null; this.projectsByRoot.clear(); }); return client; @@ -2080,6 +2251,9 @@ export class LocalRuntimeConnectionPool { this.activeConnection = null; this.activeClient = null; this.activeRuntimePid = null; + this.activeRuntimeSyncPort = null; + this.activeRuntimePublishHealth = null; + this.activeRuntimeLastWedge = null; this.projectsByRoot.clear(); if (client) closeRuntimeClient(client); }; diff --git a/apps/desktop/src/main/services/logging/logger.ts b/apps/desktop/src/main/services/logging/logger.ts index 476742334..ca2b58cd4 100644 --- a/apps/desktop/src/main/services/logging/logger.ts +++ b/apps/desktop/src/main/services/logging/logger.ts @@ -16,6 +16,14 @@ const ROTATION_CHECK_INTERVAL_MS = 60_000; const FLUSH_INTERVAL_MS = 500; const FLUSH_BATCH_SIZE = 500; +export type FileLoggerOptions = { + maxFileBytes?: number; + rotationCheckWriteInterval?: number; + rotationCheckIntervalMs?: number; + flushIntervalMs?: number; + flushBatchSize?: number; +}; + export type Logger = { debug: (event: string, meta?: Record) => void; info: (event: string, meta?: Record) => void; @@ -50,10 +58,20 @@ function createConsoleMirror(level: LogLevel, event: string, meta?: Record { - if (writesSinceRotateCheck >= ROTATION_CHECK_WRITE_INTERVAL) return true; - return Date.now() - lastRotateCheckAt >= ROTATION_CHECK_INTERVAL_MS; + if (writesSinceRotateCheck >= rotationCheckWriteInterval) return true; + return Date.now() - lastRotateCheckAt >= rotationCheckIntervalMs; }; const ensureLogDir = (): boolean => { @@ -111,7 +129,7 @@ export function createFileLogger(logFilePath: string): Logger { const rotateIfNeeded = async (upcomingWriteBytes: number): Promise => { refreshEstimatedFileSizeIfNeeded(); const currentFileSize = estimatedFileSize ?? 0; - if (currentFileSize < MAX_LOG_FILE_BYTES && currentFileSize + upcomingWriteBytes <= MAX_LOG_FILE_BYTES) return; + if (currentFileSize < maxFileBytes && currentFileSize + upcomingWriteBytes <= maxFileBytes) return; await closeLogStream(); @@ -172,7 +190,7 @@ export function createFileLogger(logFilePath: string): Logger { flushTimer = null; } - const lines = queuedLines.splice(0, FLUSH_BATCH_SIZE); + const lines = queuedLines.splice(0, flushBatchSize); const payload = lines.join(""); const bytes = Buffer.byteLength(payload, "utf8"); flushInProgress = true; @@ -194,7 +212,7 @@ export function createFileLogger(logFilePath: string): Logger { }; const scheduleFlush = () => { - if (queuedLines.length >= FLUSH_BATCH_SIZE) { + if (queuedLines.length >= flushBatchSize) { void flush(); return; } @@ -202,7 +220,7 @@ export function createFileLogger(logFilePath: string): Logger { flushTimer = setTimeout(() => { flushTimer = null; void flush(); - }, FLUSH_INTERVAL_MS); + }, flushIntervalMs); flushTimer.unref?.(); }; diff --git a/apps/desktop/src/main/services/updates/autoUpdateService.test.ts b/apps/desktop/src/main/services/updates/autoUpdateService.test.ts index 6ccd83790..a42c40af9 100644 --- a/apps/desktop/src/main/services/updates/autoUpdateService.test.ts +++ b/apps/desktop/src/main/services/updates/autoUpdateService.test.ts @@ -319,6 +319,7 @@ describe("createAutoUpdateService", () => { expect(service.getSnapshot()).toMatchObject({ status: "ready", + latestKnownVersion: "1.2.3", version: "1.2.3", progressPercent: 100, releaseNotesUrl: "https://www.ade-app.dev/docs/changelog/v1.2.3", @@ -416,15 +417,8 @@ describe("createAutoUpdateService", () => { expect(getDiskSpace).toHaveBeenCalledWith("/Applications/ADE.app/Contents/MacOS/ADE"); expect(updater.quitAndInstall).not.toHaveBeenCalled(); expect(service.getSnapshot()).toMatchObject({ - status: "error", - errorDetails: { - kind: "insufficient_space", - phase: "install", - availableBytes: 1024 * 1024 * 1024, - requiredBytes: 2012 * 1024 * 1024, - volumePath: "/System/Volumes/Data", - preservesDownload: true, - }, + status: "ready", + parked: { reason: "install_preflight_failed" }, }); expect(fs.readdirSync(updaterCacheDir).sort()).toEqual(["pending", "update.zip"]); @@ -441,9 +435,10 @@ describe("createAutoUpdateService", () => { const updater = Object.assign(new FakeAutoUpdater(), { downloadUpdate: vi.fn(async () => undefined), }); + let installSpaceAvailable = false; const getDiskSpace = vi.fn((targetPath: string) => ({ availableBytes: targetPath === installTargetPath - ? 1024 * 1024 * 1024 + ? (installSpaceAvailable ? 4 : 1) * 1024 * 1024 * 1024 : 64 * 1024 * 1024, volumePath: targetPath === installTargetPath ? "/System/Volumes/Data" : "/Volumes/Cache", })); @@ -461,30 +456,14 @@ describe("createAutoUpdateService", () => { await expect(service.quitAndInstall()).resolves.toBe(false); expect(service.getSnapshot()).toMatchObject({ - status: "error", + status: "ready", version: "1.2.3", - errorDetails: { - kind: "insufficient_space", - phase: "install", - preservesDownload: true, - }, + parked: { reason: "install_preflight_failed" }, }); - - updater.checkForUpdates.mockImplementationOnce(async () => { - updater.emit("checking-for-update"); - updater.emit("update-available", updateInfo); - return { updateInfo }; - }); - updater.downloadUpdate.mockImplementationOnce(async () => { - updater.emit("update-downloaded", updateInfo); - }); - - service.checkForUpdates(); - service.checkForUpdates(); - - await vi.waitFor(() => expect(service.getSnapshot().status).toBe("ready")); + installSpaceAvailable = true; + await expect(service.quitAndInstall()).resolves.toBe(true); expect(updater.checkForUpdates).toHaveBeenCalledTimes(2); - expect(updater.downloadUpdate).toHaveBeenCalledTimes(1); + expect(updater.downloadUpdate).not.toHaveBeenCalled(); expect(getDiskSpace).not.toHaveBeenCalledWith(updaterCacheDir); service.dispose(); @@ -513,7 +492,10 @@ describe("createAutoUpdateService", () => { updater.emit("update-downloaded", updateInfo); await expect(service.quitAndInstall()).resolves.toBe(false); - expect(service.getSnapshot().errorDetails?.preservesDownload).toBe(true); + expect(service.getSnapshot()).toMatchObject({ + status: "ready", + parked: { reason: "install_preflight_failed" }, + }); updater.checkForUpdates.mockImplementationOnce(async () => { updater.emit("checking-for-update"); @@ -522,20 +504,12 @@ describe("createAutoUpdateService", () => { throw error; }); - service.checkForUpdates(); - - await vi.waitFor(() => { - expect(service.getSnapshot()).toMatchObject({ - status: "error", - version: "1.2.3", - releaseNotesUrl: "https://www.ade-app.dev/docs/changelog/v1.2.3", - error: "network unavailable", - errorDetails: { - kind: "network", - phase: "download", - preservesDownload: true, - }, - }); + await expect(service.quitAndInstall()).resolves.toBe(false); + expect(service.getSnapshot()).toMatchObject({ + status: "ready", + version: "1.2.3", + releaseNotesUrl: "https://www.ade-app.dev/docs/changelog/v1.2.3", + parked: { reason: "refresh_failed" }, }); expect(fs.readdirSync(updaterCacheDir).sort()).toEqual(["pending", "update.zip"]); @@ -567,12 +541,8 @@ describe("createAutoUpdateService", () => { await expect(service.quitAndInstall()).resolves.toBe(false); expect(service.getSnapshot()).toMatchObject({ - status: "error", - errorDetails: { - kind: "disk_full", - phase: "install", - preservesDownload: true, - }, + status: "ready", + parked: { reason: "handoff_failed" }, }); expect(fs.readdirSync(updaterCacheDir).sort()).toEqual(["pending", "update.zip"]); @@ -722,10 +692,10 @@ describe("createAutoUpdateService", () => { status: "installing", version: "1.2.4", }); - expectCacheEmpty(updaterCacheDir); - expect(logger.info).toHaveBeenCalledWith( + expect(fs.readdirSync(updaterCacheDir).sort()).toEqual(["pending", "update.zip"]); + expect(logger.info).not.toHaveBeenCalledWith( "autoUpdate.cache_cleaned", - expect.objectContaining({ reason: "superseded_ready_update", entriesRemoved: 2 }), + expect.objectContaining({ reason: "superseded_ready_update" }), ); service.dispose(); @@ -814,10 +784,10 @@ describe("createAutoUpdateService", () => { expect(updater.quitAndInstall).not.toHaveBeenCalled(); expect(readState(globalStatePath)).toEqual({}); expect(service.getSnapshot()).toMatchObject({ - status: "error", - error: "Could not verify the latest update before installing: network unavailable", + status: "ready", + parked: { reason: "refresh_failed" }, }); - expectCacheEmpty(updaterCacheDir); + expect(fs.readdirSync(updaterCacheDir).sort()).toEqual(["pending", "update.zip"]); expect(logger.warn).toHaveBeenCalledWith( "autoUpdate.refresh_ready_before_install_failed", expect.objectContaining({ @@ -866,15 +836,12 @@ describe("createAutoUpdateService", () => { expect(updater.quitAndInstall).not.toHaveBeenCalled(); expect(service.getSnapshot()).toMatchObject({ - status: "error", - version: "1.2.4", - errorDetails: { - kind: "disk_full", - phase: "download", - preservesDownload: false, - }, + status: "ready", + version: "1.2.3", + latestKnownVersion: "1.2.4", + parked: { reason: "refresh_failed" }, }); - expectCacheEmpty(updaterCacheDir); + expect(fs.readdirSync(updaterCacheDir).sort()).toEqual(["pending", "update.zip"]); service.dispose(); }); @@ -913,13 +880,10 @@ describe("createAutoUpdateService", () => { expect(updater.downloadUpdate).not.toHaveBeenCalled(); expect(service.getSnapshot()).toMatchObject({ - status: "error", - version: "1.2.4", - errorDetails: { - kind: "insufficient_space", - phase: "download", - requiredBytes: 1536 * 1024 * 1024, - }, + status: "ready", + version: "1.2.3", + latestKnownVersion: "1.2.4", + parked: { reason: "refresh_failed" }, }); service.dispose(); @@ -968,13 +932,9 @@ describe("createAutoUpdateService", () => { await expect(service.quitAndInstall()).resolves.toBe(false); expect(service.getSnapshot()).toMatchObject({ - status: "error", + status: "ready", version: "1.2.4", - errorDetails: { - kind: "insufficient_space", - phase: "install", - requiredBytes: 2012 * 1024 * 1024, - }, + parked: { reason: "install_preflight_failed" }, }); expect(updater.quitAndInstall).not.toHaveBeenCalled(); @@ -1034,8 +994,8 @@ describe("createAutoUpdateService", () => { await expect(service.quitAndInstall()).resolves.toBe(false); expect(service.getSnapshot()).toMatchObject({ - status: "error", - error: "The command is disabled and cannot be executed", + status: "ready", + parked: { reason: "handoff_failed" }, }); expect(readState(globalStatePath)).toEqual({}); expect(fs.readdirSync(updaterCacheDir).sort()).toEqual(["pending", "update.zip"]); @@ -1092,6 +1052,7 @@ describe("createAutoUpdateService", () => { const updaterCacheDir = makeUpdaterCacheDir(); const logger = makeLogger(); const updater = new FakeAutoUpdater(); + const rollbackQuitAndInstall = vi.fn(async () => {}); const beforeQuitAndInstall = vi.fn(async () => { throw new Error("Could not stop ADE service"); }); @@ -1104,6 +1065,7 @@ describe("createAutoUpdateService", () => { periodicCheckMs: 60_000, updater, beforeQuitAndInstall, + rollbackQuitAndInstall, }); updater.emit("update-downloaded", { @@ -1114,9 +1076,14 @@ describe("createAutoUpdateService", () => { expect(updater.quitAndInstall).not.toHaveBeenCalled(); expect(readState(globalStatePath)).toEqual({}); expect(service.getSnapshot()).toMatchObject({ - status: "error", - error: "Could not stop ADE service", + status: "ready", + version: "1.2.3", + parked: { + reason: "prepare_failed", + at: expect.any(Number), + }, }); + expect(rollbackQuitAndInstall).toHaveBeenCalledWith("prepare_failed"); expect(logger.warn).toHaveBeenCalledWith( "autoUpdate.prepare_quit_and_install_failed", expect.objectContaining({ @@ -1124,6 +1091,10 @@ describe("createAutoUpdateService", () => { message: "Could not stop ADE service", }), ); + expect(logger.warn).toHaveBeenCalledWith( + "autoUpdate.install_aborted", + expect.objectContaining({ reason: "prepare_failed", version: "1.2.3" }), + ); service.dispose(); }); @@ -1157,14 +1128,11 @@ describe("createAutoUpdateService", () => { updater.emit("error", new Error("installer failed after launch")); - expect(service.getSnapshot()).toMatchObject({ - status: "error", - error: "installer failed after launch", - errorDetails: { - kind: "installer", - phase: "install", - preservesDownload: true, - }, + await vi.waitFor(() => { + expect(service.getSnapshot()).toMatchObject({ + status: "ready", + parked: { reason: "handoff_failed" }, + }); }); expect(readState(globalStatePath)).toEqual({}); expect(fs.readdirSync(updaterCacheDir).sort()).toEqual(["pending", "update.zip"]); @@ -1172,18 +1140,160 @@ describe("createAutoUpdateService", () => { service.dispose(); }); - it("allows native handoff to outlive the prepare watchdog before reporting a stall", async () => { + it("auto-applies a ready update after two continuously idle minutes and the countdown", async () => { + vi.useFakeTimers(); + const updater = new FakeAutoUpdater(); + const productAnalyticsService = { captureInternal: vi.fn() }; + const service = createAutoUpdateService({ + logger: makeLogger(), + currentVersion: "1.2.2", + globalStatePath: makeStatePath(), + updaterCacheDir: makeUpdaterCacheDir(), + getDiskSpace: () => ({ + availableBytes: 20 * 1024 * 1024 * 1024, + volumePath: "/System/Volumes/Data", + }), + autoCheckEnabled: false, + autoApplyEnabled: true, + activityCheckMs: 5_000, + autoApplyIdleMs: 2 * 60_000, + autoApplyCountdownMs: 10_000, + getRuntimeActivitySummary: vi.fn(async () => ({ idle: true })), + productAnalyticsService: productAnalyticsService as never, + updater, + }); + updater.emit("update-downloaded", { version: "1.2.3" }); + + await vi.advanceTimersByTimeAsync(2 * 60_000); + + expect(service.getSnapshot()).toMatchObject({ + status: "ready", + autoApplyPending: { deadlineAt: expect.any(Number) }, + }); + expect(updater.quitAndInstall).not.toHaveBeenCalled(); + + await vi.advanceTimersByTimeAsync(10_000); + + expect(updater.quitAndInstall).toHaveBeenCalledWith(false, true); + expect(productAnalyticsService.captureInternal).toHaveBeenCalledWith({ + event: "ade_update_auto_applied", + surface: "desktop", + }); + + service.dispose(); + }); + + it("restarts the idle window when runtime activity resumes", async () => { + vi.useFakeTimers(); + let idle = true; + const updater = new FakeAutoUpdater(); + const service = createAutoUpdateService({ + logger: makeLogger(), + currentVersion: "1.2.2", + globalStatePath: makeStatePath(), + updaterCacheDir: makeUpdaterCacheDir(), + autoCheckEnabled: false, + autoApplyEnabled: true, + activityCheckMs: 5_000, + autoApplyIdleMs: 2 * 60_000, + getRuntimeActivitySummary: vi.fn(async () => ({ idle })), + updater, + }); + updater.emit("update-downloaded", { version: "1.2.3" }); + + await vi.advanceTimersByTimeAsync(60_000); + idle = false; + await vi.advanceTimersByTimeAsync(5_000); + idle = true; + await vi.advanceTimersByTimeAsync(124_999); + + expect(service.getSnapshot().autoApplyPending).toBeNull(); + + await vi.advanceTimersByTimeAsync(1); + expect(service.getSnapshot().autoApplyPending).toEqual({ + deadlineAt: expect.any(Number), + }); + + service.dispose(); + }); + + it("cancels an idle auto-apply countdown and suppresses it for four hours", async () => { + vi.useFakeTimers(); + const updater = new FakeAutoUpdater(); + const productAnalyticsService = { captureInternal: vi.fn() }; + const service = createAutoUpdateService({ + logger: makeLogger(), + currentVersion: "1.2.2", + globalStatePath: makeStatePath(), + updaterCacheDir: makeUpdaterCacheDir(), + autoCheckEnabled: false, + autoApplyEnabled: true, + activityCheckMs: 5_000, + autoApplyIdleMs: 2 * 60_000, + autoApplyCountdownMs: 10_000, + autoApplySuppressionMs: 4 * 60 * 60_000, + getRuntimeActivitySummary: vi.fn(async () => ({ idle: true })), + productAnalyticsService: productAnalyticsService as never, + updater, + }); + updater.emit("update-downloaded", { version: "1.2.3" }); + await vi.advanceTimersByTimeAsync(2 * 60_000); + const beforeCancel = Date.now(); + + expect(service.cancelAutoApply()).toBe(true); + + expect(service.getSnapshot()).toMatchObject({ + status: "ready", + autoApplyPending: null, + autoApplySuppressedUntil: beforeCancel + 4 * 60 * 60_000, + }); + await vi.advanceTimersByTimeAsync(10_000); + expect(updater.quitAndInstall).not.toHaveBeenCalled(); + expect(productAnalyticsService.captureInternal).toHaveBeenCalledWith({ + event: "ade_update_auto_apply_cancelled", + surface: "desktop", + }); + + service.dispose(); + }); + + it("preserves the latest known version when an updater download is cancelled", () => { + const updater = new FakeAutoUpdater(); + const service = createAutoUpdateService({ + logger: makeLogger(), + currentVersion: "1.2.2", + globalStatePath: makeStatePath(), + updaterCacheDir: makeUpdaterCacheDir(), + autoCheckEnabled: false, + updater, + }); + updater.emit("update-downloaded", { version: "1.2.3" }); + + updater.emit("update-cancelled", { version: "1.2.3" }); + + expect(service.getSnapshot()).toMatchObject({ + status: "idle", + latestKnownVersion: "1.2.3", + }); + + service.dispose(); + }); + + it("force-quits when the native handoff does not reach will-quit by the hard deadline", async () => { vi.useFakeTimers(); const globalStatePath = makeStatePath(); const updaterCacheDir = makeUpdaterCacheDir(); const updater = new FakeAutoUpdater(); + const logger = makeLogger(); + const forceQuit = vi.fn(); const service = createAutoUpdateService({ - logger: makeLogger(), + logger, currentVersion: "1.2.2", globalStatePath, updaterCacheDir, installWatchdogMs: 1_000, - nativeHandoffWatchdogMs: 5_000, + quitDeadlineMs: 5_000, + forceQuit, getDiskSpace: () => ({ availableBytes: 20 * 1024 * 1024 * 1024, volumePath: "/System/Volumes/Data", @@ -1205,15 +1315,15 @@ describe("createAutoUpdateService", () => { await vi.advanceTimersByTimeAsync(4_000); - expect(service.getSnapshot()).toMatchObject({ - status: "error", - error: "ADE did not quit for the update. Free space if needed, then try again.", - errorDetails: { - kind: "installer", - phase: "install", - preservesDownload: true, - }, + expect(forceQuit).toHaveBeenCalledWith({ + blockedPhase: "app_quit", + blockedMs: 5_000, }); + expect(logger.error).toHaveBeenCalledWith("autoUpdate.quit_escalated", { + blockedPhase: "app_quit", + blockedMs: 5_000, + }); + expect(service.getSnapshot().status).toBe("installing"); expect(readState(globalStatePath)).toMatchObject({ pendingInstallUpdate: { targetVersion: "1.2.3" }, }); diff --git a/apps/desktop/src/main/services/updates/autoUpdateService.ts b/apps/desktop/src/main/services/updates/autoUpdateService.ts index 82a8f24a4..e1e02f464 100644 --- a/apps/desktop/src/main/services/updates/autoUpdateService.ts +++ b/apps/desktop/src/main/services/updates/autoUpdateService.ts @@ -5,11 +5,13 @@ import { autoUpdater, type UpdateInfo } from "electron-updater"; import type { AutoUpdateErrorDetails, AutoUpdateErrorKind, + AutoUpdateInstallAbortReason, AutoUpdatePhase, AutoUpdateSnapshot, RecentlyInstalledUpdate, } from "../../../shared/types"; import type { Logger } from "../logging/logger"; +import type { ProductAnalyticsService } from "../analytics/productAnalyticsService"; import { readGlobalState, writeGlobalState, type GlobalState } from "../state/globalState"; import { classifyUpdateError, @@ -22,7 +24,11 @@ import { const DEFAULT_RELEASE_NOTES_BASE_URL = "https://www.ade-app.dev"; const DEFAULT_INSTALL_WATCHDOG_MS = 30_000; -const DEFAULT_NATIVE_HANDOFF_WATCHDOG_MS = 5 * 60_000; +const DEFAULT_QUIT_DEADLINE_MS = 10_000; +const DEFAULT_AUTO_APPLY_IDLE_MS = 2 * 60_000; +const DEFAULT_AUTO_APPLY_COUNTDOWN_MS = 10_000; +const DEFAULT_AUTO_APPLY_SUPPRESSION_MS = 4 * 60 * 60_000; +const DEFAULT_ACTIVITY_CHECK_MS = 5_000; type AutoUpdaterLike = { logger: typeof autoUpdater.logger; @@ -46,7 +52,12 @@ type CreateAutoUpdateServiceArgs = { globalStatePath: string; updater?: AutoUpdaterLike; beforeQuitAndInstall?: () => void | Promise; + rollbackQuitAndInstall?: (reason: string) => void | Promise; + forceQuit?: (args: { blockedPhase: string; blockedMs: number }) => void; + getRuntimeActivitySummary?: () => Promise<{ idle: boolean }>; + productAnalyticsService?: Pick; now?: () => string; + nowMs?: () => number; releaseNotesBaseUrl?: string; startupDelayMs?: number; periodicCheckMs?: number; @@ -54,8 +65,13 @@ type CreateAutoUpdateServiceArgs = { installTargetPath?: string; getDiskSpace?: (targetPath: string) => DiskSpaceInfo; installWatchdogMs?: number; - nativeHandoffWatchdogMs?: number; + quitDeadlineMs?: number; + autoApplyIdleMs?: number; + autoApplyCountdownMs?: number; + autoApplySuppressionMs?: number; + activityCheckMs?: number; autoCheckEnabled?: boolean; + autoApplyEnabled?: boolean; }; type UpdateCheckResultLike = { @@ -79,6 +95,7 @@ export function createEmptyAutoUpdateSnapshot(currentVersion = ""): AutoUpdateSn return { status: "idle", currentVersion, + latestKnownVersion: null, version: null, progressPercent: null, bytesPerSecond: null, @@ -88,6 +105,9 @@ export function createEmptyAutoUpdateSnapshot(currentVersion = ""): AutoUpdateSn error: null, errorDetails: null, recentlyInstalled: null, + parked: null, + autoApplyPending: null, + autoApplySuppressedUntil: null, }; } @@ -197,6 +217,8 @@ function cloneSnapshot(snapshot: AutoUpdateSnapshot): AutoUpdateSnapshot { return { ...snapshot, recentlyInstalled: cloneRecentlyInstalledUpdate(snapshot.recentlyInstalled), + parked: snapshot.parked ? { ...snapshot.parked } : null, + autoApplyPending: snapshot.autoApplyPending ? { ...snapshot.autoApplyPending } : null, }; } @@ -298,6 +320,7 @@ function applyUpdateInfo( releaseNotesBaseUrl: string, ): Partial { return { + latestKnownVersion: info.version, version: info.version, releaseNotesUrl: buildReleaseNotesUrl(info.version, releaseNotesBaseUrl), error: null, @@ -311,7 +334,12 @@ export function createAutoUpdateService({ globalStatePath, updater = autoUpdater as unknown as AutoUpdaterLike, beforeQuitAndInstall, + rollbackQuitAndInstall, + forceQuit, + getRuntimeActivitySummary, + productAnalyticsService, now = () => new Date().toISOString(), + nowMs = () => Date.now(), releaseNotesBaseUrl = DEFAULT_RELEASE_NOTES_BASE_URL, startupDelayMs = 5_000, periodicCheckMs = 30 * 60 * 1_000, @@ -319,8 +347,13 @@ export function createAutoUpdateService({ installTargetPath = process.execPath, getDiskSpace = readDiskSpace, installWatchdogMs = DEFAULT_INSTALL_WATCHDOG_MS, - nativeHandoffWatchdogMs = DEFAULT_NATIVE_HANDOFF_WATCHDOG_MS, + quitDeadlineMs = DEFAULT_QUIT_DEADLINE_MS, + autoApplyIdleMs = DEFAULT_AUTO_APPLY_IDLE_MS, + autoApplyCountdownMs = DEFAULT_AUTO_APPLY_COUNTDOWN_MS, + autoApplySuppressionMs = DEFAULT_AUTO_APPLY_SUPPRESSION_MS, + activityCheckMs = DEFAULT_ACTIVITY_CHECK_MS, autoCheckEnabled = true, + autoApplyEnabled = autoCheckEnabled && process.env.ADE_DISABLE_AUTO_UPDATE_APPLY !== "1", }: CreateAutoUpdateServiceArgs) { updater.logger = null; // Manual download is required so ADE can preflight the updater cache volume @@ -379,6 +412,7 @@ export function createAutoUpdateService({ // call sets this; subsequent calls return the same promise so we never // race the actual install. let quitAndInstallPromise: Promise | null = null; + let installReadySnapshot: AutoUpdateSnapshot | null = null; let ignoredDownloadVersion: string | null = null; let readyRefreshInProgress = false; const readyRefreshFailure: { @@ -388,7 +422,13 @@ export function createAutoUpdateService({ let compressedUpdateBytes: number | null = null; let compressedUpdateVersion: string | null = null; let preservedDownloadRetry: PreservedDownloadRetry | null = null; - let installWatchdog: ReturnType | null = null; + let quitDeadlineTimer: ReturnType | null = null; + let activityCheckTimer: ReturnType | null = null; + let autoApplyDeadlineTimer: ReturnType | null = null; + let idleSinceMs: number | null = null; + let installQuitArmed = false; + let activityCheckFailed = false; + let activityCheckInProgress = false; const listeners = new Set<(snapshot: AutoUpdateSnapshot) => void>(); function emit(): void { @@ -399,8 +439,17 @@ export function createAutoUpdateService({ } function patchSnapshot(partial: Partial): void { - snapshot = { ...snapshot, ...partial }; + const previousStatus = snapshot.status; + let nextSnapshot = { ...snapshot, ...partial }; + if (nextSnapshot.status !== "ready") { + resetAutoApplyTracking(false); + nextSnapshot = { ...nextSnapshot, autoApplyPending: null }; + } + snapshot = nextSnapshot; emit(); + if (snapshot.status === "ready" && previousStatus !== "ready") { + scheduleActivityCheck(0); + } } function clearPendingInstallUpdate(): void { @@ -412,10 +461,22 @@ export function createAutoUpdateService({ }); } - function clearInstallWatchdog(): void { - if (!installWatchdog) return; - clearTimeout(installWatchdog); - installWatchdog = null; + function clearQuitDeadline(): void { + if (!quitDeadlineTimer) return; + clearTimeout(quitDeadlineTimer); + quitDeadlineTimer = null; + } + + function clearAutoApplyDeadline(): void { + if (!autoApplyDeadlineTimer) return; + clearTimeout(autoApplyDeadlineTimer); + autoApplyDeadlineTimer = null; + } + + function clearActivityCheck(): void { + if (!activityCheckTimer) return; + clearTimeout(activityCheckTimer); + activityCheckTimer = null; } function rememberUpdateSize(info: UpdateInfo | null | undefined): void { @@ -566,6 +627,7 @@ export function createAutoUpdateService({ if (isTerminalSnapshotStatus()) { return { status: snapshot.status, + latestKnownVersion: snapshot.latestKnownVersion, version: snapshot.version, progressPercent: snapshot.progressPercent, bytesPerSecond: snapshot.bytesPerSecond, @@ -574,10 +636,14 @@ export function createAutoUpdateService({ releaseNotesUrl: snapshot.releaseNotesUrl, error: null, errorDetails: null, + parked: snapshot.parked, + autoApplyPending: snapshot.autoApplyPending, + autoApplySuppressedUntil: snapshot.autoApplySuppressedUntil, }; } return { status: idleStatus, + latestKnownVersion: snapshot.latestKnownVersion, version: null, progressPercent: null, bytesPerSecond: null, @@ -586,6 +652,8 @@ export function createAutoUpdateService({ releaseNotesUrl: null, error: null, errorDetails: null, + parked: null, + autoApplyPending: null, }; } @@ -608,16 +676,20 @@ export function createAutoUpdateService({ }); return; } - cleanupUpdaterCacheDir({ - updaterCacheDir, - logger, - reason: "superseded_ready_update", - }); + if (!readyRefreshInProgress) { + cleanupUpdaterCacheDir({ + updaterCacheDir, + logger, + reason: "superseded_ready_update", + }); + } } ignoredDownloadVersion = null; rememberUpdateSize(info); patchSnapshot({ status: "checking", + parked: null, + autoApplyPending: null, progressPercent: null, bytesPerSecond: null, transferredBytes: null, @@ -666,6 +738,8 @@ export function createAutoUpdateService({ rememberUpdateSize(info); patchSnapshot({ status: "ready", + parked: null, + autoApplyPending: null, progressPercent: 100, bytesPerSecond: null, transferredBytes: null, @@ -674,8 +748,11 @@ export function createAutoUpdateService({ }); }; - const onUpdateNotAvailable = () => { + const onUpdateNotAvailable = (info?: UpdateInfo) => { logger.info("autoUpdate.update_not_available"); + if (info?.version) { + patchSnapshot({ latestKnownVersion: info.version }); + } if (!isTerminalSnapshotStatus()) { compressedUpdateBytes = null; compressedUpdateVersion = null; @@ -700,7 +777,9 @@ export function createAutoUpdateService({ }); patchSnapshot({ ...createEmptyAutoUpdateSnapshot(currentVersion), + latestKnownVersion: snapshot.latestKnownVersion ?? info.version, recentlyInstalled: snapshot.recentlyInstalled, + autoApplySuppressedUntil: snapshot.autoApplySuppressedUntil, }); }; @@ -720,9 +799,12 @@ export function createAutoUpdateService({ }; return; } - clearInstallWatchdog(); if (snapshot.status === "installing") { clearPendingInstallUpdate(); + clearQuitDeadline(); + installQuitArmed = false; + void abortInstall("handoff_failed", installReadySnapshot ?? snapshot); + return; } const preservedUpdate = preservedUpdateForRetryError(err); setErrorSnapshot({ @@ -770,6 +852,9 @@ export function createAutoUpdateService({ ) { rememberUpdateSize(updateInfo); } + if (updateInfo?.version) { + patchSnapshot({ latestKnownVersion: updateInfo.version }); + } if (snapshot.status === "ready" || ignoredDownloadVersion) { ignoredDownloadVersion = null; return; @@ -841,16 +926,10 @@ export function createAutoUpdateService({ const refreshFailure = readReadyRefreshFailure(); if (refreshFailure) { const failureMessage = formatErrorMessage(refreshFailure.error); - const error = `Could not verify the latest update before installing: ${failureMessage}`; logger.warn("autoUpdate.refresh_ready_before_install_failed", { version: readyVersion, message: failureMessage, }); - setErrorSnapshot({ - error: refreshFailure.error, - fallbackPhase: refreshFailure.phase, - message: error, - }); return false; } if (snapshot.status === "ready" && snapshot.version) { @@ -875,13 +954,6 @@ export function createAutoUpdateService({ installWatchdogMs, message: timeoutMessage, }); - clearPendingInstallUpdate(); - setErrorSnapshot({ - error: new Error(timeoutMessage), - fallbackPhase: phase, - message: timeoutMessage, - preservesDownload: true, - }); resolve({ completed: false }); }, installWatchdogMs); }); @@ -893,6 +965,181 @@ export function createAutoUpdateService({ } } + function resetAutoApplyTracking(clearPending: boolean): void { + clearActivityCheck(); + clearAutoApplyDeadline(); + idleSinceMs = null; + if (clearPending && snapshot.autoApplyPending) { + patchSnapshot({ autoApplyPending: null }); + } + } + + function scheduleActivityCheck(delayMs: number): void { + if ( + !autoApplyEnabled + || !getRuntimeActivitySummary + || snapshot.status !== "ready" + || quitAndInstallPromise != null + || installQuitArmed + || activityCheckInProgress + || activityCheckTimer + ) { + return; + } + activityCheckTimer = setTimeout(() => { + activityCheckTimer = null; + void checkAutoApplyActivity(); + }, Math.max(0, delayMs)); + activityCheckTimer.unref?.(); + } + + function clearPendingAutoApply(): void { + clearAutoApplyDeadline(); + if (!snapshot.autoApplyPending) return; + patchSnapshot({ autoApplyPending: null }); + } + + async function checkAutoApplyActivity(): Promise { + if ( + activityCheckInProgress + || !autoApplyEnabled + || !getRuntimeActivitySummary + || snapshot.status !== "ready" + || quitAndInstallPromise != null + || installQuitArmed + ) { + return; + } + const currentMs = nowMs(); + const suppressedUntil = snapshot.autoApplySuppressedUntil; + if (suppressedUntil != null && suppressedUntil > currentMs) { + idleSinceMs = null; + clearPendingAutoApply(); + scheduleActivityCheck(suppressedUntil - currentMs); + return; + } + if (suppressedUntil != null) { + patchSnapshot({ autoApplySuppressedUntil: null }); + } + + activityCheckInProgress = true; + try { + const activity = await getRuntimeActivitySummary(); + if (snapshot.status !== "ready" || quitAndInstallPromise != null || installQuitArmed) return; + if (activityCheckFailed) { + activityCheckFailed = false; + logger.info("autoUpdate.activity_check_recovered"); + } + if (!activity.idle) { + idleSinceMs = null; + clearPendingAutoApply(); + return; + } + const checkedAt = nowMs(); + idleSinceMs ??= checkedAt; + if (snapshot.autoApplyPending || checkedAt - idleSinceMs < autoApplyIdleMs) return; + const deadlineAt = checkedAt + autoApplyCountdownMs; + patchSnapshot({ autoApplyPending: { deadlineAt } }); + autoApplyDeadlineTimer = setTimeout(() => { + autoApplyDeadlineTimer = null; + if ( + snapshot.status !== "ready" + || snapshot.autoApplyPending?.deadlineAt !== deadlineAt + || (snapshot.autoApplySuppressedUntil ?? 0) > nowMs() + ) { + return; + } + patchSnapshot({ autoApplyPending: null }); + void quitAndInstall().then((started) => { + if (!started) return; + productAnalyticsService?.captureInternal({ + event: "ade_update_auto_applied", + surface: "desktop", + }); + }); + }, autoApplyCountdownMs); + autoApplyDeadlineTimer.unref?.(); + } catch (error) { + idleSinceMs = null; + clearPendingAutoApply(); + if (!activityCheckFailed) { + activityCheckFailed = true; + logger.warn("autoUpdate.activity_check_failed", { + message: formatErrorMessage(error), + }); + } + } finally { + activityCheckInProgress = false; + scheduleActivityCheck(activityCheckMs); + } + } + + async function abortInstall( + reason: AutoUpdateInstallAbortReason, + readySnapshot: AutoUpdateSnapshot, + ): Promise { + clearQuitDeadline(); + installQuitArmed = false; + clearPendingInstallUpdate(); + resetAutoApplyTracking(true); + try { + await rollbackQuitAndInstall?.(reason); + } catch (error) { + logger.error("autoUpdate.install_rollback_failed", { + reason, + message: formatErrorMessage(error), + }); + } + logger.warn("autoUpdate.install_aborted", { + reason, + version: readySnapshot.version, + }); + productAnalyticsService?.captureInternal({ + event: "ade_update_install_aborted", + surface: "desktop", + properties: { reason }, + }); + patchSnapshot({ + ...readySnapshot, + status: "ready", + latestKnownVersion: snapshot.latestKnownVersion ?? readySnapshot.latestKnownVersion, + progressPercent: 100, + bytesPerSecond: null, + transferredBytes: null, + totalBytes: null, + error: null, + errorDetails: null, + parked: { reason, at: nowMs() }, + autoApplyPending: null, + }); + installReadySnapshot = null; + scheduleActivityCheck(0); + return false; + } + + function armQuitDeadline(): void { + clearQuitDeadline(); + installQuitArmed = true; + const startedAt = nowMs(); + quitDeadlineTimer = setTimeout(() => { + quitDeadlineTimer = null; + if (!installQuitArmed) return; + const blockedMs = Math.max(0, nowMs() - startedAt); + const blockedPhase = "app_quit"; + logger.error("autoUpdate.quit_escalated", { + blockedPhase, + blockedMs, + }); + productAnalyticsService?.captureInternal({ + event: "ade_update_quit_escalated", + surface: "desktop", + properties: { blocked_ms: blockedMs }, + }); + forceQuit?.({ blockedPhase, blockedMs }); + }, quitDeadlineMs); + quitDeadlineTimer.unref?.(); + } + function dismissInstalledNotice(): void { if (!snapshot.recentlyInstalled) return; const currentState = readGlobalState(globalStatePath); @@ -905,6 +1152,88 @@ export function createAutoUpdateService({ }); } + async function quitAndInstall(): Promise { + // Mirrors checkPromise: collapse concurrent IPC/idle-timer calls onto one + // transaction so cleanup, rollback, and native handoff cannot race. + if (quitAndInstallPromise) return quitAndInstallPromise; + if (snapshot.status !== "ready" || !snapshot.version) return false; + const originalReadySnapshot = cloneSnapshot(snapshot); + resetAutoApplyTracking(true); + const run = async (): Promise => { + currentPhase = "verification"; + const refreshSucceeded = await refreshReadyUpdateBeforeInstall(); + if (!refreshSucceeded) { + return await abortInstall("refresh_failed", originalReadySnapshot); + } + const installVersion = snapshot.version; + if (!installVersion) { + return await abortInstall("refresh_failed", originalReadySnapshot); + } + installReadySnapshot = cloneSnapshot(snapshot); + currentPhase = "staging"; + if (!preflightSpace("install", installTargetPath)) { + return await abortInstall("install_preflight_failed", installReadySnapshot); + } + try { + const prepareResult = await waitForInstallStep( + Promise.resolve(beforeQuitAndInstall?.()), + "install", + "ADE could not prepare to quit for the update. Try again.", + ); + if (!prepareResult.completed) { + return await abortInstall("prepare_timeout", installReadySnapshot); + } + } catch (error) { + const message = formatErrorMessage(error); + logger.warn("autoUpdate.prepare_quit_and_install_failed", { + version: installVersion, + message, + }); + return await abortInstall("prepare_failed", installReadySnapshot); + } + writeGlobalState(globalStatePath, { + ...readGlobalState(globalStatePath), + pendingInstallUpdate: { + fromVersion: currentVersion, + targetVersion: installVersion, + releaseNotesUrl: snapshot.releaseNotesUrl, + requestedAt: now(), + }, + recentlyInstalledUpdate: undefined, + }); + logger.info("autoUpdate.quit_and_install", { version: installVersion }); + patchSnapshot({ + status: "installing", + progressPercent: 100, + error: null, + errorDetails: null, + parked: null, + autoApplyPending: null, + }); + try { + currentPhase = "install"; + // Mark the quit before entering electron-updater: it calls app.quit() + // synchronously, and ADE's before-quit handler must recognize this + // consented path instead of opening the normal quit confirmation. + armQuitDeadline(); + updater.quitAndInstall(false, true); + return true; + } catch (error) { + const message = formatErrorMessage(error); + logger.warn("autoUpdate.quit_and_install_failed", { + version: installVersion, + message, + }); + return await abortInstall("handoff_failed", installReadySnapshot); + } + }; + quitAndInstallPromise = run().finally(() => { + quitAndInstallPromise = null; + if (snapshot.status === "ready") scheduleActivityCheck(0); + }); + return quitAndInstallPromise; + } + const startupTimer = autoCheckEnabled ? setTimeout(checkForUpdates, startupDelayMs) : null; const periodicTimer = autoCheckEnabled ? setInterval(checkForUpdates, periodicCheckMs) : null; @@ -918,100 +1247,33 @@ export function createAutoUpdateService({ return () => listeners.delete(cb); }, dismissInstalledNotice, - async quitAndInstall(): Promise { - // Mirrors checkPromise: collapse concurrent IPC calls onto one in-flight - // promise so two callers cannot race the refresh + quitAndInstall pair. - if (quitAndInstallPromise) return quitAndInstallPromise; - if (snapshot.status !== "ready" || !snapshot.version) return false; - const run = async (): Promise => { - currentPhase = "verification"; - const refreshSucceeded = await refreshReadyUpdateBeforeInstall(); - if (!refreshSucceeded) return false; - // After refresh, snapshot may have been replaced; re-check version - // (refreshReadyUpdateBeforeInstall returns false if snapshot is no - // longer "ready" with a version, but be explicit for the narrowing). - const installVersion = snapshot.version; - if (!installVersion) return false; - currentPhase = "staging"; - if (!preflightSpace("install", installTargetPath)) return false; - try { - const prepareResult = await waitForInstallStep( - Promise.resolve(beforeQuitAndInstall?.()), - "install", - "ADE could not prepare to quit for the update. Try again.", - ); - if (!prepareResult.completed) return false; - } catch (error) { - const message = formatErrorMessage(error); - logger.warn("autoUpdate.prepare_quit_and_install_failed", { - version: installVersion, - message, - }); - setErrorSnapshot({ - error, - fallbackPhase: "install", - }); - return false; - } - writeGlobalState(globalStatePath, { - ...readGlobalState(globalStatePath), - pendingInstallUpdate: { - fromVersion: currentVersion, - targetVersion: installVersion, - releaseNotesUrl: snapshot.releaseNotesUrl, - requestedAt: now(), - }, - recentlyInstalledUpdate: undefined, - }); - logger.info("autoUpdate.quit_and_install", { version: installVersion }); - patchSnapshot({ - status: "installing", - progressPercent: 100, - error: null, - errorDetails: null, - }); - try { - currentPhase = "install"; - updater.quitAndInstall(false, true); - clearInstallWatchdog(); - installWatchdog = setTimeout(() => { - installWatchdog = null; - const message = "ADE did not quit for the update. Free space if needed, then try again."; - logger.warn("autoUpdate.quit_and_install_stalled", { - version: installVersion, - nativeHandoffWatchdogMs, - }); - setErrorSnapshot({ - error: new Error(message), - fallbackPhase: "install", - message, - preservesDownload: true, - }); - }, nativeHandoffWatchdogMs); - return true; - } catch (error) { - const message = formatErrorMessage(error); - logger.warn("autoUpdate.quit_and_install_failed", { - version: installVersion, - message, - }); - clearPendingInstallUpdate(); - setErrorSnapshot({ - error, - fallbackPhase: "install", - }); - return false; - } - }; - quitAndInstallPromise = run().finally(() => { - quitAndInstallPromise = null; + quitAndInstall, + cancelAutoApply(): boolean { + if (snapshot.status !== "ready" || !snapshot.autoApplyPending) return false; + const autoApplySuppressedUntil = nowMs() + autoApplySuppressionMs; + resetAutoApplyTracking(true); + patchSnapshot({ autoApplySuppressedUntil }); + productAnalyticsService?.captureInternal({ + event: "ade_update_auto_apply_cancelled", + surface: "desktop", }); - return quitAndInstallPromise; + scheduleActivityCheck(autoApplySuppressionMs); + return true; + }, + isInstallQuitArmed(): boolean { + return installQuitArmed; + }, + notifyQuitHandoffStarted(): void { + installQuitArmed = false; + installReadySnapshot = null; + clearQuitDeadline(); }, dispose() { if (startupTimer) clearTimeout(startupTimer); if (periodicTimer) clearInterval(periodicTimer); - clearInstallWatchdog(); + clearQuitDeadline(); + clearActivityCheck(); + clearAutoApplyDeadline(); listeners.clear(); updater.removeListener("checking-for-update", onCheckingForUpdate); updater.removeListener("update-available", onUpdateAvailable); diff --git a/apps/desktop/src/preload/global.d.ts b/apps/desktop/src/preload/global.d.ts index 406f07fd3..f7eeb2bc3 100644 --- a/apps/desktop/src/preload/global.d.ts +++ b/apps/desktop/src/preload/global.d.ts @@ -2340,6 +2340,7 @@ declare global { updateGetState: () => Promise; updateGetInstallImpact: () => Promise; updateQuitAndInstall: () => Promise; + updateCancelAutoApply: () => Promise; updateDismissInstalledNotice: () => Promise; onUpdateEvent: (cb: (snapshot: AutoUpdateSnapshot) => void) => () => void; perf: { diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.ts index e9b4f7e1e..00523b582 100644 --- a/apps/desktop/src/preload/preload.ts +++ b/apps/desktop/src/preload/preload.ts @@ -9001,6 +9001,8 @@ contextBridge.exposeInMainWorld("ade", { ipcRenderer.invoke(IPC.updateGetInstallImpact), updateQuitAndInstall: (): Promise => ipcRenderer.invoke(IPC.updateQuitAndInstall), + updateCancelAutoApply: (): Promise => + ipcRenderer.invoke(IPC.updateCancelAutoApply), updateDismissInstalledNotice: () => ipcRenderer.invoke(IPC.updateDismissInstalledNotice), onUpdateEvent: (cb: (snapshot: AutoUpdateSnapshot) => void) => { diff --git a/apps/desktop/src/renderer/browserMock.ts b/apps/desktop/src/renderer/browserMock.ts index 80a643c43..386ad47c6 100644 --- a/apps/desktop/src/renderer/browserMock.ts +++ b/apps/desktop/src/renderer/browserMock.ts @@ -3303,6 +3303,10 @@ if (typeof window !== "undefined" && shouldInstallBrowserMock(window)) { env: {}, localRuntime: { connectionState: "idle", + pid: null, + syncPort: null, + publishHealth: null, + lastWedge: null, runtimeMode: "primary", serviceInstall: { state: "skipped", @@ -6461,6 +6465,8 @@ if (typeof window !== "undefined" && shouldInstallBrowserMock(window)) { updateCheckForUpdates: resolved(undefined), updateGetState: resolved({ status: "idle", + currentVersion: "0.0.0", + latestKnownVersion: null, version: null, progressPercent: null, bytesPerSecond: null, @@ -6468,10 +6474,15 @@ if (typeof window !== "undefined" && shouldInstallBrowserMock(window)) { totalBytes: null, releaseNotesUrl: null, error: null, + errorDetails: null, recentlyInstalled: null, + parked: null, + autoApplyPending: null, + autoApplySuppressedUntil: null, }), updateGetInstallImpact: resolved({ connectedPhones: [] }), updateQuitAndInstall: resolved(true), + updateCancelAutoApply: resolved(false), updateDismissInstalledNotice: resolved(undefined), onUpdateEvent: noop, }; diff --git a/apps/desktop/src/renderer/components/app/AppShell.tsx b/apps/desktop/src/renderer/components/app/AppShell.tsx index f8b5bbb7a..3813671a8 100644 --- a/apps/desktop/src/renderer/components/app/AppShell.tsx +++ b/apps/desktop/src/renderer/components/app/AppShell.tsx @@ -72,6 +72,8 @@ import { cn } from "../ui/cn"; import { disposeTerminalRuntimesForProjectChange } from "../terminals/TerminalView"; import { buildPrsRouteSearch, type PrDetailRouteTab } from "../prs/prsRouteState"; import { ToastStack } from "./toast/ToastStack"; +import { AutoUpdateBanner } from "./AutoUpdateBanner"; +import { BrainRecoveryNotice } from "./BrainRecoveryNotice"; import { WorktreeOpenDialog } from "../projects/WorktreeOpenDialog"; import { useToasts } from "./toast/toastStore"; import { useLaneEventToasts } from "./toast/useLaneEventToasts"; @@ -1142,6 +1144,10 @@ export function AppShell({ children }: { children: React.ReactNode }) { + + + + {productAnalytics.consentRequired ? ( ): AutoUpdateSnapshot { + return { ...EMPTY_AUTO_UPDATE_SNAPSHOT, currentVersion: "1.2.34", ...overrides }; +} + +function installAdeMock(initial: AutoUpdateSnapshot = snapshot({})) { + const updateQuitAndInstall = vi.fn(async () => true); + const updateCancelAutoApply = vi.fn(async () => true); + // Mutable so the initial (async) getState read can't clobber an already-emitted + // state when its promise settles late. + let current = initial; + let listener: ((s: AutoUpdateSnapshot) => void) | null = null; + Object.defineProperty(window, "ade", { + configurable: true, + value: { + updateGetState: vi.fn(async () => current), + updateQuitAndInstall, + updateCancelAutoApply, + onUpdateEvent: vi.fn((cb: (s: AutoUpdateSnapshot) => void) => { + listener = cb; + return () => { + listener = null; + }; + }), + }, + }); + return { + updateQuitAndInstall, + updateCancelAutoApply, + emit(next: AutoUpdateSnapshot) { + current = next; + act(() => listener?.(next)); + }, + }; +} + +describe("describeStalenessBanner", () => { + it("returns null in idle steady state", () => { + expect(describeStalenessBanner(snapshot({ status: "idle" }))).toBeNull(); + }); + + it("flags a ready downloaded update", () => { + const banner = describeStalenessBanner(snapshot({ status: "ready", version: "1.2.35" })); + expect(banner?.kind).toBe("ready"); + expect(banner?.version).toBe("1.2.35"); + }); + + it("prefers the parked state over a ready status", () => { + const banner = describeStalenessBanner( + snapshot({ + status: "ready", + version: "1.2.35", + parked: { reason: "prepare_failed", at: 111 }, + }), + ); + expect(banner?.kind).toBe("parked"); + expect(banner?.signature).toContain("prepare_failed"); + }); + + it("does not flag a still-downloading update", () => { + expect(describeStalenessBanner(snapshot({ status: "downloading", version: "1.2.35" }))).toBeNull(); + }); +}); + +describe("AutoUpdateBanner", () => { + beforeEach(() => { + installAdeMock(); + }); + + afterEach(() => { + cleanup(); + for (const toast of getToasts()) dismissToast(toast.id); + vi.restoreAllMocks(); + Reflect.deleteProperty(window, "ade"); + }); + + it("stays hidden in steady state", async () => { + render(); + await waitFor(() => { + expect(screen.queryByRole("button", { name: /restart now/i })).toBeNull(); + }); + }); + + it("shows the ready banner and restarts on demand", async () => { + const mock = installAdeMock(snapshot({ status: "ready", version: "1.2.35" })); + render(); + + expect(await screen.findByText(/Running 1\.2\.34 · 1\.2\.35 is ready/)).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: /restart now/i })); + await waitFor(() => { + expect(mock.updateQuitAndInstall).toHaveBeenCalledTimes(1); + }); + }); + + it("shows the parked retry copy", async () => { + installAdeMock( + snapshot({ + status: "ready", + version: "1.2.35", + parked: { reason: "handoff_failed", at: 5 }, + }), + ); + render(); + expect(await screen.findByText(/Update to 1\.2\.35 didn't finish — Restart to retry/)).toBeTruthy(); + }); + + it("dismisses until the state changes", async () => { + const mock = installAdeMock(snapshot({ status: "ready", version: "1.2.35" })); + render(); + + await screen.findByText(/1\.2\.35 is ready/); + fireEvent.click(screen.getByRole("button", { name: /dismiss update banner/i })); + await waitFor(() => { + expect(screen.queryByText(/1\.2\.35 is ready/)).toBeNull(); + }); + + // A newer staged version has a different signature, so the banner returns. + mock.emit(snapshot({ status: "ready", version: "1.2.36" })); + expect(await screen.findByText(/1\.2\.36 is ready/)).toBeTruthy(); + }); + + it("shows a countdown toast and cancels the auto-apply", async () => { + const mock = installAdeMock( + snapshot({ + status: "ready", + version: "1.2.35", + autoApplyPending: { deadlineAt: Date.now() + 10_000 }, + }), + ); + render( + <> + + + , + ); + + expect(await screen.findByText(/Updating to 1\.2\.35 in \d+s/)).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: /cancel/i })); + await waitFor(() => { + expect(mock.updateCancelAutoApply).toHaveBeenCalledTimes(1); + }); + + // Clearing the pending state removes the toast. + mock.emit(snapshot({ status: "ready", version: "1.2.35" })); + await waitFor(() => { + expect(screen.queryByText(/Updating to 1\.2\.35/)).toBeNull(); + }); + }); +}); diff --git a/apps/desktop/src/renderer/components/app/AutoUpdateBanner.tsx b/apps/desktop/src/renderer/components/app/AutoUpdateBanner.tsx new file mode 100644 index 000000000..e3671df85 --- /dev/null +++ b/apps/desktop/src/renderer/components/app/AutoUpdateBanner.tsx @@ -0,0 +1,144 @@ +import { useCallback, useEffect, useState } from "react"; +import { ArrowsClockwise, WarningCircle } from "@phosphor-icons/react"; +import type { AutoUpdateSnapshot } from "../../../shared/types"; +import { useAutoUpdateSnapshot } from "./useAutoUpdateSnapshot"; +import { dismissToast, showToast } from "./toast/toastStore"; + +const AUTO_APPLY_TOAST_ID = "ade-auto-update-auto-apply"; + +type StalenessBanner = { + /** "ready" = a downloaded update is waiting; "parked" = a consented install aborted. */ + kind: "ready" | "parked"; + version: string | null; + /** + * Stable identity for this banner state. Dismissal is keyed on it so the + * banner reappears whenever the underlying state changes (new version staged, + * a fresh install abort) but stays hidden for an unchanged state. + */ + signature: string; +}; + +/** + * Decides whether the app is running behind what is staged on disk. A parked + * install (consented but aborted before the native updater took over) wins over + * a plain ready state so we surface the "didn't finish" retry copy. + */ +export function describeStalenessBanner(snapshot: AutoUpdateSnapshot): StalenessBanner | null { + if (snapshot.parked) { + return { + kind: "parked", + version: snapshot.version, + signature: `parked:${snapshot.parked.reason}:${snapshot.parked.at}:${snapshot.version ?? ""}`, + }; + } + if (snapshot.status === "ready") { + return { + kind: "ready", + version: snapshot.version, + signature: `ready:${snapshot.version ?? ""}`, + }; + } + return null; +} + +function versionText(version: string | null): string { + return version ? version : "the latest version"; +} + +/** + * App-shell staleness/update banner plus the idle-countdown toast. Both consume + * the same auto-update snapshot, so they are colocated to share one subscription + * and never disagree about the pending update. + */ +export function AutoUpdateBanner() { + const snapshot = useAutoUpdateSnapshot(); + const [dismissedSignature, setDismissedSignature] = useState(null); + const [restarting, setRestarting] = useState(false); + + const banner = describeStalenessBanner(snapshot); + const signature = banner?.signature ?? null; + + // Re-enable the Restart action whenever the banner state changes (or clears); + // a stale "restarting" flag must never stick across a new staged version. + useEffect(() => { + setRestarting(false); + }, [signature]); + + const handleRestart = useCallback(() => { + setRestarting(true); + void window.ade.updateQuitAndInstall() + .then((started) => { + if (!started) setRestarting(false); + }) + .catch(() => { + setRestarting(false); + }); + }, []); + + const handleCancelAutoApply = useCallback(() => { + dismissToast(AUTO_APPLY_TOAST_ID); + void window.ade.updateCancelAutoApply().catch(() => { + // Main process logs cancellation failures; the snapshot event reconciles. + }); + }, []); + + // Drive the countdown toast off `autoApplyPending`. Re-render once a second so + // the visible seconds tick down; the snapshot event clears it on apply/cancel. + const pending = snapshot.autoApplyPending; + const pendingVersion = snapshot.version; + useEffect(() => { + if (!pending) { + dismissToast(AUTO_APPLY_TOAST_ID); + return; + } + const renderToast = () => { + const secondsLeft = Math.max(0, Math.ceil((pending.deadlineAt - Date.now()) / 1000)); + showToast({ + id: AUTO_APPLY_TOAST_ID, + title: `Updating to ${versionText(pendingVersion)} in ${secondsLeft}s`, + tone: "info", + durationMs: 0, + action: { label: "Cancel", onClick: handleCancelAutoApply }, + }); + }; + renderToast(); + const timer = window.setInterval(renderToast, 1000); + return () => { + window.clearInterval(timer); + }; + }, [pending, pendingVersion, handleCancelAutoApply]); + + // Tidy up the toast if this component unmounts mid-countdown. + useEffect(() => () => dismissToast(AUTO_APPLY_TOAST_ID), []); + + if (!banner || signature === dismissedSignature) return null; + + return ( +
+
+ ); +} diff --git a/apps/desktop/src/renderer/components/app/AutoUpdateControl.test.tsx b/apps/desktop/src/renderer/components/app/AutoUpdateControl.test.tsx index 32bfff872..8bd259f83 100644 --- a/apps/desktop/src/renderer/components/app/AutoUpdateControl.test.tsx +++ b/apps/desktop/src/renderer/components/app/AutoUpdateControl.test.tsx @@ -10,6 +10,7 @@ import type { AppInfo, AutoUpdateSnapshot } from "../../../shared/types"; const idleSnapshot: AutoUpdateSnapshot = { status: "idle", currentVersion: "1.2.0", + latestKnownVersion: "1.2.0", version: null, progressPercent: null, bytesPerSecond: null, @@ -19,6 +20,9 @@ const idleSnapshot: AutoUpdateSnapshot = { error: null, errorDetails: null, recentlyInstalled: null, + parked: null, + autoApplyPending: null, + autoApplySuppressedUntil: null, }; function appInfoWithRuntimeSkew( diff --git a/apps/desktop/src/renderer/components/app/AutoUpdateControl.tsx b/apps/desktop/src/renderer/components/app/AutoUpdateControl.tsx index 619267737..2be1ddf9b 100644 --- a/apps/desktop/src/renderer/components/app/AutoUpdateControl.tsx +++ b/apps/desktop/src/renderer/components/app/AutoUpdateControl.tsx @@ -5,20 +5,7 @@ import type { AppInfo, AutoUpdateSnapshot } from "../../../shared/types"; import { Button } from "../ui/Button"; import { cn } from "../ui/cn"; import { AutoUpdateErrorDialog, isAutoUpdateDiskSpaceError } from "./AutoUpdateErrorDialog"; - -const EMPTY_UPDATE_SNAPSHOT: AutoUpdateSnapshot = { - status: "idle", - currentVersion: "", - version: null, - progressPercent: null, - bytesPerSecond: null, - transferredBytes: null, - totalBytes: null, - releaseNotesUrl: null, - error: null, - errorDetails: null, - recentlyInstalled: null, -}; +import { EMPTY_AUTO_UPDATE_SNAPSHOT } from "./useAutoUpdateSnapshot"; const RUNTIME_SKEW_REFRESH_MS = 15_000; type RuntimeVersionSkew = NonNullable["versionSkew"]; @@ -48,7 +35,7 @@ function runtimeSkewTitle(skew: RuntimeVersionSkew): string { } export function AutoUpdateControl() { - const [snapshot, setSnapshot] = useState(EMPTY_UPDATE_SNAPSHOT); + const [snapshot, setSnapshot] = useState(EMPTY_AUTO_UPDATE_SNAPSHOT); const [runtimeSkew, setRuntimeSkew] = useState(null); const [releaseNotesOpen, setReleaseNotesOpen] = useState(false); const [updateErrorOpen, setUpdateErrorOpen] = useState(false); diff --git a/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.test.ts b/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.test.ts new file mode 100644 index 000000000..c560ee12a --- /dev/null +++ b/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.test.ts @@ -0,0 +1,62 @@ +import { describe, expect, it } from "vitest"; +import { + formatRecoveryCommand, + formatRecoveryTime, + shouldShowRecoveryNotice, +} from "./BrainRecoveryNotice"; + +const wedge = (ts: string, lastCommand = "npm run build") => ({ + ts, + lastCommand, + blockedMs: 4200, +}); + +describe("shouldShowRecoveryNotice", () => { + it("is hidden when there is no wedge", () => { + expect(shouldShowRecoveryNotice(null, null)).toBe(false); + expect(shouldShowRecoveryNotice(undefined, null)).toBe(false); + }); + + it("shows an unacknowledged wedge", () => { + expect(shouldShowRecoveryNotice(wedge("2026-07-23T10:00:00Z"), null)).toBe(true); + }); + + it("hides a wedge that was already acknowledged", () => { + const ts = "2026-07-23T10:00:00Z"; + expect(shouldShowRecoveryNotice(wedge(ts), ts)).toBe(false); + }); + + it("re-shows when a newer wedge supersedes an old acknowledgment", () => { + expect( + shouldShowRecoveryNotice(wedge("2026-07-23T11:00:00Z"), "2026-07-23T10:00:00Z"), + ).toBe(true); + }); +}); + +describe("formatRecoveryCommand", () => { + it("falls back when the command is blank", () => { + expect(formatRecoveryCommand(" ")).toBe("background task"); + }); + + it("passes short commands through trimmed", () => { + expect(formatRecoveryCommand(" git status ")).toBe("git status"); + }); + + it("truncates long commands with an ellipsis", () => { + const long = "run-a-really-long-background-command-that-keeps-going-forever"; + const out = formatRecoveryCommand(long); + expect(out.endsWith("…")).toBe(true); + expect(out.length).toBe(48); + }); +}); + +describe("formatRecoveryTime", () => { + it("returns the raw value for an unparseable timestamp", () => { + expect(formatRecoveryTime("not-a-date")).toBe("not-a-date"); + }); + + it("formats a parseable timestamp to a short local time", () => { + const out = formatRecoveryTime("2026-07-23T10:05:00Z"); + expect(out).toMatch(/\d{1,2}:\d{2}/); + }); +}); diff --git a/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.tsx b/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.tsx new file mode 100644 index 000000000..70e3ee141 --- /dev/null +++ b/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.tsx @@ -0,0 +1,106 @@ +import { useCallback, useEffect, useState } from "react"; +import { ArrowCounterClockwise } from "@phosphor-icons/react"; +import type { LocalRuntimeStatus } from "../../../shared/types"; + +type LastWedge = NonNullable; + +/** localStorage key holding the `ts` of the most recently acknowledged wedge. */ +export const RECOVERY_ACK_STORAGE_KEY = "ade.brainRecovery.ackedTs"; + +/** + * A recovery notice shows once per distinct wedge event. Acknowledgment is the + * wedge's own `ts`, so a fresh recovery (new `ts`) reappears even after a prior + * one was dismissed, but the same event never nags twice. + */ +export function shouldShowRecoveryNotice( + lastWedge: LocalRuntimeStatus["lastWedge"] | undefined, + ackedTs: string | null, +): boolean { + if (!lastWedge) return false; + return lastWedge.ts !== ackedTs; +} + +/** "HH:MM" for the notice; falls back to the raw value if it can't be parsed. */ +export function formatRecoveryTime(ts: string): string { + const parsed = Date.parse(ts); + if (!Number.isFinite(parsed)) return ts; + return new Date(parsed).toLocaleTimeString([], { + hour: "2-digit", + minute: "2-digit", + }); +} + +/** Trims a long command to something that fits inline in the notice. */ +export function formatRecoveryCommand(command: string): string { + const trimmed = command.trim(); + if (!trimmed) return "background task"; + return trimmed.length > 48 ? `${trimmed.slice(0, 47)}…` : trimmed; +} + +function readAckedTs(): string | null { + try { + return window.localStorage.getItem(RECOVERY_ACK_STORAGE_KEY); + } catch { + return null; + } +} + +function writeAckedTs(ts: string): void { + try { + window.localStorage.setItem(RECOVERY_ACK_STORAGE_KEY, ts); + } catch { + // Ignore unavailable/quota-exceeded localStorage; the notice simply may + // reappear on the next mount, which is harmless. + } +} + +/** + * App-shell banner announcing that the brain recovered from a background + * event-loop wedge. Reads the one-shot `lastWedge` from runtime status; a wedge + * is a past event so there is no need to subscribe to changes. + */ +export function BrainRecoveryNotice() { + const [lastWedge, setLastWedge] = useState(null); + const [ackedTs, setAckedTs] = useState(() => readAckedTs()); + + useEffect(() => { + let cancelled = false; + const infoPromise = window.ade.app?.getInfo?.(); + if (!infoPromise) return; + void infoPromise + .then((info) => { + if (!cancelled) setLastWedge(info.localRuntime?.lastWedge ?? null); + }) + .catch(() => {}); + return () => { + cancelled = true; + }; + }, []); + + const handleDismiss = useCallback(() => { + if (!lastWedge) return; + writeAckedTs(lastWedge.ts); + setAckedTs(lastWedge.ts); + }, [lastWedge]); + + if (!shouldShowRecoveryNotice(lastWedge, ackedTs) || !lastWedge) return null; + + return ( +
+
+ ); +} diff --git a/apps/desktop/src/renderer/components/app/useAutoUpdateSnapshot.ts b/apps/desktop/src/renderer/components/app/useAutoUpdateSnapshot.ts new file mode 100644 index 000000000..6b61bf4b7 --- /dev/null +++ b/apps/desktop/src/renderer/components/app/useAutoUpdateSnapshot.ts @@ -0,0 +1,53 @@ +import { useEffect, useState } from "react"; +import type { AutoUpdateSnapshot } from "../../../shared/types"; + +/** Neutral steady-state snapshot before the first main-process read lands. */ +export const EMPTY_AUTO_UPDATE_SNAPSHOT: AutoUpdateSnapshot = { + status: "idle", + currentVersion: "", + latestKnownVersion: null, + version: null, + progressPercent: null, + bytesPerSecond: null, + transferredBytes: null, + totalBytes: null, + releaseNotesUrl: null, + error: null, + errorDetails: null, + recentlyInstalled: null, + parked: null, + autoApplyPending: null, + autoApplySuppressedUntil: null, +}; + +/** + * Subscribes to the auto-update snapshot (initial read + live events). Shared by + * every truthful-version surface (top-bar pill, app-shell banner, About panel) + * so they never disagree about what is running versus what is staged. + */ +export function useAutoUpdateSnapshot(): AutoUpdateSnapshot { + const [snapshot, setSnapshot] = useState(EMPTY_AUTO_UPDATE_SNAPSHOT); + + useEffect(() => { + let cancelled = false; + + void window.ade.updateGetState() + .then((next) => { + if (!cancelled) setSnapshot(next); + }) + .catch(() => { + // Best effort only; live events will fill in. + }); + + const unsubscribe = window.ade.onUpdateEvent((next) => { + if (!cancelled) setSnapshot(next); + }); + + return () => { + cancelled = true; + unsubscribe(); + }; + }, []); + + return snapshot; +} diff --git a/apps/desktop/src/renderer/components/remoteTargets/RemoteTargetList.tsx b/apps/desktop/src/renderer/components/remoteTargets/RemoteTargetList.tsx index 2957dfa48..5f64178af 100644 --- a/apps/desktop/src/renderer/components/remoteTargets/RemoteTargetList.tsx +++ b/apps/desktop/src/renderer/components/remoteTargets/RemoteTargetList.tsx @@ -34,11 +34,13 @@ import { import { PairMachineForm } from "./PairMachineForm"; import { assignMachineSections, + describePublishHealth, discoveredPairingInput, discoveredTargetInput, formatRemoteTargetError, isSshOnlyDiscovered, machineMatchesSavedTarget, + type LocalPublishHealth, type MachineSection, } from "./remoteMachineModel"; import { SavedMachineRow } from "./SavedMachineRow"; @@ -176,6 +178,8 @@ export function RemoteTargetList({ const [localMachineName, setLocalMachineName] = useState(""); const [localMachineIdentity, setLocalMachineIdentity] = useState<{ machineKey: string; deviceId: string } | null>(null); + const [localPublishHealth, setLocalPublishHealth] = + useState(null); const [pairingPrefill, setPairingPrefill] = useState(null); const [accountConnectingMachineKey, setAccountConnectingMachineKey] = useState(null); @@ -419,6 +423,40 @@ export function RemoteTargetList({ }; }, []); + // This Mac's route-publish health, refreshed periodically so a persisting + // failure's "for N min" stays truthful while the panel is open. getInfo is a + // cheap one-shot; there is no push event for the publisher's health. + useEffect(() => { + let cancelled = false; + const refresh = () => { + const infoPromise = window.ade.app?.getInfo?.(); + if (!infoPromise) return; + void infoPromise + .then((info) => { + if (cancelled) return; + const health = info.localRuntime?.publishHealth ?? null; + setLocalPublishHealth( + health + ? { state: health.state, failingSinceMs: health.failingSinceMs } + : null, + ); + }) + .catch(() => {}); + }; + refresh(); + const timer = window.setInterval(refresh, 30_000); + return () => { + cancelled = true; + window.clearInterval(timer); + }; + }, []); + + const publishHealthDisplay = useMemo( + () => describePublishHealth(localPublishHealth), + // Re-derive on each fetch; the 30s refresh advances the "for N min" count. + [localPublishHealth], + ); + const openAddMachine = useCallback(() => { setSelectedId(null); setFormPrefill(null); @@ -1047,6 +1085,29 @@ export function RemoteTargetList({ Machines
{connectedCount} connected
+ {publishHealthDisplay.kind === "healthy" ? ( +
Routes fresh
+ ) : null} + {publishHealthDisplay.kind === "failing" ? ( +
+ + + Other devices may not reach this Mac — route publish failing for{" "} + {publishHealthDisplay.minutes} min + +
+ ) : null}
-
- Installed - {info.appVersion} +
+ Running + {runningVersion} + {restartPending ? ( + + + restart pending + + ) : null}
-
- Latest - {latest ? ( - - {latest.version} - {releasedAgo ? ( - · {releasedAgo} - ) : null} +
+ Installed + + {installedVersion} + + · Latest {latestVersion} + {releasedAgo && latestVersion === (latest?.version ?? latestVersion) ? ` · ${releasedAgo}` : ""} - ) : ( - Unavailable - )} +
@@ -252,6 +277,32 @@ export function AboutSection({ embedded = false }: { embedded?: boolean } = {})
{embedded ? "Background service" : "ADE runtime service"}
+
+ Running + + {info.localRuntime.versionSkew.runtimeVersion ?? info.appVersion} + + {info.localRuntime.pid != null || info.localRuntime.syncPort != null ? ( + + {[ + info.localRuntime.pid != null ? `· pid ${info.localRuntime.pid}` : null, + info.localRuntime.syncPort != null ? `· port ${info.localRuntime.syncPort}` : null, + ] + .filter(Boolean) + .join(" ")} + + ) : null} + +
{info.localRuntime.connectionState === "connected" ? "Connected and ready." @@ -273,7 +324,27 @@ export function AboutSection({ embedded = false }: { embedded?: boolean } = {}) ) : null}
- {runtimeSkew ? ( + {restartPending ? ( +
+ + Will update when the app restarts +
+ ) : null} + {runtimeSkew && !restartPending ? (
{ env: {}, localRuntime: { connectionState: client.getStatus().state === "connected" ? "connected" : "idle", + pid: null, + syncPort: null, + publishHealth: null, + lastWedge: null, runtimeMode: "primary", versionSkew: { state: "none", @@ -204,6 +208,8 @@ export const webUpdateMethods = { async updateGetState() { return { status: "idle", + currentVersion: "", + latestKnownVersion: null, version: null, progressPercent: null, bytesPerSecond: null, @@ -211,12 +217,18 @@ export const webUpdateMethods = { totalBytes: null, releaseNotesUrl: null, error: null, + errorDetails: null, recentlyInstalled: null, + parked: null, + autoApplyPending: null, + autoApplySuppressedUntil: null, }; }, async updateCheckNow() { return { status: "idle", + currentVersion: "", + latestKnownVersion: null, version: null, progressPercent: null, bytesPerSecond: null, @@ -224,7 +236,11 @@ export const webUpdateMethods = { totalBytes: null, releaseNotesUrl: null, error: null, + errorDetails: null, recentlyInstalled: null, + parked: null, + autoApplyPending: null, + autoApplySuppressedUntil: null, }; }, async updateGetInstallImpact(): Promise { @@ -233,6 +249,9 @@ export const webUpdateMethods = { async updateQuitAndInstall() { return false; }, + async updateCancelAutoApply() { + return false; + }, }; function browserPlatform(): NodeJS.Platform { diff --git a/apps/desktop/src/renderer/webclient/adapter/index.ts b/apps/desktop/src/renderer/webclient/adapter/index.ts index fc948647c..6988e487a 100644 --- a/apps/desktop/src/renderer/webclient/adapter/index.ts +++ b/apps/desktop/src/renderer/webclient/adapter/index.ts @@ -164,6 +164,7 @@ export function createAdeWebAdapter( updateGetState: webUpdateMethods.updateGetState, updateGetInstallImpact: webUpdateMethods.updateGetInstallImpact, updateQuitAndInstall: webUpdateMethods.updateQuitAndInstall, + updateCancelAutoApply: webUpdateMethods.updateCancelAutoApply, updateDismissInstalledNotice: async () => undefined, onUpdateEvent: () => () => {}, perf: { diff --git a/apps/desktop/src/shared/adeRuntimeProtocol.ts b/apps/desktop/src/shared/adeRuntimeProtocol.ts new file mode 100644 index 000000000..1af1eabde --- /dev/null +++ b/apps/desktop/src/shared/adeRuntimeProtocol.ts @@ -0,0 +1,20 @@ +/** + * Monotonically bumpable compatibility level for the local ADE runtime RPC. + * + * A runtime is compatible with this desktop when the desktop's level falls + * inside the inclusive range advertised by the runtime. + */ +export const RUNTIME_COMPAT_LEVEL = 1; + +export function isRuntimeProtocolCompatible(runtimeInfo: { + minCompatibleProtocol: number | null; + protocolVersion: number | null; +}): boolean { + const { minCompatibleProtocol, protocolVersion } = runtimeInfo; + return minCompatibleProtocol != null + && protocolVersion != null + && minCompatibleProtocol > 0 + && protocolVersion >= minCompatibleProtocol + && minCompatibleProtocol <= RUNTIME_COMPAT_LEVEL + && RUNTIME_COMPAT_LEVEL <= protocolVersion; +} diff --git a/apps/desktop/src/shared/ipc.ts b/apps/desktop/src/shared/ipc.ts index 8296f75c2..8d1eb02d9 100644 --- a/apps/desktop/src/shared/ipc.ts +++ b/apps/desktop/src/shared/ipc.ts @@ -805,6 +805,7 @@ export const IPC = { updateGetState: "ade.update.getState", updateGetInstallImpact: "ade.update.getInstallImpact", updateQuitAndInstall: "ade.update.quitAndInstall", + updateCancelAutoApply: "ade.update.cancelAutoApply", updateDismissInstalledNotice: "ade.update.dismissInstalledNotice", updateEvent: "ade.update.event", transcriptionTranscribe: "ade.transcription.transcribe", diff --git a/apps/desktop/src/shared/types/core.ts b/apps/desktop/src/shared/types/core.ts index 0eff1b98b..030003859 100644 --- a/apps/desktop/src/shared/types/core.ts +++ b/apps/desktop/src/shared/types/core.ts @@ -3,6 +3,10 @@ // --------------------------------------------------------------------------- import type { DeeplinkEnvelope } from "../deeplinks"; +import type { + SyncAccountDirectoryLegDurations, + SyncAccountDirectoryState, +} from "./sync"; export type LocalRuntimeServiceInstallState = | "not_attempted" @@ -26,6 +30,18 @@ export type LocalRuntimeConnectionState = export type LocalRuntimeStatus = { connectionState: LocalRuntimeConnectionState; + pid?: number | null; + syncPort?: number | null; + publishHealth?: { + state: SyncAccountDirectoryState; + failingSinceMs: number | null; + lastLegDurations: SyncAccountDirectoryLegDurations; + } | null; + lastWedge?: { + lastCommand: string; + blockedMs: number; + ts: string; + } | null; /** * "isolated" means the desktop fell back to an app-owned no-sync brain * (phone sync and ADE Code attach to the channel service, which is down). @@ -179,9 +195,22 @@ export type AutoUpdateErrorDetails = { preservesDownload: boolean; }; +export const AUTO_UPDATE_INSTALL_ABORT_REASONS = [ + "refresh_failed", + "install_preflight_failed", + "prepare_failed", + "prepare_timeout", + "handoff_failed", +] as const; + +export type AutoUpdateInstallAbortReason = + (typeof AUTO_UPDATE_INSTALL_ABORT_REASONS)[number]; + export type AutoUpdateSnapshot = { status: AutoUpdateStatus; currentVersion: string; + /** Latest version observed from electron-updater's configured feed. */ + latestKnownVersion: string | null; version: string | null; progressPercent: number | null; bytesPerSecond: number | null; @@ -191,6 +220,21 @@ export type AutoUpdateSnapshot = { error: string | null; errorDetails: AutoUpdateErrorDetails | null; recentlyInstalled: RecentlyInstalledUpdate | null; + /** A consented install that aborted before the native updater could take over. */ + parked: { reason: AutoUpdateInstallAbortReason; at: number } | null; + /** Renderer-visible countdown before an idle update is applied. */ + autoApplyPending: { deadlineAt: number } | null; + /** Explicit user cancellation suppresses another idle countdown until this epoch. */ + autoApplySuppressedUntil: number | null; +}; + +export type RuntimeActivityCounts = { + activeAgentTurns: number; + activeWorkSessions: number; +}; + +export type RuntimeActivitySummary = RuntimeActivityCounts & { + idle: boolean; }; export type UpdateInstallImpactPhone = { diff --git a/apps/desktop/src/shared/types/productAnalytics.ts b/apps/desktop/src/shared/types/productAnalytics.ts index 943d7c023..8a374f816 100644 --- a/apps/desktop/src/shared/types/productAnalytics.ts +++ b/apps/desktop/src/shared/types/productAnalytics.ts @@ -10,6 +10,12 @@ export const PRODUCT_ANALYTICS_EVENTS = [ "ade_error", "ade_daily_usage_summary", "ade_analytics_budget", + "ade_update_install_aborted", + "ade_update_quit_escalated", + "ade_update_auto_applied", + "ade_update_auto_apply_cancelled", + "ade_brain_recovered", + "ade_publish_failing", ] as const; export type ProductAnalyticsEventName = (typeof PRODUCT_ANALYTICS_EVENTS)[number]; diff --git a/apps/desktop/src/shared/types/sync.ts b/apps/desktop/src/shared/types/sync.ts index 1bb185c23..dad3d14aa 100644 --- a/apps/desktop/src/shared/types/sync.ts +++ b/apps/desktop/src/shared/types/sync.ts @@ -340,9 +340,17 @@ export type SyncAccountDirectoryState = | "token_unreadable" | "invalid_directory_url" | "http_error" + | "token_timeout" + | "http_timeout" | "timeout" | "transport_error"; +export type SyncAccountDirectoryLegDurations = { + snapshot: number | null; + token: number | null; + http: number | null; +}; + /** Last account-directory publisher outcome for this machine brain. */ export type SyncAccountDirectoryHealth = { state: SyncAccountDirectoryState; @@ -353,6 +361,8 @@ export type SyncAccountDirectoryHealth = { lastHttpStatus: number | null; lastHttpReason: string | null; reachableEndpointCount: number; + lastLegDurations: SyncAccountDirectoryLegDurations; + failingSinceMs: number | null; }; export function createSyncAccountDirectoryHealth( @@ -369,6 +379,12 @@ export function createSyncAccountDirectoryHealth( lastHttpStatus: null, lastHttpReason: null, reachableEndpointCount: 0, + lastLegDurations: { + snapshot: null, + token: null, + http: null, + }, + failingSinceMs: null, ...overrides, }; } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 3c2a2f2a4..b8822f139 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -134,6 +134,16 @@ Product positioning and workflows live in [`docs/PRD.md`](../docs/PRD.md). This | `agentRegistry.ts` | Per-machine agent registry. | **Service managers.** `apps/ade-cli/src/serviceManager/installLaunchd.ts` (macOS), `installSystemd.ts` (Linux), `installWindows.ts` (Windows) register the brain as a login-time service. `index.ts` is the platform router; `common.ts` carries shared types (`ServiceManagerResult`, `ServiceManagerStatusResult`). +On macOS, an unchanged loaded launch agent is retained only after a bounded +runtime initialize probe succeeds. A failed probe takes the full unload, +predecessor termination, stale-process reap, and load path; the install result +is successful only after the old pid is gone, launchd reports a distinct new +pid, and the replacement initializes over the machine socket. The running brain +also stats its own CLI entrypoint every five minutes (configurable with +`ADE_BRAIN_FRESHNESS_INTERVAL_MS`, disabled by +`ADE_DISABLE_BRAIN_FRESHNESS=1`), hashes only after the stat changes, and uses +the brain-update service restart path after an idle grace period when the disk +hash no longer matches its baked runtime hash. **Session identity.** The runtime resolves caller role from ADE context env vars and command flags. Role vocabulary: `cto`, `orchestrator`, `agent`, `external`, `evaluator`. `ADE_DEFAULT_ROLE` is an authority ceiling, not an identity grant: `resolveSessionBoundRole` clamps a chat-bound caller that would otherwise inherit a daemon-wide `cto` role to `agent`, preserves an explicitly declared `orchestrator`, and never accepts a requested role above the runtime ceiling. SDK-backed chats receive `ADE_CHAT_SESSION_ID` plus `ADE_DEFAULT_ROLE=agent` (or `orchestrator` for a lead), and tracked provider CLI launch/resume does the same. Persistent SDK guidance names the concrete `--session ` for lifecycle commands so shared provider servers do not depend on process-global env inheritance. Browser automation adds a separate bearer capability: ADE-launched chat and owned-terminal environments receive an opaque `ADE_BROWSER_ACTOR_TOKEN` bound in Electron memory to that chat's trusted lane/project or personal tab collection. The runtime requires the token, strips caller-supplied routing, and carries it only over the authenticated desktop bridge. Electron validates it in the same process that issued it before restoring the bound scope; role alone never grants access to a human-authenticated browser profile. @@ -1276,6 +1286,7 @@ Post-packaging hardening (`apps/desktop/scripts/`): ### 15.1 Logging - **Main-process logger** — `apps/desktop/src/main/services/logging/logger.ts` (`createFileLogger`). Writes structured JSONL to `~/.ade/logs//ade-main.log`. Categories: `ipc.*`, `project.startup_task_*`, `renderer.*`, per-service telemetry. +- **Machine-brain logger** — the headless runtime reuses `createFileLogger` through `apps/ade-cli/src/services/runtime/brainLogger.ts`, writes `~/.ade/runtime/brain.jsonl` with 10 MiB `.1` rotation, and mirrors timestamped warnings/errors to stderr. - **Redaction** — all log writes pass through `redactSecrets()` / `sanitizeStructuredData()`. - **Retention** — local, indefinite until user clears. diff --git a/docs/features/remote-runtime/README.md b/docs/features/remote-runtime/README.md index 46939b5e5..1b8e991cb 100644 --- a/docs/features/remote-runtime/README.md +++ b/docs/features/remote-runtime/README.md @@ -74,10 +74,12 @@ relay payload E2E encryption is planned security work. See the trust boundary in and the one-connection local socket handed to the normal ADE Code client. - `apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts` — the local runtime connection used by desktop IPC, event streaming, sync - Settings, and local-work checks. It preserves a newer already-running machine - brain instead of replacing it with an older desktop-bundled runtime, falls - back to an isolated no-sync runtime for the old desktop window, and reports - `versionSkew` so the renderer can require a desktop update. It also spawns + Settings, and local-work checks. Runtime initialization advertises an integer + compatibility window (`minCompatibleProtocol` through `protocolVersion`). + A desktop connects normally to a newer machine brain when its own + `RUNTIME_COMPAT_LEVEL` falls inside that window; a newer incompatible brain + remains preserved and the old desktop window falls back to an isolated + no-sync runtime. It also spawns `ade serve` for non-primary sockets, tracks the per-user login service install/health state, and applies short per-call timeouts for project registration, file actions, and event polling so renderer IPC calls do not diff --git a/docs/logging.md b/docs/logging.md index e1c48fab2..3cd9ff5d8 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -22,6 +22,8 @@ Only closed event names, closed property keys, and coarse allowlisted values may Operational logs use ADE's local logging services and may include bounded diagnostic context appropriate for the local machine. They are for debugging a specific installation and must not be forwarded to PostHog. +The machine brain writes the same `{ts, level, event, meta}` JSONL format as the desktop logger to `~/.ade/runtime/brain.jsonl`, honoring `ADE_LOG_LEVEL` (default `info`). The file rotates at 10 MiB to `brain.1.jsonl`; warnings and errors are also mirrored to stderr with an ISO-8601 timestamp and uppercase level for launchd diagnostics. Account-directory publish outcomes record only bounded per-leg durations, the failing leg, and coarse failure codes such as `token_timeout` or `http_timeout`; they never include bearer tokens or response bodies. These high-frequency health events remain local operational logs and are not product analytics. + Claude compaction observations use the local structured line `agent_chat.claude_context_compaction_observed` with `sessionId`, `trigger` (`natural`, `ade_fallback`, or `recovery`), and `occupancyPctAtTrigger`. Record From ec3118682a8f72f4c7af515ae13e3c4a859e572f Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Thu, 23 Jul 2026 18:48:49 -0400 Subject: [PATCH 3/7] quality: dedupe wire parsers, registration-carried abort flags, doctor module split Applies the dual-review synthesis: shared parseRuntimePublishHealth/LastWedge replace two drifting hand-rolled parsers; observesAbort moves onto handler registrations (set-equality test guards the 48-action invariant); shared runWithAbortSignal; doctor orchestration moves out of cli.ts; dead duplicate state clear removed; pure version helpers extracted from autoUpdateService. Co-Authored-By: Claude Fable 5 --- apps/ade-cli/src/cli.test.ts | 1 + apps/ade-cli/src/cli.ts | 381 +---------------- apps/ade-cli/src/commands/doctor.test.ts | 44 +- apps/ade-cli/src/commands/doctor.ts | 404 ++++++++++++++++++ apps/ade-cli/src/services/sync/abortSignal.ts | 41 ++ .../src/services/sync/syncHostService.ts | 54 +-- .../sync/syncRemoteCommandService.test.ts | 55 +++ .../services/sync/syncRemoteCommandService.ts | 183 ++++---- .../src/main/services/ipc/registerIpc.ts | 6 +- .../localRuntimeConnectionPool.ts | 62 +-- .../updates/autoUpdateService.test.ts | 7 +- .../services/updates/autoUpdateService.ts | 77 +--- .../services/updates/autoUpdateVersions.ts | 71 +++ .../components/settings/AboutSection.tsx | 2 +- apps/desktop/src/shared/adeRuntimeProtocol.ts | 75 ++++ 15 files changed, 821 insertions(+), 642 deletions(-) create mode 100644 apps/ade-cli/src/services/sync/abortSignal.ts create mode 100644 apps/desktop/src/main/services/updates/autoUpdateVersions.ts diff --git a/apps/ade-cli/src/cli.test.ts b/apps/ade-cli/src/cli.test.ts index 1b8fa9bdb..005682086 100644 --- a/apps/ade-cli/src/cli.test.ts +++ b/apps/ade-cli/src/cli.test.ts @@ -849,6 +849,7 @@ describe("ADE CLI", () => { packageChannel: null, projectRoot: null, pid: 123, + uptimeMs: null, }; expect( diff --git a/apps/ade-cli/src/cli.ts b/apps/ade-cli/src/cli.ts index cf517b01b..e3413a2d7 100644 --- a/apps/ade-cli/src/cli.ts +++ b/apps/ade-cli/src/cli.ts @@ -24,10 +24,11 @@ import { runSkillCommand, } from "./commands/skill"; import { - evaluateDoctorRows, - type DoctorInput, + resolveDefaultDesktopAppName, + runDoctorCommand, type DoctorRow, } from "./commands/doctor"; +export { readInstalledDesktopVersion } from "./commands/doctor"; import { buildDeeplink, type DeeplinkEnvelope } from "../../desktop/src/shared/deeplinks"; import { buildPairingQrPayload } from "../../desktop/src/shared/pairingQr"; import { buildWebClientPairUrl } from "../../desktop/src/shared/webClientUrl"; @@ -14197,7 +14198,7 @@ type MachineRuntimeInfo = { packageChannel: string | null; projectRoot: string | null; pid: number | null; - uptimeMs?: number | null; + uptimeMs: number | null; }; function readMachineRuntimeInfo(value: unknown): MachineRuntimeInfo { @@ -14951,360 +14952,6 @@ async function runBrainCommand( ); } -type DoctorBrainProbe = { - brain: DoctorInput["brain"]; - syncStatus: JsonObject | null; - account: DoctorInput["account"]; - runtimeSyncPort: number | null; - runtimePublishHealth: DoctorInput["publishHealth"]; - runtimeLastWedge: DoctorInput["wedge"]; -}; - -const DOCTOR_BRAIN_DEADLINE_MS = 2_000; -const DOCTOR_ONLINE_DEADLINE_MS = 1_200; - -function doctorTimeout( - promise: Promise, - deadlineMs: number, - label: string, -): Promise { - let timer: ReturnType | null = null; - return Promise.race([ - promise, - new Promise((_resolve, reject) => { - timer = setTimeout(() => reject(new Error(`${label} timed out.`)), Math.max(1, deadlineMs)); - }), - ]).finally(() => { - if (timer) clearTimeout(timer); - }); -} - -function doctorRuntimeStatusFromInitialize(value: unknown): { - syncPort: number | null; - publishHealth: DoctorInput["publishHealth"]; - lastWedge: DoctorInput["wedge"]; -} { - const runtimeInfo = isRecord(value) && isRecord(value.runtimeInfo) - ? value.runtimeInfo - : {}; - const syncPort = typeof runtimeInfo.syncPort === "number" - && Number.isInteger(runtimeInfo.syncPort) - && runtimeInfo.syncPort > 0 - && runtimeInfo.syncPort <= 65_535 - ? runtimeInfo.syncPort - : null; - const rawPublish = isRecord(runtimeInfo.publishHealth) ? runtimeInfo.publishHealth : null; - const rawDurations = rawPublish && isRecord(rawPublish.lastLegDurations) - ? rawPublish.lastLegDurations - : null; - const publishHealth = rawPublish && rawDurations && asString(rawPublish.state) - ? { - state: asString(rawPublish.state)! as NonNullable["state"], - failingSinceMs: - typeof rawPublish.failingSinceMs === "number" && Number.isFinite(rawPublish.failingSinceMs) - ? Math.max(0, rawPublish.failingSinceMs) - : null, - lastLegDurations: { - snapshot: finiteDoctorDuration(rawDurations.snapshot), - token: finiteDoctorDuration(rawDurations.token), - http: finiteDoctorDuration(rawDurations.http), - }, - lastSuccessAt: - typeof rawPublish.lastSuccessAt === "number" && Number.isFinite(rawPublish.lastSuccessAt) - ? Math.max(0, rawPublish.lastSuccessAt) - : null, - skipReason: asString(rawPublish.skipReason), - } - : null; - const rawWedge = isRecord(runtimeInfo.lastWedge) ? runtimeInfo.lastWedge : null; - const lastWedge = rawWedge - && asString(rawWedge.lastCommand) - && typeof rawWedge.blockedMs === "number" - && Number.isFinite(rawWedge.blockedMs) - && asString(rawWedge.ts) - ? { - lastCommand: asString(rawWedge.lastCommand)!, - blockedMs: Math.max(0, Math.floor(rawWedge.blockedMs)), - ts: asString(rawWedge.ts)!, - } - : null; - return { syncPort, publishHealth, lastWedge }; -} - -function finiteDoctorDuration(value: unknown): number | null { - return typeof value === "number" && Number.isFinite(value) && value >= 0 - ? Math.round(value) - : null; -} - -async function probeDoctorBrain( - options: GlobalOptions, -): Promise { - const startedAt = Date.now(); - const deadlineAt = startedAt + DOCTOR_BRAIN_DEADLINE_MS; - const socketPath = await resolveMachineRuntimeSocketPath(options.socketPath); - let client: SocketJsonRpcClient | null = null; - try { - const remaining = () => Math.max(1, deadlineAt - Date.now()); - client = await doctorTimeout( - SocketJsonRpcClient.connect(socketPath, remaining(), "ADE brain"), - remaining(), - "ADE brain connection", - ); - const initializeResult = await doctorTimeout( - client.request( - "ade/initialize", - buildInitializeParams(options, "ade-doctor"), - ), - remaining(), - "ADE brain initialize", - ); - const runtimeInfo = readMachineRuntimeInfo(initializeResult); - const runtimeStatus = doctorRuntimeStatusFromInitialize(initializeResult); - const expectedBuildHash = await doctorTimeout( - resolveExpectedMachineRuntimeBuildHash(), - remaining(), - "ADE runtime build identity", - ).catch(() => null); - const mismatchReason = machineRuntimeMismatchReason( - runtimeInfo, - expectedBuildHash, - options.role, - ); - const [syncResult, accountResult] = await doctorTimeout( - Promise.allSettled([ - client.request("sync.getStatus", { includeTransferReadiness: false }), - client.request("account.call", { action: "status", args: {} }), - ]), - remaining(), - "ADE brain health reads", - ); - const syncStatus = syncResult.status === "fulfilled" && isRecord(syncResult.value) - ? syncResult.value - : null; - const accountValue = accountResult.status === "fulfilled" - ? unwrapActionEnvelope(accountResult.value) - : null; - const accountStatus = isRecord(accountValue) ? accountValue : null; - return { - brain: { - running: true, - version: runtimeInfo.version, - buildHash: runtimeInfo.buildHash, - pid: runtimeInfo.pid, - uptimeMs: runtimeInfo.uptimeMs ?? null, - mismatchReason, - error: null, - }, - syncStatus, - account: { - signedIn: typeof accountStatus?.signedIn === "boolean" ? accountStatus.signedIn : null, - source: asString(accountStatus?.source), - error: accountResult.status === "rejected" - ? accountResult.reason instanceof Error - ? accountResult.reason.message - : String(accountResult.reason) - : accountStatus - ? null - : "Account status unavailable.", - }, - runtimeSyncPort: runtimeStatus.syncPort, - runtimePublishHealth: runtimeStatus.publishHealth, - runtimeLastWedge: runtimeStatus.lastWedge, - }; - } catch (error) { - return { - brain: { - running: false, - version: null, - buildHash: null, - pid: null, - uptimeMs: null, - mismatchReason: null, - error: error instanceof Error ? error.message : String(error), - }, - syncStatus: null, - account: { - signedIn: null, - source: null, - error: "Brain unavailable.", - }, - runtimeSyncPort: null, - runtimePublishHealth: null, - runtimeLastWedge: null, - }; - } finally { - try { - client?.destroy(); - } catch {} - } -} - -export function readInstalledDesktopVersion( - appPaths?: string[], -): { version: string | null; path: string | null } { - if (process.platform !== "darwin" && !appPaths) { - return { version: null, path: null }; - } - const explicitPath = process.env.ADE_DESKTOP_APP_PATH?.trim(); - const preferredName = resolveDefaultDesktopAppName(); - const candidates = appPaths ?? [ - ...(explicitPath ? [explicitPath] : []), - `/Applications/${preferredName}.app`, - "/Applications/ADE.app", - "/Applications/ADE Beta.app", - "/Applications/ADE Alpha.app", - ]; - for (const appPath of Array.from(new Set(candidates))) { - const infoPlist = path.join(appPath, "Contents", "Info.plist"); - if (!fs.existsSync(infoPlist)) continue; - try { - const raw = fs.readFileSync(infoPlist, "utf8"); - const xmlMatch = /\s*CFBundleShortVersionString\s*<\/key>\s*\s*([^<]+?)\s*<\/string>/i.exec(raw); - if (xmlMatch?.[1]?.trim()) { - return { version: xmlMatch[1].trim(), path: appPath }; - } - } catch { - // Binary plists are handled by plutil below. - } - const result = spawnSync( - "/usr/bin/plutil", - ["-extract", "CFBundleShortVersionString", "raw", "-o", "-", infoPlist], - { encoding: "utf8", timeout: 500 }, - ); - const version = typeof result.stdout === "string" ? result.stdout.trim() : ""; - if (result.status === 0 && version) { - return { version, path: appPath }; - } - } - return { version: null, path: null }; -} - -async function readLatestDesktopVersionOnline(): Promise { - const controller = new AbortController(); - const timer = setTimeout(() => controller.abort(), DOCTOR_ONLINE_DEADLINE_MS); - try { - const { fetchAdeLatestRelease } = await import( - "../../desktop/src/main/services/github/githubService" - ); - const release = await doctorTimeout( - fetchAdeLatestRelease({ - fetchImpl: ((input: string, init?: RequestInit) => - fetch(input, { ...init, signal: controller.signal })) as never, - }), - DOCTOR_ONLINE_DEADLINE_MS, - "Latest release check", - ); - return release?.version ?? null; - } catch { - return null; - } finally { - clearTimeout(timer); - } -} - -function readLatestKnownDesktopVersionFromDisk(runtimeDir: string): string | null { - try { - const parsed = JSON.parse( - fs.readFileSync(path.join(runtimeDir, "update-status.json"), "utf8"), - ) as Record; - return asString(parsed.version); - } catch { - return null; - } -} - -async function runDoctorCommand( - online: boolean, - options: GlobalOptions, -): Promise<{ - ok: boolean; - checkedAt: string; - online: boolean; - rows: DoctorRow[]; - app: DoctorInput["app"]; - brain: DoctorInput["brain"]; - wedge: DoctorInput["wedge"]; - syncPort: number | null; - portDiagnoses: DoctorInput["portDiagnoses"]; - publishHealth: DoctorInput["publishHealth"]; - relayHealth: DoctorInput["relayHealth"]; - account: DoctorInput["account"]; -}> { - const nowMs = Date.now(); - const layout = resolveMachineAdeLayout(); - const diskLatestKnownVersion = readLatestKnownDesktopVersionFromDisk(layout.runtimeDir); - const [installedApp, latestKnownVersion, brainProbe] = await Promise.all([ - Promise.resolve(readInstalledDesktopVersion()), - online - ? readLatestDesktopVersionOnline().then((version) => version ?? diskLatestKnownVersion) - : Promise.resolve(diskLatestKnownVersion), - probeDoctorBrain(options), - ]); - const syncRouteHealth = brainProbe.syncStatus && isRecord(brainProbe.syncStatus.routeHealth) - ? brainProbe.syncStatus.routeHealth - : null; - const listener = syncRouteHealth && isRecord(syncRouteHealth.listener) - ? syncRouteHealth.listener - : null; - const syncPort = typeof listener?.port === "number" && Number.isInteger(listener.port) - ? listener.port - : brainProbe.runtimeSyncPort; - const rawPublishHealth = syncRouteHealth && isRecord(syncRouteHealth.accountDirectory) - ? syncRouteHealth.accountDirectory - : null; - const publishHealth = rawPublishHealth - ? doctorRuntimeStatusFromInitialize({ - runtimeInfo: { publishHealth: rawPublishHealth }, - }).publishHealth - : brainProbe.runtimePublishHealth; - const relayHealth = syncRouteHealth && isRecord(syncRouteHealth.relay) - ? syncRouteHealth.relay as DoctorInput["relayHealth"] - : null; - let portDiagnoses: DoctorInput["portDiagnoses"] = []; - if (brainProbe.brain.running && syncPort != null && syncPort !== DEFAULT_SYNC_HOST_PORT) { - const { inspectSyncListenerPort } = await import("./services/sync/sharedSyncListener"); - portDiagnoses = [ - inspectSyncListenerPort(DEFAULT_SYNC_HOST_PORT), - inspectSyncListenerPort(DEFAULT_SYNC_HOST_PORT + 1), - inspectSyncListenerPort(DEFAULT_SYNC_HOST_PORT + 2), - ]; - } - const wedge = readBrainLoopWatchdogLastWedge(layout.runtimeDir) - ?? brainProbe.runtimeLastWedge; - const input: DoctorInput = { - nowMs, - app: { - installedVersion: installedApp.version, - latestKnownVersion, - path: installedApp.path, - online, - }, - brain: brainProbe.brain, - wedge, - syncPort, - portDiagnoses, - publishHealth, - relayHealth, - account: brainProbe.account, - }; - const rows = evaluateDoctorRows(input); - return { - ok: !rows.some((row) => row.status === "fail"), - checkedAt: new Date(nowMs).toISOString(), - online, - rows, - app: input.app, - brain: input.brain, - wedge, - syncPort, - portDiagnoses, - publishHealth, - relayHealth, - account: input.account, - }; -} - async function runDesktopCommand(rest: string[]): Promise { const args = [...rest]; const sub = firstPositional(args) ?? "open"; @@ -15339,15 +14986,6 @@ async function runDesktopCommand(rest: string[]): Promise { }; } -function resolveDefaultDesktopAppName(): string { - const explicit = process.env.ADE_DESKTOP_APP_NAME?.trim(); - if (explicit) return explicit; - const channel = process.env.ADE_PACKAGE_CHANNEL?.trim().toLowerCase(); - if (channel === "alpha") return "ADE Alpha"; - if (channel === "beta") return "ADE Beta"; - return "ADE"; -} - async function runNativeRpcStdio(options: GlobalOptions): Promise { const previousRole = process.env.ADE_DEFAULT_ROLE; process.env.ADE_DEFAULT_ROLE = options.role; @@ -19878,7 +19516,16 @@ async function runCli( } } if (plan.kind === "doctor") { - const result = await runDoctorCommand(plan.online, parsed.options); + const result = await runDoctorCommand(plan.online, parsed.options, { + resolveMachineRuntimeSocketPath, + connectRuntime: (socketPath, timeoutMs, label) => + SocketJsonRpcClient.connect(socketPath, timeoutMs, label), + buildInitializeParams, + readMachineRuntimeInfo, + resolveExpectedMachineRuntimeBuildHash, + machineRuntimeMismatchReason, + unwrapActionEnvelope, + }); return { output: formatOutput(result, parsed.options, "doctor"), exitCode: result.ok ? 0 : 1, diff --git a/apps/ade-cli/src/commands/doctor.test.ts b/apps/ade-cli/src/commands/doctor.test.ts index 18bfe98fd..f2be9ce91 100644 --- a/apps/ade-cli/src/commands/doctor.test.ts +++ b/apps/ade-cli/src/commands/doctor.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { compareDoctorVersions, + doctorRuntimeStatusFromInitialize, evaluateDoctorRows, type DoctorInput, } from "./doctor"; @@ -54,6 +55,48 @@ function healthyInput(): DoctorInput { } describe("doctor row evaluation", () => { + it("parses the complete runtime publish and wedge health envelope", () => { + expect(doctorRuntimeStatusFromInitialize({ + runtimeInfo: { + syncPort: 8789, + publishHealth: { + state: " published ", + failingSinceMs: -1, + lastLegDurations: { + snapshot: 12.4, + token: 34.6, + http: -1, + }, + lastSuccessAt: 123_456.5, + skipReason: " signed out ", + }, + lastWedge: { + lastCommand: " chat.send ", + blockedMs: 16_500.8, + ts: " 2026-07-23T12:00:00.000Z ", + }, + }, + })).toEqual({ + syncPort: 8789, + publishHealth: { + state: "published", + failingSinceMs: 0, + lastLegDurations: { + snapshot: 12, + token: 35, + http: null, + }, + lastSuccessAt: 123_456.5, + skipReason: "signed out", + }, + lastWedge: { + lastCommand: "chat.send", + blockedMs: 16_500, + ts: "2026-07-23T12:00:00.000Z", + }, + }); + }); + it("marks a healthy machine with no red rows", () => { const rows = evaluateDoctorRows(healthyInput()); @@ -141,4 +184,3 @@ describe("doctor row evaluation", () => { expect(compareDoctorVersions("next", "1.2.35")).toBeNull(); }); }); - diff --git a/apps/ade-cli/src/commands/doctor.ts b/apps/ade-cli/src/commands/doctor.ts index ed1b19e9f..60522ea61 100644 --- a/apps/ade-cli/src/commands/doctor.ts +++ b/apps/ade-cli/src/commands/doctor.ts @@ -1,3 +1,11 @@ +import { spawnSync } from "node:child_process"; +import fs from "node:fs"; +import path from "node:path"; +import { + parseRuntimeLastWedge, + parseRuntimePublishHealth, + type RuntimePublishHealth, +} from "../../../desktop/src/shared/adeRuntimeProtocol"; import type { SyncAccountDirectoryHealth, SyncRouteHealth, @@ -5,6 +13,9 @@ import type { import type { BrainLoopWatchdogBreadcrumb, } from "../services/runtime/brainLoopWatchdog"; +import { readBrainLoopWatchdogLastWedge } from "../services/runtime/brainLoopWatchdog"; +import { resolveMachineAdeLayout } from "../services/projects/machineLayout"; +import { DEFAULT_SYNC_HOST_PORT } from "../services/sync/syncProtocol"; import type { SyncListenerPortDiagnosis, } from "../services/sync/sharedSyncListener"; @@ -64,10 +75,326 @@ export type DoctorInput = { }; }; +export type DoctorCommandOptions = { + role: "cto" | "orchestrator" | "agent" | "external" | "evaluator"; + socketPath: string | null; +}; + +export type DoctorMachineRuntimeInfo = { + version: string | null; + buildHash: string | null; + defaultRole: string | null; + packageChannel: string | null; + projectRoot: string | null; + pid: number | null; + uptimeMs: number | null; +}; + +type DoctorRpcClient = { + request(method: string, params?: unknown): Promise; + destroy(): void; +}; + +export type DoctorCommandDependencies< + Options extends DoctorCommandOptions = DoctorCommandOptions, +> = { + resolveMachineRuntimeSocketPath( + socketPathOverride?: string | null, + ): Promise; + connectRuntime( + socketPath: string, + timeoutMs: number, + label?: string, + ): Promise; + buildInitializeParams( + options: Options, + clientName: string, + ): Record; + readMachineRuntimeInfo(value: unknown): DoctorMachineRuntimeInfo; + resolveExpectedMachineRuntimeBuildHash(): Promise; + machineRuntimeMismatchReason( + runtimeInfo: DoctorMachineRuntimeInfo, + expectedBuildHash: string | null, + expectedDefaultRole: Options["role"], + ): string | null; + unwrapActionEnvelope(value: unknown): unknown; +}; + +export type DoctorCommandResult = { + ok: boolean; + checkedAt: string; + online: boolean; + rows: DoctorRow[]; + app: DoctorInput["app"]; + brain: DoctorInput["brain"]; + wedge: DoctorInput["wedge"]; + syncPort: number | null; + portDiagnoses: DoctorInput["portDiagnoses"]; + publishHealth: DoctorInput["publishHealth"]; + relayHealth: DoctorInput["relayHealth"]; + account: DoctorInput["account"]; +}; + +type DoctorBrainProbe = { + brain: DoctorInput["brain"]; + syncStatus: Record | null; + account: DoctorInput["account"]; + runtimeSyncPort: number | null; + runtimePublishHealth: DoctorInput["publishHealth"]; + runtimeLastWedge: DoctorInput["wedge"]; +}; + +const DOCTOR_BRAIN_DEADLINE_MS = 2_000; +const DOCTOR_ONLINE_DEADLINE_MS = 1_200; const DAY_MS = 24 * 60 * 60 * 1_000; const RECENT_PUBLISH_MS = 2 * 60 * 1_000; const PUBLISH_FAILURE_RED_MS = 2 * 60 * 1_000; +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function asString(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function doctorTimeout( + promise: Promise, + deadlineMs: number, + label: string, +): Promise { + let timer: ReturnType | null = null; + return Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error(`${label} timed out.`)), Math.max(1, deadlineMs)); + }), + ]).finally(() => { + if (timer) clearTimeout(timer); + }); +} + +function toDoctorPublishHealth( + health: RuntimePublishHealth | null, +): DoctorInput["publishHealth"] { + return health + ? { + ...health, + state: health.state.trim() as NonNullable["state"], + } + : null; +} + +export function doctorRuntimeStatusFromInitialize(value: unknown): { + syncPort: number | null; + publishHealth: DoctorInput["publishHealth"]; + lastWedge: DoctorInput["wedge"]; +} { + const runtimeInfo = isRecord(value) && isRecord(value.runtimeInfo) + ? value.runtimeInfo + : {}; + const syncPort = typeof runtimeInfo.syncPort === "number" + && Number.isInteger(runtimeInfo.syncPort) + && runtimeInfo.syncPort > 0 + && runtimeInfo.syncPort <= 65_535 + ? runtimeInfo.syncPort + : null; + return { + syncPort, + publishHealth: toDoctorPublishHealth( + parseRuntimePublishHealth(runtimeInfo.publishHealth), + ), + lastWedge: parseRuntimeLastWedge(runtimeInfo.lastWedge), + }; +} + +async function probeDoctorBrain( + options: Options, + dependencies: DoctorCommandDependencies, +): Promise { + const startedAt = Date.now(); + const deadlineAt = startedAt + DOCTOR_BRAIN_DEADLINE_MS; + const socketPath = await dependencies.resolveMachineRuntimeSocketPath(options.socketPath); + let client: DoctorRpcClient | null = null; + try { + const remaining = () => Math.max(1, deadlineAt - Date.now()); + client = await doctorTimeout( + dependencies.connectRuntime(socketPath, remaining(), "ADE brain"), + remaining(), + "ADE brain connection", + ); + const initializeResult = await doctorTimeout( + client.request( + "ade/initialize", + dependencies.buildInitializeParams(options, "ade-doctor"), + ), + remaining(), + "ADE brain initialize", + ); + const runtimeInfo = dependencies.readMachineRuntimeInfo(initializeResult); + const runtimeStatus = doctorRuntimeStatusFromInitialize(initializeResult); + const expectedBuildHash = await doctorTimeout( + dependencies.resolveExpectedMachineRuntimeBuildHash(), + remaining(), + "ADE runtime build identity", + ).catch(() => null); + const mismatchReason = dependencies.machineRuntimeMismatchReason( + runtimeInfo, + expectedBuildHash, + options.role, + ); + const [syncResult, accountResult] = await doctorTimeout( + Promise.allSettled([ + client.request("sync.getStatus", { includeTransferReadiness: false }), + client.request("account.call", { action: "status", args: {} }), + ]), + remaining(), + "ADE brain health reads", + ); + const syncStatus = syncResult.status === "fulfilled" && isRecord(syncResult.value) + ? syncResult.value + : null; + const accountValue = accountResult.status === "fulfilled" + ? dependencies.unwrapActionEnvelope(accountResult.value) + : null; + const accountStatus = isRecord(accountValue) ? accountValue : null; + return { + brain: { + running: true, + version: runtimeInfo.version, + buildHash: runtimeInfo.buildHash, + pid: runtimeInfo.pid, + uptimeMs: runtimeInfo.uptimeMs, + mismatchReason, + error: null, + }, + syncStatus, + account: { + signedIn: typeof accountStatus?.signedIn === "boolean" ? accountStatus.signedIn : null, + source: asString(accountStatus?.source), + error: accountResult.status === "rejected" + ? accountResult.reason instanceof Error + ? accountResult.reason.message + : String(accountResult.reason) + : accountStatus + ? null + : "Account status unavailable.", + }, + runtimeSyncPort: runtimeStatus.syncPort, + runtimePublishHealth: runtimeStatus.publishHealth, + runtimeLastWedge: runtimeStatus.lastWedge, + }; + } catch (error) { + return { + brain: { + running: false, + version: null, + buildHash: null, + pid: null, + uptimeMs: null, + mismatchReason: null, + error: error instanceof Error ? error.message : String(error), + }, + syncStatus: null, + account: { + signedIn: null, + source: null, + error: "Brain unavailable.", + }, + runtimeSyncPort: null, + runtimePublishHealth: null, + runtimeLastWedge: null, + }; + } finally { + try { + client?.destroy(); + } catch {} + } +} + +export function resolveDefaultDesktopAppName(): string { + const explicit = process.env.ADE_DESKTOP_APP_NAME?.trim(); + if (explicit) return explicit; + const channel = process.env.ADE_PACKAGE_CHANNEL?.trim().toLowerCase(); + if (channel === "alpha") return "ADE Alpha"; + if (channel === "beta") return "ADE Beta"; + return "ADE"; +} + +export function readInstalledDesktopVersion( + appPaths?: string[], +): { version: string | null; path: string | null } { + if (process.platform !== "darwin" && !appPaths) { + return { version: null, path: null }; + } + const explicitPath = process.env.ADE_DESKTOP_APP_PATH?.trim(); + const preferredName = resolveDefaultDesktopAppName(); + const candidates = appPaths ?? [ + ...(explicitPath ? [explicitPath] : []), + `/Applications/${preferredName}.app`, + "/Applications/ADE.app", + "/Applications/ADE Beta.app", + "/Applications/ADE Alpha.app", + ]; + for (const appPath of Array.from(new Set(candidates))) { + const infoPlist = path.join(appPath, "Contents", "Info.plist"); + if (!fs.existsSync(infoPlist)) continue; + try { + const raw = fs.readFileSync(infoPlist, "utf8"); + const xmlMatch = /\s*CFBundleShortVersionString\s*<\/key>\s*\s*([^<]+?)\s*<\/string>/i.exec(raw); + if (xmlMatch?.[1]?.trim()) { + return { version: xmlMatch[1].trim(), path: appPath }; + } + } catch { + // Binary plists are handled by plutil below. + } + const result = spawnSync( + "/usr/bin/plutil", + ["-extract", "CFBundleShortVersionString", "raw", "-o", "-", infoPlist], + { encoding: "utf8", timeout: 500 }, + ); + const version = typeof result.stdout === "string" ? result.stdout.trim() : ""; + if (result.status === 0 && version) { + return { version, path: appPath }; + } + } + return { version: null, path: null }; +} + +async function readLatestDesktopVersionOnline(): Promise { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), DOCTOR_ONLINE_DEADLINE_MS); + try { + const { fetchAdeLatestRelease } = await import( + "../../../desktop/src/main/services/github/githubService" + ); + const release = await doctorTimeout( + fetchAdeLatestRelease({ + fetchImpl: ((input: string, init?: RequestInit) => + fetch(input, { ...init, signal: controller.signal })) as never, + }), + DOCTOR_ONLINE_DEADLINE_MS, + "Latest release check", + ); + return release?.version ?? null; + } catch { + return null; + } finally { + clearTimeout(timer); + } +} + +function readLatestKnownDesktopVersionFromDisk(runtimeDir: string): string | null { + try { + const parsed = JSON.parse( + fs.readFileSync(path.join(runtimeDir, "update-status.json"), "utf8"), + ) as Record; + return asString(parsed.version); + } catch { + return null; + } +} + function normalizedVersionParts(value: string): number[] | null { const match = /^v?(\d+(?:\.\d+){0,3})/.exec(value.trim()); return match ? match[1]!.split(".").map((part) => Number.parseInt(part, 10)) : null; @@ -333,3 +660,80 @@ export function evaluateDoctorRows(input: DoctorInput): DoctorRow[] { accountRow(input.account), ]; } + +export async function runDoctorCommand( + online: boolean, + options: Options, + dependencies: DoctorCommandDependencies, +): Promise { + const nowMs = Date.now(); + const layout = resolveMachineAdeLayout(); + const diskLatestKnownVersion = readLatestKnownDesktopVersionFromDisk(layout.runtimeDir); + const [installedApp, latestKnownVersion, brainProbe] = await Promise.all([ + Promise.resolve(readInstalledDesktopVersion()), + online + ? readLatestDesktopVersionOnline().then((version) => version ?? diskLatestKnownVersion) + : Promise.resolve(diskLatestKnownVersion), + probeDoctorBrain(options, dependencies), + ]); + const syncRouteHealth = brainProbe.syncStatus && isRecord(brainProbe.syncStatus.routeHealth) + ? brainProbe.syncStatus.routeHealth + : null; + const listener = syncRouteHealth && isRecord(syncRouteHealth.listener) + ? syncRouteHealth.listener + : null; + const syncPort = typeof listener?.port === "number" && Number.isInteger(listener.port) + ? listener.port + : brainProbe.runtimeSyncPort; + const rawPublishHealth = syncRouteHealth && isRecord(syncRouteHealth.accountDirectory) + ? syncRouteHealth.accountDirectory + : null; + const publishHealth = rawPublishHealth + ? toDoctorPublishHealth(parseRuntimePublishHealth(rawPublishHealth)) + : brainProbe.runtimePublishHealth; + const relayHealth = syncRouteHealth && isRecord(syncRouteHealth.relay) + ? syncRouteHealth.relay as DoctorInput["relayHealth"] + : null; + let portDiagnoses: DoctorInput["portDiagnoses"] = []; + if (brainProbe.brain.running && syncPort != null && syncPort !== DEFAULT_SYNC_HOST_PORT) { + const { inspectSyncListenerPort } = await import("../services/sync/sharedSyncListener"); + portDiagnoses = [ + inspectSyncListenerPort(DEFAULT_SYNC_HOST_PORT), + inspectSyncListenerPort(DEFAULT_SYNC_HOST_PORT + 1), + inspectSyncListenerPort(DEFAULT_SYNC_HOST_PORT + 2), + ]; + } + const wedge = readBrainLoopWatchdogLastWedge(layout.runtimeDir) + ?? brainProbe.runtimeLastWedge; + const input: DoctorInput = { + nowMs, + app: { + installedVersion: installedApp.version, + latestKnownVersion, + path: installedApp.path, + online, + }, + brain: brainProbe.brain, + wedge, + syncPort, + portDiagnoses, + publishHealth, + relayHealth, + account: brainProbe.account, + }; + const rows = evaluateDoctorRows(input); + return { + ok: !rows.some((row) => row.status === "fail"), + checkedAt: new Date(nowMs).toISOString(), + online, + rows, + app: input.app, + brain: input.brain, + wedge, + syncPort, + portDiagnoses, + publishHealth, + relayHealth, + account: input.account, + }; +} diff --git a/apps/ade-cli/src/services/sync/abortSignal.ts b/apps/ade-cli/src/services/sync/abortSignal.ts new file mode 100644 index 000000000..ff9fc3c0a --- /dev/null +++ b/apps/ade-cli/src/services/sync/abortSignal.ts @@ -0,0 +1,41 @@ +export function abortSignalError( + signal: AbortSignal, + fallbackMessage: string, +): Error { + if (signal.reason instanceof Error) return signal.reason; + const error = new Error( + typeof signal.reason === "string" && signal.reason.trim() + ? signal.reason + : fallbackMessage, + ); + error.name = "AbortError"; + return error; +} + +export async function runWithAbortSignal( + run: () => Promise | T, + signal: AbortSignal | undefined, + fallbackMessage: string, +): Promise { + if (!signal) return await run(); + if (signal.aborted) throw abortSignalError(signal, fallbackMessage); + return await new Promise((resolve, reject) => { + let settled = false; + const settle = (complete: () => void): void => { + if (settled) return; + settled = true; + signal.removeEventListener("abort", onAbort); + complete(); + }; + const onAbort = (): void => { + settle(() => reject(abortSignalError(signal, fallbackMessage))); + }; + signal.addEventListener("abort", onAbort, { once: true }); + Promise.resolve() + .then(run) + .then( + (value) => settle(() => resolve(value)), + (error) => settle(() => reject(error)), + ); + }); +} diff --git a/apps/ade-cli/src/services/sync/syncHostService.ts b/apps/ade-cli/src/services/sync/syncHostService.ts index 64bf8925a..ed6bf8cb2 100644 --- a/apps/ade-cli/src/services/sync/syncHostService.ts +++ b/apps/ade-cli/src/services/sync/syncHostService.ts @@ -3,6 +3,7 @@ import http from "node:http"; import { execFile, spawn, type ChildProcess } from "node:child_process"; import os from "node:os"; import path from "node:path"; +import { runWithAbortSignal } from "./abortSignal"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { @@ -858,41 +859,6 @@ function sanitizeRemoteAddress(remoteAddress: string | null | undefined): string return value.startsWith("::ffff:") ? value.slice("::ffff:".length) : value; } -function syncOperationAbortError(signal: AbortSignal): Error { - if (signal.reason instanceof Error) return signal.reason; - const error = new Error( - typeof signal.reason === "string" && signal.reason.trim() - ? signal.reason - : "Sync operation aborted.", - ); - error.name = "AbortError"; - return error; -} - -async function runWithSyncOperationSignal( - run: () => Promise | T, - signal: AbortSignal, -): Promise { - if (signal.aborted) throw syncOperationAbortError(signal); - return await new Promise((resolve, reject) => { - let settled = false; - const settle = (complete: () => void): void => { - if (settled) return; - settled = true; - signal.removeEventListener("abort", onAbort); - complete(); - }; - const onAbort = (): void => settle(() => reject(syncOperationAbortError(signal))); - signal.addEventListener("abort", onAbort, { once: true }); - Promise.resolve() - .then(run) - .then( - (value) => settle(() => resolve(value)), - (error) => settle(() => reject(error)), - ); - }); -} - function ensureBootstrapToken(filePath: string): string { fs.mkdirSync(path.dirname(filePath), { recursive: true }); if (!fs.existsSync(filePath)) { @@ -7255,13 +7221,14 @@ export function createSyncHostService(args: SyncHostServiceArgs) { barrier.captureAttempt += 1; const session = args.sessionService.get(sessionId); const transcriptSnapshot = session - ? await runWithSyncOperationSignal( + ? await runWithAbortSignal( () => args.ptyService.readTranscriptSnapshot({ sessionId, maxBytes, alignStartToSafeBoundary: true, }), signal, + "Sync operation aborted.", ) : null; if (!isCurrentTerminalSnapshotBarrier(peer, sessionId, barrier, lifecycleGeneration)) break; @@ -7405,7 +7372,7 @@ export function createSyncHostService(args: SyncHostServiceArgs) { Math.min(beforeOffset, transcriptWindow.endOffset), ); const requestedStartOffset = Math.max(transcriptWindow.startOffset, endOffset - pageBytes); - const range = await runWithSyncOperationSignal( + const range = await runWithAbortSignal( () => args.ptyService.readTranscriptRange({ sessionId, startOffset: requestedStartOffset, @@ -7413,6 +7380,7 @@ export function createSyncHostService(args: SyncHostServiceArgs) { alignStartToSafeBoundary: true, }), signal, + "Sync operation aborted.", ); if (!range) { sendRequired(peer, "terminal_history", refused, envelope.requestId); @@ -7468,9 +7436,10 @@ export function createSyncHostService(args: SyncHostServiceArgs) { // subscription at all, or the periodic pump would stream the ACTIVE // project's transcript for the same session id after the empty ack. const personalTranscriptPath = personalChatRequested - ? await runWithSyncOperationSignal( + ? await runWithAbortSignal( () => args.personalChatScope?.transcriptPath?.(sessionId).catch(() => null) ?? null, signal, + "Sync operation aborted.", ) : null; const foreignScope = personalChatRequested @@ -7513,16 +7482,18 @@ export function createSyncHostService(args: SyncHostServiceArgs) { // derive turn state from the streamed status events instead. const resolveLiveStatusFields = async (): Promise<{ turnActive?: boolean }> => { if (personalChatRequested) { - const turnActive = await runWithSyncOperationSignal( + const turnActive = await runWithAbortSignal( () => args.personalChatScope?.isTurnActive?.(sessionId).catch(() => false) ?? false, signal, + "Sync operation aborted.", ); return typeof turnActive === "boolean" ? { turnActive } : {}; } if (foreignScope.kind !== "local") return {}; - const liveSummary = await runWithSyncOperationSignal( + const liveSummary = await runWithAbortSignal( () => args.agentChatService?.getSessionSummary(sessionId).catch(() => null) ?? null, signal, + "Sync operation aborted.", ); return liveSummary ? { turnActive: liveSummary.status === "active" } : {}; }; @@ -7572,9 +7543,10 @@ export function createSyncHostService(args: SyncHostServiceArgs) { let truncated: boolean; let transcriptSize: number; if (foreignTranscriptPath) { - const foreignSnapshot = await runWithSyncOperationSignal( + const foreignSnapshot = await runWithAbortSignal( () => readForeignChatSnapshot(foreignTranscriptPath, maxBytes), signal, + "Sync operation aborted.", ); events = foreignSnapshot.events; truncated = foreignSnapshot.truncated; diff --git a/apps/ade-cli/src/services/sync/syncRemoteCommandService.test.ts b/apps/ade-cli/src/services/sync/syncRemoteCommandService.test.ts index 515f752eb..cd6dd204b 100644 --- a/apps/ade-cli/src/services/sync/syncRemoteCommandService.test.ts +++ b/apps/ade-cli/src/services/sync/syncRemoteCommandService.test.ts @@ -120,6 +120,61 @@ function makePairingConnectInfo( } describe("createSyncRemoteCommandService", () => { + it("preserves the exact remote-command set that observes execution aborts", () => { + const { service } = createService(); + + expect(new Set(service.getAbortObservingActions())).toEqual(new Set([ + "chat.resolveSmartLinkPreview", + "chat.getChatEventHistory", + "chat.getTranscript", + "chat.getSubagentTranscript", + "chat.getMainTranscript", + "chat.getChatEventHistoryPage", + "agentChat.getEventHistoryPage", + "github.getStatus", + "github.getRemoteStatus", + "ai.getStatus", + "prs.list", + "prs.listOpenForRepo", + "prs.getForLane", + "prs.refresh", + "prs.getDetail", + "prs.getAiSummary", + "prs.getIntegrationResolutionState", + "prs.listProposals", + "prs.getQueueState", + "prs.listQueueStates", + "prs.getMergeContext", + "prs.getMergeContexts", + "prs.listWithConflicts", + "prs.listSnapshots", + "prs.getStatus", + "prs.getChecks", + "prs.getReviews", + "prs.getComments", + "prs.getFiles", + "prs.getGitHubSnapshot", + "prs.getReviewThreads", + "prs.getActionRuns", + "prs.getActivity", + "prs.getDeployments", + "prs.getDetailByGithub", + "prs.getFilesByGithub", + "prs.getCommitsByGithub", + "prs.getActionRunsByGithub", + "prs.getActivityByGithub", + "prs.getStatusByGithub", + "prs.getChecksByGithub", + "prs.getReviewsByGithub", + "prs.getCommentsByGithub", + "prs.getReviewThreadsByGithub", + "prs.getMobileGithubDetail", + "prs.preflightCreateLaneFromPrBranch", + "prs.listIntegrationWorkflows", + "prs.getMobileSnapshot", + ])); + }); + it("caps ai.getStatus probes at 30 seconds", async () => { vi.useFakeTimers(); const getStatus = vi.fn(() => new Promise(() => {})); diff --git a/apps/ade-cli/src/services/sync/syncRemoteCommandService.ts b/apps/ade-cli/src/services/sync/syncRemoteCommandService.ts index 6ce70c5e8..c97a4c1eb 100644 --- a/apps/ade-cli/src/services/sync/syncRemoteCommandService.ts +++ b/apps/ade-cli/src/services/sync/syncRemoteCommandService.ts @@ -2,6 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import { createHash, randomUUID } from "node:crypto"; import { fileURLToPath } from "node:url"; +import { runWithAbortSignal } from "./abortSignal"; import type { AgentChatCreateArgs, AgentChatCreateScheduledWorkArgs, @@ -389,6 +390,7 @@ export type SyncRemoteCommandExecutionContext = { type RegisteredRemoteCommand = { descriptor: SyncRemoteCommandDescriptor; + observesAbort: boolean; handler: ( args: Record, context: SyncRemoteCommandExecutionContext, @@ -397,59 +399,6 @@ type RegisteredRemoteCommand = { export const AI_STATUS_REMOTE_COMMAND_TIMEOUT_MS = 30_000; -function remoteCommandAbortError(signal: AbortSignal): Error { - if (signal.reason instanceof Error) return signal.reason; - const error = new Error( - typeof signal.reason === "string" && signal.reason.trim() - ? signal.reason - : "Remote command aborted.", - ); - error.name = "AbortError"; - return error; -} - -async function runWithRemoteCommandSignal( - run: () => Promise | T, - signal?: AbortSignal, -): Promise { - if (!signal) return await run(); - if (signal.aborted) throw remoteCommandAbortError(signal); - return await new Promise((resolve, reject) => { - let settled = false; - const settle = (complete: () => void): void => { - if (settled) return; - settled = true; - signal.removeEventListener("abort", onAbort); - complete(); - }; - const onAbort = (): void => settle(() => reject(remoteCommandAbortError(signal))); - signal.addEventListener("abort", onAbort, { once: true }); - Promise.resolve() - .then(run) - .then( - (value) => settle(() => resolve(value)), - (error) => settle(() => reject(error)), - ); - }); -} - -function commandHandlerObservesAbort(action: string): boolean { - return action === "ai.getStatus" - || action === "chat.resolveSmartLinkPreview" - || action === "chat.getTranscript" - || action === "chat.getChatEventHistory" - || action === "chat.getSubagentTranscript" - || action === "chat.getMainTranscript" - || action === "chat.getChatEventHistoryPage" - || action === "agentChat.getEventHistoryPage" - || action === "github.getStatus" - || action === "github.getRemoteStatus" - || action === "prs.refresh" - || action === "prs.preflightCreateLaneFromPrBranch" - || action.startsWith("prs.get") - || action.startsWith("prs.list"); -} - async function runAiStatusWithTimeout( run: () => Promise, parentSignal?: AbortSignal, @@ -467,7 +416,11 @@ async function runAiStatusWithTimeout( }, AI_STATUS_REMOTE_COMMAND_TIMEOUT_MS); timer.unref?.(); try { - return await runWithRemoteCommandSignal(run, controller.signal); + return await runWithAbortSignal( + run, + controller.signal, + "Remote command aborted.", + ); } finally { clearTimeout(timer); parentSignal?.removeEventListener("abort", onParentAbort); @@ -3562,9 +3515,13 @@ async function buildLaneDetailPayload(args: SyncRemoteCommandServiceArgs, laneId }; } +type RemoteCommandRegistrationPolicy = SyncRemoteCommandPolicy & { + observesAbort?: boolean; +}; + type RemoteCommandRegistrar = ( action: SyncRemoteCommandAction, - policy: SyncRemoteCommandPolicy, + policy: RemoteCommandRegistrationPolicy, handler: ( payload: Record, context: SyncRemoteCommandExecutionContext, @@ -3960,7 +3917,7 @@ function registerProcessRemoteCommands({ args, register }: RemoteCommandRegistra } function registerChatRemoteCommands({ args, register }: RemoteCommandRegistrationDeps): void { - register("chat.resolveSmartLinkPreview", { viewerAllowed: true }, async (payload) => { + register("chat.resolveSmartLinkPreview", { viewerAllowed: true, observesAbort: true }, async (payload) => { const url = requireString(payload.url, "chat.resolveSmartLinkPreview requires url."); const linearIssueTracker = await getConnectedLinearIssueTracker(args); return resolveSmartLinkPreview({ @@ -4110,7 +4067,7 @@ function registerChatRemoteCommands({ args, register }: RemoteCommandRegistratio paused, }); }); - register("chat.getChatEventHistory", { viewerAllowed: true }, async (payload): Promise => { + register("chat.getChatEventHistory", { viewerAllowed: true, observesAbort: true }, async (payload): Promise => { const agentChatService = requireService(args.agentChatService, "Agent chat service not available."); const sessionId = requireString(payload.sessionId, "chat.getChatEventHistory requires sessionId."); const maxEvents = asOptionalNumber(payload.maxEvents); @@ -4124,7 +4081,7 @@ function registerChatRemoteCommands({ args, register }: RemoteCommandRegistratio Object.keys(options).length > 0 ? options : undefined, ); }); - register("chat.getTranscript", { viewerAllowed: true }, async (payload) => { + register("chat.getTranscript", { viewerAllowed: true, observesAbort: true }, async (payload) => { const agentChatService = requireService(args.agentChatService, "Agent chat service not available."); const parsed = parseGetTranscriptArgs(payload); @@ -4157,11 +4114,11 @@ function registerChatRemoteCommands({ args, register }: RemoteCommandRegistratio nextCursor: hasMore ? String(oldestReturnedIndex) : null, }; }); - register("chat.getSubagentTranscript", { viewerAllowed: true, queueable: false }, async (payload) => + register("chat.getSubagentTranscript", { viewerAllowed: true, queueable: false, observesAbort: true }, async (payload) => requireService(args.agentChatService, "Agent chat service not available.").getSubagentTranscript( parseAgentChatSubagentTranscriptArgs(payload), )); - register("chat.getMainTranscript", { viewerAllowed: true, queueable: false }, async (payload) => + register("chat.getMainTranscript", { viewerAllowed: true, queueable: false, observesAbort: true }, async (payload) => requireService(args.agentChatService, "Agent chat service not available.").getMainTranscript( parseAgentChatMainTranscriptArgs(payload), )); @@ -4187,8 +4144,8 @@ function registerChatRemoteCommands({ args, register }: RemoteCommandRegistratio // beyond the hydrated tail). The canonical action mirrors the desktop/TUI // ADE action surface; the legacy agentChat.* name remains for older mobile // clients that learned the first sync-only spelling. - register("chat.getChatEventHistoryPage", { viewerAllowed: true }, getChatEventHistoryPage); - register("agentChat.getEventHistoryPage", { viewerAllowed: true }, getChatEventHistoryPage); + register("chat.getChatEventHistoryPage", { viewerAllowed: true, observesAbort: true }, getChatEventHistoryPage); + register("agentChat.getEventHistoryPage", { viewerAllowed: true, observesAbort: true }, getChatEventHistoryPage); register("chat.create", { viewerAllowed: true, queueable: true }, async (payload) => { const agentChatService = requireService(args.agentChatService, "Agent chat service not available."); const parsed = parseAgentChatCreateArgs(payload); @@ -4780,11 +4737,11 @@ function registerMiscRemoteCommands({ args, register }: RemoteCommandRegistratio )); register("history.listOperations", { viewerAllowed: true }, async (payload) => requireService(args.operationService, "Operation service not available.").list(parseListOperationsArgs(payload))); - register("github.getStatus", { viewerAllowed: true }, async (payload): Promise => + register("github.getStatus", { viewerAllowed: true, observesAbort: true }, async (payload): Promise => requireService(args.githubService, "GitHub service not available.").getStatus({ forceRefresh: payload.forceRefresh === true, })); - register("github.getRemoteStatus", { viewerAllowed: true }, async (): Promise<{ repo: GitHubRepoRef | null; hasOrigin: boolean }> => + register("github.getRemoteStatus", { viewerAllowed: true, observesAbort: true }, async (): Promise<{ repo: GitHubRepoRef | null; hasOrigin: boolean }> => requireService(args.githubService, "GitHub service not available.").getRemoteStatus()); register("github.publishCurrentProject", { viewerAllowed: true }, async (payload): Promise => { const { owner, name, description, isPrivate } = parsePublishCurrentProjectArgs(payload); @@ -4801,7 +4758,7 @@ function registerMiscRemoteCommands({ args, register }: RemoteCommandRegistratio requireService(args.projectConfigService, "Project config service not available.").save( parseProjectConfigSaveArgs(payload).candidate, )); - register("ai.getStatus", { viewerAllowed: true }, async (payload, context) => { + register("ai.getStatus", { viewerAllowed: true, observesAbort: true }, async (payload, context) => { try { return await runAiStatusWithTimeout( () => buildAiSettingsStatus(args.aiIntegrationService, { @@ -4829,9 +4786,9 @@ function registerMiscRemoteCommands({ args, register }: RemoteCommandRegistratio } function registerPrAndDeeplinkRemoteCommands({ args, register }: RemoteCommandRegistrationDeps): void { - register("prs.list", { viewerAllowed: true }, async () => args.prService.listAll()); - register("prs.listOpenForRepo", { viewerAllowed: true }, async () => args.prService.listOpenPullRequests()); - register("prs.getForLane", { viewerAllowed: true }, async (payload) => + register("prs.list", { viewerAllowed: true, observesAbort: true }, async () => args.prService.listAll()); + register("prs.listOpenForRepo", { viewerAllowed: true, observesAbort: true }, async () => args.prService.listOpenPullRequests()); + register("prs.getForLane", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getForLane(parseLaneIdArgs(payload, "prs.getForLane").laneId)); // Manual per-badge ⟳ sync + focus reconcile, so the web/mobile surfaces reach // the same heal path as desktop (both are already in ADE_ACTION_ALLOWLIST.pr). @@ -4839,7 +4796,7 @@ function registerPrAndDeeplinkRemoteCommands({ args, register }: RemoteCommandRe args.prService.syncLanePr(parseLaneIdArgs(payload, "prs.syncLanePr").laneId)); register("prs.reconcileOnFocus", { viewerAllowed: true }, async (payload) => args.prService.reconcileOnFocus({ force: payload?.force === true })); - register("prs.refresh", { viewerAllowed: true }, async (payload) => { + register("prs.refresh", { viewerAllowed: true, observesAbort: true }, async (payload) => { const prId = asTrimmedString(payload.prId); const prIds = asStringArray(payload.prIds); let refreshArgs: { prId?: string; prIds?: string[] } = {}; @@ -4886,16 +4843,16 @@ function registerPrAndDeeplinkRemoteCommands({ args, register }: RemoteCommandRe } return await args.dispatchDeeplinkUrl(url); }); - register("prs.getDetail", { viewerAllowed: true }, async (payload) => args.prService.getDetail(requirePrId(payload, "prs.getDetail"))); + register("prs.getDetail", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getDetail(requirePrId(payload, "prs.getDetail"))); register("prs.postReviewComment", { viewerAllowed: true, queueable: true }, async (payload) => args.prService.postReviewComment(parsePostPrReviewCommentArgs(payload))); - register("prs.getAiSummary", { viewerAllowed: true }, async (payload) => + register("prs.getAiSummary", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prSummaryService?.getSummary(requirePrId(payload, "prs.getAiSummary")) ?? null); register("prs.regenerateAiSummary", { viewerAllowed: true, queueable: true }, async (payload) => requireService(args.prSummaryService, "PR summary service not available.").regenerateSummary( requirePrId(payload, "prs.regenerateAiSummary"), )); - register("prs.getIntegrationResolutionState", { viewerAllowed: true }, async (payload) => + register("prs.getIntegrationResolutionState", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getIntegrationResolutionState( requireString(payload.proposalId, "prs.getIntegrationResolutionState requires proposalId."), )); @@ -4903,29 +4860,29 @@ function registerPrAndDeeplinkRemoteCommands({ args, register }: RemoteCommandRe args.prService.delete(parseDeletePrArgs(payload))); register("prs.cleanupBranch", { viewerAllowed: false, queueable: true }, async (payload) => args.prService.cleanupBranch(parseCleanupPrBranchArgs(payload))); - register("prs.listProposals", { viewerAllowed: true }, async () => args.prService.listIntegrationProposals()); - register("prs.getQueueState", { viewerAllowed: true }, async (payload) => + register("prs.listProposals", { viewerAllowed: true, observesAbort: true }, async () => args.prService.listIntegrationProposals()); + register("prs.getQueueState", { viewerAllowed: true, observesAbort: true }, async (payload) => args.queueLandingService?.getQueueStateByGroup( requireString(payload.groupId, "prs.getQueueState requires groupId."), ) ?? null); - register("prs.listQueueStates", { viewerAllowed: true }, async (payload) => + register("prs.listQueueStates", { viewerAllowed: true, observesAbort: true }, async (payload) => args.queueLandingService?.listQueueStates(payload) ?? []); - register("prs.getMergeContext", { viewerAllowed: true }, async (payload) => + register("prs.getMergeContext", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getMergeContext(requirePrId(payload, "prs.getMergeContext"))); - register("prs.getMergeContexts", { viewerAllowed: true }, async (payload) => + register("prs.getMergeContexts", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getMergeContexts(asStringArray(payload.prIds))); - register("prs.listWithConflicts", { viewerAllowed: true }, async (payload) => + register("prs.listWithConflicts", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.listWithConflicts({ includeConflictAnalysis: payload.includeConflictAnalysis === true })); - register("prs.listSnapshots", { viewerAllowed: true }, async (payload) => + register("prs.listSnapshots", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.listSnapshots({ ...(asTrimmedString(payload.prId) ? { prId: asTrimmedString(payload.prId)! } : {}), })); - register("prs.getStatus", { viewerAllowed: true }, async (payload) => args.prService.getStatus(requirePrId(payload, "prs.getStatus"))); - register("prs.getChecks", { viewerAllowed: true }, async (payload) => args.prService.getChecks(requirePrId(payload, "prs.getChecks"))); - register("prs.getReviews", { viewerAllowed: true }, async (payload) => args.prService.getReviews(requirePrId(payload, "prs.getReviews"))); - register("prs.getComments", { viewerAllowed: true }, async (payload) => args.prService.getComments(requirePrId(payload, "prs.getComments"))); - register("prs.getFiles", { viewerAllowed: true }, async (payload) => args.prService.getFiles(requirePrId(payload, "prs.getFiles"))); - register("prs.getGitHubSnapshot", { viewerAllowed: true }, async (payload) => + register("prs.getStatus", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getStatus(requirePrId(payload, "prs.getStatus"))); + register("prs.getChecks", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getChecks(requirePrId(payload, "prs.getChecks"))); + register("prs.getReviews", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getReviews(requirePrId(payload, "prs.getReviews"))); + register("prs.getComments", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getComments(requirePrId(payload, "prs.getComments"))); + register("prs.getFiles", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getFiles(requirePrId(payload, "prs.getFiles"))); + register("prs.getGitHubSnapshot", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getGithubSnapshot({ force: payload.force === true, includeExternalClosed: payload.includeExternalClosed === true, @@ -4935,30 +4892,30 @@ function registerPrAndDeeplinkRemoteCommands({ args, register }: RemoteCommandRe ? { historyPageLimit: Math.max(1, Math.floor(payload.historyPageLimit)) } : {}), })); - register("prs.getReviewThreads", { viewerAllowed: true }, async (payload) => args.prService.getReviewThreads(requirePrId(payload, "prs.getReviewThreads"))); - register("prs.getActionRuns", { viewerAllowed: true }, async (payload) => args.prService.getActionRuns(requirePrId(payload, "prs.getActionRuns"))); - register("prs.getActivity", { viewerAllowed: true }, async (payload) => args.prService.getActivity(requirePrId(payload, "prs.getActivity"))); - register("prs.getDeployments", { viewerAllowed: true }, async (payload) => args.prService.getDeployments(requirePrId(payload, "prs.getDeployments"))); + register("prs.getReviewThreads", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getReviewThreads(requirePrId(payload, "prs.getReviewThreads"))); + register("prs.getActionRuns", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getActionRuns(requirePrId(payload, "prs.getActionRuns"))); + register("prs.getActivity", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getActivity(requirePrId(payload, "prs.getActivity"))); + register("prs.getDeployments", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getDeployments(requirePrId(payload, "prs.getDeployments"))); // Coordinate-based PR reads for PRs that are not mapped to an ADE lane (no DB // row). The preload sends these `*ByGithub` runtime actions before falling // back to IPC, so the socket runtime must register them alongside the // row-based reads. Args are GitHub coordinates: { repoOwner, repoName, githubPrNumber }. - register("prs.getDetailByGithub", { viewerAllowed: true }, async (payload) => args.prService.getDetailByGithub(requirePrGithubCoords(payload, "prs.getDetailByGithub"))); - register("prs.getFilesByGithub", { viewerAllowed: true }, async (payload) => args.prService.getFilesByGithub(requirePrGithubCoords(payload, "prs.getFilesByGithub"))); - register("prs.getCommitsByGithub", { viewerAllowed: true }, async (payload) => args.prService.getCommitsByGithub(requirePrGithubCoords(payload, "prs.getCommitsByGithub"))); - register("prs.getActionRunsByGithub", { viewerAllowed: true }, async (payload) => args.prService.getActionRunsByGithub(requirePrGithubCoords(payload, "prs.getActionRunsByGithub"))); - register("prs.getActivityByGithub", { viewerAllowed: true }, async (payload) => args.prService.getActivityByGithub(requirePrGithubCoords(payload, "prs.getActivityByGithub"))); - register("prs.getStatusByGithub", { viewerAllowed: true }, async (payload) => args.prService.getStatusByGithub(requirePrGithubCoords(payload, "prs.getStatusByGithub"))); - register("prs.getChecksByGithub", { viewerAllowed: true }, async (payload) => args.prService.getChecksByGithub(requirePrGithubCoords(payload, "prs.getChecksByGithub"))); - register("prs.getReviewsByGithub", { viewerAllowed: true }, async (payload) => args.prService.getReviewsByGithub(requirePrGithubCoords(payload, "prs.getReviewsByGithub"))); - register("prs.getCommentsByGithub", { viewerAllowed: true }, async (payload) => args.prService.getCommentsByGithub(requirePrGithubCoords(payload, "prs.getCommentsByGithub"))); - register("prs.getReviewThreadsByGithub", { viewerAllowed: true }, async (payload) => args.prService.getReviewThreadsByGithub(requirePrGithubCoords(payload, "prs.getReviewThreadsByGithub"))); - register("prs.getMobileGithubDetail", { viewerAllowed: true }, async (payload) => + register("prs.getDetailByGithub", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getDetailByGithub(requirePrGithubCoords(payload, "prs.getDetailByGithub"))); + register("prs.getFilesByGithub", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getFilesByGithub(requirePrGithubCoords(payload, "prs.getFilesByGithub"))); + register("prs.getCommitsByGithub", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getCommitsByGithub(requirePrGithubCoords(payload, "prs.getCommitsByGithub"))); + register("prs.getActionRunsByGithub", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getActionRunsByGithub(requirePrGithubCoords(payload, "prs.getActionRunsByGithub"))); + register("prs.getActivityByGithub", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getActivityByGithub(requirePrGithubCoords(payload, "prs.getActivityByGithub"))); + register("prs.getStatusByGithub", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getStatusByGithub(requirePrGithubCoords(payload, "prs.getStatusByGithub"))); + register("prs.getChecksByGithub", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getChecksByGithub(requirePrGithubCoords(payload, "prs.getChecksByGithub"))); + register("prs.getReviewsByGithub", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getReviewsByGithub(requirePrGithubCoords(payload, "prs.getReviewsByGithub"))); + register("prs.getCommentsByGithub", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getCommentsByGithub(requirePrGithubCoords(payload, "prs.getCommentsByGithub"))); + register("prs.getReviewThreadsByGithub", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getReviewThreadsByGithub(requirePrGithubCoords(payload, "prs.getReviewThreadsByGithub"))); + register("prs.getMobileGithubDetail", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.getMobileGithubDetail(requirePrGithubCoords(payload, "prs.getMobileGithubDetail"))); register("prs.createFromLane", { viewerAllowed: true, queueable: true }, async (payload) => args.prService.createFromLane(parseCreatePrArgs(payload))); register("prs.createQueue", { viewerAllowed: true, queueable: true }, async (payload) => args.prService.createQueuePrs(parseCreateQueuePrsArgs(payload))); register("prs.linkToLane", { viewerAllowed: true, queueable: true }, async (payload) => args.prService.linkToLane(parseLinkPrToLaneArgs(payload))); - register("prs.preflightCreateLaneFromPrBranch", { viewerAllowed: true }, async (payload) => args.prService.preflightCreateLaneFromPrBranch(parseCreateLaneFromPrBranchArgs(payload))); + register("prs.preflightCreateLaneFromPrBranch", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.preflightCreateLaneFromPrBranch(parseCreateLaneFromPrBranchArgs(payload))); register("prs.createLaneFromPrBranch", { viewerAllowed: true, queueable: true }, async (payload) => args.prService.createLaneFromPrBranch(parseCreateLaneFromPrBranchArgs(payload))); register("prs.draftDescription", { viewerAllowed: true, queueable: true }, async (payload) => args.prService.draftDescription(parseDraftPrDescriptionArgs(payload))); @@ -5012,7 +4969,7 @@ function registerPrAndDeeplinkRemoteCommands({ args, register }: RemoteCommandRe args.prService.simulateIntegration(parseSimulateIntegrationArgs(payload))); register("prs.commitIntegration", { viewerAllowed: true, queueable: true }, async (payload) => args.prService.commitIntegration(parseCommitIntegrationArgs(payload))); - register("prs.listIntegrationWorkflows", { viewerAllowed: true }, async (payload) => + register("prs.listIntegrationWorkflows", { viewerAllowed: true, observesAbort: true }, async (payload) => args.prService.listIntegrationWorkflows(parseListIntegrationWorkflowsArgs(payload))); register("prs.updateIntegrationProposal", { viewerAllowed: true, queueable: true }, async (payload) => { args.prService.updateIntegrationProposal(parseUpdateIntegrationProposalArgs(payload)); @@ -5060,7 +5017,7 @@ function registerPrAndDeeplinkRemoteCommands({ args, register }: RemoteCommandRe await args.prService.reorderQueuePrs(parseReorderQueuePrsArgs(payload)); return { ok: true }; }); - register("prs.getMobileSnapshot", { viewerAllowed: true }, async () => args.prService.getMobileSnapshot()); + register("prs.getMobileSnapshot", { viewerAllowed: true, observesAbort: true }, async () => args.prService.getMobileSnapshot()); } export function createSyncRemoteCommandService(args: SyncRemoteCommandServiceArgs) { @@ -5068,15 +5025,17 @@ export function createSyncRemoteCommandService(args: SyncRemoteCommandServiceArg const register = ( action: SyncRemoteCommandAction, - policy: SyncRemoteCommandPolicy, + policy: RemoteCommandRegistrationPolicy, handler: ( payload: Record, context: SyncRemoteCommandExecutionContext, ) => Promise, scope: SyncRemoteCommandDescriptor["scope"] = "project", ) => { + const { observesAbort = false, ...descriptorPolicy } = policy; registry.set(action, { - descriptor: { action, scope, policy }, + descriptor: { action, scope, policy: descriptorPolicy }, + observesAbort, handler, }); }; @@ -5183,6 +5142,12 @@ export function createSyncRemoteCommandService(args: SyncRemoteCommandServiceArg return [...registry.values()].map((entry) => entry.descriptor); }, + getAbortObservingActions(): SyncRemoteCommandAction[] { + return [...registry.values()] + .filter((entry) => entry.observesAbort) + .map((entry) => entry.descriptor.action as SyncRemoteCommandAction); + }, + getPolicy(action: string): SyncRemoteCommandPolicy | null { return registry.get(action as SyncRemoteCommandAction)?.descriptor.policy ?? null; }, @@ -5206,8 +5171,12 @@ export function createSyncRemoteCommandService(args: SyncRemoteCommandServiceArg policy: handler.descriptor.policy, }); const run = () => handler.handler(commandArgs, context); - return commandHandlerObservesAbort(payload.action) - ? await runWithRemoteCommandSignal(run, context.signal) + return handler.observesAbort + ? await runWithAbortSignal( + run, + context.signal, + "Remote command aborted.", + ) : await run(); }, }; diff --git a/apps/desktop/src/main/services/ipc/registerIpc.ts b/apps/desktop/src/main/services/ipc/registerIpc.ts index 1b1f972ab..02392a995 100644 --- a/apps/desktop/src/main/services/ipc/registerIpc.ts +++ b/apps/desktop/src/main/services/ipc/registerIpc.ts @@ -1,11 +1,13 @@ import { app, BrowserWindow, clipboard, desktopCapturer, dialog, ipcMain, nativeImage, shell, systemPreferences } from "electron"; import type { IpcMainInvokeEvent } from "electron"; import { - buildGithubReleaseUrl, - compareUpdateVersions, createEmptyAutoUpdateSnapshot, type createAutoUpdateService, } from "../updates/autoUpdateService"; +import { + buildGithubReleaseUrl, + compareUpdateVersions, +} from "../updates/autoUpdateVersions"; import { spawn } from "node:child_process"; import { randomUUID } from "node:crypto"; import fs from "node:fs"; diff --git a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts index a16bad7ee..6b42d03fb 100644 --- a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts +++ b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts @@ -6,7 +6,11 @@ import os from "node:os"; import path from "node:path"; import { app } from "electron"; import { isAdeRuntimeNamedPipePath } from "../../../shared/adeRuntimeIpc"; -import { isRuntimeProtocolCompatible } from "../../../shared/adeRuntimeProtocol"; +import { + isRuntimeProtocolCompatible, + parseRuntimeLastWedge, + parseRuntimePublishHealth, +} from "../../../shared/adeRuntimeProtocol"; import type { RemoteRuntimeActionRequest, RemoteRuntimeActionResult, @@ -466,22 +470,7 @@ export function readLocalRuntimeInfo(value: unknown): { const defaultRole = info.defaultRole; const pid = info.pid; const syncPort = info.syncPort; - const rawPublishHealth = info.publishHealth; - const publishHealthRecord = rawPublishHealth && typeof rawPublishHealth === "object" && !Array.isArray(rawPublishHealth) - ? rawPublishHealth as Record - : null; - const rawLegDurations = publishHealthRecord?.lastLegDurations; - const legDurationsRecord = rawLegDurations && typeof rawLegDurations === "object" && !Array.isArray(rawLegDurations) - ? rawLegDurations as Record - : null; - const publishState = publishHealthRecord?.state; - const rawLastWedge = info.lastWedge; - const lastWedgeRecord = rawLastWedge && typeof rawLastWedge === "object" && !Array.isArray(rawLastWedge) - ? rawLastWedge as Record - : null; - const lastWedgeCommand = lastWedgeRecord?.lastCommand; - const lastWedgeBlockedMs = lastWedgeRecord?.blockedMs; - const lastWedgeTs = lastWedgeRecord?.ts; + const parsedPublishHealth = parseRuntimePublishHealth(info.publishHealth); const minCompatibleProtocol = info.minCompatibleProtocol; const protocolVersion = info.protocolVersion; return { @@ -497,34 +486,14 @@ export function readLocalRuntimeInfo(value: unknown): { ? syncPort : null, publishHealth: - typeof publishState === "string" && publishState.trim() && legDurationsRecord - ? { - state: publishState as NonNullable["state"], - failingSinceMs: - typeof publishHealthRecord?.failingSinceMs === "number" - && Number.isFinite(publishHealthRecord.failingSinceMs) - ? Math.max(0, publishHealthRecord.failingSinceMs) - : null, - lastLegDurations: { - snapshot: finiteDuration(legDurationsRecord.snapshot), - token: finiteDuration(legDurationsRecord.token), - http: finiteDuration(legDurationsRecord.http), - }, - } - : null, - lastWedge: - typeof lastWedgeCommand === "string" - && lastWedgeCommand.trim() - && typeof lastWedgeBlockedMs === "number" - && Number.isFinite(lastWedgeBlockedMs) - && typeof lastWedgeTs === "string" - && lastWedgeTs.trim() + parsedPublishHealth ? { - lastCommand: lastWedgeCommand.trim(), - blockedMs: Math.max(0, Math.floor(lastWedgeBlockedMs)), - ts: lastWedgeTs.trim(), + state: parsedPublishHealth.state, + failingSinceMs: parsedPublishHealth.failingSinceMs, + lastLegDurations: parsedPublishHealth.lastLegDurations, } : null, + lastWedge: parseRuntimeLastWedge(info.lastWedge), minCompatibleProtocol: typeof minCompatibleProtocol === "number" && Number.isInteger(minCompatibleProtocol) @@ -540,12 +509,6 @@ export function readLocalRuntimeInfo(value: unknown): { }; } -function finiteDuration(value: unknown): number | null { - return typeof value === "number" && Number.isFinite(value) && value >= 0 - ? Math.round(value) - : null; -} - export function computeLocalRuntimeBuildHash(cliPath = resolveCliScriptPath()): string | null { try { const content = fs.readFileSync(cliPath); @@ -2202,9 +2165,6 @@ export class LocalRuntimeConnectionPool { this.activeRuntimeSyncPort = null; this.activeRuntimePublishHealth = null; this.activeRuntimeLastWedge = null; - this.activeRuntimeSyncPort = null; - this.activeRuntimePublishHealth = null; - this.activeRuntimeLastWedge = null; this.projectsByRoot.clear(); }); return client; diff --git a/apps/desktop/src/main/services/updates/autoUpdateService.test.ts b/apps/desktop/src/main/services/updates/autoUpdateService.test.ts index a42c40af9..c289df06c 100644 --- a/apps/desktop/src/main/services/updates/autoUpdateService.test.ts +++ b/apps/desktop/src/main/services/updates/autoUpdateService.test.ts @@ -3,7 +3,12 @@ import fs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { buildGithubReleaseUrl, buildReleaseNotesUrl, compareUpdateVersions, createAutoUpdateService } from "./autoUpdateService"; +import { createAutoUpdateService } from "./autoUpdateService"; +import { + buildGithubReleaseUrl, + buildReleaseNotesUrl, + compareUpdateVersions, +} from "./autoUpdateVersions"; import { classifyUpdateError, estimateUpdateRequiredBytes } from "./autoUpdateErrors"; import type { Logger } from "../logging/logger"; diff --git a/apps/desktop/src/main/services/updates/autoUpdateService.ts b/apps/desktop/src/main/services/updates/autoUpdateService.ts index e1e02f464..a911881d0 100644 --- a/apps/desktop/src/main/services/updates/autoUpdateService.ts +++ b/apps/desktop/src/main/services/updates/autoUpdateService.ts @@ -21,8 +21,13 @@ import { updateDownloadBytes, type DiskSpaceInfo, } from "./autoUpdateErrors"; +import { + buildGithubReleaseUrl, + buildReleaseNotesUrl, + compareUpdateVersions, + DEFAULT_RELEASE_NOTES_BASE_URL, +} from "./autoUpdateVersions"; -const DEFAULT_RELEASE_NOTES_BASE_URL = "https://www.ade-app.dev"; const DEFAULT_INSTALL_WATCHDOG_MS = 30_000; const DEFAULT_QUIT_DEADLINE_MS = 10_000; const DEFAULT_AUTO_APPLY_IDLE_MS = 2 * 60_000; @@ -111,57 +116,6 @@ export function createEmptyAutoUpdateSnapshot(currentVersion = ""): AutoUpdateSn }; } -function parseVersion(version: string): { - core: number[]; - prerelease: string[]; -} { - const withoutBuild = version.trim().replace(/^v/i, "").split("+")[0] ?? ""; - const [coreText = "", prereleaseText = ""] = withoutBuild.split("-", 2); - return { - core: coreText.split(".").map((part) => { - const parsed = Number.parseInt(part, 10); - return Number.isFinite(parsed) ? parsed : 0; - }), - prerelease: prereleaseText ? prereleaseText.split(".") : [], - }; -} - -function comparePrereleaseIdentifier(left: string, right: string): number { - const leftNumeric = /^\d+$/.test(left); - const rightNumeric = /^\d+$/.test(right); - if (leftNumeric && rightNumeric) { - return Number(left) - Number(right); - } - if (leftNumeric !== rightNumeric) { - return leftNumeric ? -1 : 1; - } - return left.localeCompare(right); -} - -export function compareUpdateVersions(left: string, right: string): number { - const leftVersion = parseVersion(left); - const rightVersion = parseVersion(right); - const coreLength = Math.max(leftVersion.core.length, rightVersion.core.length, 3); - for (let index = 0; index < coreLength; index += 1) { - const delta = (leftVersion.core[index] ?? 0) - (rightVersion.core[index] ?? 0); - if (delta !== 0) return delta; - } - - if (leftVersion.prerelease.length === 0 && rightVersion.prerelease.length > 0) return 1; - if (leftVersion.prerelease.length > 0 && rightVersion.prerelease.length === 0) return -1; - const prereleaseLength = Math.max(leftVersion.prerelease.length, rightVersion.prerelease.length); - for (let index = 0; index < prereleaseLength; index += 1) { - const leftPart = leftVersion.prerelease[index]; - const rightPart = rightVersion.prerelease[index]; - if (leftPart == null && rightPart == null) return 0; - if (leftPart == null) return -1; - if (rightPart == null) return 1; - const delta = comparePrereleaseIdentifier(leftPart, rightPart); - if (delta !== 0) return delta; - } - return 0; -} - function isUpdateCheckResultLike(result: unknown): result is UpdateCheckResultLike { return Boolean( result @@ -178,25 +132,6 @@ function extractDownloadPromise(result: unknown): Promise | null { : null; } -export function buildReleaseNotesUrl( - version: string, - baseUrl = DEFAULT_RELEASE_NOTES_BASE_URL, -): string | null { - const normalizedVersion = version.trim().replace(/^v/i, ""); - const normalizedBaseUrl = baseUrl.trim().replace(/\/+$/, ""); - if (!normalizedVersion || !normalizedBaseUrl) return null; - return `${normalizedBaseUrl}/docs/changelog/${encodeURIComponent(`v${normalizedVersion}`)}`; -} - -// Deterministic GitHub release page for a version tag, e.g. -// https://github.com/arul28/ADE/releases/tag/v1.2.18 — the same repo the -// updater feed points at above. -export function buildGithubReleaseUrl(version: string): string | null { - const normalizedVersion = version.trim().replace(/^v/i, ""); - if (!normalizedVersion) return null; - return `https://github.com/arul28/ADE/releases/tag/${encodeURIComponent(`v${normalizedVersion}`)}`; -} - function cloneRecentlyInstalledUpdate( update: RecentlyInstalledUpdate | null, ): RecentlyInstalledUpdate | null { diff --git a/apps/desktop/src/main/services/updates/autoUpdateVersions.ts b/apps/desktop/src/main/services/updates/autoUpdateVersions.ts new file mode 100644 index 000000000..2df9b95f4 --- /dev/null +++ b/apps/desktop/src/main/services/updates/autoUpdateVersions.ts @@ -0,0 +1,71 @@ +export const DEFAULT_RELEASE_NOTES_BASE_URL = "https://www.ade-app.dev"; + +function parseVersion(version: string): { + core: number[]; + prerelease: string[]; +} { + const withoutBuild = version.trim().replace(/^v/i, "").split("+")[0] ?? ""; + const [coreText = "", prereleaseText = ""] = withoutBuild.split("-", 2); + return { + core: coreText.split(".").map((part) => { + const parsed = Number.parseInt(part, 10); + return Number.isFinite(parsed) ? parsed : 0; + }), + prerelease: prereleaseText ? prereleaseText.split(".") : [], + }; +} + +function comparePrereleaseIdentifier(left: string, right: string): number { + const leftNumeric = /^\d+$/.test(left); + const rightNumeric = /^\d+$/.test(right); + if (leftNumeric && rightNumeric) { + return Number(left) - Number(right); + } + if (leftNumeric !== rightNumeric) { + return leftNumeric ? -1 : 1; + } + return left.localeCompare(right); +} + +export function compareUpdateVersions(left: string, right: string): number { + const leftVersion = parseVersion(left); + const rightVersion = parseVersion(right); + const coreLength = Math.max(leftVersion.core.length, rightVersion.core.length, 3); + for (let index = 0; index < coreLength; index += 1) { + const delta = (leftVersion.core[index] ?? 0) - (rightVersion.core[index] ?? 0); + if (delta !== 0) return delta; + } + + if (leftVersion.prerelease.length === 0 && rightVersion.prerelease.length > 0) return 1; + if (leftVersion.prerelease.length > 0 && rightVersion.prerelease.length === 0) return -1; + const prereleaseLength = Math.max(leftVersion.prerelease.length, rightVersion.prerelease.length); + for (let index = 0; index < prereleaseLength; index += 1) { + const leftPart = leftVersion.prerelease[index]; + const rightPart = rightVersion.prerelease[index]; + if (leftPart == null && rightPart == null) return 0; + if (leftPart == null) return -1; + if (rightPart == null) return 1; + const delta = comparePrereleaseIdentifier(leftPart, rightPart); + if (delta !== 0) return delta; + } + return 0; +} + +export function buildReleaseNotesUrl( + version: string, + baseUrl = DEFAULT_RELEASE_NOTES_BASE_URL, +): string | null { + const normalizedVersion = version.trim().replace(/^v/i, ""); + const normalizedBaseUrl = baseUrl.trim().replace(/\/+$/, ""); + if (!normalizedVersion || !normalizedBaseUrl) return null; + return `${normalizedBaseUrl}/docs/changelog/${encodeURIComponent(`v${normalizedVersion}`)}`; +} + +// Deterministic GitHub release page for a version tag, e.g. +// https://github.com/arul28/ADE/releases/tag/v1.2.18 — the same repo the +// updater feed targets. +export function buildGithubReleaseUrl(version: string): string | null { + const normalizedVersion = version.trim().replace(/^v/i, ""); + if (!normalizedVersion) return null; + return `https://github.com/arul28/ADE/releases/tag/${encodeURIComponent(`v${normalizedVersion}`)}`; +} diff --git a/apps/desktop/src/renderer/components/settings/AboutSection.tsx b/apps/desktop/src/renderer/components/settings/AboutSection.tsx index e44a25fae..cbc2e2c95 100644 --- a/apps/desktop/src/renderer/components/settings/AboutSection.tsx +++ b/apps/desktop/src/renderer/components/settings/AboutSection.tsx @@ -246,7 +246,7 @@ export function AboutSection({ embedded = false }: { embedded?: boolean } = {}) {installedVersion} · Latest {latestVersion} - {releasedAgo && latestVersion === (latest?.version ?? latestVersion) ? ` · ${releasedAgo}` : ""} + {releasedAgo && latest != null && latestVersion === latest.version ? ` · ${releasedAgo}` : ""}
diff --git a/apps/desktop/src/shared/adeRuntimeProtocol.ts b/apps/desktop/src/shared/adeRuntimeProtocol.ts index 1af1eabde..647cd3b1d 100644 --- a/apps/desktop/src/shared/adeRuntimeProtocol.ts +++ b/apps/desktop/src/shared/adeRuntimeProtocol.ts @@ -1,3 +1,8 @@ +import type { + SyncAccountDirectoryLegDurations, + SyncAccountDirectoryState, +} from "./types/sync"; + /** * Monotonically bumpable compatibility level for the local ADE runtime RPC. * @@ -6,6 +11,76 @@ */ export const RUNTIME_COMPAT_LEVEL = 1; +export type RuntimePublishHealth = { + state: SyncAccountDirectoryState; + failingSinceMs: number | null; + lastLegDurations: SyncAccountDirectoryLegDurations; + lastSuccessAt: number | null; + skipReason: string | null; +}; + +export type RuntimeLastWedge = { + lastCommand: string; + blockedMs: number; + ts: string; +}; + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function finiteDuration(value: unknown): number | null { + return typeof value === "number" && Number.isFinite(value) && value >= 0 + ? Math.round(value) + : null; +} + +function asTrimmedString(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +export function parseRuntimePublishHealth(raw: unknown): RuntimePublishHealth | null { + if (!isRecord(raw) || !isRecord(raw.lastLegDurations)) return null; + const state = asTrimmedString(raw.state); + if (!state) return null; + return { + state: raw.state as SyncAccountDirectoryState, + failingSinceMs: + typeof raw.failingSinceMs === "number" && Number.isFinite(raw.failingSinceMs) + ? Math.max(0, raw.failingSinceMs) + : null, + lastLegDurations: { + snapshot: finiteDuration(raw.lastLegDurations.snapshot), + token: finiteDuration(raw.lastLegDurations.token), + http: finiteDuration(raw.lastLegDurations.http), + }, + lastSuccessAt: + typeof raw.lastSuccessAt === "number" && Number.isFinite(raw.lastSuccessAt) + ? Math.max(0, raw.lastSuccessAt) + : null, + skipReason: asTrimmedString(raw.skipReason), + }; +} + +export function parseRuntimeLastWedge(raw: unknown): RuntimeLastWedge | null { + if (!isRecord(raw)) return null; + const lastCommand = asTrimmedString(raw.lastCommand); + const ts = asTrimmedString(raw.ts); + if ( + !lastCommand + || typeof raw.blockedMs !== "number" + || !Number.isFinite(raw.blockedMs) + || !ts + ) { + return null; + } + return { + lastCommand, + blockedMs: Math.max(0, Math.floor(raw.blockedMs)), + ts, + }; +} + export function isRuntimeProtocolCompatible(runtimeInfo: { minCompatibleProtocol: number | null; protocolVersion: number | null; From afe480a36f92f3af370c7c135c41e3414e13934f Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:00:50 -0400 Subject: [PATCH 4/7] test: analytics gate, docs parity, README doctor rewrite, test consolidation Documents the six reliability/update PostHog events (taxonomy + bounded volumes), adds the Reliability-incidents insight to the dashboard spec (validated, provisioner tests green), updates six internal docs for the reliability release, rewrites the stale README doctor section, and merges the two remoteMachineModel test files into one. Co-Authored-By: Claude Fable 5 --- apps/ade-cli/README.md | 29 ++++---- .../remoteMachineModel.publishHealth.test.ts | 67 ------------------ ...unt.test.ts => remoteMachineModel.test.ts} | 63 +++++++++++++++++ docs/ARCHITECTURE.md | 21 +++++- .../onboarding-and-settings/README.md | 68 +++++++++++++++++-- .../desktop-auto-update.md | 50 ++++++++++++++ docs/features/remote-runtime/README.md | 23 ++++++- docs/features/storage-and-recovery/README.md | 37 ++++++++++ docs/features/sync-and-multi-device/README.md | 56 ++++++++++++--- docs/logging.md | 10 ++- scripts/posthog/dashboard-spec.mjs | 24 ++++++- scripts/posthog/provision.test.mjs | 14 ++-- 12 files changed, 352 insertions(+), 110 deletions(-) delete mode 100644 apps/desktop/src/renderer/components/remoteTargets/remoteMachineModel.publishHealth.test.ts rename apps/desktop/src/renderer/components/remoteTargets/{remoteMachineModel.account.test.ts => remoteMachineModel.test.ts} (81%) diff --git a/apps/ade-cli/README.md b/apps/ade-cli/README.md index 4552e67ed..57992c919 100644 --- a/apps/ade-cli/README.md +++ b/apps/ade-cli/README.md @@ -340,6 +340,7 @@ ade machines list --text ade machines connect --project ADE ade machines hop --session chat-1 ade doctor --json +ade doctor --online --text # also check the latest desktop release over the network ade projects list --text ade projects inspect /path/to/checkout --json # classify a path (repo root vs linked/ADE-managed worktree) and find its owning project + existing lane ade init @@ -516,19 +517,21 @@ Provider credentials, GitHub tokens, Linear tokens, and computer-use policy remain separate and are read from ADE project settings and their existing secure stores. -`ade doctor` reports local-only readiness metadata by default: - -- CLI version, Node/runtime version, project root, workspace root, `.ade` initialization, and config file presence. -- Machine endpoint path, whether the endpoint exists, and whether this invocation is using an attached runtime, desktop bridge, or headless mode. -- RPC tool count, ADE action count, and action counts by domain. -- Git repository readiness and GitHub readiness signals from local remotes, `gh` availability, and token environment presence. -- Linear readiness from the active project's `.ade/secrets` credential store (`linear.token.v1`), a legacy project-scoped encrypted token file, or headless environment variables. -- Provider/model readiness from local ADE config, API-key provider references, and provider CLI availability. -- Computer-use readiness from local platform capabilities. -- Sync and Relay readiness, including a relay end-to-end self-probe verdict (`relay self-probe`). When a local ADE brain is running the probe performs one round-trip through ADE Relay and reports `ready`, a `FAILED:` verdict, or a graceful skip; without a running brain (or without a validated relay bridge) it reports skipped/unavailable and never fails the run. `ade sync status` surfaces the same verdict on its `relay end-to-end` line. -- Packaged/PATH status for the `ade` binary and concrete next actions. - -Default doctor / auth checks do not call provider, GitHub, or Linear networks. They report presence and local readiness only, without printing secret values. The one network touch is the optional relay end-to-end self-probe above, which runs only when a local brain and validated relay bridge are present. +`ade doctor` inspects the installed app and machine-brain health and prints one +status row (`ok` / `warn` / `fail`) per check. It exits non-zero when any row is +`fail`. The rows are: + +- **App** — the installed ADE desktop version (read from the `.app` bundle on disk) against the latest known version. Latest-known comes from the on-disk `update-status.json` by default; pass `--online` to also fetch the latest release from GitHub (short timeout, best-effort). `warn` when the install is behind or missing. +- **Brain** — whether the machine brain responds on its socket, plus its version, pid, and uptime. `fail` when it is not responding or when its build identity does not match the expected runtime for this CLI/role. +- **Wedge history** — the last brain-loop watchdog wedge that was recovered (blocking command and how long it blocked), read from the runtime dir or the brain's reported `lastWedge`. `warn` when the most recent wedge is within the last 24h. +- **Sync port** — the sync host port the brain bound. `ok` on the default port, `warn` when bound elsewhere (with the base-port holders it found), `fail` when the brain is up but reported no port. +- **Publish health** — account-directory publish state from the brain's sync route health. `ok` when a publish succeeded recently, `fail` when it has been failing for ≥2 min, otherwise `warn`, with the slowest publish leg annotated. +- **Relay** — relay route health as already computed by the brain. `ok` when the relay control is connected, the bridge is validated, and the end-to-end round-trip is verified; `fail` when the route is not fully validated; `warn` when relay is disabled or route health is unavailable. +- **Account** — whether this machine's brain is signed in to an ADE account (and the credential source), read via the brain's `account.call status`. `warn` when signed out or unavailable. + +Default doctor does not call provider, GitHub, or Linear networks — it talks only +to the local brain over its socket, and never prints secret values. The one +optional network touch is the `--online` desktop-release lookup above. Agents starting an unfamiliar ADE session should begin with: diff --git a/apps/desktop/src/renderer/components/remoteTargets/remoteMachineModel.publishHealth.test.ts b/apps/desktop/src/renderer/components/remoteTargets/remoteMachineModel.publishHealth.test.ts deleted file mode 100644 index 2413545b1..000000000 --- a/apps/desktop/src/renderer/components/remoteTargets/remoteMachineModel.publishHealth.test.ts +++ /dev/null @@ -1,67 +0,0 @@ -import { describe, expect, it } from "vitest"; -import { - PUBLISH_FAILING_ALARM_MS, - describePublishHealth, - type LocalPublishHealth, -} from "./remoteMachineModel"; - -const NOW = 1_700_000_000_000; - -function health(overrides: Partial): LocalPublishHealth { - return { state: "published", failingSinceMs: null, ...overrides }; -} - -describe("describePublishHealth", () => { - it("reports none when there is no publish health", () => { - expect(describePublishHealth(null, NOW)).toEqual({ kind: "none" }); - expect(describePublishHealth(undefined, NOW)).toEqual({ kind: "none" }); - }); - - it("reports healthy when routes are published", () => { - expect(describePublishHealth(health({ state: "published" }), NOW)).toEqual({ - kind: "healthy", - }); - }); - - it("stays silent for non-publishing states", () => { - for (const state of [ - "sync_disabled", - "no_active_sync_scope", - "not_host", - "account_signed_out", - "machine_key_unavailable", - "missing_pairing_connect_info", - ] as const) { - expect( - describePublishHealth(health({ state, failingSinceMs: NOW - 60 * 60_000 }), NOW), - ).toEqual({ kind: "none" }); - } - }); - - it("does not alarm on a failure without a start time", () => { - expect( - describePublishHealth(health({ state: "http_error", failingSinceMs: null }), NOW), - ).toEqual({ kind: "none" }); - }); - - it("does not alarm before the failure has persisted two minutes", () => { - const failingSinceMs = NOW - (PUBLISH_FAILING_ALARM_MS - 1); - expect( - describePublishHealth(health({ state: "timeout", failingSinceMs }), NOW), - ).toEqual({ kind: "none" }); - }); - - it("alarms once a failure has persisted at least two minutes", () => { - const failingSinceMs = NOW - 12 * 60_000; - expect( - describePublishHealth(health({ state: "transport_error", failingSinceMs }), NOW), - ).toEqual({ kind: "failing", minutes: 12 }); - }); - - it("floors the failing minutes", () => { - const failingSinceMs = NOW - (5 * 60_000 + 59_000); - expect( - describePublishHealth(health({ state: "http_timeout", failingSinceMs }), NOW), - ).toEqual({ kind: "failing", minutes: 5 }); - }); -}); diff --git a/apps/desktop/src/renderer/components/remoteTargets/remoteMachineModel.account.test.ts b/apps/desktop/src/renderer/components/remoteTargets/remoteMachineModel.test.ts similarity index 81% rename from apps/desktop/src/renderer/components/remoteTargets/remoteMachineModel.account.test.ts rename to apps/desktop/src/renderer/components/remoteTargets/remoteMachineModel.test.ts index ac717554b..f0fdee39c 100644 --- a/apps/desktop/src/renderer/components/remoteTargets/remoteMachineModel.account.test.ts +++ b/apps/desktop/src/renderer/components/remoteTargets/remoteMachineModel.test.ts @@ -7,8 +7,11 @@ import type { } from "../../../shared/types"; import { DEFAULT_ADE_TUNNEL_RELAY_URL } from "../../../shared/accountDirectory"; import { + PUBLISH_FAILING_ALARM_MS, accountMachineMatchesTarget, assignMachineSections, + describePublishHealth, + type LocalPublishHealth, } from "./remoteMachineModel"; function accountMachine(overrides: Partial = {}): AdeAccountMachine { @@ -300,3 +303,63 @@ describe("assignMachineSections — account machines", () => { expect(sections.unavailable).toHaveLength(0); }); }); +const NOW = 1_700_000_000_000; + +function health(overrides: Partial): LocalPublishHealth { + return { state: "published", failingSinceMs: null, ...overrides }; +} + +describe("describePublishHealth", () => { + it("reports none when there is no publish health", () => { + expect(describePublishHealth(null, NOW)).toEqual({ kind: "none" }); + expect(describePublishHealth(undefined, NOW)).toEqual({ kind: "none" }); + }); + + it("reports healthy when routes are published", () => { + expect(describePublishHealth(health({ state: "published" }), NOW)).toEqual({ + kind: "healthy", + }); + }); + + it("stays silent for non-publishing states", () => { + for (const state of [ + "sync_disabled", + "no_active_sync_scope", + "not_host", + "account_signed_out", + "machine_key_unavailable", + "missing_pairing_connect_info", + ] as const) { + expect( + describePublishHealth(health({ state, failingSinceMs: NOW - 60 * 60_000 }), NOW), + ).toEqual({ kind: "none" }); + } + }); + + it("does not alarm on a failure without a start time", () => { + expect( + describePublishHealth(health({ state: "http_error", failingSinceMs: null }), NOW), + ).toEqual({ kind: "none" }); + }); + + it("does not alarm before the failure has persisted two minutes", () => { + const failingSinceMs = NOW - (PUBLISH_FAILING_ALARM_MS - 1); + expect( + describePublishHealth(health({ state: "timeout", failingSinceMs }), NOW), + ).toEqual({ kind: "none" }); + }); + + it("alarms once a failure has persisted at least two minutes", () => { + const failingSinceMs = NOW - 12 * 60_000; + expect( + describePublishHealth(health({ state: "transport_error", failingSinceMs }), NOW), + ).toEqual({ kind: "failing", minutes: 12 }); + }); + + it("floors the failing minutes", () => { + const failingSinceMs = NOW - (5 * 60_000 + 59_000); + expect( + describePublishHealth(health({ state: "http_timeout", failingSinceMs }), NOW), + ).toEqual({ kind: "failing", minutes: 5 }); + }); +}); diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index b8822f139..4ff649704 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -143,7 +143,22 @@ also stats its own CLI entrypoint every five minutes (configurable with `ADE_BRAIN_FRESHNESS_INTERVAL_MS`, disabled by `ADE_DISABLE_BRAIN_FRESHNESS=1`), hashes only after the stat changes, and uses the brain-update service restart path after an idle grace period when the disk -hash no longer matches its baked runtime hash. +hash no longer matches its baked runtime hash (`brainFreshnessMonitor.ts`). + +**Event-loop watchdog.** `apps/ade-cli/src/services/runtime/brainLoopWatchdog.ts` +runs a small unref'd worker thread that the main thread heartbeats every second +with the name of the currently running command +(`trackBrainLoopWatchdogCommand`). If the heartbeat stalls past the threshold +(`ADE_LOOP_WATCHDOG_MS`, default 15 s) without a matching sleep/suspend signal, +the worker atomically writes an `event-loop-wedge.json` breadcrumb (the wedged +command, blocked duration, timestamp) under the runtime dir and `SIGKILL`s the +brain so launchd can restart it. On the next boot the watchdog promotes any +breadcrumb to `last-wedge.json`, logs `brain.recovered_from_wedge`, and emits a +deduped `ade_brain_recovered` analytics event; the recovered wedge is surfaced +to clients through `runtimeInfo.lastWedge` (and the desktop `BrainRecoveryNotice` +banner). Disabled with `ADE_DISABLE_LOOP_WATCHDOG=1` and off under vitest unless +forced. The brain's own log stream is written by +`apps/ade-cli/src/services/runtime/brainLogger.ts` (see §15.1). **Session identity.** The runtime resolves caller role from ADE context env vars and command flags. Role vocabulary: `cto`, `orchestrator`, `agent`, `external`, `evaluator`. `ADE_DEFAULT_ROLE` is an authority ceiling, not an identity grant: `resolveSessionBoundRole` clamps a chat-bound caller that would otherwise inherit a daemon-wide `cto` role to `agent`, preserves an explicitly declared `orchestrator`, and never accepts a requested role above the runtime ceiling. SDK-backed chats receive `ADE_CHAT_SESSION_ID` plus `ADE_DEFAULT_ROLE=agent` (or `orchestrator` for a lead), and tracked provider CLI launch/resume does the same. Persistent SDK guidance names the concrete `--session ` for lifecycle commands so shared provider servers do not depend on process-global env inheritance. Browser automation adds a separate bearer capability: ADE-launched chat and owned-terminal environments receive an opaque `ADE_BROWSER_ACTOR_TOKEN` bound in Electron memory to that chat's trusted lane/project or personal tab collection. The runtime requires the token, strips caller-supplied routing, and carries it only over the authenticated desktop bridge. Electron validates it in the same process that issued it before restoring the bound scope; role alone never grants access to a human-authenticated browser profile. @@ -169,6 +184,8 @@ ade brain update status --text Use `ADE_VERSION=vX.Y.Z` for a pinned release or `ADE_INSTALL_DIR` to choose the destination directory. The installer defaults to `$ADE_HOME/bin/ade`; both install and `ade brain update` verify downloaded runtime assets against `SHA256SUMS`. `ade brain update` stages the next release under `$ADE_HOME/runtime/updates/`, verifies the staged binary against the staged native deps, promotes the binary/deps into place, and restarts the per-user brain service. +**Health check (`ade doctor [--online] [--text]`).** `apps/ade-cli/src/commands/doctor.ts` connects to the machine brain over the local socket (bounded ~2 s) and prints one status row (`ok` / `warn` / `fail`) per subsystem: **App** (installed desktop version from the `.app` `Info.plist` vs the latest known version — read from disk, or from GitHub with `--online`), **Brain** (running version/pid/uptime plus any build-hash or role mismatch), **Wedge history** (the most recent recovered event-loop wedge, if any), **Sync port** (whether the shared listener bound the default `8787`, and the holders of the base ports when it drifted), **Publish health** (the account-directory publisher's last-leg durations and slowest leg), **Relay** (end-to-end verified vs a classified failure), and **Account** (signed-in state and source). The command exits non-zero when any row is `fail`. The row-evaluation logic (`evaluateDoctorRows`) is pure and dependency-injected so the desktop connection-doctor card and the CLI share one verdict. + **Install + PATH wiring (when the desktop ships `ade`).** On macOS / Linux the desktop installer drops the launcher at `$HOME/.local/bin/ade`; on Windows it lands at `%LOCALAPPDATA%\ADE\bin\ade.cmd`. After a successful install on Windows, the packaged `.cmd` installer adds the target directory to HKCU `Environment\Path` when needed and broadcasts an environment-change notification. After a successful install on POSIX, `ensureUserBinOnShellPath` appends a marked `export PATH="$HOME/.local/bin:$PATH"` block to the user's shell rc (`.zshrc` for zsh, `.bashrc` for bash, `.profile` otherwise) iff (a) the install dir isn't already on the inherited `PATH` and (b) the file doesn't already contain the marker / line / target dir. The install IPC reply tells the renderer which profile was edited so the Settings/Onboarding UI can prompt the user to open a new terminal or `source` it. **Windows packaging.** The installer lays down `ade-cli-windows-wrapper.cmd` plus an `ade-cli-install-path.cmd` helper alongside the bundled Electron Node runtime. The helper installs `%LOCALAPPDATA%\ADE\bin\ade.cmd`, updates the user PATH when needed, and then `ade` works from a new normal Windows shell without a global Node install. See §14.4 for the packaging flow. @@ -711,7 +728,7 @@ Most services described here live under `apps/desktop/src/main/services/ | `state/` | `kvDb.ts`, `crsqliteExtension.ts`, `dbMaintenanceApi.ts`, `globalState.ts`, `projectState.ts`, `onConflictAudit.ts` | SQLite schema + open (WAL + `synchronous = NORMAL`), CRR extension loader, global state file, per-project state init. `kvDb` also attaches the optional `maintenance` (`DbMaintenanceApi`) handle — retention prunes, zero-peers-only cr-sqlite compaction, and fragmentation-gated vacuum — whose interface and shared retention constants live in `dbMaintenanceApi.ts` and are invoked by the storage doctor. `globalState.upsertRecentProject` accepts `preserveRecentOrder` so reactivating an already-known project (by app focus, deep link, etc.) refreshes its `lastOpenedAt` in place instead of jumping it to the front of the recents list. Recent projects use stable keys: local rows are keyed by absolute root path, remote rows by `remote::`, so a remote path string never collides with a local project. Pinned rows are retained above normal recency ordering and survive beyond the cap. `model_picker_favorites` and `model_picker_recents` are per-project CRR tables shared by desktop, TUI, and iOS; they are primary-key-only so CRR can convert them, with the recents cap enforced in `modelPickerStore.ts`. `AdeDb.sync.discardUnpublishedChangesForTables(tableNames)` lets a service clear local CRR state for specific tables without leaking those clears to sync peers — it records the cleared tables and `through_db_version` in the local-only `local_crr_change_suppressions` table, and `exportChangesSince` filters local-site rows for those tables at or below that version on the way out. The local-only excluded set (still kept out of replication) includes that suppression table itself, the snapshot caches, `local_worktree_residual_cleanups`, `pr_auto_link_ignores`, `pull_request_ai_summaries`, and `runtime_processes`. `crsql_changes` DELETE statements run through a helper that swallows the read-only-table error the cr-sqlite extension raises when a CRR-managed table is wiped, with a `db.crr_changes_cleanup_skipped` warn log instead of failing the migration. | | `sync/` | `syncService.ts`, `syncHostService.ts`, `syncPeerService.ts`, `syncRemoteCommandService.ts`, `syncProtocol.ts`, `deviceRegistryService.ts`, `syncPairingStore.ts` | **Thin delegation to the ADE runtime's sync service.** The authoritative sync service now lives in `apps/ade-cli/src/services/sync/`; the desktop main-process instances default to a non-host viewer role for legacy state and tests. The old in-process host is disabled unless `ADE_ENABLE_DESKTOP_SYNC_HOST=1` (diagnostics only). Wire formats — WebSocket envelope, remote command routing, device registry, pairing secrets — are the same across both implementations. Viewer joins clear the local `devices` + `sync_cluster_state` rows and then call `db.sync.discardUnpublishedChangesForTables(["devices", "sync_cluster_state"])` so the resulting DELETE rows do not leak back to other peers; the peer client follows up with `syncPeerService.acknowledgeLocalDbVersion()` to advance the outbound cursor past the suppressed range. | | `tests/` | `testService.ts` | Test-suite execution + run history. | -| `updates/` | `autoUpdateService.ts` | Electron auto-update wrapper around `electron-updater`. Owns the renderer-visible `AutoUpdateSnapshot` (`idle \| checking \| downloading \| ready \| installing \| error`), uses `compareUpdateVersions` (SemVer-aware) to dedupe / supersede staged installers and to reconcile `pendingInstallUpdate` against the running version on next boot. Packaged builds schedule startup/periodic checks; source/dev launches construct the service without auto-check timers so missing `app-update.yml` never surfaces as a renderer error. ADE manually starts downloads after a cache-volume capacity preflight, checks the installed-app volume again before staging, classifies disk/quota/network/verification/permission/installer failures in the shared snapshot, preserves verified downloads when safe, and bounds the native installer handoff with a watchdog. `quitAndInstall()` re-checks the staged version before calling `updater.quitAndInstall(false, true)`. See [desktop auto-update disk-space behavior](./features/onboarding-and-settings/desktop-auto-update.md). | +| `updates/` | `autoUpdateService.ts`, `autoUpdateVersions.ts` | Electron auto-update wrapper around `electron-updater`. Owns the renderer-visible `AutoUpdateSnapshot` (`idle \| checking \| downloading \| ready \| installing \| error`, plus `currentVersion` / `latestKnownVersion` for the truthful-version surfaces), uses `compareUpdateVersions` (the SemVer-aware comparator in `autoUpdateVersions.ts`) to dedupe / supersede staged installers and to reconcile `pendingInstallUpdate` against the running version on next boot. Packaged builds schedule startup/periodic checks; source/dev launches construct the service without auto-check timers so missing `app-update.yml` never surfaces as a renderer error. ADE manually starts downloads after a cache-volume capacity preflight, checks the installed-app volume again before staging, classifies disk/quota/network/verification/permission/installer failures in the shared snapshot, preserves verified downloads when safe, and bounds the native installer handoff with a watchdog. The install is transactional: `quitAndInstall()` re-checks the staged version, and a consent that aborts before the native updater takes over lands in `snapshot.parked` (a typed `AutoUpdateInstallAbortReason`) so the shell banner offers a retry instead of silently losing the update. When the runtime reports no active agent turns or work sessions (`RuntimeActivitySummary.idle`), a staged update is auto-applied after an idle grace period and a renderer-visible countdown (`autoApplyPending`); an explicit user cancel suppresses the next countdown (`autoApplySuppressedUntil`). Disable auto-apply with `ADE_DISABLE_AUTO_UPDATE_APPLY=1`. `autoUpdateVersions.ts` also builds the changelog (`buildReleaseNotesUrl`) and GitHub release (`buildGithubReleaseUrl`) links. See [desktop auto-update disk-space behavior](./features/onboarding-and-settings/desktop-auto-update.md). | | `storage/` | `diskPressure.ts`, `volume.ts`, `storageInsightsService.ts`, `historyCompression.ts`, `storageLedger.ts`, `storageDbBreakdown.ts`, `storageMaintenanceJournal.ts` | Disk-full/recovery hardening + the storage doctor. `diskPressure` samples all ADE storage roots, classifies pressure with recovery hysteresis, and gates write-producing operation classes via `canPerform(kind)` (enforced at each start boundary in `agentChatService` / `ptyService` / `processService` and the compressor). `storageInsightsService` builds the categorized Settings > Storage snapshot and preview-confirmed, link-safe cleanup, and runs the scheduled **storage doctor** maintenance sweep (`runMaintenanceNow` + post-boot/daily timers) that compresses history, reaps safe staging/backups/iOS build data, and invokes the kvDb DB-maintenance hooks. `storageLedger` is the declared bounding policy for every table/directory (with a CI coverage cross-check against `ADE_LAYOUT_DEFINITIONS`); `storageDbBreakdown` maps `dbstat` rows into the project-database breakdown; `storageMaintenanceJournal` reads/writes the 30-run doctor journal. `historyCompression` losslessly gzip-compresses inactive old transcripts/logs after byte-identity verification and exposes the transparent `.gz` read/reinflate helpers. Constructed in both `main.ts` and the `ade` runtime `bootstrap.ts`. See [features/storage-and-recovery/README.md](./features/storage-and-recovery/README.md). | | `usage/` | `usageTrackingService.ts`, `providerQuotaParsers.ts`, `usageStatsStore.ts`, `usageLedgerWorkerClient.ts`, `budgetCapService.ts`, `ledgers/localUsageLedgers.ts` | Live provider quota/cost accounting, budget enforcement, and retrospective activity stats. `usageTrackingService.ts` owns polling, pacing, provider/GitHub cache orchestration, and `getAdeUsageStats`; `providerQuotaParsers.ts` normalizes Claude and Codex quota payload variants and classifies Codex windows by advertised duration rather than assuming the provider's primary/secondary positions. The stats read returns cached expensive sources plus live project-DB aggregates immediately, marks the result `refreshing` when stale, and revalidates provider ledgers / GitHub in the background. Expensive local-ledger aggregation runs through `usageLedgerWorkerClient.ts` in a separate process so it cannot block terminal input, project switching, or sync; packaged desktop/CLI builds ship a sidecar while the static runtime uses the equivalent embedded entrypoint. `usageStatsStore.ts` aggregates AI calls, sessions, lanes, code movement, artifacts, automations, workers, streaks, and the local-only cross-client `usage_events` ledger. Local provider scanners live under `usage/ledgers/`. Budget caps can match a rule scope while `usd-per-run` evaluates usage records keyed to the active run id. For runtime-backed projects, the machine brain is the sole quota poller and the renderer consumes its pushed snapshot; `main.ts` does not start a competing project-context tracker. Threshold state remains shared at module level for the unbound/local contexts, and `main.ts` adds a final IPC-level dedup gate with a 10-minute TTL per `provider:threshold:resetCycle` key. | | `perf/` | `perfLog.ts`, `perfIpc.ts`, `metricsSampler.ts`, `aggregator.ts` | Opt-in local performance harness. `ADE_PERF_RUN_ID` opens a JSONL event log, samples Electron process metrics, records IPC durations, accepts renderer perf marks/web-vitals, and aggregates each run into `summary.json`. | diff --git a/docs/features/onboarding-and-settings/README.md b/docs/features/onboarding-and-settings/README.md index 2d79a08d6..c73800e13 100644 --- a/docs/features/onboarding-and-settings/README.md +++ b/docs/features/onboarding-and-settings/README.md @@ -266,7 +266,10 @@ Renderer — settings: - `apps/desktop/src/renderer/components/settings/AboutSection.tsx` — installed ADE version, packaged/dev badge, latest GitHub release lookup, release notes link, manual update check button, and ADE runtime - install / health status when available. + install / health status when available. It consumes the shared + `useAutoUpdateSnapshot` hook so "Installed" reads truthfully: the running + build normally, but the staged version when a download is `ready` or + `parked`, with "Latest" showing `latestKnownVersion`. - `apps/desktop/src/renderer/components/settings/AdeCliSection.tsx` — surfaces `window.ade.adeCli.getStatus()` / `installForUser()`. Status carries `terminalInstalled`, `agentPathReady`, @@ -516,24 +519,36 @@ Auto-update (top-bar control, not a settings tab): electron-updater wrapper that owns the renderer-visible `AutoUpdateSnapshot` (`status: "idle" | "checking" | "downloading" | "ready" | "installing" | "error"`, version, progress, recently - installed notice). Tracks superseded downloads against the current - ready version via `compareUpdateVersions` (a SemVer-aware - comparator that handles `v` prefixes, missing patch, and - prerelease ordering) so a same-or-older `update-available` while a + installed notice, plus the `parked` / `autoApplyPending` / + `autoApplySuppressedUntil` fields and `currentVersion` / + `latestKnownVersion` for the truthful-version surfaces). Tracks superseded + downloads against the current ready version via `compareUpdateVersions` + (the SemVer-aware comparator in `autoUpdateVersions.ts` that handles + `v` prefixes, missing patch, and prerelease ordering) so a same-or-older + `update-available` while a newer build is already staged is logged and ignored instead of clobbering the staged installer; packaged builds schedule startup and periodic update checks, while dev/source launches leave those timers off to avoid surfacing missing-updater-config errors; if the new build is strictly newer, the cached installer dir is wiped and the snapshot transitions back through `downloading`. `quitAndInstall()` is - asynchronous: it gates on the current snapshot being `ready`, + transactional and asynchronous: it gates on the current snapshot being `ready`, re-runs `updater.checkForUpdates()` with `allowReady: true` to confirm the staged installer is still the latest, and only then flips the snapshot to `installing`, persists the `pendingInstallUpdate` global-state row, and calls `updater.quitAndInstall(false, true)`. If the refresh check fails, it surfaces the error, drops the cache, and clears the pending - install. On the next launch, `reconcilePersistedUpdateState` + install. A consent that aborts before the native updater takes over sets + `snapshot.parked` with a typed `AutoUpdateInstallAbortReason` + (`refresh_failed`, `install_preflight_failed`, `prepare_failed`, + `prepare_timeout`, `handoff_failed`) so the shell banner can offer a retry. + When the runtime reports `RuntimeActivitySummary.idle` (no active agent turns + or work sessions), a staged update is auto-applied after an idle grace period + plus a renderer-visible countdown (`autoApplyPending`); an explicit cancel + suppresses the next countdown (`autoApplySuppressedUntil`), and + `ADE_DISABLE_AUTO_UPDATE_APPLY=1` turns auto-apply off. On the next launch, + `reconcilePersistedUpdateState` matches the running version against `pendingInstallUpdate` using the same SemVer comparator (so `>=` target counts as installed, even if the running build is one ahead), populates @@ -560,6 +575,34 @@ Auto-update (top-bar control, not a settings tab): on GitHub" button that opens `recentlyInstalled.githubReleaseUrl` (the GitHub release page). Each button is shown only when its URL is present; opening either link also dismisses the notice. +- `apps/desktop/src/renderer/components/app/useAutoUpdateSnapshot.ts` — the + shared subscription hook (initial `updateGetState()` read + live + `onUpdateEvent`). Every truthful-version surface — the top-bar pill, the + app-shell banner, and the About panel — consumes it so they never disagree + about what is running versus what is staged. +- `apps/desktop/src/renderer/components/app/AutoUpdateBanner.tsx` — the + app-shell staleness banner plus the idle-auto-apply countdown toast, + colocated so both read one snapshot subscription. Renders a `parked` state as + "Update to vX didn't finish — Restart to retry" (parked wins over a plain + `ready`) and a plain `ready` state as "Running vCurrent · vNext is ready", + each with a **Restart now** action; dismissal is keyed on a stable signature + so it reappears on a newer staged version or a fresh abort. The countdown + toast is driven off `autoApplyPending` with a **Cancel** action wired to + `updateCancelAutoApply()`. +- `apps/desktop/src/renderer/components/app/BrainRecoveryNotice.tsx` — the + app-shell notice shown once per distinct machine-brain event-loop recovery. + It reads the one-shot `localRuntime.lastWedge` from `app.getInfo()`, + announces "ADE recovered from a background issue … a stuck task (…) was + restarted", and acknowledges by persisting the wedge `ts` to `localStorage` + so the same event never nags twice while a fresh recovery (new `ts`) + reappears. The wedge itself is produced by the brain event-loop watchdog + (see [ARCHITECTURE.md §2.1](../../ARCHITECTURE.md) and + [Storage and recovery](../storage-and-recovery/README.md)). +- `apps/desktop/src/main/services/updates/autoUpdateVersions.ts` — the pure + SemVer helpers shared by the service and the `ade doctor` CLI: + `compareUpdateVersions` (core + prerelease ordering, `v`-prefix and + missing-patch tolerant), `buildReleaseNotesUrl` (the docs changelog link), + and `buildGithubReleaseUrl` (the tagged GitHub release page). ## Detail docs @@ -775,6 +818,17 @@ Onboarding and settings follow a simple rule: the middle of quitAndInstall. New status checks should treat `ready` and `installing` symmetrically when deciding whether to cancel or override the staged update. +- **A parked install is not a failure.** An aborted consent lands in + `snapshot.parked`, not `error` — the download is still staged and the shell + banner offers a **Restart now** retry. Keep parked distinct from the disk / + network / verification error classification, and let a parked state win over a + plain `ready` state when both describe the same version. +- **Idle auto-apply needs the runtime activity summary.** Auto-apply only arms + when the service can read `RuntimeActivitySummary.idle` and the machine has + been continuously idle for the grace period; renewed activity clears the + countdown, and an explicit cancel sets `autoApplySuppressedUntil`. It is off + under `ADE_DISABLE_AUTO_UPDATE_APPLY=1` and on dev/source launches that have no + auto-check timers. ## Cross-links diff --git a/docs/features/onboarding-and-settings/desktop-auto-update.md b/docs/features/onboarding-and-settings/desktop-auto-update.md index 37252063b..6baa9f5af 100644 --- a/docs/features/onboarding-and-settings/desktop-auto-update.md +++ b/docs/features/onboarding-and-settings/desktop-auto-update.md @@ -60,3 +60,53 @@ watchdog, while the native Squirrel handoff has a separate five-minute watchdog so loopback transfer and staging are not mistaken for a stalled quit. Handoff timeouts retain the pending-install marker because Squirrel may still complete; an explicit updater error clears it. + +## Truthful version surfaces + +Every version surface reads from one shared snapshot so they can never disagree +about what is running versus what is staged. `AutoUpdateSnapshot` carries both +`currentVersion` (the running build) and `latestKnownVersion` (the newest +version `electron-updater` has observed from its configured feed), plus the +staged `version`, `parked`, and `autoApplyPending` fields below. +`useAutoUpdateSnapshot` (`renderer/components/app/useAutoUpdateSnapshot.ts`) +does the initial `updateGetState()` read and subscribes to `onUpdateEvent`; the +top-bar pill (`AutoUpdateControl`), the app-shell banner (`AutoUpdateBanner`), +and the Settings About panel (`AboutSection`) all consume it. About shows the +running version as "Installed" and `latestKnownVersion` as "Latest", but swaps +"Installed" to the staged version when a download is `ready` or `parked`, so the +user sees the version they will get after the next restart rather than a stale +"you're up to date". + +## Transactional install and the parked banner + +`quitAndInstall()` is transactional. Before flipping the snapshot to +`installing` it re-runs `updater.checkForUpdates({ allowReady: true })` to +confirm the staged installer is still the latest, then persists +`pendingInstallUpdate` and calls `updater.quitAndInstall(false, true)`. A +consent that aborts before the native updater can take over does not silently +vanish: the snapshot records `parked: { reason, at }` where `reason` is a typed +`AutoUpdateInstallAbortReason` — `refresh_failed`, `install_preflight_failed`, +`prepare_failed`, `prepare_timeout`, or `handoff_failed`. The app-shell +`AutoUpdateBanner` renders a parked state as "Update to vX didn't finish — +Restart to retry" (a parked state wins over a plain `ready` state), and a plain +`ready` state as "Running vCurrent · vNext is ready", each with a **Restart +now** action wired to `updateQuitAndInstall()`. Banner dismissal is keyed on a +stable signature so it reappears when a newer version stages or a fresh abort +occurs, but stays hidden for an unchanged state. + +## Idle auto-apply + +When auto-apply is enabled (packaged builds with auto-check on, unless +`ADE_DISABLE_AUTO_UPDATE_APPLY=1`), a `ready` update is applied on its own once +the machine is quiet. The service polls the runtime's +`RuntimeActivitySummary` — `idle` is true only when there are no active agent +turns and no active work sessions. After the runtime has been continuously idle +for the idle grace period, the snapshot gets an `autoApplyPending: { deadlineAt }` +countdown; the `AutoUpdateBanner` renders a "Updating to vX in Ns" toast that +ticks down each second. Reaching the deadline while still `ready` and idle calls +the same `quitAndInstall()` path and emits `ade_update_auto_applied`. Any +renewed activity clears the pending countdown, and an explicit user **Cancel** +(`updateCancelAutoApply`) sets `autoApplySuppressedUntil` so another countdown +is not started until that epoch passes. `installing` remains a sticky status +throughout: the service ignores `update-not-available` / `checking-for-update` / +`error` while a quitAndInstall is in flight. diff --git a/docs/features/remote-runtime/README.md b/docs/features/remote-runtime/README.md index 1b8e991cb..5a0af87c0 100644 --- a/docs/features/remote-runtime/README.md +++ b/docs/features/remote-runtime/README.md @@ -83,7 +83,14 @@ relay payload E2E encryption is planned security work. See the trust boundary in `ade serve` for non-primary sockets, tracks the per-user login service install/health state, and applies short per-call timeouts for project registration, file actions, and event polling so renderer IPC calls do not - wait for the desktop handler timeout. + wait for the desktop handler timeout. `LocalRuntimeStatus` now also carries + the brain's `pid`, its bound `syncPort`, the account-directory `publishHealth` + slice (state + `failingSinceMs` + last-leg durations), and the one-shot + `lastWedge` recovered by the event-loop watchdog. The Machines panel renders + publish health as a This-Mac indicator (`remoteMachineModel.describePublishHealth`, + which reads inactive states as "none" and only alarms a real failure after it + has persisted ~2 minutes), and the app shell reads `lastWedge` for the + `BrainRecoveryNotice` banner. - `apps/desktop/src/main/services/runtime/lastFailureStore.ts` — bounded typed project/machine failure reports used when the background service exits before desktop IPC can obtain a normal runtime error. @@ -103,7 +110,14 @@ relay payload E2E encryption is planned security work. See the trust boundary in connected / available / unavailable sections, Pair and SSH entry paths, share-this-machine and connection-doctor cards, saved/discovered machine rows, route and latency status, SSH host-key trust, structured connection - errors, project picker, and the dirty-local-work warning. + errors, project picker, the dirty-local-work warning, and the This-Mac + route-publish health indicator. `remoteMachineModel.ts` + (`describePublishHealth`) is the pure classifier for that indicator: the + publishing `published` state reads healthy, the non-publishing states + (`sync_disabled`, `not_host`, `account_signed_out`, `machine_key_unavailable`, + …) read as "none", and every other state is a failure that only alarms once it + has persisted at least `PUBLISH_FAILING_ALARM_MS` (2 min) so a transient blip + stays quiet. - `apps/desktop/src/renderer/components/projects/RemoteProjectOpenDialog.tsx` — confirmation dialog before opening a remote project, surfaces local matches with uncommitted changes. @@ -227,7 +241,10 @@ run a local repair against data owned by the remote machine. See the host signs the client's challenge nonce over an ephemeral X25519 exchange, the client verifies that signature against the directory key before releasing any account credential, and both the account attestation - and the returned paired credentials travel ChaCha20-Poly1305-sealed. A + and the returned paired credentials travel AEAD-sealed under a negotiated + cipher — ChaCha20-Poly1305, or AES-256-GCM when a packaged Electron's bundled + BoringSSL lacks ChaCha20-Poly1305, with the chosen cipher bound into the + signed challenge so it cannot be downgraded. A host-identity verification failure aborts adoption immediately (no route is retried); hosts without a published key remain relay-only-adoptable. Successful adoption saves the returned DPoP-bound credentials either way. diff --git a/docs/features/storage-and-recovery/README.md b/docs/features/storage-and-recovery/README.md index 59ae74dd8..cfaf7d5dc 100644 --- a/docs/features/storage-and-recovery/README.md +++ b/docs/features/storage-and-recovery/README.md @@ -13,6 +13,12 @@ | `apps/desktop/src/main/services/runtime/lastFailureStore.ts` | Stores typed project/machine failures, keeps one previous report, counts repeated signatures, and computes crash-loop startup backoff. | | `apps/ade-cli/src/services/runtime/failureLogDeduper.ts` | Emits the first repeated brain failure immediately and only periodic occurrence summaries afterward. | | `apps/ade-cli/src/services/runtime/runtimeLogMaintenance.ts` | Bounds launchd stdout/stderr with tail-copy plus in-place truncation. | +| `apps/ade-cli/src/services/runtime/brainLoopWatchdog.ts` | Worker-thread **event-loop watchdog** for the machine brain. Heartbeats every second with the current command name; a stall past `ADE_LOOP_WATCHDOG_MS` (15 s default) that is not a sleep/suspend writes an `event-loop-wedge.json` breadcrumb and `SIGKILL`s the brain. On next boot it promotes the breadcrumb to `last-wedge.json`, logs `brain.recovered_from_wedge`, and emits a deduped recovery event. Disable with `ADE_DISABLE_LOOP_WATCHDOG=1`. | +| `apps/ade-cli/src/services/runtime/brainFreshnessMonitor.ts` | The running brain stats its own CLI entrypoint every 5 min (`ADE_BRAIN_FRESHNESS_INTERVAL_MS`), hashes only after the stat changes, and — when the on-disk hash no longer matches the baked runtime hash — waits for the brain to go idle (bounded) before triggering the brain-update service restart so an in-place upgrade takes effect without interrupting active work. Disable with `ADE_DISABLE_BRAIN_FRESHNESS=1`. | +| `apps/ade-cli/src/services/runtime/runtimeBuildIdentity.ts` | `computeRuntimeBuildHash` / `computeRuntimeBuildHashAsync` — the SHA-256 of the CLI entrypoint used as the brain build identity by the freshness monitor and the desktop compatibility handshake. | +| `apps/ade-cli/src/services/runtime/brainLogger.ts` | The machine-brain logger: reuses the desktop `createFileLogger` to write `~/.ade/runtime/brain.jsonl` (10 MiB `.1` rotation) and additionally mirrors timestamped `warn`/`error` lines to stderr so launchd captures them. | +| `apps/ade-cli/src/commands/doctor.ts` | `ade doctor [--online]` — connects to the brain over the local socket and prints one `ok`/`warn`/`fail` row per subsystem (App version, Brain, Wedge history, Sync port, Publish health, Relay, Account); exits non-zero on any `fail`. `evaluateDoctorRows` is pure and dependency-injected so the desktop connection-doctor card and the CLI share one verdict. | +| `apps/desktop/src/shared/adeRuntimeProtocol.ts` | Shared runtime-protocol contract: `RUNTIME_COMPAT_LEVEL` + `isRuntimeProtocolCompatible` (the integer compatibility-window check), and the tolerant parsers `parseRuntimePublishHealth` / `parseRuntimeLastWedge` that decode `runtimeInfo.publishHealth` and `runtimeInfo.lastWedge` for the connection pool, the doctor, and the desktop status surfaces. | | `apps/desktop/src/main/services/runtime/projectRecoveryService.ts` | Brain-independent diagnosis and ordered repair: space, ownership, database validation, migration recovery, service restart, endpoint/project verification, and chat reconciliation. | | `apps/desktop/src/main/services/storage/diskPressure.ts` | Samples all ADE storage roots, classifies pressure with recovery hysteresis, and gates write-producing operation classes via `canPerform(kind)`. Exports the `DiskPressureMonitor` type and refusal-message copy. | | `apps/desktop/src/main/services/storage/volume.ts` | `readVolumeSpace(dir)` (statfs free/total bytes) and `isNoSpaceError(err)` (ENOSPC/EDQUOT and disk-full message detection), shared by the pressure monitor and the database-open error classifier. | @@ -103,6 +109,37 @@ establishes exclusive ownership, runs `quick_check`, opens the database so the pre-migration recovery pass can resolve staging, restarts the service, verifies the endpoint and project RPC, then counts chat records and continuity warnings. +### Brain resilience: watchdog, freshness, and recovery notice + +The machine brain guards its own liveness. The **event-loop watchdog** +(`brainLoopWatchdog.ts`) runs an unref'd worker thread that the main thread +heartbeats every second with the name of the command currently running +(`trackBrainLoopWatchdogCommand`). If the heartbeat stalls past +`ADE_LOOP_WATCHDOG_MS` (15 s default) — and the gap is a genuine wedge, not a +laptop sleep/suspend, which the worker distinguishes by comparing wall-clock and +monotonic deltas — the worker atomically writes an `event-loop-wedge.json` +breadcrumb (wedged command, blocked ms, timestamp) under the runtime dir and +`SIGKILL`s the brain so launchd restarts it. A hung command therefore recovers +in seconds instead of stranding every client. + +On the next boot the watchdog promotes any breadcrumb to `last-wedge.json`, +logs `brain.recovered_from_wedge`, and emits a deduped `ade_brain_recovered` +analytics event. The recovered wedge is exposed to clients through +`runtimeInfo.lastWedge` (parsed by `adeRuntimeProtocol.parseRuntimeLastWedge`, +surfaced on `LocalRuntimeStatus.lastWedge`), and the desktop app shell shows it +once per distinct `ts` in `BrainRecoveryNotice` ("ADE recovered from a +background issue … a stuck task was restarted"). `ade doctor` reports the same +`last-wedge.json` as its Wedge-history row. + +Separately, the **freshness monitor** (`brainFreshnessMonitor.ts`) keeps a +long-lived login-service brain from drifting behind an in-place binary upgrade: +it stats its own CLI entrypoint every 5 min, hashes only on a stat change +(`runtimeBuildIdentity.computeRuntimeBuildHashAsync`), and when the disk hash no +longer matches the baked runtime hash it waits for the brain to be idle (bounded +by a max wait) before requesting the brain-update service restart. A transient +restart failure keeps the previous stat baseline so the unchanged replacement is +re-checked on the next probe rather than the check disabling itself. + ### Disk pressure and enforcement The monitor samples every configured project/machine root and uses the most diff --git a/docs/features/sync-and-multi-device/README.md b/docs/features/sync-and-multi-device/README.md index fe248d90d..c4623ad0d 100644 --- a/docs/features/sync-and-multi-device/README.md +++ b/docs/features/sync-and-multi-device/README.md @@ -449,7 +449,13 @@ Canonical files (`apps/ade-cli/src/services/sync/`): a `hello` whose `auth.kind` is `account_sealed`: the host signs the client's nonce over an ephemeral X25519 exchange with its `machineIdentitySigningStore` key, derives the session key, unseals the account credentials from the sealed - hello, and returns the paired credentials in a sealed `hello_ok`; a completed + hello, and returns the paired credentials in a sealed `hello_ok`. The seal + AEAD is negotiated from the client's advertised `supportedAeads` against the + host's `supportedAdoptChannelAeads()` — the host picks the first mutual + choice, echoes it in the `account_challenge_ok`, and binds it into the + signature input, so a packaged Electron without ChaCha20-Poly1305 negotiates + `aes-256-gcm` instead of failing to connect; a client advertising an AEAD set + with no host overlap is rejected rather than silently downgraded. A completed single-use challenge is required, well-formed challenges feed no rate limiter while malformed/anomalous ones charge a per-IP + global cooldown, and unsealed `account_sealed` adoption is the one account path allowed over a direct @@ -550,8 +556,17 @@ Canonical files (`apps/ade-cli/src/services/sync/`): - `sharedSyncListener.ts` — the brain-level WebSocket listener shared across per-project host services. Binds once (preferred-port retry: ~8 attempts over ~3.2 s on the saved port before falling back to a - port scan, so a brain restart does not drift the port phones saved) - and is handed between hosts on project switch: the new host adopts + port scan, so a brain restart does not drift the port phones saved). + On an `EADDRINUSE` for a port in the sync range (`DEFAULT_SYNC_HOST_PORT` + 8787 through `SYNC_HOST_MAX_PORT` 8999) it runs **sync-port zombie + reaping**: it diagnoses the port's holders (`inspectSyncListenerPort`), + and if a stale ADE brain owns it, re-confirms the same pid + process + start-time on a second diagnosis (guarding against pid reuse), terminates + that holder, logs `sync_listener.zombie_reaped`, and retries the freed port + once — so a dead-but-port-holding sibling brain cannot force the new brain + onto a drifted port that phones never saved. The same diagnosis feeds the + `ade doctor` Sync-port row. The listener is + handed between hosts on project switch: the new host adopts the open sockets — peer metadata carried over, pairing auth re-validated against the pairing store, changeset cursors recomputed from the peer's per-site cursor map, chat/terminal/roster subscriptions @@ -618,7 +633,13 @@ Canonical files (`apps/ade-cli/src/services/sync/`): `envelope_chunk` frames for peers that declared the `chunkedEnvelopes` hello capability (`SYNC_CHUNKED_ENVELOPES_CAPABILITY`); legacy peers get the single - full frame. Protocol version is `1`. Default host port is `8787`. + full frame. Protocol version is `1`. Default host port is `8787`, + and `SYNC_HOST_MAX_PORT` (`8999`) bounds the sync range the shared + listener will zombie-reap a stale ADE holder from. +- `abortSignal.ts` — the shared cancellation helper (`runWithAbortSignal`, + `abortSignalError`) used across the sync command paths so a registration- or + caller-carried `AbortSignal` rejects in-flight work with a consistent + `AbortError` instead of each call site re-implementing the wiring. - `syncRemoteCommandService.ts` (~4,600 lines) — command registry (lanes, chat, git, PR, sessions, conflicts, files, `usage.getAdeStats`, `usage.getQuotaSnapshot`, `usage.refreshQuota`, @@ -730,9 +751,18 @@ Canonical files (`apps/ade-cli/src/services/sync/`): `ade-adopt-v1` primitives used on both sides of sealed account adoption: X25519 ephemeral key generation, the canonical challenge signature input, HKDF-SHA256 session-key derivation over the X25519 shared secret + nonce, - ChaCha20-Poly1305 `seal`/`unseal` with context-bound AAD, and Ed25519 + AEAD `seal`/`unseal` with context-bound AAD, and Ed25519 sign/verify against the raw published key. Also imported by - `machineIdentitySigningStore.ts` for SPKI↔raw key conversion. + `machineIdentitySigningStore.ts` for SPKI↔raw key conversion. The AEAD is + **negotiated**: `ADOPT_CHANNEL_AEADS` lists `chacha20-poly1305` then + `aes-256-gcm`, and `supportedAdoptChannelAeads()` probes which of them the + running crypto backend can actually construct (cached). This is what lets a + packaged Electron whose bundled BoringSSL lacks ChaCha20-Poly1305 still + account-connect: the client advertises the AEADs it supports in the challenge, + the host picks the first it also supports, and the chosen AEAD is folded into + the challenge signature input (`aead` field) so it cannot be downgraded by an + on-path attacker. A client that sends no AEAD list, and both sides by default, + fall back to `chacha20-poly1305`. - `syncCloudRelayStore.ts` — persists the cloud tunnel-relay identity at `~/.ade/secrets/sync-cloud-relay.json` (lazily-minted 32-hex `machineKey` + HMAC `secret`, chmod `0600`). The identity is stable in normal operation. @@ -1472,13 +1502,21 @@ feature is merged or because a deliberately isolated-port host is running. nonce and an ephemeral X25519 public key; the host replies `account_challenge_ok` with its own X25519 ephemeral key and an Ed25519 signature (from `machineIdentitySigningStore`) over the canonical - `ade-adopt-v1 | hostDeviceId | nonce | clientEph | hostEph | ts` string. The + `ade-adopt-v1 | hostDeviceId | nonce | clientEph | hostEph | ts[ | aead]` + string. The client **verifies that signature against the directory-published `pubkey` before releasing any credential**, so a machine cannot be impersonated on a - LAN. Both sides derive the same ChaCha20-Poly1305 key via HKDF-SHA256 over the + LAN. Both sides derive the same AEAD session key via HKDF-SHA256 over the X25519 shared secret and nonce; the client then sends a `hello` with `auth.kind = "account_sealed"` carrying the sealed account attestation, and the host returns the minted paired credentials in a sealed `hello_ok`. The + seal cipher is **negotiated**: the client advertises the AEADs its crypto + backend supports (`chacha20-poly1305`, `aes-256-gcm`), the host chooses the + first it also supports, echoes it in `account_challenge_ok`, and **binds the + chosen AEAD into the signed challenge string** so it cannot be downgraded on + the wire — this is what lets a packaged Electron whose bundled BoringSSL lacks + ChaCha20-Poly1305 adopt over `aes-256-gcm` instead of failing. A client whose + advertised set does not overlap the host's is rejected. The challenge is single-use and TTL-bounded (60 s); it is required before a sealed hello is accepted. `ade-adopt-v1` protects the exchanged *credentials* (bearer, DPoP proof, minted secret), not the confidentiality of the subsequent session: @@ -1617,7 +1655,7 @@ feature is merged or because a deliberately isolated-port host is running. | Device-bound pairing (DPoP, Secure Enclave P-256) | Implemented (host + brain ingress; `requireDpop` / `ADE_SYNC_REQUIRE_DPOP`) | | Cloud tunnel relay (off-LAN transport, `relay` candidate) | Implemented whenever the host is signed in, with no separate toggle and with same-account per-connection proof (`syncTunnelClientService` + `apps/tunnel-relay`) | | Relay end-to-end self-probe + zombie-control detection (honest relay publication) | Implemented (`syncRelaySelfProbe`, JSON control keepalive, `sync.runSelfProbe`, `ade doctor` relay check) | -| Sealed account adoption over direct routes (`ade-adopt-v1`, host `pubkey` identity, relay → tailnet → LAN fallback) | Implemented (`machineIdentitySigningStore` + `adoptChannelCrypto`; desktop + iOS clients) | +| Sealed account adoption over direct routes (`ade-adopt-v1`, host `pubkey` identity, relay → tailnet → LAN fallback, negotiated ChaCha20-Poly1305 / AES-256-GCM AEAD) | Implemented (`machineIdentitySigningStore` + `adoptChannelCrypto`; desktop + iOS clients) | | Push notifications + Live Activities (APNs relay) | Implemented (see `push-notifications.md`; on-device E2E needs a physical iPhone) | | Tailscale integration | Implemented (address candidate + mDNS TXT + per-node `tailscale serve` publication on the live sync port) | | Clean, published lane + Work chat handoff between connected desktops | Implemented ([contract](./cross-machine-session-handoff.md)) | diff --git a/docs/logging.md b/docs/logging.md index 3cd9ff5d8..8442ee8fc 100644 --- a/docs/logging.md +++ b/docs/logging.md @@ -78,6 +78,14 @@ The public contract is `apps/desktop/src/shared/types/productAnalytics.ts`. The - `ade_error` - `ade_daily_usage_summary` - `ade_analytics_budget` +- `ade_update_install_aborted` +- `ade_update_quit_escalated` +- `ade_update_auto_applied` +- `ade_update_auto_apply_cancelled` +- `ade_brain_recovered` +- `ade_publish_failing` + +The update and reliability events are low-frequency by construction: the four `ade_update_*` events fire at most once per install attempt or idle-apply cycle (daily caps 10–20, minute caps 3–6); `ade_brain_recovered` fires once per wedge recovery at brain startup; `ade_publish_failing` is edge-triggered once per sustained failure episode (first crossing of two minutes), never per attempt. Properties are closed enums and bounded numbers — `reason` is allowlisted to the abort-reason constant, `last_command` is a closed sync-action slug, and `leg`/`code` are the coarse publish classifications. Worst-case combined volume is a handful of events on a very bad day, inside the shared ceiling. The default machine-wide ceiling is 200 accepted events per UTC day, shared across desktop, runtime, TUI, hosted web, and API-originated aggregates. Each event also has a tighter per-day and per-minute ceiling. Capture ingress is capped, noisy events use persisted deduplication windows, the in-memory transport queue is bounded, and the previous day's accepted/drop totals are summarized in at most two budget events per day. @@ -129,7 +137,7 @@ The full-access personal key belongs only in encrypted ADE secrets. When running ## PostHog dashboards -`scripts/posthog/dashboard-spec.mjs` is the declarative source of truth. `scripts/posthog/provision.mjs` validates and idempotently upserts the managed objects. The project currently has five managed dashboards and thirty managed insights: +`scripts/posthog/dashboard-spec.mjs` is the declarative source of truth. `scripts/posthog/provision.mjs` validates and idempotently upserts the managed objects. The project currently has five managed dashboards and thirty-one managed insights: - ADE · Growth and retention - ADE · Surface and feature adoption diff --git a/scripts/posthog/dashboard-spec.mjs b/scripts/posthog/dashboard-spec.mjs index f08003d53..23b427184 100644 --- a/scripts/posthog/dashboard-spec.mjs +++ b/scripts/posthog/dashboard-spec.mjs @@ -11,6 +11,12 @@ export const EVENTS = Object.freeze({ ERROR: "ade_error", DAILY_USAGE_SUMMARY: "ade_daily_usage_summary", ANALYTICS_BUDGET: "ade_analytics_budget", + UPDATE_INSTALL_ABORTED: "ade_update_install_aborted", + UPDATE_QUIT_ESCALATED: "ade_update_quit_escalated", + UPDATE_AUTO_APPLIED: "ade_update_auto_applied", + UPDATE_AUTO_APPLY_CANCELLED: "ade_update_auto_apply_cancelled", + BRAIN_RECOVERED: "ade_brain_recovered", + PUBLISH_FAILING: "ade_publish_failing", MARKETING_APP_OPENED: "ade_marketing_app_opened", MARKETING_SCREEN_VIEWED: "ade_marketing_screen_viewed", MARKETING_FEATURE_USED: "ade_marketing_feature_used", @@ -501,6 +507,22 @@ export const dashboardSpec = Object.freeze({ breakdown: PROPERTIES.OUTCOME, }), ), + insight( + "reliability-incidents", + "Reliability incidents", + "Brain wedge recoveries, sustained route-publish failures, and update-flow aborts/escalations. Coarse counts only; command names are closed action slugs and no payload content is ever attached.", + trends({ + series: [ + eventNode(EVENTS.BRAIN_RECOVERED, "Brain recovered from wedge"), + eventNode(EVENTS.PUBLISH_FAILING, "Route publish failing"), + eventNode(EVENTS.UPDATE_INSTALL_ABORTED, "Update install aborted"), + eventNode(EVENTS.UPDATE_QUIT_ESCALATED, "Update quit escalated"), + eventNode(EVENTS.UPDATE_AUTO_APPLIED, "Update auto-applied"), + ], + interval: "day", + dateFrom: "-30d", + }), + ), insight( "analytics-sent-vs-dropped", "Delayed local budget rollups", @@ -540,7 +562,7 @@ export const dashboardSpec = Object.freeze({ insight( "monthly-analytics-volume", "30-day ingested analytics volume", - "PostHog's actual ingested count across ADE's closed 19-event catalog. The goal line marks the current 1,000,000-event monthly Product Analytics free allowance. This is an instrumentation health view, not the account billing meter: stray or abusive events sent with the public project token are outside this chart.", + "PostHog's actual ingested count across ADE's closed 25-event catalog. The goal line marks the current 1,000,000-event monthly Product Analytics free allowance. This is an instrumentation health view, not the account billing meter: stray or abusive events sent with the public project token are outside this chart.", trends({ series: ALL_INGESTED_EVENTS.map((event) => eventNode(event, event)), formula: ALL_INGESTED_EVENTS_FORMULA, diff --git a/scripts/posthog/provision.test.mjs b/scripts/posthog/provision.test.mjs index 3295bf7d9..4feee4228 100644 --- a/scripts/posthog/provision.test.mjs +++ b/scripts/posthog/provision.test.mjs @@ -25,7 +25,7 @@ function findInsight(key) { } test("dashboard spec is valid and excludes replay queries", () => { - assert.deepEqual(validateDashboardSpec(), { dashboards: 5, insights: 30 }); + assert.deepEqual(validateDashboardSpec(), { dashboards: 5, insights: 31 }); assert.doesNotMatch(JSON.stringify(dashboardSpec), /\$snapshot|session replay|recording_property/i); }); @@ -76,8 +76,8 @@ test("dashboard queries match the bounded instrumentation semantics", () => { const ingestedVolume = findInsight("monthly-analytics-volume"); assert.equal(ingestedVolume.name, "30-day ingested analytics volume"); - assert.equal(ingestedVolume.query.source.series.length, 19); - assert.equal(ingestedVolume.query.source.trendsFilter.formula, "A+B+C+D+E+F+G+H+I+J+K+L+M+N+O+P+Q+R+S"); + assert.equal(ingestedVolume.query.source.series.length, 25); + assert.equal(ingestedVolume.query.source.trendsFilter.formula, "A+B+C+D+E+F+G+H+I+J+K+L+M+N+O+P+Q+R+S+T+U+V+W+X+Y"); }); test("config requires HTTPS and a numeric project ID", () => { @@ -169,7 +169,7 @@ test("dry run plans every object without issuing writes", async () => { output: { write(chunk) { output += chunk; } }, }); - assert.deepEqual(summary, { created: 35, updated: 0, unchanged: 0 }); + assert.deepEqual(summary, { created: 36, updated: 0, unchanged: 0 }); assert.equal(writes.length, 0); assert.match(output, /would create dashboard: ADE · Growth and retention/); assert.match(output, /would create dashboard: ADE · Marketing acquisition/); @@ -223,11 +223,11 @@ test("managed objects that already match are not rewritten", async () => { }; const first = await provisionDashboards({ api, output: { write() {} } }); - assert.deepEqual(first, { created: 35, updated: 0, unchanged: 0 }); - assert.equal(writes.length, 35); + assert.deepEqual(first, { created: 36, updated: 0, unchanged: 0 }); + assert.equal(writes.length, 36); writes.length = 0; const second = await provisionDashboards({ api, output: { write() {} } }); - assert.deepEqual(second, { created: 0, updated: 0, unchanged: 35 }); + assert.deepEqual(second, { created: 0, updated: 0, unchanged: 36 }); assert.equal(writes.length, 0); }); From d9635d547ac0ecf4c38f050d65f3b3b160ac3894 Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:33:15 -0400 Subject: [PATCH 5/7] ship: fix CLI bundle electron leak, guard update hook, deflake pool tests Extracts the electron-free release feed out of githubService so ade doctor's online check can't drag electron into the CLI bundle; the update snapshot hook tolerates hosts without the update IPC surface; the pool tests wait for ade/initialize readiness instead of bare socket accept. Co-Authored-By: Claude Fable 5 --- apps/ade-cli/src/commands/doctor.ts | 2 +- .../main/services/github/adeReleaseFeed.ts | 63 +++++++++++++++++++ .../src/main/services/github/githubService.ts | 62 +----------------- .../localRuntimeConnectionPool.test.ts | 22 +++---- .../components/app/useAutoUpdateSnapshot.ts | 21 ++++--- 5 files changed, 89 insertions(+), 81 deletions(-) create mode 100644 apps/desktop/src/main/services/github/adeReleaseFeed.ts diff --git a/apps/ade-cli/src/commands/doctor.ts b/apps/ade-cli/src/commands/doctor.ts index 60522ea61..05fdc952c 100644 --- a/apps/ade-cli/src/commands/doctor.ts +++ b/apps/ade-cli/src/commands/doctor.ts @@ -366,7 +366,7 @@ async function readLatestDesktopVersionOnline(): Promise { const timer = setTimeout(() => controller.abort(), DOCTOR_ONLINE_DEADLINE_MS); try { const { fetchAdeLatestRelease } = await import( - "../../../desktop/src/main/services/github/githubService" + "../../../desktop/src/main/services/github/adeReleaseFeed" ); const release = await doctorTimeout( fetchAdeLatestRelease({ diff --git a/apps/desktop/src/main/services/github/adeReleaseFeed.ts b/apps/desktop/src/main/services/github/adeReleaseFeed.ts new file mode 100644 index 000000000..52a53b030 --- /dev/null +++ b/apps/desktop/src/main/services/github/adeReleaseFeed.ts @@ -0,0 +1,63 @@ +export const ADE_RELEASE_REPO = { owner: "arul28", name: "ADE" } as const; + +export type AdeLatestRelease = { + version: string; + tagName: string; + htmlUrl: string | null; + publishedAt: string | null; +}; + +type AdeReleaseFetch = ( + input: string | URL, + init?: RequestInit, +) => Promise; + +function optionalString(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +/** + * Fetches the latest stable ADE release without depending on Electron or the + * desktop GitHub service. GitHub's `releases/latest` endpoint excludes drafts + * and prereleases. + */ +export async function fetchAdeLatestRelease(options?: { + token?: string | null; + repo?: { owner: string; name: string }; + fetchImpl?: AdeReleaseFetch; +}): Promise { + const repo = options?.repo ?? ADE_RELEASE_REPO; + const doFetch = options?.fetchImpl ?? fetch; + const headers: Record = { + accept: "application/vnd.github+json", + "user-agent": "ade-desktop", + }; + const token = options?.token?.trim(); + if (token) headers.authorization = `Bearer ${token}`; + + let response: Response; + try { + response = await doFetch( + `https://api.github.com/repos/${encodeURIComponent(repo.owner)}/${encodeURIComponent(repo.name)}/releases/latest`, + { method: "GET", headers }, + ); + } catch { + return null; + } + if (!response.ok) { + void response.body?.cancel().catch(() => {}); + return null; + } + + const payload = (await response.json().catch(() => null)) as Record | null; + if (!payload) return null; + + const tagName = optionalString(payload.tag_name); + if (!tagName) return null; + return { + version: tagName.replace(/^v/i, "").trim() || tagName, + tagName, + htmlUrl: optionalString(payload.html_url), + publishedAt: optionalString(payload.published_at), + }; +} diff --git a/apps/desktop/src/main/services/github/githubService.ts b/apps/desktop/src/main/services/github/githubService.ts index 22adffa59..510645492 100644 --- a/apps/desktop/src/main/services/github/githubService.ts +++ b/apps/desktop/src/main/services/github/githubService.ts @@ -26,6 +26,8 @@ import { createGitHubAppUserAuthService } from "./githubAppUserAuthService"; import { nowIso, asString } from "../shared/utils"; +export { fetchAdeLatestRelease } from "./adeReleaseFeed"; + const AUTH_STORE_FILE_NAME = "github-token.v1.bin"; const MACHINE_TOKEN_KEY = "github.token.v1"; const GITHUB_API_TIMEOUT_MS = 20_000; @@ -442,66 +444,6 @@ export function parseNextLink(linkHeader: string | null): string | null { return null; } -export const ADE_RELEASE_REPO = { owner: "arul28", name: "ADE" } as const; - -export type AdeLatestRelease = { - version: string; - tagName: string; - htmlUrl: string | null; - publishedAt: string | null; -}; - -/** - * Fetches the latest *stable* GitHub release for the ADE repo. The - * `releases/latest` endpoint already excludes drafts and prereleases, so this - * always reflects the newest shipping build. Returns null on any failure - * (network, missing auth on a private repo, no published release) so callers - * can degrade gracefully instead of surfacing errors. - */ -export async function fetchAdeLatestRelease(options?: { - token?: string | null; - repo?: { owner: string; name: string }; - fetchImpl?: typeof fetchGitHub; -}): Promise { - const repo = options?.repo ?? ADE_RELEASE_REPO; - const doFetch = options?.fetchImpl ?? fetchGitHub; - const headers: Record = { - accept: "application/vnd.github+json", - "user-agent": "ade-desktop", - }; - const token = options?.token?.trim(); - if (token) headers.authorization = `Bearer ${token}`; - - let response: Response; - try { - response = await doFetch( - `https://api.github.com/repos/${encodeURIComponent(repo.owner)}/${encodeURIComponent(repo.name)}/releases/latest`, - { method: "GET", headers }, - ); - } catch { - return null; - } - if (!response.ok) { - releaseGitHubResponse(response); - return null; - } - - const payload = (await response.json().catch(() => null)) as Record | null; - if (!payload) return null; - - const tagName = asString(payload.tag_name).trim(); - if (!tagName) return null; - const version = tagName.replace(/^v/i, "").trim() || tagName; - const htmlUrl = asString(payload.html_url).trim(); - const publishedAt = asString(payload.published_at).trim(); - return { - version, - tagName, - htmlUrl: htmlUrl || null, - publishedAt: publishedAt || null, - }; -} - export function createGithubService({ logger, projectRoot, diff --git a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts index 4e0bfda8f..2ec728971 100644 --- a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts +++ b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts @@ -125,19 +125,19 @@ function withTsxNodeOptions(value: string | undefined, loaderPath: string): stri } async function waitForRuntimeSocket(socketPath: string, timeoutMs = 10_000): Promise { - const startedAt = Date.now(); - let lastError: Error | null = null; - while (Date.now() - startedAt < timeoutMs) { + await vi.waitFor(async () => { + let client: RawRuntimeSocketClient | null = null; try { - const client = await RawRuntimeSocketClient.connect(socketPath); - client.close(); - return; - } catch (error) { - lastError = error instanceof Error ? error : new Error(String(error)); - await new Promise((resolve) => setTimeout(resolve, 100)); + client = await RawRuntimeSocketClient.connect(socketPath); + await client.request("ade/initialize", { + protocolVersion: "2025-06-18", + clientName: "local-runtime-test-readiness", + identity: { role: "external", callerId: "local-runtime-test-readiness" }, + }); + } finally { + client?.close(); } - } - throw lastError ?? new Error(`ADE service socket did not become available: ${socketPath}`); + }, { timeout: timeoutMs, interval: 100 }); } function startServeProcess(args: { diff --git a/apps/desktop/src/renderer/components/app/useAutoUpdateSnapshot.ts b/apps/desktop/src/renderer/components/app/useAutoUpdateSnapshot.ts index 6b61bf4b7..048687f47 100644 --- a/apps/desktop/src/renderer/components/app/useAutoUpdateSnapshot.ts +++ b/apps/desktop/src/renderer/components/app/useAutoUpdateSnapshot.ts @@ -31,21 +31,24 @@ export function useAutoUpdateSnapshot(): AutoUpdateSnapshot { useEffect(() => { let cancelled = false; - void window.ade.updateGetState() - .then((next) => { - if (!cancelled) setSnapshot(next); - }) - .catch(() => { - // Best effort only; live events will fill in. - }); + const initialSnapshot = window.ade.updateGetState?.(); + if (initialSnapshot) { + void initialSnapshot + .then((next) => { + if (!cancelled) setSnapshot(next); + }) + .catch(() => { + // Best effort only; live events will fill in. + }); + } - const unsubscribe = window.ade.onUpdateEvent((next) => { + const unsubscribe = window.ade.onUpdateEvent?.((next) => { if (!cancelled) setSnapshot(next); }); return () => { cancelled = true; - unsubscribe(); + unsubscribe?.(); }; }, []); From 2a05b4e508c230ec5b10c1bca06f971842863233 Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Thu, 23 Jul 2026 21:39:54 -0400 Subject: [PATCH 6/7] ship: deterministic pool tests, deadline re-check, fresh runtime health, review fixes Pins ADE_DEFAULT_ROLE and injects service status in the newer-brain pool tests (Linux CI inherited no role and rejected the fake brain); re-checks activity at the auto-apply deadline; refreshes publishHealth/lastWedge on getInfo instead of caching initialize data; recovery notice subscribes before reading; About separates Downloaded from Installed; doctor degrades slow health reads without reporting the brain down; watchdog SIGKILLs from finally; cancel suppresses countdown ticks until confirmed. Co-Authored-By: Claude Fable 5 --- apps/ade-cli/src/cli.ts | 10 ++- apps/ade-cli/src/commands/doctor.test.ts | 59 +++++++++++++- apps/ade-cli/src/commands/doctor.ts | 13 +++- .../ade-cli/src/multiProjectRpcServer.test.ts | 34 +++++++++ apps/ade-cli/src/multiProjectRpcServer.ts | 63 +++++++++------ .../accountMachinePublisherService.test.ts | 36 +++++++++ .../account/accountMachinePublisherService.ts | 5 +- .../runtime/brainFreshnessMonitor.test.ts | 4 +- .../services/runtime/brainFreshnessMonitor.ts | 8 +- .../runtime/brainLoopWatchdog.test.ts | 5 ++ .../src/services/runtime/brainLoopWatchdog.ts | 73 ++++++++++-------- apps/desktop/src/main/main.ts | 3 + .../src/main/services/ipc/registerIpc.ts | 5 +- .../localRuntimeConnectionPool.test.ts | 76 ++++++++++++++++++- .../localRuntimeConnectionPool.ts | 59 ++++++++++++++ .../updates/autoUpdateService.test.ts | 39 ++++++++++ .../services/updates/autoUpdateService.ts | 57 ++++++++++---- apps/desktop/src/preload/global.d.ts | 4 + apps/desktop/src/preload/preload.ts | 9 +++ apps/desktop/src/renderer/browserMock.ts | 1 + .../components/app/AutoUpdateBanner.test.tsx | 39 ++++++++++ .../components/app/AutoUpdateBanner.tsx | 7 +- .../app/BrainRecoveryNotice.test.ts | 56 +++++++++++++- .../components/app/BrainRecoveryNotice.tsx | 27 +++++-- .../components/settings/AboutSection.test.ts | 17 +++++ .../components/settings/AboutSection.tsx | 62 +++++++++++---- apps/desktop/src/shared/ipc.ts | 1 + 27 files changed, 661 insertions(+), 111 deletions(-) create mode 100644 apps/desktop/src/renderer/components/settings/AboutSection.test.ts diff --git a/apps/ade-cli/src/cli.ts b/apps/ade-cli/src/cli.ts index e3413a2d7..7bef15218 100644 --- a/apps/ade-cli/src/cli.ts +++ b/apps/ade-cli/src/cli.ts @@ -15102,7 +15102,7 @@ async function runServe( { ProjectRegistry }, { ProjectScopeRegistry }, { PersonalChatScope }, - { createMultiProjectRpcRequestHandler }, + { createMultiProjectRpcRequestHandler, readMachineRuntimeActivitySummary }, { createSharedSyncListener }, { resolveMobileProjectIconDataUrl }, { createBrainProjectActionsSyncHandler }, @@ -15789,7 +15789,13 @@ async function runServe( runningHash: process.env.ADE_RUNTIME_BUILD_HASH?.trim() || preparedServiceCommand.buildHash, - isIdle: () => !hasActiveHeadlessConnections(states), + isIdle: async () => ( + await readMachineRuntimeActivitySummary({ + projectRegistry, + scopeRegistry, + personalChatScope, + }) + ).idle, logger: { warn: (event, fields) => headlessProjectLogger.warn(event, fields), diff --git a/apps/ade-cli/src/commands/doctor.test.ts b/apps/ade-cli/src/commands/doctor.test.ts index f2be9ce91..12d4d159f 100644 --- a/apps/ade-cli/src/commands/doctor.test.ts +++ b/apps/ade-cli/src/commands/doctor.test.ts @@ -1,14 +1,19 @@ -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { compareDoctorVersions, doctorRuntimeStatusFromInitialize, evaluateDoctorRows, + probeDoctorBrain, type DoctorInput, } from "./doctor"; import { createSyncAccountDirectoryHealth } from "../../../desktop/src/shared/types/sync"; const NOW = Date.parse("2026-07-23T12:00:00.000Z"); +afterEach(() => { + vi.useRealTimers(); +}); + function healthyInput(): DoctorInput { return { nowMs: NOW, @@ -55,6 +60,58 @@ function healthyInput(): DoctorInput { } describe("doctor row evaluation", () => { + it("keeps an initialized brain reachable when later health reads time out", async () => { + vi.useFakeTimers(); + const never = new Promise(() => {}); + const request = vi.fn((method: string) => method === "ade/initialize" + ? Promise.resolve({ + runtimeInfo: { + version: "1.2.35", + buildHash: "build", + pid: 123, + uptimeMs: 90_000, + syncPort: 8787, + }, + }) + : never); + const resultPromise = probeDoctorBrain( + { role: "cto", socketPath: null }, + { + resolveMachineRuntimeSocketPath: async () => "/tmp/ade.sock", + connectRuntime: async () => ({ request, destroy: vi.fn() }), + buildInitializeParams: () => ({}), + readMachineRuntimeInfo: () => ({ + version: "1.2.35", + buildHash: "build", + defaultRole: "cto", + packageChannel: null, + projectRoot: null, + pid: 123, + uptimeMs: 90_000, + }), + resolveExpectedMachineRuntimeBuildHash: async () => "build", + machineRuntimeMismatchReason: () => null, + unwrapActionEnvelope: (value) => value, + }, + ); + + await vi.advanceTimersByTimeAsync(2_000); + const result = await resultPromise; + + expect(result.brain).toMatchObject({ + running: true, + version: "1.2.35", + pid: 123, + error: null, + }); + expect(result.runtimeSyncPort).toBe(8787); + expect(result.syncStatus).toBeNull(); + expect(result.account).toMatchObject({ + signedIn: null, + error: "ADE brain health reads timed out.", + }); + }); + it("parses the complete runtime publish and wedge health envelope", () => { expect(doctorRuntimeStatusFromInitialize({ runtimeInfo: { diff --git a/apps/ade-cli/src/commands/doctor.ts b/apps/ade-cli/src/commands/doctor.ts index 05fdc952c..a27a26f80 100644 --- a/apps/ade-cli/src/commands/doctor.ts +++ b/apps/ade-cli/src/commands/doctor.ts @@ -208,7 +208,7 @@ export function doctorRuntimeStatusFromInitialize(value: unknown): { }; } -async function probeDoctorBrain( +export async function probeDoctorBrain( options: Options, dependencies: DoctorCommandDependencies, ): Promise { @@ -250,7 +250,16 @@ async function probeDoctorBrain( ]), remaining(), "ADE brain health reads", - ); + ).catch((error): [ + PromiseRejectedResult, + PromiseRejectedResult, + ] => { + const reason = error instanceof Error ? error : new Error(String(error)); + return [ + { status: "rejected", reason }, + { status: "rejected", reason }, + ]; + }); const syncStatus = syncResult.status === "fulfilled" && isRecord(syncResult.value) ? syncResult.value : null; diff --git a/apps/ade-cli/src/multiProjectRpcServer.test.ts b/apps/ade-cli/src/multiProjectRpcServer.test.ts index 4bbf1a32b..b83e17dbc 100644 --- a/apps/ade-cli/src/multiProjectRpcServer.test.ts +++ b/apps/ade-cli/src/multiProjectRpcServer.test.ts @@ -7,6 +7,7 @@ import { createEventBuffer } from "./eventBuffer"; import { createMultiProjectRpcRequestHandler, decorateProjectListWithIcons, + readMachineRuntimeActivitySummary, } from "./multiProjectRpcServer"; import * as gitModule from "../../desktop/src/main/services/git/git"; import { ProjectRegistry } from "./services/projects/projectRegistry"; @@ -125,6 +126,39 @@ function makeRuntime(label: string) { } describe("multi-project RPC server", () => { + it("summarizes booted project and personal-chat work independently of client sockets", async () => { + const summary = await readMachineRuntimeActivitySummary({ + projectRegistry: { + list: () => [{ projectId: "project-1" }], + } as never, + scopeRegistry: { + getIfBooted: () => Promise.resolve({ + runtime: { + agentChatService: { hasActiveWorkloads: () => true }, + ptyService: { + list: () => [ + { runtimeState: "running" }, + { runtimeState: "exited" }, + ], + }, + }, + }), + } as never, + personalChatScope: { + activitySummary: async () => ({ + activeAgentTurns: 0, + activeWorkSessions: 1, + }), + }, + }); + + expect(summary).toEqual({ + idle: false, + activeAgentTurns: 1, + activeWorkSessions: 2, + }); + }); + it("keeps the complete inline icon catalog below its hard wire budget", async () => { const records = Array.from({ length: 8 }, (_, index) => ({ rootPath: `/project-${index}`, diff --git a/apps/ade-cli/src/multiProjectRpcServer.ts b/apps/ade-cli/src/multiProjectRpcServer.ts index 8502b3433..e096747d7 100644 --- a/apps/ade-cli/src/multiProjectRpcServer.ts +++ b/apps/ade-cli/src/multiProjectRpcServer.ts @@ -19,6 +19,7 @@ import type { ListMyGitHubReposInput, ProjectBrowseInput, RuntimeActivityCounts, + RuntimeActivitySummary, } from "../../desktop/src/shared/types"; import type { BufferedEvent } from "./eventBuffer"; import { computeRuntimeBuildHash as hashRuntimeBuild } from "./services/runtime/runtimeBuildIdentity"; @@ -115,6 +116,38 @@ export type MultiProjectRpcHandlerOptions = { reconcileAccountOwnership?: typeof reconcileAccountOwnedMachineTrust; }; +export async function readMachineRuntimeActivitySummary(args: { + projectRegistry: Pick; + scopeRegistry: Pick; + personalChatScope: Partial>; +}): Promise { + const projectActivity = await Promise.all( + args.projectRegistry.list().map(async (record) => { + const pendingScope = args.scopeRegistry.getIfBooted(record.projectId); + if (!pendingScope) return { activeAgentTurns: 0, activeWorkSessions: 0 }; + const scope = await pendingScope.catch(() => null); + if (!scope) return { activeAgentTurns: 0, activeWorkSessions: 0 }; + return summarizeRuntimeActivity(scope.runtime); + }), + ); + const personalActivity = typeof args.personalChatScope.activitySummary === "function" + ? await args.personalChatScope.activitySummary() + : { activeAgentTurns: 0, activeWorkSessions: 0 }; + const activeAgentTurns = projectActivity.reduce( + (total, activity) => total + activity.activeAgentTurns, + personalActivity.activeAgentTurns, + ); + const activeWorkSessions = projectActivity.reduce( + (total, activity) => total + activity.activeWorkSessions, + personalActivity.activeWorkSessions, + ); + return { + idle: activeAgentTurns === 0 && activeWorkSessions === 0, + activeAgentTurns, + activeWorkSessions, + }; +} + const RUNTIME_METHODS = new Set([ "ade/initialize", "ade/initialized", @@ -1193,31 +1226,11 @@ export function createMultiProjectRpcRequestHandler( } if (method === "runtime.activitySummary") { - const projectActivity = await Promise.all( - projectRegistry.list().map(async (record) => { - const pendingScope = scopeRegistry.getIfBooted(record.projectId); - if (!pendingScope) return { activeAgentTurns: 0, activeWorkSessions: 0 }; - const scope = await pendingScope.catch(() => null); - if (!scope) return { activeAgentTurns: 0, activeWorkSessions: 0 }; - return summarizeRuntimeActivity(scope.runtime); - }), - ); - const personalActivity = typeof personalChatScope.activitySummary === "function" - ? await personalChatScope.activitySummary() - : { activeAgentTurns: 0, activeWorkSessions: 0 }; - const activeAgentTurns = projectActivity.reduce( - (total, activity) => total + activity.activeAgentTurns, - personalActivity.activeAgentTurns, - ); - const activeWorkSessions = projectActivity.reduce( - (total, activity) => total + activity.activeWorkSessions, - personalActivity.activeWorkSessions, - ); - return { - idle: activeAgentTurns === 0 && activeWorkSessions === 0, - activeAgentTurns, - activeWorkSessions, - }; + return await readMachineRuntimeActivitySummary({ + projectRegistry, + scopeRegistry, + personalChatScope, + }); } if (method === "projects.list") { diff --git a/apps/ade-cli/src/services/account/accountMachinePublisherService.test.ts b/apps/ade-cli/src/services/account/accountMachinePublisherService.test.ts index 861df172d..bdefcf734 100644 --- a/apps/ade-cli/src/services/account/accountMachinePublisherService.test.ts +++ b/apps/ade-cli/src/services/account/accountMachinePublisherService.test.ts @@ -423,6 +423,42 @@ describe("account machine publisher health", () => { expect(captureAnalytics).toHaveBeenCalledTimes(2); }); + it("starts a new publish-failure analytics episode after a benign skip", async () => { + let clock = 0; + let syncEnabled = true; + const captureAnalytics = vi.fn(); + const service = createAccountMachinePublisherService({ + getAccessToken: async () => "account-token", + getAccountStatus: () => ({ signedIn: true, sessionReadState: "available" as const }), + getSnapshot: async () => snapshot(), + getMachineKey: () => "machine-studio", + directoryBaseUrl: () => "https://directory.example", + isSyncEnabled: () => syncEnabled, + fetchImpl: vi.fn(async () => new Response(null, { status: 503 })), + now: () => clock, + captureAnalytics, + }); + + await service.publishNow(); + clock = 121_000; + await service.publishNow(); + syncEnabled = false; + clock = 130_000; + await service.publishNow(); + expect(service.getPublisherHealth()).toMatchObject({ + state: "sync_disabled", + failingSinceMs: null, + }); + + syncEnabled = true; + clock = 200_000; + await service.publishNow(); + clock = 321_000; + await service.publishNow(); + + expect(captureAnalytics).toHaveBeenCalledTimes(2); + }); + it("bounds a stalled token leg and reports token_timeout without starting HTTP", async () => { vi.useFakeTimers(); const warn = vi.fn(); diff --git a/apps/ade-cli/src/services/account/accountMachinePublisherService.ts b/apps/ade-cli/src/services/account/accountMachinePublisherService.ts index a06a3de6e..4ccfbe27d 100644 --- a/apps/ade-cli/src/services/account/accountMachinePublisherService.ts +++ b/apps/ade-cli/src/services/account/accountMachinePublisherService.ts @@ -446,11 +446,10 @@ export function createAccountMachinePublisherService(options: { ? health.failingSinceMs ?? args.attemptAt : null, }; - if (state === "published") { + if (health.failingSinceMs == null) { publishFailureAnalyticsEmitted = false; } else if ( - health.failingSinceMs != null - && !publishFailureAnalyticsEmitted + !publishFailureAnalyticsEmitted && args.attemptAt - health.failingSinceMs >= PUBLISH_FAILURE_ANALYTICS_THRESHOLD_MS ) { publishFailureAnalyticsEmitted = true; diff --git a/apps/ade-cli/src/services/runtime/brainFreshnessMonitor.test.ts b/apps/ade-cli/src/services/runtime/brainFreshnessMonitor.test.ts index 512d0ebb4..364127c13 100644 --- a/apps/ade-cli/src/services/runtime/brainFreshnessMonitor.test.ts +++ b/apps/ade-cli/src/services/runtime/brainFreshnessMonitor.test.ts @@ -37,10 +37,11 @@ describe("brain freshness monitor", () => { const sleep = vi.fn(async () => { idle = true; }); + const isIdle = vi.fn(async () => idle); const monitor = createBrainFreshnessMonitor({ filePath: "/tmp/cli.cjs", runningHash: "running-hash", - isIdle: () => idle, + isIdle, restart, logger: { warn }, stat, @@ -59,6 +60,7 @@ describe("brain freshness monitor", () => { expect(computeHash).toHaveBeenCalledTimes(1); expect(sleep).toHaveBeenCalledTimes(1); + expect(isIdle).toHaveBeenCalledTimes(2); expect(restart).toHaveBeenCalledTimes(1); expect(warn).toHaveBeenCalledWith("brain.freshness_mismatch", { runningHash: "running-hash", diff --git a/apps/ade-cli/src/services/runtime/brainFreshnessMonitor.ts b/apps/ade-cli/src/services/runtime/brainFreshnessMonitor.ts index ba5f9f56c..20407d1fb 100644 --- a/apps/ade-cli/src/services/runtime/brainFreshnessMonitor.ts +++ b/apps/ade-cli/src/services/runtime/brainFreshnessMonitor.ts @@ -17,7 +17,7 @@ type BrainFileStat = { export type BrainFreshnessMonitorOptions = { filePath: string; runningHash: string | null; - isIdle: () => boolean; + isIdle: () => boolean | Promise; restart: () => void | Promise; logger: BrainFreshnessLogger; env?: NodeJS.ProcessEnv; @@ -87,11 +87,13 @@ export function createBrainFreshnessMonitor( diskHash: string, ): Promise => { const deadline = now() + maxIdleWaitMs; - while (!stopped && !options.isIdle() && now() < deadline) { + let idle = await options.isIdle(); + while (!stopped && !idle && now() < deadline) { await sleep(Math.min(idlePollMs, Math.max(1, deadline - now()))); + idle = await options.isIdle(); } if (stopped) return; - if (!options.isIdle()) { + if (!idle) { options.logger.warn("brain.freshness_restart_forced", { runningHash, diskHash, diff --git a/apps/ade-cli/src/services/runtime/brainLoopWatchdog.test.ts b/apps/ade-cli/src/services/runtime/brainLoopWatchdog.test.ts index b6b435b59..66153b25b 100644 --- a/apps/ade-cli/src/services/runtime/brainLoopWatchdog.test.ts +++ b/apps/ade-cli/src/services/runtime/brainLoopWatchdog.test.ts @@ -5,6 +5,7 @@ import { describe, expect, it, vi } from "vitest"; import { BRAIN_LOOP_WATCHDOG_BREADCRUMB_FILE, BRAIN_LOOP_WATCHDOG_LAST_WEDGE_FILE, + buildBrainLoopWatchdogWorkerSource, evaluateBrainLoopWatchdog, readBrainLoopWatchdogLastWedge, recoverBrainLoopWatchdogBreadcrumb, @@ -12,6 +13,10 @@ import { } from "./brainLoopWatchdog"; describe("brainLoopWatchdog", () => { + it("builds a valid worker program from the tested watchdog evaluator", () => { + expect(() => new Function(buildBrainLoopWatchdogWorkerSource())).not.toThrow(); + }); + it("detects a main-thread heartbeat gap while the worker check cadence stays healthy", () => { expect(evaluateBrainLoopWatchdog({ nowWallMs: 20_000, diff --git a/apps/ade-cli/src/services/runtime/brainLoopWatchdog.ts b/apps/ade-cli/src/services/runtime/brainLoopWatchdog.ts index ecfc8ae58..9cbb0f355 100644 --- a/apps/ade-cli/src/services/runtime/brainLoopWatchdog.ts +++ b/apps/ade-cli/src/services/runtime/brainLoopWatchdog.ts @@ -135,11 +135,13 @@ export function trackBrainLoopWatchdogCommand(action: string): () => void { }; } -function buildWorkerSource(): string { +export function buildBrainLoopWatchdogWorkerSource(): string { return String.raw` const fs = require("node:fs"); const path = require("node:path"); const { parentPort, workerData } = require("node:worker_threads"); + const BRAIN_LOOP_WATCHDOG_HEARTBEAT_MS = ${BRAIN_LOOP_WATCHDOG_HEARTBEAT_MS}; + const evaluateBrainLoopWatchdog = ${evaluateBrainLoopWatchdog.toString()}; const monotonicMs = () => Number(process.hrtime.bigint() / 1000000n); let lastHeartbeatWallMs = null; @@ -154,24 +156,38 @@ function buildWorkerSource(): string { blockedMs: Math.max(0, Math.floor(blockedMs)), ts: new Date().toISOString(), }; - fs.mkdirSync(workerData.runtimeDir, { recursive: true, mode: 0o700 }); const breadcrumbPath = path.join(workerData.runtimeDir, workerData.breadcrumbFile); - const tempPath = breadcrumbPath + "." + process.pid + "." + Date.now() + ".tmp"; - fs.writeFileSync(tempPath, JSON.stringify(breadcrumb) + "\n", { - encoding: "utf8", - mode: 0o600, - }); + let tempPath = null; try { + fs.mkdirSync(workerData.runtimeDir, { recursive: true, mode: 0o700 }); + tempPath = breadcrumbPath + "." + process.pid + "." + Date.now() + ".tmp"; + fs.writeFileSync(tempPath, JSON.stringify(breadcrumb) + "\n", { + encoding: "utf8", + mode: 0o600, + }); if (process.platform === "win32" && fs.existsSync(breadcrumbPath)) { fs.unlinkSync(breadcrumbPath); } fs.renameSync(tempPath, breadcrumbPath); } catch (error) { - try { fs.unlinkSync(tempPath); } catch {} - throw error; + if (tempPath) { + try { fs.unlinkSync(tempPath); } catch {} + } + try { + fs.writeSync( + 2, + "brain.event_loop_breadcrumb_failed " + + JSON.stringify({ error: error instanceof Error ? error.message : String(error) }) + + "\n", + ); + } catch {} + } finally { + try { + fs.writeSync(2, "brain.event_loop_blocked " + JSON.stringify(breadcrumb) + "\n"); + } finally { + process.kill(process.pid, "SIGKILL"); + } } - fs.writeSync(2, "brain.event_loop_blocked " + JSON.stringify(breadcrumb) + "\n"); - process.kill(process.pid, "SIGKILL"); }; parentPort.on("message", (message) => { @@ -192,33 +208,26 @@ function buildWorkerSource(): string { return; } - const wallSinceCheck = Math.max(0, nowWallMs - previousCheckWallMs); - const monotonicSinceCheck = Math.max(0, nowMonotonicMs - previousCheckMonotonicMs); - const sleepGapFloorMs = Math.max( - workerData.thresholdMs, - workerData.checkIntervalMs * 4, - ); - const checkGapDeltaMs = Math.abs(wallSinceCheck - monotonicSinceCheck); - const slept = wallSinceCheck > sleepGapFloorMs - && monotonicSinceCheck > sleepGapFloorMs - && checkGapDeltaMs <= Math.max( - 2000, - Math.max(wallSinceCheck, monotonicSinceCheck) * 0.25, - ); + const evaluation = evaluateBrainLoopWatchdog({ + nowWallMs, + nowMonotonicMs, + lastHeartbeatWallMs, + lastHeartbeatMonotonicMs, + previousCheckWallMs, + previousCheckMonotonicMs, + thresholdMs: workerData.thresholdMs, + checkIntervalMs: workerData.checkIntervalMs, + }); previousCheckWallMs = nowWallMs; previousCheckMonotonicMs = nowMonotonicMs; - if (slept) { + if (evaluation.slept) { lastHeartbeatWallMs = nowWallMs; lastHeartbeatMonotonicMs = nowMonotonicMs; return; } - const blockedMs = Math.floor(Math.min( - Math.max(0, nowWallMs - lastHeartbeatWallMs), - Math.max(0, nowMonotonicMs - lastHeartbeatMonotonicMs), - )); - if (blockedMs > workerData.thresholdMs) { - writeBreadcrumbAndKill(blockedMs); + if (evaluation.blocked) { + writeBreadcrumbAndKill(evaluation.blockedMs); } }, workerData.checkIntervalMs); `; @@ -250,7 +259,7 @@ export function startBrainLoopWatchdog(args: { let disposed = false; let worker: Worker; try { - worker = new Worker(buildWorkerSource(), { + worker = new Worker(buildBrainLoopWatchdogWorkerSource(), { eval: true, workerData: { runtimeDir: args.runtimeDir, diff --git a/apps/desktop/src/main/main.ts b/apps/desktop/src/main/main.ts index 70bfb82c1..d3bce639d 100644 --- a/apps/desktop/src/main/main.ts +++ b/apps/desktop/src/main/main.ts @@ -1390,6 +1390,9 @@ app.whenReady().then(async () => { const localRuntimePool = new LocalRuntimeConnectionPool(app.getVersion(), localRuntimeLogger, { preferServiceRepair: shouldRepairRuntimeServiceOnFallback, desktopBridgeAuthToken: builtInBrowserBridgeServer?.authToken ?? null, + onRuntimeStatusChange: (status) => { + broadcast(IPC.appRuntimeStatusChanged, status); + }, onRuntimeModeChange: (mode) => { localRuntimeLogger.warn("local_runtime.runtime_mode_changed", { mode }); if (!Notification.isSupported()) return; diff --git a/apps/desktop/src/main/services/ipc/registerIpc.ts b/apps/desktop/src/main/services/ipc/registerIpc.ts index 02392a995..53a0b635d 100644 --- a/apps/desktop/src/main/services/ipc/registerIpc.ts +++ b/apps/desktop/src/main/services/ipc/registerIpc.ts @@ -3690,6 +3690,9 @@ export function registerIpc({ ); ipcMain.handle(IPC.appGetInfo, async (): Promise => { + const localRuntimeStatus = localRuntimeConnectionPool + ? await localRuntimeConnectionPool.getFreshStatus() + : null; return { appVersion: app.getVersion(), isPackaged: app.isPackaged, @@ -3706,7 +3709,7 @@ export function registerIpc({ nodeEnv: process.env.NODE_ENV, viteDevServerUrl: process.env.VITE_DEV_SERVER_URL }, - localRuntime: localRuntimeConnectionPool?.getStatus() ?? null + localRuntime: localRuntimeStatus }; }); diff --git a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts index 2ec728971..2cc6d653e 100644 --- a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts +++ b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.test.ts @@ -153,6 +153,18 @@ function startServeProcess(args: { }); } +function runningTestServiceStatus() { + return { + ok: true, + serviceName: "com.ade.runtime", + action: "status" as const, + installed: true, + running: true, + path: "/test/com.ade.runtime", + message: "ADE test service is running.", + }; +} + function removeTempDir(dir: string): void { fs.rmSync(dir, { recursive: true, force: true }); } @@ -399,6 +411,10 @@ describe("local runtime connection pool", () => { ADE_HOME: adeHome, ADE_RUNTIME_SOCKET_PATH: socketPath, ADE_CLI_VERSION: "2.0.0", + // A real desktop-spawned brain always runs with the CTO role. Pin it + // here instead of inheriting ADE_DEFAULT_ROLE from the test host: ADE + // sessions set it on macOS, while GitHub's Linux runner does not. + ADE_DEFAULT_ROLE: "cto", NODE_OPTIONS: withTsxNodeOptions(originalEnv.NODE_OPTIONS, tsxLoaderPath), }; const daemon = startServeProcess({ @@ -413,7 +429,9 @@ describe("local runtime connection pool", () => { warn: vi.fn(), error: vi.fn(), }; - const pool = new LocalRuntimeConnectionPool("1.0.0", logger as never); + const pool = new LocalRuntimeConnectionPool("1.0.0", logger as never, { + queryServiceStatus: runningTestServiceStatus, + }); try { await waitForRuntimeSocket(socketPath); @@ -630,6 +648,60 @@ describe("local runtime connection pool", () => { pool.dispose(); }); + it("refreshes live publish and wedge health when status is read", async () => { + const onRuntimeStatusChange = vi.fn(); + const call = vi.fn(async () => ({ + runtimeInfo: { + version: "1.2.35", + pid: 4321, + syncPort: 8789, + publishHealth: { + state: "http_timeout", + failingSinceMs: 456_000, + lastLegDurations: { snapshot: 15, token: 45, http: 9_500 }, + }, + lastWedge: { + lastCommand: "chat.send", + blockedMs: 17_000, + ts: "2026-07-23T13:00:00.000Z", + }, + }, + })); + const pool = new LocalRuntimeConnectionPool("1.2.35", { + debug: vi.fn(), + info: vi.fn(), + warn: vi.fn(), + error: vi.fn(), + } as never, { + queryServiceStatus: runningTestServiceStatus, + onRuntimeStatusChange, + }); + (pool as unknown as { activeClient: unknown }).activeClient = { call }; + + const status = await pool.getFreshStatus(); + + expect(call).toHaveBeenCalledWith("runtime/info", {}, { timeoutMs: 2_000 }); + expect(status).toMatchObject({ + connectionState: "connected", + pid: 4321, + syncPort: 8789, + publishHealth: { + state: "http_timeout", + failingSinceMs: 456_000, + lastLegDurations: { snapshot: 15, token: 45, http: 9_500 }, + }, + lastWedge: { + lastCommand: "chat.send", + blockedMs: 17_000, + ts: "2026-07-23T13:00:00.000Z", + }, + }); + expect(onRuntimeStatusChange).toHaveBeenCalledWith(expect.objectContaining({ + lastWedge: expect.objectContaining({ lastCommand: "chat.send" }), + })); + pool.dispose(); + }); + it("retries the service install from isolated recovery and reports runtime mode transitions", async () => { const modeChanges: string[] = []; const pool = new LocalRuntimeConnectionPool("1.2.3", { @@ -2081,6 +2153,7 @@ describe("local runtime connection pool", () => { ADE_HOME: adeHome, ADE_RUNTIME_SOCKET_PATH: socketPath, ADE_CLI_VERSION: "2.0.0", + ADE_DEFAULT_ROLE: "cto", NODE_OPTIONS: withTsxNodeOptions(originalEnv.NODE_OPTIONS, tsxLoaderPath), }; const daemon = startServeProcess({ @@ -2110,6 +2183,7 @@ describe("local runtime connection pool", () => { pool = new LocalRuntimeConnectionPool("1.0.0", logger as never, { disableSync: true, preferServiceRepair: true, + queryServiceStatus: runningTestServiceStatus, }); const internals = pool as unknown as { tryConnect: (socketPath: string) => Promise<{ socketPath: string } | null>; diff --git a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts index 6b42d03fb..29d47359c 100644 --- a/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts +++ b/apps/desktop/src/main/services/localRuntime/localRuntimeConnectionPool.ts @@ -86,6 +86,7 @@ type LocalRuntimeConnectionPoolOptions = { preferServiceRepair?: boolean; desktopBridgeAuthToken?: string | null; queryServiceStatus?: () => ServiceManagerStatusResult; + onRuntimeStatusChange?: (status: LocalRuntimeStatus) => void; /** * Invoked when the pool enters or leaves isolated (no-sync fallback) mode. * "isolated" fires once per degradation, "primary" once per recovery. @@ -96,6 +97,7 @@ type LocalRuntimeConnectionPoolOptions = { type LocalRuntimeNodePathOptions = PackagedRuntimeNodePathOptions; const LOCAL_RUNTIME_SERVICE_UNINSTALL_TIMEOUT_MS = 20_000; +const LOCAL_RUNTIME_STATUS_REFRESH_TIMEOUT_MS = 2_000; const PLACEHOLDER_RUNTIME_VERSION = "0.0.0"; const LOCAL_RUNTIME_OUTPUT_LINE_MAX_CHARS = 4_000; const LOCAL_RUNTIME_OUTPUT_BUFFER_MAX_CHARS = 16_000; @@ -792,6 +794,7 @@ export class LocalRuntimeConnectionPool { }; private serviceHealthCheckedAtMs = 0; private serviceInstallPromise: Promise | null = null; + private runtimeStatusRefreshPromise: Promise | null = null; // Rolling 24 h aggregate of slow (>500 ms) or errored daemon action calls. // Feeds the machine-level runtime-health diagnostic surfaced in Settings. private slowActionSamples: SlowActionSample[] = []; @@ -808,6 +811,10 @@ export class LocalRuntimeConnectionPool { getStatus(): LocalRuntimeStatus { this.refreshServiceHealthIfStale(); + return this.statusSnapshot(); + } + + private statusSnapshot(): LocalRuntimeStatus { return { connectionState: this.activeClient ? "connected" @@ -830,6 +837,55 @@ export class LocalRuntimeConnectionPool { }; } + async getFreshStatus(): Promise { + await this.refreshRuntimeDiagnostics(); + return this.getStatus(); + } + + private async refreshRuntimeDiagnostics(): Promise { + if (this.runtimeStatusRefreshPromise) { + await this.runtimeStatusRefreshPromise; + return; + } + const client = this.activeClient; + if (!client) return; + const refresh = (async () => { + try { + const value = await client.call("runtime/info", {}, { + timeoutMs: LOCAL_RUNTIME_STATUS_REFRESH_TIMEOUT_MS, + }); + if (this.activeClient !== client) return; + const runtimeInfo = readLocalRuntimeInfo(value); + if (runtimeInfo.version == null && runtimeInfo.pid == null) return; + this.activeRuntimePid = runtimeInfo.pid; + this.activeRuntimeSyncPort = runtimeInfo.syncPort; + this.activeRuntimePublishHealth = runtimeInfo.publishHealth; + this.activeRuntimeLastWedge = runtimeInfo.lastWedge; + this.emitRuntimeStatusChange(); + } catch (error) { + this.logger.debug("local_runtime.status_refresh_failed", { + error: error instanceof Error ? error.message : String(error), + }); + } + })().finally(() => { + if (this.runtimeStatusRefreshPromise === refresh) { + this.runtimeStatusRefreshPromise = null; + } + }); + this.runtimeStatusRefreshPromise = refresh; + await refresh; + } + + private emitRuntimeStatusChange(): void { + try { + this.options.onRuntimeStatusChange?.(this.statusSnapshot()); + } catch (error) { + this.logger.warn("local_runtime.status_listener_failed", { + error: error instanceof Error ? error.message : String(error), + }); + } + } + getRuntimeProcessIds(): number[] { const pids = [ this.activeRuntimePid, @@ -2044,6 +2100,7 @@ export class LocalRuntimeConnectionPool { error: error instanceof Error ? error.message : String(error), }); } + this.emitRuntimeStatusChange(); } private async tryRecoverFromIsolatedRuntime(primarySocketPath: string): Promise { @@ -2152,6 +2209,7 @@ export class LocalRuntimeConnectionPool { this.activeRuntimeSyncPort = runtimeInfo.syncPort; this.activeRuntimePublishHealth = runtimeInfo.publishHealth; this.activeRuntimeLastWedge = runtimeInfo.lastWedge; + this.emitRuntimeStatusChange(); client.onDisconnect((error) => { if (this.activeClient !== client && this.activeConnection?.client !== client) return; this.logger.warn("local_runtime.disconnected", { @@ -2166,6 +2224,7 @@ export class LocalRuntimeConnectionPool { this.activeRuntimePublishHealth = null; this.activeRuntimeLastWedge = null; this.projectsByRoot.clear(); + this.emitRuntimeStatusChange(); }); return client; } diff --git a/apps/desktop/src/main/services/updates/autoUpdateService.test.ts b/apps/desktop/src/main/services/updates/autoUpdateService.test.ts index c289df06c..63d5f36b4 100644 --- a/apps/desktop/src/main/services/updates/autoUpdateService.test.ts +++ b/apps/desktop/src/main/services/updates/autoUpdateService.test.ts @@ -1188,6 +1188,45 @@ describe("createAutoUpdateService", () => { service.dispose(); }); + it("rechecks activity at the deadline and rearms after newly active work becomes idle", async () => { + vi.useFakeTimers(); + let idle = true; + const getRuntimeActivitySummary = vi.fn(async () => ({ idle })); + const updater = new FakeAutoUpdater(); + const service = createAutoUpdateService({ + logger: makeLogger(), + currentVersion: "1.2.2", + globalStatePath: makeStatePath(), + updaterCacheDir: makeUpdaterCacheDir(), + autoCheckEnabled: false, + autoApplyEnabled: true, + activityCheckMs: 1_000, + autoApplyIdleMs: 10_000, + autoApplyCountdownMs: 2_000, + getRuntimeActivitySummary, + updater, + }); + updater.emit("update-downloaded", { version: "1.2.3" }); + + await vi.advanceTimersByTimeAsync(10_000); + expect(service.getSnapshot().autoApplyPending).toEqual({ + deadlineAt: expect.any(Number), + }); + const checksBeforeDeadline = getRuntimeActivitySummary.mock.calls.length; + idle = false; + await vi.advanceTimersByTimeAsync(2_000); + + expect(getRuntimeActivitySummary.mock.calls.length).toBeGreaterThan(checksBeforeDeadline); + expect(updater.quitAndInstall).not.toHaveBeenCalled(); + expect(service.getSnapshot().autoApplyPending).toBeNull(); + + idle = true; + await vi.advanceTimersByTimeAsync(13_000); + expect(updater.quitAndInstall).toHaveBeenCalledWith(false, true); + + service.dispose(); + }); + it("restarts the idle window when runtime activity resumes", async () => { vi.useFakeTimers(); let idle = true; diff --git a/apps/desktop/src/main/services/updates/autoUpdateService.ts b/apps/desktop/src/main/services/updates/autoUpdateService.ts index a911881d0..ac2e61160 100644 --- a/apps/desktop/src/main/services/updates/autoUpdateService.ts +++ b/apps/desktop/src/main/services/updates/autoUpdateService.ts @@ -934,6 +934,47 @@ export function createAutoUpdateService({ patchSnapshot({ autoApplyPending: null }); } + async function applyAutoUpdateAtDeadline(deadlineAt: number): Promise { + if ( + !getRuntimeActivitySummary + || snapshot.status !== "ready" + || snapshot.autoApplyPending?.deadlineAt !== deadlineAt + || (snapshot.autoApplySuppressedUntil ?? 0) > nowMs() + ) { + return; + } + try { + const activity = await getRuntimeActivitySummary(); + if ( + snapshot.status !== "ready" + || snapshot.autoApplyPending?.deadlineAt !== deadlineAt + || (snapshot.autoApplySuppressedUntil ?? 0) > nowMs() + ) { + return; + } + if (!activity.idle) { + idleSinceMs = null; + clearPendingAutoApply(); + scheduleActivityCheck(activityCheckMs); + return; + } + patchSnapshot({ autoApplyPending: null }); + const started = await quitAndInstall(); + if (!started) return; + productAnalyticsService?.captureInternal({ + event: "ade_update_auto_applied", + surface: "desktop", + }); + } catch (error) { + idleSinceMs = null; + clearPendingAutoApply(); + logger.warn("autoUpdate.deadline_activity_check_failed", { + message: formatErrorMessage(error), + }); + scheduleActivityCheck(activityCheckMs); + } + } + async function checkAutoApplyActivity(): Promise { if ( activityCheckInProgress @@ -977,21 +1018,7 @@ export function createAutoUpdateService({ patchSnapshot({ autoApplyPending: { deadlineAt } }); autoApplyDeadlineTimer = setTimeout(() => { autoApplyDeadlineTimer = null; - if ( - snapshot.status !== "ready" - || snapshot.autoApplyPending?.deadlineAt !== deadlineAt - || (snapshot.autoApplySuppressedUntil ?? 0) > nowMs() - ) { - return; - } - patchSnapshot({ autoApplyPending: null }); - void quitAndInstall().then((started) => { - if (!started) return; - productAnalyticsService?.captureInternal({ - event: "ade_update_auto_applied", - surface: "desktop", - }); - }); + void applyAutoUpdateAtDeadline(deadlineAt); }, autoApplyCountdownMs); autoApplyDeadlineTimer.unref?.(); } catch (error) { diff --git a/apps/desktop/src/preload/global.d.ts b/apps/desktop/src/preload/global.d.ts index f7eeb2bc3..272f48e39 100644 --- a/apps/desktop/src/preload/global.d.ts +++ b/apps/desktop/src/preload/global.d.ts @@ -25,6 +25,7 @@ import type { AdoptAttachedLaneArgs, UnregisteredLaneCandidate, AppInfo, + LocalRuntimeStatus, AppWelcomeVideoState, AppResourceUsageSnapshot, LatestReleaseInfo, @@ -704,6 +705,9 @@ declare global { ping: () => Promise<"pong">; setDockBadgeCount: (count: number) => Promise<{ ok: true }>; getInfo: () => Promise; + onRuntimeStatusChanged: ( + cb: (status: LocalRuntimeStatus) => void, + ) => () => void; getResourceUsage: () => Promise; getRuntimeHealth: () => Promise; getLatestRelease: () => Promise; diff --git a/apps/desktop/src/preload/preload.ts b/apps/desktop/src/preload/preload.ts index 00523b582..779ac34dd 100644 --- a/apps/desktop/src/preload/preload.ts +++ b/apps/desktop/src/preload/preload.ts @@ -49,6 +49,7 @@ import type { AdoptAttachedLaneArgs, UnregisteredLaneCandidate, AppInfo, + LocalRuntimeStatus, AppWelcomeVideoState, AppResourceUsageSnapshot, LatestReleaseInfo, @@ -3274,6 +3275,14 @@ contextBridge.exposeInMainWorld("ade", { setDockBadgeCount: async (count: number): Promise<{ ok: true }> => ipcRenderer.invoke(IPC.appSetDockBadgeCount, { count }), getInfo: async (): Promise => ipcRenderer.invoke(IPC.appGetInfo), + onRuntimeStatusChanged: (cb: (status: LocalRuntimeStatus) => void) => { + const listener = ( + _event: Electron.IpcRendererEvent, + payload: LocalRuntimeStatus, + ) => cb(payload); + ipcRenderer.on(IPC.appRuntimeStatusChanged, listener); + return () => ipcRenderer.removeListener(IPC.appRuntimeStatusChanged, listener); + }, getResourceUsage: async (): Promise => ipcRenderer.invoke(IPC.appGetResourceUsage), getRuntimeHealth: async (): Promise => diff --git a/apps/desktop/src/renderer/browserMock.ts b/apps/desktop/src/renderer/browserMock.ts index 386ad47c6..8bb2f7012 100644 --- a/apps/desktop/src/renderer/browserMock.ts +++ b/apps/desktop/src/renderer/browserMock.ts @@ -3328,6 +3328,7 @@ if (typeof window !== "undefined" && shouldInstallBrowserMock(window)) { }, }, }), + onRuntimeStatusChanged: () => () => {}, getResourceUsage: resolved({ sampledAt: now, processCount: 1, diff --git a/apps/desktop/src/renderer/components/app/AutoUpdateBanner.test.tsx b/apps/desktop/src/renderer/components/app/AutoUpdateBanner.test.tsx index 8d9f0333d..a194d2fd7 100644 --- a/apps/desktop/src/renderer/components/app/AutoUpdateBanner.test.tsx +++ b/apps/desktop/src/renderer/components/app/AutoUpdateBanner.test.tsx @@ -80,6 +80,7 @@ describe("AutoUpdateBanner", () => { afterEach(() => { cleanup(); for (const toast of getToasts()) dismissToast(toast.id); + vi.useRealTimers(); vi.restoreAllMocks(); Reflect.deleteProperty(window, "ade"); }); @@ -156,4 +157,42 @@ describe("AutoUpdateBanner", () => { expect(screen.queryByText(/Updating to 1\.2\.35/)).toBeNull(); }); }); + + it("does not let a countdown tick restore the toast while cancellation is in flight", async () => { + vi.useFakeTimers(); + let resolveCancel!: (value: boolean) => void; + const cancelPromise = new Promise((resolve) => { + resolveCancel = resolve; + }); + const mock = installAdeMock( + snapshot({ + status: "ready", + version: "1.2.35", + autoApplyPending: { deadlineAt: Date.now() + 10_000 }, + }), + ); + mock.updateCancelAutoApply.mockImplementation(() => cancelPromise); + render( + <> + + + , + ); + await act(async () => { + await Promise.resolve(); + await Promise.resolve(); + }); + + expect(screen.getByText(/Updating to 1\.2\.35 in \d+s/)).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: /cancel/i })); + expect(screen.queryByText(/Updating to 1\.2\.35/)).toBeNull(); + + await act(async () => { + await vi.advanceTimersByTimeAsync(1_000); + }); + expect(screen.queryByText(/Updating to 1\.2\.35/)).toBeNull(); + + mock.emit(snapshot({ status: "ready", version: "1.2.35" })); + resolveCancel(true); + }); }); diff --git a/apps/desktop/src/renderer/components/app/AutoUpdateBanner.tsx b/apps/desktop/src/renderer/components/app/AutoUpdateBanner.tsx index e3671df85..87ee8e7c9 100644 --- a/apps/desktop/src/renderer/components/app/AutoUpdateBanner.tsx +++ b/apps/desktop/src/renderer/components/app/AutoUpdateBanner.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useState } from "react"; +import { useCallback, useEffect, useRef, useState } from "react"; import { ArrowsClockwise, WarningCircle } from "@phosphor-icons/react"; import type { AutoUpdateSnapshot } from "../../../shared/types"; import { useAutoUpdateSnapshot } from "./useAutoUpdateSnapshot"; @@ -54,6 +54,7 @@ export function AutoUpdateBanner() { const snapshot = useAutoUpdateSnapshot(); const [dismissedSignature, setDismissedSignature] = useState(null); const [restarting, setRestarting] = useState(false); + const cancelRequestedRef = useRef(false); const banner = describeStalenessBanner(snapshot); const signature = banner?.signature ?? null; @@ -76,8 +77,10 @@ export function AutoUpdateBanner() { }, []); const handleCancelAutoApply = useCallback(() => { + cancelRequestedRef.current = true; dismissToast(AUTO_APPLY_TOAST_ID); void window.ade.updateCancelAutoApply().catch(() => { + cancelRequestedRef.current = false; // Main process logs cancellation failures; the snapshot event reconciles. }); }, []); @@ -89,9 +92,11 @@ export function AutoUpdateBanner() { useEffect(() => { if (!pending) { dismissToast(AUTO_APPLY_TOAST_ID); + cancelRequestedRef.current = false; return; } const renderToast = () => { + if (cancelRequestedRef.current) return; const secondsLeft = Math.max(0, Math.ceil((pending.deadlineAt - Date.now()) / 1000)); showToast({ id: AUTO_APPLY_TOAST_ID, diff --git a/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.test.ts b/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.test.ts index c560ee12a..211856eb7 100644 --- a/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.test.ts +++ b/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.test.ts @@ -1,7 +1,13 @@ -import { describe, expect, it } from "vitest"; +/* @vitest-environment jsdom */ + +import React from "react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { act, cleanup, render, screen, waitFor } from "@testing-library/react"; import { + BrainRecoveryNotice, formatRecoveryCommand, formatRecoveryTime, + RECOVERY_ACK_STORAGE_KEY, shouldShowRecoveryNotice, } from "./BrainRecoveryNotice"; @@ -11,6 +17,54 @@ const wedge = (ts: string, lastCommand = "npm run build") => ({ blockedMs: 4200, }); +afterEach(() => { + cleanup(); + window.localStorage.removeItem(RECOVERY_ACK_STORAGE_KEY); + Reflect.deleteProperty(window, "ade"); +}); + +describe("BrainRecoveryNotice lifecycle", () => { + it("subscribes before the initial read and preserves a reconnect event that wins the race", async () => { + const order: string[] = []; + let statusListener: ((status: never) => void) | null = null; + let resolveInfo: ((info: never) => void) | null = null; + Object.defineProperty(window, "ade", { + configurable: true, + value: { + app: { + onRuntimeStatusChanged: vi.fn((listener: (status: never) => void) => { + order.push("subscribe"); + statusListener = listener; + return vi.fn(); + }), + getInfo: vi.fn(() => { + order.push("read"); + return new Promise((resolve) => { + resolveInfo = resolve; + }); + }), + }, + }, + }); + + render(React.createElement(BrainRecoveryNotice)); + await waitFor(() => expect(order).toEqual(["subscribe", "read"])); + + act(() => { + statusListener?.({ + lastWedge: wedge("2026-07-23T11:00:00Z", "chat.send"), + } as never); + }); + expect(await screen.findByText(/chat\.send/)).toBeTruthy(); + + await act(async () => { + resolveInfo?.({ localRuntime: { lastWedge: null } } as never); + await Promise.resolve(); + }); + expect(screen.getByText(/chat\.send/)).toBeTruthy(); + }); +}); + describe("shouldShowRecoveryNotice", () => { it("is hidden when there is no wedge", () => { expect(shouldShowRecoveryNotice(null, null)).toBe(false); diff --git a/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.tsx b/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.tsx index 70e3ee141..aa8b00d8c 100644 --- a/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.tsx +++ b/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.tsx @@ -56,8 +56,7 @@ function writeAckedTs(ts: string): void { /** * App-shell banner announcing that the brain recovered from a background - * event-loop wedge. Reads the one-shot `lastWedge` from runtime status; a wedge - * is a past event so there is no need to subscribe to changes. + * event-loop wedge. */ export function BrainRecoveryNotice() { const [lastWedge, setLastWedge] = useState(null); @@ -65,15 +64,27 @@ export function BrainRecoveryNotice() { useEffect(() => { let cancelled = false; + let statusRevision = 0; + const unsubscribe = window.ade.app.onRuntimeStatusChanged((status) => { + statusRevision += 1; + if (!cancelled) setLastWedge(status.lastWedge ?? null); + }); + const readRevision = statusRevision; const infoPromise = window.ade.app?.getInfo?.(); - if (!infoPromise) return; - void infoPromise - .then((info) => { - if (!cancelled) setLastWedge(info.localRuntime?.lastWedge ?? null); - }) - .catch(() => {}); + if (infoPromise) { + void infoPromise + .then((info) => { + // Subscribe before reading. If a reconnect status arrived while the + // snapshot was in flight, do not let that older read erase it. + if (!cancelled && statusRevision === readRevision) { + setLastWedge(info.localRuntime?.lastWedge ?? null); + } + }) + .catch(() => {}); + } return () => { cancelled = true; + unsubscribe(); }; }, []); diff --git a/apps/desktop/src/renderer/components/settings/AboutSection.test.ts b/apps/desktop/src/renderer/components/settings/AboutSection.test.ts new file mode 100644 index 000000000..4c2f33a4a --- /dev/null +++ b/apps/desktop/src/renderer/components/settings/AboutSection.test.ts @@ -0,0 +1,17 @@ +import { describe, expect, it } from "vitest"; +import { resolveAboutVersionState } from "./AboutSection"; + +describe("resolveAboutVersionState", () => { + it("keeps a downloaded update distinct from the installed app version", () => { + expect(resolveAboutVersionState("1.2.34", { + status: "ready", + version: "1.2.35", + parked: null, + })).toEqual({ + runningVersion: "1.2.34", + installedVersion: "1.2.34", + downloadedVersion: "1.2.35", + restartPending: true, + }); + }); +}); diff --git a/apps/desktop/src/renderer/components/settings/AboutSection.tsx b/apps/desktop/src/renderer/components/settings/AboutSection.tsx index cbc2e2c95..8afa9263f 100644 --- a/apps/desktop/src/renderer/components/settings/AboutSection.tsx +++ b/apps/desktop/src/renderer/components/settings/AboutSection.tsx @@ -113,6 +113,29 @@ function formatReleasedAgo(iso: string | null): string | null { return `released ${years} year${years === 1 ? "" : "s"} ago`; } +export function resolveAboutVersionState( + appVersion: string, + snapshot: Pick, +): { + runningVersion: string; + installedVersion: string; + downloadedVersion: string | null; + restartPending: boolean; +} { + const hasDownloadedUpdate = snapshot.status === "ready" || Boolean(snapshot.parked); + const downloadedVersion = hasDownloadedUpdate && snapshot.version + ? snapshot.version + : null; + return { + runningVersion: appVersion, + // electron-updater's ready state is only a cached download. Until the + // native installer succeeds, the app bundle on disk remains appVersion. + installedVersion: appVersion, + downloadedVersion, + restartPending: downloadedVersion != null && downloadedVersion !== appVersion, + }; +} + export function AboutSection({ embedded = false }: { embedded?: boolean } = {}) { const [info, setInfo] = useState(null); const [latest, setLatest] = useState(null); @@ -181,18 +204,13 @@ export function AboutSection({ embedded = false }: { embedded?: boolean } = {}) ? null : info.localRuntime.versionSkew; - // Truthful version split: "Running" is the version of the process the user is - // actually using (getInfo); "Installed" is what is staged on disk and will run - // after a restart; "Latest" is what the update feed advertises. When a - // consented install parked or a download is ready, the running process lags - // what is staged — that is the state the incident hid behind an "up to date". - const hasStagedUpdate = updateSnapshot.status === "ready" || Boolean(updateSnapshot.parked); - const runningVersion = info.appVersion; - const installedVersion = hasStagedUpdate && updateSnapshot.version - ? updateSnapshot.version - : info.appVersion; + const { + runningVersion, + installedVersion, + downloadedVersion, + restartPending, + } = resolveAboutVersionState(info.appVersion, updateSnapshot); const latestVersion = updateSnapshot.latestKnownVersion ?? latest?.version ?? info.appVersion; - const restartPending = hasStagedUpdate && installedVersion !== runningVersion; let pill: React.ReactNode = null; if (isDev) { @@ -244,12 +262,26 @@ export function AboutSection({ embedded = false }: { embedded?: boolean } = {}) Installed {installedVersion} - - · Latest {latestVersion} - {releasedAgo && latest != null && latestVersion === latest.version ? ` · ${releasedAgo}` : ""} - + {!downloadedVersion ? ( + + · Latest {latestVersion} + {releasedAgo && latest != null && latestVersion === latest.version ? ` · ${releasedAgo}` : ""} + + ) : null}
+ {downloadedVersion ? ( +
+ Downloaded + + {downloadedVersion} + + · Latest {latestVersion} + {releasedAgo && latest != null && latestVersion === latest.version ? ` · ${releasedAgo}` : ""} + + +
+ ) : null} {(updateAvailable || !isDev) ? ( diff --git a/apps/desktop/src/shared/ipc.ts b/apps/desktop/src/shared/ipc.ts index 8d1eb02d9..0dac35aa1 100644 --- a/apps/desktop/src/shared/ipc.ts +++ b/apps/desktop/src/shared/ipc.ts @@ -2,6 +2,7 @@ export const IPC = { appPing: "ade.app.ping", appSetDockBadgeCount: "ade.app.setDockBadgeCount", appGetInfo: "ade.app.getInfo", + appRuntimeStatusChanged: "ade.app.runtimeStatusChanged", appGetResourceUsage: "ade.app.getResourceUsage", appGetRuntimeHealth: "ade.app.getRuntimeHealth", storageGetPressure: "ade.storage.getPressure", From 05cd6fcad0432c4a609fea24c89ea49e0d5374bb Mon Sep 17 00:00:00 2001 From: Arul Sharma <31745423+arul28@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:17:05 -0400 Subject: [PATCH 7/7] ship: guard runtime-status subscription, bound publish body reads, isolate shared token refresh BrainRecoveryNotice tolerates hosts without onRuntimeStatusChanged; a declined auto-apply cancel restores the countdown toast; publish response bodies are read under a 5s bound with stream cancel so a stalled body can't pin the heartbeat; both coalesced token refreshes run under service-owned cancellation so one caller's abort no longer rejects every other caller. Co-Authored-By: Claude Fable 5 --- .../services/account/accountAuthService.ts | 60 +++++++++++++++---- .../account/accountMachinePublisherService.ts | 26 +++++++- .../components/app/AutoUpdateBanner.tsx | 14 +++-- .../components/app/BrainRecoveryNotice.tsx | 4 +- 4 files changed, 86 insertions(+), 18 deletions(-) diff --git a/apps/ade-cli/src/services/account/accountAuthService.ts b/apps/ade-cli/src/services/account/accountAuthService.ts index c20fc11e0..f6863a097 100644 --- a/apps/ade-cli/src/services/account/accountAuthService.ts +++ b/apps/ade-cli/src/services/account/accountAuthService.ts @@ -11,6 +11,7 @@ import { warnDevelopmentClerkIgnored, } from "../../../../desktop/src/shared/accountDirectory"; import type { SyncCredentialStore } from "../credentials/credentialStore"; +import { runWithAbortSignal } from "../sync/abortSignal"; export const ACCOUNT_SESSION_CREDENTIAL_KEY = "account.session.v1"; @@ -26,6 +27,10 @@ const USERINFO_REQUEST_TIMEOUT_MS = 5_000; // grant after invalid_grant. This path only runs after a definitive rejection. const REFRESH_ROTATION_WAIT_MS = 6_000; const REFRESH_ROTATION_POLL_MS = 50; +// Upper bound for the process-wide coalesced token exchange. It is owned by +// the service, not by any caller, so one caller's abort cannot cancel the +// refresh other callers are awaiting. +const SHARED_REFRESH_TIMEOUT_MS = 30_000; const ACCOUNT_TOKEN_ENV_KEY = "ADE_ACCOUNT_TOKEN"; const PROVISIONED_ACCOUNT_TOKEN_PREFIX = "ade_account_v1."; const SUCCESS_HTML = ` @@ -1748,6 +1753,15 @@ export function createAccountAuthService(args: { const credentialAtRefresh = envCredential; const epochAtRefresh = envCredentialEpoch; const refreshTokenAtRefresh = envRefreshToken ?? inspected.token; + // Same caller-isolation rule as the session refresh below: the shared + // env-token exchange runs under service-owned cancellation. + const envSharedRefresh = new AbortController(); + const envSharedTimer = setTimeout( + () => envSharedRefresh.abort(new Error("The shared ADE_ACCOUNT_TOKEN refresh timed out.")), + SHARED_REFRESH_TIMEOUT_MS, + ); + envSharedTimer.unref?.(); + const envSharedSignal = envSharedRefresh.signal; let refreshPromise: Promise | null = null; refreshPromise = (async (): Promise => { let config: AccountOAuthConfig; @@ -1767,7 +1781,7 @@ export function createAccountAuthService(args: { token = await postTokenForm({ fetchImpl, tokenUrl: `${config.issuer}/oauth/token`, - signal, + signal: envSharedSignal, body: { grant_type: "refresh_token", refresh_token: refreshTokenAtRefresh, @@ -1775,7 +1789,7 @@ export function createAccountAuthService(args: { }, }); } catch { - signal?.throwIfAborted(); + envSharedSignal.throwIfAborted(); throw new Error( "ADE_ACCOUNT_TOKEN refresh failed. Replace it with a newly provisioned token from `ade account token create`.", ); @@ -1797,16 +1811,24 @@ export function createAccountAuthService(args: { envSession, "loopback", config, - { signal }, + { signal: envSharedSignal }, ); envSession = refreshed; return refreshed.accessToken; - })(); + })().finally(() => { + clearTimeout(envSharedTimer); + }); envRefreshInFlight = refreshPromise; } const refreshPromise = envRefreshInFlight; try { - return await refreshPromise; + // Race the caller's own signal; the shared exchange keeps running for + // every other caller when this one aborts. + return await runWithAbortSignal( + () => refreshPromise, + signal ?? undefined, + "The account token request was aborted.", + ); } finally { if (envRefreshInFlight === refreshPromise) envRefreshInFlight = null; } @@ -1830,6 +1852,16 @@ export function createAccountAuthService(args: { const epochAtJoin = authEpoch; if (!refreshInFlight) { + // The coalesced exchange is shared by every caller; a single caller's + // abort must not cancel it for the rest. It runs under service-owned + // cancellation, and each caller races its own signal at the join below. + const sharedRefresh = new AbortController(); + const sharedRefreshTimer = setTimeout( + () => sharedRefresh.abort(new Error("The shared account token refresh timed out.")), + SHARED_REFRESH_TIMEOUT_MS, + ); + sharedRefreshTimer.unref?.(); + const sharedSignal = sharedRefresh.signal; refreshInFlight = (async () => { if (!sessionSnapshot.raw) return null; let refreshSnapshot = { @@ -1866,7 +1898,7 @@ export function createAccountAuthService(args: { // rotating refresh exchange may not have persisted its replacement // by the time Clerk rejects our old token, so briefly poll before // declaring the grant dead. - const rotation = await waitForRefreshRotation(refreshSnapshot, signal); + const rotation = await waitForRefreshRotation(refreshSnapshot, sharedSignal); if (rotation.kind === "rotated" && attempt === 0) { refreshSnapshot = rotation.snapshot; continue; @@ -1889,7 +1921,7 @@ export function createAccountAuthService(args: { refreshSnapshot.session, undefined, config, - { fetchUserinfo: false, obtainedAtMs, signal }, + { fetchUserinfo: false, obtainedAtMs, signal: sharedSignal }, ); if (authEpoch !== epochAtJoin) return null; if (!persistRefreshedSessionIfCurrent(refreshed, refreshSnapshot.raw)) { @@ -1907,10 +1939,10 @@ export function createAccountAuthService(args: { refreshSnapshot.session, undefined, config, - { obtainedAtMs, signal }, + { obtainedAtMs, signal: sharedSignal }, ); } catch (error) { - if (signal?.aborted) throw error; + if (sharedSignal.aborted) throw error; // The rotated credential and verified prior subject are already // durable. Optional profile enrichment must not make that successful // refresh unusable. @@ -1922,11 +1954,19 @@ export function createAccountAuthService(args: { ? enriched : readSession(); })().finally(() => { + clearTimeout(sharedRefreshTimer); refreshInFlight = null; }); } - const refreshed = await refreshInFlight; + // Join the shared exchange but race the caller's own signal: an aborting + // caller leaves; the refresh keeps running for everyone else. + const joinedRefresh = refreshInFlight; + const refreshed = await runWithAbortSignal( + () => joinedRefresh, + signal, + "The account token request was aborted.", + ); if (authEpoch !== epochAtJoin || !refreshed) { // A peer process may have won the refresh CAS. Its replacement token // satisfies this forced refresh; forcing another exchange here would diff --git a/apps/ade-cli/src/services/account/accountMachinePublisherService.ts b/apps/ade-cli/src/services/account/accountMachinePublisherService.ts index 4ccfbe27d..979f65c29 100644 --- a/apps/ade-cli/src/services/account/accountMachinePublisherService.ts +++ b/apps/ade-cli/src/services/account/accountMachinePublisherService.ts @@ -35,6 +35,7 @@ export const ACCOUNT_MACHINE_RELAY_STATE_POLL_MS = 2_000; export const ACCOUNT_MACHINE_RETRY_BACKOFF_MS = [1_000, 2_000, 5_000, 10_000, 20_000] as const; const DEFAULT_REQUEST_TIMEOUT_MS = 8_000; const DEFAULT_TOKEN_TIMEOUT_MS = 10_000; +const BODY_READ_TIMEOUT_MS = 5_000; const SLOW_PUBLISH_LEG_MS = 2_000; const PUBLISH_INFO_INTERVAL = 10; export const PUBLISH_FAILURE_ANALYTICS_THRESHOLD_MS = 120_000; @@ -793,11 +794,32 @@ export function createAccountMachinePublisherService(options: { return; } + // The request timer is cleared once headers arrive, so a stalled BODY + // would otherwise pin the inFlight promise forever and silently stop the + // heartbeat. Bound every body read and cancel the stream on expiry. + const readHttpReasonBounded = async (response: Response): Promise => { + let timer: ReturnType | null = null; + const expiry = new Promise((resolve) => { + timer = setTimeout(() => { + void response.body?.cancel().catch(() => {}); + resolve(null); + }, BODY_READ_TIMEOUT_MS); + }); + try { + return await Promise.race([ + readAccountDirectoryHttpReason(response).catch(() => null), + expiry, + ]); + } finally { + if (timer) clearTimeout(timer); + } + }; + try { let response = await sendRegistration(accessToken, registration, baseUrl); let firstUnauthorizedReason: string | null = null; if (response.status === 401) { - firstUnauthorizedReason = await readAccountDirectoryHttpReason(response).catch(() => null); + firstUnauthorizedReason = await readHttpReasonBounded(response); let refreshedToken: string | null = null; try { refreshedToken = (await runTokenLeg("token_refresh_401", true))?.trim() || null; @@ -832,7 +854,7 @@ export function createAccountMachinePublisherService(options: { // Always drain the final response, even when the first 401 already // supplied the user-facing reason. Leaving a replacement 401 body // unread can prevent the HTTP connection from being reused. - const responseReason = await readAccountDirectoryHttpReason(response).catch(() => null); + const responseReason = await readHttpReasonBounded(response); const httpReason = response.status === 401 && firstUnauthorizedReason ? firstUnauthorizedReason : responseReason; diff --git a/apps/desktop/src/renderer/components/app/AutoUpdateBanner.tsx b/apps/desktop/src/renderer/components/app/AutoUpdateBanner.tsx index 87ee8e7c9..50ee622a8 100644 --- a/apps/desktop/src/renderer/components/app/AutoUpdateBanner.tsx +++ b/apps/desktop/src/renderer/components/app/AutoUpdateBanner.tsx @@ -79,10 +79,16 @@ export function AutoUpdateBanner() { const handleCancelAutoApply = useCallback(() => { cancelRequestedRef.current = true; dismissToast(AUTO_APPLY_TOAST_ID); - void window.ade.updateCancelAutoApply().catch(() => { - cancelRequestedRef.current = false; - // Main process logs cancellation failures; the snapshot event reconciles. - }); + void window.ade.updateCancelAutoApply?.().then( + (accepted) => { + // A declined cancel must not leave the countdown invisible while + // auto-apply proceeds; the snapshot event reconciles accepted ones. + if (accepted === false) cancelRequestedRef.current = false; + }, + () => { + cancelRequestedRef.current = false; + }, + ); }, []); // Drive the countdown toast off `autoApplyPending`. Re-render once a second so diff --git a/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.tsx b/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.tsx index aa8b00d8c..0507fb99f 100644 --- a/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.tsx +++ b/apps/desktop/src/renderer/components/app/BrainRecoveryNotice.tsx @@ -65,7 +65,7 @@ export function BrainRecoveryNotice() { useEffect(() => { let cancelled = false; let statusRevision = 0; - const unsubscribe = window.ade.app.onRuntimeStatusChanged((status) => { + const unsubscribe = window.ade.app?.onRuntimeStatusChanged?.((status) => { statusRevision += 1; if (!cancelled) setLastWedge(status.lastWedge ?? null); }); @@ -84,7 +84,7 @@ export function BrainRecoveryNotice() { } return () => { cancelled = true; - unsubscribe(); + unsubscribe?.(); }; }, []);